diff --git a/.github/.markdownlint.yaml b/.github/.markdownlint.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a93d89c46ad4601505dd75ac252816fa4af5c40 --- /dev/null +++ b/.github/.markdownlint.yaml @@ -0,0 +1,210 @@ +# Default state for all rules +default: true + +# Path to configuration file to extend +extends: null + +# MD001/heading-increment/header-increment - Heading levels should only increment by one level at a time +MD001: true + +# MD002/first-heading-h1/first-header-h1 - First heading should be a top-level heading +MD002: + # Heading level + level: 1 + +# MD003/heading-style/header-style - Heading style +MD003: + # Heading style + style: "consistent" + +# MD004/ul-style - Unordered list style +MD004: + # List style + style: "consistent" + +# MD005/list-indent - Inconsistent indentation for list items at the same level +MD005: false + +# MD006/ul-start-left - Consider starting bulleted lists at the beginning of the line +MD006: false + +# MD007/ul-indent - Unordered list indentation +MD007: false + +# MD009/no-trailing-spaces - Trailing spaces +MD009: + # Spaces for line break + br_spaces: 2 + # Allow spaces for empty lines in list items + list_item_empty_lines: false + # Include unnecessary breaks + strict: false + +# MD010/no-hard-tabs - Hard tabs +MD010: false + +# MD011/no-reversed-links - Reversed link syntax +MD011: true + +# MD012/no-multiple-blanks - Multiple consecutive blank lines +MD012: + # Consecutive blank lines + maximum: 2 + +# MD013/line-length - Line length +MD013: + # Number of characters + line_length: 120 + # Number of characters for headings + heading_line_length: 120 + # Number of characters for code blocks + code_block_line_length: 150 + # Include code blocks + code_blocks: true + # Include tables + tables: true + # Include headings + headings: true + # Include headings + headers: true + # Strict length checking + strict: false + # Stern length checking + stern: false + +# MD014/commands-show-output - Dollar signs used before commands without showing output +MD014: true + +# MD018/no-missing-space-atx - No space after hash on atx style heading +MD018: true + +# MD019/no-multiple-space-atx - Multiple spaces after hash on atx style heading +MD019: true + +# MD020/no-missing-space-closed-atx - No space inside hashes on closed atx style heading +MD020: true + +# MD021/no-multiple-space-closed-atx - Multiple spaces inside hashes on closed atx style heading +MD021: true + +# MD022/blanks-around-headings/blanks-around-headers - Headings should be surrounded by blank lines +MD022: false + +# MD023/heading-start-left/header-start-left - Headings must start at the beginning of the line +MD023: true + +# MD024/no-duplicate-heading/no-duplicate-header - Multiple headings with the same content +MD024: false + +# MD025/single-title/single-h1 - Multiple top-level headings in the same document +MD025: false + +# MD026/no-trailing-punctuation - Trailing punctuation in heading +MD026: + # Punctuation characters + punctuation: ".,;:!。,;:!" + +# MD027/no-multiple-space-blockquote - Multiple spaces after blockquote symbol +MD027: true + +# MD028/no-blanks-blockquote - Blank line inside blockquote +MD028: true + +# MD029/ol-prefix - Ordered list item prefix +MD029: + # List style + style: "one_or_ordered" + +# MD030/list-marker-space - Spaces after list markers +MD030: + # Spaces for single-line unordered list items + ul_single: 1 + # Spaces for single-line ordered list items + ol_single: 1 + # Spaces for multi-line unordered list items + ul_multi: 1 + # Spaces for multi-line ordered list items + ol_multi: 1 + +# MD031/blanks-around-fences - Fenced code blocks should be surrounded by blank lines +MD031: false + +# MD032/blanks-around-lists - Lists should be surrounded by blank lines +MD032: false + +# MD033/no-inline-html - Inline HTML +MD033: false + +# MD034/no-bare-urls - Bare URL used +MD034: false + +# MD035/hr-style - Horizontal rule style +MD035: + # Horizontal rule style + style: "consistent" + +# MD036/no-emphasis-as-heading/no-emphasis-as-header - Emphasis used instead of a heading +MD036: false + +# MD037/no-space-in-emphasis - Spaces inside emphasis markers +MD037: true + +# MD038/no-space-in-code - Spaces inside code span elements +MD038: true + +# MD039/no-space-in-links - Spaces inside link text +MD039: true + +# MD040/fenced-code-language - Fenced code blocks should have a language specified +MD040: false + +# MD041/first-line-heading/first-line-h1 - First line in a file should be a top-level heading +MD041: false + +# MD042/no-empty-links - No empty links +MD042: true + +# MD043/required-headings/required-headers - Required heading structure +MD043: false + +# MD044/proper-names - Proper names should have the correct capitalization +MD044: + # List of proper names + names: ["Polkadot", "Substrate", "Cumulus", "Parity"] + # Include code blocks + code_blocks: false + # Include HTML elements + html_elements: false + +# MD045/no-alt-text - Images should have alternate text (alt text) +MD045: false + +# MD046/code-block-style - Code block style +MD046: + # Block style + style: "consistent" + +# MD047/single-trailing-newline - Files should end with a single newline character +MD047: true + +# MD048/code-fence-style - Code fence style +MD048: + # Code fence style + style: "consistent" + +# MD049/emphasis-style - Emphasis style should be consistent +MD049: false + +# MD050/strong-style - Strong style should be consistent +MD050: + # Strong style + style: "consistent" + +# MD051/link-fragments - Link fragments should be valid +MD051: false + +# MD052/reference-links-images - Reference links and images should use a label that is defined +MD052: false + +# MD053/link-image-reference-definitions - Link and image reference definitions should be needed +MD053: false diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..fdaa0c8628f766e241ba9043698b611a0bd78811 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,75 @@ +# Lists some code owners. +# +# A codeowner just oversees some part of the codebase. If an owned file is changed then the +# corresponding codeowner receives a review request. An approval of the codeowner might be +# required for merging a PR (depends on repository settings). +# +# For details about syntax, see: +# https://help.github.com/en/articles/about-code-owners +# But here are some important notes: +# +# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` +# which can be everywhere. +# - Multiple owners are supported. +# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind, +# that handles might work better because they are more recognizable on GitHub, +# eyou can use them for mentioning unlike an email. +# - The latest matching rule, if multiple, takes precedence. + +# CI +/.github/ @paritytech/ci @paritytech/release-engineering +/.gitlab-ci.yml @paritytech/ci +/.gitlab/ @paritytech/ci + +# XCM +/polkadot/xcm/ @paritytech/xcm + +# WASM executor, low-level client <-> WASM interface and other WASM-related code +/substrate/client/allocator/ @koute +/substrate/client/executor/ @koute +/substrate/primitives/panic-handler/ @koute +/substrate/primitives/runtime-interface/ @koute +/substrate/primitives/wasm-interface/ @koute +/substrate/utils/wasm-builder/ @koute + +# Systems-related bits and bobs on the client side +/substrate/client/sysinfo/ @koute +/substrate/client/tracing/ @koute + +# Documentation audit +/substrate/primitives/runtime @paritytech/docs-audit +/substrate/primitives/arithmetic @paritytech/docs-audit +# /primitives/core (to be added later) +# /primitives/io (to be added later) + +# FRAME +/substrate/frame/ @paritytech/frame-coders @paritytech/docs-audit +/substrate/frame/nfts/ @jsidorenko @paritytech/docs-audit +/substrate/frame/state-trie-migration/ @paritytech/frame-coders @cheme +/substrate/frame/uniques/ @jsidorenko @paritytech/docs-audit + +# GRANDPA, BABE, consensus stuff +/substrate/client/consensus/babe/ @andresilva +/substrate/client/consensus/grandpa/ @andresilva +/substrate/client/consensus/pow/ @sorpaas +/substrate/client/consensus/slots/ @andresilva +/substrate/frame/babe/ @andresilva +/substrate/frame/grandpa/ @andresilva +/substrate/primitives/consensus/pow/ @sorpaas + +# BEEFY, MMR +/substrate/frame/beefy/ @acatangiu +/substrate/frame/beefy-mmr/ @acatangiu +/substrate/frame/merkle-mountain-range/ @acatangiu +/substrate/primitives/merkle-mountain-range/ @acatangiu + +# Contracts +/substrate/frame/contracts/ @athei @paritytech/docs-audit + +# NPoS and election +/substrate/frame/election-provider-multi-phase/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/election-provider-support/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/elections-phragmen/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/nomination-pools/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/staking/ @paritytech/staking-core @paritytech/docs-audit +/substrate/primitives/npos-elections/ @paritytech/staking-core @paritytech/docs-audit diff --git a/cumulus/.github/ISSUE_TEMPLATE/blank.md b/.github/ISSUE_TEMPLATE/blank.md similarity index 100% rename from cumulus/.github/ISSUE_TEMPLATE/blank.md rename to .github/ISSUE_TEMPLATE/blank.md diff --git a/substrate/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml similarity index 92% rename from substrate/.github/ISSUE_TEMPLATE/bug.yaml rename to .github/ISSUE_TEMPLATE/bug_report.yaml index ae40df08eca761dc75a4a66cd3912666ba457912..f828a5d9d8931580713ba49a25c83643a74cf090 100644 --- a/substrate/.github/ISSUE_TEMPLATE/bug.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -1,6 +1,7 @@ name: Bug Report description: Let us know about an issue you experienced with this software -# labels: ["some existing label","another one"] +labels: [ I2-bug, I10-unconfirmed ] + body: - type: checkboxes attributes: @@ -20,7 +21,7 @@ body: id: bug attributes: label: Description of bug - # description: What seems to be the problem? + description: What seems to be the problem? # placeholder: Describe the problem. validations: required: true diff --git a/substrate/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml similarity index 100% rename from substrate/.github/ISSUE_TEMPLATE/config.yml rename to .github/ISSUE_TEMPLATE/config.yml diff --git a/substrate/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml similarity index 98% rename from substrate/.github/ISSUE_TEMPLATE/feature.yaml rename to .github/ISSUE_TEMPLATE/feature.yaml index 6a59522ab4b4829fae056fcadf006393cbd58a1a..828e8b461ccc85179a2e71511a08f60f9d7588a8 100644 --- a/substrate/.github/ISSUE_TEMPLATE/feature.yaml +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -1,6 +1,6 @@ name: Feature Request description: Submit your requests and suggestions to improve! -labels: ["J0-enhancement"] +labels: [ I5-enhancement ] body: - type: checkboxes id: existing diff --git a/.github/pr-custom-review.yml b/.github/pr-custom-review.yml index d37ea4cf0958e60cf4701f897af7581cae54b9f5..ac13d862a4ac2aee487de33de56fe34d6507c0dc 100644 --- a/.github/pr-custom-review.yml +++ b/.github/pr-custom-review.yml @@ -8,7 +8,7 @@ rules: check_type: changed_files condition: include: ^\.gitlab-ci\.yml|^docker/.*|^\.github/.*|^\.gitlab/.*|^\.config/nextest.toml|^\.cargo/.* - exclude: ^./gitlab/pipeline/zombienet.yml$ + exclude: ^\.gitlab/pipeline/zombienet.* min_approvals: 2 teams: - ci @@ -19,7 +19,7 @@ rules: condition: include: .* # excluding files from 'Runtime files' and 'CI files' rules - exclude: ^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/assets/(asset-hub-kusama|asset-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/bridge-hubs/(bridge-hub-kusama|bridge-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/collectives/collectives-polkadot/src/[^/]+\.rs$|^cumulus/parachains/common/src/[^/]+\.rs$|^substrate/frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*))|^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^\.gitlab-ci\.yml|^(?!.*\.dic$|.*spellcheck\.toml$)scripts/ci/.*|^\.github/.* + exclude: ^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/assets/(asset-hub-kusama|asset-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/bridge-hubs/(bridge-hub-kusama|bridge-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/collectives/collectives-polkadot/src/[^/]+\.rs$|^cumulus/parachains/common/src/[^/]+\.rs$|^substrate/frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*))|^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^\.gitlab-ci\.yml|^docker/.*|^\.github/.*|^\.gitlab/.*|^\.config/nextest.toml|^\.cargo/.* min_approvals: 2 teams: - core-devs @@ -39,7 +39,7 @@ rules: # if there are any changes in the bridges subtree (in case of backport changes back to bridges repo) - name: Bridges subtree files check_type: changed_files - condition: ^cumulus/bridges/.* + condition: ^bridges/.* min_approvals: 1 teams: - bridges-core diff --git a/.gitlab/common/lib.sh b/.github/scripts/common/lib.sh similarity index 76% rename from .gitlab/common/lib.sh rename to .github/scripts/common/lib.sh index ba5b171487288928f779f2db034f8d989213c647..b0f9cb32063a43cda58d2e6e8f25e0ff66a134eb 100755 --- a/.gitlab/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -96,7 +96,7 @@ structure_message() { # access_token: see https://matrix.org/docs/guides/client-server-api/ # Usage: send_message $body (json formatted) $room_id $access_token send_message() { -curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" + curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" } # Pretty-printing functions @@ -193,3 +193,74 @@ check_bootnode(){ echo " Bootnode appears unreachable" return 1 } + +# Assumes the ENV are set: +# - RELEASE_ID +# - GITHUB_TOKEN +# - REPO in the form paritytech/polkadot +fetch_release_artifacts() { + echo "Release ID : $RELEASE_ID" + echo "Repo : $REPO" + echo "Binary : $BINARY" + + curl -L -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json + + # Get Asset ids + ids=($(jq -r '.assets[].id' < release.json )) + count=$(jq '.assets|length' < release.json ) + + # Fetch artifacts + mkdir -p "./release-artifacts/${BINARY}" + pushd "./release-artifacts/${BINARY}" > /dev/null + + iter=1 + for id in "${ids[@]}" + do + echo " - $iter/$count: downloading asset id: $id..." + curl -s -OJ -L -H "Accept: application/octet-stream" \ + -H "Authorization: Token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${REPO}/releases/assets/$id" + iter=$((iter + 1)) + done + + pwd + ls -al --color + popd > /dev/null +} + +# Check the checksum for a given binary +function check_sha256() { + echo "Checking SHA256 for $1" + shasum -qc $1.sha256 +} + +# Import GPG keys of the release team members +# This is done in parallel as it can take a while sometimes +function import_gpg_keys() { + GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"} + SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798" + WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2" + EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3" + MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF" + MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF" + + echo "Importing GPG keys from $GPG_KEYSERVER in parallel" + for key in $SEC $WILL $EGOR $MARA $MORGAN; do + ( + echo "Importing GPG key $key" + gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key + echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust; + ) & + done + wait +} + +# Check the GPG signature for a given binary +function check_gpg() { + echo "Checking GPG Signature for $1" + gpg --no-tty --verify -q $1.asc $1 +} diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml index 3da699a354f69cf31ecd4c2870a39741abd255d1..4d0afefc47aace08bff9af796f9fb9c30dee99a7 100644 --- a/.github/workflows/check-licenses.yml +++ b/.github/workflows/check-licenses.yml @@ -14,7 +14,7 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout sources - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - uses: actions/setup-node@v3.8.1 with: node-version: "18.x" diff --git a/.github/workflows/check-markdown.yml b/.github/workflows/check-markdown.yml new file mode 100644 index 0000000000000000000000000000000000000000..f1e46ca273515d9081f9c5a7f6b8b4ff902c444a --- /dev/null +++ b/.github/workflows/check-markdown.yml @@ -0,0 +1,33 @@ +name: Check Markdown + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + packages: read + +jobs: + lint-markdown: + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + - uses: actions/setup-node@v3.8.1 + with: + node-version: "18.x" + registry-url: "https://npm.pkg.github.com" + scope: "@paritytech" + + - name: Install tooling + run: | + npm install -g markdownlint-cli + markdownlint --version + + - name: Check Markdown + env: + CONFIG: .github/.markdownlint.yaml + run: | + markdownlint --config "$CONFIG" --ignore target . diff --git a/.github/workflows/check-prdoc.yml b/.github/workflows/check-prdoc.yml new file mode 100644 index 0000000000000000000000000000000000000000..d153184941ac1e05f341ae49c4c8d4e212c0d85b --- /dev/null +++ b/.github/workflows/check-prdoc.yml @@ -0,0 +1,49 @@ +name: Check PRdoc + +on: + pull_request: + types: [labeled, opened, synchronize, unlabeled] + +env: + IMAGE: paritytech/prdoc:latest + API_BASE: https://api.github.com/repos + REPO: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR: ${{ github.event.pull_request.number }} + MOUNT: /prdoc + ENGINE: docker + +jobs: + check-prdoc: + runs-on: ubuntu-latest + steps: + - name: Pull image + run: | + echo "Pulling $IMAGE" + docker pull $IMAGE + docker run --rm $IMAGE --version + + - name: Check if PRdoc is required + id: get-labels + run: | + # Fetch the labels for the PR under test + echo "Fetch the labels for $API_BASE/${REPO}/pulls/${GITHUB_PR}" + labels=$( curl -H "Authorization: token ${GITHUB_TOKEN}" -s "$API_BASE/${REPO}/pulls/${GITHUB_PR}" | jq '.labels | .[] | .name' | tr "\n" ",") + echo "Labels: ${labels}" + echo "labels=${labels}" >> "$GITHUB_OUTPUT" + + - name: No PRdoc required + if: ${{ contains(steps.get-labels.outputs.labels, 'R0') }} + run: | + echo "PR detected as silent, no PRdoc is required, exiting..." + exit 0 + + - name: Checkout repo + if: ${{ !contains(steps.get-labels.outputs.labels, 'R0') }} + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4.0.0 + + - name: PRdoc check for PR#${{ github.event.pull_request.number }} + if: ${{ !contains(steps.get-labels.outputs.labels, 'R0') }} + run: | + echo "Checking for PR#${GITHUB_PR} in $MOUNT" + $ENGINE run --rm -v $PWD/prdoc:/doc $IMAGE check -n ${GITHUB_PR} || true diff --git a/.github/workflows/fmt-check.yml b/.github/workflows/fmt-check.yml index fd4b72061b925dfb3f6e4193a6b1fb97c0de75d8..df785404036e138c78cf1818620dd9db8c30a243 100644 --- a/.github/workflows/fmt-check.yml +++ b/.github/workflows/fmt-check.yml @@ -16,7 +16,7 @@ jobs: container: image: paritytech/ci-unified:bullseye-1.70.0-2023-05-23-v20230706 steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: Cargo fmt run: cargo +nightly fmt --all -- --check diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml new file mode 100644 index 0000000000000000000000000000000000000000..d01d78631e19b71cef7f450808ea707b061b8d82 --- /dev/null +++ b/.github/workflows/release-50_publish-docker.yml @@ -0,0 +1,285 @@ +name: Release - Publish Docker Image + +# This workflow listens to published releases or can be triggered manually. +# It builds and published releases and rc candidates. + +on: + #TODO: activate automated run later + # release: + # types: + # - published + workflow_dispatch: + inputs: + image_type: + description: Type of the image to be published + required: true + default: rc + type: choice + options: + - rc + - release + + binary: + description: Binary to be published + required: true + default: polkadot + type: choice + options: + - polkadot + - polkadot-parachain + + release_id: + description: | + Release ID. + You can find it using the command: + curl -s \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ + jq '.[] | { name: .name, id: .id }' + required: true + type: string + + registry: + description: Container registry + required: true + type: string + default: docker.io + + # The owner is often the same than the Docker Hub username but does ont have to be. + # In our case, it is not. + owner: + description: Owner of the container image repo + required: true + type: string + default: parity + + version: + description: version to build/release + default: v0.9.18 + required: true + +permissions: + contents: write + +env: + RELEASE_ID: ${{ inputs.release_id }} + ENGINE: docker + REGISTRY: ${{ inputs.registry }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} + REPO: ${{ github.repository }} + BINARY: ${{ inputs.binary }} + # EVENT_ACTION: ${{ github.event.action }} + EVENT_NAME: ${{ github.event_name }} + IMAGE_TYPE: ${{ inputs.image_type }} + +jobs: + fetch-artifacts: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build + if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + #TODO: this step will be needed when automated triggering will work + #this step runs only if the workflow is triggered automatically when new release is published + # if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} + # run: | + # mkdir -p release-artifacts && cd release-artifacts + + # for f in $BINARY $BINARY.asc $BINARY.sha256; do + # URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f" + # echo " - Fetching $f from $URL" + # wget "$URL" -O "$f" + # done + # chmod a+x $BINARY + # ls -al + + - name: Fetch rc artifacts or release artifacts based on release id + #this step runs only if the workflow is triggered manually + if: ${{ env.EVENT_NAME == 'workflow_dispatch' }} + run: | + . ./.github/scripts/common/lib.sh + + fetch_release_artifacts + + - name: Cache the artifacts + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + key: artifacts-${{ env.BINARY }}-${{ github.sha }} + path: | + ./release-artifacts/${{ env.BINARY }}/**/* + + build-container: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build + if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }} + runs-on: ubuntu-latest + needs: fetch-artifacts + + steps: + - name: Checkout sources + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + - name: Get artifacts from cache + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + key: artifacts-${{ env.BINARY }}-${{ github.sha }} + fail-on-cache-miss: true + path: | + ./release-artifacts/${{ env.BINARY }}/**/* + + - name: Check sha256 ${{ env.BINARY }} + working-directory: ./release-artifacts/${{ env.BINARY }} + run: | + . ../../.github/scripts/common/lib.sh + + echo "Checking binary $BINARY" + check_sha256 $BINARY && echo "OK" || echo "ERR" + + - name: Check GPG ${{ env.BINARY }} + working-directory: ./release-artifacts/${{ env.BINARY }} + run: | + . ../../.github/scripts/common/lib.sh + import_gpg_keys + check_gpg $BINARY + + - name: Fetch rc commit and tag + if: ${{ env.IMAGE_TYPE == 'rc' }} + id: fetch_rc_refs + run: | + release=release-${{ inputs.release_id }} && \ + echo "release=${release}" >> $GITHUB_OUTPUT + + commit=$(git rev-parse --short HEAD) && \ + echo "commit=${commit}" >> $GITHUB_OUTPUT + + tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ + [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ + echo "No tag, doing without" + + - name: Fetch release tags + working-directory: ./release-artifacts/${{ env.BINARY }} + if: ${{ env.IMAGE_TYPE == 'release'}} + id: fetch_release_refs + run: | + chmod a+rx $BINARY + VERSION=$(./$BINARY --version | awk '{ print $2 }' ) + release=$( echo $VERSION | cut -f1 -d- ) + echo "tag=latest" >> $GITHUB_OUTPUT + echo "release=${release}" >> $GITHUB_OUTPUT + + - name: Build Injected Container image for polkadot rc + if: ${{ env.BINARY == 'polkadot' }} + env: + ARTIFACTS_FOLDER: ./release-artifacts + IMAGE_NAME: ${{ env.BINARY }} + OWNER: ${{ env.DOCKER_OWNER }} + TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} + run: | + ls -al + echo "Building container for $BINARY" + ./docker/scripts/build-injected.sh + + - name: Build Injected Container image for polkadot-parachain + if: ${{ env.BINARY == 'polkadot-parachain' }} + env: + ARTIFACTS_FOLDER: ./release-artifacts + IMAGE_NAME: ${{ env.BINARY }} + OWNER: ${{ env.DOCKER_OWNER }} + DOCKERFILE: docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile + TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} + run: | + ls -al + mkdir -p $ARTIFACTS_FOLDER/specs + cp cumulus/parachains/chain-specs/*.json $ARTIFACTS_FOLDER/specs + + echo "Building container for $BINARY" + ./docker/scripts/build-injected.sh + + - name: Login to Dockerhub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.CUMULUS_DOCKERHUB_USERNAME }} + password: ${{ secrets.CUMULUS_DOCKERHUB_TOKEN }} + + - name: Push Container image for ${{ env.BINARY }} + id: docker_push + run: | + $ENGINE images | grep ${BINARY} + $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} + + - name: Check version for the published image for ${{ env.BINARY }} + env: + RELEASE_TAG: ${{ steps.fetch_rc_refs.outputs.release || steps.fetch_release_refs.outputs.release }} + run: | + echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" + $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version + + fetch-latest-debian-package-version: # this job will be triggered for polkadot release build + if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }} + runs-on: ubuntu-latest + outputs: + polkadot_apt_version: ${{ steps.fetch-latest-apt.outputs.polkadot_apt_version }} + container: + image: paritytech/parity-keyring + options: --user root + steps: + - name: Get version + id: fetch-latest-apt + run: | + apt update + apt show polkadot + version=$(apt show polkadot 2>/dev/null | grep "Version:" | awk '{print $2}') + echo "polkadot_apt_version=v$version" >> $GITHUB_OUTPUT + echo "You passed ${{ inputs.version }} but this is ignored" + echo "We use the version from the Debian Package: $version" + + build-polkadot-release-container: # this job will be triggered for polkadot release build + if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }} + runs-on: ubuntu-latest + needs: fetch-latest-debian-package-version + steps: + - name: Checkout sources + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Cache Docker layers + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to Docker Hub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.POLKADOT_DOCKERHUB_USERNAME }} + password: ${{ secrets.POLKADOT_DOCKERHUB_TOKEN }} + + - name: Fetch values + id: fetch-data + run: | + date=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + echo "date=$date" >> $GITHUB_OUTPUT + + - name: Build and push + id: docker_build + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + push: true + file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile + # TODO: The owner should be used below but buildx does not resolve the VARs + # TODO: It would be good to get rid of this GHA that we don't really need. + tags: | + parity/polkadot:latest + parity/polkadot:${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }} + build-args: | + VCS_REF=${{ github.ref }} + POLKADOT_VERSION=${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }} + BUILD_DATE=${{ steps.fetch-data.outputs.date }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.gitignore b/.gitignore index bd7f34b481041163b45c4c75ad4757d28ec0ca20..35e02e706b4261dcac2a3f9c6686245a01de9a8a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .local .vscode .wasm-binaries +*.adoc *.bin *.iml *.orig diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e0465ba1eb1b350e105a67703bd51ce13f544ae..10dd69f12a77c27a7d3de083f5063bd9f6a0e931 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: "paritytech/ci-unified:bullseye-1.70.0-2023-05-23-v20230706" + CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE] # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master" @@ -30,7 +30,7 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.65" + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.67" DOCKER_IMAGES_VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" default: @@ -114,7 +114,7 @@ default: - !reference [.rust-info-script, script] - !reference [.rusty-cachier, before_script] tags: - - linux-docker-vm-c2 + - linux-docker # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier # rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description @@ -197,8 +197,6 @@ default: extends: .build-refs include: - # weights jobs - # - gitlab/pipeline/weights.yml # check jobs - .gitlab/pipeline/check.yml # test jobs @@ -211,11 +209,14 @@ include: - .gitlab/pipeline/publish.yml # zombienet jobs - .gitlab/pipeline/zombienet.yml - # # timestamp handler + # timestamp handler - project: parity/infrastructure/ci_cd/shared ref: v0.2 file: /common/timestamp.yml - + # ci image + - project: parity/infrastructure/ci_cd/shared + ref: main + file: /common/ci-unified.yml # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 636f7e47afad81fb9b99ada0eeaad768ec17bedf..5b53798c403dd3ad82d908f7d7fe32956bcd6838 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -77,26 +77,6 @@ build-malus: - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - cp -r ./docker/* ./artifacts -build-staking-miner: - stage: build - extends: - - .docker-env - - .common-refs - # - .collect-artifacts - # DAG - needs: - - job: build-malus - artifacts: false - script: - - time cargo build -q --locked --release --package staging-staking-miner - # # pack artifacts - # - mkdir -p ./artifacts - # - mv ./target/release/staking-miner ./artifacts/. - # - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - # - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - # - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - # - cp -r ./scripts/* ./artifacts - build-rustdoc: stage: build extends: @@ -105,20 +85,40 @@ build-rustdoc: - .run-immediately variables: SKIP_WASM_BUILD: 1 - # artifacts: - # name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - # when: on_success - # expire_in: 1 days - # paths: - # - ./crate-docs/ + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" + when: on_success + expire_in: 1 days + paths: + - ./crate-docs/ script: # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo doc --workspace --no-deps + - time cargo doc --features try-runtime,experimental --workspace --no-deps - rm -f ./target/doc/.lock - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs + # Inject Simple Analytics (https://www.simpleanalytics.com/) privacy preserving tracker into + # all .html files + - | + inject_simple_analytics() { + local path="$1" + local script_content="" + + # Function that inject script into the head of an html file using sed. + process_file() { + local file="$1" + echo "Adding Simple Analytics script to $file" + sed -i "s||$script_content|" "$file" + } + export -f process_file + # xargs runs process_file in seperate shells without access to outer variables. + # to make script_content available inside process_file, export it as an env var here. + export script_content + + # Modify .html files in parallel using xargs, otherwise it can take a long time. + find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {} + } + inject_simple_analytics "./crate-docs" - echo "" > ./crate-docs/index.html build-implementers-guide: @@ -127,7 +127,7 @@ build-implementers-guide: - .kubernetes-env - .common-refs - .run-immediately - # - .collect-artifacts + - .collect-artifacts # git depth is set on purpose: https://github.com/paritytech/polkadot/issues/6284 variables: GIT_STRATEGY: clone @@ -146,7 +146,7 @@ build-short-benchmark: - .run-immediately - .collect-artifacts script: - - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot + - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build --bin polkadot --workspace - mkdir -p artifacts - target/release/polkadot --version - cp ./target/release/polkadot ./artifacts/ @@ -166,7 +166,7 @@ build-linux-stable-cumulus: RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - - time cargo build --release --locked --bin polkadot-parachain + - time cargo build --release --locked -p polkadot-parachain-bin --bin polkadot-parachain - echo "___Packing the artifacts___" - mkdir -p ./artifacts - mv ./target/release/polkadot-parachain ./artifacts/. @@ -186,7 +186,7 @@ build-test-parachain: RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - - time cargo build --release --locked --bin test-parachain + - time cargo build --release --locked -p cumulus-test-service --bin test-parachain - echo "___Packing the artifacts___" - mkdir -p ./artifacts - mv ./target/release/test-parachain ./artifacts/. @@ -273,7 +273,7 @@ build-short-benchmark-cumulus: - .run-immediately - .collect-artifacts script: - - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot-parachain + - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build -p polkadot-parachain-bin --bin polkadot-parachain --workspace - mkdir -p artifacts - target/release/polkadot-parachain --version - cp ./target/release/polkadot-parachain ./artifacts/ @@ -299,7 +299,7 @@ build-linux-substrate: # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589 - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA" script: - - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release + - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p node-cli - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate - echo -n "Substrate version = " - if [ "${CI_COMMIT_TAG}" ]; then @@ -309,7 +309,7 @@ build-linux-substrate: cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION; fi - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - cp -r ./docker/substrate_injected.Dockerfile ./artifacts/substrate/ + - cp -r ./docker/dockerfiles/substrate_injected.Dockerfile ./artifacts/substrate/ # - printf '\n# building node-template\n\n' # - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz @@ -341,7 +341,7 @@ build-subkey-linux: extends: .build-subkey # DAG needs: - - job: build-staking-miner + - job: build-malus artifacts: false # tbd # build-subkey-macos: diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index f88f5808637b89f452fdc77fb3031d40788cc0ba..5cc2337bf40978ea704aed262ca2f296e028f645 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -19,7 +19,7 @@ check-try-runtime: - time cargo check --locked -p parachain-template-node --features try-runtime # add after https://github.com/paritytech/substrate/pull/14502 is merged # experimental code may rely on try-runtime and vice-versa - - time cargo check --locked --features try-runtime,experimental + - time cargo check --locked --all --features try-runtime,experimental cargo-fmt-manifest: stage: check @@ -103,7 +103,6 @@ test-rust-feature-propagation: - zepter lint propagate-feature --feature try-runtime --left-side-feature-missing=ignore --workspace --feature-enables-dep="try-runtime:frame-try-runtime" --locked - zepter lint propagate-feature --feature runtime-benchmarks --left-side-feature-missing=ignore --workspace --feature-enables-dep="runtime-benchmarks:frame-benchmarking" --locked - zepter lint propagate-feature --feature std --left-side-feature-missing=ignore --workspace --locked - allow_failure: true # Experimental # More info can be found here: https://github.com/paritytech/polkadot/pull/5865 .check-runtime-migration: diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index ed18082344f02b719471e50b21d8db0765343a5e..a03d407c040904a6a1a4fd7095052a039d0019e0 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -1,6 +1,68 @@ # This file is part of .gitlab-ci.yml # Here are all jobs that are executed during "publish" stage +publish-rustdoc: + stage: publish + extends: .kubernetes-env + variables: + CI_IMAGE: node:18 + GIT_DEPTH: 100 + RUSTDOCS_DEPLOY_REFS: "master" + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "master" + needs: + - job: build-rustdoc + artifacts: true + - job: build-implementers-guide + artifacts: true + script: + # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we + # exit immediately. + # Putting spaces at the front and back to ensure we are not matching just any substring, but the + # whole space-separated value. + # setup ssh + - eval $(ssh-agent) + - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + - mkdir ~/.ssh && touch ~/.ssh/known_hosts + - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # Set git config + - git config user.email "devops-team@parity.io" + - git config user.name "${GITHUB_USER}" + - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch origin gh-pages + # Save README and docs + - cp -r ./crate-docs/ /tmp/doc/ + - cp -r ./artifacts/book/ /tmp/ + - cp README.md /tmp/doc/ + # we don't need to commit changes because we copy docs to /tmp + - git checkout gh-pages --force + # Enable if docs needed for other refs + # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS + # - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud + # - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME} + # Ensure the destination dir doesn't exist. + - rm -rf ${CI_COMMIT_REF_NAME} + - rm -rf book/ + - mv -f /tmp/doc ${CI_COMMIT_REF_NAME} + # dir for implementors guide + - mkdir -p book + - mv /tmp/book/html/* book/ + # Upload files + - git add --all + # `git commit` has an exit code of > 0 if there is nothing to commit. + # This causes GitLab to exit immediately and marks this job failed. + # We don't want to mark the entire job failed if there's nothing to + # publish though, hence the `|| true`. + - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" || + echo "___Nothing to commit___" + - git push origin gh-pages --force + after_script: + - rm -rf .git/ ./* + # cumulus .build-push-image: @@ -35,7 +97,7 @@ build-push-image-polkadot-parachain-debug: - job: build-linux-stable-cumulus artifacts: true variables: - DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug" build-push-image-test-parachain: @@ -48,7 +110,7 @@ build-push-image-test-parachain: - job: build-test-parachain artifacts: true variables: - DOCKERFILE: "docker/test-parachain_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/test-parachain_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/test-parachain" # publish-s3: # stage: publish @@ -114,7 +176,7 @@ build-push-image-polkadot-debug: - job: build-linux-stable artifacts: true variables: - DOCKERFILE: "docker/polkadot_injected_debug.Dockerfile" + DOCKERFILE: "docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile" IMAGE_NAME: "docker.io/paritypr/polkadot-debug" build-push-image-colander: @@ -127,7 +189,7 @@ build-push-image-colander: - job: build-test-collators artifacts: true variables: - DOCKERFILE: "docker/collator_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/collator_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/colander" build-push-image-malus: @@ -140,7 +202,7 @@ build-push-image-malus: - job: build-malus artifacts: true variables: - DOCKERFILE: "docker/malus_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/malus_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/malus" build-push-image-substrate-pr: @@ -153,7 +215,7 @@ build-push-image-substrate-pr: - job: build-linux-substrate artifacts: true variables: - DOCKERFILE: "docker/substrate_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/substrate" # old way @@ -201,7 +263,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +# # docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile # DOCKERFILE: polkadot_injected_debug.Dockerfile # IMAGE_NAME: docker.io/paritypr/polkadot-debug # needs: @@ -230,7 +292,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/collator_injected.Dockerfile +# # docker/dockerfiles/collator_injected.Dockerfile # DOCKERFILE: collator_injected.Dockerfile # IMAGE_NAME: docker.io/paritypr/colander # needs: @@ -258,7 +320,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/malus_injected.Dockerfile +# # docker/dockerfiles/malus_injected.Dockerfile # DOCKERFILE: malus_injected.Dockerfile # IMAGE_NAME: docker.io/paritypr/malus # needs: @@ -274,24 +336,6 @@ build-push-image-substrate-pr: # # this artifact is used in zombienet-tests job # dotenv: ./artifacts/malus.env -# publish-staking-miner-image: -# stage: publish -# extends: -# - .kubernetes-env -# - .build-push-image -# - .publish-refs -# variables: -# CI_IMAGE: ${BUILDAH_IMAGE} -# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# IMAGE_NAME: docker.io/paritytech/staking-miner -# GIT_STRATEGY: none -# DOCKER_USER: ${Docker_Hub_User_Parity} -# DOCKER_PASS: ${Docker_Hub_Pass_Parity} -# needs: -# - job: build-staking-miner -# artifacts: true - # substrate # publish-substrate-image-pr: diff --git a/.gitlab/pipeline/short-benchmarks.yml b/.gitlab/pipeline/short-benchmarks.yml index 81601fba32acfcc57415d1da57bb7b5a7c724956..5b0ea276773dd28e5e86a18a6b3c10b3bd103cde 100644 --- a/.gitlab/pipeline/short-benchmarks.yml +++ b/.gitlab/pipeline/short-benchmarks.yml @@ -5,7 +5,7 @@ # run short-benchmarks for relay chain runtimes from polkadot -short-benchmark-polkadot: &short-bench +short-benchmark-westend: &short-bench stage: short-benchmarks extends: - .docker-env @@ -14,21 +14,11 @@ short-benchmark-polkadot: &short-bench - job: build-short-benchmark artifacts: true variables: - RUNTIME: polkadot + RUNTIME: westend tags: - benchmark script: - - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - -short-benchmark-kusama: - <<: *short-bench - variables: - RUNTIME: kusama - -short-benchmark-westend: - <<: *short-bench - variables: - RUNTIME: westend + - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 # run short-benchmarks for system parachain runtimes from cumulus @@ -45,7 +35,7 @@ short-benchmark-westend: tags: - benchmark script: - - ./artifacts/polkadot-parachain benchmark pallet --wasm-execution compiled --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 short-benchmark-asset-hub-polkadot: <<: *short-bench-cumulus diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index c81750d49f9295063378ccfb724af467bcb3c630..3a4f5e71247ca33fa1d3655d1f1a01fd0dffb3bf 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -23,28 +23,53 @@ test-linux-stable: - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}" # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged # "upgrade_version_checks_should_work" is currently failing - # "receive_rate_limit_is_enforced"and "benchmark_block_works" can be found in test-linux-stable-additional-tests - # they fail if run with other tests - # "rx::tests::sent_views_include_finalized_number_update", "follow_chain_works", "create_snapshot_works" and "block_execution_works" - # can be found in test-linux-stable-slow - | time cargo nextest run \ - -E 'all() & !test(upgrade_version_checks_should_work) & !test(receive_rate_limit_is_enforced) & !test(benchmark_block_works) & !test(rx::tests::sent_views_include_finalized_number_update) & !test(follow_chain_works) & !test(create_snapshot_works) & !test(block_execution_works)' \ --workspace \ --locked \ --release \ --no-fail-fast \ - --features runtime-benchmarks,try-runtime,experimental \ + --features try-runtime,experimental \ --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} + # Upload tests results to Elasticsearch + - echo "Upload test results to Elasticsearch" + - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json + - | + curl -v -XPOST --http1.1 \ + -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \ + https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \ + -H 'Content-Type: application/json' \ + -d @target/nextest/default/junit.json || echo "failed to upload junit report" # run runtime-api tests with `enable-staging-api` feature on the 1st node - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi - # todo: add flacky-test collector + artifacts: + when: always + paths: + - target/nextest/default/junit.xml + reports: + junit: target/nextest/default/junit.xml test-linux-oldkernel-stable: extends: test-linux-stable tags: - oldkernel-vm +# https://github.com/paritytech/ci_cd/issues/864 +test-linux-stable-runtime-benchmarks: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + - .pipeline-stopper-artifacts + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet + # can be used to run all tests # test-linux-stable-all: # stage: test @@ -71,7 +96,7 @@ test-linux-oldkernel-stable: # --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} # # todo: add flacky-test collector -# for some reasons these tests fail if run with all tests +# TODO: remove me test-linux-stable-additional-tests: stage: test extends: @@ -85,16 +110,11 @@ test-linux-stable-additional-tests: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - | - time cargo nextest run \ - -E 'test(receive_rate_limit_is_enforced) + test(benchmark_block_works)' \ - --workspace \ - --locked \ - --release \ - --features runtime-benchmarks,try-runtime - allow_failure: true + # tests were moved to test-linux-stable + # the jobs should be removed + - exit 0 -# these ones can be really slow so it's better to run them separately +# TODO: remove me test-linux-stable-slow: stage: test # remove after cache is setup @@ -110,14 +130,9 @@ test-linux-stable-slow: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - | - time cargo nextest run \ - -E 'test(rx::tests::sent_views_include_finalized_number_update) + test(follow_chain_works) + test(create_snapshot_works) + test(block_execution_works)' \ - --workspace \ - --locked \ - --release \ - --features runtime-benchmarks,try-runtime - allow_failure: true + # tests were moved to test-linux-stable + # the jobs should be removed + - exit 0 # takes about 1,5h without cache # can be used to check that nextest works correctly @@ -156,7 +171,7 @@ test-doc: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - time cargo test --doc + - time cargo test --doc --workspace test-rustdoc: stage: test @@ -202,8 +217,6 @@ test-node-metrics: - time cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > artifacts/log.txt - # FIXME! - allow_failure: true test-deterministic-wasm: stage: test @@ -215,7 +228,15 @@ test-deterministic-wasm: - job: test-frame-ui artifacts: false script: - - .gitlab/test_deterministic_wasm.sh + # build runtime + - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime + # make checksum + - sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 + - cargo clean + # build again + - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime + # confirm checksum + - sha256sum -c checksum.sha256 cargo-check-benches: stage: test @@ -333,7 +354,7 @@ quick-benchmarks: WASM_BUILD_NO_COLOR: 1 WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" script: - - time cargo run --locked --release --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - time cargo run --locked --release -p node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 test-frame-examples-compile-to-wasm: # into one job diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 87b821742c675b39294c8bfccead1058cf3f9e33..e420baf486aa9bd3fa22c4127f1d773e44b98d8c 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -7,15 +7,25 @@ - export BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" # from build-linux-stable job - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} + - export CUMULUS_IMAGE="docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}" - export MALUS_IMAGE="${MALUS_IMAGE}":${PIPELINE_IMAGE_TAG} + - IMAGE_AVAILABLE=$(curl -o /dev/null -w "%{http_code}" -I -L -s https://registry.hub.docker.com/v2/repositories/parity/polkadot/tags/${BUILD_RELEASE_VERSION}) + - if [ $IMAGE_AVAILABLE -eq 200 ]; then + export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}"; + else + echo "Getting the image to use as SECONDARY, using ${BUILD_RELEASE_VERSION} as base"; + VERSIONS=$(curl -L -s 'https://registry.hub.docker.com/v2/repositories/parity/polkadot/tags/' | jq -r '.results[].name'| grep -E "v[0-9]" |grep -vE "[0-9]-"); + VERSION_TO_USE=$(echo "${BUILD_RELEASE_VERSION}\n$VERSIONS"|sort -r|grep -A1 "${BUILD_RELEASE_VERSION}"|tail -1); + export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${VERSION_TO_USE}"; + fi - echo "Zombienet Tests Config" - echo "gh-dir ${GH_DIR}" - echo "local-dir ${LOCAL_DIR}" - echo "polkadot image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}" - echo "polkadot secondary image ${ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE}" - echo "colander image ${COL_IMAGE}" + - echo "cumulus image ${CUMULUS_IMAGE}" - echo "malus image ${MALUS_IMAGE}" stage: zombienet image: "${ZOMBIENET_IMAGE}" @@ -28,6 +38,8 @@ artifacts: true - job: build-push-image-colander artifacts: true + - job: build-push-image-polkadot-parachain-debug + artifacts: true extends: - .kubernetes-env - .zombienet-refs diff --git a/.gitlab/test_deterministic_wasm.sh b/.gitlab/test_deterministic_wasm.sh deleted file mode 100755 index 4f1d2981ff2b4535b7fba056f9f67500965a9359..0000000000000000000000000000000000000000 --- a/.gitlab/test_deterministic_wasm.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -e - -#shellcheck source=../common/lib.sh -source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/common/lib.sh" - -# build runtime -WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime -# make checksum -sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 - -cargo clean - -# build again -WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime -# confirm checksum -sha256sum -c checksum.sha256 diff --git a/cumulus/BRIDGES.md b/BRIDGES.md similarity index 74% rename from cumulus/BRIDGES.md rename to BRIDGES.md index 8766de92c17e3b5b0d4596453cbf6bfce1893297..a6f00aec09283e10d1a697bcef3f523881941663 100644 --- a/cumulus/BRIDGES.md +++ b/BRIDGES.md @@ -1,13 +1,13 @@ -# Using Parity Bridges Common dependency (`git subtree`). +# Using Parity Bridges Common dependency (`git subtree`) In `./bridges` sub-directory you can find a `git subtree` imported version of: -[parity-bridges-common](https://github.com/paritytech/parity-bridges-common/) repository. +[`parity-bridges-common`](https://github.com/paritytech/parity-bridges-common/) repository. (For regular Cumulus contributor 1. is relevant) \ (For Cumulus maintainer 1. and 2. are relevant) \ (For Bridges team 1. and 2. and 3. are relevant) -# 1. How to fix broken Bridges code? +## How to fix broken Bridges code? To fix Bridges code simply create a commit in current (`Cumulus`) repo. Best if the commit is isolated to changes in `./bridges` sub-directory, because it makes @@ -16,7 +16,7 @@ it easier to import that change back to upstream repo. (Any changes to `bridges` subtree require Bridges team approve and they should manage backport to Bridges repo) -# 2. How to pull latest Bridges code to the `bridges` subtree +## How to pull latest Bridges code to the `bridges` subtree (in practice) The `bridges` repo has a stabilized branch `polkadot-staging` dedicated for releasing. @@ -25,7 +25,7 @@ The `bridges` repo has a stabilized branch `polkadot-staging` dedicated for rele cd # this will update new git branches from bridges repo -# there could be unresolved conflicts, but dont worry, +# there could be unresolved conflicts, but don't worry, # lots of them are caused because of removed unneeded files with patch step, BRANCH=polkadot-staging ./scripts/bridges_update_subtree.sh fetch @@ -45,9 +45,9 @@ BRANCH=polkadot-staging ./scripts/bridges_update_subtree.sh fetch # so after all conflicts are solved and patch passes and compiles, # then we need to finish merge with: git merge --continue -```` +``` -# 3. How to pull latest Bridges code or contribute back? +## How to pull latest Bridges code or contribute back? (in theory) Note that it's totally fine to ping the **Bridges Team** to do that for you. The point @@ -58,34 +58,34 @@ If you still would like to either update the code to match latest code from the or create an upstream PR read below. The following commands should be run in the current (`polkadot`) repo. -1. Add Bridges repo as a local remote: +### Add Bridges repo as a local remote ``` -$ git remote add -f bridges git@github.com:paritytech/parity-bridges-common.git +git remote add -f bridges git@github.com:paritytech/parity-bridges-common.git ``` If you plan to contribute back, consider forking the repository on Github and adding your personal fork as a remote as well. ``` -$ git remote add -f my-bridges git@github.com:tomusdrw/parity-bridges-common.git +git remote add -f my-bridges git@github.com:tomusdrw/parity-bridges-common.git ``` -2. To update Bridges: +### To update Bridges +``` +git fetch bridges polkadot-staging +git subtree pull --prefix=bridges bridges polkadot-staging --squash ``` -$ git fetch bridges polkadot-staging -$ git subtree pull --prefix=bridges bridges polkadot-staging --squash -```` We use `--squash` to avoid adding individual commits and rather squashing them all into one. -3. Clean unneeded files here: +### Clean unneeded files here ``` ./bridges/scripts/verify-pallets-build.sh --ignore-git-state --no-revert ``` -4. Contributing back to Bridges (creating upstream PR) +### Contributing back to Bridges (creating upstream PR) ``` -$ git subtree push --prefix=bridges my-bridges polkadot-staging +git subtree push --prefix=bridges my-bridges polkadot-staging ``` This command will push changes to your personal fork of Bridges repo, from where you can simply create a PR to the main repo. diff --git a/Cargo.lock b/Cargo.lock index 8af7f4f872173104165b415a38fc48e340cbca44..ddb19c24e1f6a1cc962566dcaa68620df3dd22fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,7 +494,7 @@ dependencies = [ "ark-ff", "ark-std", "tracing", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "ark-secret-scalar" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ec", "ark-ff", @@ -561,7 +561,7 @@ dependencies = [ [[package]] name = "ark-transcript" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ff", "ark-serialize", @@ -617,7 +617,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.27", + "time", ] [[package]] @@ -633,7 +633,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.27", + "time", ] [[package]] @@ -737,7 +737,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "kusama-runtime-constants", "log", "pallet-asset-conversion", @@ -833,7 +833,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-asset-tx-payment", "pallet-assets", @@ -926,7 +926,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-asset-conversion", "pallet-asset-conversion-tx-payment", @@ -990,7 +990,7 @@ dependencies = [ "cumulus-test-relay-sproof-builder", "frame-support", "frame-system", - "hex-literal 0.4.1", + "hex-literal", "pallet-assets", "pallet-balances", "pallet-collator-selection", @@ -1138,7 +1138,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1160,7 +1160,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1177,7 +1177,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1240,13 +1240,12 @@ dependencies = [ [[package]] name = "bandersnatch_vrfs" version = "0.0.1" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-bls12-381", "ark-ec", "ark-ed-on-bls12-381-bandersnatch", "ark-ff", - "ark-scale", "ark-serialize", "ark-std", "dleq_vrf", @@ -1352,7 +1351,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -1636,7 +1635,7 @@ dependencies = [ "finality-grandpa", "frame-support", "hex", - "hex-literal 0.4.1", + "hex-literal", "parity-scale-codec", "scale-info", "serde", @@ -1666,7 +1665,7 @@ dependencies = [ "bp-runtime", "frame-support", "hex", - "hex-literal 0.4.1", + "hex-literal", "parity-scale-codec", "scale-info", "serde", @@ -1728,7 +1727,7 @@ dependencies = [ "bp-runtime", "frame-support", "hex", - "hex-literal 0.4.1", + "hex-literal", "parity-scale-codec", "scale-info", "sp-runtime", @@ -1754,7 +1753,7 @@ dependencies = [ "frame-support", "frame-system", "hash-db", - "hex-literal 0.4.1", + "hex-literal", "impl-trait-for-tuples", "log", "num-traits", @@ -1778,7 +1777,7 @@ dependencies = [ "bp-parachains", "bp-polkadot-core", "bp-runtime", - "ed25519-dalek 1.0.1", + "ed25519-dalek", "finality-grandpa", "parity-scale-codec", "sp-application-crypto", @@ -1832,7 +1831,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "kusama-runtime-constants", "log", "pallet-aura", @@ -1895,7 +1894,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-aura", "pallet-authorship", @@ -1990,7 +1989,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-aura", "pallet-authorship", @@ -2274,9 +2273,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b40ccee03b5175c18cde8f37e7d2a33bcef6f8ec8f7cc0d81090d1bb380949c9" +checksum = "03915af431787e6ffdcc74c645077518c6b6e01f80b761e0fbbfa288536311b3" dependencies = [ "smallvec", ] @@ -2334,7 +2333,7 @@ name = "chain-spec-builder" version = "2.0.0" dependencies = [ "ansi_term", - "clap 4.4.2", + "clap 4.4.4", "node-cli", "rand 0.8.5", "sc-chain-spec", @@ -2345,15 +2344,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.27" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56b4c72906975ca04becb8a30e102dfecddd0c06181e3e95ddc444be28881f8" +checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", - "time 0.1.45", "wasm-bindgen", "windows-targets 0.48.5", ] @@ -2465,9 +2463,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive 4.4.2", @@ -2475,9 +2473,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -2491,7 +2489,7 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", ] [[package]] @@ -2516,7 +2514,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -2566,6 +2564,7 @@ dependencies = [ "frame-support", "integration-tests-common", "pallet-assets", + "pallet-balances", "pallet-core-fellowship", "pallet-salary", "pallet-xcm", @@ -2600,7 +2599,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-alliance", "pallet-aura", @@ -2608,6 +2607,7 @@ dependencies = [ "pallet-balances", "pallet-collator-selection", "pallet-collective", + "pallet-collective-content", "pallet-core-fellowship", "pallet-multisig", "pallet-preimage", @@ -2705,7 +2705,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof?rev=0e948f3#0e948f3c28cbacecdd3020403c4841c0eb339213" +source = "git+https://github.com/w3f/ring-proof?rev=8657210#86572101f4210647984ab4efedba6b3fcc890895" dependencies = [ "ark-ec", "ark-ff", @@ -2714,6 +2714,7 @@ dependencies = [ "ark-std", "fflonk", "merlin 3.0.0", + "rand_chacha 0.3.1", ] [[package]] @@ -2815,7 +2816,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "kusama-runtime-constants", "log", "pallet-aura", @@ -3086,7 +3087,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.2", + "clap 4.4.4", "criterion-plot", "futures", "is-terminal", @@ -3251,7 +3252,7 @@ dependencies = [ name = "cumulus-client-cli" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "parity-scale-codec", "sc-chain-spec", "sc-cli", @@ -3545,7 +3546,7 @@ dependencies = [ "environmental", "frame-support", "frame-system", - "hex-literal 0.4.1", + "hex-literal", "impl-trait-for-tuples", "lazy_static", "log", @@ -3575,7 +3576,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -3900,14 +3901,6 @@ dependencies = [ "sp-trie", ] -[[package]] -name = "cumulus-test-relay-validation-worker-provider" -version = "0.1.0" -dependencies = [ - "polkadot-node-core-pvf", - "toml 0.7.6", -] - [[package]] name = "cumulus-test-runtime" version = "0.1.0" @@ -3944,7 +3937,7 @@ name = "cumulus-test-service" version = "0.1.0" dependencies = [ "async-trait", - "clap 4.4.2", + "clap 4.4.4", "criterion 0.5.1", "cumulus-client-cli", "cumulus-client-consensus-common", @@ -3959,7 +3952,6 @@ dependencies = [ "cumulus-relay-chain-minimal-node", "cumulus-test-client", "cumulus-test-relay-sproof-builder", - "cumulus-test-relay-validation-worker-provider", "cumulus-test-runtime", "frame-system", "frame-system-rpc-runtime-api", @@ -4067,7 +4059,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -4107,7 +4099,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -4124,7 +4116,7 @@ checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -4422,7 +4414,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -4434,7 +4426,7 @@ checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" [[package]] name = "dleq_vrf" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ec", "ark-ff", @@ -4465,18 +4457,18 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "docify" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029de870d175d11969524d91a3fb2cbf6d488b853bff99d41cf65e533ac7d9d2" +checksum = "76ee528c501ddd15d5181997e9518e59024844eac44fd1e40cb20ddb2a8562fa" dependencies = [ "docify_macros", ] [[package]] name = "docify_macros" -version = "0.2.1" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac43324656a1b05eb0186deb51f27d2d891c704c37f34de281ef6297ba193e5" +checksum = "0ca01728ab2679c464242eca99f94e2ce0514b52ac9ad950e2ed03fca991231c" dependencies = [ "common-path", "derive-syn-parse", @@ -4484,7 +4476,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.31", + "syn 2.0.37", "termcolor", "toml 0.7.6", "walkdir", @@ -4561,15 +4553,6 @@ dependencies = [ "spki 0.7.2", ] -[[package]] -name = "ed25519" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" -dependencies = [ - "signature 1.6.4", -] - [[package]] name = "ed25519" version = "2.2.2" @@ -4580,20 +4563,6 @@ dependencies = [ "signature 2.1.0", ] -[[package]] -name = "ed25519-dalek" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" -dependencies = [ - "curve25519-dalek 3.2.0", - "ed25519 1.5.3", - "rand 0.7.3", - "serde", - "sha2 0.9.9", - "zeroize", -] - [[package]] name = "ed25519-dalek" version = "2.0.0" @@ -4601,7 +4570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ "curve25519-dalek 4.0.0", - "ed25519 2.2.2", + "ed25519", "rand_core 0.6.4", "serde", "sha2 0.10.7", @@ -4629,7 +4598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e83e509bcd060ca4b54b72bde5bb306cb2088cb01e14797ebae90a24f70f5f7" dependencies = [ "curve25519-dalek 4.0.0", - "ed25519 2.2.2", + "ed25519", "hashbrown 0.14.0", "hex", "rand_core 0.6.4", @@ -4728,18 +4697,18 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] name = "enumn" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b893c4eb2dc092c811165f84dc7447fae16fb66521717968c34c509b39b1a5c5" +checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -4862,12 +4831,6 @@ dependencies = [ "futures", ] -[[package]] -name = "exitcode" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" - [[package]] name = "expander" version = "0.0.4" @@ -4880,18 +4843,6 @@ dependencies = [ "quote", ] -[[package]] -name = "expander" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3774182a5df13c3d1690311ad32fbe913feef26baba609fa2dd5f72042bd2ab6" -dependencies = [ - "blake2", - "fs-err", - "proc-macro2", - "quote", -] - [[package]] name = "expander" version = "2.0.0" @@ -4902,7 +4853,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -5192,7 +5143,7 @@ dependencies = [ "Inflector", "array-bytes", "chrono", - "clap 4.4.2", + "clap 4.4.4", "comfy-table", "frame-benchmarking", "frame-support", @@ -5258,7 +5209,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", - "syn 2.0.31", + "syn 2.0.37", "trybuild", ] @@ -5284,7 +5235,7 @@ dependencies = [ name = "frame-election-solution-type-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "frame-election-provider-solution-type", "frame-election-provider-support", "frame-support", @@ -5410,7 +5361,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -5421,7 +5372,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -5430,7 +5381,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -5653,7 +5604,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -5839,9 +5790,11 @@ dependencies = [ name = "glutton-runtime" version = "1.0.0" dependencies = [ + "cumulus-pallet-aura-ext", "cumulus-pallet-parachain-system", "cumulus-pallet-xcm", "cumulus-primitives-core", + "cumulus-primitives-timestamp", "frame-benchmarking", "frame-executive", "frame-support", @@ -5849,14 +5802,17 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", + "pallet-aura", "pallet-glutton", "pallet-sudo", + "pallet-timestamp", "parachain-info", "parachains-common", "parity-scale-codec", "scale-info", "sp-api", "sp-block-builder", + "sp-consensus-aura", "sp-core", "sp-inherents", "sp-offchain", @@ -6004,12 +5960,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - [[package]] name = "hex-literal" version = "0.4.1" @@ -6941,6 +6891,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "layout-rs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1164ef87cb9607c2d887216eca79f0fc92895affe1789bba805dd38d829584e0" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -7130,7 +7089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "276bb57e7af15d8f100d3c11cbdd32c6752b7eef4ba7a18ecf464972c07abcce" dependencies = [ "bs58 0.4.0", - "ed25519-dalek 2.0.0", + "ed25519-dalek", "log", "multiaddr", "multihash", @@ -7660,7 +7619,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -7674,7 +7633,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -7685,7 +7644,7 @@ checksum = "c12469fc165526520dff2807c2975310ab47cf7190a45b99b49a7dc8befab17b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -7696,7 +7655,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -7720,15 +7679,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - [[package]] name = "matches" version = "0.1.10" @@ -8192,7 +8142,7 @@ name = "node-bench" version = "0.9.0-dev" dependencies = [ "array-bytes", - "clap 4.4.2", + "clap 4.4.4", "derive_more", "fs_extra", "futures", @@ -8229,7 +8179,7 @@ version = "3.0.0-dev" dependencies = [ "array-bytes", "assert_cmd", - "clap 4.4.2", + "clap 4.4.4", "clap_complete", "criterion 0.4.0", "frame-benchmarking-cli", @@ -8355,7 +8305,7 @@ dependencies = [ name = "node-inspect" version = "0.9.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "parity-scale-codec", "sc-cli", "sc-client-api", @@ -8409,7 +8359,7 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "generate-bags", "kitchensink-runtime", ] @@ -8418,7 +8368,7 @@ dependencies = [ name = "node-template" version = "4.0.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "frame-benchmarking", "frame-benchmarking-cli", "frame-system", @@ -8461,7 +8411,7 @@ dependencies = [ name = "node-template-release" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "flate2", "fs_extra", "glob", @@ -8572,16 +8522,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num" version = "0.4.1" @@ -8756,9 +8696,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "orchestra" -version = "0.0.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "227585216d05ba65c7ab0a0450a3cf2cbd81a98862a54c4df8e14d5ac6adb015" +checksum = "46d78e1deb2a8d54fc1f063a544130db4da31dfe4d5d3b493186424910222a76" dependencies = [ "async-trait", "dyn-clonable", @@ -8773,12 +8713,16 @@ dependencies = [ [[package]] name = "orchestra-proc-macro" -version = "0.0.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2871aadd82a2c216ee68a69837a526dfe788ecbe74c4c5038a6acdbff6653066" +checksum = "d035b1f968d91a826f2e34a9d6d02cb2af5aa7ca39ebd27922d850ab4b2dd2c6" dependencies = [ - "expander 0.0.6", - "itertools 0.10.5", + "anyhow", + "expander 2.0.0", + "fs-err", + "indexmap 2.0.0", + "itertools 0.11.0", + "layout-rs", "petgraph", "proc-macro-crate", "proc-macro2", @@ -8801,12 +8745,6 @@ version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "3.5.0" @@ -9346,6 +9284,21 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-collective-content" +version = "0.1.0" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-contracts" version = "4.0.0-dev" @@ -9403,7 +9356,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -10474,7 +10427,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -10540,6 +10493,7 @@ dependencies = [ name = "pallet-sudo" version = "4.0.0-dev" dependencies = [ + "docify", "frame-benchmarking", "frame-support", "frame-system", @@ -10570,6 +10524,7 @@ dependencies = [ name = "pallet-timestamp" version = "4.0.0-dev" dependencies = [ + "docify", "frame-benchmarking", "frame-support", "frame-system", @@ -10862,7 +10817,7 @@ dependencies = [ name = "parachain-template-node" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "color-print", "cumulus-client-cli", "cumulus-client-collator", @@ -10934,7 +10889,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-aura", "pallet-authorship", @@ -10978,6 +10933,7 @@ dependencies = [ "cumulus-primitives-utility", "frame-support", "frame-system", + "kusama-runtime-constants", "log", "num-traits", "pallet-asset-tx-payment", @@ -10986,8 +10942,12 @@ dependencies = [ "pallet-balances", "pallet-collator-selection", "parity-scale-codec", + "polkadot-core-primitives", "polkadot-primitives", + "polkadot-runtime-constants", + "rococo-runtime-constants", "scale-info", + "smallvec", "sp-consensus-aura", "sp-core", "sp-io", @@ -10997,6 +10957,7 @@ dependencies = [ "staging-xcm-builder", "staging-xcm-executor", "substrate-wasm-builder", + "westend-runtime-constants", ] [[package]] @@ -11012,7 +10973,7 @@ dependencies = [ "cumulus-test-relay-sproof-builder", "frame-support", "frame-system", - "hex-literal 0.4.1", + "hex-literal", "pallet-assets", "pallet-balances", "pallet-collator-selection", @@ -11257,7 +11218,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-asset-tx-payment", "pallet-assets", @@ -11333,7 +11294,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -11374,7 +11335,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -11457,7 +11418,7 @@ dependencies = [ [[package]] name = "polkadot" -version = "1.0.0" +version = "1.1.0" dependencies = [ "assert_cmd", "color-eyre", @@ -11564,6 +11525,7 @@ name = "polkadot-availability-recovery" version = "1.0.0" dependencies = [ "assert_matches", + "async-trait", "env_logger 0.9.3", "fatality", "futures", @@ -11590,9 +11552,9 @@ dependencies = [ [[package]] name = "polkadot-cli" -version = "1.0.0" +version = "1.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "frame-benchmarking-cli", "futures", "log", @@ -12055,7 +12017,7 @@ dependencies = [ "assert_matches", "futures", "futures-timer", - "hex-literal 0.3.4", + "hex-literal", "libc", "parity-scale-codec", "pin-project", @@ -12072,7 +12034,6 @@ dependencies = [ "slotmap", "sp-core", "sp-maybe-compressed-blob", - "sp-tracing", "sp-wasm-interface", "substrate-build-script-utils", "tempfile", @@ -12297,9 +12258,10 @@ dependencies = [ "parking_lot 0.12.1", "polkadot-node-subsystem", "polkadot-node-subsystem-util", - "polkadot-overseer", "polkadot-primitives", + "sc-client-api", "sc-keystore", + "sc-utils", "sp-application-crypto", "sp-core", "sp-keyring", @@ -12319,6 +12281,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-primitives", "polkadot-statement-table", + "sc-client-api", "sc-network", "sc-transaction-pool-api", "smallvec", @@ -12356,11 +12319,13 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", "prioritized-metered-channel", "rand 0.8.5", + "sc-client-api", "schnellru", "sp-application-crypto", "sp-core", @@ -12384,6 +12349,7 @@ dependencies = [ "polkadot-node-metrics", "polkadot-node-network-protocol", "polkadot-node-primitives", + "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-types", "polkadot-primitives", "polkadot-primitives-test-helpers", @@ -12398,7 +12364,7 @@ dependencies = [ [[package]] name = "polkadot-parachain-bin" -version = "1.0.0" +version = "1.1.0" dependencies = [ "assert_cmd", "asset-hub-kusama-runtime", @@ -12408,7 +12374,7 @@ dependencies = [ "bridge-hub-kusama-runtime", "bridge-hub-polkadot-runtime", "bridge-hub-rococo-runtime", - "clap 4.4.2", + "clap 4.4.4", "collectives-polkadot-runtime", "color-print", "contracts-rococo-runtime", @@ -12426,7 +12392,7 @@ dependencies = [ "frame-benchmarking-cli", "futures", "glutton-runtime", - "hex-literal 0.4.1", + "hex-literal", "jsonrpsee", "log", "nix 0.26.2", @@ -12517,7 +12483,7 @@ name = "polkadot-primitives" version = "1.0.0" dependencies = [ "bitvec", - "hex-literal 0.4.1", + "hex-literal", "parity-scale-codec", "polkadot-core-primitives", "polkadot-parachain-primitives", @@ -12594,7 +12560,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-authority-discovery", "pallet-authorship", @@ -12694,7 +12660,7 @@ dependencies = [ "frame-support", "frame-support-test", "frame-system", - "hex-literal 0.4.1", + "hex-literal", "impl-trait-for-tuples", "libsecp256k1", "log", @@ -12773,7 +12739,8 @@ dependencies = [ "frame-support-test", "frame-system", "futures", - "hex-literal 0.4.1", + "hex-literal", + "impl-trait-for-tuples", "log", "pallet-authority-discovery", "pallet-authorship", @@ -12785,6 +12752,7 @@ dependencies = [ "pallet-timestamp", "pallet-vesting", "parity-scale-codec", + "polkadot-core-primitives", "polkadot-parachain-primitives", "polkadot-primitives", "polkadot-primitives-test-helpers", @@ -12827,9 +12795,8 @@ dependencies = [ "frame-system", "frame-system-rpc-runtime-api", "futures", - "hex-literal 0.4.1", + "hex-literal", "is_executable", - "kusama-runtime-constants", "kvdb", "kvdb-rocksdb", "log", @@ -12875,9 +12842,6 @@ dependencies = [ "polkadot-parachain-primitives", "polkadot-primitives", "polkadot-rpc", - "polkadot-runtime", - "polkadot-runtime-common", - "polkadot-runtime-constants", "polkadot-runtime-parachains", "polkadot-statement-distribution", "polkadot-test-client", @@ -12933,7 +12897,6 @@ dependencies = [ "sp-transaction-pool", "sp-version", "sp-weights", - "staging-kusama-runtime", "substrate-prometheus-endpoint", "tempfile", "thiserror", @@ -13021,7 +12984,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.2", + "clap 4.4.4", "color-eyre", "futures", "futures-timer", @@ -13056,7 +13019,7 @@ dependencies = [ "frame-support", "frame-system", "frame-system-rpc-runtime-api", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-authority-discovery", "pallet-authorship", @@ -13167,7 +13130,7 @@ dependencies = [ name = "polkadot-voter-bags" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "generate-bags", "polkadot-runtime", "sp-io", @@ -13347,7 +13310,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -13367,9 +13330,9 @@ dependencies = [ [[package]] name = "prioritized-metered-channel" -version = "0.2.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382698e48a268c832d0b181ed438374a6bb708a82a8ca273bb0f61c74cf209c4" +checksum = "e99f0c89bd88f393aab44a4ab949351f7bc7e7e1179d11ecbfe50cbe4c47e342" dependencies = [ "coarsetime", "crossbeam-queue", @@ -13429,14 +13392,14 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -13475,7 +13438,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -13790,7 +13753,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.27", + "time", "x509-parser 0.13.2", "yasna", ] @@ -13803,7 +13766,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.27", + "time", "yasna", ] @@ -13866,7 +13829,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -13929,7 +13892,7 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" name = "remote-ext-tests-bags-list" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "frame-system", "kusama-runtime-constants", "log", @@ -14017,13 +13980,14 @@ dependencies = [ [[package]] name = "ring" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof?rev=0e948f3#0e948f3c28cbacecdd3020403c4841c0eb339213" +source = "git+https://github.com/w3f/ring-proof?rev=8657210#86572101f4210647984ab4efedba6b3fcc890895" dependencies = [ "ark-ec", "ark-ff", "ark-poly", "ark-serialize", "ark-std", + "blake2", "common", "fflonk", "merlin 3.0.0", @@ -14130,7 +14094,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-authority-discovery", "pallet-authorship", @@ -14628,7 +14592,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -14637,7 +14601,7 @@ version = "0.10.0-dev" dependencies = [ "array-bytes", "chrono", - "clap 4.4.2", + "clap 4.4.4", "fdlimit", "futures", "futures-timer", @@ -14695,6 +14659,7 @@ dependencies = [ "sp-statement-store", "sp-storage", "sp-test-primitives", + "sp-trie", "substrate-prometheus-endpoint", "substrate-test-runtime", "thiserror", @@ -14809,17 +14774,14 @@ dependencies = [ "num-traits", "parity-scale-codec", "parking_lot 0.12.1", - "rand_chacha 0.2.2", "sc-block-builder", "sc-client-api", "sc-consensus", "sc-consensus-epochs", "sc-consensus-slots", - "sc-network", "sc-network-test", "sc-telemetry", "sc-transaction-pool-api", - "scale-info", "sp-api", "sp-application-crypto", "sp-block-builder", @@ -15133,7 +15095,7 @@ dependencies = [ "substrate-test-runtime", "tempfile", "tracing", - "tracing-subscriber 0.2.25", + "tracing-subscriber", "wat", ] @@ -15741,7 +15703,7 @@ dependencies = [ name = "sc-storage-monitor" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "fs4", "log", "sc-client-db", @@ -15831,7 +15793,7 @@ dependencies = [ "thiserror", "tracing", "tracing-log", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -15841,7 +15803,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -16114,20 +16076,25 @@ dependencies = [ name = "seedling-runtime" version = "0.1.0" dependencies = [ + "cumulus-pallet-aura-ext", "cumulus-pallet-parachain-system", "cumulus-pallet-solo-to-para", "cumulus-primitives-core", + "cumulus-primitives-timestamp", "frame-executive", "frame-support", "frame-system", + "pallet-aura", "pallet-balances", "pallet-sudo", + "pallet-timestamp", "parachain-info", "parachains-common", "parity-scale-codec", "scale-info", "sp-api", "sp-block-builder", + "sp-consensus-aura", "sp-core", "sp-inherents", "sp-offchain", @@ -16195,7 +16162,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -16209,9 +16176,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -16261,7 +16228,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -16358,6 +16325,7 @@ dependencies = [ name = "shell-runtime" version = "0.1.0" dependencies = [ + "cumulus-pallet-aura-ext", "cumulus-pallet-parachain-system", "cumulus-pallet-xcm", "cumulus-primitives-core", @@ -16365,12 +16333,15 @@ dependencies = [ "frame-support", "frame-system", "frame-try-runtime", + "pallet-aura", + "pallet-timestamp", "parachain-info", "parachains-common", "parity-scale-codec", "scale-info", "sp-api", "sp-block-builder", + "sp-consensus-aura", "sp-core", "sp-inherents", "sp-offchain", @@ -16410,18 +16381,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signal-hook-tokio" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e" -dependencies = [ - "futures-core", - "libc", - "signal-hook", - "tokio", -] - [[package]] name = "signature" version = "1.6.4" @@ -16701,7 +16660,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -17039,7 +16998,6 @@ name = "sp-core" version = "21.0.0" dependencies = [ "array-bytes", - "arrayvec 0.7.4", "bandersnatch_vrfs", "bitflags 1.3.2", "blake2", @@ -17102,7 +17060,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -17146,7 +17104,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -17188,7 +17146,7 @@ name = "sp-io" version = "23.0.0" dependencies = [ "bytes", - "ed25519-dalek 2.0.0", + "ed25519-dalek", "libsecp256k1", "log", "parity-scale-codec", @@ -17284,7 +17242,7 @@ dependencies = [ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "honggfuzz", "rand 0.8.5", "sp-npos-elections", @@ -17377,7 +17335,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -17475,7 +17433,7 @@ version = "4.0.0-dev" dependencies = [ "aes-gcm 0.10.2", "curve25519-dalek 4.0.0", - "ed25519-dalek 2.0.0", + "ed25519-dalek", "hkdf", "parity-scale-codec", "rand 0.8.5", @@ -17541,7 +17499,7 @@ dependencies = [ "sp-std", "tracing", "tracing-core", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -17617,7 +17575,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -17725,7 +17683,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "kusama-runtime-constants", "log", "pallet-authority-discovery", @@ -17823,47 +17781,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "staging-staking-miner" -version = "1.0.0" -dependencies = [ - "assert_cmd", - "clap 4.4.2", - "exitcode", - "frame-election-provider-support", - "frame-remote-externalities", - "frame-support", - "frame-system", - "futures-util", - "jsonrpsee", - "log", - "pallet-balances", - "pallet-election-provider-multi-phase", - "pallet-staking", - "pallet-transaction-payment", - "parity-scale-codec", - "paste", - "polkadot-core-primitives", - "polkadot-runtime", - "polkadot-runtime-common", - "sc-transaction-pool-api", - "serde", - "serde_json", - "signal-hook", - "signal-hook-tokio", - "sp-core", - "sp-npos-elections", - "sp-runtime", - "sp-state-machine", - "sp-version", - "staging-kusama-runtime", - "sub-tokens", - "thiserror", - "tokio", - "tracing-subscriber 0.3.17", - "westend-runtime", -] - [[package]] name = "staging-xcm" version = "1.0.0" @@ -17872,7 +17789,7 @@ dependencies = [ "derivative", "environmental", "hex", - "hex-literal 0.4.1", + "hex-literal", "impl-trait-for-tuples", "log", "parity-scale-codec", @@ -18038,19 +17955,11 @@ dependencies = [ "webrtc-util", ] -[[package]] -name = "sub-tokens" -version = "0.1.0" -source = "git+https://github.com/paritytech/substrate-debug-kit?branch=master#e12503ab781e913735dc389865a3b8b4a6c6399d" -dependencies = [ - "separator", -] - [[package]] name = "subkey" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "sc-cli", ] @@ -18092,7 +18001,7 @@ dependencies = [ name = "substrate-frame-cli" version = "4.0.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "frame-support", "frame-system", "sc-cli", @@ -18441,9 +18350,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -18551,7 +18460,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "futures", "futures-timer", "log", @@ -18570,7 +18479,6 @@ dependencies = [ "sp-keyring", "substrate-test-utils", "test-parachain-adder", - "test-parachain-adder-collator", "tokio", ] @@ -18600,7 +18508,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.4", "futures", "futures-timer", "log", @@ -18619,7 +18527,6 @@ dependencies = [ "sp-keyring", "substrate-test-utils", "test-parachain-undying", - "test-parachain-undying-collator", "tokio", ] @@ -18655,9 +18562,9 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] @@ -18684,13 +18591,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.47" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -18762,17 +18669,6 @@ dependencies = [ "tikv-jemalloc-sys", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.27" @@ -18881,7 +18777,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -19062,7 +18958,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -19105,7 +19001,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -19138,7 +19034,7 @@ dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers 0.0.1", + "matchers", "parking_lot 0.11.2", "regex", "serde", @@ -19152,29 +19048,11 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "tracing-subscriber" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" -dependencies = [ - "matchers 0.1.0", - "nu-ansi-term", - "once_cell", - "regex", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - [[package]] name = "trie-bench" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f54b4f9d51d368e62cf7e0730c7c1e18fc658cc84333656bab5b328f44aa964" +checksum = "a4680cb226e31d2a096592d0edecdda91cc371743002f80c0f8cf80219819b3b" dependencies = [ "criterion 0.4.0", "hash-db", @@ -19188,9 +19066,9 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767abe6ffed88a1889671a102c2861ae742726f52e0a5a425b92c9fbfa7e9c85" +checksum = "ff28e0f815c2fea41ebddf148e008b077d2faddb026c9555b29696114d602642" dependencies = [ "hash-db", "hashbrown 0.13.2", @@ -19276,7 +19154,7 @@ version = "0.10.0-dev" dependencies = [ "assert_cmd", "async-trait", - "clap 4.4.2", + "clap 4.4.4", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -19643,12 +19521,6 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -19678,7 +19550,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -19712,7 +19584,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -20151,7 +20023,7 @@ dependencies = [ "sha2 0.10.7", "stun", "thiserror", - "time 0.3.27", + "time", "tokio", "turn", "url", @@ -20348,7 +20220,7 @@ dependencies = [ "frame-system-benchmarking", "frame-system-rpc-runtime-api", "frame-try-runtime", - "hex-literal 0.4.1", + "hex-literal", "log", "pallet-authority-discovery", "pallet-authorship", @@ -20759,7 +20631,7 @@ dependencies = [ "ring 0.16.20", "rusticata-macros", "thiserror", - "time 0.3.27", + "time", ] [[package]] @@ -20777,7 +20649,7 @@ dependencies = [ "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.27", + "time", ] [[package]] @@ -20848,7 +20720,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] @@ -20947,7 +20819,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.27", + "time", ] [[package]] @@ -20967,7 +20839,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.37", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4db27b98e90788775663aed31cbc10c851316318..d1078e3c86a82a3129e2d8cb2bf965fdbf0f36d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,31 +8,31 @@ license = "GPL-3.0-only" resolver = "2" members = [ - "cumulus/bridges/bin/runtime-common", - "cumulus/bridges/modules/grandpa", - "cumulus/bridges/modules/messages", - "cumulus/bridges/modules/parachains", - "cumulus/bridges/modules/relayers", - "cumulus/bridges/modules/xcm-bridge-hub-router", - "cumulus/bridges/primitives/chain-asset-hub-kusama", - "cumulus/bridges/primitives/chain-asset-hub-polkadot", - "cumulus/bridges/primitives/chain-bridge-hub-cumulus", - "cumulus/bridges/primitives/chain-bridge-hub-kusama", - "cumulus/bridges/primitives/chain-bridge-hub-polkadot", - "cumulus/bridges/primitives/chain-bridge-hub-rococo", - "cumulus/bridges/primitives/chain-bridge-hub-wococo", - "cumulus/bridges/primitives/chain-kusama", - "cumulus/bridges/primitives/chain-polkadot", - "cumulus/bridges/primitives/chain-rococo", - "cumulus/bridges/primitives/chain-wococo", - "cumulus/bridges/primitives/header-chain", - "cumulus/bridges/primitives/messages", - "cumulus/bridges/primitives/parachains", - "cumulus/bridges/primitives/polkadot-core", - "cumulus/bridges/primitives/relayers", - "cumulus/bridges/primitives/runtime", - "cumulus/bridges/primitives/test-utils", - "cumulus/bridges/primitives/xcm-bridge-hub-router", + "bridges/bin/runtime-common", + "bridges/modules/grandpa", + "bridges/modules/messages", + "bridges/modules/parachains", + "bridges/modules/relayers", + "bridges/modules/xcm-bridge-hub-router", + "bridges/primitives/chain-asset-hub-kusama", + "bridges/primitives/chain-asset-hub-polkadot", + "bridges/primitives/chain-bridge-hub-cumulus", + "bridges/primitives/chain-bridge-hub-kusama", + "bridges/primitives/chain-bridge-hub-polkadot", + "bridges/primitives/chain-bridge-hub-rococo", + "bridges/primitives/chain-bridge-hub-wococo", + "bridges/primitives/chain-kusama", + "bridges/primitives/chain-polkadot", + "bridges/primitives/chain-rococo", + "bridges/primitives/chain-wococo", + "bridges/primitives/header-chain", + "bridges/primitives/messages", + "bridges/primitives/parachains", + "bridges/primitives/polkadot-core", + "bridges/primitives/relayers", + "bridges/primitives/runtime", + "bridges/primitives/test-utils", + "bridges/primitives/xcm-bridge-hub-router", "cumulus/client/cli", "cumulus/client/collator", "cumulus/client/consensus/aura", @@ -92,7 +92,6 @@ members = [ "cumulus/primitives/utility", "cumulus/test/client", "cumulus/test/relay-sproof-builder", - "cumulus/test/relay-validation-worker-provider", "cumulus/test/runtime", "cumulus/test/service", "cumulus/xcm/xcm-emulator", @@ -172,7 +171,6 @@ members = [ "polkadot/statement-table", "polkadot/utils/generate-bags", "polkadot/utils/remote-ext-tests/bags-list", - "polkadot/utils/staking-miner", "polkadot/xcm", "polkadot/xcm/pallet-xcm", "polkadot/xcm/pallet-xcm-benchmarks", @@ -452,6 +450,7 @@ members = [ "substrate/utils/prometheus", "substrate/utils/wasm-builder", ] +default-members = [ "polkadot" ] [profile.release] # Polkadot runtime requires unwinding. diff --git a/README.md b/README.md index a0ef181827983bf89b32f8f955e470f222eb2864..56b3481bafc038f7d3dd0606558e9e37243dcd53 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -> NOTE: We have recently made significant changes to our repository structure. In order to -streamline our development process and foster better contributions, we have merged three separate -repositories Cumulus, Substrate and Polkadot into this repository. Read more about the changes [ +> NOTE: We have recently made significant changes to our repository structure. In order to streamline our development +process and foster better contributions, we have merged three separate repositories Cumulus, Substrate and Polkadot into +this repository. Read more about the changes [ here](https://polkadot-public.notion.site/Polkadot-SDK-FAQ-fbc4cecc2c46443fb37b9eeec2f0d85f). # Polkadot SDK @@ -9,27 +9,29 @@ here](https://polkadot-public.notion.site/Polkadot-SDK-FAQ-fbc4cecc2c46443fb37b9 [![StackExchange](https://img.shields.io/badge/StackExchange-Community%20&%20Support-222222?logo=stackexchange)](https://substrate.stackexchange.com/) -The Polkadot SDK repository provides all the resources needed to start building on the Polkadot -network, a multi-chain blockchain platform that enables different blockchains to interoperate and -share information in a secure and scalable way. The Polkadot SDK comprises three main pieces of -software: +The Polkadot SDK repository provides all the resources needed to start building on the Polkadot network, a multi-chain +blockchain platform that enables different blockchains to interoperate and share information in a secure and scalable +way. The Polkadot SDK comprises three main pieces of software: ## [Polkadot](./polkadot/) -[![PolkadotForum](https://img.shields.io/badge/Polkadot_Forum-e6007a?logo=polkadot)](https://forum.polkadot.network/) [![Polkadot-license](https://img.shields.io/badge/License-GPL3-blue)](./polkadot/LICENSE) +[![PolkadotForum](https://img.shields.io/badge/Polkadot_Forum-e6007a?logo=polkadot)](https://forum.polkadot.network/) +[![Polkadot-license](https://img.shields.io/badge/License-GPL3-blue)](./polkadot/LICENSE) -Implementation of a node for the https://polkadot.network in Rust, using the Substrate framework. -This directory currently contains runtimes for the Polkadot, Kusama, Westend, and Rococo networks. -In the future, these will be relocated to the [`runtimes`](https://github.com/polkadot-fellows/runtimes/) repository. +Implementation of a node for the https://polkadot.network in Rust, using the Substrate framework. This directory +currently contains runtimes for the Polkadot, Kusama, Westend, and Rococo networks. In the future, these will be +relocated to the [`runtimes`](https://github.com/polkadot-fellows/runtimes/) repository. ## [Substrate](./substrate/) - [![SubstrateRustDocs](https://img.shields.io/badge/Rust_Docs-Substrate-24CC85?logo=rust)](https://paritytech.github.io/substrate/master/substrate/index.html) [![Substrate-license](https://img.shields.io/badge/License-GPL3%2FApache2.0-blue)](./substrate/README.md#LICENSE) + [![SubstrateRustDocs](https://img.shields.io/badge/Rust_Docs-Substrate-24CC85?logo=rust)](https://paritytech.github.io/substrate/master/substrate/index.html) + [![Substrate-license](https://img.shields.io/badge/License-GPL3%2FApache2.0-blue)](./substrate/README.md#LICENSE) -Substrate is the primary blockchain SDK used by developers to create the parachains that make up -the Polkadot network. Additionally, it allows for the development of self-sovereign blockchains -that operate completely independently of Polkadot. +Substrate is the primary blockchain SDK used by developers to create the parachains that make up the Polkadot network. +Additionally, it allows for the development of self-sovereign blockchains that operate completely independently of +Polkadot. ## [Cumulus](./cumulus/) -[![CumulusRustDocs](https://img.shields.io/badge/Rust_Docs-Cumulus-222222?logo=rust)](https://paritytech.github.io/cumulus/cumulus_client_collator/index.html) [![Cumulus-license](https://img.shields.io/badge/License-GPL3-blue)](./cumulus/LICENSE) +[![CumulusRustDocs](https://img.shields.io/badge/Rust_Docs-Cumulus-222222?logo=rust)](https://paritytech.github.io/cumulus/cumulus_client_collator/index.html) +[![Cumulus-license](https://img.shields.io/badge/License-GPL3-blue)](./cumulus/LICENSE) Cumulus is a set of tools for writing Substrate-based Polkadot parachains. @@ -37,10 +39,10 @@ Cumulus is a set of tools for writing Substrate-based Polkadot parachains. Below are the primary upstream dependencies utilized in this project: -- [parity-scale-codec](https://crates.io/crates/parity-scale-codec) -- [parity-db](https://crates.io/crates/parity-db) -- [parity-common](https://github.com/paritytech/parity-common) -- [trie](https://github.com/paritytech/trie) +- [`parity-scale-codec`](https://crates.io/crates/parity-scale-codec) +- [`parity-db`](https://crates.io/crates/parity-db) +- [`parity-common`](https://github.com/paritytech/parity-common) +- [`trie`](https://github.com/paritytech/trie) ## Security @@ -48,9 +50,11 @@ The security policy and procedures can be found in [docs/SECURITY.md](./docs/SEC ## Contributing & Code of Conduct -Ensure you follow our [contribution guidelines](./docs/CONTRIBUTING.md). In every interaction and contribution, this project adheres to the [Contributor Covenant Code of Conduct](./docs/CODE_OF_CONDUCT.md). +Ensure you follow our [contribution guidelines](./docs/CONTRIBUTING.md). In every interaction and contribution, this +project adheres to the [Contributor Covenant Code of Conduct](./docs/CODE_OF_CONDUCT.md). ## Additional Resources -- For monitoring upcoming changes and current proposals related to the technical implementation of the Polkadot network, visit the [`Requests for Comment (RFC)`](https://github.com/polkadot-fellows/RFCs) repository. While it's maintained by the Polkadot Fellowship, the RFC process welcomes contributions from everyone. - +- For monitoring upcoming changes and current proposals related to the technical implementation of the Polkadot network, + visit the [`Requests for Comment (RFC)`](https://github.com/polkadot-fellows/RFCs) repository. While it's maintained + by the Polkadot Fellowship, the RFC process welcomes contributions from everyone. diff --git a/cumulus/bridges/.gitignore b/bridges/.gitignore similarity index 100% rename from cumulus/bridges/.gitignore rename to bridges/.gitignore diff --git a/cumulus/bridges/CODE_OF_CONDUCT.md b/bridges/CODE_OF_CONDUCT.md similarity index 96% rename from cumulus/bridges/CODE_OF_CONDUCT.md rename to bridges/CODE_OF_CONDUCT.md index 70541fb72fa25af86a4268b947cdeb880d1d739c..23411da2e048c758d56d511c792e020a37d0ee0d 100644 --- a/cumulus/bridges/CODE_OF_CONDUCT.md +++ b/bridges/CODE_OF_CONDUCT.md @@ -34,9 +34,9 @@ of preference. We see that blockchains are naturally community platforms with u ultimate decision makers. We assert that good software will maximise user agency by facilitate user-expression on the network. As such: -- This project will strive to give users as much choice as is both reasonable and possible over what +* This project will strive to give users as much choice as is both reasonable and possible over what protocol they adhere to; but -- use of the project's technical forums, commenting systems, pull requests and issue trackers as a +* use of the project's technical forums, commenting systems, pull requests and issue trackers as a means to express individual protocol preferences is forbidden. ## Our Responsibilities diff --git a/cumulus/bridges/LICENSE b/bridges/LICENSE similarity index 100% rename from cumulus/bridges/LICENSE rename to bridges/LICENSE diff --git a/cumulus/bridges/README.md b/bridges/README.md similarity index 70% rename from cumulus/bridges/README.md rename to bridges/README.md index 2f8c5ca9abb293699fc9510ed60ef4be117dd783..da46fe67d924acb2afffcf971bacb60b560f0cd5 100644 --- a/cumulus/bridges/README.md +++ b/bridges/README.md @@ -2,11 +2,10 @@ This is a collection of components for building bridges. -These components include Substrate pallets for syncing headers, passing arbitrary messages, as well -as libraries for building relayers to provide cross-chain communication capabilities. +These components include Substrate pallets for syncing headers, passing arbitrary messages, as well as libraries for +building relayers to provide cross-chain communication capabilities. -Three bridge nodes are also available. The nodes can be used to run test networks which bridge other -Substrate chains. +Three bridge nodes are also available. The nodes can be used to run test networks which bridge other Substrate chains. 🚧 The bridges are currently under construction - a hardhat is recommended beyond this point 🚧 @@ -21,8 +20,8 @@ Substrate chains. ## Installation -To get up and running you need both stable and nightly Rust. Rust nightly is used to build the Web -Assembly (WASM) runtime for the node. You can configure the WASM support as so: +To get up and running you need both stable and nightly Rust. Rust nightly is used to build the Web Assembly (WASM) +runtime for the node. You can configure the WASM support as so: ```bash rustup install nightly @@ -38,8 +37,8 @@ cargo build --all cargo test --all ``` -Also you can build the repo with -[Parity CI Docker image](https://github.com/paritytech/scripts/tree/master/dockerfiles/bridges-ci): +Also you can build the repo with [Parity CI Docker +image](https://github.com/paritytech/scripts/tree/master/dockerfiles/bridges-ci): ```bash docker pull paritytech/bridges-ci:production @@ -57,16 +56,14 @@ docker run --rm -it -w /shellhere/parity-bridges-common \ If you want to reproduce other steps of CI process you can use the following [guide](https://github.com/paritytech/scripts#reproduce-ci-locally). -If you need more information about setting up your development environment [Substrate's -Installation page](https://docs.substrate.io/main-docs/install/) is a good -resource. +If you need more information about setting up your development environment [Substrate's Installation +page](https://docs.substrate.io/main-docs/install/) is a good resource. ## High-Level Architecture -This repo has support for bridging foreign chains together using a combination of Substrate pallets -and external processes called relayers. A bridge chain is one that is able to follow the consensus -of a foreign chain independently. For example, consider the case below where we want to bridge two -Substrate based chains. +This repo has support for bridging foreign chains together using a combination of Substrate pallets and external +processes called relayers. A bridge chain is one that is able to follow the consensus of a foreign chain independently. +For example, consider the case below where we want to bridge two Substrate based chains. ``` +---------------+ +---------------+ @@ -82,19 +79,19 @@ Substrate based chains. +---------------+ ``` -The Millau chain must be able to accept Rialto headers and verify their integrity. It does this by -using a runtime module designed to track GRANDPA finality. Since two blockchains can't interact -directly they need an external service, called a relayer, to communicate. The relayer will subscribe -to new Rialto headers via RPC and submit them to the Millau chain for verification. +The Millau chain must be able to accept Rialto headers and verify their integrity. It does this by using a runtime +module designed to track GRANDPA finality. Since two blockchains can't interact directly they need an external service, +called a relayer, to communicate. The relayer will subscribe to new Rialto headers via RPC and submit them to the Millau +chain for verification. -Take a look at [Bridge High Level Documentation](./docs/high-level-overview.md) for more in-depth -description of the bridge interaction. +Take a look at [Bridge High Level Documentation](./docs/high-level-overview.md) for more in-depth description of the +bridge interaction. ## Project Layout -Here's an overview of how the project is laid out. The main bits are the `bin`, which is the actual -"blockchain", the `modules` which are used to build the blockchain's logic (a.k.a the runtime) and -the `relays` which are used to pass messages between chains. +Here's an overview of how the project is laid out. The main bits are the `bin`, which is the actual "blockchain", the +`modules` which are used to build the blockchain's logic (a.k.a the runtime) and the `relays` which are used to pass +messages between chains. ``` ├── bin // Node and Runtime for the various Substrate chains @@ -117,16 +114,16 @@ the `relays` which are used to pass messages between chains. ## Running the Bridge -To run the Bridge you need to be able to connect the bridge relay node to the RPC interface of nodes -on each side of the bridge (source and target chain). +To run the Bridge you need to be able to connect the bridge relay node to the RPC interface of nodes on each side of the +bridge (source and target chain). There are 2 ways to run the bridge, described below: -- building & running from source: with this option, you'll be able to run the bridge between two standalone -chains that are running GRANDPA finality gadget to achieve finality; +- building & running from source: with this option, you'll be able to run the bridge between two standalone chains that +are running GRANDPA finality gadget to achieve finality; -- running a Docker Compose setup: this is a recommended option, where you'll see bridges with parachains, -complex relays and more. +- running a Docker Compose setup: this is a recommended option, where you'll see bridges with parachains, complex relays +and more. ### Using the Source @@ -141,16 +138,15 @@ cargo build -p substrate-relay ### Running a Dev network -We will launch a dev network to demonstrate how to relay a message between two Substrate based -chains (named Rialto and Millau). +We will launch a dev network to demonstrate how to relay a message between two Substrate based chains (named Rialto and +Millau). -To do this we will need two nodes, two relayers which will relay headers, and two relayers which -will relay messages. +To do this we will need two nodes, two relayers which will relay headers, and two relayers which will relay messages. #### Running from local scripts -To run a simple dev network you can use the scripts located in the -[`deployments/local-scripts` folder](./deployments/local-scripts). +To run a simple dev network you can use the scripts located in the [`deployments/local-scripts` +folder](./deployments/local-scripts). First, we must run the two Substrate nodes. @@ -167,8 +163,8 @@ After the nodes are up we can run the header relayers. ./deployments/local-scripts/relay-rialto-to-millau.sh ``` -At this point you should see the relayer submitting headers from the Millau Substrate chain to the -Rialto Substrate chain. +At this point you should see the relayer submitting headers from the Millau Substrate chain to the Rialto Substrate +chain. ``` # Header Relayer Logs @@ -192,20 +188,23 @@ You will also see the message lane relayers listening for new messages. [Millau_to_Rialto_MessageLane_00000000] [date] DEBUG bridge Asking Millau::ReceivingConfirmationsDelivery about best message nonces [...] [date] INFO bridge Synced Some(2) of Some(3) nonces in Millau::MessagesDelivery -> Rialto::MessagesDelivery race [...] [date] DEBUG bridge Asking Millau::MessagesDelivery about message nonces -[...] [date] DEBUG bridge Received best nonces from Millau::ReceivingConfirmationsDelivery: TargetClientNonces { latest_nonce: 0, nonces_data: () } +[...] [date] DEBUG bridge Received best nonces from Millau::ReceivingConfirmationsDelivery: TargetClientNonces { + latest_nonce: 0, nonces_data: () } [...] [date] DEBUG bridge Asking Millau::ReceivingConfirmationsDelivery about finalized message nonces -[...] [date] DEBUG bridge Received finalized nonces from Millau::ReceivingConfirmationsDelivery: TargetClientNonces { latest_nonce: 0, nonces_data: () } +[...] [date] DEBUG bridge Received finalized nonces from Millau::ReceivingConfirmationsDelivery: TargetClientNonces { + latest_nonce: 0, nonces_data: () } [...] [date] DEBUG bridge Received nonces from Millau::MessagesDelivery: SourceClientNonces { new_nonces: {}, confirmed_nonce: Some(0) } [...] [date] DEBUG bridge Asking Millau node about its state -[...] [date] DEBUG bridge Received state from Millau node: ClientState { best_self: HeaderId(1593, 0xacac***), best_finalized_self: HeaderId(1590, 0x0be81d...), best_finalized_peer_at_best_self: HeaderId(0, 0xdcdd89...) } +[...] [date] DEBUG bridge Received state from Millau node: ClientState { best_self: HeaderId(1593, 0xacac***), best_finalized_self: + HeaderId(1590, 0x0be81d...), best_finalized_peer_at_best_self: HeaderId(0, 0xdcdd89...) } ``` To send a message see the ["How to send a message" section](#how-to-send-a-message). ### How to send a message -In this section we'll show you how to quickly send a bridge message. The message is just an encoded XCM -`Trap(43)` message. +In this section we'll show you how to quickly send a bridge message. The message is just an encoded XCM `Trap(43)` +message. ```bash # In `parity-bridges-common` folder @@ -222,20 +221,20 @@ TRACE bridge Sent transaction to Millau node: 0x5e68... And at the Rialto node logs you'll something like this: ``` -... runtime::bridge-messages: Received messages: total=1, valid=1. Weight used: Weight(ref_time: 1215065371, proof_size: 48559)/Weight(ref_time: 1215065371, proof_size: 54703). -``` +... runtime::bridge-messages: Received messages: total=1, valid=1. Weight used: Weight(ref_time: 1215065371, proof_size: + 48559)/Weight(ref_time: 1215065371, proof_size: 54703). +``` -It means that the message has been delivered and dispatched. Message may be dispatched with an -error, though - the goal of our test bridge is to ensure that messages are successfully delivered -and all involved components are working. +It means that the message has been delivered and dispatched. Message may be dispatched with an error, though - the goal +of our test bridge is to ensure that messages are successfully delivered and all involved components are working. ## Full Network Docker Compose Setup -For a more sophisticated deployment which includes bidirectional header sync, message passing, -monitoring dashboards, etc. see the [Deployments README](./deployments/README.md). +For a more sophisticated deployment which includes bidirectional header sync, message passing, monitoring dashboards, +etc. see the [Deployments README](./deployments/README.md). -You should note that you can find images for all the bridge components published on -[Docker Hub](https://hub.docker.com/u/paritytech). +You should note that you can find images for all the bridge components published on [Docker +Hub](https://hub.docker.com/u/paritytech). To run a Rialto node for example, you can use the following command: @@ -247,13 +246,12 @@ docker run -p 30333:30333 -p 9933:9933 -p 9944:9944 \ ## Community -Main hangout for the community is [Element](https://element.io/) (formerly Riot). Element is a chat -server like, for example, Discord. Most discussions around Polkadot and Substrate happen -in various Element "rooms" (channels). So, joining Element might be a good idea, anyway. +Main hangout for the community is [Element](https://element.io/) (formerly Riot). Element is a chat server like, for +example, Discord. Most discussions around Polkadot and Substrate happen in various Element "rooms" (channels). So, +joining Element might be a good idea, anyway. -If you are interested in information exchange and development of Polkadot related bridges please -feel free to join the [Polkadot Bridges](https://app.element.io/#/room/#bridges:web3.foundation) -Element channel. +If you are interested in information exchange and development of Polkadot related bridges please feel free to join the +[Polkadot Bridges](https://app.element.io/#/room/#bridges:web3.foundation) Element channel. -The [Substrate Technical](https://app.element.io/#/room/#substrate-technical:matrix.org) Element -channel is most suited for discussions regarding Substrate itself. +The [Substrate Technical](https://app.element.io/#/room/#substrate-technical:matrix.org) Element channel is most suited +for discussions regarding Substrate itself. diff --git a/bridges/SECURITY.md b/bridges/SECURITY.md new file mode 100644 index 0000000000000000000000000000000000000000..9f215c88765474e6b211882296c8cf190f216780 --- /dev/null +++ b/bridges/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +Thanks for helping make the Parity ecosystem more secure. Security is one of our first priorities. + +## Reporting a vulnerability + +If you find something that can be treated as a security vulnerability, please do not use the issue tracker or discuss it +in the public forum as it can cause more damage, rather than giving real help to the ecosystem. + +Security vulnerabilities should be reported by the [contact form](https://security-submission.parity.io/). + +If you think that your report might be eligible for the Bug Bounty Program, please mark this during the submission. +Please check up-to-date [Parity Bug Bounty Program rules](https://www.parity.io/bug-bounty) to find out the information +about our Bug Bounty Program. + +**Warning**: This is an unified SECURITY.md file for Paritytech GitHub Organization. The presence of this file does not +mean that this repository is covered by the Bug Bounty program. Please always check the Bug Bounty Program scope for +information. diff --git a/cumulus/bridges/bin/runtime-common/Cargo.toml b/bridges/bin/runtime-common/Cargo.toml similarity index 72% rename from cumulus/bridges/bin/runtime-common/Cargo.toml rename to bridges/bin/runtime-common/Cargo.toml index b139f835c1a013b049f5c621f571d4377f234475..5c3fefc69ce710b1af10721ad41d1e5144b00f5a 100644 --- a/cumulus/bridges/bin/runtime-common/Cargo.toml +++ b/bridges/bin/runtime-common/Cargo.toml @@ -29,24 +29,24 @@ pallet-bridge-relayers = { path = "../../modules/relayers", default-features = f # Substrate dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -pallet-transaction-payment = { path = "../../../../substrate/frame/transaction-payment", default-features = false } -pallet-utility = { path = "../../../../substrate/frame/utility", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-io = { path = "../../../../substrate/primitives/io", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +pallet-transaction-payment = { path = "../../../substrate/frame/transaction-payment", default-features = false } +pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-io = { path = "../../../substrate/primitives/io", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } # Polkadot dependencies -xcm = { package = "staging-xcm", path = "../../../../polkadot/xcm", default-features = false } -xcm-builder = { package = "staging-xcm-builder", path = "../../../../polkadot/xcm/xcm-builder", default-features = false } +xcm = { package = "staging-xcm", path = "../../../polkadot/xcm", default-features = false } +xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder", default-features = false } [dev-dependencies] bp-test-utils = { path = "../../primitives/test-utils" } -pallet-balances = { path = "../../../../substrate/frame/balances" } +pallet-balances = { path = "../../../substrate/frame/balances" } [features] default = [ "std" ] diff --git a/cumulus/bridges/bin/runtime-common/src/integrity.rs b/bridges/bin/runtime-common/src/integrity.rs similarity index 96% rename from cumulus/bridges/bin/runtime-common/src/integrity.rs rename to bridges/bin/runtime-common/src/integrity.rs index 290c22f835d2a488e4f312f0a4811f730925fc15..d3827a14dd6cc24e088a8d05d26aba9d769eb213 100644 --- a/cumulus/bridges/bin/runtime-common/src/integrity.rs +++ b/bridges/bin/runtime-common/src/integrity.rs @@ -27,7 +27,6 @@ use codec::Encode; use frame_support::{storage::generator::StorageValue, traits::Get, weights::Weight}; use frame_system::limits; use pallet_bridge_messages::WeightInfoExt as _; -use sp_runtime::traits::SignedExtension; /// Macro that ensures that the runtime configuration and chain primitives crate are sharing /// the same types (nonce, block number, hash, hasher, account id and header). @@ -347,15 +346,3 @@ pub fn check_message_lane_weights< ); } } - -/// Check that the `AdditionalSigned` type of a wrapped runtime is the same as the one of the -/// corresponding actual runtime. -/// -/// This method doesn't perform any `assert`. If the condition is not true it will generate a -/// compile-time error. -pub fn check_additional_signed() -where - SignedExt: SignedExtension, - IndirectSignedExt: SignedExtension, -{ -} diff --git a/cumulus/bridges/bin/runtime-common/src/lib.rs b/bridges/bin/runtime-common/src/lib.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/lib.rs rename to bridges/bin/runtime-common/src/lib.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages.rs b/bridges/bin/runtime-common/src/messages.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages.rs rename to bridges/bin/runtime-common/src/messages.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_api.rs b/bridges/bin/runtime-common/src/messages_api.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_api.rs rename to bridges/bin/runtime-common/src/messages_api.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_benchmarking.rs b/bridges/bin/runtime-common/src/messages_benchmarking.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_benchmarking.rs rename to bridges/bin/runtime-common/src/messages_benchmarking.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_call_ext.rs b/bridges/bin/runtime-common/src/messages_call_ext.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_call_ext.rs rename to bridges/bin/runtime-common/src/messages_call_ext.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_generation.rs b/bridges/bin/runtime-common/src/messages_generation.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_generation.rs rename to bridges/bin/runtime-common/src/messages_generation.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_xcm_extension.rs b/bridges/bin/runtime-common/src/messages_xcm_extension.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_xcm_extension.rs rename to bridges/bin/runtime-common/src/messages_xcm_extension.rs diff --git a/cumulus/bridges/bin/runtime-common/src/mock.rs b/bridges/bin/runtime-common/src/mock.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/mock.rs rename to bridges/bin/runtime-common/src/mock.rs diff --git a/cumulus/bridges/bin/runtime-common/src/parachains_benchmarking.rs b/bridges/bin/runtime-common/src/parachains_benchmarking.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/parachains_benchmarking.rs rename to bridges/bin/runtime-common/src/parachains_benchmarking.rs diff --git a/cumulus/bridges/bin/runtime-common/src/priority_calculator.rs b/bridges/bin/runtime-common/src/priority_calculator.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/priority_calculator.rs rename to bridges/bin/runtime-common/src/priority_calculator.rs diff --git a/cumulus/bridges/bin/runtime-common/src/refund_relayer_extension.rs b/bridges/bin/runtime-common/src/refund_relayer_extension.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/refund_relayer_extension.rs rename to bridges/bin/runtime-common/src/refund_relayer_extension.rs diff --git a/cumulus/bridges/docs/complex-relay.html b/bridges/docs/complex-relay.html similarity index 100% rename from cumulus/bridges/docs/complex-relay.html rename to bridges/docs/complex-relay.html diff --git a/cumulus/bridges/docs/grandpa-finality-relay.html b/bridges/docs/grandpa-finality-relay.html similarity index 100% rename from cumulus/bridges/docs/grandpa-finality-relay.html rename to bridges/docs/grandpa-finality-relay.html diff --git a/bridges/docs/high-level-overview.md b/bridges/docs/high-level-overview.md new file mode 100644 index 0000000000000000000000000000000000000000..42efc8100bd080763c22ea3e4e813f3c3c87db37 --- /dev/null +++ b/bridges/docs/high-level-overview.md @@ -0,0 +1,184 @@ +# High-Level Bridge Documentation + +This document gives a brief, abstract description of main components that may be found in this repository. If you want +to see how we're using them to build Rococo <> Wococo (Kusama <> Polkadot) bridge, please refer to the [Polkadot <> +Kusama Bridge](./polkadot-kusama-bridge-overview.md). + +## Purpose + +This repo contains all components required to build a trustless connection between standalone Substrate chains, that are +using GRANDPA finality, their parachains or any combination of those. On top of this connection, we offer a messaging +pallet that provides means to organize messages exchange. + +On top of that layered infrastructure, anyone may build their own bridge applications - e.g. [XCM +messaging](./polkadot-kusama-bridge-overview.md), [encoded calls +messaging](https://github.com/paritytech/parity-bridges-common/releases/tag/encoded-calls-messaging) and so on. + +## Terminology + +Even though we support (and require) two-way bridging, the documentation will generally talk about a one-sided +interaction. That's to say, we will only talk about syncing finality proofs and messages from a _source_ chain to a +_target_ chain. This is because the two-sided interaction is really just the one-sided interaction with the source and +target chains switched. + +The bridge has both on-chain (pallets) and offchain (relayers) components. + +## On-chain components + +On-chain bridge components are pallets that are deployed at the chain runtime. Finality pallets require deployment at +the target chain, while messages pallet needs to be deployed at both, source and target chains. + +### Bridge GRANDPA Finality Pallet + +A GRANDPA light client of the source chain built into the target chain's runtime. It provides a "source of truth" about +the source chain headers which have been finalized. This is useful for higher level applications. + +The pallet tracks current GRANDPA authorities set and only accepts finality proofs (GRANDPA justifications), generated +by the current authorities set. The GRANDPA protocol itself requires current authorities set to generate explicit +justification for the header that enacts next authorities set. Such headers and their finality proofs are called +mandatory in the pallet and relayer pays no fee for such headers submission. + +The pallet does not require all headers to be imported or provided. The relayer itself chooses which headers he wants to +submit (with the exception of mandatory headers). + +More: [pallet level documentation and code](../modules/grandpa/). + +### Bridge Parachains Finality Pallet + +Parachains are not supposed to have their own finality, so we can't use bridge GRANDPA pallet to verify their finality +proofs. Instead, they rely on their relay chain finality. The parachain header is considered final, when it is accepted +by the [`paras` +pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras) +at its relay chain. Obviously, the relay chain block, where it is accepted, must also be finalized by the relay chain +GRANDPA gadget. + +That said, the bridge parachains pallet accepts storage proof of one or several parachain heads, inserted to the +[`Heads`](https://github.com/paritytech/polkadot/blob/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras/mod.rs#L642) +map of the [`paras` +pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras). +To verify this storage proof, the pallet uses relay chain header, imported earlier by the bridge GRANDPA pallet. + +The pallet may track multiple parachains at once and those parachains may use different primitives. So the parachain +header decoding never happens at the pallet level. For maintaining the headers order, the pallet uses relay chain header +number. + +More: [pallet level documentation and code](../modules/parachains/). + +### Bridge Messages Pallet + +The pallet is responsible for queuing messages at the source chain and receiving the messages proofs at the target +chain. The messages are sent to the particular _lane_, where they are guaranteed to be received in the same order they +are sent. The pallet supports many lanes. + +The lane has two ends. Outbound lane end is storing number of messages that have been sent and the number of messages +that have been received. Inbound lane end stores the number of messages that have been received and also a map that maps +messages to relayers that have delivered those messages to the target chain. + +The pallet has three main entrypoints: +- the `send_message` may be used by the other runtime pallets to send the messages; +- the `receive_messages_proof` is responsible for parsing the messages proof and handing messages over to the dispatch +code; +- the `receive_messages_delivery_proof` is responsible for parsing the messages delivery proof and rewarding relayers +that have delivered the message. + +Many things are abstracted by the pallet: +- the message itself may mean anything, the pallet doesn't care about its content; +- the message dispatch happens during delivery, but it is decoupled from the pallet code; +- the messages proof and messages delivery proof are verified outside of the pallet; +- the relayers incentivization scheme is defined outside of the pallet. + +Outside of the messaging pallet, we have a set of adapters, where messages and delivery proofs are regular storage +proofs. The proofs are generated at the bridged chain and require bridged chain finality. So messages pallet, in this +case, depends on one of the finality pallets. The messages are XCM messages and we are using XCM executor to dispatch +them on receival. You may find more info in [Polkadot <> Kusama Bridge](./polkadot-kusama-bridge-overview.md) document. + +More: [pallet level documentation and code](../modules/messages/). + +### Bridge Relayers Pallet + +The pallet is quite simple. It just registers relayer rewards and has an entrypoint to collect them. When the rewards +are registered and the reward amount is configured outside of the pallet. + +More: [pallet level documentation and code](../modules/relayers/). + +## Offchain Components + +Offchain bridge components are separate processes, called relayers. Relayers are connected both to the source chain and +target chain nodes. Relayers are reading state of the source chain, compare it to the state of the target chain and, if +state at target chain needs to be updated, submits target chain transaction. + +### GRANDPA Finality Relay + +The task of relay is to submit source chain GRANDPA justifications and their corresponding headers to the Bridge GRANDPA +Finality Pallet, deployed at the target chain. For that, the relay subscribes to the source chain GRANDPA justifications +stream and submits every new justification it sees to the target chain GRANDPA light client. In addition, relay is +searching for mandatory headers and submits their justifications - without that the pallet will be unable to move +forward. + +More: [GRANDPA Finality Relay Sequence Diagram](./grandpa-finality-relay.html), [pallet level documentation and +code](../relays/finality/). + +### Parachains Finality Relay + +The relay connects to the source _relay_ chain and the target chain nodes. It doesn't need to connect to the tracked +parachain nodes. The relay looks at the +[`Heads`](https://github.com/paritytech/polkadot/blob/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras/mod.rs#L642) +map of the [`paras` +pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras) +in source chain, and compares the value with the best parachain head, stored in the bridge parachains pallet at the +target chain. If new parachain head appears at the relay chain block `B`, the relay process **waits** until header `B` +or one of its ancestors appears at the target chain. Once it is available, the storage proof of the map entry is +generated and is submitted to the target chain. + +As its on-chain component (which requires bridge GRANDPA pallet to be deployed nearby), the parachains finality relay +requires GRANDPA finality relay to be running in parallel. Without it, the header `B` or any of its children's finality +at source won't be relayed at target, and target chain won't be able to verify generated storage proof. + +More: [Parachains Finality Relay Sequence Diagram](./parachains-finality-relay.html), [code](../relays/parachains/). + +### Messages Relay + +Messages relay is actually two relays that are running in a single process: messages delivery relay and delivery +confirmation relay. Even though they are more complex and have many caveats, the overall algorithm is the same as in +other relays. + +Message delivery relay connects to the source chain and looks at the outbound lane end, waiting until new messages are +queued there. Once they appear at the source block `B`, the relay start waiting for the block `B` or its descendant +appear at the target chain. Then the messages storage proof is generated and submitted to the bridge messages pallet at +the target chain. In addition, the transaction may include the storage proof of the outbound lane state - that proves +that relayer rewards have been paid and this data (map of relay accounts to the delivered messages) may be pruned from +the inbound lane state at the target chain. + +Delivery confirmation relay connects to the target chain and starts watching the inbound lane end. When new messages are +delivered to the target chain, the corresponding _source chain account_ is inserted to the map in the inbound lane data. +Relay detects that, say, at the target chain block `B` and waits until that block or its descendant appears at the +source chain. Once that happens, the relay crafts a storage proof of that data and sends it to the messages pallet, +deployed at the source chain. + +As you can see, the messages relay also requires finality relay to be operating in parallel. Since messages relay +submits transactions to both source and target chains, it requires both _source-to-target_ and _target-to-source_ +finality relays. They can be GRANDPA finality relays or GRANDPA+parachains finality relays, depending on the type of +connected chain. + +More: [Messages Relay Sequence Diagram](./messages-relay.html), [pallet level documentation and +code](../relays/messages/). + +### Complex Relay + +Every relay transaction has its cost. The only transaction, that is "free" to relayer is when the mandatory GRANDPA +header is submitted. The relay that feeds the bridge with every relay chain and/or parachain head it sees, will have to +pay a (quite large) cost. And if no messages are sent through the bridge, that is just waste of money. + +We have a special relay mode, called _complex relay_, where relay mostly sleeps and only submits transactions that are +required for the messages/confirmations delivery. This mode starts two message relays (in both directions). All required +finality relays are also started in a special _on-demand_ mode. In this mode they do not submit any headers without +special request. As always, the only exception is when GRANDPA finality relay sees the mandatory header - it is +submitted without such request. + +The message relays are watching their lanes and when, at some block `B`, they see new messages/confirmations to be +delivered, they are asking on-demand relays to relay this block `B`. On-demand relays does that and then message relay +may perform its job. If on-demand relay is a parachain finality relay, it also runs its own on-demand GRANDPA relay, +which is used to relay required relay chain headers. + +More: [Complex Relay Sequence Diagram](./complex-relay.html), +[code](../relays/bin-substrate/src/cli/relay_headers_and_messages/). diff --git a/cumulus/bridges/docs/messages-relay.html b/bridges/docs/messages-relay.html similarity index 100% rename from cumulus/bridges/docs/messages-relay.html rename to bridges/docs/messages-relay.html diff --git a/cumulus/bridges/docs/parachains-finality-relay.html b/bridges/docs/parachains-finality-relay.html similarity index 100% rename from cumulus/bridges/docs/parachains-finality-relay.html rename to bridges/docs/parachains-finality-relay.html diff --git a/cumulus/bridges/docs/polkadot-kusama-bridge-overview.md b/bridges/docs/polkadot-kusama-bridge-overview.md similarity index 56% rename from cumulus/bridges/docs/polkadot-kusama-bridge-overview.md rename to bridges/docs/polkadot-kusama-bridge-overview.md index b469720f65b2bb5b8b941ec34e89df505d13ac67..08036f0b0722b869786ae3d0abfc6ae7ea7c2c18 100644 --- a/cumulus/bridges/docs/polkadot-kusama-bridge-overview.md +++ b/bridges/docs/polkadot-kusama-bridge-overview.md @@ -1,35 +1,35 @@ # Polkadot <> Kusama Bridge Overview -This document describes how we use all components, described in the [High-Level Bridge Documentation](./high-level-overview.md), -to build the XCM bridge between Kusama and Polkadot. In this case, our components merely work as a XCM transport -(like XCMP/UMP/HRMP), between chains that are not a part of the same consensus system. +This document describes how we use all components, described in the [High-Level Bridge +Documentation](./high-level-overview.md), to build the XCM bridge between Kusama and Polkadot. In this case, our +components merely work as a XCM transport (like XCMP/UMP/HRMP), between chains that are not a part of the same consensus +system. The overall architecture may be seen in [this diagram](./polkadot-kusama-bridge.html). ## Bridge Hubs -All operations at relay chain are expensive. Ideally all non-mandatory transactions must happen on parachains. -That's why we are planning to have two parachains - Polkadot Bridge Hub under Polkadot consensus and Kusama -Bridge Hub under Kusama consensus. +All operations at relay chain are expensive. Ideally all non-mandatory transactions must happen on parachains. That's +why we are planning to have two parachains - Polkadot Bridge Hub under Polkadot consensus and Kusama Bridge Hub under +Kusama consensus. -The Bridge Hub will have all required bridge pallets in its runtime. We hope that later, other teams will be able to -use our bridge hubs too and have their pallets there. +The Bridge Hub will have all required bridge pallets in its runtime. We hope that later, other teams will be able to use +our bridge hubs too and have their pallets there. -The Bridge Hub will use the base token of the ecosystem - KSM at Kusama Bridge Hub and DOT at Polkadot Bridge Hub. -The runtime will have minimal set of non-bridge pallets, so there's not much you can do directly on bridge hubs. +The Bridge Hub will use the base token of the ecosystem - KSM at Kusama Bridge Hub and DOT at Polkadot Bridge Hub. The +runtime will have minimal set of non-bridge pallets, so there's not much you can do directly on bridge hubs. ## Connecting Parachains -You won't be able to directly use bridge hub transactions to send XCM messages over the bridge. Instead, you'll need -to use other parachains transactions, which will use HRMP to deliver messages to the Bridge Hub. The Bridge Hub will -just queue these messages in its outbound lane, which is dedicated to deliver messages between two parachains. +You won't be able to directly use bridge hub transactions to send XCM messages over the bridge. Instead, you'll need to +use other parachains transactions, which will use HRMP to deliver messages to the Bridge Hub. The Bridge Hub will just +queue these messages in its outbound lane, which is dedicated to deliver messages between two parachains. -Our first planned bridge will connect the Polkadot and Kusama Asset Hubs. A bridge between those two -parachains would allow Asset Hub Polkadot accounts to hold wrapped KSM tokens and Asset Hub Kusama -accounts to hold wrapped DOT tokens. +Our first planned bridge will connect the Polkadot and Kusama Asset Hubs. A bridge between those two parachains would +allow Asset Hub Polkadot accounts to hold wrapped KSM tokens and Asset Hub Kusama accounts to hold wrapped DOT tokens. -For that bridge (pair of parachains under different consensus systems) we'll be using the lane 00000000. Later, -when other parachains will join the bridge, they will be using other lanes for their messages. +For that bridge (pair of parachains under different consensus systems) we'll be using the lane 00000000. Later, when +other parachains will join the bridge, they will be using other lanes for their messages. ## Running Relayers @@ -38,9 +38,9 @@ justifications to the bridge hubs at the other side. It'll also relay finalized Hub heads. This will only happen when messages will be queued at hubs. So most of time relayer will be idle. There's no any active relayer sets, or something like that. Anyone may start its own relayer and relay queued messages. -We are not against that and, as always, appreciate any community efforts. Of course, running relayer has the cost. -Apart from paying for the CPU and network, the relayer pays for transactions at both sides of the bridge. We have -a mechanism for rewarding relayers. +We are not against that and, as always, appreciate any community efforts. Of course, running relayer has the cost. Apart +from paying for the CPU and network, the relayer pays for transactions at both sides of the bridge. We have a mechanism +for rewarding relayers. ### Compensating the Cost of Message Delivery Transactions @@ -56,51 +56,49 @@ is the relayer, which is following our rules: - we compensate the cost of message delivery transactions that have actually delivered the messages. So if your transaction has claimed to deliver messages `[42, 43, 44]`, but, because of some reasons, has actually delivered - messages `[42, 43]`, the transaction will be free for relayer. If it has not delivered any messages, then - the relayer pays the full cost of the transaction; + messages `[42, 43]`, the transaction will be free for relayer. If it has not delivered any messages, then the relayer + pays the full cost of the transaction; - we compensate the cost of message delivery and all required finality calls, if they are part of the same [`frame_utility::batch_all`](https://github.com/paritytech/substrate/blob/891d6a5c870ab88521183facafc811a203bb6541/frame/utility/src/lib.rs#L326) - transaction. Of course, the calls inside the batch must be linked - e.g. the submitted parachain head must be used - to prove messages. Relay header must be used to prove parachain head finality. If one of calls fails, or if they - are not linked together, the relayer pays the full transaction cost. + transaction. Of course, the calls inside the batch must be linked - e.g. the submitted parachain head must be used to + prove messages. Relay header must be used to prove parachain head finality. If one of calls fails, or if they are not + linked together, the relayer pays the full transaction cost. Please keep in mind that the fee of "zero-cost" transactions is still withdrawn from the relayer account. But the -compensation is registered in the `pallet_bridge_relayers::RelayerRewards` map at the target bridge hub. The relayer -may later claim all its rewards later, using the `pallet_bridge_relayers::claim_rewards` call. +compensation is registered in the `pallet_bridge_relayers::RelayerRewards` map at the target bridge hub. The relayer may +later claim all its rewards later, using the `pallet_bridge_relayers::claim_rewards` call. *A side note*: why we don't simply set the cost of useful transactions to zero? That's because the bridge has its cost. If we won't take any fees, it would mean that the sender is not obliged to pay for its messages. And Bridge Hub -collators (and, maybe, "treasury") are not receiving any payment for including transactions. More about this later, -in the [Who is Rewarding Relayers](#who-is-rewarding-relayers) section. +collators (and, maybe, "treasury") are not receiving any payment for including transactions. More about this later, in +the [Who is Rewarding Relayers](#who-is-rewarding-relayers) section. ### Message Delivery Confirmation Rewards In addition to the "zero-cost" message delivery transactions, the relayer is also rewarded for: -- delivering every message. The reward is registered during delivery confirmation transaction at the Source Bridge - Hub.; +- delivering every message. The reward is registered during delivery confirmation transaction at the Source Bridge Hub.; -- submitting delivery confirmation transaction. The relayer may submit delivery confirmation that e.g. confirms - delivery of four messages, of which the only one (or zero) messages is actually delivered by this relayer. It - receives some fee for confirming messages, delivered by other relayers. +- submitting delivery confirmation transaction. The relayer may submit delivery confirmation that e.g. confirms delivery + of four messages, of which the only one (or zero) messages is actually delivered by this relayer. It receives some fee + for confirming messages, delivered by other relayers. Both rewards may be claimed using the `pallet_bridge_relayers::claim_rewards` call at the Source Bridge Hub. ### Who is Rewarding Relayers Obviously, there should be someone who is paying relayer rewards. We want bridge transactions to have a cost, so we -can't use fees for rewards. Instead, the parachains using the bridge, use sovereign accounts on both sides -of the bridge to cover relayer rewards. +can't use fees for rewards. Instead, the parachains using the bridge, use sovereign accounts on both sides of the bridge +to cover relayer rewards. -Bridged Parachains will have sovereign accounts at bridge hubs. For example, the Kusama Asset Hub will -have an account at the Polkadot Bridge Hub. The Polkadot Asset Hub will have an account at the Kusama -Bridge Hub. The sovereign accounts are used as a source of funds when the relayer is calling the -`pallet_bridge_relayers::claim_rewards`. +Bridged Parachains will have sovereign accounts at bridge hubs. For example, the Kusama Asset Hub will have an account +at the Polkadot Bridge Hub. The Polkadot Asset Hub will have an account at the Kusama Bridge Hub. The sovereign accounts +are used as a source of funds when the relayer is calling the `pallet_bridge_relayers::claim_rewards`. -Since messages lane is only used by the pair of parachains, there's no collision between different bridges. E.g. -Kusama Asset Hub will only reward relayers that are delivering messages from Kusama Asset Hub. The Kusama Asset Hub sovereign account -is not used to cover rewards of bridging with some other Polkadot Parachain. +Since messages lane is only used by the pair of parachains, there's no collision between different bridges. E.g. Kusama +Asset Hub will only reward relayers that are delivering messages from Kusama Asset Hub. The Kusama Asset Hub sovereign +account is not used to cover rewards of bridging with some other Polkadot Parachain. ### Multiple Relayers and Rewards @@ -108,25 +106,24 @@ Our goal is to incentivize running honest relayers. But we have no relayers sets message delivery transaction, hoping that the cost of this transaction will be compensated. So what if some message is currently queued and two relayers are submitting two identical message delivery transactions at once? Without any special means, the cost of first included transaction will be compensated and the cost of the other one won't. A honest, -but unlucky relayer will lose some money. In addition, we'll waste some portion of block size and weight, which -may be used by other useful transactions. +but unlucky relayer will lose some money. In addition, we'll waste some portion of block size and weight, which may be +used by other useful transactions. -To solve the problem, we have two signed extensions ([generate_bridge_reject_obsolete_headers_and_messages! {}](../bin/runtime-common/src/lib.rs) -and [RefundRelayerForMessagesFromParachain](../bin/runtime-common/src/refund_relayer_extension.rs)), that are -preventing bridge transactions with obsolete data from including into the block. We are rejecting following -transactions: +To solve the problem, we have two signed extensions ([generate_bridge_reject_obsolete_headers_and_messages! +{}](../bin/runtime-common/src/lib.rs) and +[RefundRelayerForMessagesFromParachain](../bin/runtime-common/src/refund_relayer_extension.rs)), that are preventing +bridge transactions with obsolete data from including into the block. We are rejecting following transactions: - transactions, that are submitting the GRANDPA justification for the best finalized header, or one of its ancestors; - transactions, that are submitting the proof of the current best parachain head, or one of its ancestors; -- transactions, that are delivering already delivered messages. If at least one of messages is not yet delivered, - the transaction is not rejected; +- transactions, that are delivering already delivered messages. If at least one of messages is not yet delivered, the + transaction is not rejected; -- transactions, that are confirming delivery of already confirmed messages. If at least one of confirmations is new, - the transaction is not rejected; +- transactions, that are confirming delivery of already confirmed messages. If at least one of confirmations is new, the + transaction is not rejected; - [`frame_utility::batch_all`](https://github.com/paritytech/substrate/blob/891d6a5c870ab88521183facafc811a203bb6541/frame/utility/src/lib.rs#L326) - transactions, that have both finality and message delivery calls. All restrictions from the - [Compensating the Cost of Message Delivery Transactions](#compensating-the-cost-of-message-delivery-transactions) - are applied. + transactions, that have both finality and message delivery calls. All restrictions from the [Compensating the Cost of + Message Delivery Transactions](#compensating-the-cost-of-message-delivery-transactions) are applied. diff --git a/cumulus/bridges/docs/polkadot-kusama-bridge.html b/bridges/docs/polkadot-kusama-bridge.html similarity index 100% rename from cumulus/bridges/docs/polkadot-kusama-bridge.html rename to bridges/docs/polkadot-kusama-bridge.html diff --git a/cumulus/bridges/modules/grandpa/Cargo.toml b/bridges/modules/grandpa/Cargo.toml similarity index 65% rename from cumulus/bridges/modules/grandpa/Cargo.toml rename to bridges/modules/grandpa/Cargo.toml index 87eda6b29a703599dcc8afd6a2b88191e458a3c9..05d6a8e5c26b0d82f5dc80ce5be21d9a53b46d33 100644 --- a/cumulus/bridges/modules/grandpa/Cargo.toml +++ b/bridges/modules/grandpa/Cargo.toml @@ -20,20 +20,20 @@ bp-header-chain = { path = "../../primitives/header-chain", default-features = f # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } # Optional Benchmarking Dependencies bp-test-utils = { path = "../../primitives/test-utils", default-features = false, optional = true } -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } [dev-dependencies] -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/grandpa/README.md b/bridges/modules/grandpa/README.md similarity index 100% rename from cumulus/bridges/modules/grandpa/README.md rename to bridges/modules/grandpa/README.md diff --git a/cumulus/bridges/modules/grandpa/src/benchmarking.rs b/bridges/modules/grandpa/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/benchmarking.rs rename to bridges/modules/grandpa/src/benchmarking.rs diff --git a/cumulus/bridges/modules/grandpa/src/call_ext.rs b/bridges/modules/grandpa/src/call_ext.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/call_ext.rs rename to bridges/modules/grandpa/src/call_ext.rs diff --git a/cumulus/bridges/modules/grandpa/src/lib.rs b/bridges/modules/grandpa/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/lib.rs rename to bridges/modules/grandpa/src/lib.rs diff --git a/cumulus/bridges/modules/grandpa/src/mock.rs b/bridges/modules/grandpa/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/mock.rs rename to bridges/modules/grandpa/src/mock.rs diff --git a/cumulus/bridges/modules/grandpa/src/storage_types.rs b/bridges/modules/grandpa/src/storage_types.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/storage_types.rs rename to bridges/modules/grandpa/src/storage_types.rs diff --git a/cumulus/bridges/modules/grandpa/src/weights.rs b/bridges/modules/grandpa/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/weights.rs rename to bridges/modules/grandpa/src/weights.rs diff --git a/cumulus/bridges/modules/messages/Cargo.toml b/bridges/modules/messages/Cargo.toml similarity index 67% rename from cumulus/bridges/modules/messages/Cargo.toml rename to bridges/modules/messages/Cargo.toml index 5eecdb147fada8100866280af3c81b7ede77605f..7b7ea06198102ece6d93313413451a7c0294ef8b 100644 --- a/cumulus/bridges/modules/messages/Cargo.toml +++ b/bridges/modules/messages/Cargo.toml @@ -19,17 +19,17 @@ bp-runtime = { path = "../../primitives/runtime", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-test-utils = { path = "../../primitives/test-utils" } -pallet-balances = { path = "../../../../substrate/frame/balances" } -sp-io = { path = "../../../../substrate/primitives/io" } +pallet-balances = { path = "../../../substrate/frame/balances" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/bridges/modules/messages/README.md b/bridges/modules/messages/README.md new file mode 100644 index 0000000000000000000000000000000000000000..457d5f5facfa70fdb11d05c5d544e75eb44f975f --- /dev/null +++ b/bridges/modules/messages/README.md @@ -0,0 +1,215 @@ +# Bridge Messages Pallet + +The messages pallet is used to deliver messages from source chain to target chain. Message is (almost) opaque to the +module and the final goal is to hand message to the message dispatch mechanism. + +## Contents + +- [Overview](#overview) +- [Message Workflow](#message-workflow) +- [Integrating Message Lane Module into Runtime](#integrating-messages-module-into-runtime) +- [Non-Essential Functionality](#non-essential-functionality) +- [Weights of Module Extrinsics](#weights-of-module-extrinsics) + +## Overview + +Message lane is an unidirectional channel, where messages are sent from source chain to the target chain. At the same +time, a single instance of messages module supports both outbound lanes and inbound lanes. So the chain where the module +is deployed (this chain), may act as a source chain for outbound messages (heading to a bridged chain) and as a target +chain for inbound messages (coming from a bridged chain). + +Messages module supports multiple message lanes. Every message lane is identified with a 4-byte identifier. Messages +sent through the lane are assigned unique (for this lane) increasing integer value that is known as nonce ("number that +can only be used once"). Messages that are sent over the same lane are guaranteed to be delivered to the target chain in +the same order they're sent from the source chain. In other words, message with nonce `N` will be delivered right before +delivering a message with nonce `N+1`. + +Single message lane may be seen as a transport channel for single application (onchain, offchain or mixed). At the same +time the module itself never dictates any lane or message rules. In the end, it is the runtime developer who defines +what message lane and message mean for this runtime. + +In our [Kusama<>Polkadot bridge](../../docs/polkadot-kusama-bridge-overview.md) we are using lane as a channel of +communication between two parachains of different relay chains. For example, lane `[0, 0, 0, 0]` is used for Polkadot <> +Kusama Asset Hub communications. Other lanes may be used to bridge other parachains. + +## Message Workflow + +The pallet is not intended to be used by end users and provides no public calls to send the message. Instead, it +provides runtime-internal method that allows other pallets (or other runtime code) to queue outbound messages. + +The message "appears" when some runtime code calls the `send_message()` method of the pallet. The submitter specifies +the lane that they're willing to use and the message itself. If some fee must be paid for sending the message, it must +be paid outside of the pallet. If a message passes all checks (that include, for example, message size check, disabled +lane check, ...), the nonce is assigned and the message is stored in the module storage. The message is in an +"undelivered" state now. + +We assume that there are external, offchain actors, called relayers, that are submitting module related transactions to +both target and source chains. The pallet itself has no assumptions about relayers incentivization scheme, but it has +some callbacks for paying rewards. See [Integrating Messages Module into +runtime](#Integrating-Messages-Module-into-runtime) for details. + +Eventually, some relayer would notice this message in the "undelivered" state and it would decide to deliver this +message. Relayer then crafts `receive_messages_proof()` transaction (aka delivery transaction) for the messages module +instance, deployed at the target chain. Relayer provides its account id at the source chain, the proof of message (or +several messages), the number of messages in the transaction and their cumulative dispatch weight. Once a transaction is +mined, the message is considered "delivered". + +Once a message is delivered, the relayer may want to confirm delivery back to the source chain. There are two reasons +why it would want to do that. The first is that we intentionally limit number of "delivered", but not yet "confirmed" +messages at inbound lanes (see [What about other Constants in the Messages Module Configuration +Trait](#What-about-other-Constants-in-the-Messages-Module-Configuration-Trait) for explanation). So at some point, the +target chain may stop accepting new messages until relayers confirm some of these. The second is that if the relayer +wants to be rewarded for delivery, it must prove the fact that it has actually delivered the message. And this proof may +only be generated after the delivery transaction is mined. So relayer crafts the `receive_messages_delivery_proof()` +transaction (aka confirmation transaction) for the messages module instance, deployed at the source chain. Once this +transaction is mined, the message is considered "confirmed". + +The "confirmed" state is the final state of the message. But there's one last thing related to the message - the fact +that it is now "confirmed" and reward has been paid to the relayer (or at least callback for this has been called), must +be confirmed to the target chain. Otherwise, we may reach the limit of "unconfirmed" messages at the target chain and it +will stop accepting new messages. So relayer sometimes includes a nonce of the latest "confirmed" message in the next +`receive_messages_proof()` transaction, proving that some messages have been confirmed. + +## Integrating Messages Module into Runtime + +As it has been said above, the messages module supports both outbound and inbound message lanes. So if we will integrate +a module in some runtime, it may act as the source chain runtime for outbound messages and as the target chain runtime +for inbound messages. In this section, we'll sometimes refer to the chain we're currently integrating with, as "this +chain" and the other chain as "bridged chain". + +Messages module doesn't simply accept transactions that are claiming that the bridged chain has some updated data for +us. Instead of this, the module assumes that the bridged chain is able to prove that updated data in some way. The proof +is abstracted from the module and may be of any kind. In our Substrate-to-Substrate bridge we're using runtime storage +proofs. Other bridges may use transaction proofs, Substrate header digests or anything else that may be proved. + +**IMPORTANT NOTE**: everything below in this chapter describes details of the messages module configuration. But if +you're interested in well-probed and relatively easy integration of two Substrate-based chains, you may want to look at +the [bridge-runtime-common](../../bin/runtime-common/) crate. This crate is providing a lot of helpers for integration, +which may be directly used from within your runtime. Then if you'll decide to change something in this scheme, get back +here for detailed information. + +### General Information + +The messages module supports instances. Every module instance is supposed to bridge this chain and some bridged chain. +To bridge with another chain, using another instance is suggested (this isn't forced anywhere in the code, though). Keep +in mind, that the pallet may be used to build virtual channels between multiple chains, as we do in our [Polkadot <> +Kusama bridge](../../docs/polkadot-kusama-bridge-overview.md). There, the pallet actually bridges only two parachains - +Kusama Bridge Hub and Polkadot Bridge Hub. However, other Kusama and Polkadot parachains are able to send (XCM) messages +to their Bridge Hubs. The messages will be delivered to the other side of the bridge and routed to the proper +destination parachain within the bridged chain consensus. + +Message submitters may track message progress by inspecting module events. When Message is accepted, the +`MessageAccepted` event is emitted. The event contains both message lane identifier and nonce that has been assigned to +the message. When a message is delivered to the target chain, the `MessagesDelivered` event is emitted from the +`receive_messages_delivery_proof()` transaction. The `MessagesDelivered` contains the message lane identifier and +inclusive range of delivered message nonces. + +The pallet provides no means to get the result of message dispatch at the target chain. If that is required, it must be +done outside of the pallet. For example, XCM messages, when dispatched, have special instructions to send some data back +to the sender. Other dispatchers may use similar mechanism for that. +### How to plug-in Messages Module to Send Messages to the Bridged Chain? + +The `pallet_bridge_messages::Config` trait has 3 main associated types that are used to work with outbound messages. The +`pallet_bridge_messages::Config::TargetHeaderChain` defines how we see the bridged chain as the target for our outbound +messages. It must be able to check that the bridged chain may accept our message - like that the message has size below +maximal possible transaction size of the chain and so on. And when the relayer sends us a confirmation transaction, this +implementation must be able to parse and verify the proof of messages delivery. Normally, you would reuse the same +(configurable) type on all chains that are sending messages to the same bridged chain. + +The `pallet_bridge_messages::Config::LaneMessageVerifier` defines a single callback to verify outbound messages. The +simplest callback may just accept all messages. But in this case you'll need to answer many questions first. Who will +pay for the delivery and confirmation transaction? Are we sure that someone will ever deliver this message to the +bridged chain? Are we sure that we don't bloat our runtime storage by accepting this message? What if the message is +improperly encoded or has some fields set to invalid values? Answering all those (and similar) questions would lead to +correct implementation. + +There's another thing to consider when implementing type for use in +`pallet_bridge_messages::Config::LaneMessageVerifier`. It is whether we treat all message lanes identically, or they'll +have different sets of verification rules? For example, you may reserve lane#1 for messages coming from some +'wrapped-token' pallet - then you may verify in your implementation that the origin is associated with this pallet. +Lane#2 may be reserved for 'system' messages and you may charge zero fee for such messages. You may have some rate +limiting for messages sent over the lane#3. Or you may just verify the same rules set for all outbound messages - it is +all up to the `pallet_bridge_messages::Config::LaneMessageVerifier` implementation. + +The last type is the `pallet_bridge_messages::Config::DeliveryConfirmationPayments`. When confirmation transaction is +received, we call the `pay_reward()` method, passing the range of delivered messages. You may use the +[`pallet-bridge-relayers`](../relayers/) pallet and its +[`DeliveryConfirmationPaymentsAdapter`](../relayers/src/payment_adapter.rs) adapter as a possible implementation. It +allows you to pay fixed reward for relaying the message and some of its portion for confirming delivery. + +### I have a Messages Module in my Runtime, but I Want to Reject all Outbound Messages. What shall I do? + +You should be looking at the `bp_messages::source_chain::ForbidOutboundMessages` structure +[`bp_messages::source_chain`](../../primitives/messages/src/source_chain.rs). It implements all required traits and will +simply reject all transactions, related to outbound messages. + +### How to plug-in Messages Module to Receive Messages from the Bridged Chain? + +The `pallet_bridge_messages::Config` trait has 2 main associated types that are used to work with inbound messages. The +`pallet_bridge_messages::Config::SourceHeaderChain` defines how we see the bridged chain as the source of our inbound +messages. When relayer sends us a delivery transaction, this implementation must be able to parse and verify the proof +of messages wrapped in this transaction. Normally, you would reuse the same (configurable) type on all chains that are +sending messages to the same bridged chain. + +The `pallet_bridge_messages::Config::MessageDispatch` defines a way on how to dispatch delivered messages. Apart from +actually dispatching the message, the implementation must return the correct dispatch weight of the message before +dispatch is called. + +### I have a Messages Module in my Runtime, but I Want to Reject all Inbound Messages. What shall I do? + +You should be looking at the `bp_messages::target_chain::ForbidInboundMessages` structure from the +[`bp_messages::target_chain`](../../primitives/messages/src/target_chain.rs) module. It implements all required traits +and will simply reject all transactions, related to inbound messages. + +### What about other Constants in the Messages Module Configuration Trait? + +Two settings that are used to check messages in the `send_message()` function. The +`pallet_bridge_messages::Config::ActiveOutboundLanes` is an array of all message lanes, that may be used to send +messages. All messages sent using other lanes are rejected. All messages that have size above +`pallet_bridge_messages::Config::MaximalOutboundPayloadSize` will also be rejected. + +To be able to reward the relayer for delivering messages, we store a map of message nonces range => identifier of the +relayer that has delivered this range at the target chain runtime storage. If a relayer delivers multiple consequent +ranges, they're merged into single entry. So there may be more than one entry for the same relayer. Eventually, this +whole map must be delivered back to the source chain to confirm delivery and pay rewards. So to make sure we are able to +craft this confirmation transaction, we need to: (1) keep the size of this map below a certain limit and (2) make sure +that the weight of processing this map is below a certain limit. Both size and processing weight mostly depend on the +number of entries. The number of entries is limited with the +`pallet_bridge_messages::ConfigMaxUnrewardedRelayerEntriesAtInboundLane` parameter. Processing weight also depends on +the total number of messages that are being confirmed, because every confirmed message needs to be read. So there's +another `pallet_bridge_messages::Config::MaxUnconfirmedMessagesAtInboundLane` parameter for that. + +When choosing values for these parameters, you must also keep in mind that if proof in your scheme is based on finality +of headers (and it is the most obvious option for Substrate-based chains with finality notion), then choosing too small +values for these parameters may cause significant delays in message delivery. That's because there are too many actors +involved in this scheme: 1) authorities that are finalizing headers of the target chain need to finalize header with +non-empty map; 2) the headers relayer then needs to submit this header and its finality proof to the source chain; 3) +the messages relayer must then send confirmation transaction (storage proof of this map) to the source chain; 4) when +the confirmation transaction will be mined at some header, source chain authorities must finalize this header; 5) the +headers relay then needs to submit this header and its finality proof to the target chain; 6) only now the messages +relayer may submit new messages from the source to target chain and prune the entry from the map. + +Delivery transaction requires the relayer to provide both number of entries and total number of messages in the map. +This means that the module never charges an extra cost for delivering a map - the relayer would need to pay exactly for +the number of entries+messages it has delivered. So the best guess for values of these parameters would be the pair that +would occupy `N` percent of the maximal transaction size and weight of the source chain. The `N` should be large enough +to process large maps, at the same time keeping reserve for future source chain upgrades. + +## Non-Essential Functionality + +There may be a special account in every runtime where the messages module is deployed. This account, named 'module +owner', is like a module-level sudo account - he's able to halt and resume all module operations without requiring +runtime upgrade. Calls that are related to this account are: +- `fn set_owner()`: current module owner may call it to transfer "ownership" to another account; +- `fn halt_operations()`: the module owner (or sudo account) may call this function to stop all module operations. After + this call, all message-related transactions will be rejected until further `resume_operations` call'. This call may be + used when something extraordinary happens with the bridge; +- `fn resume_operations()`: module owner may call this function to resume bridge operations. The module will resume its + regular operations after this call. + +If pallet owner is not defined, the governance may be used to make those calls. + +## Messages Relay + +We have an offchain actor, who is watching for new messages and submits them to the bridged chain. It is the messages +relay - you may look at the [crate level documentation and the code](../../relays/messages/). diff --git a/cumulus/bridges/modules/messages/src/benchmarking.rs b/bridges/modules/messages/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/benchmarking.rs rename to bridges/modules/messages/src/benchmarking.rs diff --git a/cumulus/bridges/modules/messages/src/inbound_lane.rs b/bridges/modules/messages/src/inbound_lane.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/inbound_lane.rs rename to bridges/modules/messages/src/inbound_lane.rs diff --git a/cumulus/bridges/modules/messages/src/lib.rs b/bridges/modules/messages/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/lib.rs rename to bridges/modules/messages/src/lib.rs diff --git a/cumulus/bridges/modules/messages/src/mock.rs b/bridges/modules/messages/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/mock.rs rename to bridges/modules/messages/src/mock.rs diff --git a/cumulus/bridges/modules/messages/src/outbound_lane.rs b/bridges/modules/messages/src/outbound_lane.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/outbound_lane.rs rename to bridges/modules/messages/src/outbound_lane.rs diff --git a/cumulus/bridges/modules/messages/src/weights.rs b/bridges/modules/messages/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/weights.rs rename to bridges/modules/messages/src/weights.rs diff --git a/cumulus/bridges/modules/messages/src/weights_ext.rs b/bridges/modules/messages/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/weights_ext.rs rename to bridges/modules/messages/src/weights_ext.rs diff --git a/cumulus/bridges/modules/parachains/Cargo.toml b/bridges/modules/parachains/Cargo.toml similarity index 71% rename from cumulus/bridges/modules/parachains/Cargo.toml rename to bridges/modules/parachains/Cargo.toml index 50a838edf56db3a7b94014b692eb8575b4f9e499..4eb09ed78d17f3b4da620ac6b28645bdaab88c12 100644 --- a/cumulus/bridges/modules/parachains/Cargo.toml +++ b/bridges/modules/parachains/Cargo.toml @@ -20,18 +20,18 @@ pallet-bridge-grandpa = { path = "../grandpa", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } [dev-dependencies] bp-header-chain = { path = "../../primitives/header-chain" } bp-test-utils = { path = "../../primitives/test-utils" } -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/parachains/README.md b/bridges/modules/parachains/README.md similarity index 99% rename from cumulus/bridges/modules/parachains/README.md rename to bridges/modules/parachains/README.md index 5982c65ad3166107128b9256f89f337f8de7c69e..d3f52c791ab5899c438bf902acd85d3e8a96153a 100644 --- a/cumulus/bridges/modules/parachains/README.md +++ b/bridges/modules/parachains/README.md @@ -19,7 +19,7 @@ validators. Validators validate the block and register the new parachain head in [`Heads` map](https://github.com/paritytech/polkadot/blob/88013730166ba90745ae7c9eb3e0c1be1513c7cc/runtime/parachains/src/paras/mod.rs#L645) of the [`paras`](https://github.com/paritytech/polkadot/tree/master/runtime/parachains/src/paras) pallet, deployed at the relay chain. Keep in mind that this pallet, deployed at a relay chain, is **NOT** a bridge pallet, -even though the names are similar. +even though the names are similar. And what the bridge parachains pallet does, is simply verifying storage proofs of parachain heads within that `Heads` map. It does that using relay chain header, that has been previously imported by the diff --git a/cumulus/bridges/modules/parachains/src/benchmarking.rs b/bridges/modules/parachains/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/benchmarking.rs rename to bridges/modules/parachains/src/benchmarking.rs diff --git a/cumulus/bridges/modules/parachains/src/call_ext.rs b/bridges/modules/parachains/src/call_ext.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/call_ext.rs rename to bridges/modules/parachains/src/call_ext.rs diff --git a/cumulus/bridges/modules/parachains/src/lib.rs b/bridges/modules/parachains/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/lib.rs rename to bridges/modules/parachains/src/lib.rs diff --git a/cumulus/bridges/modules/parachains/src/mock.rs b/bridges/modules/parachains/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/mock.rs rename to bridges/modules/parachains/src/mock.rs diff --git a/cumulus/bridges/modules/parachains/src/weights.rs b/bridges/modules/parachains/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/weights.rs rename to bridges/modules/parachains/src/weights.rs diff --git a/cumulus/bridges/modules/parachains/src/weights_ext.rs b/bridges/modules/parachains/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/weights_ext.rs rename to bridges/modules/parachains/src/weights_ext.rs diff --git a/cumulus/bridges/modules/relayers/Cargo.toml b/bridges/modules/relayers/Cargo.toml similarity index 66% rename from cumulus/bridges/modules/relayers/Cargo.toml rename to bridges/modules/relayers/Cargo.toml index 3a7a57e18018967c35c2d3b72d317787333cedad..46fc3bb43b1de23b5784179ccd2dbc31ae6c7230 100644 --- a/cumulus/bridges/modules/relayers/Cargo.toml +++ b/bridges/modules/relayers/Cargo.toml @@ -20,19 +20,19 @@ pallet-bridge-messages = { path = "../messages", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-arithmetic = { path = "../../../../substrate/primitives/arithmetic", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-arithmetic = { path = "../../../substrate/primitives/arithmetic", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-runtime = { path = "../../primitives/runtime" } -pallet-balances = { path = "../../../../substrate/frame/balances" } -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } -sp-runtime = { path = "../../../../substrate/primitives/runtime" } +pallet-balances = { path = "../../../substrate/frame/balances" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } +sp-runtime = { path = "../../../substrate/primitives/runtime" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/relayers/README.md b/bridges/modules/relayers/README.md similarity index 100% rename from cumulus/bridges/modules/relayers/README.md rename to bridges/modules/relayers/README.md diff --git a/cumulus/bridges/modules/relayers/src/benchmarking.rs b/bridges/modules/relayers/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/benchmarking.rs rename to bridges/modules/relayers/src/benchmarking.rs diff --git a/cumulus/bridges/modules/relayers/src/lib.rs b/bridges/modules/relayers/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/lib.rs rename to bridges/modules/relayers/src/lib.rs diff --git a/cumulus/bridges/modules/relayers/src/mock.rs b/bridges/modules/relayers/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/mock.rs rename to bridges/modules/relayers/src/mock.rs diff --git a/cumulus/bridges/modules/relayers/src/payment_adapter.rs b/bridges/modules/relayers/src/payment_adapter.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/payment_adapter.rs rename to bridges/modules/relayers/src/payment_adapter.rs diff --git a/cumulus/bridges/modules/relayers/src/stake_adapter.rs b/bridges/modules/relayers/src/stake_adapter.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/stake_adapter.rs rename to bridges/modules/relayers/src/stake_adapter.rs diff --git a/cumulus/bridges/modules/relayers/src/weights.rs b/bridges/modules/relayers/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/weights.rs rename to bridges/modules/relayers/src/weights.rs diff --git a/cumulus/bridges/modules/relayers/src/weights_ext.rs b/bridges/modules/relayers/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/weights_ext.rs rename to bridges/modules/relayers/src/weights_ext.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml b/bridges/modules/xcm-bridge-hub-router/Cargo.toml similarity index 59% rename from cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml rename to bridges/modules/xcm-bridge-hub-router/Cargo.toml index 6ad3c57ca73a788cc09d8fb9e9b1d3510027e4c2..c61cab291e142e5e1aa4680b162d288ee62284a5 100644 --- a/cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml +++ b/bridges/modules/xcm-bridge-hub-router/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false } -log = { version = "0.4.19", default-features = false } +log = { version = "0.4.20", default-features = false } scale-info = { version = "2.8.0", default-features = false, features = ["bit-vec", "derive", "serde"] } # Bridge dependencies @@ -17,21 +17,21 @@ bp-xcm-bridge-hub-router = { path = "../../primitives/xcm-bridge-hub-router", de # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } # Polkadot Dependencies -xcm = { package = "staging-xcm", path = "../../../../polkadot/xcm", default-features = false } -xcm-builder = { package = "staging-xcm-builder", path = "../../../../polkadot/xcm/xcm-builder", default-features = false } +xcm = { package = "staging-xcm", path = "../../../polkadot/xcm", default-features = false } +xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder", default-features = false } [dev-dependencies] -sp-io = { path = "../../../../substrate/primitives/io" } -sp-std = { path = "../../../../substrate/primitives/std" } +sp-io = { path = "../../../substrate/primitives/io" } +sp-std = { path = "../../../substrate/primitives/std" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs b/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs rename to bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/lib.rs b/bridges/modules/xcm-bridge-hub-router/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/lib.rs rename to bridges/modules/xcm-bridge-hub-router/src/lib.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs b/bridges/modules/xcm-bridge-hub-router/src/mock.rs similarity index 99% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs rename to bridges/modules/xcm-bridge-hub-router/src/mock.rs index cd50b98a168853eec6a37c9a4806271574454cca..58df21a6d9016a70a8414b884ccedf33d7ae1524 100644 --- a/cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs +++ b/bridges/modules/xcm-bridge-hub-router/src/mock.rs @@ -144,5 +144,5 @@ pub fn new_test_ext() -> sp_io::TestExternalities { /// Run pallet test. pub fn run_test(test: impl FnOnce() -> T) -> T { - new_test_ext().execute_with(|| test()) + new_test_ext().execute_with(test) } diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/weights.rs b/bridges/modules/xcm-bridge-hub-router/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/weights.rs rename to bridges/modules/xcm-bridge-hub-router/src/weights.rs diff --git a/cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml b/bridges/primitives/chain-asset-hub-kusama/Cargo.toml similarity index 88% rename from cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml rename to bridges/primitives/chain-asset-hub-kusama/Cargo.toml index 557f56bfb62e2b26c80a48e99f8a4c7e0898d76f..adb9a57bc134729ab753e3006093207e0de32e61 100644 --- a/cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml +++ b/bridges/primitives/chain-asset-hub-kusama/Cargo.toml @@ -11,7 +11,7 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } # Bridge Dependencies bp-xcm-bridge-hub-router = { path = "../xcm-bridge-hub-router", default-features = false } diff --git a/cumulus/bridges/primitives/chain-asset-hub-kusama/src/lib.rs b/bridges/primitives/chain-asset-hub-kusama/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-asset-hub-kusama/src/lib.rs rename to bridges/primitives/chain-asset-hub-kusama/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml b/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml similarity index 79% rename from cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml rename to bridges/primitives/chain-asset-hub-polkadot/Cargo.toml index 6fe9ffab44b800bd2308635a6ef91572fd2ded3d..857ead15b0ddf79b82fd7b8c01da767419e22e2b 100644 --- a/cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml +++ b/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml @@ -11,8 +11,8 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } # Bridge Dependencies bp-xcm-bridge-hub-router = { path = "../xcm-bridge-hub-router", default-features = false } diff --git a/cumulus/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs b/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs rename to bridges/primitives/chain-asset-hub-polkadot/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml b/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml similarity index 62% rename from cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml rename to bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml index 865cead3c878023cce8fefe3b4c64ca0c4b4c13c..24cf7236d45333254dd62073dfebcd85ec526362 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml @@ -15,13 +15,13 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } # Polkadot Dependencies -polkadot-primitives = { path = "../../../../polkadot/primitives", default-features = false } +polkadot-primitives = { path = "../../../polkadot/primitives", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs b/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs similarity index 72% rename from cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs rename to bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs index feef61c7f20a1655f86ffbcc52809bfd9fef41e5..c1dbc6db36f644c3d123a8c3c5930fce31973146 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs +++ b/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs @@ -23,10 +23,9 @@ pub use bp_polkadot_core::{ }; use bp_messages::*; +use bp_polkadot_core::SuffixedCommonSignedExtension; use bp_runtime::extensions::{ - BridgeRejectObsoleteHeadersAndMessages, ChargeTransactionPayment, CheckEra, CheckGenesis, - CheckNonZeroSender, CheckNonce, CheckSpecVersion, CheckTxVersion, CheckWeight, - GenericSignedExtension, RefundBridgedParachainMessagesSchema, + BridgeRejectObsoleteHeadersAndMessages, RefundBridgedParachainMessagesSchema, }; use frame_support::{ dispatch::DispatchClass, @@ -133,88 +132,8 @@ pub const MAX_UNREWARDED_RELAYERS_IN_CONFIRMATION_TX: MessageNonce = 1024; /// analysis (like the one above). pub const MAX_UNCONFIRMED_MESSAGES_IN_CONFIRMATION_TX: MessageNonce = 4096; -/// Extra signed extension data that is used by all bridge hubs. -pub type SignedExtra = ( - CheckNonZeroSender, - CheckSpecVersion, - CheckTxVersion, - CheckGenesis, - CheckEra, - CheckNonce, - CheckWeight, - ChargeTransactionPayment, +/// Signed extension that is used by all bridge hubs. +pub type SignedExtension = SuffixedCommonSignedExtension<( BridgeRejectObsoleteHeadersAndMessages, RefundBridgedParachainMessagesSchema, -); - -/// Signed extension that is used by all bridge hubs. -pub type SignedExtension = GenericSignedExtension; - -/// Helper trait to define some extra methods on bridge hubs signed extension (and -/// overcome Rust limitations). -pub trait BridgeHubSignedExtension { - /// Create signed extension from its components. - fn from_params( - spec_version: u32, - transaction_version: u32, - era: bp_runtime::TransactionEra, - genesis_hash: Hash, - nonce: Nonce, - tip: Balance, - ) -> Self; - - /// Return transaction nonce. - fn nonce(&self) -> Nonce; - - /// Return transaction tip. - fn tip(&self) -> Balance; -} - -impl BridgeHubSignedExtension for SignedExtension { - /// Create signed extension from its components. - fn from_params( - spec_version: u32, - transaction_version: u32, - era: bp_runtime::TransactionEra, - genesis_hash: Hash, - nonce: Nonce, - tip: Balance, - ) -> Self { - GenericSignedExtension::new( - ( - (), // non-zero sender - (), // spec version - (), // tx version - (), // genesis - era.frame_era(), // era - nonce.into(), // nonce (compact encoding) - (), // Check weight - tip.into(), // transaction payment / tip (compact encoding) - (), // bridge reject obsolete headers and msgs - (), // bridge reward to relayer for message passing - ), - Some(( - (), - spec_version, - transaction_version, - genesis_hash, - era.signed_payload(genesis_hash), - (), - (), - (), - (), - (), - )), - ) - } - - /// Return transaction nonce. - fn nonce(&self) -> Nonce { - self.payload.5 .0 - } - - /// Return transaction tip. - fn tip(&self) -> Balance { - self.payload.7 .0 - } -} +)>; diff --git a/cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml b/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml rename to bridges/primitives/chain-bridge-hub-kusama/Cargo.toml index 92b0c9bf447562b4523077a996c6ea702ccf5af7..387f5e8ade6e7dc501159266a94168dd7055a8c1 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml @@ -15,10 +15,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs b/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs rename to bridges/primitives/chain-bridge-hub-kusama/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml b/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml rename to bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml index 7468d5be60d3f53e80ee8c3b36bddaeea235a7d4..40b386e22d224da198043f44210a8c1de65af3ac 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml @@ -16,10 +16,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs b/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs rename to bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml b/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml rename to bridges/primitives/chain-bridge-hub-rococo/Cargo.toml index 8dde903701c7c6dd900d9b6a93038b1262bfd1de..05b8163e9fcaacc8f32c0b45bfafffc882e0f1be 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml @@ -15,10 +15,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs b/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs rename to bridges/primitives/chain-bridge-hub-rococo/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml b/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml rename to bridges/primitives/chain-bridge-hub-wococo/Cargo.toml index c93cdad5110b484488c0d661c894e612557b559b..17c134f4412f7d74a45e089c920bd260bcd95008 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml @@ -16,10 +16,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs b/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs rename to bridges/primitives/chain-bridge-hub-wococo/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-kusama/Cargo.toml b/bridges/primitives/chain-kusama/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-kusama/Cargo.toml rename to bridges/primitives/chain-kusama/Cargo.toml index e404bdb3d94f17bfff662e7c33c43dc9d71c44f3..2d63c3f374fb50c94e2572c09511bf20d63b61e4 100644 --- a/cumulus/bridges/primitives/chain-kusama/Cargo.toml +++ b/bridges/primitives/chain-kusama/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-kusama/src/lib.rs b/bridges/primitives/chain-kusama/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-kusama/src/lib.rs rename to bridges/primitives/chain-kusama/src/lib.rs index e234a87b6cf680e1b0b2aba39d04704b85b8c896..8c3fbd9c203e5e81ebfc248259cd823255d826f2 100644 --- a/cumulus/bridges/primitives/chain-kusama/src/lib.rs +++ b/bridges/primitives/chain-kusama/src/lib.rs @@ -57,6 +57,9 @@ impl ChainWithGrandpa for Kusama { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +// The SignedExtension used by Kusama. +pub use bp_polkadot_core::CommonSignedExtension as SignedExtension; + /// Name of the parachains pallet in the Kusama runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-polkadot/Cargo.toml b/bridges/primitives/chain-polkadot/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-polkadot/Cargo.toml rename to bridges/primitives/chain-polkadot/Cargo.toml index 4d9bf8c182f9e38db8868126f76531f12146e3f9..539b10ef9c68f490756d4f7e0453b25c9b7fc5c7 100644 --- a/cumulus/bridges/primitives/chain-polkadot/Cargo.toml +++ b/bridges/primitives/chain-polkadot/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-polkadot/src/lib.rs b/bridges/primitives/chain-polkadot/src/lib.rs similarity index 92% rename from cumulus/bridges/primitives/chain-polkadot/src/lib.rs rename to bridges/primitives/chain-polkadot/src/lib.rs index 9585fd4d71634ea9af59d6a64206e7dcfc3c80ca..d1d6f74543121afb47c5fdc423fd29aae5e2615e 100644 --- a/cumulus/bridges/primitives/chain-polkadot/src/lib.rs +++ b/bridges/primitives/chain-polkadot/src/lib.rs @@ -21,7 +21,7 @@ pub use bp_polkadot_core::*; use bp_header_chain::ChainWithGrandpa; -use bp_runtime::{decl_bridge_finality_runtime_apis, Chain}; +use bp_runtime::{decl_bridge_finality_runtime_apis, extensions::PrevalidateAttests, Chain}; use frame_support::weights::Weight; use sp_std::prelude::Vec; @@ -57,6 +57,9 @@ impl ChainWithGrandpa for Polkadot { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +/// The SignedExtension used by Polkadot. +pub type SignedExtension = SuffixedCommonSignedExtension; + /// Name of the parachains pallet in the Polkadot runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-rococo/Cargo.toml b/bridges/primitives/chain-rococo/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-rococo/Cargo.toml rename to bridges/primitives/chain-rococo/Cargo.toml index 72c6d4c03b66641b97244a0d9ff67ee666d44348..3c4d3917bc219bfd7dd65ec9e8a0c283fe834e49 100644 --- a/cumulus/bridges/primitives/chain-rococo/Cargo.toml +++ b/bridges/primitives/chain-rococo/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-rococo/src/lib.rs b/bridges/primitives/chain-rococo/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-rococo/src/lib.rs rename to bridges/primitives/chain-rococo/src/lib.rs index cf7cd16990f8642fbe039310d54de36b5859330e..1589d14ea5143da0bc0244b23f502a2b99cf9233 100644 --- a/cumulus/bridges/primitives/chain-rococo/src/lib.rs +++ b/bridges/primitives/chain-rococo/src/lib.rs @@ -61,6 +61,9 @@ parameter_types! { pub const SS58Prefix: u8 = 42; } +// The SignedExtension used by Rococo. +pub use bp_polkadot_core::CommonSignedExtension as SignedExtension; + /// Name of the parachains pallet in the Rococo runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-wococo/Cargo.toml b/bridges/primitives/chain-wococo/Cargo.toml similarity index 73% rename from cumulus/bridges/primitives/chain-wococo/Cargo.toml rename to bridges/primitives/chain-wococo/Cargo.toml index f7feb7f96eb17b7f1f16c5abc77904a3cbc26c58..05901821b366b71d40d9c28392150172be44a0b7 100644 --- a/cumulus/bridges/primitives/chain-wococo/Cargo.toml +++ b/bridges/primitives/chain-wococo/Cargo.toml @@ -17,9 +17,9 @@ bp-rococo = { path = "../chain-rococo", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-wococo/src/lib.rs b/bridges/primitives/chain-wococo/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-wococo/src/lib.rs rename to bridges/primitives/chain-wococo/src/lib.rs index c64451993ee7a8bba32b7f7cd7e00cbeab5dd048..5b5bde82690442c3a2c26b424163eeb58b5892e9 100644 --- a/cumulus/bridges/primitives/chain-wococo/src/lib.rs +++ b/bridges/primitives/chain-wococo/src/lib.rs @@ -60,6 +60,9 @@ impl ChainWithGrandpa for Wococo { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +// The SignedExtension used by Wococo. +pub use bp_rococo::CommonSignedExtension as SignedExtension; + /// Name of the With-Wococo GRANDPA pallet instance that is deployed at bridged chains. pub const WITH_WOCOCO_GRANDPA_PALLET_NAME: &str = "BridgeWococoGrandpa"; diff --git a/cumulus/bridges/primitives/header-chain/Cargo.toml b/bridges/primitives/header-chain/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/header-chain/Cargo.toml rename to bridges/primitives/header-chain/Cargo.toml index a7b53b0336dcc59c88bfd828e3af75b540cfe551..e3e83235960a8d1565d5be040ce222b632e95876 100644 --- a/cumulus/bridges/primitives/header-chain/Cargo.toml +++ b/bridges/primitives/header-chain/Cargo.toml @@ -18,11 +18,11 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false, features = ["serde"] } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false, features = ["serde"] } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-test-utils = { path = "../test-utils" } diff --git a/cumulus/bridges/primitives/header-chain/src/justification/mod.rs b/bridges/primitives/header-chain/src/justification/mod.rs similarity index 97% rename from cumulus/bridges/primitives/header-chain/src/justification/mod.rs rename to bridges/primitives/header-chain/src/justification/mod.rs index 24c453a0790c90c04b4573a85a66a276a8e6fa89..72a5f68918d9703babe1e9c263f9148c57df4340 100644 --- a/cumulus/bridges/primitives/header-chain/src/justification/mod.rs +++ b/bridges/primitives/header-chain/src/justification/mod.rs @@ -97,7 +97,11 @@ impl GrandpaJustification { } } -impl crate::FinalityProof for GrandpaJustification { +impl crate::FinalityProof for GrandpaJustification { + fn target_header_hash(&self) -> H::Hash { + self.commit.target_hash + } + fn target_header_number(&self) -> H::Number { self.commit.target_number } diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/equivocation.rs b/bridges/primitives/header-chain/src/justification/verification/equivocation.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/equivocation.rs rename to bridges/primitives/header-chain/src/justification/verification/equivocation.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/mod.rs b/bridges/primitives/header-chain/src/justification/verification/mod.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/mod.rs rename to bridges/primitives/header-chain/src/justification/verification/mod.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/optimizer.rs b/bridges/primitives/header-chain/src/justification/verification/optimizer.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/optimizer.rs rename to bridges/primitives/header-chain/src/justification/verification/optimizer.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/strict.rs b/bridges/primitives/header-chain/src/justification/verification/strict.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/strict.rs rename to bridges/primitives/header-chain/src/justification/verification/strict.rs diff --git a/cumulus/bridges/primitives/header-chain/src/lib.rs b/bridges/primitives/header-chain/src/lib.rs similarity index 98% rename from cumulus/bridges/primitives/header-chain/src/lib.rs rename to bridges/primitives/header-chain/src/lib.rs index 7008dfa6063aedaa2d4869eef87f3a076a7b2380..d2c7ec0759e884713cb37a871bbcc65cabd256af 100644 --- a/cumulus/bridges/primitives/header-chain/src/lib.rs +++ b/bridges/primitives/header-chain/src/lib.rs @@ -127,7 +127,10 @@ pub struct InitializationData { } /// Abstract finality proof that is justifying block finality. -pub trait FinalityProof: Clone + Send + Sync + Debug { +pub trait FinalityProof: Clone + Send + Sync + Debug { + /// Return hash of header that this proof is generated for. + fn target_header_hash(&self) -> Hash; + /// Return number of header that this proof is generated for. fn target_header_number(&self) -> Number; } @@ -209,7 +212,7 @@ impl TryFrom> for HeaderGrandpa /// Helper trait for finding equivocations in finality proofs. pub trait FindEquivocations { /// The type returned when encountering an error while looking for equivocations. - type Error; + type Error: Debug; /// Find equivocations. fn find_equivocations( diff --git a/cumulus/bridges/primitives/header-chain/src/storage_keys.rs b/bridges/primitives/header-chain/src/storage_keys.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/storage_keys.rs rename to bridges/primitives/header-chain/src/storage_keys.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/implementation_match.rs b/bridges/primitives/header-chain/tests/implementation_match.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/implementation_match.rs rename to bridges/primitives/header-chain/tests/implementation_match.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/equivocation.rs b/bridges/primitives/header-chain/tests/justification/equivocation.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/equivocation.rs rename to bridges/primitives/header-chain/tests/justification/equivocation.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/optimizer.rs b/bridges/primitives/header-chain/tests/justification/optimizer.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/optimizer.rs rename to bridges/primitives/header-chain/tests/justification/optimizer.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/strict.rs b/bridges/primitives/header-chain/tests/justification/strict.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/strict.rs rename to bridges/primitives/header-chain/tests/justification/strict.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/tests.rs b/bridges/primitives/header-chain/tests/tests.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/tests.rs rename to bridges/primitives/header-chain/tests/tests.rs diff --git a/cumulus/bridges/primitives/messages/Cargo.toml b/bridges/primitives/messages/Cargo.toml similarity index 78% rename from cumulus/bridges/primitives/messages/Cargo.toml rename to bridges/primitives/messages/Cargo.toml index b1fa6d575aeb60a98c879ac0d4320b2d213408da..b30d6d2559f7916e79f2a3aa2c83ff66298bb24c 100644 --- a/cumulus/bridges/primitives/messages/Cargo.toml +++ b/bridges/primitives/messages/Cargo.toml @@ -18,9 +18,9 @@ bp-header-chain = { path = "../header-chain", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/messages/src/lib.rs b/bridges/primitives/messages/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/lib.rs rename to bridges/primitives/messages/src/lib.rs diff --git a/cumulus/bridges/primitives/messages/src/source_chain.rs b/bridges/primitives/messages/src/source_chain.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/source_chain.rs rename to bridges/primitives/messages/src/source_chain.rs diff --git a/cumulus/bridges/primitives/messages/src/storage_keys.rs b/bridges/primitives/messages/src/storage_keys.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/storage_keys.rs rename to bridges/primitives/messages/src/storage_keys.rs diff --git a/cumulus/bridges/primitives/messages/src/target_chain.rs b/bridges/primitives/messages/src/target_chain.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/target_chain.rs rename to bridges/primitives/messages/src/target_chain.rs diff --git a/cumulus/bridges/primitives/parachains/Cargo.toml b/bridges/primitives/parachains/Cargo.toml similarity index 72% rename from cumulus/bridges/primitives/parachains/Cargo.toml rename to bridges/primitives/parachains/Cargo.toml index 978296954bb7f2bcc716018eaa232f03562e6e89..ca69523dde37a68ea75dfbc23b5258d04c5510b9 100644 --- a/cumulus/bridges/primitives/parachains/Cargo.toml +++ b/bridges/primitives/parachains/Cargo.toml @@ -19,10 +19,10 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/parachains/src/lib.rs b/bridges/primitives/parachains/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/parachains/src/lib.rs rename to bridges/primitives/parachains/src/lib.rs diff --git a/cumulus/bridges/primitives/polkadot-core/Cargo.toml b/bridges/primitives/polkadot-core/Cargo.toml similarity index 69% rename from cumulus/bridges/primitives/polkadot-core/Cargo.toml rename to bridges/primitives/polkadot-core/Cargo.toml index 2563adaf219c9909c5e45f97fa3003057cc6cabc..aa7eb8024fbf2c7613264af8daa5fefe0b24f7cc 100644 --- a/cumulus/bridges/primitives/polkadot-core/Cargo.toml +++ b/bridges/primitives/polkadot-core/Cargo.toml @@ -19,11 +19,11 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/polkadot-core/src/lib.rs b/bridges/primitives/polkadot-core/src/lib.rs similarity index 81% rename from cumulus/bridges/primitives/polkadot-core/src/lib.rs rename to bridges/primitives/polkadot-core/src/lib.rs index b35d97ec79ae2275a5ab885a2f7eb4948e45969d..af39b5ab9babae2b2e6858bff83eaf8c29ef74bb 100644 --- a/cumulus/bridges/primitives/polkadot-core/src/lib.rs +++ b/bridges/primitives/polkadot-core/src/lib.rs @@ -17,7 +17,15 @@ #![cfg_attr(not(feature = "std"), no_std)] use bp_messages::MessageNonce; -use bp_runtime::{Chain, EncodedOrDecodedCall, StorageMapKeyProvider}; +use bp_runtime::{ + self, + extensions::{ + ChargeTransactionPayment, CheckEra, CheckGenesis, CheckNonZeroSender, CheckNonce, + CheckSpecVersion, CheckTxVersion, CheckWeight, GenericSignedExtension, + SignedExtensionSchema, + }, + Chain, EncodedOrDecodedCall, StorageMapKeyProvider, TransactionEra, +}; use frame_support::{ dispatch::DispatchClass, parameter_types, @@ -272,6 +280,99 @@ impl AccountInfoStorageMapKeyProvider { } } +/// Extra signed extension data that is used by most chains. +pub type CommonSignedExtra = ( + CheckNonZeroSender, + CheckSpecVersion, + CheckTxVersion, + CheckGenesis, + CheckEra, + CheckNonce, + CheckWeight, + ChargeTransactionPayment, +); + +/// Extra signed extension data that starts with `CommonSignedExtra`. +pub type SuffixedCommonSignedExtension = + GenericSignedExtension<(CommonSignedExtra, Suffix)>; + +/// Helper trait to define some extra methods on `SuffixedCommonSignedExtension`. +pub trait SuffixedCommonSignedExtensionExt { + /// Create signed extension from its components. + fn from_params( + spec_version: u32, + transaction_version: u32, + era: TransactionEra, + genesis_hash: Hash, + nonce: Nonce, + tip: Balance, + extra: (Suffix::Payload, Suffix::AdditionalSigned), + ) -> Self; + + /// Return transaction nonce. + fn nonce(&self) -> Nonce; + + /// Return transaction tip. + fn tip(&self) -> Balance; +} + +impl SuffixedCommonSignedExtensionExt for SuffixedCommonSignedExtension +where + Suffix: SignedExtensionSchema, +{ + fn from_params( + spec_version: u32, + transaction_version: u32, + era: TransactionEra, + genesis_hash: Hash, + nonce: Nonce, + tip: Balance, + extra: (Suffix::Payload, Suffix::AdditionalSigned), + ) -> Self { + GenericSignedExtension::new( + ( + ( + (), // non-zero sender + (), // spec version + (), // tx version + (), // genesis + era.frame_era(), // era + nonce.into(), // nonce (compact encoding) + (), // Check weight + tip.into(), // transaction payment / tip (compact encoding) + ), + extra.0, + ), + Some(( + ( + (), + spec_version, + transaction_version, + genesis_hash, + era.signed_payload(genesis_hash), + (), + (), + (), + ), + extra.1, + )), + ) + } + + fn nonce(&self) -> Nonce { + let common_payload = self.payload.0; + common_payload.5 .0 + } + + fn tip(&self) -> Balance { + let common_payload = self.payload.0; + common_payload.7 .0 + } +} + +/// Signed extension that is used by most chains. +pub type CommonSignedExtension = SuffixedCommonSignedExtension<()>; + #[cfg(test)] mod tests { use super::*; diff --git a/cumulus/bridges/primitives/polkadot-core/src/parachains.rs b/bridges/primitives/polkadot-core/src/parachains.rs similarity index 100% rename from cumulus/bridges/primitives/polkadot-core/src/parachains.rs rename to bridges/primitives/polkadot-core/src/parachains.rs diff --git a/cumulus/bridges/primitives/relayers/Cargo.toml b/bridges/primitives/relayers/Cargo.toml similarity index 74% rename from cumulus/bridges/primitives/relayers/Cargo.toml rename to bridges/primitives/relayers/Cargo.toml index bc674d4cb773682acdae3e7f90e546dd239cd6fb..99cd79c68417205f11d329bcce8637022aaa6e51 100644 --- a/cumulus/bridges/primitives/relayers/Cargo.toml +++ b/bridges/primitives/relayers/Cargo.toml @@ -17,9 +17,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/relayers/src/lib.rs b/bridges/primitives/relayers/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/relayers/src/lib.rs rename to bridges/primitives/relayers/src/lib.rs diff --git a/cumulus/bridges/primitives/relayers/src/registration.rs b/bridges/primitives/relayers/src/registration.rs similarity index 100% rename from cumulus/bridges/primitives/relayers/src/registration.rs rename to bridges/primitives/relayers/src/registration.rs diff --git a/cumulus/bridges/primitives/runtime/Cargo.toml b/bridges/primitives/runtime/Cargo.toml similarity index 57% rename from cumulus/bridges/primitives/runtime/Cargo.toml rename to bridges/primitives/runtime/Cargo.toml index f6134d6e3328004ff1e7cc50128abfe75dd5d014..4454066b59faed0d57b1ab8d06f619949007b5d4 100644 --- a/cumulus/bridges/primitives/runtime/Cargo.toml +++ b/bridges/primitives/runtime/Cargo.toml @@ -17,15 +17,15 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-io = { path = "../../../../substrate/primitives/io", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-state-machine = { path = "../../../../substrate/primitives/state-machine", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-io = { path = "../../../substrate/primitives/io", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-state-machine = { path = "../../../substrate/primitives/state-machine", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } +trie-db = { version = "0.28.0", default-features = false } [dev-dependencies] hex-literal = "0.4" diff --git a/cumulus/bridges/primitives/runtime/src/chain.rs b/bridges/primitives/runtime/src/chain.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/chain.rs rename to bridges/primitives/runtime/src/chain.rs diff --git a/cumulus/bridges/primitives/runtime/src/extensions.rs b/bridges/primitives/runtime/src/extensions.rs similarity index 94% rename from cumulus/bridges/primitives/runtime/src/extensions.rs rename to bridges/primitives/runtime/src/extensions.rs index 253350d17e74faa13c97ab2a0ee1876fce69693f..44eeaad93c91603b34713ca477b5623f4e07aad0 100644 --- a/cumulus/bridges/primitives/runtime/src/extensions.rs +++ b/bridges/primitives/runtime/src/extensions.rs @@ -35,7 +35,12 @@ pub trait SignedExtensionSchema: Encode + Decode + Debug + Eq + Clone + StaticTy type AdditionalSigned: Encode + Debug + Eq + Clone + StaticTypeInfo; } -// An implementation of `SignedExtensionSchema` using generic params. +impl SignedExtensionSchema for () { + type Payload = (); + type AdditionalSigned = (); +} + +/// An implementation of `SignedExtensionSchema` using generic params. #[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, TypeInfo)] pub struct GenericSignedExtensionSchema(PhantomData<(P, S)>); @@ -72,6 +77,9 @@ pub type CheckWeight = GenericSignedExtensionSchema<(), ()>; /// The `SignedExtensionSchema` for `pallet_transaction_payment::ChargeTransactionPayment`. pub type ChargeTransactionPayment = GenericSignedExtensionSchema, ()>; +/// The `SignedExtensionSchema` for `polkadot-runtime-common::PrevalidateAttests`. +pub type PrevalidateAttests = GenericSignedExtensionSchema<(), ()>; + /// The `SignedExtensionSchema` for `BridgeRejectObsoleteHeadersAndMessages`. pub type BridgeRejectObsoleteHeadersAndMessages = GenericSignedExtensionSchema<(), ()>; @@ -99,7 +107,7 @@ pub struct GenericSignedExtension { // It may be set to `None` if extensions are decoded. We are never reconstructing transactions // (and it makes no sense to do that) => decoded version of `SignedExtensions` is only used to // read fields of the `payload`. And when resigning transaction, we're reconstructing - // `SignedExtensions` from the scratch. + // `SignedExtensions` from scratch. additional_signed: Option, } diff --git a/cumulus/bridges/primitives/runtime/src/lib.rs b/bridges/primitives/runtime/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/lib.rs rename to bridges/primitives/runtime/src/lib.rs diff --git a/cumulus/bridges/primitives/runtime/src/messages.rs b/bridges/primitives/runtime/src/messages.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/messages.rs rename to bridges/primitives/runtime/src/messages.rs diff --git a/cumulus/bridges/primitives/runtime/src/storage_proof.rs b/bridges/primitives/runtime/src/storage_proof.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/storage_proof.rs rename to bridges/primitives/runtime/src/storage_proof.rs diff --git a/cumulus/bridges/primitives/runtime/src/storage_types.rs b/bridges/primitives/runtime/src/storage_types.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/storage_types.rs rename to bridges/primitives/runtime/src/storage_types.rs diff --git a/cumulus/bridges/primitives/test-utils/Cargo.toml b/bridges/primitives/test-utils/Cargo.toml similarity index 57% rename from cumulus/bridges/primitives/test-utils/Cargo.toml rename to bridges/primitives/test-utils/Cargo.toml index fc6e5859141ebe6dfa17bc854e1740eb9a90907c..7081ce90f97b90c3eb437d297fae5cc9f65cfe61 100644 --- a/cumulus/bridges/primitives/test-utils/Cargo.toml +++ b/bridges/primitives/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" authors.workspace = true edition.workspace = true license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -publish = false [dependencies] bp-header-chain = { path = "../header-chain", default-features = false } @@ -12,14 +11,14 @@ bp-parachains = { path = "../parachains", default-features = false } bp-polkadot-core = { path = "../polkadot-core", default-features = false } bp-runtime = { path = "../runtime", default-features = false } codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false } -ed25519-dalek = { version = "1.0", default-features = false, features = ["u64_backend"] } +ed25519-dalek = { version = "2.0", default-features = false } finality-grandpa = { version = "0.16.2", default-features = false } -sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +sp-application-crypto = { path = "../../../substrate/primitives/application-crypto", default-features = false } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/test-utils/src/keyring.rs b/bridges/primitives/test-utils/src/keyring.rs similarity index 81% rename from cumulus/bridges/primitives/test-utils/src/keyring.rs rename to bridges/primitives/test-utils/src/keyring.rs index b99132de3ec36bf0f58a454635876708258d1c0f..eabf9c784eb8182f2824cd4c62defe4b92657450 100644 --- a/cumulus/bridges/primitives/test-utils/src/keyring.rs +++ b/bridges/primitives/test-utils/src/keyring.rs @@ -18,7 +18,7 @@ use bp_header_chain::{justification::JustificationVerificationContext, AuthoritySet}; use codec::Encode; -use ed25519_dalek::{Keypair, PublicKey, SecretKey, Signature}; +use ed25519_dalek::{Signature, SigningKey, VerifyingKey}; use finality_grandpa::voter_set::VoterSet; use sp_consensus_grandpa::{AuthorityId, AuthorityList, AuthorityWeight, SetId}; use sp_runtime::RuntimeDebug; @@ -37,29 +37,15 @@ pub const FERDIE: Account = Account(5); pub struct Account(pub u16); impl Account { - pub fn public(&self) -> PublicKey { - (&self.secret()).into() + pub fn public(&self) -> VerifyingKey { + self.pair().verifying_key() } - pub fn secret(&self) -> SecretKey { + pub fn pair(&self) -> SigningKey { let data = self.0.encode(); let mut bytes = [0_u8; 32]; bytes[0..data.len()].copy_from_slice(&data); - SecretKey::from_bytes(&bytes) - .expect("A static array of the correct length is a known good.") - } - - pub fn pair(&self) -> Keypair { - let mut pair: [u8; 64] = [0; 64]; - - let secret = self.secret(); - pair[..32].copy_from_slice(&secret.to_bytes()); - - let public = self.public(); - pair[32..].copy_from_slice(&public.to_bytes()); - - Keypair::from_bytes(&pair) - .expect("We expect the SecretKey to be good, so this must also be good.") + SigningKey::from_bytes(&bytes) } pub fn sign(&self, msg: &[u8]) -> Signature { diff --git a/cumulus/bridges/primitives/test-utils/src/lib.rs b/bridges/primitives/test-utils/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/test-utils/src/lib.rs rename to bridges/primitives/test-utils/src/lib.rs diff --git a/cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml b/bridges/primitives/xcm-bridge-hub-router/Cargo.toml similarity index 76% rename from cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml rename to bridges/primitives/xcm-bridge-hub-router/Cargo.toml index df2b00563deb1df184cf5969f81ea82d5816ae6a..725a7d94564e226110453b35cdf4f0161c12c113 100644 --- a/cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml +++ b/bridges/primitives/xcm-bridge-hub-router/Cargo.toml @@ -11,8 +11,8 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["bit-vec", "derive"] } # Substrate Dependencies -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/xcm-bridge-hub-router/src/lib.rs b/bridges/primitives/xcm-bridge-hub-router/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/xcm-bridge-hub-router/src/lib.rs rename to bridges/primitives/xcm-bridge-hub-router/src/lib.rs diff --git a/cumulus/bridges/scripts/verify-pallets-build.sh b/bridges/scripts/verify-pallets-build.sh similarity index 93% rename from cumulus/bridges/scripts/verify-pallets-build.sh rename to bridges/scripts/verify-pallets-build.sh index 1ea3dbe8626030eaeb644ce80a0d58fd2760f038..b8ac09e26b8bcd78e4a0c3022c098f2ebc93afd3 100755 --- a/cumulus/bridges/scripts/verify-pallets-build.sh +++ b/bridges/scripts/verify-pallets-build.sh @@ -89,17 +89,26 @@ rm -rf $BRIDGES_FOLDER/scripts/update-weights-setup.sh rm -rf $BRIDGES_FOLDER/scripts/update_substrate.sh rm -rf $BRIDGES_FOLDER/tools rm -f $BRIDGES_FOLDER/.dockerignore +rm -f $BRIDGES_FOLDER/local.Dockerfile.dockerignore rm -f $BRIDGES_FOLDER/deny.toml rm -f $BRIDGES_FOLDER/.gitlab-ci.yml rm -f $BRIDGES_FOLDER/.editorconfig rm -f $BRIDGES_FOLDER/Cargo.toml rm -f $BRIDGES_FOLDER/ci.Dockerfile +rm -f $BRIDGES_FOLDER/local.Dockerfile rm -f $BRIDGES_FOLDER/CODEOWNERS rm -f $BRIDGES_FOLDER/Dockerfile +rm -f $BRIDGES_FOLDER/rustfmt.toml # let's fix Cargo.toml a bit (it'll be helpful if we are in the bridges repo) if [[ ! -f "Cargo.toml" ]]; then cat > Cargo.toml <<-CARGO_TOML + [workspace.package] + authors = ["Parity Technologies "] + edition = "2021" + repository = "https://github.com/paritytech/parity-bridges-common.git" + license = "GPL-3.0-only" + [workspace] resolver = "2" diff --git a/cumulus/.github/ISSUE_TEMPLATE/release-client.md b/cumulus/.github/ISSUE_TEMPLATE/release-client.md deleted file mode 100644 index bb7f206157674634e395631eb2434659c5ad026e..0000000000000000000000000000000000000000 --- a/cumulus/.github/ISSUE_TEMPLATE/release-client.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Release Checklist for Client -about: Release Checklist for Client -title: Release Checklist for Client {{ env.VERSION }} ---- - -# Release Checklist - Client - -### Client Release - -- [ ] build a new `polkadot-parachain` binary and publish it to S3 -- [ ] new `polkadot-parachain` version has [run on the network](../../docs/release.md#burnin) - without issue for at least 12h -- [ ] a draft release has been created in the [Github Releases page](https://github.com/paritytech/cumulus/releases) with the relevant release-notes -- [ ] the [build artifacts](../../docs/release.md#build-artifacts) have been added to the - draft-release. - ---- - -Read more about the [release documentation](../../docs/release.md). diff --git a/cumulus/.github/ISSUE_TEMPLATE/release-runtime.md b/cumulus/.github/ISSUE_TEMPLATE/release-runtime.md deleted file mode 100644 index 0f3543759afd924ca6c74011ed6e64dc1e5005f4..0000000000000000000000000000000000000000 --- a/cumulus/.github/ISSUE_TEMPLATE/release-runtime.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -name: Release Checklist for Runtime -about: Release Checklist for Runtime -title: Release Checklist for Runtime {{ env.VERSION }} ---- - -# Release Checklist - Runtimes - -**All** following checks must be completed before publishing a new release. -The release process is owned and led by @paritytech/release-engineering team. -The checks marked with :crab: are meant to be checked by [a runtime engineer](https://github.com/paritytech/cumulus/issues/1761). - -## Runtimes Release - -### Codebase -These checks should be performed on the codebase. - -- [ ] the [`spec_version`](https://github.com/paritytech/cumulus/blob/master/docs/release.md#spec-version) has been incremented since the - last release for any native runtimes from any existing use on public (non-private/test) networks -- [ ] :crab: previously [completed migrations](https://github.com/paritytech/cumulus/blob/master/docs/release.md#old-migrations-removed) are removed for any public (non-private/test) networks -- [ ] pallet and [extrinsic ordering](https://github.com/paritytech/cumulus/blob/master/docs/release.md#extrinsic-ordering--storage) as well as `SignedExtension`s have stayed - the same. Bump `transaction_version` otherwise -- [ ] the [benchmarks](https://github.com/paritytech/ci_cd/wiki/Benchmarks:-cumulus) ran -- [ ] the weights have been updated for any modified runtime logic -- [ ] :crab: the new weights are sane, there are no significant (>50%) drops or rises with no reason -- [ ] :crab: XCM config is compatible with the configurations and versions of relevant interlocutors, like the Relay Chain. - -### On the release branch - -The following checks can be performed after we have forked off to the release-candidate branch or started an additional release candidate branch (rc-2, rc-3, etc) - -- [ ] Verify [new migrations](https://github.com/paritytech/cumulus/blob/master/docs/release.md#new-migrations) complete successfully, and the - runtime state is correctly updated for any public (non-private/test) - networks -- [ ] Run [integration tests](https://github.com/paritytech/cumulus/blob/master/docs/release.md#integration-tests), and make sure they pass. -- [ ] Push runtime upgrade to Asset Hub Westend and verify network stability -- [ ] Push runtime upgrade to Collectives and verify network stability -- [ ] Push runtime upgrade to Bridge-Hub-Kusama and verify network stability - - -### Github - -- [ ] Check that a draft release has been created at the [Github Releases page](https://github.com/paritytech/cumulus/releases) with relevant [release - notes](https://github.com/paritytech/cumulus/blob/master/docs/release.md#release-notes) -- [ ] Check that [build artifacts](https://github.com/paritytech/cumulus/blob/master/docs/release.md#build-artifacts) have been added to the - draft-release. - -# Post release - -- [ ] :crab: all commits (runtime version bumps, fixes) on this release branch have been merged back to master. - ---- - -Read more about the [release documentation](https://github.com/paritytech/cumulus/blob/master/docs/release.md). diff --git a/cumulus/.github/workflows/release-50_publish-docker.yml b/cumulus/.github/workflows/release-50_publish-docker.yml deleted file mode 100644 index 6ad943c3903ca597b38dedf64061e62a6c39cf6e..0000000000000000000000000000000000000000 --- a/cumulus/.github/workflows/release-50_publish-docker.yml +++ /dev/null @@ -1,206 +0,0 @@ -name: Release - Publish Docker Image - -# This workflow listens to pubished releases or can be triggered manually. -# It includes releases and rc candidates. -# It fetches the binaries, checks sha256 and GPG -# signatures, then builds an injected docker -# image and publishes it. - -on: - release: - types: - - published - workflow_dispatch: - inputs: - release_id: - description: | - Release ID. - You can find it using the command: - curl -s \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ - jq '.[] | { name: .name, id: .id }' - required: true - type: string - image_type: - description: Type of the image to be published - required: true - default: rc - type: choice - options: - - rc - - release - registry: - description: Container registry - required: true - type: string - default: docker.io - owner: - description: Owner of the container image repo - required: true - type: string - default: parity - -env: - RELEASE_ID: ${{ inputs.release_id }} - ENGINE: docker - REGISTRY: ${{ inputs.registry }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} - REPO: ${{ github.repository }} - BINARY: polkadot-parachain - EVENT_ACTION: ${{ github.event.action }} - EVENT_NAME: ${{ github.event_name }} - IMAGE_TYPE: ${{ inputs.image_type }} - -jobs: - fetch-artifacts: - runs-on: ubuntu-latest - - steps: - - name: Checkout sources - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - - name: Prepare temp folder - run: | - TMP=$(mktemp -d) - echo "TMP=$TMP" >> "$GITHUB_ENV" - pwd - ls -al "$TMP" - - - name: Fetch lib.sh from polkadot repo - working-directory: ${{ env.TMP }} - run: | - curl -O -L \ - -H "Accept: application/vnd.github.v3.raw" \ - https://raw.githubusercontent.com/paritytech/polkadot/master/scripts/ci/common/lib.sh - - chmod a+x lib.sh - ls -al - - - name: Fetch release artifacts based on final release tag - #this step runs only if the workflow is triggered automatically when new release is published - if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} - run: | - mkdir -p release-artifacts && cd release-artifacts - - for f in $BINARY $BINARY.asc $BINARY.sha256; do - URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f" - echo " - Fetching $f from $URL" - wget "$URL" -O "$f" - done - chmod a+x $BINARY - cp -f ${TMP}/lib.sh . - ls -al - - - name: Fetch rc artifacts or release artifacts based on release id - #this step runs only if the workflow is triggered manually - if: ${{ env.EVENT_NAME == 'workflow_dispatch' }} - run: | - . ${TMP}/lib.sh - - fetch_release_artifacts - - chmod a+x release-artifacts/$BINARY - ls -al - - cp -f ${TMP}/lib.sh release-artifacts/ - - - name: Cache the artifacts - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - path: | - ./release-artifacts/**/* - - build-container: - runs-on: ubuntu-latest - needs: fetch-artifacts - - steps: - - name: Checkout sources - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - - name: Get artifacts from cache - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - fail-on-cache-miss: true - path: | - ./release-artifacts/**/* - - - name: Check sha256 ${{ env.BINARY }} - working-directory: ./release-artifacts - run: | - . ./lib.sh - - echo "Checking binary $BINARY" - check_sha256 $BINARY && echo "OK" || echo "ERR" - - - name: Check GPG ${{ env.BINARY }} - working-directory: ./release-artifacts - run: | - . ./lib.sh - import_gpg_keys - check_gpg $BINARY - - - name: Build Injected Container image for ${{ env.BINARY }} - env: - IMAGE_NAME: ${{ env.BINARY }} - OWNER: ${{ env.DOCKER_OWNER }} - run: | - ls -al - echo "Building container for $BINARY" - ./docker/scripts/build-injected-image.sh - - - name: Fetch rc commit and tag - if: ${{ env.IMAGE_TYPE == 'rc' }} - id: fetch_rc_refs - run: | - release=release-${{ inputs.release_id }} && \ - echo "release=${release}" >> $GITHUB_OUTPUT - - commit=$(git rev-parse --short HEAD) && \ - echo "commit=${commit}" >> $GITHUB_OUTPUT - - tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ - [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ - echo "No tag, doing without" - - - name: Fetch release tags - if: ${{ env.IMAGE_TYPE == 'release' || env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} - id: fetch_release_refs - run: | - VERSION=$(docker run --pull never --rm $DOCKER_OWNER/$BINARY --version | awk '{ print $2 }' ) - release=$( echo $VERSION | cut -f1 -d- ) - echo "tag=latest" >> $GITHUB_OUTPUT - echo "release=${release}" >> $GITHUB_OUTPUT - - - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Tag and Push Container image for ${{ env.BINARY }} - id: docker_push - env: - TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} - run: | - TAGS=${TAGS[@]:-latest} - IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" - - echo "The image ${BINARY} will be tagged with ${TAG_ARRAY[*]}" - for TAG in "${TAG_ARRAY[@]}"; do - $ENGINE tag ${DOCKER_OWNER}/${BINARY} ${DOCKER_OWNER}/${BINARY}:${TAG} - $ENGINE push ${DOCKER_OWNER}/${BINARY}:${TAG} - done - - $ENGINE images | grep ${BINARY} - - - name: Check version for the published image for ${{ env.BINARY }} - env: - RELEASE_TAG: ${{ steps.fetch_rc_refs.outputs.release || steps.fetch_release_refs.outputs.release }} - run: | - echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" - $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version diff --git a/cumulus/CODEOWNERS b/cumulus/CODEOWNERS deleted file mode 100644 index 3e5e8bd062dda4d1fd5675b49daaf7186b3a034e..0000000000000000000000000000000000000000 --- a/cumulus/CODEOWNERS +++ /dev/null @@ -1,23 +0,0 @@ -# Lists some code owners. -# -# A codeowner just oversees some part of the codebase. If an owned file is changed then the -# corresponding codeowner receives a review request. An approval of the codeowner might be -# required for merging a PR (depends on repository settings). -# -# For details about syntax, see: -# https://help.github.com/en/articles/about-code-owners -# But here are some important notes: -# -# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` -# which can be everywhere. -# - Multiple owners are supported. -# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind, -# that handles might work better because they are more recognizable on GitHub, -# eyou can use them for mentioning unlike an email. -# - The latest matching rule, if multiple, takes precedence. - -# CI -/.github/ @paritytech/ci @paritytech/release-engineering -/.gitlab-ci.yml @paritytech/ci -/scripts/ci/ @paritytech/ci @paritytech/release-engineering - diff --git a/cumulus/README.md b/cumulus/README.md index 419e293a0abd3a032de6e591d60b7006c64aa628..19f9f3f113dd06655c418a4caf6b30a9cad6ee9f 100644 --- a/cumulus/README.md +++ b/cumulus/README.md @@ -2,59 +2,53 @@ [![Doc](https://img.shields.io/badge/cumulus%20docs-master-brightgreen)](https://paritytech.github.io/cumulus/) -This repository contains both the Cumulus SDK and also specific chains implemented on top of this -SDK. +This repository contains both the Cumulus SDK and also specific chains implemented on top of this SDK. -If you only want to run a **Polkadot Parachain Node**, check out our [container -section](./docs/container.md). +If you only want to run a **Polkadot Parachain Node**, check out our [container section](./docs/container.md). ## Cumulus SDK -A set of tools for writing [Substrate](https://substrate.io/)-based -[Polkadot](https://wiki.polkadot.network/en/) -[parachains](https://wiki.polkadot.network/docs/en/learn-parachains). Refer to the included -[overview](docs/overview.md) for architectural details, and the [Connect to a relay chain how-to -guide](https://docs.substrate.io/reference/how-to-guides/parachains/connect-to-a-relay-chain/) for a -guided walk-through of using these tools. +A set of tools for writing [Substrate](https://substrate.io/)-based [Polkadot](https://wiki.polkadot.network/en/) +[parachains](https://wiki.polkadot.network/docs/en/learn-parachains). Refer to the included [overview](docs/overview.md) +for architectural details, and the [Connect to a relay chain how-to +guide](https://docs.substrate.io/reference/how-to-guides/parachains/connect-to-a-relay-chain/) for a guided walk-through +of using these tools. -It's easy to write blockchains using Substrate, and the overhead of writing parachains' -distribution, p2p, database, and synchronization layers should be just as low. This project aims to -make it easy to write parachains for Polkadot by leveraging the power of Substrate. +It's easy to write blockchains using Substrate, and the overhead of writing parachains' distribution, p2p, database, and +synchronization layers should be just as low. This project aims to make it easy to write parachains for Polkadot by +leveraging the power of Substrate. -Cumulus clouds are shaped sort of like dots; together they form a system that is intricate, -beautiful and functional. +Cumulus clouds are shaped sort of like dots; together they form a system that is intricate, beautiful and functional. ### Consensus [`parachain-consensus`](https://github.com/paritytech/polkadot-sdk/blob/master/cumulus/client/consensus/common/src/parachain_consensus.rs) -is a [consensus engine](https://docs.substrate.io/v3/advanced/consensus) for Substrate that follows -a Polkadot [relay chain](https://wiki.polkadot.network/docs/en/learn-architecture#relay-chain). This -will run a Polkadot node internally, and dictate to the client and synchronization algorithms which -chain to follow, -[finalize](https://wiki.polkadot.network/docs/en/learn-consensus#probabilistic-vs-provable-finality), -and treat as best. +is a [consensus engine](https://docs.substrate.io/v3/advanced/consensus) for Substrate that follows a Polkadot [relay +chain](https://wiki.polkadot.network/docs/en/learn-architecture#relay-chain). This will run a Polkadot node internally, +and dictate to the client and synchronization algorithms which chain to follow, +[finalize](https://wiki.polkadot.network/docs/en/learn-consensus#probabilistic-vs-provable-finality), and treat as best. ### Collator -A Polkadot [collator](https://wiki.polkadot.network/docs/en/learn-collator) for the parachain is -implemented by the `polkadot-parachain` binary (previously called `polkadot-collator`). +A Polkadot [collator](https://wiki.polkadot.network/docs/en/learn-collator) for the parachain is implemented by the +`polkadot-parachain` binary (previously called `polkadot-collator`). -You may run `polkadot-parachain` locally after building it or using one of the container option -described [here](./docs/container.md). +You may run `polkadot-parachain` locally after building it or using one of the container option described +[here](./docs/container.md). -### Relay Chain Interaction -To operate a parachain node, a connection to the corresponding relay -chain is necessary. This can be achieved in one of three ways: -1. Run a full relay chain node within the parachain node (default) -2. Connect to an external relay chain node via WebSocket RPC +### Relay Chain Interaction +To operate a parachain node, a connection to the corresponding relay chain is necessary. This can be achieved in one of +three ways: +1. Run a full relay chain node within the parachain node (default) +2. Connect to an external relay chain node via WebSocket RPC 3. Run a light client for the relay chain #### In-process Relay Chain Node -If an external relay chain node is not specified (default behavior), then a full relay chain node is -spawned within the same process. +If an external relay chain node is not specified (default behavior), then a full relay chain node is spawned within the +same process. -This node has all of the typical components of a regular Polkadot node and will have to fully sync -with the relay chain to work. +This node has all of the typical components of a regular Polkadot node and will have to fully sync with the relay chain +to work. ##### Example command ```bash @@ -66,19 +60,16 @@ polkadot-parachain \ ``` #### External Relay Chain Node -An external relay chain node is connected via WebsSocket RPC by using the -`--relay-chain-rpc-urls` command line argument. This option accepts one or more -space-separated WebSocket URLs to a full relay chain node. By default, only the -first URL will be used, with the rest as a backup in case the connection to the -first node is lost. +An external relay chain node is connected via WebsSocket RPC by using the `--relay-chain-rpc-urls` command line +argument. This option accepts one or more space-separated WebSocket URLs to a full relay chain node. By default, only +the first URL will be used, with the rest as a backup in case the connection to the first node is lost. -Parachain nodes using this feature won't have to fully sync with the relay chain -to work, so in general they will use fewer system resources. +Parachain nodes using this feature won't have to fully sync with the relay chain to work, so in general they will use +fewer system resources. -**Note:** At this time, any parachain nodes using this feature will still spawn a -significantly cut-down relay chain node in-process. Even though they lack the -majority of normal Polkadot subsystems, they will still need to connect directly -to the relay chain network. +**Note:** At this time, any parachain nodes using this feature will still spawn a significantly cut-down relay chain +node in-process. Even though they lack the majority of normal Polkadot subsystems, they will still need to connect +directly to the relay chain network. ##### Example command @@ -94,17 +85,15 @@ polkadot-parachain \ ``` #### Relay Chain Light Client -An internal relay chain light client provides a fast and lightweight approach -for connecting to the relay chain network. It provides relay chain notifications -and facilitates runtime calls. +An internal relay chain light client provides a fast and lightweight approach for connecting to the relay chain network. +It provides relay chain notifications and facilitates runtime calls. -To specify which chain the light client should connect to, users need to supply -a relay chain chain-spec as part of the relay chain arguments. +To specify which chain the light client should connect to, users need to supply a relay chain chain-spec as part of the +relay chain arguments. -**Note:** At this time, any parachain nodes using this feature will still spawn -a significantly cut-down relay chain node in-process. Even though they lack the -majority of normal Polkadot subsystems, they will still need to connect directly -to the relay chain network. +**Note:** At this time, any parachain nodes using this feature will still spawn a significantly cut-down relay chain +node in-process. Even though they lack the majority of normal Polkadot subsystems, they will still need to connect +directly to the relay chain network. ##### Example command @@ -118,23 +107,22 @@ polkadot-parachain \ ``` ## Installation and Setup -Before building Cumulus SDK based nodes / runtimes prepare your environment by -following Substrate [installation instructions](https://docs.substrate.io/main-docs/install/). +Before building Cumulus SDK based nodes / runtimes prepare your environment by following Substrate [installation +instructions](https://docs.substrate.io/main-docs/install/). -To launch a local network, you can use [zombienet](https://github.com/paritytech/zombienet) -for quick setup and experimentation or follow the [manual setup](#manual-setup). +To launch a local network, you can use [zombienet](https://github.com/paritytech/zombienet) for quick setup and +experimentation or follow the [manual setup](#manual-setup). ### Zombienet -We use Zombienet to spin up networks for integration tests and local networks. -Follow [these installation steps](https://github.com/paritytech/zombienet#requirements-by-provider) -to set it up on your machine. A simple network specification with two relay chain -nodes and one collator is located at [zombienet/examples/small_network.toml](zombienet/examples/small_network.toml). +We use Zombienet to spin up networks for integration tests and local networks. Follow [these installation +steps](https://github.com/paritytech/zombienet#requirements-by-provider) to set it up on your machine. A simple network +specification with two relay chain nodes and one collator is located at +[zombienet/examples/small_network.toml](zombienet/examples/small_network.toml). #### Which provider should I use? Zombienet offers multiple providers to run networks. Choose the one that best fits your needs: - **Podman:** Choose this if you want to spin up a network quick and easy. -- **Native:** Choose this if you want to develop and deploy your changes. Requires compilation -of the binaries. +- **Native:** Choose this if you want to develop and deploy your changes. Requires compilation of the binaries. - **Kubernetes:** Choose this for advanced use-cases or running on cloud-infrastructure. #### How to run @@ -183,13 +171,16 @@ cargo build --release --bin polkadot-parachain ./target/release/polkadot-parachain export-genesis-wasm > genesis-wasm # Collator1 -./target/release/polkadot-parachain --collator --alice --force-authoring --tmp --port 40335 --rpc-port 9946 -- --chain ../polkadot/rococo-local-cfde.json --port 30335 +./target/release/polkadot-parachain --collator --alice --force-authoring \ + --tmp --port 40335 --rpc-port 9946 -- --chain ../polkadot/rococo-local-cfde.json --port 30335 # Collator2 -./target/release/polkadot-parachain --collator --bob --force-authoring --tmp --port 40336 --rpc-port 9947 -- --chain ../polkadot/rococo-local-cfde.json --port 30336 +./target/release/polkadot-parachain --collator --bob --force-authoring \ + --tmp --port 40336 --rpc-port 9947 -- --chain ../polkadot/rococo-local-cfde.json --port 30336 # Parachain Full Node 1 -./target/release/polkadot-parachain --tmp --port 40337 --rpc-port 9948 -- --chain ../polkadot/rococo-local-cfde.json --port 30337 +./target/release/polkadot-parachain --tmp --port 40337 --rpc-port 9948 -- \ + --chain ../polkadot/rococo-local-cfde.json --port 30337 ``` #### Register the parachain @@ -199,8 +190,8 @@ cargo build --release --bin polkadot-parachain ## Asset Hub 🪙 -This repository also contains the Asset Hub runtimes. Asset Hub is a system parachain -providing an asset store for the Polkadot ecosystem. +This repository also contains the Asset Hub runtimes. Asset Hub is a system parachain providing an asset store for the +Polkadot ecosystem. ### Build & Launch a Node @@ -228,20 +219,18 @@ See [the `contracts-rococo` readme](parachains/runtimes/contracts/contracts-roco See [the `bridge-hubs` readme](parachains/runtimes/bridge-hubs/README.md) for details. ## Rococo 👑 -[Rococo](https://polkadot.js.org/apps/?rpc=wss://rococo-rpc.polkadot.io) is becoming a -[Community Parachain Testbed](https://polkadot.network/blog/rococo-revamp-becoming-a-community-parachain-testbed/) -for parachain teams in the Polkadot ecosystem. It supports multiple parachains with the -differentiation of long-term connections and recurring short-term connections, to see -which parachains are currently connected and how long they will be connected for -[see here](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-rpc.polkadot.io#/parachains). +[Rococo](https://polkadot.js.org/apps/?rpc=wss://rococo-rpc.polkadot.io) is becoming a [Community Parachain +Testbed](https://polkadot.network/blog/rococo-revamp-becoming-a-community-parachain-testbed/) for parachain teams in the +Polkadot ecosystem. It supports multiple parachains with the differentiation of long-term connections and recurring +short-term connections, to see which parachains are currently connected and how long they will be connected for [see +here](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-rpc.polkadot.io#/parachains). -Rococo is an elaborate style of design and the name describes the painstaking effort that -has gone into this project. +Rococo is an elaborate style of design and the name describes the painstaking effort that has gone into this project. ### Build & Launch Rococo Collators -Collators are similar to validators in the relay chain. These nodes build the blocks that -will eventually be included by the relay chain for a parachain. +Collators are similar to validators in the relay chain. These nodes build the blocks that will eventually be included by +the relay chain for a parachain. To run a Rococo collator you will need to compile the following binary: @@ -250,8 +239,7 @@ To run a Rococo collator you will need to compile the following binary: cargo build --release --locked --bin polkadot-parachain ``` -Once the executable is built, launch collators for each parachain (repeat once each for chain -`tick`, `trick`, `track`): +Once the executable is built, launch collators for each parachain (repeat once each for chain `tick`, `trick`, `track`): ```bash ./target/release/polkadot-parachain --chain $CHAIN --validator @@ -261,10 +249,10 @@ You can also build [using a container](./docs/container.md). ### Parachains -* [Asset Hub](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-statemint-rpc.polkadot.io#/explorer) -* [Contracts on Rococo](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-contracts-rpc.polkadot.io#/explorer) -* [RILT](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo.kilt.io#/explorer) +- [Asset Hub](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-statemint-rpc.polkadot.io#/explorer) +- [Contracts on Rococo](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-contracts-rpc.polkadot.io#/explorer) +- [RILT](https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo.kilt.io#/explorer) -The network uses horizontal message passing (HRMP) to enable communication between -parachains and the relay chain and, in turn, between parachains. This means that every -message is sent to the relay chain, and from the relay chain to its destination parachain. +The network uses horizontal message passing (HRMP) to enable communication between parachains and the relay chain and, +in turn, between parachains. This means that every message is sent to the relay chain, and from the relay chain to its +destination parachain. diff --git a/cumulus/bridges/SECURITY.md b/cumulus/bridges/SECURITY.md deleted file mode 100644 index 65f2f3bff05dd94786f7f3c8d084af2ca9fe236f..0000000000000000000000000000000000000000 --- a/cumulus/bridges/SECURITY.md +++ /dev/null @@ -1,14 +0,0 @@ -# Security Policy - -Thanks for helping make the Parity ecosystem more secure. Security is one of our first priorities. - -## Reporting a vulnerability - -If you find something that can be treated as a security vulnerability, please do not use the issue tracker or discuss it in the public forum as it can cause more damage, rather than giving real help to the ecosystem. - -Security vulnerabilities should be reported by the [contact form](https://security-submission.parity.io/). - -If you think that your report might be eligible for the Bug Bounty Program, please mark this during the submission. Please check up-to-date [Parity Bug Bounty Program rules](https://www.parity.io/bug-bounty) to find out the information about our Bug Bounty Program. - -**Warning**: This is an unified SECURITY.md file for Paritytech GitHub Organization. The presence of this file does not mean that this repository is covered by the Bug Bounty program. Please always check the Bug Bounty Program scope for information. - diff --git a/cumulus/bridges/docs/high-level-overview.md b/cumulus/bridges/docs/high-level-overview.md deleted file mode 100644 index 449224124afd77a4e53d55e7cf6b3b9a65961802..0000000000000000000000000000000000000000 --- a/cumulus/bridges/docs/high-level-overview.md +++ /dev/null @@ -1,181 +0,0 @@ -# High-Level Bridge Documentation - -This document gives a brief, abstract description of main components that may be found in this repository. -If you want to see how we're using them to build Rococo <> Wococo (Kusama <> Polkadot) bridge, please -refer to the [Polkadot <> Kusama Bridge](./polkadot-kusama-bridge-overview.md). - -## Purpose - -This repo contains all components required to build a trustless connection between standalone Substrate chains, -that are using GRANDPA finality, their parachains or any combination of those. On top of this connection, we -offer a messaging pallet that provides means to organize messages exchange. - -On top of that layered infrastructure, anyone may build their own bridge applications - e.g. [XCM messaging](./polkadot-kusama-bridge-overview.md), -[encoded calls messaging](https://github.com/paritytech/parity-bridges-common/releases/tag/encoded-calls-messaging) and so on. - -## Terminology - -Even though we support (and require) two-way bridging, the documentation will generally talk about -a one-sided interaction. That's to say, we will only talk about syncing finality proofs and messages -from a _source_ chain to a _target_ chain. This is because the two-sided interaction is really just the -one-sided interaction with the source and target chains switched. - -The bridge has both on-chain (pallets) and offchain (relayers) components. - -## On-chain components - -On-chain bridge components are pallets that are deployed at the chain runtime. Finality pallets require -deployment at the target chain, while messages pallet needs to be deployed at both, source -and target chains. - -### Bridge GRANDPA Finality Pallet - -A GRANDPA light client of the source chain built into the target chain's runtime. It provides a "source of truth" -about the source chain headers which have been finalized. This is useful for higher level applications. - -The pallet tracks current GRANDPA authorities set and only accepts finality proofs (GRANDPA justifications), -generated by the current authorities set. The GRANDPA protocol itself requires current authorities set to -generate explicit justification for the header that enacts next authorities set. Such headers and their finality -proofs are called mandatory in the pallet and relayer pays no fee for such headers submission. - -The pallet does not require all headers to be imported or provided. The relayer itself chooses which headers -he wants to submit (with the exception of mandatory headers). - -More: [pallet level documentation and code](../modules/grandpa/). - -### Bridge Parachains Finality Pallet - -Parachains are not supposed to have their own finality, so we can't use bridge GRANDPA pallet to verify their -finality proofs. Instead, they rely on their relay chain finality. The parachain header is considered final, -when it is accepted by the [`paras` pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras) -at its relay chain. Obviously, the relay chain block, where it is accepted, must also be finalized by the relay -chain GRANDPA gadget. - -That said, the bridge parachains pallet accepts storage proof of one or several parachain heads, inserted to the -[`Heads`](https://github.com/paritytech/polkadot/blob/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras/mod.rs#L642) -map of the [`paras` pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras). -To verify this storage proof, the pallet uses relay chain header, imported earlier by the bridge GRANDPA pallet. - -The pallet may track multiple parachains at once and those parachains may use different primitives. So the -parachain header decoding never happens at the pallet level. For maintaining the headers order, the pallet -uses relay chain header number. - -More: [pallet level documentation and code](../modules/parachains/). - -### Bridge Messages Pallet - -The pallet is responsible for queuing messages at the source chain and receiving the messages proofs at the -target chain. The messages are sent to the particular _lane_, where they are guaranteed to be received in the -same order they are sent. The pallet supports many lanes. - -The lane has two ends. Outbound lane end is storing number of messages that have been sent and the number of -messages that have been received. Inbound lane end stores the number of messages that have been received and -also a map that maps messages to relayers that have delivered those messages to the target chain. - -The pallet has three main entrypoints: -- the `send_message` may be used by the other runtime pallets to send the messages; -- the `receive_messages_proof` is responsible for parsing the messages proof and handing messages over to the -dispatch code; -- the `receive_messages_delivery_proof` is responsible for parsing the messages delivery proof and rewarding -relayers that have delivered the message. - -Many things are abstracted by the pallet: -- the message itself may mean anything, the pallet doesn't care about its content; -- the message dispatch happens during delivery, but it is decoupled from the pallet code; -- the messages proof and messages delivery proof are verified outside of the pallet; -- the relayers incentivization scheme is defined outside of the pallet. - -Outside of the messaging pallet, we have a set of adapters, where messages and delivery proofs are regular -storage proofs. The proofs are generated at the bridged chain and require bridged chain finality. So messages -pallet, in this case, depends on one of the finality pallets. The messages are XCM messages and we are using -XCM executor to dispatch them on receival. You may find more info in [Polkadot <> Kusama Bridge](./polkadot-kusama-bridge-overview.md) -document. - -More: [pallet level documentation and code](../modules/messages/). - -### Bridge Relayers Pallet - -The pallet is quite simple. It just registers relayer rewards and has an entrypoint to collect them. When -the rewards are registered and the reward amount is configured outside of the pallet. - -More: [pallet level documentation and code](../modules/relayers/). - -## Offchain Components - -Offchain bridge components are separate processes, called relayers. Relayers are connected both to the -source chain and target chain nodes. Relayers are reading state of the source chain, compare it to the -state of the target chain and, if state at target chain needs to be updated, submits target chain -transaction. - -### GRANDPA Finality Relay - -The task of relay is to submit source chain GRANDPA justifications and their corresponding headers to -the Bridge GRANDPA Finality Pallet, deployed at the target chain. For that, the relay subscribes to -the source chain GRANDPA justifications stream and submits every new justification it sees to the -target chain GRANDPA light client. In addition, relay is searching for mandatory headers and -submits their justifications - without that the pallet will be unable to move forward. - -More: [GRANDPA Finality Relay Sequence Diagram](./grandpa-finality-relay.html), [pallet level documentation and code](../relays/finality/). - -### Parachains Finality Relay - -The relay connects to the source _relay_ chain and the target chain nodes. It doesn't need to connect to the -tracked parachain nodes. The relay looks at the [`Heads`](https://github.com/paritytech/polkadot/blob/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras/mod.rs#L642) -map of the [`paras` pallet](https://github.com/paritytech/polkadot/tree/1a034bd6de0e76721d19aed02a538bcef0787260/runtime/parachains/src/paras) -in source chain, and compares the value with the best parachain head, stored in the bridge parachains pallet at -the target chain. If new parachain head appears at the relay chain block `B`, the relay process **waits** -until header `B` or one of its ancestors appears at the target chain. Once it is available, the storage -proof of the map entry is generated and is submitted to the target chain. - -As its on-chain component (which requires bridge GRANDPA pallet to be deployed nearby), the parachains -finality relay requires GRANDPA finality relay to be running in parallel. Without it, the header `B` or -any of its children's finality at source won't be relayed at target, and target chain -won't be able to verify generated storage proof. - -More: [Parachains Finality Relay Sequence Diagram](./parachains-finality-relay.html), [code](../relays/parachains/). - -### Messages Relay - -Messages relay is actually two relays that are running in a single process: messages delivery relay and -delivery confirmation relay. Even though they are more complex and have many caveats, the overall algorithm -is the same as in other relays. - -Message delivery relay connects to the source chain and looks at the outbound lane end, waiting until new -messages are queued there. Once they appear at the source block `B`, the relay start waiting for the block -`B` or its descendant appear at the target chain. Then the messages storage proof is generated and submitted -to the bridge messages pallet at the target chain. In addition, the transaction may include the storage proof -of the outbound lane state - that proves that relayer rewards have been paid and this data (map of relay -accounts to the delivered messages) may be pruned from the inbound lane state at the target chain. - -Delivery confirmation relay connects to the target chain and starts watching the inbound lane end. When new -messages are delivered to the target chain, the corresponding _source chain account_ is inserted to the -map in the inbound lane data. Relay detects that, say, at the target chain block `B` and waits until that -block or its descendant appears at the source chain. Once that happens, the relay crafts a storage proof of -that data and sends it to the messages pallet, deployed at the source chain. - -As you can see, the messages relay also requires finality relay to be operating in parallel. Since messages -relay submits transactions to both source and target chains, it requires both _source-to-target_ and -_target-to-source_ finality relays. They can be GRANDPA finality relays or GRANDPA+parachains finality relays, -depending on the type of connected chain. - -More: [Messages Relay Sequence Diagram](./messages-relay.html), [pallet level documentation and code](../relays/messages/). - -### Complex Relay - -Every relay transaction has its cost. The only transaction, that is "free" to relayer is when the mandatory -GRANDPA header is submitted. The relay that feeds the bridge with every relay chain and/or parachain head it -sees, will have to pay a (quite large) cost. And if no messages are sent through the bridge, that is just -waste of money. - -We have a special relay mode, called _complex relay_, where relay mostly sleeps and only submits transactions -that are required for the messages/confirmations delivery. This mode starts two message relays (in both -directions). All required finality relays are also started in a special _on-demand_ mode. In this mode they -do not submit any headers without special request. As always, the only exception is when GRANDPA finality -relay sees the mandatory header - it is submitted without such request. - -The message relays are watching their lanes and when, at some block `B`, they see new messages/confirmations -to be delivered, they are asking on-demand relays to relay this block `B`. On-demand relays does that and -then message relay may perform its job. If on-demand relay is a parachain finality relay, it also runs its -own on-demand GRANDPA relay, which is used to relay required relay chain headers. - -More: [Complex Relay Sequence Diagram](./complex-relay.html), [code](../relays/bin-substrate/src/cli/relay_headers_and_messages/). diff --git a/cumulus/bridges/modules/messages/README.md b/cumulus/bridges/modules/messages/README.md deleted file mode 100644 index b5250d0dca0b23be3943fecb3926be4ea7097be0..0000000000000000000000000000000000000000 --- a/cumulus/bridges/modules/messages/README.md +++ /dev/null @@ -1,242 +0,0 @@ -# Bridge Messages Pallet - -The messages pallet is used to deliver messages from source chain to target chain. Message is -(almost) opaque to the module and the final goal is to hand message to the message dispatch -mechanism. - -## Contents - -- [Overview](#overview) -- [Message Workflow](#message-workflow) -- [Integrating Message Lane Module into Runtime](#integrating-messages-module-into-runtime) -- [Non-Essential Functionality](#non-essential-functionality) -- [Weights of Module Extrinsics](#weights-of-module-extrinsics) - -## Overview - -Message lane is an unidirectional channel, where messages are sent from source chain to the target -chain. At the same time, a single instance of messages module supports both outbound lanes and -inbound lanes. So the chain where the module is deployed (this chain), may act as a source chain for -outbound messages (heading to a bridged chain) and as a target chain for inbound messages (coming -from a bridged chain). - -Messages module supports multiple message lanes. Every message lane is identified with a 4-byte -identifier. Messages sent through the lane are assigned unique (for this lane) increasing integer -value that is known as nonce ("number that can only be used once"). Messages that are sent over the -same lane are guaranteed to be delivered to the target chain in the same order they're sent from -the source chain. In other words, message with nonce `N` will be delivered right before delivering a -message with nonce `N+1`. - -Single message lane may be seen as a transport channel for single application (onchain, offchain or -mixed). At the same time the module itself never dictates any lane or message rules. In the end, it -is the runtime developer who defines what message lane and message mean for this runtime. - -In our [Kusama<>Polkadot bridge](../../docs/polkadot-kusama-bridge-overview.md) we are using lane -as a channel of communication between two parachains of different relay chains. For example, lane -`[0, 0, 0, 0]` is used for Polkadot <> Kusama Asset Hub communications. Other lanes may be used to -bridge other parachains. - -## Message Workflow - -The pallet is not intended to be used by end users and provides no public calls to send the message. -Instead, it provides runtime-internal method that allows other pallets (or other runtime code) to queue -outbound messages. - -The message "appears" when some runtime code calls the `send_message()` method of the pallet. -The submitter specifies the lane that they're willing to use and the message itself. If some fee must -be paid for sending the message, it must be paid outside of the pallet. If a message passes all checks -(that include, for example, message size check, disabled lane check, ...), the nonce is assigned and -the message is stored in the module storage. The message is in an "undelivered" state now. - -We assume that there are external, offchain actors, called relayers, that are submitting module -related transactions to both target and source chains. The pallet itself has no assumptions about -relayers incentivization scheme, but it has some callbacks for paying rewards. See -[Integrating Messages Module into runtime](#Integrating-Messages-Module-into-runtime) -for details. - -Eventually, some relayer would notice this message in the "undelivered" state and it would decide to -deliver this message. Relayer then crafts `receive_messages_proof()` transaction (aka delivery -transaction) for the messages module instance, deployed at the target chain. Relayer provides -its account id at the source chain, the proof of message (or several messages), the number of -messages in the transaction and their cumulative dispatch weight. Once a transaction is mined, the -message is considered "delivered". - -Once a message is delivered, the relayer may want to confirm delivery back to the source chain. -There are two reasons why it would want to do that. The first is that we intentionally limit number -of "delivered", but not yet "confirmed" messages at inbound lanes -(see [What about other Constants in the Messages Module Configuration Trait](#What-about-other-Constants-in-the-Messages-Module-Configuration-Trait) for explanation). -So at some point, the target chain may stop accepting new messages until relayers confirm some of -these. The second is that if the relayer wants to be rewarded for delivery, it must prove the fact -that it has actually delivered the message. And this proof may only be generated after the delivery -transaction is mined. So relayer crafts the `receive_messages_delivery_proof()` transaction (aka -confirmation transaction) for the messages module instance, deployed at the source chain. Once -this transaction is mined, the message is considered "confirmed". - -The "confirmed" state is the final state of the message. But there's one last thing related to the -message - the fact that it is now "confirmed" and reward has been paid to the relayer (or at least -callback for this has been called), must be confirmed to the target chain. Otherwise, we may reach -the limit of "unconfirmed" messages at the target chain and it will stop accepting new messages. So -relayer sometimes includes a nonce of the latest "confirmed" message in the next -`receive_messages_proof()` transaction, proving that some messages have been confirmed. - -## Integrating Messages Module into Runtime - -As it has been said above, the messages module supports both outbound and inbound message lanes. -So if we will integrate a module in some runtime, it may act as the source chain runtime for -outbound messages and as the target chain runtime for inbound messages. In this section, we'll -sometimes refer to the chain we're currently integrating with, as "this chain" and the other -chain as "bridged chain". - -Messages module doesn't simply accept transactions that are claiming that the bridged chain has -some updated data for us. Instead of this, the module assumes that the bridged chain is able to -prove that updated data in some way. The proof is abstracted from the module and may be of any kind. -In our Substrate-to-Substrate bridge we're using runtime storage proofs. Other bridges may use -transaction proofs, Substrate header digests or anything else that may be proved. - -**IMPORTANT NOTE**: everything below in this chapter describes details of the messages module -configuration. But if you're interested in well-probed and relatively easy integration of two -Substrate-based chains, you may want to look at the -[bridge-runtime-common](../../bin/runtime-common/) crate. This crate is providing a lot of -helpers for integration, which may be directly used from within your runtime. Then if you'll decide -to change something in this scheme, get back here for detailed information. - -### General Information - -The messages module supports instances. Every module instance is supposed to bridge this chain -and some bridged chain. To bridge with another chain, using another instance is suggested (this -isn't forced anywhere in the code, though). Keep in mind, that the pallet may be used to build -virtual channels between multiple chains, as we do in our [Polkadot <> Kusama bridge](../../docs/polkadot-kusama-bridge-overview.md). -There, the pallet actually bridges only two parachains - Kusama Bridge Hub and Polkadot -Bridge Hub. However, other Kusama and Polkadot parachains are able to send (XCM) messages to their -Bridge Hubs. The messages will be delivered to the other side of the bridge and routed to the proper -destination parachain within the bridged chain consensus. - -Message submitters may track message progress by inspecting module events. When Message is accepted, -the `MessageAccepted` event is emitted. The event contains both message lane identifier and nonce that -has been assigned to the message. When a message is delivered to the target chain, the `MessagesDelivered` -event is emitted from the `receive_messages_delivery_proof()` transaction. The `MessagesDelivered` contains -the message lane identifier and inclusive range of delivered message nonces. - -The pallet provides no means to get the result of message dispatch at the target chain. If that is -required, it must be done outside of the pallet. For example, XCM messages, when dispatched, have -special instructions to send some data back to the sender. Other dispatchers may use similar -mechanism for that. -### How to plug-in Messages Module to Send Messages to the Bridged Chain? - -The `pallet_bridge_messages::Config` trait has 3 main associated types that are used to work with -outbound messages. The `pallet_bridge_messages::Config::TargetHeaderChain` defines how we see the -bridged chain as the target for our outbound messages. It must be able to check that the bridged -chain may accept our message - like that the message has size below maximal possible transaction -size of the chain and so on. And when the relayer sends us a confirmation transaction, this -implementation must be able to parse and verify the proof of messages delivery. Normally, you would -reuse the same (configurable) type on all chains that are sending messages to the same bridged -chain. - -The `pallet_bridge_messages::Config::LaneMessageVerifier` defines a single callback to verify outbound -messages. The simplest callback may just accept all messages. But in this case you'll need to answer -many questions first. Who will pay for the delivery and confirmation transaction? Are we sure that -someone will ever deliver this message to the bridged chain? Are we sure that we don't bloat our -runtime storage by accepting this message? What if the message is improperly encoded or has some -fields set to invalid values? Answering all those (and similar) questions would lead to correct -implementation. - -There's another thing to consider when implementing type for use in -`pallet_bridge_messages::Config::LaneMessageVerifier`. It is whether we treat all message lanes -identically, or they'll have different sets of verification rules? For example, you may reserve -lane#1 for messages coming from some 'wrapped-token' pallet - then you may verify in your -implementation that the origin is associated with this pallet. Lane#2 may be reserved for 'system' -messages and you may charge zero fee for such messages. You may have some rate limiting for messages -sent over the lane#3. Or you may just verify the same rules set for all outbound messages - it is -all up to the `pallet_bridge_messages::Config::LaneMessageVerifier` implementation. - -The last type is the `pallet_bridge_messages::Config::DeliveryConfirmationPayments`. When confirmation -transaction is received, we call the `pay_reward()` method, passing the range of delivered messages. -You may use the [`pallet-bridge-relayers`](../relayers/) pallet and its -[`DeliveryConfirmationPaymentsAdapter`](../relayers/src/payment_adapter.rs) adapter as a possible -implementation. It allows you to pay fixed reward for relaying the message and some of its portion -for confirming delivery. - -### I have a Messages Module in my Runtime, but I Want to Reject all Outbound Messages. What shall I do? - -You should be looking at the `bp_messages::source_chain::ForbidOutboundMessages` structure -[`bp_messages::source_chain`](../../primitives/messages/src/source_chain.rs). It implements -all required traits and will simply reject all transactions, related to outbound messages. - -### How to plug-in Messages Module to Receive Messages from the Bridged Chain? - -The `pallet_bridge_messages::Config` trait has 2 main associated types that are used to work with -inbound messages. The `pallet_bridge_messages::Config::SourceHeaderChain` defines how we see the -bridged chain as the source of our inbound messages. When relayer sends us a delivery transaction, -this implementation must be able to parse and verify the proof of messages wrapped in this -transaction. Normally, you would reuse the same (configurable) type on all chains that are sending -messages to the same bridged chain. - -The `pallet_bridge_messages::Config::MessageDispatch` defines a way on how to dispatch delivered -messages. Apart from actually dispatching the message, the implementation must return the correct -dispatch weight of the message before dispatch is called. - -### I have a Messages Module in my Runtime, but I Want to Reject all Inbound Messages. What shall I do? - -You should be looking at the `bp_messages::target_chain::ForbidInboundMessages` structure from -the [`bp_messages::target_chain`](../../primitives/messages/src/target_chain.rs) module. It -implements all required traits and will simply reject all transactions, related to inbound messages. - -### What about other Constants in the Messages Module Configuration Trait? - -Two settings that are used to check messages in the `send_message()` function. The -`pallet_bridge_messages::Config::ActiveOutboundLanes` is an array of all message lanes, that -may be used to send messages. All messages sent using other lanes are rejected. All messages that have -size above `pallet_bridge_messages::Config::MaximalOutboundPayloadSize` will also be rejected. - -To be able to reward the relayer for delivering messages, we store a map of message nonces range => -identifier of the relayer that has delivered this range at the target chain runtime storage. If a -relayer delivers multiple consequent ranges, they're merged into single entry. So there may be more -than one entry for the same relayer. Eventually, this whole map must be delivered back to the source -chain to confirm delivery and pay rewards. So to make sure we are able to craft this confirmation -transaction, we need to: (1) keep the size of this map below a certain limit and (2) make sure that -the weight of processing this map is below a certain limit. Both size and processing weight mostly -depend on the number of entries. The number of entries is limited with the -`pallet_bridge_messages::ConfigMaxUnrewardedRelayerEntriesAtInboundLane` parameter. Processing weight -also depends on the total number of messages that are being confirmed, because every confirmed -message needs to be read. So there's another -`pallet_bridge_messages::Config::MaxUnconfirmedMessagesAtInboundLane` parameter for that. - -When choosing values for these parameters, you must also keep in mind that if proof in your scheme -is based on finality of headers (and it is the most obvious option for Substrate-based chains with -finality notion), then choosing too small values for these parameters may cause significant delays -in message delivery. That's because there are too many actors involved in this scheme: 1) authorities -that are finalizing headers of the target chain need to finalize header with non-empty map; 2) the -headers relayer then needs to submit this header and its finality proof to the source chain; 3) the -messages relayer must then send confirmation transaction (storage proof of this map) to the source -chain; 4) when the confirmation transaction will be mined at some header, source chain authorities -must finalize this header; 5) the headers relay then needs to submit this header and its finality -proof to the target chain; 6) only now the messages relayer may submit new messages from the source -to target chain and prune the entry from the map. - -Delivery transaction requires the relayer to provide both number of entries and total number of -messages in the map. This means that the module never charges an extra cost for delivering a map - -the relayer would need to pay exactly for the number of entries+messages it has delivered. So the -best guess for values of these parameters would be the pair that would occupy `N` percent of the -maximal transaction size and weight of the source chain. The `N` should be large enough to process -large maps, at the same time keeping reserve for future source chain upgrades. - -## Non-Essential Functionality - -There may be a special account in every runtime where the messages module is deployed. This -account, named 'module owner', is like a module-level sudo account - he's able to halt and -resume all module operations without requiring runtime upgrade. Calls that are related to this -account are: -- `fn set_owner()`: current module owner may call it to transfer "ownership" to another account; -- `fn halt_operations()`: the module owner (or sudo account) may call this function to stop all - module operations. After this call, all message-related transactions will be rejected until - further `resume_operations` call'. This call may be used when something extraordinary happens with - the bridge; -- `fn resume_operations()`: module owner may call this function to resume bridge operations. The - module will resume its regular operations after this call. - -If pallet owner is not defined, the governance may be used to make those calls. - -## Messages Relay - -We have an offchain actor, who is watching for new messages and submits them to the bridged chain. -It is the messages relay - you may look at the [crate level documentation and the code](../../relays/messages/). diff --git a/cumulus/bridges/rustfmt.toml b/cumulus/bridges/rustfmt.toml deleted file mode 100644 index 082150daf04ee39ada660c315fd0f5bbcf99dea0..0000000000000000000000000000000000000000 --- a/cumulus/bridges/rustfmt.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Basic -hard_tabs = true -max_width = 100 -use_small_heuristics = "Max" -# Imports -imports_granularity = "Crate" -reorder_imports = true -# Consistency -newline_style = "Unix" -# Format comments -comment_width = 100 -wrap_comments = true -# Misc -chain_width = 80 -spaces_around_ranges = false -binop_separator = "Back" -reorder_impl_items = false -match_arm_leading_pipes = "Preserve" -match_arm_blocks = false -match_block_trailing_comma = true -trailing_comma = "Vertical" -trailing_semicolon = false -use_field_init_shorthand = true - diff --git a/cumulus/client/cli/Cargo.toml b/cumulus/client/cli/Cargo.toml index 6c44d5f5fd3b6631a7cfa0449d9d04adaa300a1f..c45b669fc6d1e58cad16dde5e45111fe5a9a74d4 100644 --- a/cumulus/client/cli/Cargo.toml +++ b/cumulus/client/cli/Cargo.toml @@ -5,7 +5,7 @@ authors.workspace = true edition.workspace = true [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } url = "2.4.0" diff --git a/cumulus/client/consensus/aura/src/collators/lookahead.rs b/cumulus/client/consensus/aura/src/collators/lookahead.rs index 58d8e9d1a062f7538b498f34c1f510593d2f5412..57cd646fbcdef58fca265454eeeb677c35beee7b 100644 --- a/cumulus/client/consensus/aura/src/collators/lookahead.rs +++ b/cumulus/client/consensus/aura/src/collators/lookahead.rs @@ -34,7 +34,8 @@ use codec::{Codec, Encode}; use cumulus_client_collator::service::ServiceInterface as CollatorServiceInterface; use cumulus_client_consensus_common::{ - self as consensus_common, ParachainBlockImportMarker, ParentSearchParams, + self as consensus_common, load_abridged_host_configuration, ParachainBlockImportMarker, + ParentSearchParams, }; use cumulus_client_consensus_proposer::ProposerInterface; use cumulus_primitives_aura::AuraUnincludedSegmentApi; @@ -44,11 +45,13 @@ use cumulus_primitives_core::{ use cumulus_relay_chain_interface::RelayChainInterface; use polkadot_node_primitives::SubmitCollationParams; -use polkadot_node_subsystem::messages::CollationGenerationMessage; +use polkadot_node_subsystem::messages::{ + CollationGenerationMessage, RuntimeApiMessage, RuntimeApiRequest, +}; use polkadot_overseer::Handle as OverseerHandle; use polkadot_primitives::{CollatorPair, Id as ParaId, OccupiedCoreAssumption}; -use futures::prelude::*; +use futures::{channel::oneshot, prelude::*}; use sc_client_api::{backend::AuxStore, BlockBackend, BlockOf}; use sc_consensus::BlockImport; use sc_consensus_aura::standalone as aura_internal; @@ -180,6 +183,17 @@ where while let Some(relay_parent_header) = import_notifications.next().await { let relay_parent = relay_parent_header.hash(); + if !is_para_scheduled(relay_parent, params.para_id, &mut params.overseer_handle).await { + tracing::trace!( + target: crate::LOG_TARGET, + ?relay_parent, + ?params.para_id, + "Para is not scheduled on any core, skipping import notification", + ); + + continue + } + let max_pov_size = match params .relay_client .persisted_validation_data( @@ -416,16 +430,68 @@ where Some(SlotClaim::unchecked::

(author_pub, slot, timestamp)) } +/// Reads allowed ancestry length parameter from the relay chain storage at the given relay parent. +/// +/// Falls back to 0 in case of an error. async fn max_ancestry_lookback( - _relay_parent: PHash, - _relay_client: &impl RelayChainInterface, + relay_parent: PHash, + relay_client: &impl RelayChainInterface, ) -> usize { - // TODO [https://github.com/paritytech/cumulus/issues/2706] - // We need to read the relay-chain state to know what the maximum - // age truly is, but that depends on those pallets existing. - // - // For now, just provide the conservative value of '2'. - // Overestimating can cause problems, as we'd be building on forks of the - // chain that can never get included. Underestimating is less of an issue. - 2 + match load_abridged_host_configuration(relay_parent, relay_client).await { + Ok(Some(config)) => config.async_backing_params.allowed_ancestry_len as usize, + Ok(None) => { + tracing::error!( + target: crate::LOG_TARGET, + "Active config is missing in relay chain storage", + ); + 0 + }, + Err(err) => { + tracing::error!( + target: crate::LOG_TARGET, + ?err, + ?relay_parent, + "Failed to read active config from relay chain client", + ); + 0 + }, + } +} + +// Checks if there exists a scheduled core for the para at the provided relay parent. +// +// Falls back to `false` in case of an error. +async fn is_para_scheduled( + relay_parent: PHash, + para_id: ParaId, + overseer_handle: &mut OverseerHandle, +) -> bool { + let (tx, rx) = oneshot::channel(); + let request = RuntimeApiRequest::AvailabilityCores(tx); + overseer_handle + .send_msg(RuntimeApiMessage::Request(relay_parent, request), "LookaheadCollator") + .await; + + let cores = match rx.await { + Ok(Ok(cores)) => cores, + Ok(Err(error)) => { + tracing::error!( + target: crate::LOG_TARGET, + ?error, + ?relay_parent, + "Failed to query availability cores runtime API", + ); + return false + }, + Err(oneshot::Canceled) => { + tracing::error!( + target: crate::LOG_TARGET, + ?relay_parent, + "Sender for availability cores runtime request dropped", + ); + return false + }, + }; + + cores.iter().any(|core| core.para_id() == Some(para_id)) } diff --git a/cumulus/client/consensus/common/src/lib.rs b/cumulus/client/consensus/common/src/lib.rs index edd97cf02a2557b372f3a88234267a0a4e639929..08bceabb2bd4a49ac8917e82caf1b97cd9eac183 100644 --- a/cumulus/client/consensus/common/src/lib.rs +++ b/cumulus/client/consensus/common/src/lib.rs @@ -20,8 +20,8 @@ use polkadot_primitives::{ }; use cumulus_primitives_core::{ - relay_chain::{BlockId as RBlockId, OccupiedCoreAssumption}, - ParaId, + relay_chain::{self, BlockId as RBlockId, OccupiedCoreAssumption}, + AbridgedHostConfiguration, ParaId, }; use cumulus_relay_chain_interface::{RelayChainError, RelayChainInterface}; @@ -211,6 +211,7 @@ pub trait ParachainBlockImportMarker {} impl ParachainBlockImportMarker for ParachainBlockImport {} /// Parameters when searching for suitable parents to build on top of. +#[derive(Debug)] pub struct ParentSearchParams { /// The relay-parent that is intended to be used. pub relay_parent: PHash, @@ -228,6 +229,7 @@ pub struct ParentSearchParams { } /// A potential parent block returned from [`find_potential_parents`] +#[derive(Debug, PartialEq)] pub struct PotentialParent { /// The hash of the block. pub hash: B::Hash, @@ -412,3 +414,18 @@ pub fn relay_slot_and_timestamp( }) .ok() } + +/// Reads abridged host configuration from the relay chain storage at the given relay parent. +pub async fn load_abridged_host_configuration( + relay_parent: PHash, + relay_client: &impl RelayChainInterface, +) -> Result, RelayChainError> { + relay_client + .get_storage_by_key(relay_parent, relay_chain::well_known_keys::ACTIVE_CONFIG) + .await? + .map(|bytes| { + AbridgedHostConfiguration::decode(&mut &bytes[..]) + .map_err(RelayChainError::DeserializationError) + }) + .transpose() +} diff --git a/cumulus/client/consensus/common/src/tests.rs b/cumulus/client/consensus/common/src/tests.rs index 15586d81d9bfff8d177cf392c89a4c90606ad3bf..22d3dd3abd49dfbe4a0bb548ce4b2c35cf5ed839 100644 --- a/cumulus/client/consensus/common/src/tests.rs +++ b/cumulus/client/consensus/common/src/tests.rs @@ -19,7 +19,10 @@ use crate::*; use async_trait::async_trait; use codec::Encode; use cumulus_client_pov_recovery::RecoveryKind; -use cumulus_primitives_core::{relay_chain::BlockId, InboundDownwardMessage, InboundHrmpMessage}; +use cumulus_primitives_core::{ + relay_chain::{self, BlockId}, + CumulusDigestItem, InboundDownwardMessage, InboundHrmpMessage, +}; use cumulus_relay_chain_interface::{ CommittedCandidateReceipt, OccupiedCoreAssumption, OverseerHandle, PHeader, ParaId, RelayChainInterface, RelayChainResult, SessionIndex, StorageValue, ValidatorId, @@ -42,12 +45,21 @@ use std::{ time::Duration, }; +fn relay_block_num_from_hash(hash: &PHash) -> relay_chain::BlockNumber { + hash.to_low_u64_be() as u32 +} + +fn relay_hash_from_block_num(block_number: relay_chain::BlockNumber) -> PHash { + PHash::from_low_u64_be(block_number as u64) +} + struct RelaychainInner { new_best_heads: Option>, finalized_heads: Option>, new_best_heads_sender: mpsc::UnboundedSender

, finalized_heads_sender: mpsc::UnboundedSender
, relay_chain_hash_to_header: HashMap, + relay_chain_hash_to_header_pending: HashMap, } impl RelaychainInner { @@ -61,6 +73,7 @@ impl RelaychainInner { new_best_heads: Some(new_best_heads), finalized_heads: Some(finalized_heads), relay_chain_hash_to_header: Default::default(), + relay_chain_hash_to_header_pending: Default::default(), } } } @@ -110,20 +123,17 @@ impl RelayChainInterface for Relaychain { &self, hash: PHash, _: ParaId, - _: OccupiedCoreAssumption, + assumption: OccupiedCoreAssumption, ) -> RelayChainResult> { - Ok(Some(PersistedValidationData { - parent_head: self - .inner - .lock() - .unwrap() - .relay_chain_hash_to_header - .get(&hash) - .unwrap() - .encode() - .into(), - ..Default::default() - })) + let inner = self.inner.lock().unwrap(); + let relay_to_header = match assumption { + OccupiedCoreAssumption::Included => &inner.relay_chain_hash_to_header_pending, + _ => &inner.relay_chain_hash_to_header, + }; + let Some(parent_head) = relay_to_header.get(&hash).map(|head| head.encode().into()) else { + return Ok(None) + }; + Ok(Some(PersistedValidationData { parent_head, ..Default::default() })) } async fn candidate_pending_availability( @@ -135,7 +145,7 @@ impl RelayChainInterface for Relaychain { } async fn session_index_for_child(&self, _: PHash) -> RelayChainResult { - unimplemented!("Not needed for test") + Ok(0) } async fn import_notification_stream( @@ -210,8 +220,23 @@ impl RelayChainInterface for Relaychain { .boxed()) } - async fn header(&self, _block_id: BlockId) -> RelayChainResult> { - unimplemented!("Not needed for test") + async fn header(&self, block_id: BlockId) -> RelayChainResult> { + let number = match block_id { + BlockId::Hash(hash) => relay_block_num_from_hash(&hash), + BlockId::Number(block_number) => block_number, + }; + let parent_hash = number + .checked_sub(1) + .map(relay_hash_from_block_num) + .unwrap_or_else(|| PHash::zero()); + + Ok(Some(PHeader { + parent_hash, + number, + digest: sp_runtime::Digest::default(), + state_root: PHash::zero(), + extrinsics_root: PHash::zero(), + })) } } @@ -238,6 +263,7 @@ fn build_block( sproof: RelayStateSproofBuilder, at: Option, timestamp: Option, + relay_parent: Option, ) -> Block { let builder = match at { Some(at) => match timestamp { @@ -249,10 +275,17 @@ fn build_block( let mut block = builder.build().unwrap().block; - // Simulate some form of post activity (like a Seal or Other generic things). - // This is mostly used to exercise the `LevelMonitor` correct behavior. - // (in practice we want that header post-hash != pre-hash) - block.header.digest.push(sp_runtime::DigestItem::Other(vec![1, 2, 3])); + if let Some(relay_parent) = relay_parent { + block + .header + .digest + .push(CumulusDigestItem::RelayParent(relay_parent).to_digest_item()); + } else { + // Simulate some form of post activity (like a Seal or Other generic things). + // This is mostly used to exercise the `LevelMonitor` correct behavior. + // (in practice we want that header post-hash != pre-hash) + block.header.digest.push(sp_runtime::DigestItem::Other(vec![1, 2, 3])); + } block } @@ -292,13 +325,14 @@ fn build_and_import_block_ext>( importer: &mut I, at: Option, timestamp: Option, + relay_parent: Option, ) -> Block { let sproof = match at { None => sproof_with_best_parent(client), Some(at) => sproof_with_parent_by_hash(client, at), }; - let block = build_block(client, sproof, at, timestamp); + let block = build_block(client, sproof, at, timestamp, relay_parent); import_block_sync(importer, block.clone(), origin, import_as_best); block } @@ -311,6 +345,7 @@ fn build_and_import_block(mut client: Arc, import_as_best: bool) -> Bloc &mut client, None, None, + None, ) } @@ -372,7 +407,7 @@ fn follow_new_best_with_dummy_recovery_works() { let header = client.header(best).ok().flatten().expect("No header for best"); sproof_with_parent(HeadData(header.encode())) }; - let block = build_block(&*client, sproof, None, None); + let block = build_block(&*client, sproof, None, None, None); let block_clone = block.clone(); let client_clone = client.clone(); @@ -638,6 +673,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, None, None, + None, ); let id0 = block0.header.hash(); @@ -650,6 +686,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -664,6 +701,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id10), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -692,6 +730,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -711,6 +750,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(2 * LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -749,6 +789,7 @@ fn restore_limit_monitor() { &mut para_import, None, None, + None, ); let id00 = block00.header.hash(); @@ -761,6 +802,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id00), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -775,6 +817,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id10), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -809,6 +852,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id00), Some(LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -830,3 +874,487 @@ fn restore_limit_monitor() { })); assert_eq!(*monitor.freshness.get(&block13.header.hash()).unwrap(), monitor.import_counter); } + +#[test] +fn find_potential_parents_in_allowed_ancestry() { + sp_tracing::try_init_simple(); + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(relay_parent, block.header().clone()); + } + + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth: 0, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 1); + let parent = &potential_parents[0]; + + assert_eq!(parent.hash, block.hash()); + assert_eq!(&parent.header, block.header()); + assert_eq!(parent.depth, 0); + assert!(parent.aligned_with_pending); + + // New block is not pending or included. + let block_relay_parent = relay_hash_from_block_num(11); + let search_relay_parent = relay_hash_from_block_num(13); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(search_relay_parent, block.header().clone()); + } + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(block.header().hash()), + None, + Some(block_relay_parent), + ); + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 2, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + + assert_eq!(potential_parents.len(), 2); + let parent = &potential_parents[1]; + + assert_eq!(parent.hash, block.hash()); + assert_eq!(&parent.header, block.header()); + assert_eq!(parent.depth, 1); + assert!(parent.aligned_with_pending); + + // Reduce allowed ancestry. + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 1); + let parent = &potential_parents[0]; + assert_ne!(parent.hash, block.hash()); +} + +/// Tests that pending availability block is always potential parent. +#[test] +fn find_potential_pending_parent() { + sp_tracing::try_init_simple(); + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + let relay_parent = relay_hash_from_block_num(12); + let pending_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(included_block.header().hash()), + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + let search_relay_parent = relay_hash_from_block_num(15); + { + let relay_inner = &mut relay_chain.inner.lock().unwrap(); + relay_inner + .relay_chain_hash_to_header + .insert(search_relay_parent, included_block.header().clone()); + relay_inner + .relay_chain_hash_to_header_pending + .insert(search_relay_parent, pending_block.header().clone()); + } + + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 2); + let included_parent = &potential_parents[0]; + + assert_eq!(included_parent.hash, included_block.hash()); + assert_eq!(&included_parent.header, included_block.header()); + assert_eq!(included_parent.depth, 0); + assert!(included_parent.aligned_with_pending); + + let pending_parent = &potential_parents[1]; + + assert_eq!(pending_parent.hash, pending_block.hash()); + assert_eq!(&pending_parent.header, pending_block.header()); + assert_eq!(pending_parent.depth, 1); + assert!(pending_parent.aligned_with_pending); +} + +#[test] +fn find_potential_parents_with_max_depth() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(relay_parent, included_block.header().clone()); + } + + let mut blocks = Vec::new(); + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + blocks.push(block); + } + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), max_depth + 1); + let expected_parents: Vec<_> = + std::iter::once(&included_block).chain(blocks.iter().take(max_depth)).collect(); + + for i in 0..(max_depth + 1) { + let parent = &potential_parents[i]; + let expected = &expected_parents[i]; + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + assert_eq!(parent.depth, i); + assert!(parent.aligned_with_pending); + } + } +} + +#[test] +fn find_potential_parents_aligned_with_pending() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + // Choose different relay parent for alternative chain to get new hashes. + let search_relay_parent = relay_hash_from_block_num(11); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + let pending_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(included_block.header().hash()), + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let relay_inner = &mut relay_chain.inner.lock().unwrap(); + relay_inner + .relay_chain_hash_to_header + .insert(search_relay_parent, included_block.header().clone()); + relay_inner + .relay_chain_hash_to_header_pending + .insert(search_relay_parent, pending_block.header().clone()); + } + + // Build two sibling chains from the included block. + let mut aligned_blocks = Vec::new(); + let mut parent = pending_block.header().hash(); + for _ in 1..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + aligned_blocks.push(block); + } + + let mut alt_blocks = Vec::new(); + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + Some(parent), + None, + Some(search_relay_parent), + ); + parent = block.header().hash(); + alt_blocks.push(block); + } + + // Ignore alternative branch: + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), max_depth + 1); + let expected_parents: Vec<_> = [&included_block, &pending_block] + .into_iter() + .chain(aligned_blocks.iter()) + .take(max_depth + 1) + .collect(); + + for i in 0..(max_depth + 1) { + let parent = &potential_parents[i]; + let expected = &expected_parents[i]; + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + assert_eq!(parent.depth, i); + assert!(parent.aligned_with_pending); + } + } + + // Do not ignore: + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: false, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + + let expected_len = 2 * max_depth + 1; + assert_eq!(potential_parents.len(), expected_len); + let expected_aligned: Vec<_> = [&included_block, &pending_block] + .into_iter() + .chain(aligned_blocks.iter()) + .take(max_depth + 1) + .collect(); + let expected_alt = alt_blocks.iter().take(max_depth); + + let expected_parents: Vec<_> = + expected_aligned.clone().into_iter().chain(expected_alt).collect(); + // Check correctness. + assert_eq!(expected_parents.len(), expected_len); + + for i in 0..expected_len { + let parent = &potential_parents[i]; + let expected = expected_parents + .iter() + .find(|block| block.header().hash() == parent.hash) + .expect("missing parent"); + + let is_aligned = expected_aligned.contains(&expected); + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + + assert_eq!(parent.aligned_with_pending, is_aligned); + } + } +} + +/// Tests that no potential parent gets discarded if there's no pending availability block. +#[test] +fn find_potential_parents_aligned_no_pending() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + // Choose different relay parent for alternative chain to get new hashes. + let search_relay_parent = relay_hash_from_block_num(11); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(search_relay_parent, included_block.header().clone()); + } + + // Build two sibling chains from the included block. + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + } + + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + Some(parent), + None, + Some(search_relay_parent), + ); + parent = block.header().hash(); + } + + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents_aligned = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, + max_depth, + ignore_alternative_branches: false, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 2 * max_depth + 1); + assert_eq!(potential_parents, potential_parents_aligned); + } +} diff --git a/cumulus/client/consensus/proposer/Cargo.toml b/cumulus/client/consensus/proposer/Cargo.toml index f7edbc695e3864dc21c8b34a4eab7087998b1e6e..29720a8f4791124536095ffba44189762de47f90 100644 --- a/cumulus/client/consensus/proposer/Cargo.toml +++ b/cumulus/client/consensus/proposer/Cargo.toml @@ -8,7 +8,7 @@ edition.workspace = true [dependencies] anyhow = "1.0" async-trait = "0.1.73" -thiserror = "1.0.47" +thiserror = "1.0.48" # Substrate sp-consensus = { path = "../../../../substrate/primitives/consensus/common" } diff --git a/cumulus/client/relay-chain-inprocess-interface/Cargo.toml b/cumulus/client/relay-chain-inprocess-interface/Cargo.toml index c198b22cad4a8bd0bed0bafd1ba5993cff54b733..39eda5075e29e9477ced48a37db3d45e00c67705 100644 --- a/cumulus/client/relay-chain-inprocess-interface/Cargo.toml +++ b/cumulus/client/relay-chain-inprocess-interface/Cargo.toml @@ -37,7 +37,7 @@ sp-keyring = { path = "../../../substrate/primitives/keyring" } # Polkadot polkadot-primitives = { path = "../../../polkadot/primitives" } polkadot-test-client = { path = "../../../polkadot/node/test/client" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } # Cumulus cumulus-test-service = { path = "../../test/service" } diff --git a/cumulus/client/relay-chain-interface/Cargo.toml b/cumulus/client/relay-chain-interface/Cargo.toml index b81cc1b4780761fa86ca819653b90159f19749e9..3da7ab0b0e8212d7a05d972ca3fb7c82a0955bd3 100644 --- a/cumulus/client/relay-chain-interface/Cargo.toml +++ b/cumulus/client/relay-chain-interface/Cargo.toml @@ -16,6 +16,6 @@ sc-client-api = { path = "../../../substrate/client/api" } futures = "0.3.28" async-trait = "0.1.73" -thiserror = "1.0.47" +thiserror = "1.0.48" jsonrpsee-core = "0.16.2" parity-scale-codec = "3.6.4" diff --git a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs index 491758c1329a3427f9e206b8f118053cb4f304bf..bea2fc330a24e3c9875f2359cde5f4388de0e6ae 100644 --- a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs +++ b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs @@ -36,13 +36,14 @@ use polkadot_node_network_protocol::{ use polkadot_node_subsystem_util::metrics::{prometheus::Registry, Metrics}; use polkadot_overseer::{ BlockInfo, DummySubsystem, Handle, Overseer, OverseerConnector, OverseerHandle, SpawnGlue, - KNOWN_LEAVES_CACHE_SIZE, + UnpinHandle, KNOWN_LEAVES_CACHE_SIZE, }; use polkadot_primitives::CollatorPair; use sc_authority_discovery::Service as AuthorityDiscoveryService; use sc_network::NetworkStateInfo; use sc_service::TaskManager; +use sc_utils::mpsc::tracing_unbounded; use sp_runtime::traits::Block as BlockT; use cumulus_primitives_core::relay_chain::{Block, Hash as PHash}; @@ -221,20 +222,25 @@ async fn forward_collator_events( ) -> Result<(), RelayChainError> { let mut finality = client.finality_notification_stream().await?.fuse(); let mut imports = client.import_notification_stream().await?.fuse(); + // Collators do no need to pin any specific blocks + let (dummy_sink, _) = tracing_unbounded("does-not-matter", 42); + let dummy_unpin_handle = UnpinHandle::new(Default::default(), dummy_sink); loop { select! { f = finality.next() => { match f { Some(header) => { + let hash = header.hash(); tracing::info!( target: "minimal-polkadot-node", "Received finalized block via RPC: #{} ({} -> {})", header.number, header.parent_hash, - header.hash() + hash, ); - let block_info = BlockInfo { hash: header.hash(), parent_hash: header.parent_hash, number: header.number }; + let unpin_handle = dummy_unpin_handle.clone(); + let block_info = BlockInfo { hash, parent_hash: header.parent_hash, number: header.number, unpin_handle }; handle.block_finalized(block_info).await; } None => return Err(RelayChainError::GenericError("Relay chain finality stream ended.".to_string())), @@ -243,14 +249,16 @@ async fn forward_collator_events( i = imports.next() => { match i { Some(header) => { + let hash = header.hash(); tracing::info!( target: "minimal-polkadot-node", "Received imported block via RPC: #{} ({} -> {})", header.number, header.parent_hash, - header.hash() + hash, ); - let block_info = BlockInfo { hash: header.hash(), parent_hash: header.parent_hash, number: header.number }; + let unpin_handle = dummy_unpin_handle.clone(); + let block_info = BlockInfo { hash, parent_hash: header.parent_hash, number: header.number, unpin_handle }; handle.block_imported(block_info).await; } None => return Err(RelayChainError::GenericError("Relay chain import stream ended.".to_string())), diff --git a/cumulus/client/relay-chain-rpc-interface/Cargo.toml b/cumulus/client/relay-chain-rpc-interface/Cargo.toml index 9797c512505ce38ac4dcb60466403a80d0f78a29..0f09377e106ce4274ddf0d59028476af3127a51b 100644 --- a/cumulus/client/relay-chain-rpc-interface/Cargo.toml +++ b/cumulus/client/relay-chain-rpc-interface/Cargo.toml @@ -32,12 +32,12 @@ jsonrpsee = { version = "0.16.2", features = ["ws-client"] } tracing = "0.1.37" async-trait = "0.1.73" url = "2.4.0" -serde_json = "1.0.105" +serde_json = "1.0.107" serde = "1.0.188" schnellru = "0.2.1" smoldot = { version = "0.11.0", default_features = false, features = ["std"]} smoldot-light = { version = "0.9.0", default_features = false, features = ["std"] } either = "1.8.1" -thiserror = "1.0.38" +thiserror = "1.0.48" rand = "0.8.5" pin-project = "1.1.3" diff --git a/cumulus/client/service/src/lib.rs b/cumulus/client/service/src/lib.rs index 211a5cc3b79bc7b8d59aed9300b0a442f5372de8..82890666ebae3f197e291019d215ea9d446ef71f 100644 --- a/cumulus/client/service/src/lib.rs +++ b/cumulus/client/service/src/lib.rs @@ -484,6 +484,7 @@ where import_queue, block_announce_validator_builder: Some(Box::new(move |_| block_announce_validator)), warp_sync_params, + block_relay: None, }) } diff --git a/cumulus/docker/docker-compose.yml b/cumulus/docker/docker-compose.yml deleted file mode 100644 index 8344ad43bb4cbac17b5b3f1d7cc5de9264f36050..0000000000000000000000000000000000000000 --- a/cumulus/docker/docker-compose.yml +++ /dev/null @@ -1,129 +0,0 @@ -version: '3.7' -services: - node_alice: - image: "polkadot:${BRANCH:-cumulus-branch}" - ports: - - "30333:30333" - - "9933:9933" - - "9944:9944" - volumes: - - "polkadot-data-alice:/data" - - type: bind - source: ./test/parachain/chain-specs/polkadot_chainspec.json - target: /chainspec.json - read_only: true - command: > - polkadot - --chain=/chainspec.json - --base-path=/data - --port 30333 - --rpc-port 9933 - --ws-port 9944 - --rpc-external - --rpc-cors all - --ws-external - --alice - networks: - testing_net: - ipv4_address: 172.28.1.1 - aliases: - - alice - - node_bob: - image: "polkadot:${BRANCH:-cumulus-branch}" - ports: - - "30344:30333" - - "9935:9933" - - "9945:9944" - volumes: - - "polkadot-data-bob:/data" - - type: bind - source: ./test/parachain/chain-specs/polkadot_chainspec.json - target: /chainspec.json - read_only: true - command: > - polkadot - --chain=/chainspec.json - --base-path=/data - --port 30333 - --rpc-port 9933 - --ws-port 9944 - --rpc-external - --ws-external - --rpc-cors all - --bob - networks: - testing_net: - ipv4_address: 172.28.1.2 - aliases: - - bob - - genesis_state: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - image: "ctpc:latest" - volumes: - - "genesis-state:/data" - command: > - polkadot-parachain - export-genesis-state - /data/genesis-state - - collator: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - target: collator - image: "ctpc:collator" - volumes: - - "collator-data:/data" - depends_on: - - node_alice - - node_bob - command: > - inject_bootnodes.sh - --base-path=/data - networks: - testing_net: - - runtime: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - target: runtime - image: "ctpc:runtime" - volumes: - - "parachain-runtime:/runtime" - - - registrar: - build: - context: . - dockerfile: ./docker/parachain-registrar.dockerfile - image: para-reg:latest - volumes: - - "genesis-state:/genesis" - - "parachain-runtime:/runtime" - depends_on: - - node_alice - - runtime - - genesis_state - networks: - testing_net: - - -volumes: - polkadot-data-alice: - polkadot-data-bob: - collator-data: - genesis-state: - parachain-runtime: - - -networks: - testing_net: - ipam: - driver: default - config: - - subnet: 172.28.0.0/16 diff --git a/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile b/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile deleted file mode 100644 index a2e32049f5bb4fccb3d7bf1ce3b04ffea69fd9e4..0000000000000000000000000000000000000000 --- a/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ - # apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - # add user and link ~/.local/share/polkadot-parachain to /data - useradd -m -u 10000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \ - mkdir -p /data /polkadot-parachain/.local/share && \ - chown -R polkadot-parachain:polkadot-parachain /data && \ - ln -s /data /polkadot-parachain/.local/share/polkadot-parachain && \ - mkdir -p /specs - -# add polkadot-parachain binary to the docker image -COPY ./artifacts/polkadot-parachain /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER polkadot-parachain - -# check if executable works in this container -RUN /usr/local/bin/polkadot-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot-parachain"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/cumulus/docker/polkadot-parachain_builder.Containerfile b/cumulus/docker/polkadot-parachain_builder.Containerfile deleted file mode 100644 index 159bcb323693865d924b93d674af770f8a6bdf41..0000000000000000000000000000000000000000 --- a/cumulus/docker/polkadot-parachain_builder.Containerfile +++ /dev/null @@ -1,36 +0,0 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile -# This is the build stage for polkadot-parachain. Here we create the binary in a temporary image. -FROM docker.io/paritytech/ci-linux:production as builder - -WORKDIR /cumulus -COPY . /cumulus - -RUN cargo build --release --locked -p polkadot-parachain - -# This is the 2nd stage: a very small image where we copy the Polkadot binary." -FROM docker.io/library/ubuntu:20.04 - -LABEL io.parity.image.type="builder" \ - io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.description="Multistage Docker image for polkadot-parachain" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot-parachain_builder.Dockerfile" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus" - -COPY --from=builder /cumulus/target/release/polkadot-parachain /usr/local/bin - -RUN useradd -m -u 1000 -U -s /bin/sh -d /cumulus polkadot-parachain && \ - mkdir -p /data /cumulus/.local/share && \ - chown -R polkadot-parachain:polkadot-parachain /data && \ - ln -s /data /cumulus/.local/share/polkadot-parachain && \ -# unclutter and minimize the attack surface - rm -rf /usr/bin /usr/sbin && \ -# check if executable works in this container - /usr/local/bin/polkadot-parachain --version - -USER polkadot-parachain - -EXPOSE 30333 9933 9944 9615 -VOLUME ["/data"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/cumulus/docker/test-parachain_injected.Dockerfile b/cumulus/docker/test-parachain_injected.Dockerfile deleted file mode 100644 index 6056c504604e3be218b123791065624c32f4e87a..0000000000000000000000000000000000000000 --- a/cumulus/docker/test-parachain_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Test parachain for Zombienet" \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/test-parachain_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ - # apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - # add user and link ~/.local/share/test-parachain to /data - useradd -m -u 10000 -U -s /bin/sh -d /test-parachain test-parachain && \ - mkdir -p /data /test-parachain/.local/share && \ - chown -R test-parachain:test-parachain /data && \ - ln -s /data /test-parachain/.local/share/test-parachain && \ - mkdir -p /specs - -# add test-parachain binary to the docker image -COPY ./artifacts/test-parachain /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER test-parachain - -# check if executable works in this container -RUN /usr/local/bin/test-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/test-parachain"] - -ENTRYPOINT ["/usr/local/bin/test-parachain"] diff --git a/cumulus/docs/release.md b/cumulus/docs/release.md index b04c4e844c4eda09fa82a4febeda9a47d0f527c9..38d1915013b07d67ae074a0f7606c6fd58526e3b 100644 --- a/cumulus/docs/release.md +++ b/cumulus/docs/release.md @@ -37,8 +37,8 @@ performed during the release process. ### Burn In -Ensure that Parity DevOps has run the new release on Westend and Kusama Asset Hub collators for 12h -prior to publishing the release. +Ensure that Parity DevOps has run the new release on Westend and Kusama Asset Hub collators for 12h prior to publishing +the release. ### Build Artifacts @@ -75,56 +75,61 @@ function of the appropriate pallets. ### Extrinsic Ordering & Storage -Offline signing libraries depend on a consistent ordering of call indices and -functions. Compare the metadata of the current and new runtimes and ensure that -the `module index, call index` tuples map to the same set of functions. It also checks if there have been any changes in `storage`. In case of a breaking change, increase `transaction_version`. +Offline signing libraries depend on a consistent ordering of call indices and functions. Compare the metadata of the +current and new runtimes and ensure that the `module index, call index` tuples map to the same set of functions. It also +checks if there have been any changes in `storage`. In case of a breaking change, increase `transaction_version`. -To verify the order has not changed, manually start the following [Github Action](https://github.com/paritytech/cumulus/actions/workflows/extrinsic-ordering-check-from-bin.yml). It takes around a minute to run and will produce the report as artifact you need to manually check. +To verify the order has not changed, manually start the following [Github +Action](https://github.com/paritytech/cumulus/actions/workflows/extrinsic-ordering-check-from-bin.yml). It takes around +a minute to run and will produce the report as artifact you need to manually check. To run it, in the _Run Workflow_ dropdown: 1. **Use workflow from**: to ignore, leave `master` as default -2. **The WebSocket url of the reference node**: - - Asset Hub Polkadot: `wss://statemint-rpc.polkadot.io` +2. **The WebSocket url of the reference node**: - Asset Hub Polkadot: `wss://statemint-rpc.polkadot.io` - Asset Hub Kusama: `wss://statemine-rpc.polkadot.io` - Asset Hub Westend: `wss://westmint-rpc.polkadot.io` -3. **A url to a Linux binary for the node containing the runtime to test**: Paste the URL of the latest release-candidate binary from the draft-release on Github. The binary has to previously be uploaded to S3 (Github url link to the binary is constantly changing) +3. **A url to a Linux binary for the node containing the runtime to test**: Paste the URL of the latest + release-candidate binary from the draft-release on Github. The binary has to previously be uploaded to S3 (Github url + link to the binary is constantly changing) - E.g: https://releases.parity.io/cumulus/v0.9.270-rc3/polkadot-parachain -4. **The name of the chain under test. Usually, you would pass a local chain**: - - Asset Hub Polkadot: `asset-hub-polkadot-local` +4. **The name of the chain under test. Usually, you would pass a local chain**: - Asset Hub Polkadot: + `asset-hub-polkadot-local` - Asset Hub Kusama: `asset-hub-kusama-local` - Asset Hub Westend: `asset-hub-westend-local` 5. Click **Run workflow** -When the workflow is done, click on it and download the zip artifact, inside you'll find an `output.txt` file. The things to look for in the output are lines like: +When the workflow is done, click on it and download the zip artifact, inside you'll find an `output.txt` file. The +things to look for in the output are lines like: - `[Identity] idx 28 -> 25 (calls 15)` - indicates the index for Identity has changed - `[+] Society, Recovery` - indicates the new version includes 2 additional modules/pallets. - If no indices have changed, every modules line should look something like `[Identity] idx 25 (calls 15)` -**Note**: Adding new functions to the runtime does not constitute a breaking change -as long as the indexes did not change. +**Note**: Adding new functions to the runtime does not constitute a breaking change as long as the indexes did not +change. -**Note**: Extrinsic function signatures changes (adding/removing & ordering arguments) are not caught by the job, so those changes should be reviewed "manually" +**Note**: Extrinsic function signatures changes (adding/removing & ordering arguments) are not caught by the job, so +those changes should be reviewed "manually" ### Benchmarks -The Benchmarks can now be started from the CI. First find the CI pipeline from [here](https://gitlab.parity.io/parity/mirrors/cumulus/-/pipelines?page=1&scope=all&ref=release-parachains-v9220) and pick the latest. -[Guide](https://github.com/paritytech/ci_cd/wiki/Benchmarks:-cumulus) +The Benchmarks can now be started from the CI. First find the CI pipeline from +[here](https://gitlab.parity.io/parity/mirrors/cumulus/-/pipelines?page=1&scope=all&ref=release-parachains-v9220) and +pick the latest. [Guide](https://github.com/paritytech/ci_cd/wiki/Benchmarks:-cumulus) ### Integration Tests Until https://github.com/paritytech/ci_cd/issues/499 is done, tests will have to be run manually. -1. Go to https://github.com/paritytech/parachains-integration-tests and check out the release branch. -E.g. https://github.com/paritytech/parachains-integration-tests/tree/release-v9270-v0.9.27 -for `release-parachains-v0.9.270` +1. Go to https://github.com/paritytech/parachains-integration-tests and check out the release branch. E.g. +https://github.com/paritytech/parachains-integration-tests/tree/release-v9270-v0.9.27 for `release-parachains-v0.9.270` 2. Clone `release-parachains-` branch from Cumulus 3. `cargo build --release` 4. Copy `./target/polkadot-parachain` to `./bin` -5. Clone `it/release--fast-sudo` from Polkadot -In case the branch does not exists (it is a manual process): cherry pick paritytech/polkadot@791c8b8 and run -`find . -type f -name "*.toml" -print0 | xargs -0 sed -i '' -e 's/polkadot-vX.X.X/polkadot-v/g'` +5. Clone `it/release--fast-sudo` from Polkadot In case the branch does not exists (it is a manual process): + cherry pick `paritytech/polkadot@791c8b8` and run: + `find . -type f -name "*.toml" -print0 | xargs -0 sed -i '' -e 's/polkadot-vX.X.X/polkadot-v/g'` 6. `cargo build --release --features fast-runtime` 7. Copy `./target/polkadot` into `./bin` (in Cumulus) 8. Run the tests: - - Asset Hub Polkadot: `yarn zombienet-test -c ./examples/statemint/config.toml -t ./examples/statemint` - - Asset Hub Kusama: `yarn zombienet-test -c ./examples/statemine/config.toml -t ./examples/statemine` + - Asset Hub Polkadot: `yarn zombienet-test -c ./examples/statemint/config.toml -t ./examples/statemint` + - Asset Hub Kusama: `yarn zombienet-test -c ./examples/statemine/config.toml -t ./examples/statemine` diff --git a/cumulus/pallets/collator-selection/README.md b/cumulus/pallets/collator-selection/README.md index 9718db58b37e9ae1c81b5b627fe27e51129cf418..811207fd8c0f916f31d08d27fc53499572b6bef4 100644 --- a/cumulus/pallets/collator-selection/README.md +++ b/cumulus/pallets/collator-selection/README.md @@ -1 +1 @@ -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/cumulus/pallets/parachain-system/Cargo.toml b/cumulus/pallets/parachain-system/Cargo.toml index 3063083c0ee49d5f6e88db287de529a8eef0c80d..5470dce47480e3d5f3287ec078f886eb0302aaa9 100644 --- a/cumulus/pallets/parachain-system/Cargo.toml +++ b/cumulus/pallets/parachain-system/Cargo.toml @@ -11,7 +11,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = environmental = { version = "1.1.4", default-features = false } impl-trait-for-tuples = "0.2.1" log = { version = "0.4.20", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +trie-db = { version = "0.28.0", default-features = false } scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate diff --git a/cumulus/pallets/parachain-system/proc-macro/Cargo.toml b/cumulus/pallets/parachain-system/proc-macro/Cargo.toml index e5e7da81d2aea4cc34833e09b517228bb7c5e0b1..a1510847fc2f58681b4fa9d4364648d7db24c299 100644 --- a/cumulus/pallets/parachain-system/proc-macro/Cargo.toml +++ b/cumulus/pallets/parachain-system/proc-macro/Cargo.toml @@ -9,7 +9,7 @@ description = "Proc macros provided by the parachain-system pallet" proc-macro = true [dependencies] -syn = "2.0.31" +syn = "2.0.37" proc-macro2 = "1.0.64" quote = "1.0.33" proc-macro-crate = "1.3.1" diff --git a/cumulus/pallets/parachain-system/src/lib.rs b/cumulus/pallets/parachain-system/src/lib.rs index a8e9a0bf9ae4460b7cda5ef1f7b49bb5c0629eef..a7e59a61c9be2a74025eb3014a11f1632f73ff15 100644 --- a/cumulus/pallets/parachain-system/src/lib.rs +++ b/cumulus/pallets/parachain-system/src/lib.rs @@ -1400,12 +1400,12 @@ impl Pallet { CustomValidationHeadData::::put(head_data); } - /// Open HRMP channel for using it in benchmarks. + /// Open HRMP channel for using it in benchmarks or tests. /// /// The caller assumes that the pallet will accept regular outbound message to the sibling /// `target_parachain` after this call. No other assumptions are made. - #[cfg(feature = "runtime-benchmarks")] - pub fn open_outbound_hrmp_channel_for_benchmarks(target_parachain: ParaId) { + #[cfg(any(feature = "runtime-benchmarks", feature = "std"))] + pub fn open_outbound_hrmp_channel_for_benchmarks_or_tests(target_parachain: ParaId) { RelevantMessagingState::::put(MessagingStateSnapshot { dmq_mqc_head: Default::default(), relay_dispatch_queue_remaining_capacity: Default::default(), diff --git a/cumulus/pallets/xcmp-queue/Cargo.toml b/cumulus/pallets/xcmp-queue/Cargo.toml index de4ad61de9e090c3095009cde286d37137db34ce..5aab57da91b4f731567bd81d384cab646beb936d 100644 --- a/cumulus/pallets/xcmp-queue/Cargo.toml +++ b/cumulus/pallets/xcmp-queue/Cargo.toml @@ -38,7 +38,7 @@ pallet-balances = { path = "../../../substrate/frame/balances" } xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder" } # Cumulus -cumulus-pallet-parachain-system = { path = "../parachain-system" } +cumulus-pallet-parachain-system = { path = "../parachain-system", features = ["parameterized-consensus-hook"] } [features] default = [ "std" ] diff --git a/cumulus/pallets/xcmp-queue/src/tests.rs b/cumulus/pallets/xcmp-queue/src/tests.rs index 2929e8452bef6095aa1e2ba4f168d85482f0b3b2..ba005a5badfeb69472b09cddd1f93f6dc0ac42dd 100644 --- a/cumulus/pallets/xcmp-queue/src/tests.rs +++ b/cumulus/pallets/xcmp-queue/src/tests.rs @@ -14,9 +14,9 @@ // limitations under the License. use super::*; -use cumulus_primitives_core::XcmpMessageHandler; +use cumulus_primitives_core::{ParaId, XcmpMessageHandler}; use frame_support::{assert_noop, assert_ok}; -use mock::{new_test_ext, RuntimeCall, RuntimeOrigin, Test, XcmpQueue}; +use mock::{new_test_ext, ParachainSystem, RuntimeCall, RuntimeOrigin, Test, XcmpQueue}; use sp_runtime::traits::BadOrigin; #[test] @@ -341,3 +341,32 @@ fn xcmp_queue_consumes_dest_and_msg_on_ok_validate() { ); }); } + +#[test] +fn xcmp_queue_send_xcm_works() { + new_test_ext().execute_with(|| { + let sibling_para_id = ParaId::from(12345); + let dest = (Parent, X1(Parachain(sibling_para_id.into()))).into(); + let msg = Xcm(vec![ClearOrigin]); + + // try to send without opened HRMP channel to the sibling_para_id + assert_eq!( + send_xcm::(dest, msg.clone()), + Err(SendError::Transport("NoChannel")), + ); + + // open HRMP channel to the sibling_para_id + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(sibling_para_id); + + // check empty outbound queue + assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); + + // now send works + assert_ok!(send_xcm::(dest, msg)); + + // check outbound queue contains message/page for sibling_para_id + assert!(XcmpQueue::take_outbound_messages(usize::MAX) + .iter() + .any(|(para_id, _)| para_id == &sibling_para_id)); + }) +} diff --git a/cumulus/parachain-template/README.md b/cumulus/parachain-template/README.md index 6dcc70c538290e16587025b652131954c8264ea3..2d71bbd71f3615a071710d9402c83f43621893d1 100644 --- a/cumulus/parachain-template/README.md +++ b/cumulus/parachain-template/README.md @@ -19,4 +19,4 @@ parathreads [here](https://wiki.polkadot.network/docs/learn-parathreads). 🧙 Learn about how to use this template and run your own parachain testnet for it in the -[Devhub Cumulus Tutorial](https://docs.substrate.io/tutorials/v3/cumulus/start-relay/). \ No newline at end of file +[Devhub Cumulus Tutorial](https://docs.substrate.io/tutorials/v3/cumulus/start-relay/). diff --git a/cumulus/parachain-template/node/Cargo.toml b/cumulus/parachain-template/node/Cargo.toml index b1f1946e3ed457f04a9b2623969df02d6016d842..223a78dacc49f5f479c7e351a7ee53768535b4ee 100644 --- a/cumulus/parachain-template/node/Cargo.toml +++ b/cumulus/parachain-template/node/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" publish = false [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } log = "0.4.20" codec = { package = "parity-scale-codec", version = "3.0.0" } serde = { version = "1.0.188", features = ["derive"] } diff --git a/cumulus/parachain-template/runtime/Cargo.toml b/cumulus/parachain-template/runtime/Cargo.toml index f26b7be23533a284aaf2d1bdbb2232daaf7509e5..4e51f16dca168fbcc94309a055e27908e1420ef7 100644 --- a/cumulus/parachain-template/runtime/Cargo.toml +++ b/cumulus/parachain-template/runtime/Cargo.toml @@ -128,6 +128,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", diff --git a/cumulus/parachains/common/Cargo.toml b/cumulus/parachains/common/Cargo.toml index 0c863b7295b4c216761b2d4127911849bd2b6e3c..963de03fa16aeec446d9afca50bb26bcaf37a1d7 100644 --- a/cumulus/parachains/common/Cargo.toml +++ b/cumulus/parachains/common/Cargo.toml @@ -13,6 +13,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", features = ["derive log = { version = "0.4.19", default-features = false } scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } num-traits = { version = "0.2", default-features = false} +smallvec = "1.11.0" # Substrate frame-support = { path = "../../../substrate/frame/support", default-features = false } @@ -28,6 +29,11 @@ sp-runtime = { path = "../../../substrate/primitives/runtime", default-features sp-std = { path = "../../../substrate/primitives/std", default-features = false } # Polkadot +kusama-runtime-constants = { path = "../../../polkadot/runtime/kusama/constants", default-features = false} +polkadot-runtime-constants = { path = "../../../polkadot/runtime/polkadot/constants", default-features = false} +rococo-runtime-constants = { path = "../../../polkadot/runtime/rococo/constants", default-features = false} +westend-runtime-constants = { path = "../../../polkadot/runtime/westend/constants", default-features = false} +polkadot-core-primitives = { path = "../../../polkadot/core-primitives", default-features = false} polkadot-primitives = { path = "../../../polkadot/primitives", default-features = false} xcm = { package = "staging-xcm", path = "../../../polkadot/xcm", default-features = false} xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder", default-features = false} @@ -52,19 +58,38 @@ std = [ "cumulus-primitives-utility/std", "frame-support/std", "frame-system/std", + "kusama-runtime-constants/std", "log/std", "pallet-asset-tx-payment/std", "pallet-assets/std", "pallet-authorship/std", "pallet-balances/std", "pallet-collator-selection/std", + "polkadot-core-primitives/std", "polkadot-primitives/std", + "polkadot-runtime-constants/std", + "rococo-runtime-constants/std", "sp-consensus-aura/std", "sp-core/std", "sp-io/std", "sp-runtime/std", "sp-std/std", + "westend-runtime-constants/std", "xcm-builder/std", "xcm-executor/std", "xcm/std", ] + +runtime-benchmarks = [ + "cumulus-primitives-utility/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "pallet-asset-tx-payment/runtime-benchmarks", + "pallet-assets/runtime-benchmarks", + "pallet-balances/runtime-benchmarks", + "pallet-collator-selection/runtime-benchmarks", + "polkadot-primitives/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", + "xcm-builder/runtime-benchmarks", + "xcm-executor/runtime-benchmarks", +] diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/constants.rs b/cumulus/parachains/common/src/kusama.rs similarity index 95% rename from cumulus/parachains/runtimes/contracts/contracts-rococo/src/constants.rs rename to cumulus/parachains/common/src/kusama.rs index 9b0fe5182a25243f4e9cd9d6d3ff21676f88025b..308f7d081cedcc90aba2d6c12ed25e13e716bbb4 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/constants.rs +++ b/cumulus/parachains/common/src/kusama.rs @@ -13,6 +13,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +/// Consensus-related. +pub mod consensus { + /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included + /// into the relay chain. + pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; + /// How many parachain blocks are processed by the relay chain per parent. Limits the + /// number of blocks authored per slot. + pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; + /// Relay chain slot duration, in milliseconds. + pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; +} + +/// Constants relating to KSM. pub mod currency { use kusama_runtime_constants as constants; use polkadot_core_primitives::Balance; @@ -31,7 +44,7 @@ pub mod currency { } } -/// Fee-related. +/// Constants related to Kusama fee payment. pub mod fee { use frame_support::{ pallet_prelude::Weight, @@ -75,8 +88,8 @@ pub mod fee { impl WeightToFeePolynomial for RefTimeToFee { type Balance = Balance; fn polynomial() -> WeightToFeeCoefficients { - // in Kusama, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Rococo Contracts, we map to 1/10 of that, or 1/100 CENT + // In Kusama, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: + // The standard system parachain configuration is 1/10 of that, as in 1/100 CENT. let p = super::currency::CENTS; let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); @@ -107,15 +120,3 @@ pub mod fee { } } } - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/common/src/lib.rs b/cumulus/parachains/common/src/lib.rs index 797010d49a0715c2851f3f802b4317e8653a4916..cb2ac1a1e3e45d7af1cba07c8dddbba1837cf199 100644 --- a/cumulus/parachains/common/src/lib.rs +++ b/cumulus/parachains/common/src/lib.rs @@ -16,6 +16,10 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod impls; +pub mod kusama; +pub mod polkadot; +pub mod rococo; +pub mod westend; pub mod xcm_config; pub use constants::*; pub use opaque::*; diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/constants.rs b/cumulus/parachains/common/src/polkadot.rs similarity index 90% rename from cumulus/parachains/runtimes/collectives/collectives-polkadot/src/constants.rs rename to cumulus/parachains/common/src/polkadot.rs index 46b562ea4de795650512366fb411a9129fc18b35..4f459b9bb5a3448affc564a0a5282e03bc07c7a3 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/constants.rs +++ b/cumulus/parachains/common/src/polkadot.rs @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +/// Universally recognized accounts. pub mod account { use frame_support::PalletId; @@ -26,8 +27,25 @@ pub mod account { /// It is used as a temporarily place to deposit a slashed imbalance /// before the teleport to the Treasury. pub const REFERENDA_PALLET_ID: PalletId = PalletId(*b"py/refer"); + /// Ambassador Referenda pallet ID. + /// It is used as a temporarily place to deposit a slashed imbalance + /// before the teleport to the Treasury. + pub const AMBASSADOR_REFERENDA_PALLET_ID: PalletId = PalletId(*b"py/amref"); +} + +/// Consensus-related. +pub mod consensus { + /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included + /// into the relay chain. + pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; + /// How many parachain blocks are processed by the relay chain per parent. Limits the + /// number of blocks authored per slot. + pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; + /// Relay chain slot duration, in milliseconds. + pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; } +/// Constants relating to DOT. pub mod currency { use polkadot_core_primitives::Balance; use polkadot_runtime_constants as constants; @@ -41,12 +59,12 @@ pub mod currency { pub const MILLICENTS: Balance = constants::currency::MILLICENTS; pub const fn deposit(items: u32, bytes: u32) -> Balance { - // 1/100 of Polkadot. + // 1/100 of Polkadot constants::currency::deposit(items, bytes) / 100 } } -/// Fee-related. +/// Constants related to Polkadot fee payment. pub mod fee { use frame_support::{ pallet_prelude::Weight, @@ -90,8 +108,8 @@ pub mod fee { impl WeightToFeePolynomial for RefTimeToFee { type Balance = Balance; fn polynomial() -> WeightToFeeCoefficients { - // in Polkadot, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in a parachain, we map to 1/10 of that, or 1/100 CENT + // In Polkadot, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: + // The standard system parachain configuration is 1/10 of that, as in 1/100 CENT. let p = super::currency::CENTS; let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); @@ -122,15 +140,3 @@ pub mod fee { } } } - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/constants.rs b/cumulus/parachains/common/src/rococo.rs similarity index 96% rename from cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/constants.rs rename to cumulus/parachains/common/src/rococo.rs index 80620feaedc6986e07b05e43aa4a33ba1ce81baa..6e31def4b55b923f1596793e6cb114163551c017 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/constants.rs +++ b/cumulus/parachains/common/src/rococo.rs @@ -73,8 +73,8 @@ pub mod fee { impl WeightToFeePolynomial for RefTimeToFee { type Balance = Balance; fn polynomial() -> WeightToFeeCoefficients { - // in Rococo, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Bridge Hub, we map to 1/10 of that, or 1/100 CENT + // In Rococo, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: + // The standard system parachain configuration is 1/10 of that, as in 1/100 CENT. let p = super::currency::CENTS; let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/constants.rs b/cumulus/parachains/common/src/westend.rs similarity index 96% rename from cumulus/parachains/runtimes/assets/asset-hub-westend/src/constants.rs rename to cumulus/parachains/common/src/westend.rs index b2629ef10fcc4c139b9cbf69aacadb2a84820ab7..9d3e0bd1a0e2a51d2dde47b5be329be0bc8b5d38 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/constants.rs +++ b/cumulus/parachains/common/src/westend.rs @@ -75,8 +75,8 @@ pub mod fee { impl WeightToFeePolynomial for RefTimeToFee { type Balance = Balance; fn polynomial() -> WeightToFeeCoefficients { - // in Westend, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Asset Hub, we map to 1/10 of that, or 1/100 CENT + // In Westend, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: + // The standard system parachain configuration is 1/10 of that, as in 1/100 CENT. let p = super::currency::CENTS; let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); diff --git a/cumulus/parachains/integration-tests/e2e/collectives/README.md b/cumulus/parachains/integration-tests/e2e/collectives/README.md index 98ea77aac60642ac7001297bc5d3024124cd7822..9c4efe7c9504835814e1a693dd8d57710da1bc3b 100644 --- a/cumulus/parachains/integration-tests/e2e/collectives/README.md +++ b/cumulus/parachains/integration-tests/e2e/collectives/README.md @@ -1,19 +1,23 @@ -E2E tests concerning Polkadot Governance and the Collectives Parachain. The tests run by the Parachain Integration Tests [tool](https://github.com/paritytech/parachains-integration-tests/). +E2E tests concerning Polkadot Governance and the Collectives Parachain. The tests run by the Parachain Integration Tests +[tool](https://github.com/paritytech/parachains-integration-tests/). -## Requirements +# Requirements The tests require some changes to the regular production runtime builds: -RelayChain runtime: +## RelayChain runtime 1. Alice has SUDO -2. Public Referenda `StakingAdmin`, `FellowshipAdmin` tracks settings (see the corresponding keys of the `TRACKS_DATA` constant in the `governance::tracks` module of the Relay Chain runtime crate): +2. Public Referenda `StakingAdmin`, `FellowshipAdmin` tracks settings (see the corresponding keys of the `TRACKS_DATA` + constant in the `governance::tracks` module of the Relay Chain runtime crate): ``` yaml prepare_period: 5 Block, decision_period: 1 Block, confirm_period: 1 Block, min_enactment_period: 1 Block, ``` -Collectives runtime: -1. Fellowship Referenda `Fellows` track settings (see the corresponding key of the `TRACKS_DATA` constant in the `fellowship::tracks` module of the Collectives runtime crate): + +## Collectives runtime +1. Fellowship Referenda `Fellows` track settings (see the corresponding key of the `TRACKS_DATA` constant in the + `fellowship::tracks` module of the Collectives runtime crate): ``` yaml prepare_period: 5 Block, decision_period: 1 Block, diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml index d45a201a4d051e341f9a0c88ecfe073c90549598..788b2482be877e424052a563a86885b34fb1ead7 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml @@ -3,6 +3,7 @@ name = "asset-hub-kusama-integration-tests" version = "1.0.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Asset Hub Kusama runtime integration tests with xcm-emulator" publish = false @@ -44,6 +45,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/lib.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/lib.rs index 9325ca54b068f6b2d9e7cdaa9b3a833cb41ff4ea..ad74aa2301fc34d478b9f719f9ccfddb44b79fc8 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use codec::Encode; pub use frame_support::{ diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/hrmp_channels.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/hrmp_channels.rs index 8647c1aa0081044500a1f8efaf9b444b90d752bb..7bb64333db52fbee59ba5d83bbf9cfab841c1220 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/hrmp_channels.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/hrmp_channels.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -160,12 +159,6 @@ fn force_open_hrmp_channel_for_system_para_works() { // Parachain A init values let para_a_id = PenpalKusamaA::para_id(); - let fund_amount = KUSAMA_ED * 1000_000_000; - - // Fund Parachain's Sovereign accounts to be able to reserve the deposit - let para_a_sovereign_account = Kusama::fund_para_sovereign(fund_amount, para_a_id); - let system_para_sovereign_account = Kusama::fund_para_sovereign(fund_amount, system_para_id); - Kusama::execute_with(|| { assert_ok!(::Hrmp::force_open_hrmp_channel( relay_root_origin, @@ -180,14 +173,6 @@ fn force_open_hrmp_channel_for_system_para_works() { assert_expected_events!( Kusama, vec![ - // Sender deposit is reserved for System Parachain's Sovereign account - RuntimeEvent::Balances(pallet_balances::Event::Reserved { who, .. }) =>{ - who: *who == system_para_sovereign_account, - }, - // Recipient deposit is reserved for Parachain's Sovereign account - RuntimeEvent::Balances(pallet_balances::Event::Reserved { who, .. }) =>{ - who: *who == para_a_sovereign_account, - }, // HRMP channel forced opened RuntimeEvent::Hrmp( polkadot_runtime_parachains::hrmp::Event::HrmpChannelForceOpened( diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/mod.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/mod.rs index 41d8406581930a48630818f4ba9c63497863f3bf..b3089a3b38269bfbb6d6377099ea4acae4a034b1 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/mod.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/mod.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. mod hrmp_channels; mod reserve_transfer; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/reserve_transfer.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/reserve_transfer.rs index 26b3cdf68b1b817f30b20846db8bb7453b0751ee..645dca5035b1d8d385d9e01c31c2570dae14c2a4 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/reserve_transfer.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/reserve_transfer.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/send.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/send.rs index 598256db83a0ee6cacc6e593c8b3e5ef0f3dfa9b..5891b694c8e436104b508cce7b67b8266460cc3e 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/send.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/send.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -174,8 +173,6 @@ fn send_xcm_from_para_to_system_para_paying_fee_with_assets_works() { PenpalKusamaA::assert_xcm_pallet_sent(); }); - PenpalKusamaA::execute_with(|| {}); - AssetHubKusama::execute_with(|| { type RuntimeEvent = ::RuntimeEvent; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/set_xcm_versions.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/set_xcm_versions.rs index 155ada2ec2f3384b00c43a954eca4cbd4570611d..a7af96096cdde4a9fc6bddb86874c02ca65a1c97 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/set_xcm_versions.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/set_xcm_versions.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/swap.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/swap.rs index aad93db9922e4db838b8c2a56849028bc3a7f697..3a67b5435828184eb49c9ffbda814803d4115048 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/swap.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/swap.rs @@ -1,22 +1,21 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; -use asset_hub_kusama_runtime::constants::currency::EXISTENTIAL_DEPOSIT; use frame_support::{instances::Instance2, BoundedVec}; +use parachains_common::kusama::currency::EXISTENTIAL_DEPOSIT; use sp_runtime::{DispatchError, ModuleError}; #[test] @@ -183,11 +182,9 @@ fn swap_locally_on_chain_using_foreign_assets() { .encode() .into(); - let buy_execution_fee_amount = - asset_hub_kusama_runtime::constants::fee::WeightToFee::weight_to_fee(&Weight::from_parts( - 10_100_000_000_000, - 300_000, - )); + let buy_execution_fee_amount = parachains_common::kusama::fee::WeightToFee::weight_to_fee( + &Weight::from_parts(10_100_000_000_000, 300_000), + ); let buy_execution_fee = MultiAsset { id: Concrete(MultiLocation { parents: 1, interior: Here }), fun: Fungible(buy_execution_fee_amount), diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/teleport.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/teleport.rs index b671fd5b44480fa509ce1c47525d3dbc91db3e50..f69878f35435ded3da456756328350a1c286a471 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/teleport.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/src/tests/teleport.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #![allow(dead_code)] // diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml index 3d5d23e9963625fa664e0fb0f34ee0f4b31bb613..023e8b84f11b4a9e8330b12f6cec0c60967e9cae 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml @@ -3,6 +3,7 @@ name = "asset-hub-polkadot-integration-tests" version = "1.0.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Asset Hub Polkadot runtime integration tests with xcm-emulator" publish = false @@ -43,6 +44,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/lib.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/lib.rs index 6cbf6ab1ccb0eb516bb00cc810d51a19f99fee47..e8ba8e44f25c529f17cae594008d2f3e3fb2e80c 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use codec::Encode; pub use frame_support::{ diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/hrmp_channels.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/hrmp_channels.rs index 337e4ba6113a15d6ad9dee05c6d683b028f77208..a6286c619f64ced17ceddcab7d3dfc44b2e862d1 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/hrmp_channels.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/hrmp_channels.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -160,12 +159,6 @@ fn force_open_hrmp_channel_for_system_para_works() { // Parachain A init values let para_a_id = PenpalPolkadotA::para_id(); - let fund_amount = POLKADOT_ED * 1000_000_000; - - // Fund Parachain's Sovereign accounts to be able to reserve the deposit - let system_para_sovereign_account = Polkadot::fund_para_sovereign(fund_amount, system_para_id); - let para_a_sovereign_account = Polkadot::fund_para_sovereign(fund_amount, para_a_id); - Polkadot::execute_with(|| { assert_ok!(::Hrmp::force_open_hrmp_channel( relay_root_origin, @@ -180,14 +173,6 @@ fn force_open_hrmp_channel_for_system_para_works() { assert_expected_events!( Polkadot, vec![ - // Sender deposit is reserved for System Parachain's Sovereign account - RuntimeEvent::Balances(pallet_balances::Event::Reserved { who, .. }) =>{ - who: *who == system_para_sovereign_account, - }, - // Recipient deposit is reserved for Parachain's Sovereign account - RuntimeEvent::Balances(pallet_balances::Event::Reserved { who, .. }) =>{ - who: *who == para_a_sovereign_account, - }, // HRMP channel forced opened RuntimeEvent::Hrmp( polkadot_runtime_parachains::hrmp::Event::HrmpChannelForceOpened( diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/mod.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/mod.rs index 547b59deadcb80360bfee2673df0c3274a56de8e..c22de4f1c3ebd10a3fd1135718812bd09c53bbe0 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/mod.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/mod.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. mod hrmp_channels; mod reserve_transfer; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/reserve_transfer.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/reserve_transfer.rs index e6722d585bc0906360327240b96fac37575137f3..e53693d85d2bbc48ae9fc5ee473c55613cc76d63 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/reserve_transfer.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/reserve_transfer.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/send.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/send.rs index 978377b0fda3b0934a7d4160730d97152b19dfb4..244b428a7523b36680e08696ca28f69cef8a2fc4 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/send.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/send.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -177,8 +176,6 @@ fn send_xcm_from_para_to_system_para_paying_fee_with_assets_works() { PenpalPolkadotA::assert_xcm_pallet_sent(); }); - PenpalPolkadotA::execute_with(|| {}); - AssetHubPolkadot::execute_with(|| { type RuntimeEvent = ::RuntimeEvent; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/set_xcm_versions.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/set_xcm_versions.rs index 287bfa35ae9c69c07d81bfa23fb84a2084239d00..e121c4167993fbfedfd6ba8f7cbe51417e212591 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/set_xcm_versions.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/set_xcm_versions.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/teleport.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/teleport.rs index 166f73137e750fa0b62492c427f962c472d25ef8..644c51d75b6620429ebdbcab1a1bdc8b37cce410 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/teleport.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/src/tests/teleport.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #![allow(dead_code)] // @@ -187,7 +186,6 @@ fn limited_teleport_native_assets_from_relay_to_system_para_works() { /// Limited Teleport of native asset from System Parachain to Relay Chain /// should work when there is enough balance in Relay Chain's `CheckAccount` #[test] -#[cfg(feature = "FIXME-IGNORED")] // fn limited_teleport_native_assets_back_from_system_para_to_relay_works() { // Dependency - Relay Chain's `CheckAccount` should have enough balance limited_teleport_native_assets_from_relay_to_system_para_works(); @@ -226,7 +224,6 @@ fn limited_teleport_native_assets_back_from_system_para_to_relay_works() { /// Limited Teleport of native asset from System Parachain to Relay Chain /// should't work when there is not enough balance in Relay Chain's `CheckAccount` #[test] -#[cfg(feature = "FIXME-IGNORED")] // fn limited_teleport_native_assets_from_system_para_to_relay_fails() { // Init values for Relay Chain let amount_to_send: Balance = ASSET_HUB_POLKADOT_ED * 1000; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml index 34d34009ebd079f5a77aefd6c2812e5cdfa0f4db..80c41c24aa75ef4c166bd778935e2178b9529484 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml @@ -3,6 +3,7 @@ name = "asset-hub-westend-integration-tests" version = "1.0.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Asset Hub Westend runtime integration tests with xcm-emulator" publish = false @@ -44,6 +45,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/lib.rs index 2c89c0f9dd46982c2099f7f36e5a16f1724c6f82..6e0f3434aedf3dd7a5f630ad40729e6a61e9ba47 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use codec::Encode; pub use frame_support::{ diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/mod.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/mod.rs index e2a60e0b300443c49dec7e27551ac8641dc8ab29..b3841af0e6c38372b8fb621fac468b25bdec63a1 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/mod.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/mod.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. mod reserve_transfer; mod send; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/reserve_transfer.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/reserve_transfer.rs index 67fc53a826aa9185857ae6497a5c32670fb2a413..51fac43be1255948a9ae72044bececa3d907f2c5 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/reserve_transfer.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/reserve_transfer.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/send.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/send.rs index f2040351e5dbf624750deb31204ff2fc3a2f3b26..424d222bef3811966f55d8fbac954f33f065ffc7 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/send.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/send.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -116,8 +115,6 @@ fn send_xcm_from_para_to_system_para_paying_fee_with_assets_works() { PenpalWestendA::assert_xcm_pallet_sent(); }); - PenpalWestendA::execute_with(|| {}); - AssetHubWestend::execute_with(|| { type RuntimeEvent = ::RuntimeEvent; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/set_xcm_versions.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/set_xcm_versions.rs index 57632527174566cbdc7996c8197389b8fd4a5232..2720095aac00d5d15c43e77224b30b7387079897 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/set_xcm_versions.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/set_xcm_versions.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/swap.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/swap.rs index 1c4dd9d76837c7291c7e409bfe2776975b0762de..7d1615c9e29182c97146ca46c966f0ae85991b11 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/swap.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/swap.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. use crate::*; @@ -171,11 +170,9 @@ fn swap_locally_on_chain_using_foreign_assets() { .encode() .into(); - let buy_execution_fee_amount = - asset_hub_westend_runtime::constants::fee::WeightToFee::weight_to_fee(&Weight::from_parts( - 10_100_000_000_000, - 300_000, - )); + let buy_execution_fee_amount = parachains_common::westend::fee::WeightToFee::weight_to_fee( + &Weight::from_parts(10_100_000_000_000, 300_000), + ); let buy_execution_fee = MultiAsset { id: Concrete(MultiLocation { parents: 1, interior: Here }), fun: Fungible(buy_execution_fee_amount), diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/teleport.rs b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/teleport.rs index 233dc32b13b5ce70abc522c9bc03d6e9360dae1c..8de73a7420c6b69954b2fe295a3b5f6bf4e7219c 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/teleport.rs +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/src/tests/teleport.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #![allow(dead_code)] // diff --git a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml index 12a2ed51591fc54a636781b00653a79f7a471498..c02c96255e1837bcdbd0755b669e5985721e0c16 100644 --- a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml @@ -3,6 +3,7 @@ name = "bridge-hub-rococo-integration-tests" version = "1.0.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Bridge Hub Rococo runtime integration tests with xcm-emulator" publish = false @@ -24,8 +25,8 @@ pallet-xcm = { path = "../../../../../../polkadot/xcm/pallet-xcm", default-featu parachains-common = { path = "../../../../common" } cumulus-pallet-xcmp-queue = { path = "../../../../../pallets/xcmp-queue", default-features = false} cumulus-pallet-dmp-queue = { path = "../../../../../pallets/dmp-queue", default-features = false} -pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false} -bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false} +pallet-bridge-messages = { path = "../../../../../../bridges/modules/messages", default-features = false} +bp-messages = { path = "../../../../../../bridges/primitives/messages", default-features = false} # Local xcm-emulator = { path = "../../../../../xcm/xcm-emulator", default-features = false} diff --git a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/lib.rs index 0a923ec04de76006224e66a70a72fe00b2d4ef99..122d65461159d90a84888b792f35e89decf7c5be 100644 --- a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use bp_messages::LaneId; pub use frame_support::assert_ok; diff --git a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/example.rs b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/example.rs index 127292829fd5d1cd3b9f773868bf98f13e21fbd0..f24e13bb71b875d8d68be161e5dc833198f93c39 100644 --- a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/example.rs +++ b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/example.rs @@ -1,98 +1,98 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +use crate::*; #[test] -#[ignore] fn example() { - // // Init tests variables - // // XcmPallet send arguments - // let sudo_origin = ::RuntimeOrigin::root(); - // let destination = Rococo::child_location_of(BridgeHubRococo::para_id()).into(); - // let weight_limit = WeightLimit::Unlimited; - // let check_origin = None; + // Init tests variables + // XcmPallet send arguments + let sudo_origin = ::RuntimeOrigin::root(); + let destination = Rococo::child_location_of(BridgeHubRococo::para_id()).into(); + let weight_limit = WeightLimit::Unlimited; + let check_origin = None; - // let remote_xcm = Xcm(vec![ClearOrigin]); + let remote_xcm = Xcm(vec![ClearOrigin]); - // let xcm = VersionedXcm::from(Xcm(vec![ - // UnpaidExecution { weight_limit, check_origin }, - // ExportMessage { - // network: WococoId, - // destination: X1(Parachain(AssetHubWococo::para_id().into())), - // xcm: remote_xcm, - // }, - // ])); + let xcm = VersionedXcm::from(Xcm(vec![ + UnpaidExecution { weight_limit, check_origin }, + ExportMessage { + network: WococoId, + destination: X1(Parachain(AssetHubWococo::para_id().into())), + xcm: remote_xcm, + }, + ])); - // //Rococo Global Consensus - // // Send XCM message from Relay Chain to Bridge Hub source Parachain - // Rococo::execute_with(|| { - // assert_ok!(::XcmPallet::send( - // sudo_origin, - // bx!(destination), - // bx!(xcm), - // )); + //Rococo Global Consensus + // Send XCM message from Relay Chain to Bridge Hub source Parachain + Rococo::execute_with(|| { + assert_ok!(::XcmPallet::send( + sudo_origin, + bx!(destination), + bx!(xcm), + )); - // type RuntimeEvent = ::RuntimeEvent; + type RuntimeEvent = ::RuntimeEvent; - // assert_expected_events!( - // Rococo, - // vec![ - // RuntimeEvent::XcmPallet(pallet_xcm::Event::Sent { .. }) => {}, - // ] - // ); - // }); - // // Receive XCM message in Bridge Hub source Parachain - // BridgeHubRococo::execute_with(|| { - // type RuntimeEvent = ::RuntimeEvent; + assert_expected_events!( + Rococo, + vec![ + RuntimeEvent::XcmPallet(pallet_xcm::Event::Sent { .. }) => {}, + ] + ); + }); + // Receive XCM message in Bridge Hub source Parachain + BridgeHubRococo::execute_with(|| { + type RuntimeEvent = ::RuntimeEvent; - // assert_expected_events!( - // BridgeHubRococo, - // vec![ - // RuntimeEvent::DmpQueue(cumulus_pallet_dmp_queue::Event::ExecutedDownward { - // outcome: Outcome::Complete(_), - // .. - // }) => {}, - // RuntimeEvent::BridgeWococoMessages(pallet_bridge_messages::Event::MessageAccepted { - // lane_id: LaneId([0, 0, 0, 1]), - // nonce: 1, - // }) => {}, - // ] - // ); - // }); + assert_expected_events!( + BridgeHubRococo, + vec![ + RuntimeEvent::DmpQueue(cumulus_pallet_dmp_queue::Event::ExecutedDownward { + outcome: Outcome::Complete(_), + .. + }) => {}, + RuntimeEvent::BridgeWococoMessages(pallet_bridge_messages::Event::MessageAccepted { + lane_id: LaneId([0, 0, 0, 1]), + nonce: 1, + }) => {}, + ] + ); + }); - // // Wococo GLobal Consensus - // // Receive XCM message in Bridge Hub target Parachain - // BridgeHubWococo::execute_with(|| { - // type RuntimeEvent = ::RuntimeEvent; + // Wococo GLobal Consensus + // Receive XCM message in Bridge Hub target Parachain + BridgeHubWococo::execute_with(|| { + type RuntimeEvent = ::RuntimeEvent; - // assert_expected_events!( - // BridgeHubWococo, - // vec![ - // RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::XcmpMessageSent { .. }) => {}, - // ] - // ); - // }); - // // Receive embeded XCM message within `ExportMessage` in Parachain destination - // AssetHubWococo::execute_with(|| { - // type RuntimeEvent = ::RuntimeEvent; + assert_expected_events!( + BridgeHubWococo, + vec![ + RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::XcmpMessageSent { .. }) => {}, + ] + ); + }); + // Receive embeded XCM message within `ExportMessage` in Parachain destination + AssetHubWococo::execute_with(|| { + type RuntimeEvent = ::RuntimeEvent; - // assert_expected_events!( - // AssetHubWococo, - // vec![ - // RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::Fail { .. }) => {}, - // ] - // ); - // }); + assert_expected_events!( + AssetHubWococo, + vec![ + RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::Fail { .. }) => {}, + ] + ); + }); } diff --git a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/mod.rs b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/mod.rs index 6e11743aecbf36cca86eda552528d8da00ee6b4b..48347557ae77df661d8c41726cb3fe7f24738678 100644 --- a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/mod.rs +++ b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/src/tests/mod.rs @@ -1,17 +1,16 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. mod example; diff --git a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/Cargo.toml b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/Cargo.toml index ee0e254befc5153aecfe909b25f89bae6481f615..99caccc81590ec39a83e6a6ef7a280b454165921 100644 --- a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/Cargo.toml @@ -3,6 +3,7 @@ name = "collectives-polkadot-integration-tests" version = "0.1.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Polkadot Collectives parachain runtime integration tests based on xcm-emulator" publish = false @@ -14,6 +15,7 @@ sp-runtime = { path = "../../../../../../substrate/primitives/runtime", default- frame-support = { path = "../../../../../../substrate/frame/support", default-features = false} sp-core = { path = "../../../../../../substrate/primitives/core", default-features = false} pallet-assets = { path = "../../../../../../substrate/frame/assets", default-features = false} +pallet-balances = { path = "../../../../../../substrate/frame/balances", default-features = false} pallet-core-fellowship = { path = "../../../../../../substrate/frame/core-fellowship", default-features = false} pallet-salary = { path = "../../../../../../substrate/frame/salary", default-features = false} diff --git a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/lib.rs b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/lib.rs index ad2b5a501117516b1064b36cb567b2e3983e41f2..aa716c7c94855acb016056733bcd6111fa8b5599 100644 --- a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use codec::Encode; pub use frame_support::{assert_ok, sp_runtime::AccountId32}; diff --git a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/ambassador.rs b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/ambassador.rs new file mode 100644 index 0000000000000000000000000000000000000000..d9fd78fbcbffe0fd8a73c447c8c0cdf762448fae --- /dev/null +++ b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/ambassador.rs @@ -0,0 +1,65 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests concerning the Ambassador Program. + +use crate::*; +use collectives_polkadot_runtime::ambassador::AmbassadorSalaryPaymaster; +use frame_support::traits::{fungible::Mutate, tokens::Pay}; +use sp_core::crypto::Ss58Codec; +use xcm_emulator::TestExt; + +#[test] +fn pay_salary() { + let pay_from: AccountId = + ::from_string("5DS1Gaf6R9eFAV8QyeZP9P89kTkJMurxv3y3J3TTMu8p8VCX") + .unwrap(); + let pay_to = Polkadot::account_id_of(ALICE); + let pay_amount = 90000000000; + + AssetHubPolkadot::execute_with(|| { + type AssetHubBalances = ::Balances; + + assert_ok!(>::mint_into(&pay_from, pay_amount * 2)); + }); + + Collectives::execute_with(|| { + type RuntimeEvent = ::RuntimeEvent; + + assert_ok!(AmbassadorSalaryPaymaster::pay(&pay_to, (), pay_amount)); + assert_expected_events!( + Collectives, + vec![ + RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::XcmpMessageSent { .. }) => {}, + ] + ); + }); + + AssetHubPolkadot::execute_with(|| { + type RuntimeEvent = ::RuntimeEvent; + + assert_expected_events!( + AssetHubPolkadot, + vec![ + RuntimeEvent::Balances(pallet_balances::Event::Transfer { from, to, amount }) => { + from: from == &pay_from, + to: to == &pay_to, + amount: amount == &pay_amount, + }, + RuntimeEvent::XcmpQueue(cumulus_pallet_xcmp_queue::Event::Success { .. }) => {}, + ] + ); + }); +} diff --git a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/fellowship.rs b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/fellowship.rs index 26fa55acb0d66142729d087b10592b5b05a1740b..c08a660205f6829ed50b5643ab191b4c3005ce73 100644 --- a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/fellowship.rs +++ b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/fellowship.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. //! Integration tests concerning the Fellowship. @@ -58,11 +57,8 @@ fn pay_salary() { ); }); - Collectives::execute_with(|| {}); - AssetHubPolkadot::execute_with(|| { type RuntimeEvent = ::RuntimeEvent; - assert_expected_events!( AssetHubPolkadot, vec![ diff --git a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/mod.rs b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/mod.rs index a9445ac8ec7eb191f067b6e95bf41730957a0b95..42d2432d2237b83b229cd8803b5d0262bbb4e23a 100644 --- a/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/mod.rs +++ b/cumulus/parachains/integration-tests/emulated/collectives/collectives-polkadot/src/tests/mod.rs @@ -1,17 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +mod ambassador; mod fellowship; diff --git a/cumulus/parachains/integration-tests/emulated/common/Cargo.toml b/cumulus/parachains/integration-tests/emulated/common/Cargo.toml index a8002158dbc139e53c9ce44b8e1b9a5054ef25b0..f1db26c2a540d4d7c2ca29e0c68860d12b2cf8b5 100644 --- a/cumulus/parachains/integration-tests/emulated/common/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/common/Cargo.toml @@ -3,6 +3,7 @@ name = "integration-tests-common" version = "1.0.0" authors.workspace = true edition.workspace = true +license = "Apache-2.0" description = "Common resources for integration testing with xcm-emulator" publish = false @@ -55,9 +56,9 @@ xcm-emulator = { path = "../../../../xcm/xcm-emulator", default-features = false cumulus-pallet-dmp-queue = { path = "../../../../pallets/dmp-queue" } cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-features = false} cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system" } -bp-messages = { path = "../../../../bridges/primitives/messages" } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages" } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common" } +bp-messages = { path = "../../../../../bridges/primitives/messages" } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages" } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common" } [features] runtime-benchmarks = [ @@ -79,6 +80,7 @@ runtime-benchmarks = [ "pallet-message-queue/runtime-benchmarks", "pallet-staking/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "penpal-runtime/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/common/src/constants.rs b/cumulus/parachains/integration-tests/emulated/common/src/constants.rs index fedc46fabfff1f6f55f274c4bbf848505afb367b..8725ebd140b97272630430de324177bf5cae1484 100644 --- a/cumulus/parachains/integration-tests/emulated/common/src/constants.rs +++ b/cumulus/parachains/integration-tests/emulated/common/src/constants.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Substrate use beefy_primitives::ecdsa_crypto::AuthorityId as BeefyId; @@ -632,7 +631,7 @@ pub mod rococo { pub mod asset_hub_polkadot { use super::*; pub const PARA_ID: u32 = 1000; - pub const ED: Balance = asset_hub_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = asset_hub_polkadot_runtime::RuntimeGenesisConfig { @@ -689,7 +688,7 @@ pub mod asset_hub_polkadot { pub mod asset_hub_westend { use super::*; pub const PARA_ID: u32 = 1000; - pub const ED: Balance = asset_hub_westend_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::westend::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = asset_hub_westend_runtime::RuntimeGenesisConfig { @@ -746,7 +745,7 @@ pub mod asset_hub_westend { pub mod asset_hub_kusama { use super::*; pub const PARA_ID: u32 = 1000; - pub const ED: Balance = asset_hub_kusama_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::kusama::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = asset_hub_kusama_runtime::RuntimeGenesisConfig { @@ -864,7 +863,7 @@ pub mod penpal { pub mod collectives { use super::*; pub const PARA_ID: u32 = 1001; - pub const ED: Balance = collectives_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = collectives_polkadot_runtime::RuntimeGenesisConfig { @@ -921,7 +920,7 @@ pub mod collectives { pub mod bridge_hub_kusama { use super::*; pub const PARA_ID: u32 = 1002; - pub const ED: Balance = bridge_hub_kusama_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::kusama::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = bridge_hub_kusama_runtime::RuntimeGenesisConfig { @@ -978,7 +977,7 @@ pub mod bridge_hub_kusama { pub mod bridge_hub_polkadot { use super::*; pub const PARA_ID: u32 = 1002; - pub const ED: Balance = bridge_hub_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = bridge_hub_polkadot_runtime::RuntimeGenesisConfig { @@ -1035,7 +1034,7 @@ pub mod bridge_hub_polkadot { pub mod bridge_hub_rococo { use super::*; pub const PARA_ID: u32 = 1013; - pub const ED: Balance = bridge_hub_rococo_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + pub const ED: Balance = parachains_common::rococo::currency::EXISTENTIAL_DEPOSIT; pub fn genesis() -> Storage { let genesis_config = bridge_hub_rococo_runtime::RuntimeGenesisConfig { diff --git a/cumulus/parachains/integration-tests/emulated/common/src/impls.rs b/cumulus/parachains/integration-tests/emulated/common/src/impls.rs index f13da0016205cceaa2599f4e23b2e6b8963c84b1..eed61d94171107a278e13b8fca8ba9a94a547f21 100644 --- a/cumulus/parachains/integration-tests/emulated/common/src/impls.rs +++ b/cumulus/parachains/integration-tests/emulated/common/src/impls.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub use codec::{Decode, Encode}; pub use paste; diff --git a/cumulus/parachains/integration-tests/emulated/common/src/lib.rs b/cumulus/parachains/integration-tests/emulated/common/src/lib.rs index 49751136f7ba0016c022359eaf789516c0ac0243..7461165f2a198e8f8b871850545bf50f6e37e885 100644 --- a/cumulus/parachains/integration-tests/emulated/common/src/lib.rs +++ b/cumulus/parachains/integration-tests/emulated/common/src/lib.rs @@ -1,18 +1,17 @@ // Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. +// SPDX-License-Identifier: Apache-2.0 -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. pub mod constants; pub mod impls; diff --git a/cumulus/parachains/pallets/collective-content/Cargo.toml b/cumulus/parachains/pallets/collective-content/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..1c831ac7268a54533acf38188b3dcf73edcec6c1 --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "pallet-collective-content" +version = "0.1.0" +authors = ["Parity Technologies "] +edition = "2021" +description = "Managed content" + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +scale-info = { version = "2.3.0", default-features = false, features = ["derive"] } + +frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", optional = true, default-features = false } +frame-support = { path = "../../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../../substrate/frame/system", default-features = false } + +sp-core = { path = "../../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../../substrate/primitives/std", default-features = false } + +[dev-dependencies] +sp-io = { path = "../../../../substrate/primitives/io", default-features = false } + +[features] +default = [ "std" ] +runtime-benchmarks = [ + "frame-benchmarking/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", +] + +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] + +std = [ + "codec/std", + "frame-benchmarking/std", + "frame-support/std", + "frame-system/std", + "scale-info/std", + "sp-core/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] diff --git a/cumulus/parachains/pallets/collective-content/src/benchmarking.rs b/cumulus/parachains/pallets/collective-content/src/benchmarking.rs new file mode 100644 index 0000000000000000000000000000000000000000..1f145f725b139b4e93481a9c69b0715471f7ec0b --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/src/benchmarking.rs @@ -0,0 +1,88 @@ +// Copyright (C) 2023 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The pallet benchmarks. + +use super::{Pallet as CollectiveContent, *}; +use frame_benchmarking::{impl_benchmark_test_suite, v2::*}; +use frame_support::traits::EnsureOrigin; + +fn assert_last_event, I: 'static>(generic_event: >::RuntimeEvent) { + frame_system::Pallet::::assert_last_event(generic_event.into()); +} + +/// returns CID hash of 68 bytes of given `i`. +fn create_cid(i: u8) -> OpaqueCid { + let cid: OpaqueCid = [i; 68].to_vec().try_into().unwrap(); + cid +} + +#[instance_benchmarks] +mod benchmarks { + use super::*; + + #[benchmark] + fn set_charter() -> Result<(), BenchmarkError> { + let cid: OpaqueCid = create_cid(1); + let origin = + T::CharterOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, cid.clone()); + + assert_eq!(Charter::::get(), Some(cid.clone())); + assert_last_event::(Event::NewCharterSet { cid }.into()); + Ok(()) + } + + #[benchmark] + fn announce() -> Result<(), BenchmarkError> { + let expire_at = DispatchTime::<_>::At(10u32.into()); + let now = frame_system::Pallet::::block_number(); + let cid: OpaqueCid = create_cid(1); + let origin = T::AnnouncementOrigin::try_successful_origin() + .map_err(|_| BenchmarkError::Weightless)?; + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, cid.clone(), Some(expire_at.clone())); + + assert_eq!(>::count(), 1); + assert_last_event::( + Event::AnnouncementAnnounced { cid, expire_at: expire_at.evaluate(now) }.into(), + ); + + Ok(()) + } + + #[benchmark] + fn remove_announcement() -> Result<(), BenchmarkError> { + let cid: OpaqueCid = create_cid(1); + let origin = T::AnnouncementOrigin::try_successful_origin() + .map_err(|_| BenchmarkError::Weightless)?; + CollectiveContent::::announce(origin.clone(), cid.clone(), None) + .expect("could not publish an announcement"); + assert_eq!(>::count(), 1); + + #[extrinsic_call] + _(origin as T::RuntimeOrigin, cid.clone()); + + assert_eq!(>::count(), 0); + assert_last_event::(Event::AnnouncementRemoved { cid }.into()); + + Ok(()) + } + + impl_benchmark_test_suite!(CollectiveContent, super::mock::new_bench_ext(), super::mock::Test); +} diff --git a/cumulus/parachains/pallets/collective-content/src/lib.rs b/cumulus/parachains/pallets/collective-content/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a685858accb67868837e734ba9808741a4f7ea1 --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/src/lib.rs @@ -0,0 +1,206 @@ +// Copyright (C) 2023 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Managed Collective Content Pallet +//! +//! The pallet provides the functionality to store different types of content. This would typically +//! be used by an on-chain collective, such as the Polkadot Alliance or Ambassador Program. +//! +//! The pallet stores content as an [OpaqueCid], which should correspond to some off-chain hosting +//! service, such as IPFS, and contain any type of data. Each type of content has its own origin +//! from which it can be managed. The origins are configurable in the runtime. Storing content does +//! not require a deposit, as it is expected to be managed by a trusted collective. +//! +//! Content types: +//! +//! - Collective [charter](pallet::Charter): A single document (`OpaqueCid`) managed by +//! [CharterOrigin](pallet::Config::CharterOrigin). +//! - Collective [announcements](pallet::Announcements): A list of announcements managed by +//! [AnnouncementOrigin](pallet::Config::AnnouncementOrigin). + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; +pub mod weights; + +pub use pallet::*; +pub use weights::WeightInfo; + +use frame_support::{traits::schedule::DispatchTime, BoundedVec}; +use sp_core::ConstU32; +use sp_std::prelude::*; + +/// IPFS compatible CID. +// Worst case 2 bytes base and codec, 2 bytes hash type and size, 64 bytes hash digest. +pub type OpaqueCid = BoundedVec>; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::{ensure, pallet_prelude::*}; + use frame_system::pallet_prelude::*; + use sp_runtime::{traits::BadOrigin, Saturating}; + + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(0); + + #[pallet::pallet] + #[pallet::storage_version(STORAGE_VERSION)] + pub struct Pallet(PhantomData<(T, I)>); + + /// The module configuration trait. + #[pallet::config] + pub trait Config: frame_system::Config { + /// The overarching event type. + type RuntimeEvent: From> + + IsType<::RuntimeEvent>; + + /// Default lifetime for an announcement before it expires. + type AnnouncementLifetime: Get>; + + /// The origin to control the collective announcements. + type AnnouncementOrigin: EnsureOrigin; + + /// Maximum number of announcements in the storage. + #[pallet::constant] + type MaxAnnouncements: Get; + + /// The origin to control the collective charter. + type CharterOrigin: EnsureOrigin; + + /// Weight information needed for the pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::error] + pub enum Error { + /// The announcement is not found. + MissingAnnouncement, + /// Number of announcements exceeds `MaxAnnouncementsCount`. + TooManyAnnouncements, + /// Cannot expire in the past. + InvalidExpiration, + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event, I: 'static = ()> { + /// A new charter has been set. + NewCharterSet { cid: OpaqueCid }, + /// A new announcement has been made. + AnnouncementAnnounced { cid: OpaqueCid, expire_at: BlockNumberFor }, + /// An on-chain announcement has been removed. + AnnouncementRemoved { cid: OpaqueCid }, + } + + /// The collective charter. + #[pallet::storage] + pub type Charter, I: 'static = ()> = StorageValue<_, OpaqueCid, OptionQuery>; + + /// The collective announcements. + #[pallet::storage] + pub type Announcements, I: 'static = ()> = + CountedStorageMap<_, Blake2_128Concat, OpaqueCid, BlockNumberFor, OptionQuery>; + + #[pallet::call] + impl, I: 'static> Pallet { + /// Set the collective charter. + /// + /// Parameters: + /// - `origin`: Must be the [Config::CharterOrigin]. + /// - `cid`: [CID](super::OpaqueCid) of the IPFS document of the collective charter. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::set_charter())] + pub fn set_charter(origin: OriginFor, cid: OpaqueCid) -> DispatchResult { + T::CharterOrigin::ensure_origin(origin)?; + + Charter::::put(&cid); + + Self::deposit_event(Event::::NewCharterSet { cid }); + Ok(()) + } + + /// Publish an announcement. + /// + /// Parameters: + /// - `origin`: Must be the [Config::AnnouncementOrigin]. + /// - `cid`: [CID](super::OpaqueCid) of the IPFS document to announce. + /// - `maybe_expire`: Expiration block of the announcement. If `None` + /// [`Config::AnnouncementLifetime`] + /// used as a default. + #[pallet::call_index(1)] + #[pallet::weight(T::WeightInfo::announce())] + pub fn announce( + origin: OriginFor, + cid: OpaqueCid, + maybe_expire: Option>>, + ) -> DispatchResult { + T::AnnouncementOrigin::ensure_origin(origin)?; + + let now = frame_system::Pallet::::block_number(); + let expire_at = maybe_expire + .map_or(now.saturating_add(T::AnnouncementLifetime::get()), |e| e.evaluate(now)); + ensure!(expire_at > now, Error::::InvalidExpiration); + ensure!( + T::MaxAnnouncements::get() > >::count(), + Error::::TooManyAnnouncements + ); + + >::insert(cid.clone(), expire_at); + + Self::deposit_event(Event::::AnnouncementAnnounced { cid, expire_at }); + Ok(()) + } + + /// Remove an announcement. + /// + /// Transaction fee refunded for expired announcements. + /// + /// Parameters: + /// - `origin`: Must be the [Config::AnnouncementOrigin] or signed for expired + /// announcements. + /// - `cid`: [CID](super::OpaqueCid) of the IPFS document to remove. + #[pallet::call_index(2)] + #[pallet::weight(T::WeightInfo::remove_announcement())] + pub fn remove_announcement( + origin: OriginFor, + cid: OpaqueCid, + ) -> DispatchResultWithPostInfo { + let maybe_who = match T::AnnouncementOrigin::try_origin(origin) { + Ok(_) => None, + Err(origin) => Some(ensure_signed(origin)?), + }; + let expire_at = >::get(cid.clone()) + .ok_or(Error::::MissingAnnouncement)?; + let now = frame_system::Pallet::::block_number(); + ensure!(maybe_who.is_none() || now >= expire_at, BadOrigin); + + >::remove(cid.clone()); + + Self::deposit_event(Event::::AnnouncementRemoved { cid }); + + if now >= expire_at { + return Ok(Pays::No.into()) + } + Ok(Pays::Yes.into()) + } + } +} diff --git a/cumulus/parachains/pallets/collective-content/src/mock.rs b/cumulus/parachains/pallets/collective-content/src/mock.rs new file mode 100644 index 0000000000000000000000000000000000000000..2ae5943f332acf61b60792563b7d3f9b1e808c70 --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/src/mock.rs @@ -0,0 +1,107 @@ +// Copyright (C) 2023 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Test utilities. + +pub use crate as pallet_collective_content; +use crate::WeightInfo; +use frame_support::{ + ord_parameter_types, parameter_types, + traits::{ConstU32, ConstU64}, + weights::Weight, +}; +use frame_system::EnsureSignedBy; +use sp_runtime::{traits::IdentityLookup, BuildStorage}; + +frame_support::construct_runtime!( + pub enum Test { + System: frame_system, + CollectiveContent: pallet_collective_content, + } +); + +type AccountId = u64; +type Block = frame_system::mocking::MockBlock; + +ord_parameter_types! { + pub const CharterManager: u64 = 1; + pub const AnnouncementManager: u64 = 2; + pub const SomeAccount: u64 = 3; +} + +parameter_types! { + pub const AnnouncementLifetime: u64 = 100; + pub const MaxAnnouncements: u32 = 5; +} + +impl pallet_collective_content::Config for Test { + type RuntimeEvent = RuntimeEvent; + type AnnouncementLifetime = AnnouncementLifetime; + type AnnouncementOrigin = EnsureSignedBy; + type MaxAnnouncements = MaxAnnouncements; + type CharterOrigin = EnsureSignedBy; + type WeightInfo = CCWeightInfo; +} + +impl frame_system::Config for Test { + type BaseCallFilter = (); + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type Nonce = u64; + type Block = Block; + type Hash = sp_core::H256; + type Hashing = sp_runtime::traits::BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = (); + type OnSetCode = (); + type MaxConsumers = ConstU32<16>; +} +pub struct CCWeightInfo; +impl WeightInfo for CCWeightInfo { + fn set_charter() -> Weight { + Weight::zero() + } + fn announce() -> Weight { + Weight::zero() + } + fn remove_announcement() -> Weight { + Weight::zero() + } +} + +// Build test environment. +pub fn new_test_ext() -> sp_io::TestExternalities { + let t = RuntimeGenesisConfig::default().build_storage().unwrap().into(); + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext +} + +#[cfg(feature = "runtime-benchmarks")] +pub fn new_bench_ext() -> sp_io::TestExternalities { + RuntimeGenesisConfig::default().build_storage().unwrap().into() +} diff --git a/cumulus/parachains/pallets/collective-content/src/tests.rs b/cumulus/parachains/pallets/collective-content/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..4910b30b89af84430a60e2eab011ed2c5c6a3d03 --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/src/tests.rs @@ -0,0 +1,204 @@ +// Copyright (C) 2023 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tests. + +use super::{mock::*, *}; +use frame_support::{assert_noop, assert_ok, error::BadOrigin, pallet_prelude::Pays}; + +/// returns CID hash of 68 bytes of given `i`. +fn create_cid(i: u8) -> OpaqueCid { + let cid: OpaqueCid = [i; 68].to_vec().try_into().unwrap(); + cid +} + +#[test] +fn set_charter_works() { + new_test_ext().execute_with(|| { + // wrong origin. + let origin = RuntimeOrigin::signed(SomeAccount::get()); + let cid = create_cid(1); + assert_noop!(CollectiveContent::set_charter(origin, cid), BadOrigin); + + // success. + let origin = RuntimeOrigin::signed(CharterManager::get()); + let cid = create_cid(2); + + assert_ok!(CollectiveContent::set_charter(origin, cid.clone())); + assert_eq!(Charter::::get(), Some(cid.clone())); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::NewCharterSet { cid })); + + // reset. success. + let origin = RuntimeOrigin::signed(CharterManager::get()); + let cid = create_cid(3); + + assert_ok!(CollectiveContent::set_charter(origin, cid.clone())); + assert_eq!(Charter::::get(), Some(cid.clone())); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::NewCharterSet { cid })); + }); +} + +#[test] +fn announce_works() { + new_test_ext().execute_with(|| { + let now = frame_system::Pallet::::block_number(); + // wrong origin. + let origin = RuntimeOrigin::signed(SomeAccount::get()); + let cid = create_cid(1); + + assert_noop!(CollectiveContent::announce(origin, cid, None), BadOrigin); + + // success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(2); + let maybe_expire_at = None; + + assert_ok!(CollectiveContent::announce(origin, cid.clone(), maybe_expire_at)); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementAnnounced { + cid, + expire_at: now.saturating_add(AnnouncementLifetime::get()), + })); + + // one more. success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(3); + let maybe_expire_at = None; + + assert_ok!(CollectiveContent::announce(origin, cid.clone(), maybe_expire_at)); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementAnnounced { + cid, + expire_at: now.saturating_add(AnnouncementLifetime::get()), + })); + + // one more with expire. success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(4); + let maybe_expire_at = DispatchTime::<_>::After(10); + + assert_ok!(CollectiveContent::announce(origin, cid.clone(), Some(maybe_expire_at))); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementAnnounced { + cid, + expire_at: maybe_expire_at.evaluate(now), + })); + + // one more with later expire. success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(5); + let maybe_expire_at = DispatchTime::<_>::At(now + 20); + + assert_ok!(CollectiveContent::announce(origin, cid.clone(), Some(maybe_expire_at))); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementAnnounced { + cid, + expire_at: maybe_expire_at.evaluate(now), + })); + + // one more with earlier expire. success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(6); + let maybe_expire_at = DispatchTime::<_>::At(now + 5); + + assert_ok!(CollectiveContent::announce(origin, cid.clone(), Some(maybe_expire_at))); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementAnnounced { + cid, + expire_at: maybe_expire_at.evaluate(now), + })); + + // one more with earlier expire. success. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(7); + let maybe_expire_at = DispatchTime::<_>::At(now + 5); + + assert_eq!(>::count(), MaxAnnouncements::get()); + assert_noop!( + CollectiveContent::announce(origin, cid.clone(), Some(maybe_expire_at)), + Error::::TooManyAnnouncements + ); + }); +} + +#[test] +fn remove_announcement_works() { + new_test_ext().execute_with(|| { + // wrong origin. + let origin = RuntimeOrigin::signed(CharterManager::get()); + let cid = create_cid(8); + + assert_noop!( + CollectiveContent::remove_announcement(origin, cid), + Error::::MissingAnnouncement + ); + + // missing announcement. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(9); + + assert_noop!( + CollectiveContent::remove_announcement(origin, cid), + Error::::MissingAnnouncement + ); + + // wrong origin. announcement not yet expired. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(10); + assert_ok!(CollectiveContent::announce(origin.clone(), cid.clone(), None)); + assert!(>::contains_key(cid.clone())); + + let origin = RuntimeOrigin::signed(SomeAccount::get()); + assert_noop!(CollectiveContent::remove_announcement(origin, cid.clone()), BadOrigin); + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + assert_ok!(CollectiveContent::remove_announcement(origin, cid)); + + // success. + + // remove first announcement and assert. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(11); + assert_ok!(CollectiveContent::announce(origin.clone(), cid.clone(), None)); + assert!(>::contains_key(cid.clone())); + + let info = CollectiveContent::remove_announcement(origin.clone(), cid.clone()).unwrap(); + assert_eq!(info.pays_fee, Pays::Yes); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementRemoved { + cid: cid.clone(), + })); + assert_noop!( + CollectiveContent::remove_announcement(origin, cid.clone()), + Error::::MissingAnnouncement + ); + assert!(!>::contains_key(cid)); + + // remove expired announcement and assert. + let origin = RuntimeOrigin::signed(AnnouncementManager::get()); + let cid = create_cid(12); + assert_ok!(CollectiveContent::announce( + origin.clone(), + cid.clone(), + Some(DispatchTime::<_>::At(10)) + )); + assert!(>::contains_key(cid.clone())); + + System::set_block_number(11); + let origin = RuntimeOrigin::signed(SomeAccount::get()); + let info = CollectiveContent::remove_announcement(origin.clone(), cid.clone()).unwrap(); + assert_eq!(info.pays_fee, Pays::No); + System::assert_last_event(RuntimeEvent::CollectiveContent(Event::AnnouncementRemoved { + cid: cid.clone(), + })); + assert_noop!( + CollectiveContent::remove_announcement(origin, cid), + Error::::MissingAnnouncement + ); + }); +} diff --git a/cumulus/parachains/pallets/collective-content/src/weights.rs b/cumulus/parachains/pallets/collective-content/src/weights.rs new file mode 100644 index 0000000000000000000000000000000000000000..7ebbaabb5acc2451dc9323874c4f912f98ad8fef --- /dev/null +++ b/cumulus/parachains/pallets/collective-content/src/weights.rs @@ -0,0 +1,41 @@ +// Copyright (C) 2023 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The pallet weight info trait and its unit implementation. + +use frame_support::weights::Weight; + +/// Weights information needed for the pallet. +pub trait WeightInfo { + /// Returns the weight of the set_charter extrinsic. + fn set_charter() -> Weight; + /// Returns the weight of the announce extrinsic. + fn announce() -> Weight; + /// Returns the weight of the remove_announcement extrinsic. + fn remove_announcement() -> Weight; +} + +/// Unit implementation of the [WeightInfo]. +impl WeightInfo for () { + fn set_charter() -> Weight { + Weight::zero() + } + fn announce() -> Weight { + Weight::zero() + } + fn remove_announcement() -> Weight { + Weight::zero() + } +} diff --git a/cumulus/parachains/pallets/parachain-info/Cargo.toml b/cumulus/parachains/pallets/parachain-info/Cargo.toml index 4ed11e3943980c0de0af6deacdd73aa528dc0044..931df9d9273d8cf0d511ff321973593ecf563657 100644 --- a/cumulus/parachains/pallets/parachain-info/Cargo.toml +++ b/cumulus/parachains/pallets/parachain-info/Cargo.toml @@ -3,7 +3,6 @@ authors.workspace = true edition.workspace = true name = "parachain-info" version = "0.1.0" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml index 53555499842fde9684b3a5b33087ca8de80643ee..1d0c7bc98563417169a3a17586de695e1ba36a3a 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml @@ -100,6 +100,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -118,6 +119,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/constants.rs b/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/constants.rs deleted file mode 100644 index 8daf8fda4b4a2e645ee24af8525cf15ef2b95e89..0000000000000000000000000000000000000000 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/constants.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod currency { - use kusama_runtime_constants as constants; - use polkadot_core_primitives::Balance; - - /// The existential deposit. Set to 1/10 of its parent Relay Chain. - pub const EXISTENTIAL_DEPOSIT: Balance = constants::currency::EXISTENTIAL_DEPOSIT / 10; - - pub const UNITS: Balance = constants::currency::UNITS; - pub const CENTS: Balance = constants::currency::CENTS; - pub const GRAND: Balance = constants::currency::GRAND; - pub const MILLICENTS: Balance = constants::currency::MILLICENTS; - - pub const fn deposit(items: u32, bytes: u32) -> Balance { - // map to 1/100 of what the kusama relay chain charges (v9020) - constants::currency::deposit(items, bytes) / 100 - } -} - -/// Fee-related. -pub mod fee { - use frame_support::weights::{ - constants::ExtrinsicBaseWeight, FeePolynomial, Weight, WeightToFeeCoefficient, - WeightToFeeCoefficients, WeightToFeePolynomial, - }; - use polkadot_core_primitives::Balance; - use smallvec::smallvec; - pub use sp_runtime::Perbill; - - /// The block saturation level. Fees will be updates based on this value. - pub const TARGET_BLOCK_FULLNESS: Perbill = Perbill::from_percent(25); - - /// Handles converting a weight scalar to a fee value, based on the scale and granularity of the - /// node's balance type. - /// - /// This should typically create a mapping between the following ranges: - /// - [0, MAXIMUM_BLOCK_WEIGHT] - /// - [Balance::min, Balance::max] - /// - /// Yet, it can be used for any other sort of change to weight-fee. Some examples being: - /// - Setting it to `0` will essentially disable the weight fee. - /// - Setting it to `1` will cause the literal `#[weight = x]` values to be charged. - pub struct WeightToFee; - impl frame_support::weights::WeightToFee for WeightToFee { - type Balance = Balance; - - fn weight_to_fee(weight: &Weight) -> Self::Balance { - let time_poly: FeePolynomial = RefTimeToFee::polynomial().into(); - let proof_poly: FeePolynomial = ProofSizeToFee::polynomial().into(); - - // Take the maximum instead of the sum to charge by the more scarce resource. - time_poly.eval(weight.ref_time()).max(proof_poly.eval(weight.proof_size())) - } - } - - /// Maps the reference time component of `Weight` to a fee. - pub struct RefTimeToFee; - impl WeightToFeePolynomial for RefTimeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // in Kusama, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Asset Hub, we map to 1/10 of that, or 1/100 CENT - let p = super::currency::CENTS; - let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } - - /// Maps the proof size component of `Weight` to a fee. - pub struct ProofSizeToFee; - impl WeightToFeePolynomial for ProofSizeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // Map 10kb proof to 1 CENT. - let p = super::currency::CENTS; - let q = 10_000; - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } -} - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs index afccc5c068b5e7b6629d1a8e03d29497d93d28ca..828d1b4750a3e0436bc18f7926489a04a4b2bb9e 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs @@ -24,7 +24,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod weights; pub mod xcm_config; @@ -50,7 +49,6 @@ use sp_version::NativeVersion; use sp_version::RuntimeVersion; use codec::{Decode, Encode, MaxEncodedLen}; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, @@ -70,8 +68,10 @@ use pallet_asset_conversion_tx_payment::AssetConversionAdapter; use pallet_nfts::PalletFeatures; pub use parachains_common as common; use parachains_common::{ - impls::DealWithFees, AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, BlockNumber, - Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, + impls::DealWithFees, + kusama::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, BlockNumber, Hash, Header, Nonce, + Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; use sp_runtime::RuntimeDebug; @@ -1395,8 +1395,9 @@ fn ensure_key_ss58() { #[cfg(test)] mod tests { - use super::{constants::fee, *}; + use super::*; use crate::{CENTS, MILLICENTS}; + use parachains_common::kusama::fee; use sp_runtime::traits::Zero; use sp_weights::WeightToFee; diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/tests/tests.rs b/cumulus/parachains/runtimes/assets/asset-hub-kusama/tests/tests.rs index 6d9eccdf3744778af680575cbac79490732ad1ea..7d49b56e461a0a3919e2130dbb6d84f6888349ad 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/tests/tests.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-kusama/tests/tests.rs @@ -21,7 +21,6 @@ use asset_hub_kusama_runtime::xcm_config::{ AssetFeeAsExistentialDepositMultiplierFeeCharger, KsmLocation, TrustBackedAssetsPalletLocation, }; pub use asset_hub_kusama_runtime::{ - constants::fee::WeightToFee, xcm_config::{CheckingAccount, ForeignCreatorsSovereignAccountOf, XcmConfig}, AllPalletsWithoutSystem, AssetDeposit, Assets, Balances, ExistentialDeposit, ForeignAssets, ForeignAssetsInstance, MetadataDepositBase, MetadataDepositPerByte, ParachainSystem, Runtime, @@ -35,7 +34,9 @@ use frame_support::{ traits::fungibles::InspectEnumerable, weights::{Weight, WeightToFee as WeightToFeeT}, }; -use parachains_common::{AccountId, AssetIdForTrustBackedAssets, AuraId, Balance}; +use parachains_common::{ + kusama::fee::WeightToFee, AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, +}; use sp_runtime::traits::MaybeEquivalence; use xcm::latest::prelude::*; use xcm_executor::traits::{Identity, JustTry, WeightTrader}; diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml index dfc3d5416cc3c7c2ebc51dee2aa68a495a6a5079..4a6ce4cbf593108fdc06c81853cd1dafaa7d3fde 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml @@ -88,6 +88,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -105,6 +106,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/constants.rs b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/constants.rs deleted file mode 100644 index d430e38f1af15b423b2173778a1cb6376fa17835..0000000000000000000000000000000000000000 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/constants.rs +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod currency { - use polkadot_core_primitives::Balance; - use polkadot_runtime_constants as constants; - - /// The existential deposit. Set to 1/10 of its parent Relay Chain. - pub const EXISTENTIAL_DEPOSIT: Balance = constants::currency::EXISTENTIAL_DEPOSIT / 10; - - pub const UNITS: Balance = constants::currency::UNITS; - pub const DOLLARS: Balance = constants::currency::DOLLARS; - pub const CENTS: Balance = constants::currency::CENTS; - pub const MILLICENTS: Balance = constants::currency::MILLICENTS; - - pub const fn deposit(items: u32, bytes: u32) -> Balance { - // 1/100 of Polkadot - constants::currency::deposit(items, bytes) / 100 - } -} - -/// Fee-related. -pub mod fee { - use frame_support::weights::{ - constants::ExtrinsicBaseWeight, FeePolynomial, WeightToFeeCoefficient, - WeightToFeeCoefficients, WeightToFeePolynomial, - }; - use polkadot_core_primitives::Balance; - use smallvec::smallvec; - pub use sp_runtime::Perbill; - use sp_weights::Weight; - - /// The block saturation level. Fees will be updates based on this value. - pub const TARGET_BLOCK_FULLNESS: Perbill = Perbill::from_percent(25); - - /// Handles converting a weight scalar to a fee value, based on the scale and granularity of the - /// node's balance type. - /// - /// This should typically create a mapping between the following ranges: - /// - [0, MAXIMUM_BLOCK_WEIGHT] - /// - [Balance::min, Balance::max] - /// - /// Yet, it can be used for any other sort of change to weight-fee. Some examples being: - /// - Setting it to `0` will essentially disable the weight fee. - /// - Setting it to `1` will cause the literal `#[weight = x]` values to be charged. - pub struct WeightToFee; - impl frame_support::weights::WeightToFee for WeightToFee { - type Balance = Balance; - - fn weight_to_fee(weight: &Weight) -> Self::Balance { - let time_poly: FeePolynomial = RefTimeToFee::polynomial().into(); - let proof_poly: FeePolynomial = ProofSizeToFee::polynomial().into(); - - // Take the maximum instead of the sum to charge by the more scarce resource. - time_poly.eval(weight.ref_time()).max(proof_poly.eval(weight.proof_size())) - } - } - - /// Maps the reference time component of `Weight` to a fee. - pub struct RefTimeToFee; - impl WeightToFeePolynomial for RefTimeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // in Polkadot, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Asset Hub, we map to 1/10 of that, or 1/100 CENT - let p = super::currency::CENTS; - let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } - - /// Maps the proof size component of `Weight` to a fee. - pub struct ProofSizeToFee; - impl WeightToFeePolynomial for ProofSizeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // Map 10kb proof to 1 CENT. - let p = super::currency::CENTS; - let q = 10_000; - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } -} - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs index 7275209802f15d9110db2e0faace126362a8a52a..0051af21f9a32b67ff87b0ca3b82373ff4778715 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs @@ -59,7 +59,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod weights; pub mod xcm_config; @@ -82,7 +81,6 @@ use sp_version::NativeVersion; use sp_version::RuntimeVersion; use codec::{Decode, Encode, MaxEncodedLen}; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, @@ -102,6 +100,7 @@ use pallet_nfts::PalletFeatures; pub use parachains_common as common; use parachains_common::{ impls::{AssetsToBlockAuthor, DealWithFees}, + polkadot::{consensus::*, currency::*, fee::WeightToFee}, AccountId, AssetHubPolkadotAuraId as AuraId, AssetIdForTrustBackedAssets, Balance, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, @@ -1228,8 +1227,9 @@ cumulus_pallet_parachain_system::register_validate_block! { #[cfg(test)] mod tests { - use super::{constants::fee, *}; + use super::*; use crate::{CENTS, MILLICENTS}; + use parachains_common::polkadot::fee; use sp_runtime::traits::Zero; use sp_weights::WeightToFee; diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/xcm_config.rs b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/xcm_config.rs index d59507e4bd07d4f5a8953c1ad62814a2ebbc2a37..65cf62a610f472105450db14429c5ae29b9b6843 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/xcm_config.rs @@ -186,6 +186,9 @@ match_types! { pub type FellowshipSalaryPallet: impl Contains = { MultiLocation { parents: 1, interior: X2(Parachain(1001), PalletInstance(64)) } }; + pub type AmbassadorSalaryPallet: impl Contains = { + MultiLocation { parents: 1, interior: X2(Parachain(1001), PalletInstance(74)) } + }; } /// A call filter for the XCM Transact instruction. This is a temporary measure until we properly @@ -372,6 +375,7 @@ pub type Barrier = TrailingSetTopicAsId< ParentOrParentsPlurality, FellowsPlurality, FellowshipSalaryPallet, + AmbassadorSalaryPallet, )>, // Subscriptions for version tracking are OK. AllowSubscriptionsFrom, diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/tests/tests.rs b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/tests/tests.rs index 3eab6723ec20cd23a70f78d6e8afe93abe93199b..7200ebc16a2875336fc50e711386cab41df6c9b4 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/tests/tests.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/tests/tests.rs @@ -22,10 +22,9 @@ use asset_hub_polkadot_runtime::xcm_config::{ ForeignCreatorsSovereignAccountOf, TrustBackedAssetsPalletLocation, XcmConfig, }; pub use asset_hub_polkadot_runtime::{ - constants::fee::WeightToFee, AllPalletsWithoutSystem, AssetDeposit, Assets, Balances, - ExistentialDeposit, ForeignAssets, ForeignAssetsInstance, MetadataDepositBase, - MetadataDepositPerByte, ParachainSystem, Runtime, RuntimeCall, RuntimeEvent, SessionKeys, - System, TrustBackedAssetsInstance, + AllPalletsWithoutSystem, AssetDeposit, Assets, Balances, ExistentialDeposit, ForeignAssets, + ForeignAssetsInstance, MetadataDepositBase, MetadataDepositPerByte, ParachainSystem, Runtime, + RuntimeCall, RuntimeEvent, SessionKeys, System, TrustBackedAssetsInstance, }; use asset_test_utils::{CollatorSessionKeys, ExtBuilder}; use codec::{Decode, Encode}; @@ -36,7 +35,8 @@ use frame_support::{ weights::{Weight, WeightToFee as WeightToFeeT}, }; use parachains_common::{ - AccountId, AssetHubPolkadotAuraId as AuraId, AssetIdForTrustBackedAssets, Balance, + polkadot::fee::WeightToFee, AccountId, AssetHubPolkadotAuraId as AuraId, + AssetIdForTrustBackedAssets, Balance, }; use sp_runtime::traits::MaybeEquivalence; use xcm::latest::prelude::*; diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml index 8c25842cbf3817c31c8e87b32025c1a5fa28c5de..f436ae9537f46b5266111f39bb039f841c3d4388 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml @@ -92,6 +92,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -110,6 +111,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs index b5f33e4e01458c5310207d59d2065b3886e4d677..4887fce1b0a4a5d633907cae17697b7793e781c8 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs @@ -24,7 +24,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod weights; pub mod xcm_config; @@ -36,7 +35,6 @@ use assets_common::{ AssetIdForTrustBackedAssetsConvert, }; use codec::{Decode, Encode, MaxEncodedLen}; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; use frame_support::{ construct_runtime, @@ -57,8 +55,10 @@ use pallet_asset_conversion_tx_payment::AssetConversionAdapter; use pallet_nfts::PalletFeatures; pub use parachains_common as common; use parachains_common::{ - impls::DealWithFees, AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, BlockNumber, - Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, + impls::DealWithFees, + westend::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, BlockNumber, Hash, Header, Nonce, + Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; use sp_api::impl_runtime_apis; diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/tests/tests.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/tests/tests.rs index b2bb511182eb2091dc1fd5c4d9a95baa0e51e970..599ff90e254aa5f4fb745e153b8a3e9b9addb0e8 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/tests/tests.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/tests/tests.rs @@ -17,13 +17,6 @@ //! Tests for the Westmint (Westend Assets Hub) chain. -pub use asset_hub_westend_runtime::{ - constants::fee::WeightToFee, - xcm_config::{CheckingAccount, TrustBackedAssetsPalletLocation, XcmConfig}, - AllowMultiAssetPools, AssetDeposit, Assets, Balances, ExistentialDeposit, ForeignAssets, - ForeignAssetsInstance, ParachainSystem, Runtime, SessionKeys, System, - TrustBackedAssetsInstance, -}; use asset_hub_westend_runtime::{ xcm_config::{ AssetFeeAsExistentialDepositMultiplierFeeCharger, ForeignCreatorsSovereignAccountOf, @@ -32,6 +25,12 @@ use asset_hub_westend_runtime::{ AllPalletsWithoutSystem, MetadataDepositBase, MetadataDepositPerByte, RuntimeCall, RuntimeEvent, }; +pub use asset_hub_westend_runtime::{ + xcm_config::{CheckingAccount, TrustBackedAssetsPalletLocation, XcmConfig}, + AllowMultiAssetPools, AssetDeposit, Assets, Balances, ExistentialDeposit, ForeignAssets, + ForeignAssetsInstance, ParachainSystem, Runtime, SessionKeys, System, + TrustBackedAssetsInstance, +}; use asset_test_utils::{CollatorSessionKeys, ExtBuilder, XcmReceivedFrom}; use codec::{Decode, DecodeLimit, Encode}; use cumulus_primitives_utility::ChargeWeightInFungibles; @@ -40,7 +39,9 @@ use frame_support::{ traits::fungibles::InspectEnumerable, weights::{Weight, WeightToFee as WeightToFeeT}, }; -use parachains_common::{AccountId, AssetIdForTrustBackedAssets, AuraId, Balance}; +use parachains_common::{ + westend::fee::WeightToFee, AccountId, AssetIdForTrustBackedAssets, AuraId, Balance, +}; use sp_io; use sp_runtime::traits::MaybeEquivalence; use std::convert::Into; diff --git a/cumulus/parachains/runtimes/assets/common/Cargo.toml b/cumulus/parachains/runtimes/assets/common/Cargo.toml index a8b5f3f8c6b9db84e8c18b654cc0379a641ced1c..0cd5de2ddcd47df05cf3f9dc76c39ada47ac0cb1 100644 --- a/cumulus/parachains/runtimes/assets/common/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/common/Cargo.toml @@ -56,6 +56,7 @@ runtime-benchmarks = [ "pallet-asset-conversion/runtime-benchmarks", "pallet-asset-tx-payment/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml b/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml index 7fee710f000c0d2bd44f6ddfba81e0b5c7eb1d3f..0a27535ed22bd447774fbf13636f1a2c09a5699c 100644 --- a/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "1.0.0" authors.workspace = true edition.workspace = true description = "Test utils for Asset Hub runtimes." -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } diff --git a/cumulus/parachains/runtimes/bridge-hubs/README.md b/cumulus/parachains/runtimes/bridge-hubs/README.md index 1520065b7e3a47656c7215bdf2c3883c03c0c379..487c601ef84023310fe7e15c5642ca060c9e550b 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/README.md +++ b/cumulus/parachains/runtimes/bridge-hubs/README.md @@ -1,26 +1,26 @@ - [Bridge-hub Parachains](#bridge-hub-parachains) - * [Requirements for local run/testing](#requirements-for-local-runtesting) - * [How to test local Rococo <-> Wococo bridge](#how-to-test-local-rococo---wococo-bridge) - + [Run chains (Rococo + BridgeHub, Wococo + BridgeHub) with zombienet](#run-chains-rococo--bridgehub-wococo--bridgehub-with-zombienet) - + [Run relayer (BridgeHubRococo, BridgeHubWococo)](#run-relayer-bridgehubrococo-bridgehubwococo) + - [Requirements for local run/testing](#requirements-for-local-runtesting) + - [How to test local Rococo <-> Wococo bridge](#how-to-test-local-rococo---wococo-bridge) + - [Run chains (Rococo + BridgeHub, Wococo + BridgeHub) with + zombienet](#run-chains-rococo--bridgehub-wococo--bridgehub-with-zombienet) + - [Run relayer (BridgeHubRococo, BridgeHubWococo)](#run-relayer-bridgehubrococo-bridgehubwococo) - [Run with script (alternative 1)](#run-with-script-alternative-1) - [Run with binary (alternative 2)](#run-with-binary-alternative-2) - + [Send messages - transfer asset over bridge](#send-messages---transfer-asset-over-bridge) - * [How to test live BridgeHubRococo/BridgeHubWococo](#how-to-test-live-bridgehubrococobridgehubwococo) - * [How to test local BridgeHubKusama/BridgeHubPolkadot](#how-to-test-local-bridgehubkusamabridgehubpolkadot) + - [Send messages - transfer asset over bridge](#send-messages---transfer-asset-over-bridge) + - [How to test live BridgeHubRococo/BridgeHubWococo](#how-to-test-live-bridgehubrococobridgehubwococo) + - [How to test local BridgeHubKusama/BridgeHubPolkadot](#how-to-test-local-bridgehubkusamabridgehubpolkadot) # Bridge-hub Parachains -_BridgeHub(s)_ are **_system parachains_** that will house trustless bridges from the local -ecosystem to others. -The current trustless bridges planned for the BridgeHub(s) are: +_BridgeHub(s)_ are **_system parachains_** that will house trustless bridges from the local ecosystem to others. The +current trustless bridges planned for the BridgeHub(s) are: - `BridgeHubPolkadot` system parachain: 1. Polkadot <-> Kusama bridge 2. Polkadot <-> Ethereum bridge (Snowbridge) - `BridgeHubKusama` system parachain: 1. Kusama <-> Polkadot bridge - 2. Kusama <-> Ethereum bridge - The high-level responsibilities of each bridge living on BridgeHub: + 2. Kusama <-> Ethereum bridge The high-level + responsibilities of each bridge living on BridgeHub: - sync finality proofs between relay chains (or equivalent) - sync finality proofs between BridgeHub parachains - pass (XCM) messages between different BridgeHub parachains @@ -192,43 +192,40 @@ RUST_LOG=runtime=trace,rpc=trace,bridge=trace \ ``` **Check relay-chain headers relaying:** -- Rococo parachain: - - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8943#/chainstate - - Pallet: **bridgeWococoGrandpa** - - Keys: **bestFinalized()** -- Wococo parachain: - - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8945#/chainstate - - Pallet: **bridgeRococoGrandpa** - - Keys: **bestFinalized()** +- Rococo parachain: - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8943#/chainstate - Pallet: + **bridgeWococoGrandpa** - Keys: **bestFinalized()** +- Wococo parachain: - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8945#/chainstate - Pallet: + **bridgeRococoGrandpa** - Keys: **bestFinalized()** **Check parachain headers relaying:** -- Rococo parachain: - - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8943#/chainstate - - Pallet: **bridgeWococoParachain** - - Keys: **bestParaHeads()** -- Wococo parachain: - - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8945#/chainstate - - Pallet: **bridgeRococoParachain** - - Keys: **bestParaHeads()** +- Rococo parachain: - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8943#/chainstate - Pallet: + **bridgeWococoParachain** - Keys: **bestParaHeads()** +- Wococo parachain: - https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A8945#/chainstate - Pallet: + **bridgeRococoParachain** - Keys: **bestParaHeads()** ### Send messages - transfer asset over bridge TODO: see `# !!! READ HERE` above ## How to test live BridgeHubRococo/BridgeHubWococo -(here is still deployed older PoC from branch `origin/bko-transfer-asset-via-bridge`, which uses custom extrinsic, which is going to be replaced by `pallet_xcm` usage) +(here is still deployed older PoC from branch `origin/bko-transfer-asset-via-bridge`, which uses custom extrinsic, which +is going to be replaced by `pallet_xcm` usage) - uses account seed on Live Rococo:Rockmine2 ``` cd ./scripts/bridges_rococo_wococo.sh transfer-asset-from-asset-hub-rococo ``` -- open explorers: - - Rockmine2 (see events `xcmpQueue.XcmpMessageSent`, `bridgeTransfer.ReserveAssetsDeposited`, `bridgeTransfer.TransferInitiated`) https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fws-rococo-rockmine2-collator-node-0.parity-testnet.parity.io#/explorer - - BridgeHubRococo (see `bridgeWococoMessages.MessageAccepted`) https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-bridge-hub-rpc.polkadot.io#/explorer - - BridgeHubWococo (see `bridgeRococoMessages.MessagesReceived`) https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fwococo-bridge-hub-rpc.polkadot.io#/explorer - - Wockmint (see `xcmpQueue.Success` for `transfer-asset` and `xcmpQueue.Fail` for `ping-via-bridge`) https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fwococo-wockmint-rpc.polkadot.io#/explorer - - BridgeHubRococo (see `bridgeWococoMessages.MessagesDelivered`) +- open explorers: - Rockmine2 (see events `xcmpQueue.XcmpMessageSent`, `bridgeTransfer.ReserveAssetsDeposited`, + `bridgeTransfer.TransferInitiated`) + https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fws-rococo-rockmine2-collator-node-0.parity-testnet.parity.io#/explorer + - BridgeHubRococo (see `bridgeWococoMessages.MessageAccepted`) + https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Frococo-bridge-hub-rpc.polkadot.io#/explorer - BridgeHubWococo (see + `bridgeRococoMessages.MessagesReceived`) + https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fwococo-bridge-hub-rpc.polkadot.io#/explorer - Wockmint (see + `xcmpQueue.Success` for `transfer-asset` and `xcmpQueue.Fail` for `ping-via-bridge`) + https://polkadot.js.org/apps/?rpc=wss%3A%2F%2Fwococo-wockmint-rpc.polkadot.io#/explorer - BridgeHubRococo (see + `bridgeWococoMessages.MessagesDelivered`) ## How to test local BridgeHubKusama/BridgeHubPolkadot diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml index 91eb7adc61feeafb4d060e59039a40df999eaa3d..67c3fa37df0480793df8c7a8a0730f34a4533a71 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml @@ -137,6 +137,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -148,6 +149,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/constants.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/constants.rs deleted file mode 100644 index 760bf7fb6d1f9d31d00a5510834e40174c96ddd9..0000000000000000000000000000000000000000 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/constants.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod currency { - use kusama_runtime_constants as constants; - use polkadot_core_primitives::Balance; - - /// The existential deposit. Set to 1/10 of its parent Relay Chain. - pub const EXISTENTIAL_DEPOSIT: Balance = constants::currency::EXISTENTIAL_DEPOSIT / 10; - - pub const UNITS: Balance = constants::currency::UNITS; - pub const CENTS: Balance = constants::currency::CENTS; - pub const MILLICENTS: Balance = constants::currency::MILLICENTS; - - pub const fn deposit(items: u32, bytes: u32) -> Balance { - // map to 1/100 of what the kusama relay chain charges (v9020) - constants::currency::deposit(items, bytes) / 100 - } -} - -/// Fee-related. -pub mod fee { - use frame_support::{ - pallet_prelude::Weight, - weights::{ - constants::ExtrinsicBaseWeight, FeePolynomial, WeightToFeeCoefficient, - WeightToFeeCoefficients, WeightToFeePolynomial, - }, - }; - use polkadot_core_primitives::Balance; - use smallvec::smallvec; - pub use sp_runtime::Perbill; - - /// Handles converting a weight scalar to a fee value, based on the scale and granularity of the - /// node's balance type. - /// - /// This should typically create a mapping between the following ranges: - /// - [0, MAXIMUM_BLOCK_WEIGHT] - /// - [Balance::min, Balance::max] - /// - /// Yet, it can be used for any other sort of change to weight-fee. Some examples being: - /// - Setting it to `0` will essentially disable the weight fee. - /// - Setting it to `1` will cause the literal `#[weight = x]` values to be charged. - pub struct WeightToFee; - impl frame_support::weights::WeightToFee for WeightToFee { - type Balance = Balance; - - fn weight_to_fee(weight: &Weight) -> Self::Balance { - let time_poly: FeePolynomial = RefTimeToFee::polynomial().into(); - let proof_poly: FeePolynomial = ProofSizeToFee::polynomial().into(); - - // Take the maximum instead of the sum to charge by the more scarce resource. - time_poly.eval(weight.ref_time()).max(proof_poly.eval(weight.proof_size())) - } - } - - /// Maps the reference time component of `Weight` to a fee. - pub struct RefTimeToFee; - impl WeightToFeePolynomial for RefTimeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // in Kusama, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Bridge Hub, we map to 1/10 of that, or 1/100 CENT - let p = super::currency::CENTS; - let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } - - /// Maps the proof size component of `Weight` to a fee. - pub struct ProofSizeToFee; - impl WeightToFeePolynomial for ProofSizeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // Map 10kb proof to 1 CENT. - let p = super::currency::CENTS; - let q = 10_000; - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } -} - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs index 044ff845fe66d438c7c177aa90303b1847c5aae2..54b15e6b327b51eefefcb22e9987aac287401192 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs @@ -22,7 +22,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod weights; pub mod xcm_config; @@ -41,7 +40,6 @@ use sp_std::prelude::*; use sp_version::NativeVersion; use sp_version::RuntimeVersion; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, @@ -69,8 +67,10 @@ use polkadot_runtime_common::{BlockHashCount, SlowAdjustingFeeUpdate}; use weights::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight}; use parachains_common::{ - impls::DealWithFees, AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, - AVERAGE_ON_INITIALIZE_RATIO, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, + impls::DealWithFees, + kusama::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, + HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; // XCM Imports diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/tests/tests.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/tests/tests.rs index 5418e36bd120fbef6bbe520e520686613bf65974..893524e12f66230f5de2a48966344b8b93f08489 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/tests/tests.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/tests/tests.rs @@ -15,12 +15,12 @@ // along with Cumulus. If not, see . pub use bridge_hub_kusama_runtime::{ - constants::fee::WeightToFee, xcm_config::XcmConfig, AllPalletsWithoutSystem, Balances, - ExistentialDeposit, ParachainSystem, PolkadotXcm, Runtime, RuntimeEvent, SessionKeys, + xcm_config::XcmConfig, AllPalletsWithoutSystem, Balances, ExistentialDeposit, ParachainSystem, + PolkadotXcm, Runtime, RuntimeEvent, SessionKeys, }; use codec::Decode; use frame_support::parameter_types; -use parachains_common::{AccountId, AuraId}; +use parachains_common::{kusama::fee::WeightToFee, AccountId, AuraId}; const ALICE: [u8; 32] = [1u8; 32]; diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml index 52a866b7097149da2be98bf32403bbb4877f0292..77923ee74f89ed495af94ac560668186f55ee414 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml @@ -137,6 +137,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -148,6 +149,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/constants.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/constants.rs deleted file mode 100644 index 3bab7bd1eb338e94e62517cc84d8488ae4bc50c9..0000000000000000000000000000000000000000 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/constants.rs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -pub mod currency { - use polkadot_core_primitives::Balance; - use polkadot_runtime_constants as constants; - - /// The existential deposit. Set to 1/10 of its parent Relay Chain. - pub const EXISTENTIAL_DEPOSIT: Balance = constants::currency::EXISTENTIAL_DEPOSIT / 10; - - pub const UNITS: Balance = constants::currency::UNITS; - pub const CENTS: Balance = constants::currency::CENTS; - pub const MILLICENTS: Balance = constants::currency::MILLICENTS; - - pub const fn deposit(items: u32, bytes: u32) -> Balance { - // 1/100 of Polkadot - constants::currency::deposit(items, bytes) / 100 - } -} - -/// Fee-related. -pub mod fee { - use frame_support::{ - pallet_prelude::Weight, - weights::{ - constants::ExtrinsicBaseWeight, FeePolynomial, WeightToFeeCoefficient, - WeightToFeeCoefficients, WeightToFeePolynomial, - }, - }; - use polkadot_core_primitives::Balance; - use smallvec::smallvec; - pub use sp_runtime::Perbill; - - /// Handles converting a weight scalar to a fee value, based on the scale and granularity of the - /// node's balance type. - /// - /// This should typically create a mapping between the following ranges: - /// - [0, MAXIMUM_BLOCK_WEIGHT] - /// - [Balance::min, Balance::max] - /// - /// Yet, it can be used for any other sort of change to weight-fee. Some examples being: - /// - Setting it to `0` will essentially disable the weight fee. - /// - Setting it to `1` will cause the literal `#[weight = x]` values to be charged. - pub struct WeightToFee; - impl frame_support::weights::WeightToFee for WeightToFee { - type Balance = Balance; - - fn weight_to_fee(weight: &Weight) -> Self::Balance { - let time_poly: FeePolynomial = RefTimeToFee::polynomial().into(); - let proof_poly: FeePolynomial = ProofSizeToFee::polynomial().into(); - - // Take the maximum instead of the sum to charge by the more scarce resource. - time_poly.eval(weight.ref_time()).max(proof_poly.eval(weight.proof_size())) - } - } - - /// Maps the reference time component of `Weight` to a fee. - pub struct RefTimeToFee; - impl WeightToFeePolynomial for RefTimeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // in Polkadot, extrinsic base weight (smallest non-zero weight) is mapped to 1/10 CENT: - // in Bridge Hub, we map to 1/10 of that, or 1/100 CENT - let p = super::currency::CENTS; - let q = 100 * Balance::from(ExtrinsicBaseWeight::get().ref_time()); - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } - - /// Maps the proof size component of `Weight` to a fee. - pub struct ProofSizeToFee; - impl WeightToFeePolynomial for ProofSizeToFee { - type Balance = Balance; - fn polynomial() -> WeightToFeeCoefficients { - // Map 10kb proof to 1 CENT. - let p = super::currency::CENTS; - let q = 10_000; - - smallvec![WeightToFeeCoefficient { - degree: 1, - negative: false, - coeff_frac: Perbill::from_rational(p % q, q), - coeff_integer: p / q, - }] - } - } -} - -/// Consensus-related. -pub mod consensus { - /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included - /// into the relay chain. - pub const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; - /// How many parachain blocks are processed by the relay chain per parent. Limits the - /// number of blocks authored per slot. - pub const BLOCK_PROCESSING_VELOCITY: u32 = 1; - /// Relay chain slot duration, in milliseconds. - pub const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; -} diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs index f735858a93400bdb33ea59bcb8ccadd8bff2f658..dbfdc249a3cd7aad3cb5482f4f4a5e3a5493e0a0 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs @@ -22,7 +22,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod weights; pub mod xcm_config; @@ -41,7 +40,6 @@ use sp_std::prelude::*; use sp_version::NativeVersion; use sp_version::RuntimeVersion; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, @@ -70,8 +68,10 @@ use polkadot_runtime_common::{BlockHashCount, SlowAdjustingFeeUpdate}; use weights::{BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight}; use parachains_common::{ - impls::DealWithFees, AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, - AVERAGE_ON_INITIALIZE_RATIO, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, + impls::DealWithFees, + polkadot::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, + HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; // XCM Imports use xcm::latest::prelude::BodyId; diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/tests/tests.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/tests/tests.rs index 03b23cdd7ac221d8af10459e73c64d0a5f477215..0be87bd46facfc079cda4a1ea4cf6b7a34114d75 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/tests/tests.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/tests/tests.rs @@ -15,12 +15,12 @@ // along with Cumulus. If not, see . pub use bridge_hub_polkadot_runtime::{ - constants::fee::WeightToFee, xcm_config::XcmConfig, AllPalletsWithoutSystem, Balances, - ExistentialDeposit, ParachainSystem, PolkadotXcm, Runtime, RuntimeEvent, SessionKeys, + xcm_config::XcmConfig, AllPalletsWithoutSystem, Balances, ExistentialDeposit, ParachainSystem, + PolkadotXcm, Runtime, RuntimeEvent, SessionKeys, }; use codec::Decode; use frame_support::parameter_types; -use parachains_common::{AccountId, AuraId}; +use parachains_common::{polkadot::fee::WeightToFee, AccountId, AuraId}; const ALICE: [u8; 32] = [1u8; 32]; diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml index d65fb9b72cd1cdcb263eb006ff82326cb71a28c2..70c62d1a34fd9fb6f375f456ad5729b52af8e19d 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml @@ -72,26 +72,26 @@ parachain-info = { path = "../../../pallets/parachain-info", default-features = parachains-common = { path = "../../../common", default-features = false } # Bridges -bp-bridge-hub-rococo = { path = "../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } -bp-bridge-hub-wococo = { path = "../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } -bp-header-chain = { path = "../../../../bridges/primitives/header-chain", default-features = false } -bp-messages = { path = "../../../../bridges/primitives/messages", default-features = false } -bp-parachains = { path = "../../../../bridges/primitives/parachains", default-features = false } -bp-polkadot-core = { path = "../../../../bridges/primitives/polkadot-core", default-features = false } -bp-relayers = { path = "../../../../bridges/primitives/relayers", default-features = false } -bp-runtime = { path = "../../../../bridges/primitives/runtime", default-features = false } -bp-rococo = { path = "../../../../bridges/primitives/chain-rococo", default-features = false } -bp-wococo = { path = "../../../../bridges/primitives/chain-wococo", default-features = false } -pallet-bridge-grandpa = { path = "../../../../bridges/modules/grandpa", default-features = false } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages", default-features = false } -pallet-bridge-parachains = { path = "../../../../bridges/modules/parachains", default-features = false } -pallet-bridge-relayers = { path = "../../../../bridges/modules/relayers", default-features = false } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", default-features = false } +bp-bridge-hub-rococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } +bp-bridge-hub-wococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } +bp-header-chain = { path = "../../../../../bridges/primitives/header-chain", default-features = false } +bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false } +bp-parachains = { path = "../../../../../bridges/primitives/parachains", default-features = false } +bp-polkadot-core = { path = "../../../../../bridges/primitives/polkadot-core", default-features = false } +bp-relayers = { path = "../../../../../bridges/primitives/relayers", default-features = false } +bp-runtime = { path = "../../../../../bridges/primitives/runtime", default-features = false } +bp-rococo = { path = "../../../../../bridges/primitives/chain-rococo", default-features = false } +bp-wococo = { path = "../../../../../bridges/primitives/chain-wococo", default-features = false } +pallet-bridge-grandpa = { path = "../../../../../bridges/modules/grandpa", default-features = false } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false } +pallet-bridge-parachains = { path = "../../../../../bridges/modules/parachains", default-features = false } +pallet-bridge-relayers = { path = "../../../../../bridges/modules/relayers", default-features = false } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", default-features = false } [dev-dependencies] static_assertions = "1.1" bridge-hub-test-utils = { path = "../test-utils" } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", features = ["integrity-test"] } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", features = ["integrity-test"] } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } [features] @@ -173,6 +173,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -188,6 +189,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index a872e2327303c14f1361a949c1fb4036828c2ced..dbdc2133ec8881deca3600ef29fc551e83f0a45b 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -24,11 +24,9 @@ include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); pub mod bridge_hub_rococo_config; pub mod bridge_hub_wococo_config; -pub mod constants; mod weights; pub mod xcm_config; -use constants::{consensus::*, currency::*}; use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; use sp_api::impl_runtime_apis; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; @@ -79,7 +77,6 @@ use crate::{ BridgeRefundBridgeHubRococoMessages, OnBridgeHubWococoBlobDispatcher, WithBridgeHubRococoMessageBridge, }, - constants::fee::WeightToFee, xcm_config::XcmRouter, }; use bridge_runtime_common::{ @@ -87,8 +84,10 @@ use bridge_runtime_common::{ messages_xcm_extension::{XcmAsPlainPayload, XcmBlobMessageDispatch}, }; use parachains_common::{ - impls::DealWithFees, AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, - AVERAGE_ON_INITIALIZE_RATIO, HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, + impls::DealWithFees, + rococo::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, Balance, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, + HOURS, MAXIMUM_BLOCK_WEIGHT, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; use xcm_executor::XcmExecutor; @@ -1071,7 +1070,7 @@ impl_runtime_apis! { ) -> (bridge_hub_rococo_config::FromWococoBridgeHubMessagesProof, Weight) { use cumulus_primitives_core::XcmpMessageSource; assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(42.into()); + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(42.into()); prepare_message_proof_from_parachain::< Runtime, BridgeGrandpaWococoInstance, @@ -1114,7 +1113,7 @@ impl_runtime_apis! { ) -> (bridge_hub_wococo_config::FromRococoBridgeHubMessagesProof, Weight) { use cumulus_primitives_core::XcmpMessageSource; assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(42.into()); + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(42.into()); prepare_message_proof_from_parachain::< Runtime, BridgeGrandpaRococoInstance, @@ -1244,55 +1243,67 @@ cumulus_pallet_parachain_system::register_validate_block! { #[cfg(test)] mod tests { use super::*; - use bp_runtime::TransactionEra; - use bridge_hub_test_utils::test_header; use codec::Encode; - - pub type TestBlockHeader = - sp_runtime::generic::Header; + use sp_runtime::{ + generic::Era, + traits::{SignedExtension, Zero}, + }; #[test] fn ensure_signed_extension_definition_is_compatible_with_relay() { - let payload: SignedExtra = ( - frame_system::CheckNonZeroSender::new(), - frame_system::CheckSpecVersion::new(), - frame_system::CheckTxVersion::new(), - frame_system::CheckGenesis::new(), - frame_system::CheckEra::from(sp_runtime::generic::Era::Immortal), - frame_system::CheckNonce::from(10), - frame_system::CheckWeight::new(), - pallet_transaction_payment::ChargeTransactionPayment::from(10), - BridgeRejectObsoleteHeadersAndMessages {}, - ( - BridgeRefundBridgeHubRococoMessages::default(), - BridgeRefundBridgeHubWococoMessages::default(), - ), - ); - - { - use bp_bridge_hub_rococo::BridgeHubSignedExtension; - let bhr_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( - 10, - 10, - TransactionEra::Immortal, - test_header::(1).hash(), - 10, - 10, + use bp_polkadot_core::SuffixedCommonSignedExtensionExt; + + sp_io::TestExternalities::default().execute_with(|| { + frame_system::BlockHash::::insert(BlockNumber::zero(), Hash::default()); + let payload: SignedExtra = ( + frame_system::CheckNonZeroSender::new(), + frame_system::CheckSpecVersion::new(), + frame_system::CheckTxVersion::new(), + frame_system::CheckGenesis::new(), + frame_system::CheckEra::from(Era::Immortal), + frame_system::CheckNonce::from(10), + frame_system::CheckWeight::new(), + pallet_transaction_payment::ChargeTransactionPayment::from(10), + BridgeRejectObsoleteHeadersAndMessages, + ( + BridgeRefundBridgeHubRococoMessages::default(), + BridgeRefundBridgeHubWococoMessages::default(), + ), ); - assert_eq!(payload.encode(), bhr_indirect_payload.encode()); - } - { - use bp_bridge_hub_wococo::BridgeHubSignedExtension; - let bhw_indirect_payload = bp_bridge_hub_wococo::SignedExtension::from_params( - 10, - 10, - TransactionEra::Immortal, - test_header::(1).hash(), - 10, - 10, - ); - assert_eq!(payload.encode(), bhw_indirect_payload.encode()); - } + { + let bhr_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( + VERSION.spec_version, + VERSION.transaction_version, + bp_runtime::TransactionEra::Immortal, + System::block_hash(BlockNumber::zero()), + 10, + 10, + (((), ()), ((), ())), + ); + assert_eq!(payload.encode(), bhr_indirect_payload.encode()); + assert_eq!( + payload.additional_signed().unwrap().encode(), + bhr_indirect_payload.additional_signed().unwrap().encode() + ) + } + + { + let bhw_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( + VERSION.spec_version, + VERSION.transaction_version, + bp_runtime::TransactionEra::Immortal, + System::block_hash(BlockNumber::zero()), + 10, + 10, + (((), ()), ((), ())), + ); + assert_eq!(payload.encode(), bhw_indirect_payload.encode()); + assert_eq!( + payload.additional_signed().unwrap().encode(), + bhw_indirect_payload.additional_signed().unwrap().encode() + ) + } + }); } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/tests.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/tests.rs index 77e7e0382d313ac1715c866b493e2f3804fa022e..e5fe67f2a8e5b29a46adcf2a133ce9e956df9d5b 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/tests.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/tests/tests.rs @@ -19,7 +19,6 @@ use bp_polkadot_core::Signature; use bridge_hub_rococo_runtime::{ bridge_hub_rococo_config, bridge_hub_wococo_config, - constants::fee::WeightToFee, xcm_config::{RelayNetwork, XcmConfig}, AllPalletsWithoutSystem, BridgeRejectObsoleteHeadersAndMessages, DeliveryRewardInBalance, Executive, ExistentialDeposit, ParachainSystem, PolkadotXcm, RequiredStakeForStakeAndSlash, @@ -28,7 +27,7 @@ use bridge_hub_rococo_runtime::{ use codec::{Decode, Encode}; use frame_support::parameter_types; use frame_system::pallet_prelude::HeaderFor; -use parachains_common::{AccountId, AuraId, Balance}; +use parachains_common::{rococo::fee::WeightToFee, AccountId, AuraId, Balance}; use sp_keyring::AccountKeyring::Alice; use sp_runtime::{ generic::{Era, SignedPayload}, diff --git a/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml index 73678e8f91a363a3c6f5ccbcfb86d77b50543428..d56c32d8afabb3c40eb861f028c5f954889cf60d 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" authors.workspace = true edition.workspace = true description = "Utils for BridgeHub testing" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } @@ -42,20 +41,20 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false} # Bridges -bp-bridge-hub-rococo = { path = "../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } -bp-bridge-hub-wococo = { path = "../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } -bp-header-chain = { path = "../../../../bridges/primitives/header-chain", default-features = false } -bp-messages = { path = "../../../../bridges/primitives/messages", default-features = false } -bp-parachains = { path = "../../../../bridges/primitives/parachains", default-features = false } -bp-polkadot-core = { path = "../../../../bridges/primitives/polkadot-core", default-features = false } -bp-relayers = { path = "../../../../bridges/primitives/relayers", default-features = false } -bp-runtime = { path = "../../../../bridges/primitives/runtime", default-features = false } -bp-test-utils = { path = "../../../../bridges/primitives/test-utils", default-features = false } -pallet-bridge-grandpa = { path = "../../../../bridges/modules/grandpa", default-features = false } -pallet-bridge-parachains = { path = "../../../../bridges/modules/parachains", default-features = false } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages", default-features = false } -pallet-bridge-relayers = { path = "../../../../bridges/modules/relayers", default-features = false } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", default-features = false } +bp-bridge-hub-rococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } +bp-bridge-hub-wococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } +bp-header-chain = { path = "../../../../../bridges/primitives/header-chain", default-features = false } +bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false } +bp-parachains = { path = "../../../../../bridges/primitives/parachains", default-features = false } +bp-polkadot-core = { path = "../../../../../bridges/primitives/polkadot-core", default-features = false } +bp-relayers = { path = "../../../../../bridges/primitives/relayers", default-features = false } +bp-runtime = { path = "../../../../../bridges/primitives/runtime", default-features = false } +bp-test-utils = { path = "../../../../../bridges/primitives/test-utils", default-features = false } +pallet-bridge-grandpa = { path = "../../../../../bridges/modules/grandpa", default-features = false } +pallet-bridge-parachains = { path = "../../../../../bridges/modules/parachains", default-features = false } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false } +pallet-bridge-relayers = { path = "../../../../../bridges/modules/relayers", default-features = false } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml b/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml index 1f17a0ef0e69ee4837ccf48b2d53aad54808c2d3..6a5806d3b90199726e0ee3e1d6d34dfa2d6578fb 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml @@ -72,6 +72,7 @@ cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-f cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } cumulus-primitives-utility = { path = "../../../../primitives/utility", default-features = false } pallet-collator-selection = { path = "../../../../pallets/collator-selection", default-features = false } +pallet-collective-content = { path = "../../../pallets/collective-content", default-features = false } parachain-info = { path = "../../../pallets/parachain-info", default-features = false } parachains-common = { path = "../../../common", default-features = false } @@ -87,6 +88,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -94,6 +96,7 @@ runtime-benchmarks = [ "pallet-alliance/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-collator-selection/runtime-benchmarks", + "pallet-collective-content/runtime-benchmarks", "pallet-collective/runtime-benchmarks", "pallet-core-fellowship/runtime-benchmarks", "pallet-multisig/runtime-benchmarks", @@ -106,6 +109,7 @@ runtime-benchmarks = [ "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", @@ -127,6 +131,7 @@ try-runtime = [ "pallet-authorship/try-runtime", "pallet-balances/try-runtime", "pallet-collator-selection/try-runtime", + "pallet-collective-content/try-runtime", "pallet-collective/try-runtime", "pallet-core-fellowship/try-runtime", "pallet-multisig/try-runtime", @@ -168,6 +173,7 @@ std = [ "pallet-authorship/std", "pallet-balances/std", "pallet-collator-selection/std", + "pallet-collective-content/std", "pallet-collective/std", "pallet-core-fellowship/std", "pallet-multisig/std", diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/mod.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..b055ffc8abf1337e1260156cd149fe21b0573ba0 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/mod.rs @@ -0,0 +1,255 @@ +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The Ambassador Program. +//! +//! The module defines the following on-chain functionality of the Ambassador Program: +//! +//! - Managed set of program members, where every member has a [rank](ranks) +//! (via [AmbassadorCollective](pallet_ranked_collective)). +//! - Referendum functionality for the program members to propose, vote on, and execute +//! proposals on behalf of the members of a certain [rank](Origin) +//! (via [AmbassadorReferenda](pallet_referenda)). +//! - Managed content (charter, announcements) (via [pallet_collective_content]). +//! - Promotion and demotion periods, register of members' activity, and rank based salaries +//! (via [AmbassadorCore](pallet_core_fellowship)). +//! - Members' salaries (via [AmbassadorSalary](pallet_salary), requiring a member to be +//! imported or inducted into [AmbassadorCore](pallet_core_fellowship)). + +pub mod origins; +mod tracks; + +use super::*; +use crate::xcm_config::{DotAssetHub, FellowshipAdminBodyId}; +use frame_support::traits::{EitherOf, MapSuccess, TryMapSuccess}; +pub use origins::pallet_origins as pallet_ambassador_origins; +use origins::pallet_origins::{ + EnsureAmbassadorsVoice, EnsureAmbassadorsVoiceFrom, EnsureHeadAmbassadorsVoice, Origin, +}; +use parachains_common::polkadot::account; +use sp_core::ConstU128; +use sp_runtime::traits::{CheckedReduceBy, ConstU16, ConvertToValue, Replace}; +use xcm::prelude::*; +use xcm_builder::{AliasesIntoAccountId32, PayOverXcm}; + +/// The Ambassador Program's member ranks. +pub mod ranks { + use pallet_ranked_collective::Rank; + + #[allow(dead_code)] + pub const CANDIDATE: Rank = 0; + pub const AMBASSADOR_TIER_1: Rank = 1; + pub const AMBASSADOR_TIER_2: Rank = 2; + pub const SENIOR_AMBASSADOR_TIER_3: Rank = 3; + pub const SENIOR_AMBASSADOR_TIER_4: Rank = 4; + pub const HEAD_AMBASSADOR_TIER_5: Rank = 5; + pub const HEAD_AMBASSADOR_TIER_6: Rank = 6; + pub const HEAD_AMBASSADOR_TIER_7: Rank = 7; + pub const MASTER_AMBASSADOR_TIER_8: Rank = 8; + pub const MASTER_AMBASSADOR_TIER_9: Rank = 9; +} + +impl pallet_ambassador_origins::Config for Runtime {} + +pub type AmbassadorCollectiveInstance = pallet_ranked_collective::Instance2; + +/// Demotion is by any of: +/// - Root can promote arbitrarily. +/// - the FellowshipAdmin origin (i.e. token holder referendum); +/// - a senior members vote by the rank two above the current rank. +pub type DemoteOrigin = EitherOf< + frame_system::EnsureRootWithSuccess>, + EitherOf< + MapSuccess< + EnsureXcm>, + Replace>, + >, + TryMapSuccess< + EnsureAmbassadorsVoiceFrom>, + CheckedReduceBy>, + >, + >, +>; + +/// Promotion and approval (rank-retention) is by any of: +/// - Root can promote arbitrarily. +/// - the FellowshipAdmin origin (i.e. token holder referendum); +/// - a senior members vote by the rank two above the new/current rank. +/// - a member of rank `5` or above can add a candidate (rank `0`). +pub type PromoteOrigin = EitherOf< + DemoteOrigin, + TryMapSuccess< + pallet_ranked_collective::EnsureMember< + Runtime, + AmbassadorCollectiveInstance, + { ranks::HEAD_AMBASSADOR_TIER_5 }, + >, + Replace>, + >, +>; + +impl pallet_ranked_collective::Config for Runtime { + type WeightInfo = weights::pallet_ranked_collective_ambassador_collective::WeightInfo; + type RuntimeEvent = RuntimeEvent; + type PromoteOrigin = PromoteOrigin; + type DemoteOrigin = DemoteOrigin; + type Polls = AmbassadorReferenda; + type MinRankOfClass = sp_runtime::traits::Identity; + type VoteWeight = pallet_ranked_collective::Linear; +} + +parameter_types! { + pub const AlarmInterval: BlockNumber = 1; + pub const SubmissionDeposit: Balance = 0; + pub const UndecidingTimeout: BlockNumber = 7 * DAYS; + // The Ambassador Referenda pallet account, used as a temporarily place to deposit a slashed imbalance before teleport to the treasury. + pub AmbassadorPalletAccount: AccountId = account::AMBASSADOR_REFERENDA_PALLET_ID.into_account_truncating(); +} + +pub type AmbassadorReferendaInstance = pallet_referenda::Instance2; + +impl pallet_referenda::Config for Runtime { + type WeightInfo = weights::pallet_referenda_ambassador_referenda::WeightInfo; + type RuntimeCall = RuntimeCall; + type RuntimeEvent = RuntimeEvent; + type Scheduler = Scheduler; + type Currency = Balances; + // A proposal can be submitted by a member of the Ambassador Program of + // [ranks::SENIOR_AMBASSADOR_TIER_3] rank or higher. + type SubmitOrigin = pallet_ranked_collective::EnsureMember< + Runtime, + AmbassadorCollectiveInstance, + { ranks::SENIOR_AMBASSADOR_TIER_3 }, + >; + type CancelOrigin = EitherOf, EnsureHeadAmbassadorsVoice>; + type KillOrigin = EitherOf, EnsureHeadAmbassadorsVoice>; + type Slash = ToParentTreasury; + type Votes = pallet_ranked_collective::Votes; + type Tally = pallet_ranked_collective::TallyOf; + type SubmissionDeposit = SubmissionDeposit; + type MaxQueued = ConstU32<20>; + type UndecidingTimeout = UndecidingTimeout; + type AlarmInterval = AlarmInterval; + type Tracks = tracks::TracksInfo; + type Preimages = Preimage; +} + +parameter_types! { + pub const AnnouncementLifetime: BlockNumber = 180 * DAYS; + pub const MaxAnnouncements: u32 = 50; +} + +pub type AmbassadorContentInstance = pallet_collective_content::Instance1; + +impl pallet_collective_content::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type CharterOrigin = EitherOf, EnsureHeadAmbassadorsVoice>; + type AnnouncementLifetime = AnnouncementLifetime; + // An announcement can be submitted by a Senior Ambassador member or an ambassador plurality + // voice taken via referendum. + type AnnouncementOrigin = EitherOfDiverse< + pallet_ranked_collective::EnsureMember< + Runtime, + AmbassadorCollectiveInstance, + { ranks::SENIOR_AMBASSADOR_TIER_3 }, + >, + EnsureAmbassadorsVoice, + >; + type MaxAnnouncements = MaxAnnouncements; + type WeightInfo = weights::pallet_collective_content::WeightInfo; +} + +pub type AmbassadorCoreInstance = pallet_core_fellowship::Instance2; + +impl pallet_core_fellowship::Config for Runtime { + type WeightInfo = weights::pallet_core_fellowship_ambassador_core::WeightInfo; + type RuntimeEvent = RuntimeEvent; + type Members = pallet_ranked_collective::Pallet; + type Balance = Balance; + // Parameters are set by any of: + // - Root; + // - the FellowshipAdmin origin (i.e. token holder referendum); + // - a vote among all Head Ambassadors. + type ParamsOrigin = EitherOfDiverse< + EnsureXcm>, + EnsureHeadAmbassadorsVoice, + >; + // Induction (creating a candidate) is by any of: + // - Root; + // - the FellowshipAdmin origin (i.e. token holder referendum); + // - a single Head Ambassador; + // - a vote among all senior members. + type InductOrigin = EitherOfDiverse< + EnsureXcm>, + EitherOfDiverse< + pallet_ranked_collective::EnsureMember< + Runtime, + AmbassadorCollectiveInstance, + { ranks::HEAD_AMBASSADOR_TIER_5 }, + >, + EnsureAmbassadorsVoiceFrom>, + >, + >; + type ApproveOrigin = PromoteOrigin; + type PromoteOrigin = PromoteOrigin; + type EvidenceSize = ConstU32<65536>; +} + +pub type AmbassadorSalaryInstance = pallet_salary::Instance2; + +parameter_types! { + // The interior location on AssetHub for the paying account. This is the Ambassador Salary + // pallet instance (which sits at index 74). This sovereign account will need funding. + pub AmbassadorSalaryLocation: InteriorMultiLocation = PalletInstance(74).into(); +} + +/// [`PayOverXcm`] setup to pay the Ambassador salary on the AssetHub in DOT. +pub type AmbassadorSalaryPaymaster = PayOverXcm< + AmbassadorSalaryLocation, + crate::xcm_config::XcmRouter, + crate::PolkadotXcm, + ConstU32<{ 6 * HOURS }>, + AccountId, + (), + ConvertToValue, + AliasesIntoAccountId32<(), AccountId>, +>; + +impl pallet_salary::Config for Runtime { + type WeightInfo = weights::pallet_salary_ambassador_salary::WeightInfo; + type RuntimeEvent = RuntimeEvent; + + #[cfg(not(feature = "runtime-benchmarks"))] + type Paymaster = AmbassadorSalaryPaymaster; + #[cfg(feature = "runtime-benchmarks")] + type Paymaster = crate::impls::benchmarks::PayWithEnsure< + AmbassadorSalaryPaymaster, + crate::impls::benchmarks::OpenHrmpChannel>, + >; + type Members = pallet_ranked_collective::Pallet; + + #[cfg(not(feature = "runtime-benchmarks"))] + type Salary = pallet_core_fellowship::Pallet; + #[cfg(feature = "runtime-benchmarks")] + type Salary = frame_support::traits::tokens::ConvertRank< + crate::impls::benchmarks::RankToSalary, + >; + // 15 days to register for a salary payment. + type RegistrationPeriod = ConstU32<{ 15 * DAYS }>; + // 15 days to claim the salary payment. + type PayoutPeriod = ConstU32<{ 15 * DAYS }>; + // Total monthly salary budget. + type Budget = ConstU128<{ 10_000 * DOLLARS }>; +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/origins.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/origins.rs new file mode 100644 index 0000000000000000000000000000000000000000..3ce8a6b9e5d1bf7f70dda395f1998fd66d20cbeb --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/origins.rs @@ -0,0 +1,135 @@ +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The Ambassador Program's origins. + +#[frame_support::pallet] +pub mod pallet_origins { + use crate::ambassador::ranks; + use frame_support::pallet_prelude::*; + use pallet_ranked_collective::Rank; + + #[pallet::pallet] + pub struct Pallet(PhantomData); + + /// The pallet configuration trait. + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[derive(PartialEq, Eq, Clone, MaxEncodedLen, Encode, Decode, TypeInfo, RuntimeDebug)] + #[pallet::origin] + pub enum Origin { + /// Plurality voice of the [ranks::AMBASSADOR_TIER_1] members or above given via + /// referendum. + Ambassadors, + /// Plurality voice of the [ranks::AMBASSADOR_TIER_2] members or above given via + /// referendum. + AmbassadorsTier2, + /// Plurality voice of the [ranks::SENIOR_AMBASSADOR_TIER_3] members or above given via + /// referendum. + SeniorAmbassadors, + /// Plurality voice of the [ranks::SENIOR_AMBASSADOR_TIER_4] members or above given via + /// referendum. + SeniorAmbassadorsTier4, + /// Plurality voice of the [ranks::HEAD_AMBASSADOR_TIER_5] members or above given via + /// referendum. + HeadAmbassadors, + /// Plurality voice of the [ranks::HEAD_AMBASSADOR_TIER_6] members or above given via + /// referendum. + HeadAmbassadorsTier6, + /// Plurality voice of the [ranks::HEAD_AMBASSADOR_TIER_7] members or above given via + /// referendum. + HeadAmbassadorsTier7, + /// Plurality voice of the [ranks::MASTER_AMBASSADOR_TIER_8] members or above given via + /// referendum. + MasterAmbassadors, + /// Plurality voice of the [ranks::MASTER_AMBASSADOR_TIER_9] members or above given via + /// referendum. + MasterAmbassadorsTier9, + } + + impl Origin { + /// Returns the rank that the origin `self` speaks for, or `None` if it doesn't speak for + /// any. + pub fn as_voice(&self) -> Option { + Some(match &self { + Origin::Ambassadors => ranks::AMBASSADOR_TIER_1, + Origin::AmbassadorsTier2 => ranks::AMBASSADOR_TIER_2, + Origin::SeniorAmbassadors => ranks::SENIOR_AMBASSADOR_TIER_3, + Origin::SeniorAmbassadorsTier4 => ranks::SENIOR_AMBASSADOR_TIER_4, + Origin::HeadAmbassadors => ranks::HEAD_AMBASSADOR_TIER_5, + Origin::HeadAmbassadorsTier6 => ranks::HEAD_AMBASSADOR_TIER_6, + Origin::HeadAmbassadorsTier7 => ranks::HEAD_AMBASSADOR_TIER_7, + Origin::MasterAmbassadors => ranks::MASTER_AMBASSADOR_TIER_8, + Origin::MasterAmbassadorsTier9 => ranks::MASTER_AMBASSADOR_TIER_9, + }) + } + } + + /// Implementation of the [EnsureOrigin] trait for the [Origin::HeadAmbassadors] origin. + pub struct EnsureHeadAmbassadorsVoice; + impl> + From> EnsureOrigin for EnsureHeadAmbassadorsVoice { + type Success = (); + fn try_origin(o: O) -> Result { + o.into().and_then(|o| match o { + Origin::HeadAmbassadors => Ok(()), + r => Err(O::from(r)), + }) + } + + #[cfg(feature = "runtime-benchmarks")] + fn try_successful_origin() -> Result { + Ok(O::from(Origin::HeadAmbassadors)) + } + } + + /// Implementation of the [EnsureOrigin] trait for the plurality voice [Origin]s + /// from a given rank `R` with the success result of the corresponding [Rank]. + pub struct EnsureAmbassadorsVoiceFrom(PhantomData); + impl, O: Into> + From> EnsureOrigin + for EnsureAmbassadorsVoiceFrom + { + type Success = Rank; + fn try_origin(o: O) -> Result { + o.into().and_then(|o| match Origin::as_voice(&o) { + Some(r) if r >= R::get() => Ok(r), + _ => Err(O::from(o)), + }) + } + + #[cfg(feature = "runtime-benchmarks")] + fn try_successful_origin() -> Result { + ranks::MASTER_AMBASSADOR_TIER_9 + .ge(&R::get()) + .then(|| O::from(Origin::MasterAmbassadorsTier9)) + .ok_or(()) + } + } + + /// Implementation of the [EnsureOrigin] trait for the plurality voice [Origin]s with the + /// success result of the corresponding [Rank]. + pub struct EnsureAmbassadorsVoice; + impl> + From> EnsureOrigin for EnsureAmbassadorsVoice { + type Success = Rank; + fn try_origin(o: O) -> Result { + o.into().and_then(|o| Origin::as_voice(&o).ok_or(O::from(o))) + } + + #[cfg(feature = "runtime-benchmarks")] + fn try_successful_origin() -> Result { + Ok(O::from(Origin::MasterAmbassadorsTier9)) + } + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/tracks.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/tracks.rs new file mode 100644 index 0000000000000000000000000000000000000000..073d8e6ee362ad97f9fa66c612fca5e1a8123ab2 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/ambassador/tracks.rs @@ -0,0 +1,282 @@ +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The Ambassador Program's referenda voting tracks. + +use super::Origin; +use crate::{Balance, BlockNumber, RuntimeOrigin, DAYS, DOLLARS, HOURS}; +use sp_runtime::Perbill; + +/// Referendum `TrackId` type. +pub type TrackId = u16; + +/// Referendum track IDs. +pub mod constants { + use super::TrackId; + + pub const AMBASSADOR_TIER_1: TrackId = 1; + pub const AMBASSADOR_TIER_2: TrackId = 2; + pub const SENIOR_AMBASSADOR_TIER_3: TrackId = 3; + pub const SENIOR_AMBASSADOR_TIER_4: TrackId = 4; + pub const HEAD_AMBASSADOR_TIER_5: TrackId = 5; + pub const HEAD_AMBASSADOR_TIER_6: TrackId = 6; + pub const HEAD_AMBASSADOR_TIER_7: TrackId = 7; + pub const MASTER_AMBASSADOR_TIER_8: TrackId = 8; + pub const MASTER_AMBASSADOR_TIER_9: TrackId = 9; +} + +/// The type implementing the [`pallet_referenda::TracksInfo`] trait for referenda pallet. +pub struct TracksInfo; + +/// Information on the voting tracks. +impl pallet_referenda::TracksInfo for TracksInfo { + type Id = TrackId; + + type RuntimeOrigin = ::PalletsOrigin; + + /// Return the array of available tracks and their information. + fn tracks() -> &'static [(Self::Id, pallet_referenda::TrackInfo)] { + static DATA: [(TrackId, pallet_referenda::TrackInfo); 9] = [ + ( + constants::AMBASSADOR_TIER_1, + pallet_referenda::TrackInfo { + name: "ambassador tier 1", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::AMBASSADOR_TIER_2, + pallet_referenda::TrackInfo { + name: "ambassador tier 2", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::SENIOR_AMBASSADOR_TIER_3, + pallet_referenda::TrackInfo { + name: "senior ambassador tier 3", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::SENIOR_AMBASSADOR_TIER_4, + pallet_referenda::TrackInfo { + name: "senior ambassador tier 4", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::HEAD_AMBASSADOR_TIER_5, + pallet_referenda::TrackInfo { + name: "head ambassador tier 5", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::HEAD_AMBASSADOR_TIER_6, + pallet_referenda::TrackInfo { + name: "head ambassador tier 6", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::HEAD_AMBASSADOR_TIER_7, + pallet_referenda::TrackInfo { + name: "head ambassador tier 7", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::MASTER_AMBASSADOR_TIER_8, + pallet_referenda::TrackInfo { + name: "master ambassador tier 8", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ( + constants::MASTER_AMBASSADOR_TIER_9, + pallet_referenda::TrackInfo { + name: "master ambassador tier 9", + max_deciding: 10, + decision_deposit: 5 * DOLLARS, + prepare_period: 24 * HOURS, + decision_period: 7 * DAYS, + confirm_period: 24 * HOURS, + min_enactment_period: 1 * HOURS, + min_approval: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(50), + ceil: Perbill::from_percent(100), + }, + min_support: pallet_referenda::Curve::LinearDecreasing { + length: Perbill::from_percent(100), + floor: Perbill::from_percent(10), + ceil: Perbill::from_percent(50), + }, + }, + ), + ]; + &DATA[..] + } + + /// Determine the voting track for the given `origin`. + fn track_for(id: &Self::RuntimeOrigin) -> Result { + #[cfg(feature = "runtime-benchmarks")] + { + // For benchmarks, we enable a root origin. + // It is important that this is not available in production! + let root: Self::RuntimeOrigin = frame_system::RawOrigin::Root.into(); + if &root == id { + return Ok(constants::MASTER_AMBASSADOR_TIER_9) + } + } + + match Origin::try_from(id.clone()) { + Ok(Origin::Ambassadors) => Ok(constants::AMBASSADOR_TIER_1), + Ok(Origin::AmbassadorsTier2) => Ok(constants::AMBASSADOR_TIER_2), + Ok(Origin::SeniorAmbassadors) => Ok(constants::SENIOR_AMBASSADOR_TIER_3), + Ok(Origin::SeniorAmbassadorsTier4) => Ok(constants::SENIOR_AMBASSADOR_TIER_4), + Ok(Origin::HeadAmbassadors) => Ok(constants::HEAD_AMBASSADOR_TIER_5), + Ok(Origin::HeadAmbassadorsTier6) => Ok(constants::HEAD_AMBASSADOR_TIER_6), + Ok(Origin::HeadAmbassadorsTier7) => Ok(constants::HEAD_AMBASSADOR_TIER_7), + Ok(Origin::MasterAmbassadors) => Ok(constants::MASTER_AMBASSADOR_TIER_8), + Ok(Origin::MasterAmbassadorsTier9) => Ok(constants::MASTER_AMBASSADOR_TIER_9), + _ => Err(()), + } + } +} + +// implements [`frame_support::traits::Get`] for [`TracksInfo`] +pallet_referenda::impl_tracksinfo_get!(TracksInfo, Balance, BlockNumber); diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/fellowship/mod.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/fellowship/mod.rs index 9b86753312951536734f7a3e8c2987d08b27e9cc..2a2757ea5cebc28a1f85bbc28d50e0c523b54d90 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/fellowship/mod.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/fellowship/mod.rs @@ -20,11 +20,12 @@ pub(crate) mod migration; mod origins; mod tracks; use crate::{ - constants, impls::ToParentTreasury, weights, AccountId, Balance, Balances, FellowshipReferenda, - GovernanceLocation, PolkadotTreasuryAccount, Preimage, Runtime, RuntimeCall, RuntimeEvent, - RuntimeOrigin, Scheduler, DAYS, + impls::ToParentTreasury, + weights, + xcm_config::{FellowshipAdminBodyId, UsdtAssetHub}, + AccountId, Balance, Balances, FellowshipReferenda, GovernanceLocation, PolkadotTreasuryAccount, + Preimage, Runtime, RuntimeCall, RuntimeEvent, RuntimeOrigin, Scheduler, DAYS, }; -use cumulus_primitives_core::Junction::GeneralIndex; use frame_support::{ parameter_types, traits::{EitherOf, EitherOfDiverse, MapSuccess, OriginTrait, TryWithMorphedArg}, @@ -36,11 +37,10 @@ pub use origins::{ }; use pallet_ranked_collective::EnsureOfRank; use pallet_xcm::{EnsureXcm, IsVoiceOfBody}; -use polkadot_runtime_constants::{time::HOURS, xcm::body::FELLOWSHIP_ADMIN_INDEX}; +use parachains_common::{polkadot::account, HOURS}; use sp_core::{ConstU128, ConstU32}; use sp_runtime::traits::{AccountIdConversion, ConstU16, ConvertToValue, Replace, TakeFirst}; -use xcm::latest::BodyId; -use xcm_builder::{AliasesIntoAccountId32, LocatableAssetId, PayOverXcm}; +use xcm_builder::{AliasesIntoAccountId32, PayOverXcm}; #[cfg(feature = "runtime-benchmarks")] use crate::impls::benchmarks::{OpenHrmpChannel, PayWithEnsure}; @@ -62,8 +62,7 @@ pub mod ranks { parameter_types! { // Referenda pallet account, used to temporarily deposit slashed imbalance before teleporting. - pub ReferendaPalletAccount: AccountId = constants::account::REFERENDA_PALLET_ID.into_account_truncating(); - pub const FellowshipAdminBodyId: BodyId = BodyId::Index(FELLOWSHIP_ADMIN_INDEX); + pub ReferendaPalletAccount: AccountId = account::REFERENDA_PALLET_ID.into_account_truncating(); } impl pallet_fellowship_origins::Config for Runtime {} @@ -71,7 +70,7 @@ impl pallet_fellowship_origins::Config for Runtime {} pub type FellowshipReferendaInstance = pallet_referenda::Instance1; impl pallet_referenda::Config for Runtime { - type WeightInfo = weights::pallet_referenda::WeightInfo; + type WeightInfo = weights::pallet_referenda_fellowship_referenda::WeightInfo; type RuntimeCall = RuntimeCall; type RuntimeEvent = RuntimeEvent; type Scheduler = Scheduler; @@ -106,7 +105,7 @@ impl pallet_referenda::Config for Runtime { pub type FellowshipCollectiveInstance = pallet_ranked_collective::Instance1; impl pallet_ranked_collective::Config for Runtime { - type WeightInfo = weights::pallet_ranked_collective::WeightInfo; + type WeightInfo = weights::pallet_ranked_collective_fellowship_collective::WeightInfo; type RuntimeEvent = RuntimeEvent; #[cfg(not(feature = "runtime-benchmarks"))] @@ -138,7 +137,7 @@ impl pallet_ranked_collective::Config for Runtime pub type FellowshipCoreInstance = pallet_core_fellowship::Instance1; impl pallet_core_fellowship::Config for Runtime { - type WeightInfo = weights::pallet_core_fellowship::WeightInfo; + type WeightInfo = weights::pallet_core_fellowship_fellowship_core::WeightInfo; type RuntimeEvent = RuntimeEvent; type Members = pallet_ranked_collective::Pallet; type Balance = Balance; @@ -196,12 +195,6 @@ pub type FellowshipSalaryInstance = pallet_salary::Instance1; use xcm::prelude::*; parameter_types! { - pub AssetHub: MultiLocation = (Parent, Parachain(1000)).into(); - pub AssetHubUsdtId: AssetId = (PalletInstance(50), GeneralIndex(1984)).into(); - pub UsdtAsset: LocatableAssetId = LocatableAssetId { - location: AssetHub::get(), - asset_id: AssetHubUsdtId::get(), - }; // The interior location on AssetHub for the paying account. This is the Fellowship Salary // pallet instance (which sits at index 64). This sovereign account will need funding. pub Interior: InteriorMultiLocation = PalletInstance(64).into(); @@ -217,12 +210,12 @@ pub type FellowshipSalaryPaymaster = PayOverXcm< ConstU32<{ 6 * HOURS }>, AccountId, (), - ConvertToValue, + ConvertToValue, AliasesIntoAccountId32<(), AccountId>, >; impl pallet_salary::Config for Runtime { - type WeightInfo = weights::pallet_salary::WeightInfo; + type WeightInfo = weights::pallet_salary_fellowship_salary::WeightInfo; type RuntimeEvent = RuntimeEvent; #[cfg(not(feature = "runtime-benchmarks"))] diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs index c970d82cfe50d358ebe765c1c4472bb7aaab6f7f..9f4c2a6a4c94a5d28fce09dbe800b2d7226723cf 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs @@ -184,7 +184,7 @@ pub mod benchmarks { impl> EnsureSuccessful for OpenHrmpChannel { fn ensure_successful() { if let ChannelStatus::Closed = ParachainSystem::get_channel_status(I::get().into()) { - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(I::get().into()) + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(I::get().into()) } } } diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs index 5033a2d8beb1369f96dfc017cf524ce08e3c3467..d833c75d470cb75467c00c59b01d9d5c1f309b4b 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs @@ -20,7 +20,7 @@ //! //! ### Governance //! -//! As a common good parachain, Collectives defers its governance (namely, its `Root` origin), to +//! As a system parachain, Collectives defers its governance (namely, its `Root` origin), to //! its Relay Chain parent, Polkadot. //! //! ### Collator Selection @@ -36,12 +36,13 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; +pub mod ambassador; pub mod impls; mod weights; pub mod xcm_config; // Fellowship configurations. pub mod fellowship; +pub use ambassador::pallet_ambassador_origins; use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; use fellowship::{ @@ -64,12 +65,14 @@ use sp_version::NativeVersion; use sp_version::RuntimeVersion; use codec::{Decode, Encode, MaxEncodedLen}; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, parameter_types, - traits::{ConstBool, ConstU16, ConstU32, ConstU64, ConstU8, EitherOfDiverse, InstanceFilter}, + traits::{ + fungible::HoldConsideration, ConstBool, ConstU16, ConstU32, ConstU64, ConstU8, + EitherOfDiverse, InstanceFilter, LinearStoragePrice, + }, weights::{ConstantMultiplier, Weight}, PalletId, }; @@ -79,7 +82,9 @@ use frame_system::{ }; pub use parachains_common as common; use parachains_common::{ - impls::DealWithFees, AccountId, AuraId, Balance, BlockNumber, Hash, Header, Nonce, Signature, + impls::DealWithFees, + polkadot::{account::*, consensus::*, currency::*, fee::WeightToFee}, + AccountId, AuraId, Balance, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, DAYS, HOURS, MAXIMUM_BLOCK_WEIGHT, MINUTES, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; @@ -209,7 +214,7 @@ impl pallet_balances::Config for Runtime { type ReserveIdentifier = [u8; 8]; type RuntimeHoldReason = RuntimeHoldReason; type FreezeIdentifier = (); - type MaxHolds = ConstU32<0>; + type MaxHolds = ConstU32<1>; type MaxFreezes = ConstU32<0>; } @@ -289,6 +294,8 @@ pub enum ProxyType { Alliance, /// Fellowship proxy. Allows calls related to the Fellowship. Fellowship, + /// Ambassador proxy. Allows calls related to the Ambassador Program. + Ambassador, } impl Default for ProxyType { fn default() -> Self { @@ -323,6 +330,18 @@ impl InstanceFilter for ProxyType { c, RuntimeCall::FellowshipCollective { .. } | RuntimeCall::FellowshipReferenda { .. } | + RuntimeCall::FellowshipCore { .. } | + RuntimeCall::FellowshipSalary { .. } | + RuntimeCall::Utility { .. } | + RuntimeCall::Multisig { .. } + ), + ProxyType::Ambassador => matches!( + c, + RuntimeCall::AmbassadorCollective { .. } | + RuntimeCall::AmbassadorReferenda { .. } | + RuntimeCall::AmbassadorContent { .. } | + RuntimeCall::AmbassadorCore { .. } | + RuntimeCall::AmbassadorSalary { .. } | RuntimeCall::Utility { .. } | RuntimeCall::Multisig { .. } ), @@ -484,8 +503,8 @@ parameter_types! { pub const AllyDeposit: Balance = 1_000 * UNITS; // 1,000 DOT bond to join as an Ally // The Alliance pallet account, used as a temporary place to deposit a slashed imbalance // before the teleport to the Treasury. - pub AlliancePalletAccount: AccountId = constants::account::ALLIANCE_PALLET_ID.into_account_truncating(); - pub PolkadotTreasuryAccount: AccountId = constants::account::POLKADOT_TREASURY_PALLET_ID.into_account_truncating(); + pub AlliancePalletAccount: AccountId = ALLIANCE_PALLET_ID.into_account_truncating(); + pub PolkadotTreasuryAccount: AccountId = POLKADOT_TREASURY_PALLET_ID.into_account_truncating(); // The number of blocks a member must wait between giving a retirement notice and retiring. // Supposed to be greater than time required to `kick_member` with alliance motion. pub const AllianceRetirementPeriod: BlockNumber = (90 * DAYS) + ALLIANCE_MOTION_DURATION; @@ -545,6 +564,7 @@ impl pallet_scheduler::Config for Runtime { parameter_types! { pub const PreimageBaseDeposit: Balance = deposit(2, 64); pub const PreimageByteDeposit: Balance = deposit(0, 1); + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -552,8 +572,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } // Create the runtime by composing the FRAME pallets that were previously configured. @@ -589,7 +613,7 @@ construct_runtime!( Utility: pallet_utility::{Pallet, Call, Event} = 40, Multisig: pallet_multisig::{Pallet, Call, Storage, Event} = 41, Proxy: pallet_proxy::{Pallet, Call, Storage, Event} = 42, - Preimage: pallet_preimage::{Pallet, Call, Storage, Event} = 43, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 43, Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event} = 44, // The main stage. @@ -608,6 +632,14 @@ construct_runtime!( FellowshipCore: pallet_core_fellowship::::{Pallet, Call, Storage, Event} = 63, // pub type FellowshipSalaryInstance = pallet_salary::Instance1; FellowshipSalary: pallet_salary::::{Pallet, Call, Storage, Event} = 64, + + // Ambassador Program. + AmbassadorCollective: pallet_ranked_collective::::{Pallet, Call, Storage, Event} = 70, + AmbassadorReferenda: pallet_referenda::::{Pallet, Call, Storage, Event} = 71, + AmbassadorOrigins: pallet_ambassador_origins::{Origin} = 72, + AmbassadorCore: pallet_core_fellowship::::{Pallet, Call, Storage, Event} = 73, + AmbassadorSalary: pallet_salary::::{Pallet, Call, Storage, Event} = 74, + AmbassadorContent: pallet_collective_content::::{Pallet, Call, Storage, Event} = 75, } ); @@ -676,6 +708,11 @@ mod benches { [pallet_ranked_collective, FellowshipCollective] [pallet_core_fellowship, FellowshipCore] [pallet_salary, FellowshipSalary] + [pallet_referenda, AmbassadorReferenda] + [pallet_ranked_collective, AmbassadorCollective] + [pallet_collective_content, AmbassadorContent] + [pallet_core_fellowship, AmbassadorCore] + [pallet_salary, AmbassadorSalary] ); } diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/mod.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/mod.rs index 9ddf53792ea4f56fd7460f697d8ec4835dfdaa06..b5a3a892f79a56f034b60cbb3c4e142a5ff6ef16 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/mod.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/mod.rs @@ -22,13 +22,18 @@ pub mod pallet_alliance; pub mod pallet_balances; pub mod pallet_collator_selection; pub mod pallet_collective; -pub mod pallet_core_fellowship; +pub mod pallet_collective_content; +pub mod pallet_core_fellowship_ambassador_core; +pub mod pallet_core_fellowship_fellowship_core; pub mod pallet_multisig; pub mod pallet_preimage; pub mod pallet_proxy; -pub mod pallet_ranked_collective; -pub mod pallet_referenda; -pub mod pallet_salary; +pub mod pallet_ranked_collective_ambassador_collective; +pub mod pallet_ranked_collective_fellowship_collective; +pub mod pallet_referenda_ambassador_referenda; +pub mod pallet_referenda_fellowship_referenda; +pub mod pallet_salary_ambassador_salary; +pub mod pallet_salary_fellowship_salary; pub mod pallet_scheduler; pub mod pallet_session; pub mod pallet_timestamp; diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_collective_content.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_collective_content.rs new file mode 100644 index 0000000000000000000000000000000000000000..e66907b9453060a29b5e6635fa03a31e0dc782ad --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_collective_content.rs @@ -0,0 +1,94 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_collective_content` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-08-18, STEPS: `10`, REPEAT: `3`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 + +// Executed Command: +// ./target/debug/polkadot-parachain +// benchmark +// pallet +// --chain=collectives-polkadot-dev +// --steps=10 +// --repeat=3 +// --pallet=pallet_collective_content +// --extrinsic=* +// --wasm-execution=compiled +// --heap-pages=4096 +// --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_collective_content`. +pub struct WeightInfo(PhantomData); +impl pallet_collective_content::WeightInfo for WeightInfo { + /// Storage: `AmbassadorContent::Charter` (r:0 w:1) + /// Proof: `AmbassadorContent::Charter` (`max_values`: Some(1), `max_size`: Some(70), added: 565, mode: `MaxEncodedLen`) + fn set_charter() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 99_000_000 picoseconds. + Weight::from_parts(99_000_000, 0) + .saturating_add(Weight::from_parts(0, 0)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorContent::AnnouncementsCount` (r:1 w:1) + /// Proof: `AmbassadorContent::AnnouncementsCount` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorContent::NextAnnouncementExpireAt` (r:1 w:1) + /// Proof: `AmbassadorContent::NextAnnouncementExpireAt` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorContent::Announcements` (r:0 w:1) + /// Proof: `AmbassadorContent::Announcements` (`max_values`: None, `max_size`: Some(90), added: 2565, mode: `MaxEncodedLen`) + fn announce() -> Weight { + // Proof Size summary in bytes: + // Measured: `285` + // Estimated: `3507` + // Minimum execution time: 273_000_000 picoseconds. + Weight::from_parts(278_000_000, 0) + .saturating_add(Weight::from_parts(0, 3507)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorContent::Announcements` (r:1 w:1) + /// Proof: `AmbassadorContent::Announcements` (`max_values`: None, `max_size`: Some(90), added: 2565, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorContent::AnnouncementsCount` (r:1 w:1) + /// Proof: `AmbassadorContent::AnnouncementsCount` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + fn remove_announcement() -> Weight { + // Proof Size summary in bytes: + // Measured: `450` + // Estimated: `3555` + // Minimum execution time: 326_000_000 picoseconds. + Weight::from_parts(338_000_000, 0) + .saturating_add(Weight::from_parts(0, 3555)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_ambassador_core.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_ambassador_core.rs new file mode 100644 index 0000000000000000000000000000000000000000..f40940a8b25faa7c441b1ac9b237cb34e671cf17 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_ambassador_core.rs @@ -0,0 +1,223 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_core_fellowship` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 + +// Executed Command: +// target/release/polkadot-parachain +// benchmark +// pallet +// --chain=collectives-polkadot-dev +// --wasm-execution=compiled +// --pallet=pallet_core_fellowship +// --extrinsic=* +// --steps=2 +// --repeat=2 +// --json +// --header=./file_header.txt +// --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_core_fellowship`. +pub struct WeightInfo(PhantomData); +impl pallet_core_fellowship::WeightInfo for WeightInfo { + /// Storage: `AmbassadorCore::Params` (r:0 w:1) + /// Proof: `AmbassadorCore::Params` (`max_values`: Some(1), `max_size`: Some(364), added: 859, mode: `MaxEncodedLen`) + fn set_params() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 11_000_000 picoseconds. + Weight::from_parts(11_000_000, 0) + .saturating_add(Weight::from_parts(0, 0)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Params` (r:1 w:0) + /// Proof: `AmbassadorCore::Params` (`max_values`: Some(1), `max_size`: Some(364), added: 859, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:1 w:0) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:1 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + fn bump_offboard() -> Weight { + // Proof Size summary in bytes: + // Measured: `66011` + // Estimated: `69046` + // Minimum execution time: 96_000_000 picoseconds. + Weight::from_parts(111_000_000, 0) + .saturating_add(Weight::from_parts(0, 69046)) + .saturating_add(T::DbWeight::get().reads(6)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Params` (r:1 w:0) + /// Proof: `AmbassadorCore::Params` (`max_values`: Some(1), `max_size`: Some(364), added: 859, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:1 w:0) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:1 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + fn bump_demote() -> Weight { + // Proof Size summary in bytes: + // Measured: `66121` + // Estimated: `69046` + // Minimum execution time: 99_000_000 picoseconds. + Weight::from_parts(116_000_000, 0) + .saturating_add(Weight::from_parts(0, 69046)) + .saturating_add(T::DbWeight::get().reads(6)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + fn set_active() -> Weight { + // Proof Size summary in bytes: + // Measured: `360` + // Estimated: `3514` + // Minimum execution time: 21_000_000 picoseconds. + Weight::from_parts(22_000_000, 0) + .saturating_add(Weight::from_parts(0, 3514)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:0 w:1) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:0 w:1) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + fn induct() -> Weight { + // Proof Size summary in bytes: + // Measured: `118` + // Estimated: `3514` + // Minimum execution time: 36_000_000 picoseconds. + Weight::from_parts(36_000_000, 0) + .saturating_add(Weight::from_parts(0, 3514)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(5)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Params` (r:1 w:0) + /// Proof: `AmbassadorCore::Params` (`max_values`: Some(1), `max_size`: Some(364), added: 859, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:1 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:0 w:1) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:0 w:1) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + fn promote() -> Weight { + // Proof Size summary in bytes: + // Measured: `65989` + // Estimated: `69046` + // Minimum execution time: 95_000_000 picoseconds. + Weight::from_parts(110_000_000, 0) + .saturating_add(Weight::from_parts(0, 69046)) + .saturating_add(T::DbWeight::get().reads(5)) + .saturating_add(T::DbWeight::get().writes(6)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:0 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + fn offboard() -> Weight { + // Proof Size summary in bytes: + // Measured: `331` + // Estimated: `3514` + // Minimum execution time: 21_000_000 picoseconds. + Weight::from_parts(22_000_000, 0) + .saturating_add(Weight::from_parts(0, 3514)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + fn import() -> Weight { + // Proof Size summary in bytes: + // Measured: `285` + // Estimated: `3514` + // Minimum execution time: 20_000_000 picoseconds. + Weight::from_parts(21_000_000, 0) + .saturating_add(Weight::from_parts(0, 3514)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::Member` (r:1 w:1) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:1 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + fn approve() -> Weight { + // Proof Size summary in bytes: + // Measured: `65967` + // Estimated: `69046` + // Minimum execution time: 78_000_000 picoseconds. + Weight::from_parts(104_000_000, 0) + .saturating_add(Weight::from_parts(0, 69046)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorCore::Member` (r:1 w:0) + /// Proof: `AmbassadorCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCore::MemberEvidence` (r:1 w:1) + /// Proof: `AmbassadorCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) + fn submit_evidence() -> Weight { + // Proof Size summary in bytes: + // Measured: `151` + // Estimated: `69046` + // Minimum execution time: 43_000_000 picoseconds. + Weight::from_parts(44_000_000, 0) + .saturating_add(Weight::from_parts(0, 69046)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_fellowship_core.rs similarity index 87% rename from cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship.rs rename to cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_fellowship_core.rs index d053513b53accc68cc3332ee1e123b6c3a08bd83..434986b03bba84c8ef43250c2192a946de09b286 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_core_fellowship_fellowship_core.rs @@ -17,24 +17,21 @@ //! Autogenerated weights for `pallet_core_fellowship` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-07-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-ynta1nyy-project-238-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot-parachain +// target/release/polkadot-parachain // benchmark // pallet // --chain=collectives-polkadot-dev // --wasm-execution=compiled // --pallet=pallet_core_fellowship -// --no-storage-info -// --no-median-slopes -// --no-min-squares // --extrinsic=* -// --steps=50 -// --repeat=20 +// --steps=2 +// --repeat=2 // --json // --header=./file_header.txt // --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ @@ -56,8 +53,8 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 9_077_000 picoseconds. - Weight::from_parts(9_356_000, 0) + // Minimum execution time: 11_000_000 picoseconds. + Weight::from_parts(12_000_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -75,10 +72,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) fn bump_offboard() -> Weight { // Proof Size summary in bytes: - // Measured: `66111` + // Measured: `66144` // Estimated: `69046` - // Minimum execution time: 128_419_000 picoseconds. - Weight::from_parts(149_318_000, 0) + // Minimum execution time: 109_000_000 picoseconds. + Weight::from_parts(125_000_000, 0) .saturating_add(Weight::from_parts(0, 69046)) .saturating_add(T::DbWeight::get().reads(6)) .saturating_add(T::DbWeight::get().writes(4)) @@ -97,10 +94,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) fn bump_demote() -> Weight { // Proof Size summary in bytes: - // Measured: `66221` + // Measured: `66254` // Estimated: `69046` - // Minimum execution time: 127_629_000 picoseconds. - Weight::from_parts(130_928_000, 0) + // Minimum execution time: 112_000_000 picoseconds. + Weight::from_parts(114_000_000, 0) .saturating_add(Weight::from_parts(0, 69046)) .saturating_add(T::DbWeight::get().reads(6)) .saturating_add(T::DbWeight::get().writes(4)) @@ -111,10 +108,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::Member` (`max_values`: None, `max_size`: Some(49), added: 2524, mode: `MaxEncodedLen`) fn set_active() -> Weight { // Proof Size summary in bytes: - // Measured: `460` + // Measured: `493` // Estimated: `3514` - // Minimum execution time: 18_655_000 picoseconds. - Weight::from_parts(19_331_000, 0) + // Minimum execution time: 22_000_000 picoseconds. + Weight::from_parts(27_000_000, 0) .saturating_add(Weight::from_parts(0, 3514)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -131,10 +128,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) fn induct() -> Weight { // Proof Size summary in bytes: - // Measured: `218` + // Measured: `251` // Estimated: `3514` - // Minimum execution time: 28_764_000 picoseconds. - Weight::from_parts(29_385_000, 0) + // Minimum execution time: 35_000_000 picoseconds. + Weight::from_parts(36_000_000, 0) .saturating_add(Weight::from_parts(0, 3514)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(5)) @@ -155,10 +152,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) fn promote() -> Weight { // Proof Size summary in bytes: - // Measured: `66089` + // Measured: `66122` // Estimated: `69046` - // Minimum execution time: 123_179_000 picoseconds. - Weight::from_parts(125_302_000, 0) + // Minimum execution time: 97_000_000 picoseconds. + Weight::from_parts(129_000_000, 0) .saturating_add(Weight::from_parts(0, 69046)) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(6)) @@ -171,10 +168,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) fn offboard() -> Weight { // Proof Size summary in bytes: - // Measured: `431` + // Measured: `464` // Estimated: `3514` - // Minimum execution time: 19_700_000 picoseconds. - Weight::from_parts(20_319_000, 0) + // Minimum execution time: 22_000_000 picoseconds. + Weight::from_parts(22_000_000, 0) .saturating_add(Weight::from_parts(0, 3514)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(2)) @@ -185,10 +182,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) fn import() -> Weight { // Proof Size summary in bytes: - // Measured: `385` + // Measured: `418` // Estimated: `3514` - // Minimum execution time: 18_048_000 picoseconds. - Weight::from_parts(18_345_000, 0) + // Minimum execution time: 20_000_000 picoseconds. + Weight::from_parts(24_000_000, 0) .saturating_add(Weight::from_parts(0, 3514)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -201,10 +198,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) fn approve() -> Weight { // Proof Size summary in bytes: - // Measured: `66067` + // Measured: `66100` // Estimated: `69046` - // Minimum execution time: 108_578_000 picoseconds. - Weight::from_parts(111_311_000, 0) + // Minimum execution time: 89_000_000 picoseconds. + Weight::from_parts(119_000_000, 0) .saturating_add(Weight::from_parts(0, 69046)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -215,10 +212,10 @@ impl pallet_core_fellowship::WeightInfo for WeightInfo< /// Proof: `FellowshipCore::MemberEvidence` (`max_values`: None, `max_size`: Some(65581), added: 68056, mode: `MaxEncodedLen`) fn submit_evidence() -> Weight { // Proof Size summary in bytes: - // Measured: `151` + // Measured: `184` // Estimated: `69046` - // Minimum execution time: 94_484_000 picoseconds. - Weight::from_parts(97_930_000, 0) + // Minimum execution time: 43_000_000 picoseconds. + Weight::from_parts(52_000_000, 0) .saturating_add(Weight::from_parts(0, 69046)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_preimage.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_preimage.rs index cf2f0ae39da61e05a9816a445742022e6c4ba939..e9f565d9387da08f35419baa196ce7ef77e265be 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_preimage.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_preimage.rs @@ -50,6 +50,21 @@ use core::marker::PhantomData; /// Weight functions for `pallet_preimage`. pub struct WeightInfo(PhantomData); impl pallet_preimage::WeightInfo for WeightInfo { + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_000_000 picoseconds. + Weight::from_parts(2_000_000, 3593) + // Standard Error: 13_720 + .saturating_add(Weight::from_parts(17_309_199, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) + } + /// Storage: `Preimage::StatusFor` (r:1 w:1) /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) /// Storage: `Preimage::PreimageFor` (r:0 w:1) diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_ambassador_collective.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_ambassador_collective.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6372c4b89dc222ca6a5a6acd36306d79ea3ffb2 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_ambassador_collective.rs @@ -0,0 +1,177 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_ranked_collective` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 + +// Executed Command: +// target/release/polkadot-parachain +// benchmark +// pallet +// --chain=collectives-polkadot-dev +// --wasm-execution=compiled +// --pallet=pallet_ranked_collective +// --extrinsic=* +// --steps=2 +// --repeat=2 +// --json +// --header=./file_header.txt +// --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_ranked_collective`. +pub struct WeightInfo(PhantomData); +impl pallet_ranked_collective::WeightInfo for WeightInfo { + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:0 w:1) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:0 w:1) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + fn add_member() -> Weight { + // Proof Size summary in bytes: + // Measured: `42` + // Estimated: `3507` + // Minimum execution time: 21_000_000 picoseconds. + Weight::from_parts(23_000_000, 0) + .saturating_add(Weight::from_parts(0, 3507)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:11 w:11) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:11 w:11) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:11 w:11) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. + fn remove_member(r: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `508 + r * (281 ±0)` + // Estimated: `3519 + r * (2529 ±0)` + // Minimum execution time: 34_000_000 picoseconds. + Weight::from_parts(36_500_000, 0) + .saturating_add(Weight::from_parts(0, 3519)) + // Standard Error: 158_113 + .saturating_add(Weight::from_parts(16_000_000, 0).saturating_mul(r.into())) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().reads((3_u64).saturating_mul(r.into()))) + .saturating_add(T::DbWeight::get().writes(4)) + .saturating_add(T::DbWeight::get().writes((3_u64).saturating_mul(r.into()))) + .saturating_add(Weight::from_parts(0, 2529).saturating_mul(r.into())) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:0 w:1) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:0 w:1) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. + fn promote_member(r: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `210 + r * (17 ±0)` + // Estimated: `3507` + // Minimum execution time: 25_000_000 picoseconds. + Weight::from_parts(26_000_000, 0) + .saturating_add(Weight::from_parts(0, 3507)) + // Standard Error: 180_277 + .saturating_add(Weight::from_parts(650_000, 0).saturating_mul(r.into())) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:1) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:1) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IdToIndex` (r:1 w:1) + /// Proof: `AmbassadorCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::IndexToId` (r:1 w:1) + /// Proof: `AmbassadorCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) + /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. + fn demote_member(r: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `508 + r * (71 ±0)` + // Estimated: `3519` + // Minimum execution time: 34_000_000 picoseconds. + Weight::from_parts(36_500_000, 0) + .saturating_add(Weight::from_parts(0, 3519)) + // Standard Error: 335_410 + .saturating_add(Weight::from_parts(550_000, 0).saturating_mul(r.into())) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Voting` (r:1 w:1) + /// Proof: `AmbassadorCollective::Voting` (`max_values`: None, `max_size`: Some(65), added: 2540, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn vote() -> Weight { + // Proof Size summary in bytes: + // Measured: `566` + // Estimated: `317568` + // Minimum execution time: 57_000_000 picoseconds. + Weight::from_parts(60_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(5)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:0) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::VotingCleanup` (r:1 w:0) + /// Proof: `AmbassadorCollective::VotingCleanup` (`max_values`: None, `max_size`: Some(114), added: 2589, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Voting` (r:100 w:100) + /// Proof: `AmbassadorCollective::Voting` (`max_values`: None, `max_size`: Some(65), added: 2540, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 100]`. + /// The range of component `n` is `[0, 100]`. + fn cleanup_poll(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `209 + n * (52 ±0)` + // Estimated: `4365 + n * (2550 ±0)` + // Minimum execution time: 18_000_000 picoseconds. + Weight::from_parts(18_500_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + // Standard Error: 11_180 + .saturating_add(Weight::from_parts(1_335_000, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2550).saturating_mul(n.into())) + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_fellowship_collective.rs similarity index 82% rename from cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective.rs rename to cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_fellowship_collective.rs index 561edda953b9a75df47a3388245eede3ec78ba51..7515aecbb525a4b4de59bd05e8329166a5515ff1 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_ranked_collective_fellowship_collective.rs @@ -17,24 +17,21 @@ //! Autogenerated weights for `pallet_ranked_collective` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-07-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-ynta1nyy-project-238-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot-parachain +// target/release/polkadot-parachain // benchmark // pallet // --chain=collectives-polkadot-dev // --wasm-execution=compiled // --pallet=pallet_ranked_collective -// --no-storage-info -// --no-median-slopes -// --no-min-squares // --extrinsic=* -// --steps=50 -// --repeat=20 +// --steps=2 +// --repeat=2 // --json // --header=./file_header.txt // --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ @@ -62,8 +59,8 @@ impl pallet_ranked_collective::WeightInfo for WeightInf // Proof Size summary in bytes: // Measured: `142` // Estimated: `3507` - // Minimum execution time: 16_027_000 picoseconds. - Weight::from_parts(16_501_000, 0) + // Minimum execution time: 21_000_000 picoseconds. + Weight::from_parts(22_000_000, 0) .saturating_add(Weight::from_parts(0, 3507)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(4)) @@ -77,15 +74,16 @@ impl pallet_ranked_collective::WeightInfo for WeightInf /// Storage: `FellowshipCollective::IndexToId` (r:11 w:11) /// Proof: `FellowshipCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. fn remove_member(r: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `617 + r * (281 ±0)` + // Measured: `608 + r * (281 ±0)` // Estimated: `3519 + r * (2529 ±0)` - // Minimum execution time: 27_829_000 picoseconds. - Weight::from_parts(30_053_705, 0) + // Minimum execution time: 35_000_000 picoseconds. + Weight::from_parts(36_500_000, 0) .saturating_add(Weight::from_parts(0, 3519)) - // Standard Error: 26_813 - .saturating_add(Weight::from_parts(13_088_861, 0).saturating_mul(r.into())) + // Standard Error: 254_950 + .saturating_add(Weight::from_parts(15_900_000, 0).saturating_mul(r.into())) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().reads((3_u64).saturating_mul(r.into()))) .saturating_add(T::DbWeight::get().writes(4)) @@ -101,15 +99,16 @@ impl pallet_ranked_collective::WeightInfo for WeightInf /// Storage: `FellowshipCollective::IdToIndex` (r:0 w:1) /// Proof: `FellowshipCollective::IdToIndex` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. fn promote_member(r: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `314 + r * (17 ±0)` + // Measured: `310 + r * (17 ±0)` // Estimated: `3507` - // Minimum execution time: 19_762_000 picoseconds. - Weight::from_parts(20_493_905, 0) + // Minimum execution time: 25_000_000 picoseconds. + Weight::from_parts(25_500_000, 0) .saturating_add(Weight::from_parts(0, 3507)) - // Standard Error: 5_519 - .saturating_add(Weight::from_parts(349_033, 0).saturating_mul(r.into())) + // Standard Error: 70_710 + .saturating_add(Weight::from_parts(400_000, 0).saturating_mul(r.into())) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(4)) } @@ -122,15 +121,16 @@ impl pallet_ranked_collective::WeightInfo for WeightInf /// Storage: `FellowshipCollective::IndexToId` (r:1 w:1) /// Proof: `FellowshipCollective::IndexToId` (`max_values`: None, `max_size`: Some(54), added: 2529, mode: `MaxEncodedLen`) /// The range of component `r` is `[0, 10]`. + /// The range of component `r` is `[0, 10]`. fn demote_member(r: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `632 + r * (72 ±0)` + // Measured: `608 + r * (71 ±0)` // Estimated: `3519` - // Minimum execution time: 28_092_000 picoseconds. - Weight::from_parts(30_800_398, 0) + // Minimum execution time: 35_000_000 picoseconds. + Weight::from_parts(37_500_000, 0) .saturating_add(Weight::from_parts(0, 3519)) - // Standard Error: 17_223 - .saturating_add(Weight::from_parts(615_330, 0).saturating_mul(r.into())) + // Standard Error: 150_000 + .saturating_add(Weight::from_parts(350_000, 0).saturating_mul(r.into())) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(4)) } @@ -144,10 +144,10 @@ impl pallet_ranked_collective::WeightInfo for WeightInf /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn vote() -> Weight { // Proof Size summary in bytes: - // Measured: `666` + // Measured: `700` // Estimated: `317568` - // Minimum execution time: 46_255_000 picoseconds. - Weight::from_parts(47_590_000, 0) + // Minimum execution time: 57_000_000 picoseconds. + Weight::from_parts(57_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(4)) @@ -159,18 +159,19 @@ impl pallet_ranked_collective::WeightInfo for WeightInf /// Storage: `FellowshipCollective::Voting` (r:100 w:100) /// Proof: `FellowshipCollective::Voting` (`max_values`: None, `max_size`: Some(65), added: 2540, mode: `MaxEncodedLen`) /// The range of component `n` is `[0, 100]`. + /// The range of component `n` is `[0, 100]`. fn cleanup_poll(n: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `500 + n * (50 ±0)` - // Estimated: `4365 + n * (2540 ±0)` - // Minimum execution time: 14_975_000 picoseconds. - Weight::from_parts(17_408_362, 0) + // Measured: `343 + n * (52 ±0)` + // Estimated: `4365 + n * (2550 ±0)` + // Minimum execution time: 18_000_000 picoseconds. + Weight::from_parts(19_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) - // Standard Error: 3_134 - .saturating_add(Weight::from_parts(1_222_024, 0).saturating_mul(n.into())) + // Standard Error: 25_000 + .saturating_add(Weight::from_parts(1_395_000, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) - .saturating_add(Weight::from_parts(0, 2540).saturating_mul(n.into())) + .saturating_add(Weight::from_parts(0, 2550).saturating_mul(n.into())) } } diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_ambassador_referenda.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_ambassador_referenda.rs new file mode 100644 index 0000000000000000000000000000000000000000..fdc451c5d31ccae8a1195482ef32e99163c22442 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_ambassador_referenda.rs @@ -0,0 +1,536 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_referenda` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 + +// Executed Command: +// target/release/polkadot-parachain +// benchmark +// pallet +// --chain=collectives-polkadot-dev +// --wasm-execution=compiled +// --pallet=pallet_referenda +// --extrinsic=* +// --steps=2 +// --repeat=2 +// --json +// --header=./file_header.txt +// --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_referenda`. +pub struct WeightInfo(PhantomData); +impl pallet_referenda::WeightInfo for WeightInfo { + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::ReferendumCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumCount` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:0 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + fn submit() -> Weight { + // Proof Size summary in bytes: + // Measured: `255` + // Estimated: `159279` + // Minimum execution time: 32_000_000 picoseconds. + Weight::from_parts(34_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn place_decision_deposit_preparing() -> Weight { + // Proof Size summary in bytes: + // Measured: `366` + // Estimated: `317568` + // Minimum execution time: 63_000_000 picoseconds. + Weight::from_parts(68_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:0) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn place_decision_deposit_queued() -> Weight { + // Proof Size summary in bytes: + // Measured: `1165` + // Estimated: `159279` + // Minimum execution time: 97_000_000 picoseconds. + Weight::from_parts(123_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:0) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn place_decision_deposit_not_queued() -> Weight { + // Proof Size summary in bytes: + // Measured: `1173` + // Estimated: `159279` + // Minimum execution time: 104_000_000 picoseconds. + Weight::from_parts(111_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn place_decision_deposit_passing() -> Weight { + // Proof Size summary in bytes: + // Measured: `702` + // Estimated: `317568` + // Minimum execution time: 140_000_000 picoseconds. + Weight::from_parts(150_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(5)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn place_decision_deposit_failing() -> Weight { + // Proof Size summary in bytes: + // Measured: `601` + // Estimated: `317568` + // Minimum execution time: 81_000_000 picoseconds. + Weight::from_parts(82_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(5)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + fn refund_decision_deposit() -> Weight { + // Proof Size summary in bytes: + // Measured: `317` + // Estimated: `4365` + // Minimum execution time: 38_000_000 picoseconds. + Weight::from_parts(38_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + fn refund_submission_deposit() -> Weight { + // Proof Size summary in bytes: + // Measured: `167` + // Estimated: `4365` + // Minimum execution time: 17_000_000 picoseconds. + Weight::from_parts(18_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn cancel() -> Weight { + // Proof Size summary in bytes: + // Measured: `311` + // Estimated: `317568` + // Minimum execution time: 44_000_000 picoseconds. + Weight::from_parts(45_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + /// Storage: `ParachainInfo::ParachainId` (r:1 w:0) + /// Proof: `ParachainInfo::ParachainId` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `PolkadotXcm::SupportedVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SupportedVersion` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::VersionDiscoveryQueue` (r:1 w:1) + /// Proof: `PolkadotXcm::VersionDiscoveryQueue` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::SafeXcmVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SafeXcmVersion` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParachainSystem::HostConfiguration` (r:1 w:0) + /// Proof: `ParachainSystem::HostConfiguration` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParachainSystem::PendingUpwardMessages` (r:1 w:1) + /// Proof: `ParachainSystem::PendingUpwardMessages` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `AmbassadorReferenda::MetadataOf` (r:1 w:0) + /// Proof: `AmbassadorReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) + fn kill() -> Weight { + // Proof Size summary in bytes: + // Measured: `626` + // Estimated: `317568` + // Minimum execution time: 183_000_000 picoseconds. + Weight::from_parts(187_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(11)) + .saturating_add(T::DbWeight::get().writes(6)) + } + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:0) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + fn one_fewer_deciding_queue_empty() -> Weight { + // Proof Size summary in bytes: + // Measured: `140` + // Estimated: `3636` + // Minimum execution time: 12_000_000 picoseconds. + Weight::from_parts(12_000_000, 0) + .saturating_add(Weight::from_parts(0, 3636)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn one_fewer_deciding_failing() -> Weight { + // Proof Size summary in bytes: + // Measured: `1412` + // Estimated: `159279` + // Minimum execution time: 88_000_000 picoseconds. + Weight::from_parts(97_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn one_fewer_deciding_passing() -> Weight { + // Proof Size summary in bytes: + // Measured: `1412` + // Estimated: `159279` + // Minimum execution time: 87_000_000 picoseconds. + Weight::from_parts(92_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:0) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + fn nudge_referendum_requeued_insertion() -> Weight { + // Proof Size summary in bytes: + // Measured: `935` + // Estimated: `4365` + // Minimum execution time: 43_000_000 picoseconds. + Weight::from_parts(46_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:0) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + fn nudge_referendum_requeued_slide() -> Weight { + // Proof Size summary in bytes: + // Measured: `935` + // Estimated: `4365` + // Minimum execution time: 39_000_000 picoseconds. + Weight::from_parts(43_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:0) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + fn nudge_referendum_queued() -> Weight { + // Proof Size summary in bytes: + // Measured: `951` + // Estimated: `4365` + // Minimum execution time: 48_000_000 picoseconds. + Weight::from_parts(50_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:0) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::TrackQueue` (r:1 w:1) + /// Proof: `AmbassadorReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(171), added: 2646, mode: `MaxEncodedLen`) + fn nudge_referendum_not_queued() -> Weight { + // Proof Size summary in bytes: + // Measured: `959` + // Estimated: `4365` + // Minimum execution time: 42_000_000 picoseconds. + Weight::from_parts(48_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_no_deposit() -> Weight { + // Proof Size summary in bytes: + // Measured: `263` + // Estimated: `159279` + // Minimum execution time: 28_000_000 picoseconds. + Weight::from_parts(30_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_preparing() -> Weight { + // Proof Size summary in bytes: + // Measured: `311` + // Estimated: `159279` + // Minimum execution time: 26_000_000 picoseconds. + Weight::from_parts(28_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + fn nudge_referendum_timed_out() -> Weight { + // Proof Size summary in bytes: + // Measured: `208` + // Estimated: `4365` + // Minimum execution time: 19_000_000 picoseconds. + Weight::from_parts(20_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_begin_deciding_failing() -> Weight { + // Proof Size summary in bytes: + // Measured: `546` + // Estimated: `159279` + // Minimum execution time: 42_000_000 picoseconds. + Weight::from_parts(46_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::DecidingCount` (r:1 w:1) + /// Proof: `AmbassadorReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_begin_deciding_passing() -> Weight { + // Proof Size summary in bytes: + // Measured: `647` + // Estimated: `159279` + // Minimum execution time: 87_000_000 picoseconds. + Weight::from_parts(93_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(4)) + .saturating_add(T::DbWeight::get().writes(3)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_begin_confirming() -> Weight { + // Proof Size summary in bytes: + // Measured: `700` + // Estimated: `159279` + // Minimum execution time: 100_000_000 picoseconds. + Weight::from_parts(120_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_end_confirming() -> Weight { + // Proof Size summary in bytes: + // Measured: `683` + // Estimated: `159279` + // Minimum execution time: 90_000_000 picoseconds. + Weight::from_parts(100_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_continue_not_confirming() -> Weight { + // Proof Size summary in bytes: + // Measured: `700` + // Estimated: `159279` + // Minimum execution time: 77_000_000 picoseconds. + Weight::from_parts(82_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_continue_confirming() -> Weight { + // Proof Size summary in bytes: + // Measured: `704` + // Estimated: `159279` + // Minimum execution time: 68_000_000 picoseconds. + Weight::from_parts(77_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:2 w:2) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Lookup` (r:1 w:1) + /// Proof: `Scheduler::Lookup` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`) + fn nudge_referendum_approved() -> Weight { + // Proof Size summary in bytes: + // Measured: `704` + // Estimated: `317568` + // Minimum execution time: 99_000_000 picoseconds. + Weight::from_parts(104_000_000, 0) + .saturating_add(Weight::from_parts(0, 317568)) + .saturating_add(T::DbWeight::get().reads(5)) + .saturating_add(T::DbWeight::get().writes(4)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:1) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::MemberCount` (r:1 w:0) + /// Proof: `AmbassadorCollective::MemberCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) + /// Storage: `Scheduler::Agenda` (r:1 w:1) + /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) + fn nudge_referendum_rejected() -> Weight { + // Proof Size summary in bytes: + // Measured: `700` + // Estimated: `159279` + // Minimum execution time: 87_000_000 picoseconds. + Weight::from_parts(100_000_000, 0) + .saturating_add(Weight::from_parts(0, 159279)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:0) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::MetadataOf` (r:0 w:1) + /// Proof: `AmbassadorReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) + fn set_some_metadata() -> Weight { + // Proof Size summary in bytes: + // Measured: `419` + // Estimated: `4365` + // Minimum execution time: 23_000_000 picoseconds. + Weight::from_parts(25_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorReferenda::ReferendumInfoFor` (r:1 w:0) + /// Proof: `AmbassadorReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorReferenda::MetadataOf` (r:1 w:1) + /// Proof: `AmbassadorReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) + fn clear_metadata() -> Weight { + // Proof Size summary in bytes: + // Measured: `285` + // Estimated: `4365` + // Minimum execution time: 20_000_000 picoseconds. + Weight::from_parts(21_000_000, 0) + .saturating_add(Weight::from_parts(0, 4365)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_fellowship_referenda.rs similarity index 87% rename from cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda.rs rename to cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_fellowship_referenda.rs index 12d92a803cad9be870465264539c438b9a7487e7..5b4aed06899fcca022895488477ffb058ae76399 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_referenda_fellowship_referenda.rs @@ -17,24 +17,21 @@ //! Autogenerated weights for `pallet_referenda` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-07-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-ynta1nyy-project-238-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot-parachain +// target/release/polkadot-parachain // benchmark // pallet // --chain=collectives-polkadot-dev // --wasm-execution=compiled // --pallet=pallet_referenda -// --no-storage-info -// --no-median-slopes -// --no-min-squares // --extrinsic=* -// --steps=50 -// --repeat=20 +// --steps=2 +// --repeat=2 // --json // --header=./file_header.txt // --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ @@ -60,10 +57,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) fn submit() -> Weight { // Proof Size summary in bytes: - // Measured: `355` + // Measured: `389` // Estimated: `159279` - // Minimum execution time: 29_271_000 picoseconds. - Weight::from_parts(30_285_000, 0) + // Minimum execution time: 34_000_000 picoseconds. + Weight::from_parts(36_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(3)) @@ -74,10 +71,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn place_decision_deposit_preparing() -> Weight { // Proof Size summary in bytes: - // Measured: `366` + // Measured: `400` // Estimated: `317568` - // Minimum execution time: 52_128_000 picoseconds. - Weight::from_parts(53_504_000, 0) + // Minimum execution time: 64_000_000 picoseconds. + Weight::from_parts(67_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(3)) @@ -92,10 +89,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn place_decision_deposit_queued() -> Weight { // Proof Size summary in bytes: - // Measured: `2004` + // Measured: `2038` // Estimated: `159279` - // Minimum execution time: 110_018_000 picoseconds. - Weight::from_parts(114_369_000, 0) + // Minimum execution time: 99_000_000 picoseconds. + Weight::from_parts(109_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -110,10 +107,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn place_decision_deposit_not_queued() -> Weight { // Proof Size summary in bytes: - // Measured: `2045` + // Measured: `2079` // Estimated: `159279` - // Minimum execution time: 110_231_000 picoseconds. - Weight::from_parts(114_517_000, 0) + // Minimum execution time: 101_000_000 picoseconds. + Weight::from_parts(111_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -128,10 +125,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn place_decision_deposit_passing() -> Weight { // Proof Size summary in bytes: - // Measured: `802` + // Measured: `836` // Estimated: `317568` - // Minimum execution time: 195_619_000 picoseconds. - Weight::from_parts(207_157_000, 0) + // Minimum execution time: 135_000_000 picoseconds. + Weight::from_parts(153_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(4)) @@ -146,10 +143,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn place_decision_deposit_failing() -> Weight { // Proof Size summary in bytes: - // Measured: `701` + // Measured: `735` // Estimated: `317568` - // Minimum execution time: 64_020_000 picoseconds. - Weight::from_parts(65_463_000, 0) + // Minimum execution time: 78_000_000 picoseconds. + Weight::from_parts(82_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(4)) @@ -158,10 +155,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) fn refund_decision_deposit() -> Weight { // Proof Size summary in bytes: - // Measured: `317` + // Measured: `351` // Estimated: `4365` - // Minimum execution time: 30_701_000 picoseconds. - Weight::from_parts(31_528_000, 0) + // Minimum execution time: 38_000_000 picoseconds. + Weight::from_parts(39_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) @@ -170,10 +167,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) fn refund_submission_deposit() -> Weight { // Proof Size summary in bytes: - // Measured: `167` + // Measured: `201` // Estimated: `4365` - // Minimum execution time: 15_173_000 picoseconds. - Weight::from_parts(15_787_000, 0) + // Minimum execution time: 18_000_000 picoseconds. + Weight::from_parts(19_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) @@ -184,10 +181,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn cancel() -> Weight { // Proof Size summary in bytes: - // Measured: `311` + // Measured: `345` // Estimated: `317568` - // Minimum execution time: 37_886_000 picoseconds. - Weight::from_parts(38_679_000, 0) + // Minimum execution time: 45_000_000 picoseconds. + Weight::from_parts(46_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(3)) @@ -214,10 +211,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) fn kill() -> Weight { // Proof Size summary in bytes: - // Measured: `517` + // Measured: `587` // Estimated: `317568` - // Minimum execution time: 152_111_000 picoseconds. - Weight::from_parts(155_738_000, 0) + // Minimum execution time: 185_000_000 picoseconds. + Weight::from_parts(196_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(11)) .saturating_add(T::DbWeight::get().writes(6)) @@ -228,10 +225,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::DecidingCount` (`max_values`: None, `max_size`: Some(14), added: 2489, mode: `MaxEncodedLen`) fn one_fewer_deciding_queue_empty() -> Weight { // Proof Size summary in bytes: - // Measured: `140` + // Measured: `174` // Estimated: `4277` - // Minimum execution time: 10_712_000 picoseconds. - Weight::from_parts(10_976_000, 0) + // Minimum execution time: 12_000_000 picoseconds. + Weight::from_parts(15_000_000, 0) .saturating_add(Weight::from_parts(0, 4277)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -246,10 +243,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn one_fewer_deciding_failing() -> Weight { // Proof Size summary in bytes: - // Measured: `2418` + // Measured: `2452` // Estimated: `159279` - // Minimum execution time: 97_671_000 picoseconds. - Weight::from_parts(104_911_000, 0) + // Minimum execution time: 82_000_000 picoseconds. + Weight::from_parts(90_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -264,10 +261,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn one_fewer_deciding_passing() -> Weight { // Proof Size summary in bytes: - // Measured: `2418` + // Measured: `2452` // Estimated: `159279` - // Minimum execution time: 104_019_000 picoseconds. - Weight::from_parts(108_208_000, 0) + // Minimum execution time: 91_000_000 picoseconds. + Weight::from_parts(99_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -278,10 +275,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(812), added: 3287, mode: `MaxEncodedLen`) fn nudge_referendum_requeued_insertion() -> Weight { // Proof Size summary in bytes: - // Measured: `1807` + // Measured: `1841` // Estimated: `4365` - // Minimum execution time: 50_199_000 picoseconds. - Weight::from_parts(54_350_000, 0) + // Minimum execution time: 41_000_000 picoseconds. + Weight::from_parts(44_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -292,10 +289,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(812), added: 3287, mode: `MaxEncodedLen`) fn nudge_referendum_requeued_slide() -> Weight { // Proof Size summary in bytes: - // Measured: `1774` + // Measured: `1808` // Estimated: `4365` - // Minimum execution time: 52_459_000 picoseconds. - Weight::from_parts(54_382_000, 0) + // Minimum execution time: 46_000_000 picoseconds. + Weight::from_parts(55_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -308,10 +305,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(812), added: 3287, mode: `MaxEncodedLen`) fn nudge_referendum_queued() -> Weight { // Proof Size summary in bytes: - // Measured: `1790` + // Measured: `1824` // Estimated: `4365` - // Minimum execution time: 57_810_000 picoseconds. - Weight::from_parts(63_690_000, 0) + // Minimum execution time: 49_000_000 picoseconds. + Weight::from_parts(53_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -324,10 +321,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::TrackQueue` (`max_values`: None, `max_size`: Some(812), added: 3287, mode: `MaxEncodedLen`) fn nudge_referendum_not_queued() -> Weight { // Proof Size summary in bytes: - // Measured: `1831` + // Measured: `1865` // Estimated: `4365` - // Minimum execution time: 56_778_000 picoseconds. - Weight::from_parts(59_556_000, 0) + // Minimum execution time: 51_000_000 picoseconds. + Weight::from_parts(54_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -338,10 +335,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_no_deposit() -> Weight { // Proof Size summary in bytes: - // Measured: `263` + // Measured: `297` // Estimated: `159279` - // Minimum execution time: 24_377_000 picoseconds. - Weight::from_parts(27_031_000, 0) + // Minimum execution time: 28_000_000 picoseconds. + Weight::from_parts(30_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(2)) @@ -352,10 +349,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_preparing() -> Weight { // Proof Size summary in bytes: - // Measured: `311` + // Measured: `345` // Estimated: `159279` - // Minimum execution time: 24_717_000 picoseconds. - Weight::from_parts(25_578_000, 0) + // Minimum execution time: 28_000_000 picoseconds. + Weight::from_parts(29_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(2)) @@ -364,10 +361,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::ReferendumInfoFor` (`max_values`: None, `max_size`: Some(900), added: 3375, mode: `MaxEncodedLen`) fn nudge_referendum_timed_out() -> Weight { // Proof Size summary in bytes: - // Measured: `208` + // Measured: `242` // Estimated: `4365` - // Minimum execution time: 17_280_000 picoseconds. - Weight::from_parts(17_845_000, 0) + // Minimum execution time: 20_000_000 picoseconds. + Weight::from_parts(21_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) @@ -382,10 +379,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_begin_deciding_failing() -> Weight { // Proof Size summary in bytes: - // Measured: `646` + // Measured: `680` // Estimated: `159279` - // Minimum execution time: 36_996_000 picoseconds. - Weight::from_parts(37_970_000, 0) + // Minimum execution time: 42_000_000 picoseconds. + Weight::from_parts(47_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -400,10 +397,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_begin_deciding_passing() -> Weight { // Proof Size summary in bytes: - // Measured: `747` + // Measured: `781` // Estimated: `159279` - // Minimum execution time: 91_681_000 picoseconds. - Weight::from_parts(98_640_000, 0) + // Minimum execution time: 90_000_000 picoseconds. + Weight::from_parts(95_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(4)) .saturating_add(T::DbWeight::get().writes(3)) @@ -416,10 +413,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_begin_confirming() -> Weight { // Proof Size summary in bytes: - // Measured: `800` + // Measured: `834` // Estimated: `159279` - // Minimum execution time: 149_940_000 picoseconds. - Weight::from_parts(167_561_000, 0) + // Minimum execution time: 84_000_000 picoseconds. + Weight::from_parts(93_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -432,10 +429,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_end_confirming() -> Weight { // Proof Size summary in bytes: - // Measured: `783` + // Measured: `817` // Estimated: `159279` - // Minimum execution time: 157_443_000 picoseconds. - Weight::from_parts(168_023_000, 0) + // Minimum execution time: 88_000_000 picoseconds. + Weight::from_parts(98_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -448,10 +445,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_continue_not_confirming() -> Weight { // Proof Size summary in bytes: - // Measured: `800` + // Measured: `834` // Estimated: `159279` - // Minimum execution time: 155_539_000 picoseconds. - Weight::from_parts(161_877_000, 0) + // Minimum execution time: 81_000_000 picoseconds. + Weight::from_parts(93_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -464,10 +461,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_continue_confirming() -> Weight { // Proof Size summary in bytes: - // Measured: `804` + // Measured: `838` // Estimated: `159279` - // Minimum execution time: 82_000_000 picoseconds. - Weight::from_parts(87_101_000, 0) + // Minimum execution time: 74_000_000 picoseconds. + Weight::from_parts(77_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -482,10 +479,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Lookup` (`max_values`: None, `max_size`: Some(48), added: 2523, mode: `MaxEncodedLen`) fn nudge_referendum_approved() -> Weight { // Proof Size summary in bytes: - // Measured: `804` + // Measured: `838` // Estimated: `317568` - // Minimum execution time: 154_590_000 picoseconds. - Weight::from_parts(186_418_000, 0) + // Minimum execution time: 105_000_000 picoseconds. + Weight::from_parts(123_000_000, 0) .saturating_add(Weight::from_parts(0, 317568)) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(4)) @@ -498,10 +495,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `Scheduler::Agenda` (`max_values`: None, `max_size`: Some(155814), added: 158289, mode: `MaxEncodedLen`) fn nudge_referendum_rejected() -> Weight { // Proof Size summary in bytes: - // Measured: `800` + // Measured: `834` // Estimated: `159279` - // Minimum execution time: 149_822_000 picoseconds. - Weight::from_parts(164_866_000, 0) + // Minimum execution time: 90_000_000 picoseconds. + Weight::from_parts(100_000_000, 0) .saturating_add(Weight::from_parts(0, 159279)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -514,10 +511,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) fn set_some_metadata() -> Weight { // Proof Size summary in bytes: - // Measured: `386` + // Measured: `453` // Estimated: `4365` - // Minimum execution time: 21_413_000 picoseconds. - Weight::from_parts(21_938_000, 0) + // Minimum execution time: 24_000_000 picoseconds. + Weight::from_parts(24_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -528,10 +525,10 @@ impl pallet_referenda::WeightInfo for WeightInfo { /// Proof: `FellowshipReferenda::MetadataOf` (`max_values`: None, `max_size`: Some(52), added: 2527, mode: `MaxEncodedLen`) fn clear_metadata() -> Weight { // Proof Size summary in bytes: - // Measured: `285` + // Measured: `319` // Estimated: `4365` - // Minimum execution time: 18_927_000 picoseconds. - Weight::from_parts(19_423_000, 0) + // Minimum execution time: 21_000_000 picoseconds. + Weight::from_parts(23_000_000, 0) .saturating_add(Weight::from_parts(0, 4365)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_ambassador_salary.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_ambassador_salary.rs new file mode 100644 index 0000000000000000000000000000000000000000..0522420f2f5172ecc2840b70ecb9c74796bbddc3 --- /dev/null +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_ambassador_salary.rs @@ -0,0 +1,190 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_salary` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 + +// Executed Command: +// target/release/polkadot-parachain +// benchmark +// pallet +// --chain=collectives-polkadot-dev +// --wasm-execution=compiled +// --pallet=pallet_salary +// --extrinsic=* +// --steps=2 +// --repeat=2 +// --json +// --header=./file_header.txt +// --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_salary`. +pub struct WeightInfo(PhantomData); +impl pallet_salary::WeightInfo for WeightInfo { + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + fn init() -> Weight { + // Proof Size summary in bytes: + // Measured: `109` + // Estimated: `1541` + // Minimum execution time: 12_000_000 picoseconds. + Weight::from_parts(14_000_000, 0) + .saturating_add(Weight::from_parts(0, 1541)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + fn bump() -> Weight { + // Proof Size summary in bytes: + // Measured: `191` + // Estimated: `1541` + // Minimum execution time: 15_000_000 picoseconds. + Weight::from_parts(16_000_000, 0) + .saturating_add(Weight::from_parts(0, 1541)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorSalary::Status` (r:1 w:0) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Claimant` (r:1 w:1) + /// Proof: `AmbassadorSalary::Claimant` (`max_values`: None, `max_size`: Some(86), added: 2561, mode: `MaxEncodedLen`) + fn induct() -> Weight { + // Proof Size summary in bytes: + // Measured: `400` + // Estimated: `3551` + // Minimum execution time: 23_000_000 picoseconds. + Weight::from_parts(23_000_000, 0) + .saturating_add(Weight::from_parts(0, 3551)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Claimant` (r:1 w:1) + /// Proof: `AmbassadorSalary::Claimant` (`max_values`: None, `max_size`: Some(86), added: 2561, mode: `MaxEncodedLen`) + fn register() -> Weight { + // Proof Size summary in bytes: + // Measured: `467` + // Estimated: `3551` + // Minimum execution time: 27_000_000 picoseconds. + Weight::from_parts(28_000_000, 0) + .saturating_add(Weight::from_parts(0, 3551)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(2)) + } + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Claimant` (r:1 w:1) + /// Proof: `AmbassadorSalary::Claimant` (`max_values`: None, `max_size`: Some(86), added: 2561, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `ParachainInfo::ParachainId` (r:1 w:0) + /// Proof: `ParachainInfo::ParachainId` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `PolkadotXcm::QueryCounter` (r:1 w:1) + /// Proof: `PolkadotXcm::QueryCounter` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::SupportedVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SupportedVersion` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::VersionDiscoveryQueue` (r:1 w:1) + /// Proof: `PolkadotXcm::VersionDiscoveryQueue` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::SafeXcmVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SafeXcmVersion` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParachainSystem::RelevantMessagingState` (r:1 w:0) + /// Proof: `ParachainSystem::RelevantMessagingState` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `XcmpQueue::OutboundXcmpStatus` (r:1 w:1) + /// Proof: `XcmpQueue::OutboundXcmpStatus` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `XcmpQueue::OutboundXcmpMessages` (r:0 w:1) + /// Proof: `XcmpQueue::OutboundXcmpMessages` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::Queries` (r:0 w:1) + /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn payout() -> Weight { + // Proof Size summary in bytes: + // Measured: `879` + // Estimated: `4344` + // Minimum execution time: 68_000_000 picoseconds. + Weight::from_parts(72_000_000, 0) + .saturating_add(Weight::from_parts(0, 4344)) + .saturating_add(T::DbWeight::get().reads(10)) + .saturating_add(T::DbWeight::get().writes(7)) + } + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Claimant` (r:1 w:1) + /// Proof: `AmbassadorSalary::Claimant` (`max_values`: None, `max_size`: Some(86), added: 2561, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorCollective::Members` (r:1 w:0) + /// Proof: `AmbassadorCollective::Members` (`max_values`: None, `max_size`: Some(42), added: 2517, mode: `MaxEncodedLen`) + /// Storage: `ParachainInfo::ParachainId` (r:1 w:0) + /// Proof: `ParachainInfo::ParachainId` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) + /// Storage: `PolkadotXcm::QueryCounter` (r:1 w:1) + /// Proof: `PolkadotXcm::QueryCounter` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::SupportedVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SupportedVersion` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::VersionDiscoveryQueue` (r:1 w:1) + /// Proof: `PolkadotXcm::VersionDiscoveryQueue` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::SafeXcmVersion` (r:1 w:0) + /// Proof: `PolkadotXcm::SafeXcmVersion` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `ParachainSystem::RelevantMessagingState` (r:1 w:0) + /// Proof: `ParachainSystem::RelevantMessagingState` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `XcmpQueue::OutboundXcmpStatus` (r:1 w:1) + /// Proof: `XcmpQueue::OutboundXcmpStatus` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `XcmpQueue::OutboundXcmpMessages` (r:0 w:1) + /// Proof: `XcmpQueue::OutboundXcmpMessages` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `PolkadotXcm::Queries` (r:0 w:1) + /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn payout_other() -> Weight { + // Proof Size summary in bytes: + // Measured: `879` + // Estimated: `4344` + // Minimum execution time: 69_000_000 picoseconds. + Weight::from_parts(70_000_000, 0) + .saturating_add(Weight::from_parts(0, 4344)) + .saturating_add(T::DbWeight::get().reads(10)) + .saturating_add(T::DbWeight::get().writes(7)) + } + /// Storage: `AmbassadorSalary::Status` (r:1 w:1) + /// Proof: `AmbassadorSalary::Status` (`max_values`: Some(1), `max_size`: Some(56), added: 551, mode: `MaxEncodedLen`) + /// Storage: `AmbassadorSalary::Claimant` (r:1 w:1) + /// Proof: `AmbassadorSalary::Claimant` (`max_values`: None, `max_size`: Some(86), added: 2561, mode: `MaxEncodedLen`) + /// Storage: `PolkadotXcm::Queries` (r:1 w:1) + /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn check_payment() -> Weight { + // Proof Size summary in bytes: + // Measured: `479` + // Estimated: `3944` + // Minimum execution time: 27_000_000 picoseconds. + Weight::from_parts(28_000_000, 0) + .saturating_add(Weight::from_parts(0, 3944)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(3)) + } +} diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_fellowship_salary.rs similarity index 86% rename from cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary.rs rename to cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_fellowship_salary.rs index 3a6825cf6415887dc284555aaad14d52628de58c..9bb7e68d3145c339505f9c6a729eca1217c02537 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/weights/pallet_salary_fellowship_salary.rs @@ -17,24 +17,21 @@ //! Autogenerated weights for `pallet_salary` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-07-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-08-11, STEPS: `2`, REPEAT: `2`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-ynta1nyy-project-238-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 +//! HOSTNAME: `cob`, CPU: `` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("collectives-polkadot-dev")`, DB CACHE: 1024 // Executed Command: -// ./target/production/polkadot-parachain +// target/release/polkadot-parachain // benchmark // pallet // --chain=collectives-polkadot-dev // --wasm-execution=compiled // --pallet=pallet_salary -// --no-storage-info -// --no-median-slopes -// --no-min-squares // --extrinsic=* -// --steps=50 -// --repeat=20 +// --steps=2 +// --repeat=2 // --json // --header=./file_header.txt // --output=./parachains/runtimes/collectives/collectives-polkadot/src/weights/ @@ -56,8 +53,8 @@ impl pallet_salary::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `142` // Estimated: `1541` - // Minimum execution time: 10_579_000 picoseconds. - Weight::from_parts(10_898_000, 0) + // Minimum execution time: 13_000_000 picoseconds. + Weight::from_parts(17_000_000, 0) .saturating_add(Weight::from_parts(0, 1541)) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) @@ -68,8 +65,8 @@ impl pallet_salary::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `224` // Estimated: `1541` - // Minimum execution time: 12_723_000 picoseconds. - Weight::from_parts(13_221_000, 0) + // Minimum execution time: 15_000_000 picoseconds. + Weight::from_parts(18_000_000, 0) .saturating_add(Weight::from_parts(0, 1541)) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) @@ -84,8 +81,8 @@ impl pallet_salary::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `395` // Estimated: `3551` - // Minimum execution time: 18_522_000 picoseconds. - Weight::from_parts(19_120_000, 0) + // Minimum execution time: 22_000_000 picoseconds. + Weight::from_parts(25_000_000, 0) .saturating_add(Weight::from_parts(0, 3551)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(1)) @@ -100,8 +97,8 @@ impl pallet_salary::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `462` // Estimated: `3551` - // Minimum execution time: 22_270_000 picoseconds. - Weight::from_parts(23_325_000, 0) + // Minimum execution time: 26_000_000 picoseconds. + Weight::from_parts(29_000_000, 0) .saturating_add(Weight::from_parts(0, 3551)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(2)) @@ -132,11 +129,11 @@ impl pallet_salary::WeightInfo for WeightInfo { /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) fn payout() -> Weight { // Proof Size summary in bytes: - // Measured: `703` - // Estimated: `4168` - // Minimum execution time: 54_436_000 picoseconds. - Weight::from_parts(56_347_000, 0) - .saturating_add(Weight::from_parts(0, 4168)) + // Measured: `774` + // Estimated: `4239` + // Minimum execution time: 67_000_000 picoseconds. + Weight::from_parts(74_000_000, 0) + .saturating_add(Weight::from_parts(0, 4239)) .saturating_add(T::DbWeight::get().reads(10)) .saturating_add(T::DbWeight::get().writes(7)) } @@ -166,11 +163,11 @@ impl pallet_salary::WeightInfo for WeightInfo { /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) fn payout_other() -> Weight { // Proof Size summary in bytes: - // Measured: `703` - // Estimated: `4168` - // Minimum execution time: 54_140_000 picoseconds. - Weight::from_parts(56_312_000, 0) - .saturating_add(Weight::from_parts(0, 4168)) + // Measured: `774` + // Estimated: `4239` + // Minimum execution time: 66_000_000 picoseconds. + Weight::from_parts(71_000_000, 0) + .saturating_add(Weight::from_parts(0, 4239)) .saturating_add(T::DbWeight::get().reads(10)) .saturating_add(T::DbWeight::get().writes(7)) } @@ -182,11 +179,11 @@ impl pallet_salary::WeightInfo for WeightInfo { /// Proof: `PolkadotXcm::Queries` (`max_values`: None, `max_size`: None, mode: `Measured`) fn check_payment() -> Weight { // Proof Size summary in bytes: - // Measured: `478` - // Estimated: `3943` - // Minimum execution time: 24_650_000 picoseconds. - Weight::from_parts(25_242_000, 0) - .saturating_add(Weight::from_parts(0, 3943)) + // Measured: `512` + // Estimated: `3977` + // Minimum execution time: 26_000_000 picoseconds. + Weight::from_parts(27_000_000, 0) + .saturating_add(Weight::from_parts(0, 3977)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(3)) } diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/xcm_config.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/xcm_config.rs index b4db73e3ab4401da08b38ec36632cc07edce25a0..02e4913fcd9c73063a392699a71e2d09584e8bf6 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/xcm_config.rs @@ -26,15 +26,16 @@ use frame_system::EnsureRoot; use pallet_xcm::XcmPassthrough; use parachains_common::{impls::ToStakingPot, xcm_config::ConcreteNativeAssetFrom}; use polkadot_parachain_primitives::primitives::Sibling; +use polkadot_runtime_constants::xcm::body::FELLOWSHIP_ADMIN_INDEX; use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, CurrencyAdapter, DenyReserveTransferToRelayChain, DenyThenTry, EnsureXcmOrigin, FixedWeightBounds, IsConcrete, - OriginToPluralityVoice, ParentAsSuperuser, ParentIsPreset, RelayChainAsNative, - SiblingParachainAsNative, SiblingParachainConvertsVia, SignedAccountId32AsNative, - SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, - UsingComponents, WithComputedOrigin, WithUniqueTopic, + LocatableAssetId, OriginToPluralityVoice, ParentAsSuperuser, ParentIsPreset, + RelayChainAsNative, SiblingParachainAsNative, SiblingParachainConvertsVia, + SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, + TrailingSetTopicAsId, UsingComponents, WithComputedOrigin, WithUniqueTopic, }; use xcm_executor::{traits::WithOriginFilter, XcmExecutor}; @@ -46,6 +47,17 @@ parameter_types! { X2(GlobalConsensus(RelayNetwork::get().unwrap()), Parachain(ParachainInfo::parachain_id().into())); pub CheckingAccount: AccountId = PolkadotXcm::check_account(); pub const GovernanceLocation: MultiLocation = MultiLocation::parent(); + pub const FellowshipAdminBodyId: BodyId = BodyId::Index(FELLOWSHIP_ADMIN_INDEX); + pub AssetHub: MultiLocation = (Parent, Parachain(1000)).into(); + pub AssetHubUsdtId: AssetId = (PalletInstance(50), GeneralIndex(1984)).into(); + pub UsdtAssetHub: LocatableAssetId = LocatableAssetId { + location: AssetHub::get(), + asset_id: AssetHubUsdtId::get(), + }; + pub DotAssetHub: LocatableAssetId = LocatableAssetId { + location: AssetHub::get(), + asset_id: DotLocation::get().into(), + }; } /// Type for specifying how a `MultiLocation` can be converted into an `AccountId`. This is used diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml b/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml index 2e298e05ec76b8ec79a39caeeb6f74acbb52c997..f6fccc0435425adbcd8d9a067dc4de7db7760a81 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml @@ -138,6 +138,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -151,6 +152,7 @@ runtime-benchmarks = [ "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/README.md b/cumulus/parachains/runtimes/contracts/contracts-rococo/README.md index e4f15ccf92d655d8358ba460a4c7e02ae262a6fd..387bb24bb0e07ef982b89634ae98d7a663c47ef7 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/README.md +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/README.md @@ -33,8 +33,8 @@ There are also different user interfaces and command-line tools you can use to d or interact with contracts: * [Contracts UI](https://paritytech.github.io/contracts-ui/) ‒ a beginner-friendly UI for smart contract developers. -* [polkadot-js](https://polkadot.js.org/apps/) ‒ the go-to expert UI for smart contract developers. -* [cargo-contract](https://github.com/paritytech/cargo-contract) ‒ a CLI tool, ideal for scripting or your terminal workflow. +* [`polkadot-js`](https://polkadot.js.org/apps/) ‒ the go-to expert UI for smart contract developers. +* [`cargo-contract`](https://github.com/paritytech/cargo-contract) ‒ a CLI tool, ideal for scripting or your terminal workflow. If you are looking for a quickstart, we can recommend [ink!'s Guided Tutorial for Beginners](https://docs.substrate.io/tutorials/v3/ink-workshop/pt1/). diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs index 6598fd3fae0a8800fc1e2919de8cff2807da7f44..1c99393d5e52fccf427b40f232d55a833d082c9e 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/contracts.rs @@ -14,8 +14,8 @@ // limitations under the License. use crate::{ - constants::currency::deposit, Balance, Balances, RandomnessCollectiveFlip, Runtime, - RuntimeCall, RuntimeEvent, RuntimeHoldReason, Timestamp, + Balance, Balances, RandomnessCollectiveFlip, Runtime, RuntimeCall, RuntimeEvent, + RuntimeHoldReason, Timestamp, }; use frame_support::{ parameter_types, @@ -28,7 +28,7 @@ use pallet_contracts::{ }; use sp_runtime::Perbill; -pub use parachains_common::AVERAGE_ON_INITIALIZE_RATIO; +pub use parachains_common::{rococo::currency::deposit, AVERAGE_ON_INITIALIZE_RATIO}; // Prints debug output of the `contracts` pallet to stdout if the node is // started with `-lruntime::contracts=debug`. diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs index b5815ab057e6492158f1dc8726ee08f4ebdb3e3d..399ada1be2c739c6ea4931fa99c6a38f3c039dd9 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs @@ -25,7 +25,6 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -pub mod constants; mod contracts; mod weights; mod xcm_config; @@ -45,7 +44,6 @@ use sp_std::prelude::*; use sp_version::NativeVersion; use sp_version::RuntimeVersion; -use constants::{consensus::*, currency::*, fee::WeightToFee}; use frame_support::{ construct_runtime, dispatch::DispatchClass, @@ -57,9 +55,10 @@ use frame_support::{ use frame_system::limits::{BlockLength, BlockWeights}; pub use parachains_common as common; use parachains_common::{ - impls::DealWithFees, AccountId, BlockNumber, Hash, Header, Nonce, Signature, - AVERAGE_ON_INITIALIZE_RATIO, MAXIMUM_BLOCK_WEIGHT, MINUTES, NORMAL_DISPATCH_RATIO, - SLOT_DURATION, + impls::DealWithFees, + rococo::{consensus::*, currency::*, fee::WeightToFee}, + AccountId, BlockNumber, Hash, Header, Nonce, Signature, AVERAGE_ON_INITIALIZE_RATIO, + MAXIMUM_BLOCK_WEIGHT, MINUTES, NORMAL_DISPATCH_RATIO, SLOT_DURATION, }; pub use parachains_common::{AuraId, Balance}; use xcm_config::CollatorSelectionUpdateOrigin; diff --git a/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml b/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml index 0caf00340d3a3ea63549983c92d37e622a88770e..0ffe59b927f9d2435276472e5b5c5e6434b4ca77 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml @@ -16,10 +16,13 @@ frame-system = { path = "../../../../../substrate/frame/system", default-feature frame-system-rpc-runtime-api = { path = "../../../../../substrate/frame/system/rpc/runtime-api", default-features = false} frame-system-benchmarking = { path = "../../../../../substrate/frame/system/benchmarking", default-features = false, optional = true} frame-try-runtime = { path = "../../../../../substrate/frame/try-runtime", default-features = false, optional = true} +pallet-aura = { path = "../../../../../substrate/frame/aura", default-features = false} pallet-glutton = { path = "../../../../../substrate/frame/glutton", default-features = false, optional = true} pallet-sudo = { path = "../../../../../substrate/frame/sudo", default-features = false, optional = true} +pallet-timestamp = { path = "../../../../../substrate/frame/timestamp", default-features = false } sp-api = { path = "../../../../../substrate/primitives/api", default-features = false} sp-block-builder = { path = "../../../../../substrate/primitives/block-builder", default-features = false} +sp-consensus-aura = { path = "../../../../../substrate/primitives/consensus/aura", default-features = false } sp-core = { path = "../../../../../substrate/primitives/core", default-features = false} sp-inherents = { path = "../../../../../substrate/primitives/inherents", default-features = false} sp-offchain = { path = "../../../../../substrate/primitives/offchain", default-features = false} @@ -36,9 +39,11 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false} # Cumulus +cumulus-pallet-aura-ext = { path = "../../../../pallets/aura-ext", default-features = false } cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook",] } cumulus-pallet-xcm = { path = "../../../../pallets/xcm", default-features = false } cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } +cumulus-primitives-timestamp = { path = "../../../../primitives/timestamp", default-features = false } parachain-info = { path = "../../../pallets/parachain-info", default-features = false } parachains-common = { path = "../../../common", default-features = false } @@ -55,15 +60,19 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "pallet-glutton/runtime-benchmarks", "pallet-sudo?/runtime-benchmarks", + "pallet-timestamp/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", ] std = [ "codec/std", + "cumulus-pallet-aura-ext/std", "cumulus-pallet-parachain-system/std", "cumulus-pallet-xcm/std", "cumulus-primitives-core/std", + "cumulus-primitives-timestamp/std", "frame-benchmarking?/std", "frame-executive/std", "frame-support/std", @@ -71,13 +80,16 @@ std = [ "frame-system-rpc-runtime-api/std", "frame-system/std", "frame-try-runtime?/std", + "pallet-aura/std", "pallet-glutton/std", "pallet-sudo/std", + "pallet-timestamp/std", "parachain-info/std", "parachains-common/std", "scale-info/std", "sp-api/std", "sp-block-builder/std", + "sp-consensus-aura/std", "sp-core/std", "sp-inherents/std", "sp-offchain/std", @@ -92,14 +104,19 @@ std = [ "xcm/std", ] try-runtime = [ + "cumulus-pallet-aura-ext/try-runtime", "cumulus-pallet-parachain-system/try-runtime", "cumulus-pallet-xcm/try-runtime", "frame-executive/try-runtime", "frame-support/try-runtime", "frame-system/try-runtime", "frame-try-runtime/try-runtime", + "pallet-aura/try-runtime", "pallet-glutton/try-runtime", "pallet-sudo/try-runtime", + "pallet-timestamp/try-runtime", "parachain-info/try-runtime", "sp-runtime/try-runtime", ] + +experimental = [ "pallet-aura/experimental" ] diff --git a/cumulus/parachains/runtimes/glutton/glutton-kusama/src/lib.rs b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/lib.rs index dde8f747d4633bb3a8bbcd9dca471431dea99414..41cb0fceebb59c42a17b1b528830f3aa58950693 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-kusama/src/lib.rs +++ b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/lib.rs @@ -46,11 +46,12 @@ include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); pub mod weights; pub mod xcm_config; -use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use cumulus_pallet_parachain_system::RelayNumberMonotonicallyIncreases; use sp_api::impl_runtime_apis; -use sp_core::OpaqueMetadata; +pub use sp_consensus_aura::sr25519::AuthorityId as AuraId; +use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; use sp_runtime::{ - create_runtime_str, generic, + create_runtime_str, generic, impl_opaque_keys, traits::{AccountIdLookup, BlakeTwo256, Block as BlockT}, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, @@ -64,24 +65,37 @@ pub use frame_support::{ construct_runtime, dispatch::DispatchClass, parameter_types, - traits::{Everything, IsInVec, Randomness}, + traits::{ + ConstBool, ConstU32, ConstU64, ConstU8, EitherOfDiverse, Everything, IsInVec, Randomness, + }, weights::{ constants::{ BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_REF_TIME_PER_SECOND, }, IdentityFee, Weight, }, - StorageValue, + PalletId, StorageValue, }; use frame_system::{ limits::{BlockLength, BlockWeights}, EnsureRoot, }; -use parachains_common::{AccountId, Signature}; +use parachains_common::{ + kusama::consensus::{ + BLOCK_PROCESSING_VELOCITY, RELAY_CHAIN_SLOT_DURATION_MILLIS, UNINCLUDED_SEGMENT_CAPACITY, + }, + AccountId, Signature, SLOT_DURATION, +}; #[cfg(any(feature = "std", test))] pub use sp_runtime::BuildStorage; pub use sp_runtime::{Perbill, Permill}; +impl_opaque_keys! { + pub struct SessionKeys { + pub aura: Aura, + } +} + #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("glutton"), @@ -178,12 +192,35 @@ impl cumulus_pallet_parachain_system::Config for Runtime { type ReservedDmpWeight = ReservedDmpWeight; type XcmpMessageHandler = (); type ReservedXcmpWeight = (); - type CheckAssociatedRelayNumber = RelayNumberStrictlyIncreases; - type ConsensusHook = cumulus_pallet_parachain_system::consensus_hook::ExpectParentIncluded; + type CheckAssociatedRelayNumber = RelayNumberMonotonicallyIncreases; + type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< + Runtime, + RELAY_CHAIN_SLOT_DURATION_MILLIS, + BLOCK_PROCESSING_VELOCITY, + UNINCLUDED_SEGMENT_CAPACITY, + >; } impl parachain_info::Config for Runtime {} +impl cumulus_pallet_aura_ext::Config for Runtime {} + +impl pallet_timestamp::Config for Runtime { + type Moment = u64; + type OnTimestampSet = Aura; + type MinimumPeriod = ConstU64<{ SLOT_DURATION / 2 }>; + type WeightInfo = weights::pallet_timestamp::WeightInfo; +} + +impl pallet_aura::Config for Runtime { + type AuthorityId = AuraId; + type DisabledValidators = (); + type MaxAuthorities = ConstU32<100_000>; + type AllowMultipleBlocksPerSlot = ConstBool; + #[cfg(feature = "experimental")] + type SlotDuration = pallet_aura::MinimumPeriodTimesTwo; +} + impl pallet_glutton::Config for Runtime { type RuntimeEvent = RuntimeEvent; type WeightInfo = weights::pallet_glutton::WeightInfo; @@ -204,6 +241,7 @@ construct_runtime! { Pallet, Call, Config, Storage, Inherent, Event, ValidateUnsigned, } = 1, ParachainInfo: parachain_info::{Pallet, Storage, Config} = 2, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent} = 3, // DMP handler. CumulusXcm: cumulus_pallet_xcm::{Pallet, Call, Storage, Event, Origin} = 10, @@ -211,6 +249,10 @@ construct_runtime! { // The main stage. Glutton: pallet_glutton::{Pallet, Call, Storage, Event, Config} = 20, + // Collator support + Aura: pallet_aura::{Pallet, Storage, Config} = 30, + AuraExt: cumulus_pallet_aura_ext::{Pallet, Storage, Config} = 31, + // Sudo. Sudo: pallet_sudo::{Pallet, Call, Storage, Event, Config} = 255, } @@ -295,6 +337,16 @@ impl_runtime_apis! { } } + impl sp_consensus_aura::AuraApi for Runtime { + fn slot_duration() -> sp_consensus_aura::SlotDuration { + sp_consensus_aura::SlotDuration::from_millis(Aura::slot_duration()) + } + + fn authorities() -> Vec { + Aura::authorities().into_inner() + } + } + impl sp_block_builder::BlockBuilder for Runtime { fn apply_extrinsic( extrinsic: ::Extrinsic, @@ -332,12 +384,14 @@ impl_runtime_apis! { } impl sp_session::SessionKeys for Runtime { - fn decode_session_keys(_: Vec) -> Option, sp_core::crypto::KeyTypeId)>> { - Some(Vec::new()) + fn generate_session_keys(seed: Option>) -> Vec { + SessionKeys::generate(seed) } - fn generate_session_keys(_: Option>) -> Vec { - Vec::new() + fn decode_session_keys( + encoded: Vec, + ) -> Option, KeyTypeId)>> { + SessionKeys::decode_into_raw_public_keys(&encoded) } } @@ -402,5 +456,5 @@ impl_runtime_apis! { cumulus_pallet_parachain_system::register_validate_block! { Runtime = Runtime, - BlockExecutor = Executive, + BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::, } diff --git a/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/mod.rs b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/mod.rs index 990558538acfcc97590325d2573a06f6026d35fb..955010505d310cffe1660fcec0dace4c2d613181 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/mod.rs +++ b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/mod.rs @@ -15,3 +15,4 @@ // along with Cumulus. If not, see . pub mod pallet_glutton; +pub mod pallet_timestamp; diff --git a/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/pallet_timestamp.rs b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/pallet_timestamp.rs new file mode 100644 index 0000000000000000000000000000000000000000..8edae065f1b9173a3767a037c167f05420b95a70 --- /dev/null +++ b/cumulus/parachains/runtimes/glutton/glutton-kusama/src/weights/pallet_timestamp.rs @@ -0,0 +1,75 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Cumulus. + +// Cumulus is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Cumulus is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Cumulus. If not, see . + +//! Autogenerated weights for `pallet_timestamp` +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-07-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `runner-ynta1nyy-project-238-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("asset-hub-kusama-dev")`, DB CACHE: 1024 + +// Executed Command: +// ./target/production/polkadot-parachain +// benchmark +// pallet +// --chain=asset-hub-kusama-dev +// --wasm-execution=compiled +// --pallet=pallet_timestamp +// --no-storage-info +// --no-median-slopes +// --no-min-squares +// --extrinsic=* +// --steps=50 +// --repeat=20 +// --json +// --header=./file_header.txt +// --output=./parachains/runtimes/assets/asset-hub-kusama/src/weights/ + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] +#![allow(missing_docs)] + +use frame_support::{traits::Get, weights::Weight}; +use core::marker::PhantomData; + +/// Weight functions for `pallet_timestamp`. +pub struct WeightInfo(PhantomData); +impl pallet_timestamp::WeightInfo for WeightInfo { + /// Storage: `Timestamp::Now` (r:1 w:1) + /// Proof: `Timestamp::Now` (`max_values`: Some(1), `max_size`: Some(8), added: 503, mode: `MaxEncodedLen`) + /// Storage: `Aura::CurrentSlot` (r:1 w:0) + /// Proof: `Aura::CurrentSlot` (`max_values`: Some(1), `max_size`: Some(8), added: 503, mode: `MaxEncodedLen`) + fn set() -> Weight { + // Proof Size summary in bytes: + // Measured: `86` + // Estimated: `1493` + // Minimum execution time: 9_313_000 picoseconds. + Weight::from_parts(9_775_000, 0) + .saturating_add(Weight::from_parts(0, 1493)) + .saturating_add(T::DbWeight::get().reads(2)) + .saturating_add(T::DbWeight::get().writes(1)) + } + fn on_finalize() -> Weight { + // Proof Size summary in bytes: + // Measured: `57` + // Estimated: `0` + // Minimum execution time: 3_322_000 picoseconds. + Weight::from_parts(3_577_000, 0) + .saturating_add(Weight::from_parts(0, 0)) + } +} diff --git a/cumulus/parachains/runtimes/starters/seedling/Cargo.toml b/cumulus/parachains/runtimes/starters/seedling/Cargo.toml index 2cd09d3a9eb073c2ed4e88cbf9d2c73471ccd897..1b68b720d977dd6e01902c358dee9e45ceee3571 100644 --- a/cumulus/parachains/runtimes/starters/seedling/Cargo.toml +++ b/cumulus/parachains/runtimes/starters/seedling/Cargo.toml @@ -12,10 +12,13 @@ scale-info = { version = "2.9.0", default-features = false, features = ["derive" frame-executive = { path = "../../../../../substrate/frame/executive", default-features = false} frame-support = { path = "../../../../../substrate/frame/support", default-features = false} frame-system = { path = "../../../../../substrate/frame/system", default-features = false} +pallet-aura = { path = "../../../../../substrate/frame/aura", default-features = false} pallet-balances = { path = "../../../../../substrate/frame/balances", default-features = false} pallet-sudo = { path = "../../../../../substrate/frame/sudo", default-features = false} +pallet-timestamp = { path = "../../../../../substrate/frame/timestamp", default-features = false } sp-api = { path = "../../../../../substrate/primitives/api", default-features = false} sp-block-builder = { path = "../../../../../substrate/primitives/block-builder", default-features = false} +sp-consensus-aura = { path = "../../../../../substrate/primitives/consensus/aura", default-features = false } sp-core = { path = "../../../../../substrate/primitives/core", default-features = false} sp-inherents = { path = "../../../../../substrate/primitives/inherents", default-features = false} sp-offchain = { path = "../../../../../substrate/primitives/offchain", default-features = false} @@ -26,11 +29,13 @@ sp-transaction-pool = { path = "../../../../../substrate/primitives/transaction- sp-version = { path = "../../../../../substrate/primitives/version", default-features = false} # Cumulus +cumulus-pallet-aura-ext = { path = "../../../../pallets/aura-ext", default-features = false } cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook",] } cumulus-pallet-solo-to-para = { path = "../../../../pallets/solo-to-para", default-features = false } +cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } +cumulus-primitives-timestamp = { path = "../../../../primitives/timestamp", default-features = false } parachain-info = { path = "../../../pallets/parachain-info", default-features = false } parachains-common = { path = "../../../common", default-features = false } -cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } [build-dependencies] substrate-wasm-builder = { path = "../../../../../substrate/utils/wasm-builder", optional = true } @@ -39,19 +44,24 @@ substrate-wasm-builder = { path = "../../../../../substrate/utils/wasm-builder", default = [ "std" ] std = [ "codec/std", + "cumulus-pallet-aura-ext/std", "cumulus-pallet-parachain-system/std", "cumulus-pallet-solo-to-para/std", "cumulus-primitives-core/std", + "cumulus-primitives-timestamp/std", "frame-executive/std", "frame-support/std", "frame-system/std", + "pallet-aura/std", "pallet-balances/std", "pallet-sudo/std", + "pallet-timestamp/std", "parachain-info/std", "parachains-common/std", "scale-info/std", "sp-api/std", "sp-block-builder/std", + "sp-consensus-aura/std", "sp-core/std", "sp-inherents/std", "sp-offchain/std", @@ -62,3 +72,5 @@ std = [ "sp-version/std", "substrate-wasm-builder", ] + +experimental = [ "pallet-aura/experimental" ] diff --git a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs index 5f6733faf70679933fd177ee6ff2fdcbffe3500f..34e82737f82bb38409327b9b4cc96166e0ffcda0 100644 --- a/cumulus/parachains/runtimes/starters/seedling/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/seedling/src/lib.rs @@ -27,11 +27,12 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use cumulus_pallet_parachain_system::RelayNumberMonotonicallyIncreases; use sp_api::impl_runtime_apis; -use sp_core::OpaqueMetadata; +pub use sp_consensus_aura::sr25519::AuthorityId as AuraId; +use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; use sp_runtime::{ - create_runtime_str, generic, + create_runtime_str, generic, impl_opaque_keys, traits::{AccountIdLookup, BlakeTwo256, Block as BlockT}, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, @@ -46,7 +47,7 @@ pub use frame_support::{ construct_runtime, dispatch::DispatchClass, parameter_types, - traits::{IsInVec, Randomness}, + traits::{ConstBool, ConstU32, ConstU64, ConstU8, EitherOfDiverse, IsInVec, Randomness}, weights::{ constants::{ BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_REF_TIME_PER_SECOND, @@ -61,6 +62,12 @@ use parachains_common::{AccountId, Signature}; pub use sp_runtime::BuildStorage; pub use sp_runtime::{Perbill, Permill}; +impl_opaque_keys! { + pub struct SessionKeys { + pub aura: Aura, + } +} + /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { @@ -80,6 +87,15 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } +/// Maximum number of blocks simultaneously accepted by the Runtime, not yet included +/// into the relay chain. +const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; +/// How many parachain blocks are processed by the relay chain per parent. Limits the +/// number of blocks authored per slot. +const BLOCK_PROCESSING_VELOCITY: u32 = 1; +/// Relay chain slot duration, in milliseconds. +const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; + /// We assume that ~10% of the block weight is consumed by `on_initialize` handlers. /// This is used to limit the maximal weight of a single extrinsic. const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(10); @@ -174,23 +190,49 @@ impl cumulus_pallet_parachain_system::Config for Runtime { type ReservedDmpWeight = (); type XcmpMessageHandler = (); type ReservedXcmpWeight = (); - type CheckAssociatedRelayNumber = RelayNumberStrictlyIncreases; - type ConsensusHook = cumulus_pallet_parachain_system::consensus_hook::ExpectParentIncluded; + type CheckAssociatedRelayNumber = RelayNumberMonotonicallyIncreases; + type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< + Runtime, + RELAY_CHAIN_SLOT_DURATION_MILLIS, + BLOCK_PROCESSING_VELOCITY, + UNINCLUDED_SEGMENT_CAPACITY, + >; } impl parachain_info::Config for Runtime {} +impl cumulus_pallet_aura_ext::Config for Runtime {} + +impl pallet_aura::Config for Runtime { + type AuthorityId = AuraId; + type DisabledValidators = (); + type MaxAuthorities = ConstU32<100_000>; + type AllowMultipleBlocksPerSlot = ConstBool; + #[cfg(feature = "experimental")] + type SlotDuration = pallet_aura::MinimumPeriodTimesTwo; +} + +impl pallet_timestamp::Config for Runtime { + type Moment = u64; + type OnTimestampSet = Aura; + type MinimumPeriod = ConstU64<0>; + type WeightInfo = (); +} + construct_runtime! { pub enum Runtime { System: frame_system::{Pallet, Call, Storage, Config, Event}, Sudo: pallet_sudo::{Pallet, Call, Storage, Config, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, ParachainSystem: cumulus_pallet_parachain_system::{ Pallet, Call, Config, Storage, Inherent, Event, ValidateUnsigned, }, ParachainInfo: parachain_info::{Pallet, Storage, Config}, SoloToPara: cumulus_pallet_solo_to_para::{Pallet, Call, Storage, Event}, + Aura: pallet_aura::{Pallet, Storage, Config}, + AuraExt: cumulus_pallet_aura_ext::{Pallet, Storage, Config}, } } @@ -233,6 +275,16 @@ pub type Executive = frame_executive::Executive< >; impl_runtime_apis! { + impl sp_consensus_aura::AuraApi for Runtime { + fn slot_duration() -> sp_consensus_aura::SlotDuration { + sp_consensus_aura::SlotDuration::from_millis(Aura::slot_duration()) + } + + fn authorities() -> Vec { + Aura::authorities().into_inner() + } + } + impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { VERSION @@ -298,12 +350,14 @@ impl_runtime_apis! { } impl sp_session::SessionKeys for Runtime { - fn decode_session_keys(_: Vec) -> Option, sp_core::crypto::KeyTypeId)>> { - Some(Vec::new()) + fn generate_session_keys(seed: Option>) -> Vec { + SessionKeys::generate(seed) } - fn generate_session_keys(_: Option>) -> Vec { - Vec::new() + fn decode_session_keys( + encoded: Vec, + ) -> Option, KeyTypeId)>> { + SessionKeys::decode_into_raw_public_keys(&encoded) } } @@ -316,5 +370,5 @@ impl_runtime_apis! { cumulus_pallet_parachain_system::register_validate_block! { Runtime = Runtime, - BlockExecutor = Executive, + BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::, } diff --git a/cumulus/parachains/runtimes/starters/shell/Cargo.toml b/cumulus/parachains/runtimes/starters/shell/Cargo.toml index 6f9046057102fdd42b2a588a14b66a99199afcaf..46cef8d4ae083062b0c8521642ede074feb70035 100644 --- a/cumulus/parachains/runtimes/starters/shell/Cargo.toml +++ b/cumulus/parachains/runtimes/starters/shell/Cargo.toml @@ -13,8 +13,11 @@ frame-executive = { path = "../../../../../substrate/frame/executive", default-f frame-support = { path = "../../../../../substrate/frame/support", default-features = false} frame-system = { path = "../../../../../substrate/frame/system", default-features = false} frame-try-runtime = { path = "../../../../../substrate/frame/try-runtime", default-features = false, optional = true } +pallet-aura = { path = "../../../../../substrate/frame/aura", default-features = false} +pallet-timestamp = { path = "../../../../../substrate/frame/timestamp", default-features = false } sp-api = { path = "../../../../../substrate/primitives/api", default-features = false} sp-block-builder = { path = "../../../../../substrate/primitives/block-builder", default-features = false} +sp-consensus-aura = { path = "../../../../../substrate/primitives/consensus/aura", default-features = false } sp-core = { path = "../../../../../substrate/primitives/core", default-features = false} sp-inherents = { path = "../../../../../substrate/primitives/inherents", default-features = false} sp-offchain = { path = "../../../../../substrate/primitives/offchain", default-features = false} @@ -30,6 +33,7 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false} # Cumulus +cumulus-pallet-aura-ext = { path = "../../../../pallets/aura-ext", default-features = false } cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook",] } cumulus-pallet-xcm = { path = "../../../../pallets/xcm", default-features = false } cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } @@ -43,6 +47,7 @@ substrate-wasm-builder = { path = "../../../../../substrate/utils/wasm-builder", default = [ "std" ] std = [ "codec/std", + "cumulus-pallet-aura-ext/std", "cumulus-pallet-parachain-system/std", "cumulus-pallet-xcm/std", "cumulus-primitives-core/std", @@ -50,11 +55,14 @@ std = [ "frame-support/std", "frame-system/std", "frame-try-runtime?/std", + "pallet-aura/std", + "pallet-timestamp/std", "parachain-info/std", "parachains-common/std", "scale-info/std", "sp-api/std", "sp-block-builder/std", + "sp-consensus-aura/std", "sp-core/std", "sp-inherents/std", "sp-offchain/std", @@ -69,12 +77,17 @@ std = [ "xcm/std", ] try-runtime = [ + "cumulus-pallet-aura-ext/try-runtime", "cumulus-pallet-parachain-system/try-runtime", "cumulus-pallet-xcm/try-runtime", "frame-executive/try-runtime", "frame-support/try-runtime", "frame-system/try-runtime", "frame-try-runtime/try-runtime", + "pallet-aura/try-runtime", + "pallet-timestamp/try-runtime", "parachain-info/try-runtime", "sp-runtime/try-runtime", ] + +experimental = [ "pallet-aura/experimental" ] diff --git a/cumulus/parachains/runtimes/starters/shell/src/lib.rs b/cumulus/parachains/runtimes/starters/shell/src/lib.rs index ef914a246efc9f310176b3c46bf277a16b372082..477933b5c8d25b04205331ae0dfe17c42e7b555d 100644 --- a/cumulus/parachains/runtimes/starters/shell/src/lib.rs +++ b/cumulus/parachains/runtimes/starters/shell/src/lib.rs @@ -32,13 +32,14 @@ include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); pub mod xcm_config; use codec::{Decode, Encode}; -use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use cumulus_pallet_parachain_system::RelayNumberMonotonicallyIncreases; use frame_support::unsigned::TransactionValidityError; use scale_info::TypeInfo; use sp_api::impl_runtime_apis; -use sp_core::OpaqueMetadata; +pub use sp_consensus_aura::sr25519::AuthorityId as AuraId; +use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; use sp_runtime::{ - create_runtime_str, generic, + create_runtime_str, generic, impl_opaque_keys, traits::{AccountIdLookup, BlakeTwo256, Block as BlockT, DispatchInfoOf}, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, @@ -53,7 +54,7 @@ pub use frame_support::{ construct_runtime, dispatch::DispatchClass, parameter_types, - traits::{Everything, IsInVec, Randomness}, + traits::{ConstBool, ConstU32, ConstU64, ConstU8, EitherOfDiverse, IsInVec, Randomness}, weights::{ constants::{ BlockExecutionWeight, ExtrinsicBaseWeight, RocksDbWeight, WEIGHT_REF_TIME_PER_SECOND, @@ -68,6 +69,12 @@ use parachains_common::{AccountId, Signature}; pub use sp_runtime::BuildStorage; pub use sp_runtime::{Perbill, Permill}; +impl_opaque_keys! { + pub struct SessionKeys { + pub aura: Aura, + } +} + /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { @@ -87,6 +94,15 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } +/// Maximum number of blocks simultaneously accepted by the Runtime, not yet included +/// into the relay chain. +const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; +/// How many parachain blocks are processed by the relay chain per parent. Limits the +/// number of blocks authored per slot. +const BLOCK_PROCESSING_VELOCITY: u32 = 1; +/// Relay chain slot duration, in milliseconds. +const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; + /// We assume that ~10% of the block weight is consumed by `on_initialize` handlers. /// This is used to limit the maximal weight of a single extrinsic. const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(10); @@ -177,16 +193,41 @@ impl cumulus_pallet_parachain_system::Config for Runtime { type ReservedDmpWeight = ReservedDmpWeight; type XcmpMessageHandler = (); type ReservedXcmpWeight = (); - type CheckAssociatedRelayNumber = RelayNumberStrictlyIncreases; - type ConsensusHook = cumulus_pallet_parachain_system::consensus_hook::ExpectParentIncluded; + type CheckAssociatedRelayNumber = RelayNumberMonotonicallyIncreases; + type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< + Runtime, + RELAY_CHAIN_SLOT_DURATION_MILLIS, + BLOCK_PROCESSING_VELOCITY, + UNINCLUDED_SEGMENT_CAPACITY, + >; } impl parachain_info::Config for Runtime {} +impl cumulus_pallet_aura_ext::Config for Runtime {} + +impl pallet_aura::Config for Runtime { + type AuthorityId = AuraId; + type DisabledValidators = (); + type MaxAuthorities = ConstU32<100_000>; + type AllowMultipleBlocksPerSlot = ConstBool; + #[cfg(feature = "experimental")] + type SlotDuration = pallet_aura::MinimumPeriodTimesTwo; +} + +impl pallet_timestamp::Config for Runtime { + type Moment = u64; + type OnTimestampSet = Aura; + type MinimumPeriod = ConstU64<0>; + type WeightInfo = (); +} + construct_runtime! { pub enum Runtime { System: frame_system::{Pallet, Call, Storage, Config, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent}, + ParachainSystem: cumulus_pallet_parachain_system::{ Pallet, Call, Config, Storage, Inherent, Event, ValidateUnsigned, }, @@ -194,6 +235,9 @@ construct_runtime! { // DMP handler. CumulusXcm: cumulus_pallet_xcm::{Pallet, Call, Storage, Event, Origin}, + + Aura: pallet_aura::{Pallet, Storage, Config}, + AuraExt: cumulus_pallet_aura_ext::{Pallet, Storage, Config}, } } @@ -263,6 +307,16 @@ pub type Executive = frame_executive::Executive< >; impl_runtime_apis! { + impl sp_consensus_aura::AuraApi for Runtime { + fn slot_duration() -> sp_consensus_aura::SlotDuration { + sp_consensus_aura::SlotDuration::from_millis(Aura::slot_duration()) + } + + fn authorities() -> Vec { + Aura::authorities().into_inner() + } + } + impl sp_api::Core for Runtime { fn version() -> RuntimeVersion { VERSION @@ -328,12 +382,14 @@ impl_runtime_apis! { } impl sp_session::SessionKeys for Runtime { - fn decode_session_keys(_: Vec) -> Option, sp_core::crypto::KeyTypeId)>> { - Some(Vec::new()) + fn generate_session_keys(seed: Option>) -> Vec { + SessionKeys::generate(seed) } - fn generate_session_keys(_: Option>) -> Vec { - Vec::new() + fn decode_session_keys( + encoded: Vec, + ) -> Option, KeyTypeId)>> { + SessionKeys::decode_into_raw_public_keys(&encoded) } } @@ -346,5 +402,5 @@ impl_runtime_apis! { cumulus_pallet_parachain_system::register_validate_block! { Runtime = Runtime, - BlockExecutor = Executive, + BlockExecutor = cumulus_pallet_aura_ext::BlockExecutor::, } diff --git a/cumulus/parachains/runtimes/test-utils/Cargo.toml b/cumulus/parachains/runtimes/test-utils/Cargo.toml index 681e5e64d41280ef4ded74ff2bf347031e74bb04..378d9ea4c42ce27d0541b2c5ef567801d2e32bd3 100644 --- a/cumulus/parachains/runtimes/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "1.0.0" authors.workspace = true edition.workspace = true description = "Utils for Runtimes testing" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } diff --git a/cumulus/parachains/runtimes/testing/penpal/Cargo.toml b/cumulus/parachains/runtimes/testing/penpal/Cargo.toml index 99caf2f27d0391f7e942b531071bbb5c00ac6b39..547695e7621079d8bc7ca33b9fed0c2a41ac15db 100644 --- a/cumulus/parachains/runtimes/testing/penpal/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/penpal/Cargo.toml @@ -135,6 +135,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -147,6 +148,7 @@ runtime-benchmarks = [ "pallet-sudo/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs index 97d2e63370ead0cb5ee82aa5773c3b37b56f67c2..f2ffc451b10ef2f2b0f37983bb3fc633a020193f 100644 --- a/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/testing/penpal/src/xcm_config.rs @@ -163,7 +163,7 @@ pub type Barrier = TrailingSetTopicAsId< // If the message is one that immediately attemps to pay for execution, then // allow it. AllowTopLevelPaidExecutionFrom, - // Common Good Assets parachain, parent and its exec plurality get free + // System Assets parachain, parent and its exec plurality get free // execution AllowExplicitUnpaidExecutionFrom<( CommonGoodAssetsParachain, diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml index 863b9edd72581dd358e89f94ccdfd3b3675240eb..54055ca732ba994aefd2a646daaf92cc3d21364e 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml @@ -105,6 +105,7 @@ std = [ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", @@ -113,6 +114,7 @@ runtime-benchmarks = [ "pallet-sudo/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", diff --git a/cumulus/polkadot-parachain/Cargo.toml b/cumulus/polkadot-parachain/Cargo.toml index dc59b8207528f8b4fe2171fb17eb67853101b690..ac8ad53b524310d88e0fbcfb75a9fa416510ab43 100644 --- a/cumulus/polkadot-parachain/Cargo.toml +++ b/cumulus/polkadot-parachain/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "polkadot-parachain-bin" -version = "1.0.0" +version = "1.1.0" authors.workspace = true build = "build.rs" edition.workspace = true @@ -12,13 +12,13 @@ path = "src/main.rs" [dependencies] async-trait = "0.1.73" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } futures = "0.3.28" hex-literal = "0.4.1" log = "0.4.20" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.105" +serde_json = "1.0.107" # Local rococo-parachain-runtime = { path = "../parachains/runtimes/testing/rococo-parachain" } @@ -116,6 +116,7 @@ runtime-benchmarks = [ "frame-benchmarking-cli/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "glutton-runtime/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "penpal-runtime/runtime-benchmarks", "polkadot-cli/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", diff --git a/cumulus/polkadot-parachain/src/chain_spec/asset_hubs.rs b/cumulus/polkadot-parachain/src/chain_spec/asset_hubs.rs index c1fb60374aef87b3771bbbb1168d138a84d1b513..c1edeb98cd0aa3b50e1c8efbe47085144f70900e 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/asset_hubs.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/asset_hubs.rs @@ -32,11 +32,11 @@ pub type AssetHubWestendChainSpec = sc_service::GenericChainSpec; const ASSET_HUB_POLKADOT_ED: AssetHubBalance = - asset_hub_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; const ASSET_HUB_KUSAMA_ED: AssetHubBalance = - asset_hub_kusama_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::kusama::currency::EXISTENTIAL_DEPOSIT; const ASSET_HUB_WESTEND_ED: AssetHubBalance = - asset_hub_westend_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::westend::currency::EXISTENTIAL_DEPOSIT; /// Generate the session keys from individual elements. /// diff --git a/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs b/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs index 4cb81f57081a53ac815ec2884c8fac608c9214ee..5de4a49f8275122c3c86fbf18409068a653ce054 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/bridge_hubs.rs @@ -192,7 +192,7 @@ pub mod rococo { pub(crate) const BRIDGE_HUB_ROCOCO_LOCAL: &str = "bridge-hub-rococo-local"; pub(crate) const BRIDGE_HUB_ROCOCO_DEVELOPMENT: &str = "bridge-hub-rococo-dev"; const BRIDGE_HUB_ROCOCO_ED: BridgeHubBalance = - bridge_hub_rococo_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::rococo::currency::EXISTENTIAL_DEPOSIT; /// Specialized `ChainSpec` for the normal parachain runtime. pub type BridgeHubChainSpec = @@ -372,7 +372,7 @@ pub mod kusama { pub(crate) const BRIDGE_HUB_KUSAMA_LOCAL: &str = "bridge-hub-kusama-local"; pub(crate) const BRIDGE_HUB_KUSAMA_DEVELOPMENT: &str = "bridge-hub-kusama-dev"; const BRIDGE_HUB_KUSAMA_ED: BridgeHubBalance = - bridge_hub_kusama_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::kusama::currency::EXISTENTIAL_DEPOSIT; /// Specialized `ChainSpec` for the normal parachain runtime. pub type BridgeHubChainSpec = @@ -509,7 +509,7 @@ pub mod polkadot { pub(crate) const BRIDGE_HUB_POLKADOT_LOCAL: &str = "bridge-hub-polkadot-local"; pub(crate) const BRIDGE_HUB_POLKADOT_DEVELOPMENT: &str = "bridge-hub-polkadot-dev"; const BRIDGE_HUB_POLKADOT_ED: BridgeHubBalance = - bridge_hub_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; /// Specialized `ChainSpec` for the normal parachain runtime. pub type BridgeHubChainSpec = diff --git a/cumulus/polkadot-parachain/src/chain_spec/collectives.rs b/cumulus/polkadot-parachain/src/chain_spec/collectives.rs index 6126fbb114f069ec4c65d1352d6073afcc0369d0..fbf49b9535a42f632df29931e7515745414eb9c6 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/collectives.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/collectives.rs @@ -26,7 +26,7 @@ pub type CollectivesPolkadotChainSpec = sc_service::GenericChainSpec; const COLLECTIVES_POLKADOT_ED: CollectivesBalance = - collectives_polkadot_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::polkadot::currency::EXISTENTIAL_DEPOSIT; /// Generate the session keys from individual elements. /// diff --git a/cumulus/polkadot-parachain/src/chain_spec/contracts.rs b/cumulus/polkadot-parachain/src/chain_spec/contracts.rs index bf318e448f0e65018629945fa6524588d8f96184..0d5012858edc6f81a1554996fbafe4a1f6973072 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/contracts.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/contracts.rs @@ -31,7 +31,7 @@ const CONTRACTS_PARACHAIN_ID: u32 = 1002; /// The existential deposit is determined by the runtime "contracts-rococo". const CONTRACTS_ROCOCO_ED: contracts_rococo_runtime::Balance = - contracts_rococo_runtime::constants::currency::EXISTENTIAL_DEPOSIT; + parachains_common::rococo::currency::EXISTENTIAL_DEPOSIT; pub fn contracts_rococo_development_config() -> ContractsRococoChainSpec { let mut properties = sc_chain_spec::Properties::new(); diff --git a/cumulus/polkadot-parachain/src/chain_spec/glutton.rs b/cumulus/polkadot-parachain/src/chain_spec/glutton.rs index acd5b5bfbe132fe1df8c75636522763d47c21335..881fae398827a6adc1fd6195f9d69997a3a6e6af 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/glutton.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/glutton.rs @@ -16,9 +16,12 @@ use crate::chain_spec::{get_account_id_from_seed, Extensions}; use cumulus_primitives_core::ParaId; +use parachains_common::AuraId; use sc_service::ChainType; use sp_core::sr25519; +use super::get_collator_keys_from_seed; + /// Specialized `ChainSpec` for the Glutton parachain runtime. pub type GluttonChainSpec = sc_service::GenericChainSpec; @@ -30,7 +33,7 @@ pub fn glutton_development_config(para_id: ParaId) -> GluttonChainSpec { // ID "glutton_dev", ChainType::Local, - move || glutton_genesis(para_id), + move || glutton_genesis(para_id, vec![get_collator_keys_from_seed::("Alice")]), Vec::new(), None, None, @@ -47,7 +50,15 @@ pub fn glutton_local_config(para_id: ParaId) -> GluttonChainSpec { // ID "glutton_local", ChainType::Local, - move || glutton_genesis(para_id), + move || { + glutton_genesis( + para_id, + vec![ + get_collator_keys_from_seed::("Alice"), + get_collator_keys_from_seed::("Bob"), + ], + ) + }, Vec::new(), None, None, @@ -67,7 +78,15 @@ pub fn glutton_config(para_id: ParaId) -> GluttonChainSpec { // ID format!("glutton-kusama-{}", para_id).as_str(), ChainType::Live, - move || glutton_genesis(para_id), + move || { + glutton_genesis( + para_id, + vec![ + get_collator_keys_from_seed::("Alice"), + get_collator_keys_from_seed::("Bob"), + ], + ) + }, Vec::new(), None, // Protocol ID @@ -78,7 +97,10 @@ pub fn glutton_config(para_id: ParaId) -> GluttonChainSpec { ) } -fn glutton_genesis(parachain_id: ParaId) -> glutton_runtime::RuntimeGenesisConfig { +fn glutton_genesis( + parachain_id: ParaId, + collators: Vec, +) -> glutton_runtime::RuntimeGenesisConfig { glutton_runtime::RuntimeGenesisConfig { system: glutton_runtime::SystemConfig { code: glutton_runtime::WASM_BINARY @@ -94,6 +116,8 @@ fn glutton_genesis(parachain_id: ParaId) -> glutton_runtime::RuntimeGenesisConfi trash_data_count: Default::default(), ..Default::default() }, + aura: glutton_runtime::AuraConfig { authorities: collators }, + aura_ext: Default::default(), sudo: glutton_runtime::SudoConfig { key: Some(get_account_id_from_seed::("Alice")), }, diff --git a/cumulus/polkadot-parachain/src/chain_spec/seedling.rs b/cumulus/polkadot-parachain/src/chain_spec/seedling.rs index 3ebfb80d4685f67b9a43835fdc54609b5ca564dc..6a5842320976c3794557a38deffe4105af4245c5 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/seedling.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/seedling.rs @@ -16,10 +16,12 @@ use crate::chain_spec::{get_account_id_from_seed, Extensions}; use cumulus_primitives_core::ParaId; -use parachains_common::AccountId; +use parachains_common::{AccountId, AuraId}; use sc_service::ChainType; use sp_core::sr25519; +use super::get_collator_keys_from_seed; + /// Specialized `ChainSpec` for the seedling parachain runtime. pub type SeedlingChainSpec = sc_service::GenericChainSpec; @@ -33,6 +35,7 @@ pub fn get_seedling_chain_spec() -> SeedlingChainSpec { seedling_testnet_genesis( get_account_id_from_seed::("Alice"), 2000.into(), + vec![get_collator_keys_from_seed::("Alice")], ) }, Vec::new(), @@ -47,6 +50,7 @@ pub fn get_seedling_chain_spec() -> SeedlingChainSpec { fn seedling_testnet_genesis( root_key: AccountId, parachain_id: ParaId, + collators: Vec, ) -> seedling_runtime::RuntimeGenesisConfig { seedling_runtime::RuntimeGenesisConfig { system: seedling_runtime::SystemConfig { @@ -61,5 +65,7 @@ fn seedling_testnet_genesis( ..Default::default() }, parachain_system: Default::default(), + aura: seedling_runtime::AuraConfig { authorities: collators }, + aura_ext: Default::default(), } } diff --git a/cumulus/polkadot-parachain/src/chain_spec/shell.rs b/cumulus/polkadot-parachain/src/chain_spec/shell.rs index 7eb65591b12f38cb4f5de15967aa4d398a1eb183..0899c1af3b4d238262191436c0bacf80f557882b 100644 --- a/cumulus/polkadot-parachain/src/chain_spec/shell.rs +++ b/cumulus/polkadot-parachain/src/chain_spec/shell.rs @@ -16,8 +16,11 @@ use crate::chain_spec::Extensions; use cumulus_primitives_core::ParaId; +use parachains_common::AuraId; use sc_service::ChainType; +use super::get_collator_keys_from_seed; + /// Specialized `ChainSpec` for the shell parachain runtime. pub type ShellChainSpec = sc_service::GenericChainSpec; @@ -27,7 +30,9 @@ pub fn get_shell_chain_spec() -> ShellChainSpec { "Shell Local Testnet", "shell_local_testnet", ChainType::Local, - move || shell_testnet_genesis(1000.into()), + move || { + shell_testnet_genesis(1000.into(), vec![get_collator_keys_from_seed::("Alice")]) + }, Vec::new(), None, None, @@ -37,7 +42,10 @@ pub fn get_shell_chain_spec() -> ShellChainSpec { ) } -fn shell_testnet_genesis(parachain_id: ParaId) -> shell_runtime::RuntimeGenesisConfig { +fn shell_testnet_genesis( + parachain_id: ParaId, + collators: Vec, +) -> shell_runtime::RuntimeGenesisConfig { shell_runtime::RuntimeGenesisConfig { system: shell_runtime::SystemConfig { code: shell_runtime::WASM_BINARY @@ -47,5 +55,7 @@ fn shell_testnet_genesis(parachain_id: ParaId) -> shell_runtime::RuntimeGenesisC }, parachain_info: shell_runtime::ParachainInfoConfig { parachain_id, ..Default::default() }, parachain_system: Default::default(), + aura: shell_runtime::AuraConfig { authorities: collators }, + aura_ext: Default::default(), } } diff --git a/cumulus/polkadot-parachain/src/command.rs b/cumulus/polkadot-parachain/src/command.rs index 596b7baf6710ea1af53d988d6f5b3d0f93921c7d..32e363aff74be201f176bdcf2eec13cbe2dff6e9 100644 --- a/cumulus/polkadot-parachain/src/command.rs +++ b/cumulus/polkadot-parachain/src/command.rs @@ -381,7 +381,7 @@ impl SubstrateCli for RelayChainCli { } /// Creates partial components for the runtimes that are supported by the benchmarks. -macro_rules! construct_benchmark_partials { +macro_rules! construct_partials { ($config:expr, |$partials:ident| $code:expr) => { match $config.chain_spec.runtime() { Runtime::AssetHubKusama => { @@ -456,7 +456,41 @@ macro_rules! construct_benchmark_partials { )?; $code }, - _ => Err("The chain is not supported".into()), + Runtime::Shell => { + let $partials = new_partial::( + &$config, + crate::service::shell_build_import_queue, + )?; + $code + }, + Runtime::Seedling => { + let $partials = new_partial::( + &$config, + crate::service::shell_build_import_queue, + )?; + $code + }, + Runtime::ContractsRococo => { + let $partials = new_partial::( + &$config, + crate::service::contracts_rococo_build_import_queue, + )?; + $code + }, + Runtime::Penpal(_) | Runtime::Default => { + let $partials = new_partial::( + &$config, + crate::service::rococo_parachain_build_import_queue, + )?; + $code + }, + Runtime::Glutton => { + let $partials = new_partial::( + &$config, + crate::service::shell_build_import_queue, + )?; + $code + }, } }; } @@ -679,10 +713,12 @@ pub fn run() -> Result<()> { cmd.run(config, polkadot_config) }) }, - Some(Subcommand::ExportGenesisState(cmd)) => - construct_async_run!(|components, cli, cmd, config| { - Ok(async move { cmd.run(&*config.chain_spec, &*components.client) }) - }), + Some(Subcommand::ExportGenesisState(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.sync_run(|config| { + construct_partials!(config, |partials| cmd.run(&*config.chain_spec, &*partials.client)) + }) + }, Some(Subcommand::ExportGenesisWasm(cmd)) => { let runner = cli.create_runner(cmd)?; runner.sync_run(|_config| { @@ -704,7 +740,7 @@ pub fn run() -> Result<()> { .into()) }, BenchmarkCmd::Block(cmd) => runner.sync_run(|config| { - construct_benchmark_partials!(config, |partials| cmd.run(partials.client)) + construct_partials!(config, |partials| cmd.run(partials.client)) }), #[cfg(not(feature = "runtime-benchmarks"))] BenchmarkCmd::Storage(_) => @@ -716,7 +752,7 @@ pub fn run() -> Result<()> { .into()), #[cfg(feature = "runtime-benchmarks")] BenchmarkCmd::Storage(cmd) => runner.sync_run(|config| { - construct_benchmark_partials!(config, |partials| { + construct_partials!(config, |partials| { let db = partials.backend.expose_db(); let storage = partials.backend.expose_storage(); diff --git a/cumulus/primitives/utility/Cargo.toml b/cumulus/primitives/utility/Cargo.toml index 47c5442574fb6aea3a52b0ac9a11c9c6aff20fa4..9ed1e664ac217e7504d2ca6915c588b5c9c82c10 100644 --- a/cumulus/primitives/utility/Cargo.toml +++ b/cumulus/primitives/utility/Cargo.toml @@ -38,3 +38,11 @@ std = [ "xcm-executor/std", "xcm/std", ] + +runtime-benchmarks = [ + "frame-support/runtime-benchmarks", + "polkadot-runtime-common/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", + "xcm-builder/runtime-benchmarks", + "xcm-executor/runtime-benchmarks", +] diff --git a/cumulus/scripts/ci/changelog/README.md b/cumulus/scripts/ci/changelog/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5c8ee9c9b914e634e07fb6030e89a7188f1f3aaf --- /dev/null +++ b/cumulus/scripts/ci/changelog/README.md @@ -0,0 +1,77 @@ +# Changelog + +Currently, the changelog is built locally. It will be moved to CI once labels stabilize. + +For now, a bit of preparation is required before you can run the script: +- fetch the srtool digests +- store them under the `digests` folder as `-srtool-digest.json` +- ensure the `.env` file is up to date with correct information + +The content of the release notes is generated from the template files under the `scripts/ci/changelog/templates` folder. +For readability and maintenance, the template is split into several small snippets. + +Run: +``` +./bin/changelog [=HEAD] +``` + +For instance: +``` +./bin/changelog parachains-v7.0.0-rc8 +``` + +A file called `release-notes.md` will be generated and can be used for the release. + +## ENV + +You may use the following ENV for testing: + +``` +RUSTC_STABLE="rustc 1.56.1 (59eed8a2a 2021-11-01)" +RUSTC_NIGHTLY="rustc 1.57.0-nightly (51e514c0f 2021-09-12)" +PRE_RELEASE=true +HIDE_SRTOOL_ROCOCO=true +HIDE_SRTOOL_SHELL=true +REF1=statemine-v5.0.0 +REF2=HEAD +DEBUG=1 +NO_CACHE=1 +``` + +By default, the template will include all the information, including the runtime data. For clients releases, we don't +need those and they can be skipped by setting the following env: +``` +RELEASE_TYPE=client +``` + +## Considered labels + +The following list will likely evolve over time and it will be hard to keep it in sync. In any case, if you want to find +all the labels that are used, search for `meta` in the templates. Currently, the considered labels are: + +- Priority: C labels +- Audit: D labels +- E4 => new host function +- B0 => silent, not showing up +- B1-releasenotes (misc unless other labels) +- B5-client (client changes) +- B7-runtimenoteworthy (runtime changes) +- T6-XCM + +Note that labels with the same letter are mutually exclusive. A PR should not have both `B0` and `B5`, or both `C1` and +`C9`. In case of conflicts, the template will decide which label will be considered. + +## Dev and debugging + +### Hot Reload + +The following command allows **Hot Reload**: +``` +fswatch templates -e ".*\.md$" | xargs -n1 -I{} ./bin/changelog statemine-v5.0.0 +``` +### Caching + +By default, if the changelog data from Github is already present, the calls to the Github API will be skipped and the +local version of the data will be used. This is much faster. If you know that some labels have changed in Github, you +probably want to refresh the data. You can then either delete manually the `cumulus.json` file or `export NO_CACHE=1` to +force refreshing the data. diff --git a/cumulus/test/relay-sproof-builder/Cargo.toml b/cumulus/test/relay-sproof-builder/Cargo.toml index c987a2b66c9026bdc5a40a8a6fb3331258850331..e044b92f7c4af4d297c8996c404a479e2d73ec8b 100644 --- a/cumulus/test/relay-sproof-builder/Cargo.toml +++ b/cumulus/test/relay-sproof-builder/Cargo.toml @@ -3,7 +3,6 @@ name = "cumulus-test-relay-sproof-builder" version = "0.1.0" authors.workspace = true edition.workspace = true -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive" ] } diff --git a/cumulus/test/relay-validation-worker-provider/Cargo.toml b/cumulus/test/relay-validation-worker-provider/Cargo.toml deleted file mode 100644 index b7c59e8329958ad5d32d7b4ee171608302f868a8..0000000000000000000000000000000000000000 --- a/cumulus/test/relay-validation-worker-provider/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "cumulus-test-relay-validation-worker-provider" -version = "0.1.0" -authors.workspace = true -edition.workspace = true -build = "build.rs" -publish = false - -[dependencies] - -# Polkadot -polkadot-node-core-pvf = { path = "../../../polkadot/node/core/pvf", features = ["test-utils"] } - -[build-dependencies] -toml = "0.7.6" diff --git a/cumulus/test/relay-validation-worker-provider/build.rs b/cumulus/test/relay-validation-worker-provider/build.rs deleted file mode 100644 index 60bb950db1fcc94c0e558dc885785c1e3724efa2..0000000000000000000000000000000000000000 --- a/cumulus/test/relay-validation-worker-provider/build.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . - -use std::{ - env, fs, - path::{Path, PathBuf}, - process::{self, Command}, -}; -use toml::value::Table; - -/// The name of the project we will building. -const PROJECT_NAME: &str = "validation-worker"; -/// The env variable that instructs us to skip the build. -const SKIP_ENV: &str = "SKIP_BUILD"; - -fn main() { - if env::var(SKIP_ENV).is_ok() { - return - } - - let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo")); - - let project = create_project(&out_dir); - build_project(&project.join("Cargo.toml")); - - fs::copy(project.join("target/release").join(PROJECT_NAME), out_dir.join(PROJECT_NAME)) - .expect("Copies validation worker"); -} - -fn find_cargo_lock() -> PathBuf { - let mut path = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"), - ); - - loop { - if path.join("Cargo.lock").exists() { - return path.join("Cargo.lock") - } - - if !path.pop() { - panic!("Could not find `Cargo.lock`") - } - } -} - -fn create_project(out_dir: &Path) -> PathBuf { - let project_dir = out_dir.join(format!("{}-project", PROJECT_NAME)); - fs::create_dir_all(project_dir.join("src")).expect("Creates project dir and project src dir"); - - let mut project_toml = Table::new(); - - let mut package = Table::new(); - package.insert("name".into(), PROJECT_NAME.into()); - package.insert("version".into(), "1.0.0".into()); - package.insert("edition".into(), "2021".into()); - - project_toml.insert("package".into(), package.into()); - - project_toml.insert("workspace".into(), Table::new().into()); - - let mut dependencies = Table::new(); - - let mut dependency_project = Table::new(); - dependency_project.insert( - "path".into(), - env::var("CARGO_MANIFEST_DIR") - .expect("`CARGO_MANIFEST_DIR` is set by cargo") - .into(), - ); - - dependencies - .insert("cumulus-test-relay-validation-worker-provider".into(), dependency_project.into()); - - project_toml.insert("dependencies".into(), dependencies.into()); - - add_patches(&mut project_toml); - - fs::write( - project_dir.join("Cargo.toml"), - toml::to_string_pretty(&project_toml).expect("Wasm workspace toml is valid; qed"), - ) - .expect("Writes project `Cargo.toml`"); - - fs::write( - project_dir.join("src").join("main.rs"), - r#" - cumulus_test_relay_validation_worker_provider::polkadot_node_core_pvf::decl_puppet_worker_main!(); - "#, - ) - .expect("Writes `main.rs`"); - - let cargo_lock = find_cargo_lock(); - fs::copy(&cargo_lock, project_dir.join("Cargo.lock")).expect("Copies `Cargo.lock`"); - println!("cargo:rerun-if-changed={}", cargo_lock.display()); - - project_dir -} - -fn add_patches(project_toml: &mut Table) { - let workspace_toml_path = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"), - ) - .join("../../../Cargo.toml"); - - let mut workspace_toml: Table = toml::from_str( - &fs::read_to_string(&workspace_toml_path).expect("Workspace root `Cargo.toml` exists; qed"), - ) - .expect("Workspace root `Cargo.toml` is a valid toml file; qed"); - - let mut workspace_path = workspace_toml_path; - workspace_path.pop(); - - while let Some(mut patch) = - workspace_toml.remove("patch").and_then(|p| p.try_into::().ok()) - { - // Iterate over all patches and make the patch path absolute from the workspace root path. - patch - .iter_mut() - .filter_map(|p| { - p.1.as_table_mut().map(|t| t.iter_mut().filter_map(|t| t.1.as_table_mut())) - }) - .flatten() - .for_each(|p| { - p.iter_mut().filter(|(k, _)| k == &"path").for_each(|(_, v)| { - if let Some(path) = v.as_str().map(PathBuf::from) { - if path.is_relative() { - *v = workspace_path.join(path).display().to_string().into(); - } - } - }) - }); - - project_toml.insert("patch".into(), patch.into()); - } -} - -fn build_project(cargo_toml: &Path) { - let cargo = env::var("CARGO").expect("`CARGO` env variable is always set by cargo"); - - let status = Command::new(cargo) - .arg("build") - .arg("--release") - .arg(format!("--manifest-path={}", cargo_toml.display())) - // Unset the `CARGO_TARGET_DIR` to prevent a cargo deadlock (cargo locks a target dir - // exclusive). - .env_remove("CARGO_TARGET_DIR") - // Do not call us recursively. - .env(SKIP_ENV, "1") - .status(); - - match status.map(|s| s.success()) { - Ok(true) => {}, - // Use `process.exit(1)` to have a clean error output. - _ => process::exit(1), - } -} diff --git a/cumulus/test/relay-validation-worker-provider/src/lib.rs b/cumulus/test/relay-validation-worker-provider/src/lib.rs deleted file mode 100644 index 6c3f4182b03bd6ca473c3380a567dfe682ef3b4d..0000000000000000000000000000000000000000 --- a/cumulus/test/relay-validation-worker-provider/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . - -//! Provides the [`VALIDATION_WORKER`] for integration tests in Cumulus. -//! -//! The validation worker is used by the relay chain to validate parachains. This worker is placed -//! in an extra process to provide better security and to ensure that a worker can be killed etc. -//! -//! !!This should only be used for tests!! - -pub use polkadot_node_core_pvf; - -/// The path to the validation worker. -pub const VALIDATION_WORKER: &str = concat!(env!("OUT_DIR"), "/validation-worker"); diff --git a/cumulus/test/service/Cargo.toml b/cumulus/test/service/Cargo.toml index 04d53545ead716a6b47fb76b93df587fb3837071..5285376f3d59c78e1bf8f255bfd038a3839d1296 100644 --- a/cumulus/test/service/Cargo.toml +++ b/cumulus/test/service/Cargo.toml @@ -11,7 +11,7 @@ path = "src/main.rs" [dependencies] async-trait = "0.1.73" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } criterion = { version = "0.5.1", features = [ "async_tokio" ] } jsonrpsee = { version = "0.16.2", features = ["server"] } @@ -72,7 +72,6 @@ cumulus-primitives-core = { path = "../../primitives/core" } cumulus-primitives-parachain-inherent = { path = "../../primitives/parachain-inherent" } cumulus-relay-chain-inprocess-interface = { path = "../../client/relay-chain-inprocess-interface" } cumulus-relay-chain-interface = { path = "../../client/relay-chain-interface" } -cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-worker-provider" } cumulus-test-runtime = { path = "../runtime" } cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-node" } cumulus-client-pov-recovery = { path = "../../client/pov-recovery" } @@ -102,6 +101,7 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "pallet-im-online/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-cli/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-service/runtime-benchmarks", diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 3275aabc4d860d73cde4b26025d11ebfa9b05821..a721645546af7417200a7768242573abbe252397 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -903,8 +903,9 @@ pub fn run_relay_chain_validator_node( config.rpc_addr = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port)); } - polkadot_test_service::run_validator_node( - config, - Some(cumulus_test_relay_validation_worker_provider::VALIDATION_WORKER.into()), - ) + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + + polkadot_test_service::run_validator_node(config, Some(workers_path)) } diff --git a/cumulus/xcm/xcm-emulator/README.md b/cumulus/xcm/xcm-emulator/README.md index d188c99eecfcbffa9c79f56651736bd009168694..2a861a9d269be18e85d967e6dfc21904665f7dd5 100644 --- a/cumulus/xcm/xcm-emulator/README.md +++ b/cumulus/xcm/xcm-emulator/README.md @@ -13,11 +13,11 @@ As the messages do not physically go through the same messaging infrastructure there is some code that is not being tested compared to using slower E2E tests. In future it may be possible to run these XCM emulated tests as E2E tests (without changes). -As well as the XCM message transport being mocked out, so too are areas around consensus, +As well as the XCM message transport being mocked out, so too are areas around consensus, in particular things like disputes, staking and iamonline events can't be tested. ## Alternatives If you just wish to test execution of various XCM instructions -against the XCM VM then the `xcm-simulator` (in the polkadot +against the XCM VM then the `xcm-simulator` (in the Polkadot repo) is the perfect tool for this. diff --git a/cumulus/xcm/xcm-emulator/src/lib.rs b/cumulus/xcm/xcm-emulator/src/lib.rs index 59f238528fc1c3b2d76241c234ad0f88e03cd07b..9fda0632bae40639651f9d1a776cf952affcee5b 100644 --- a/cumulus/xcm/xcm-emulator/src/lib.rs +++ b/cumulus/xcm/xcm-emulator/src/lib.rs @@ -248,6 +248,14 @@ pub trait Parachain: Chain { type ParachainInfo: Get; type ParachainSystem; + fn init(); + + fn new_block(); + + fn finalize_block(); + + fn set_last_head(); + fn para_id() -> ParaId { Self::ext_wrapper(|| Self::ParachainInfo::get()) } @@ -263,8 +271,6 @@ pub trait Parachain: Chain { fn sovereign_account_id_of(location: MultiLocation) -> AccountId { Self::LocationToAccountId::convert_location(&location).unwrap() } - - fn init(); } pub trait Bridge { @@ -603,28 +609,74 @@ macro_rules! decl_test_parachains { type ParachainSystem = $crate::ParachainSystemPallet<::Runtime>; type ParachainInfo = $parachain_info; + // We run an empty block during initialisation to open HRMP channels + // and have them ready for the next block fn init() { use $crate::{Chain, HeadData, Network, NetworkComponent, Hooks, Encode, Parachain, TestExt}; + // Set the last block head for later use in the next block + Self::set_last_head(); + // Initialize a new block + Self::new_block(); + // Finalize the new block + Self::finalize_block(); + } + + fn new_block() { + use $crate::{Chain, HeadData, Network, NetworkComponent, Hooks, Encode, Parachain, TestExt}; + + let para_id = Self::para_id().into(); + + Self::ext_wrapper(|| { + // Increase Relay Chain block number + let mut relay_block_number = <$name as NetworkComponent>::Network::relay_block_number(); + relay_block_number += 1; + <$name as NetworkComponent>::Network::set_relay_block_number(relay_block_number); + + // Initialize a new Parachain block + let mut block_number = ::System::block_number(); + block_number += 1; + let parent_head_data = $crate::LAST_HEAD.with(|b| b.borrow_mut() + .get_mut(::Network::name()) + .expect("network not initialized?") + .get(¶_id) + .expect("network not initialized?") + .clone() + ); + ::System::initialize(&block_number, &parent_head_data.hash(), &Default::default()); + <::ParachainSystem as Hooks<$crate::BlockNumber>>::on_initialize(block_number); + + let _ = ::ParachainSystem::set_validation_data( + ::RuntimeOrigin::none(), + <$name as NetworkComponent>::Network::hrmp_channel_parachain_inherent_data(para_id, relay_block_number, parent_head_data), + ); + }); + } - let para_id = Self::para_id(); + fn finalize_block() { + use $crate::{Chain, Encode, Hooks, Network, NetworkComponent, Parachain, TestExt}; Self::ext_wrapper(|| { let block_number = ::System::block_number(); - let mut relay_block_number = ::Network::relay_block_number(); + ::ParachainSystem::on_finalize(block_number); + }); + + Self::set_last_head(); + } - // Get parent head data - let header = ::System::finalize(); - let parent_head_data = HeadData(header.encode()); + fn set_last_head() { + use $crate::{Chain, Encode, HeadData, Network, NetworkComponent, Parachain, TestExt}; + + let para_id = Self::para_id().into(); + + Self::ext_wrapper(|| { + // Store parent head data for use later. + let created_header = ::System::finalize(); $crate::LAST_HEAD.with(|b| b.borrow_mut() .get_mut(::Network::name()) .expect("network not initialized?") - .insert(para_id.into(), parent_head_data.clone()) + .insert(para_id, HeadData(created_header.encode())) ); - - let next_block_number = block_number + 1; - ::System::initialize(&next_block_number, &header.hash(), &Default::default()); - <::ParachainSystem as Hooks<$crate::BlockNumber>>::on_initialize(next_block_number); }); } } @@ -746,43 +798,20 @@ macro_rules! __impl_test_ext_for_parachain { // Make sure the Network is initialized <$name as NetworkComponent>::Network::init(); - let para_id = <$name>::para_id().into(); - - // Initialize block - $local_ext.with(|v| { - v.borrow_mut().execute_with(|| { - let parent_head_data = $crate::LAST_HEAD.with(|b| b.borrow_mut() - .get_mut(::Network::name()) - .expect("network not initialized?") - .get(¶_id) - .expect("network not initialized?") - .clone() - ); - - // Increase block number - let mut relay_block_number = <$name as NetworkComponent>::Network::relay_block_number(); - relay_block_number += 1; - <$name as NetworkComponent>::Network::set_relay_block_number(relay_block_number); - - let _ = ::ParachainSystem::set_validation_data( - ::RuntimeOrigin::none(), - <$name as NetworkComponent>::Network::hrmp_channel_parachain_inherent_data(para_id, relay_block_number, parent_head_data), - ); - }) - }); + // Initialize a new block + Self::new_block(); // Execute let r = $local_ext.with(|v| v.borrow_mut().execute_with(execute)); - // provide inbound DMP/HRMP messages through a side-channel. - // normally this would come through the `set_validation_data`, - // but we go around that. - <$name as NetworkComponent>::Network::process_messages(); + // Finalize the block + Self::finalize_block(); - // Finalize block and send messages if needed + let para_id = <$name>::para_id().into(); + + // Send messages if needed $local_ext.with(|v| { v.borrow_mut().execute_with(|| { - let block_number = ::System::block_number(); let mock_header = $crate::HeaderT::new( 0, Default::default(), @@ -791,16 +820,6 @@ macro_rules! __impl_test_ext_for_parachain { Default::default(), ); - // Finalize to get xcmp messages. - ::ParachainSystem::on_finalize(block_number); - // Store parent head data for use later. - let created_header = ::System::finalize(); - $crate::LAST_HEAD.with(|b| b.borrow_mut() - .get_mut(::Network::name()) - .expect("network not initialized?") - .insert(para_id.into(), $crate::HeadData(created_header.encode())) - ); - let collation_info = ::ParachainSystem::collect_collation_info(&mock_header); // send upward messages @@ -834,11 +853,6 @@ macro_rules! __impl_test_ext_for_parachain { // clean events ::System::reset_events(); - - // reinitialize before next call. - let next_block_number = block_number + 1; - ::System::initialize(&next_block_number, &created_header.hash(), &Default::default()); - <::ParachainSystem as Hooks<$crate::BlockNumber>>::on_initialize(next_block_number); }) }); diff --git a/cumulus/zombienet/tests/0007-prepare-warp-sync-db-snapshot.md b/cumulus/zombienet/tests/0007-prepare-warp-sync-db-snapshot.md index 7885dd0c0269d5772a344ba447fafd933b686563..373e480723803cf51585e38242bb1b37e41c8e15 100644 --- a/cumulus/zombienet/tests/0007-prepare-warp-sync-db-snapshot.md +++ b/cumulus/zombienet/tests/0007-prepare-warp-sync-db-snapshot.md @@ -1,22 +1,24 @@ # Database snapshot guide -For this guide we will be taking a snapshot of a parachain and relay chain. Please note we are using a local chain here `rococo_local_testnet` and `local_testnet`. Live chains will have different values +For this guide we will be taking a snapshot of a parachain and relay chain. Please note we are using a local chain here +`rococo_local_testnet` and `local_testnet`. Live chains will have different values *Please ensure that the database is not in current use, i.e no nodes are writing to it* # How to prepare database for a relaychain -To prepare snapshot for a relay chain we need to copy the database. +To prepare snapshot for a relay chain we need to copy the database. ``` mkdir -p relaychain-snapshot/alice/data/chains/rococo_local_testnet/db/ -cp -r chain-data/alice/data/chains/rococo_local_testnet/db/. relaychain-snapshot/alice/data/chains/rococo_local_testnet/db/ +cp -r chain-data/alice/data/chains/rococo_local_testnet/db/. relaychain-snapshot/alice/data/chains/rococo_local_testnet/db/ tar -C relaychain-snapshot/alice/ -czf relaychain.tgz data ``` # How to prepare database for a parachain -To prepare snapshot for a parachain we need to copy the database for both the collator node (parachain data) and validator (relay data) +To prepare snapshot for a parachain we need to copy the database for both the collator node (parachain data) and +validator (relay data) ``` #Parachain data @@ -33,5 +35,6 @@ tar -C parachain-snapshot/charlie/ -czf parachain.tgz data relay-data ``` # Restoring a snapshot -Zombienet will automatically download the `*.tgz` file to the respective folder for a run. However you can also download it manually, just ensure you extract the tar file in the correct directory, i.e. the root directory -`chain-data/charlie/` \ No newline at end of file +Zombienet will automatically download the `*.tgz` file to the respective folder for a run. However you can also download +it manually, just ensure you extract the tar file in the correct directory, i.e. the root directory +`chain-data/charlie/` diff --git a/docker/dockerfiles/binary_injected.Dockerfile b/docker/dockerfiles/binary_injected.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ac1fd5317c67cb4cbef1044a6e2e3379f8ff4662 --- /dev/null +++ b/docker/dockerfiles/binary_injected.Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/parity/base-bin + +# This file allows building a Generic container image +# based on one or multiple pre-built Linux binaries. +# Some defaults are set to polkadot but all can be overriden. + +SHELL ["/bin/bash", "-c"] + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +# That can be a single one or a comma separated list +ARG BINARY=polkadot + +ARG BIN_FOLDER=. +ARG DOC_URL=https://github.com/paritytech/polkadot-sdk +ARG DESCRIPTION="Polkadot: a platform for web3" +ARG AUTHORS="devops-team@parity.io" +ARG VENDOR="Parity Technologies" + +LABEL io.parity.image.authors=${AUTHORS} \ + io.parity.image.vendor="${VENDOR}" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="${DOC_URL}" \ + io.parity.image.description="${DESCRIPTION}" \ + io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/binary_injected.Dockerfile" + +USER root +WORKDIR /app + +# add polkadot binary to docker image +# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/ +COPY entrypoint.sh . +COPY "bin/*" "/usr/local/bin/" +RUN chmod -R a+rx "/usr/local/bin" + +USER parity +ENV BINARY=${BINARY} + +# ENTRYPOINT +ENTRYPOINT ["/app/entrypoint.sh"] + +# We call the help by default +CMD ["--help"] diff --git a/docker/collator_injected.Dockerfile b/docker/dockerfiles/collator_injected.Dockerfile similarity index 95% rename from docker/collator_injected.Dockerfile rename to docker/dockerfiles/collator_injected.Dockerfile index 6472c240f3322a667e49ff1a1a952c90a4cd85c7..0c9ea1e0ca833fd51c0553672ad85e2009ab7d3c 100644 --- a/docker/collator_injected.Dockerfile +++ b/docker/dockerfiles/collator_injected.Dockerfile @@ -10,7 +10,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Injected adder-collator Docker image" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/collator_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/collator_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/docker/docker-compose.yml b/docker/dockerfiles/docker-compose.yml similarity index 89% rename from docker/docker-compose.yml rename to docker/dockerfiles/docker-compose.yml index 8344ad43bb4cbac17b5b3f1d7cc5de9264f36050..8dc8540353fe0df08c55c98e660b3979c56bfeeb 100644 --- a/docker/docker-compose.yml +++ b/docker/dockerfiles/docker-compose.yml @@ -61,7 +61,7 @@ services: genesis_state: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile image: "ctpc:latest" volumes: - "genesis-state:/data" @@ -73,7 +73,7 @@ services: collator: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile target: collator image: "ctpc:collator" volumes: @@ -90,7 +90,7 @@ services: runtime: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile target: runtime image: "ctpc:runtime" volumes: @@ -100,7 +100,7 @@ services: registrar: build: context: . - dockerfile: ./docker/parachain-registrar.dockerfile + dockerfile: ./docker/dockerfiles/parachain-registrar.dockerfile image: para-reg:latest volumes: - "genesis-state:/genesis" diff --git a/docker/malus_injected.Dockerfile b/docker/dockerfiles/malus_injected.Dockerfile similarity index 100% rename from docker/malus_injected.Dockerfile rename to docker/dockerfiles/malus_injected.Dockerfile diff --git a/cumulus/docker/parachain-registrar.dockerfile b/docker/dockerfiles/parachain-registrar.dockerfile similarity index 89% rename from cumulus/docker/parachain-registrar.dockerfile rename to docker/dockerfiles/parachain-registrar.dockerfile index f7d77454a2b9d37080047f1351c564c87f91824d..00908395101f98652ab961f9054717794dd383af 100644 --- a/cumulus/docker/parachain-registrar.dockerfile +++ b/docker/dockerfiles/parachain-registrar.dockerfile @@ -9,7 +9,7 @@ CMD [ "--version" ] # To use the pjs build stage to access the blockchain from the host machine: # -# docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . +# docker build -f docker/dockerfiles/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . # alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944' # # Then, as long as the chain is running, you can use the polkadot-js-api CLI like: diff --git a/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile similarity index 88% rename from docker/polkadot-parachain-debug_unsigned_injected.Dockerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile index e77563b8ebf2466a7935341041fc04bb07bbce24..75cc2b9e629de4b9d8bf99e2a7bc866d3160a95a 100644 --- a/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/cumulus/" @@ -28,7 +28,7 @@ RUN apt-get update && \ apt-get clean && \ find /var/lib/apt/lists/ -type f -not -name lock -delete; \ # add user and link ~/.local/share/polkadot-parachain to /data - useradd -m -u 10000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \ + useradd -m -u 1000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \ mkdir -p /data /polkadot-parachain/.local/share && \ chown -R polkadot-parachain:polkadot-parachain /data && \ ln -s /data /polkadot-parachain/.local/share/polkadot-parachain && \ diff --git a/docker/polkadot-parachain_builder.Containerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile similarity index 89% rename from docker/polkadot-parachain_builder.Containerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile index 159bcb323693865d924b93d674af770f8a6bdf41..4d110d6af47286891c726485efd9492e2554a023 100644 --- a/docker/polkadot-parachain_builder.Containerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile @@ -1,4 +1,4 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +# This file is sourced from https://github.com/paritytech/polkadot/blob/master/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile # This is the build stage for polkadot-parachain. Here we create the binary in a temporary image. FROM docker.io/paritytech/ci-linux:production as builder @@ -14,7 +14,7 @@ LABEL io.parity.image.type="builder" \ io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.description="Multistage Docker image for polkadot-parachain" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot-parachain_builder.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Dockerfile" \ io.parity.image.documentation="https://github.com/paritytech/cumulus" COPY --from=builder /cumulus/target/release/polkadot-parachain /usr/local/bin diff --git a/cumulus/docker/injected.Dockerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile similarity index 68% rename from cumulus/docker/injected.Dockerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile index f9b11f022e75bc6ec81400b42ae8acac2385424f..16bd0f4cf3c55f521cc546655a38f709ad094c41 100644 --- a/cumulus/docker/injected.Dockerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile @@ -9,10 +9,10 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" + io.parity.image.documentation="https://github.com/paritytech/polkadot-sdk/" # show backtraces ENV RUST_BACKTRACE 1 @@ -22,8 +22,10 @@ USER root RUN mkdir -p /specs # add polkadot-parachain binary to the docker image -COPY ./release-artifacts/* /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ +COPY bin/* /usr/local/bin/ +COPY specs/* /specs/ + +RUN chmod -R a+rx "/usr/local/bin" USER parity diff --git a/docker/dockerfiles/polkadot/README.md b/docker/dockerfiles/polkadot/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e331d8984c2cee5e4b79a851809e303751e3bb57 --- /dev/null +++ b/docker/dockerfiles/polkadot/README.md @@ -0,0 +1,9 @@ +# Self built Docker image + +The Polkadot repo contains several options to build Docker images for Polkadot. + +This folder contains a self-contained image that does not require a Linux pre-built binary. + +Instead, building the image is possible on any host having docker installed and will +build Polkadot inside Docker. That also means that no Rust toolchain is required on the host +machine for the build to succeed. diff --git a/docker/dockerfiles/polkadot/docker-compose-local.yml b/docker/dockerfiles/polkadot/docker-compose-local.yml new file mode 100644 index 0000000000000000000000000000000000000000..1ff3a1ccaac252b5c8e5230b4584329ce6448865 --- /dev/null +++ b/docker/dockerfiles/polkadot/docker-compose-local.yml @@ -0,0 +1,50 @@ +version: '3' +services: + node_alice: + ports: + - "30333:30333" + - "9933:9933" + - "9944:9944" + - "9615:9615" + image: parity/polkadot:latest + volumes: + - "polkadot-data-alice:/data" + command: | + --chain=polkadot-local + --alice + -d /data + --node-key 0000000000000000000000000000000000000000000000000000000000000001 + networks: + testing_net: + ipv4_address: 172.28.1.1 + + node_bob: + ports: + - "30344:30333" + - "9935:9933" + - "9945:9944" + - "29615:9615" + image: parity/polkadot:latest + volumes: + - "polkadot-data-bob:/data" + links: + - "node_alice:alice" + command: | + --chain=polkadot-local + --bob + -d /data + --bootnodes '/ip4/172.28.1.1/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR' + networks: + testing_net: + ipv4_address: 172.28.1.2 + +volumes: + polkadot-data-alice: + polkadot-data-bob: + +networks: + testing_net: + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 diff --git a/docker/dockerfiles/polkadot/docker-compose.yml b/docker/dockerfiles/polkadot/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..524b1164796ac3f11f4fe30ec48a98e50e23c90f --- /dev/null +++ b/docker/dockerfiles/polkadot/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3' +services: + polkadot: + image: parity/polkadot:latest + + ports: + - "127.0.0.1:30333:30333/tcp" + - "127.0.0.1:9933:9933/tcp" + - "127.0.0.1:9944:9944/tcp" + - "127.0.0.1:9615:9615/tcp" + + volumes: + - "polkadot-data:/data" + + command: | + --unsafe-rpc-external + --unsafe-ws-external + --rpc-cors all + --prometheus-external + +volumes: + polkadot-data: diff --git a/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md b/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md new file mode 100644 index 0000000000000000000000000000000000000000..7e89cb55f3de5e016362cb50b5a5e5d0cc3e5713 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md @@ -0,0 +1,7 @@ +# Polkadot official Docker image + +## [Polkadot](https://polkadot.network/) + +## [GitHub](https://github.com/paritytech/polkadot) + +## [Polkadot Wiki](https://wiki.polkadot.network/) diff --git a/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile b/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f8dc374a14aab5f196952f9c33e66fa980ed50f0 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile @@ -0,0 +1,36 @@ +# This is the build stage for Polkadot. Here we create the binary in a temporary image. +FROM docker.io/paritytech/ci-linux:production as builder + +WORKDIR /polkadot +COPY . /polkadot + +RUN cargo build --locked --release + +# This is the 2nd stage: a very small image where we copy the Polkadot binary." +FROM docker.io/parity/base-bin:latest + +LABEL description="Multistage Docker image for Polkadot: a platform for web3" \ + io.parity.image.type="builder" \ + io.parity.image.authors="chevdor@gmail.com, devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.description="Polkadot: a platform for web3" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +COPY --from=builder /polkadot/target/release/polkadot /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ + mkdir -p /data /polkadot/.local/share && \ + chown -R polkadot:polkadot /data && \ + ln -s /data /polkadot/.local/share/polkadot && \ +# unclutter and minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ +# check if executable works in this container + /usr/local/bin/polkadot --version + +USER polkadot + +EXPOSE 30333 9933 9944 9615 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/local/bin/polkadot"] diff --git a/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7ad092476fec6cafbbfc3d7489a79955f2e80b99 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile @@ -0,0 +1,44 @@ +FROM docker.io/parity/base-bin + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG POLKADOT_VERSION +ARG POLKADOT_GPGKEY=9D4B2B6EB8F97156D19669A9FF0812D491B96798 +ARG GPG_KEYSERVER="keyserver.ubuntu.com" + +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="parity/polkadot" \ + io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +USER root + +# show backtraces +ENV RUST_BACKTRACE 1 + +RUN \ + apt-get update && \ + apt-get install -y --no-install-recommends polkadot=${POLKADOT_VERSION#?} && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* ; \ + mkdir -p /data /polkadot/.local/share && \ + chown -R parity:parity /data && \ + ln -s /data /polkadot/.local/share/polkadot + +USER parity + +# check if executable works in this container +RUN /usr/bin/polkadot --version +RUN /usr/lib/polkadot/polkadot-execute-worker --version +RUN /usr/lib/polkadot/polkadot-prepare-worker --version + +EXPOSE 30333 9933 9944 9615 +VOLUME ["/polkadot"] + +ENTRYPOINT ["/usr/bin/polkadot"] diff --git a/docker/polkadot_injected_debug.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile similarity index 94% rename from docker/polkadot_injected_debug.Dockerfile rename to docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile index f7f764d335a25d4517c319506b031688c122e4ac..80ce82589873dbbb75504a8a424341fe8c9d9ba2 100644 --- a/docker/polkadot_injected_debug.Dockerfile +++ b/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Polkadot: a platform for web3" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/docker/polkadot_injected_release.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile similarity index 95% rename from docker/polkadot_injected_release.Dockerfile rename to docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile index 87ae7ac27dc0c76d23d3f2752e5173803b429da8..c13f2db982a147c28991edee2d6e9aa2f1307291 100644 --- a/docker/polkadot_injected_release.Dockerfile +++ b/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile @@ -11,7 +11,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/polkadot" \ io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/docker/substrate_injected.Dockerfile b/docker/dockerfiles/substrate_injected.Dockerfile similarity index 100% rename from docker/substrate_injected.Dockerfile rename to docker/dockerfiles/substrate_injected.Dockerfile diff --git a/cumulus/docker/test-parachain-collator.dockerfile b/docker/dockerfiles/test-parachain-collator.dockerfile similarity index 96% rename from cumulus/docker/test-parachain-collator.dockerfile rename to docker/dockerfiles/test-parachain-collator.dockerfile index 9c2d8fbe5818fe241da70a401968aa45a39fbf44..0d56949152e2cf69538baf89fd3601e5ec73b53c 100644 --- a/cumulus/docker/test-parachain-collator.dockerfile +++ b/docker/dockerfiles/test-parachain-collator.dockerfile @@ -1,4 +1,4 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +# This file is sourced from https://github.com/paritytech/polkadot/blob/master/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile FROM docker.io/paritytech/ci-linux:production as builder WORKDIR /cumulus diff --git a/docker/test-parachain_injected.Dockerfile b/docker/dockerfiles/test-parachain_injected.Dockerfile similarity index 95% rename from docker/test-parachain_injected.Dockerfile rename to docker/dockerfiles/test-parachain_injected.Dockerfile index 0b345e16e4af572098fb4d5c062859bb42789cdc..e5d0df7aad673b48135ecb73928c53cf5c1a0c8a 100644 --- a/docker/test-parachain_injected.Dockerfile +++ b/docker/dockerfiles/test-parachain_injected.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Test parachain for Zombienet" \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/test-parachain_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/dockerfiles/test-parachain_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/cumulus/" diff --git a/docker/injected.Dockerfile b/docker/injected.Dockerfile deleted file mode 100644 index 93d0561ca87755052fc45f965becd76ee244065b..0000000000000000000000000000000000000000 --- a/docker/injected.Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/polkadot to /data - useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ - mkdir -p /data /polkadot/.local/share && \ - chown -R polkadot:polkadot /data && \ - ln -s /data /polkadot/.local/share/polkadot && \ - mkdir -p /specs - -# add polkadot-parachain binary to the docker image -COPY ./target/release/polkadot-parachain /usr/local/bin -COPY ./target/release/polkadot-parachain.asc /usr/local/bin -COPY ./target/release/polkadot-parachain.sha256 /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER polkadot - -# check if executable works in this container -RUN /usr/local/bin/polkadot-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/docker/parachain-registrar.dockerfile b/docker/parachain-registrar.dockerfile deleted file mode 100644 index f7d77454a2b9d37080047f1351c564c87f91824d..0000000000000000000000000000000000000000 --- a/docker/parachain-registrar.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM node:latest AS pjs - -# It would be great to depend on a more stable tag, but we need some -# as-yet-unreleased features. -RUN yarn global add @polkadot/api-cli@0.10.0-beta.14 - -ENTRYPOINT [ "polkadot-js-api" ] -CMD [ "--version" ] - -# To use the pjs build stage to access the blockchain from the host machine: -# -# docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . -# alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944' -# -# Then, as long as the chain is running, you can use the polkadot-js-api CLI like: -# -# pjs query.sudo.key - -FROM pjs -RUN apt-get update && apt-get install curl netcat -y && \ - curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \ - chmod +x /wait-for-it.sh -# the only thing left to do is to actually run the transaction. -COPY ./docker/scripts/register_para.sh /usr/bin -# unset the previous stage's entrypoint -ENTRYPOINT [] -CMD [ "/usr/bin/register_para.sh" ] diff --git a/docker/scripts/adder-collator/build-injected.sh b/docker/scripts/adder-collator/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..3a2d497413747628f3a7ab1f3bffc3502d85a3d8 --- /dev/null +++ b/docker/scripts/adder-collator/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=adder-collator,undying-collator +export ARTIFACTS_FOLDER=$1 + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/adder-collator/test-build.sh b/docker/scripts/adder-collator/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..171e0309f807971eaa79133f3de3b8d125787cbe --- /dev/null +++ b/docker/scripts/adder-collator/test-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# TODO: Switch to /bin/bash when the image is built from parity/base-bin + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which adder-collator)" /export' + +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which undying-collator)" /export' + +./build-injected.sh $TMP diff --git a/docker/scripts/build-injected.sh b/docker/scripts/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..f415cf43c0eeefecbe0fe2a0649bbe9df0e2f960 --- /dev/null +++ b/docker/scripts/build-injected.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +#set -e + +# This script allows building a Container Image from a Linux +# binary that is injected into a base-image. + +ENGINE=${ENGINE:-podman} + +if [ "$ENGINE" == "podman" ]; then + PODMAN_FLAGS="--format docker" +else + PODMAN_FLAGS="" +fi + +CONTEXT=$(mktemp -d) +REGISTRY=${REGISTRY:-docker.io} + +# The following line ensure we know the project root +PROJECT_ROOT=${PROJECT_ROOT:-$(git rev-parse --show-toplevel)} +DOCKERFILE=${DOCKERFILE:-docker/dockerfiles/binary_injected.Dockerfile} +VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)") + +#n The following VAR have default that can be overriden +DOCKER_OWNER=${DOCKER_OWNER:-parity} + +# We may get 1..n binaries, comma separated +BINARY=${BINARY:-polkadot} +IFS=',' read -r -a BINARIES <<< "$BINARY" + +VERSION=${VERSION:-$VERSION_TOML} +ARTIFACTS_FOLDER=${ARTIFACTS_FOLDER:-.} + +IMAGE=${IMAGE:-${REGISTRY}/${DOCKER_OWNER}/${BINARIES[0]}} +DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY}" +DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT} + +VCS_REF=${VCS_REF:-01234567} + +# Build the image +echo "Using engine: $ENGINE" +echo "Using Dockerfile: $DOCKERFILE" +echo "Using context: $CONTEXT" +echo "Building ${IMAGE}:latest container image for ${BINARY} v${VERSION} from ${ARTIFACTS_FOLDER} hang on!" +echo "ARTIFACTS_FOLDER=$ARTIFACTS_FOLDER" +echo "CONTEXT=$CONTEXT" + +# We need all binaries and resources available in the Container build "CONTEXT" +mkdir -p $CONTEXT/bin +for bin in "${BINARIES[@]}" +do + echo "Copying $ARTIFACTS_FOLDER/$bin to context: $CONTEXT/bin" + ls -al "$ARTIFACTS_FOLDER/$bin" + cp -r "$ARTIFACTS_FOLDER/$bin" "$CONTEXT/bin" +done + +cp "$PROJECT_ROOT/docker/scripts/entrypoint.sh" "$CONTEXT" + +if [[ "$BINARY" == "polkadot-parachain" ]]; then + mkdir -p "$CONTEXT/specs" + echo "Copying parachains chain-specs from $ARTIFACTS_FOLDER/specs to context: $CONTEXT/specs" + ls -al "$ARTIFACTS_FOLDER/specs" + cp -r "$ARTIFACTS_FOLDER/specs" "$CONTEXT/specs" +fi + +echo "Building image: ${IMAGE}" + +TAGS=${TAGS[@]:-latest} +IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" +TAG_ARGS=" " + +echo "The image ${IMAGE} will be tagged with ${TAG_ARRAY[*]}" +for tag in "${TAG_ARRAY[@]}"; do + TAG_ARGS+="--tag ${IMAGE}:${tag} " +done + +echo "$TAG_ARGS" + +# time \ +$ENGINE build \ + ${PODMAN_FLAGS} \ + --build-arg VCS_REF="${VCS_REF}" \ + --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + --build-arg IMAGE_NAME="${IMAGE}" \ + --build-arg BINARY="${BINARY}" \ + --build-arg ARTIFACTS_FOLDER="${ARTIFACTS_FOLDER}" \ + --build-arg DESCRIPTION="${DESCRIPTION}" \ + ${TAG_ARGS} \ + -f "${PROJECT_ROOT}/${DOCKERFILE}" \ + ${CONTEXT} + +echo "Your Container image for ${IMAGE} is ready" +$ENGINE images + +if [[ -z "${SKIP_IMAGE_VALIDATION}" ]]; then + echo "Check the image ${IMAGE}:${TAG_ARRAY[0]}" + $ENGINE run --rm -i "${IMAGE}:${TAG_ARRAY[0]}" --version + + echo "Query binaries" + $ENGINE run --rm -i --entrypoint /bin/bash "${IMAGE}:${TAG_ARRAY[0]}" -c "echo BINARY: ${BINARY}" +fi diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..eaa815faf6a45307047f07768826aa44f71afac0 --- /dev/null +++ b/docker/scripts/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Sanity check +if [ -z "$BINARY" ] +then + echo "BINARY ENV not defined, this should never be the case. Aborting..." + exit 1 +fi + +# If the user built the image with multiple binaries, +# we consider the first one to be the canonical one +# To start with another binary, the user can either: +# - use the --entrypoint option +# - pass the ENV BINARY with a single binary +IFS=',' read -r -a BINARIES <<< "$BINARY" +BIN0=${BINARIES[0]} +echo "Starting binary $BIN0" +$BIN0 $@ diff --git a/docker/scripts/malus/build-injected.sh b/docker/scripts/malus/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..83e30e178500fe1ec83351ab5f6c427c28f6290a --- /dev/null +++ b/docker/scripts/malus/build-injected.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=malus,polkadot-execute-worker,polkadot-prepare-worker +export ARTIFACTS_FOLDER=$1 +# export TAGS=... + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/malus/test-build.sh b/docker/scripts/malus/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..3114e9e2adf12c84663568f5fddf91e690621287 --- /dev/null +++ b/docker/scripts/malus/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + paritypr/malus:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/docker/scripts/polkadot-parachain/build-injected.sh b/docker/scripts/polkadot-parachain/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..f5c86a0351772801ea306031bc028d3eb4738c9d --- /dev/null +++ b/docker/scripts/polkadot-parachain/build-injected.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot-parachain +export ARTIFACTS_FOLDER=$1 +export DOCKERFILE="docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile" +# export TAGS=... + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/polkadot-parachain/test-build.sh b/docker/scripts/polkadot-parachain/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..1dc53bd0d0b7b7afa0e8506b49042d25db662797 --- /dev/null +++ b/docker/scripts/polkadot-parachain/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot-parachain:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/cumulus/docker/scripts/build-injected-image.sh b/docker/scripts/polkadot-parachain_build-injected-image.sh similarity index 70% rename from cumulus/docker/scripts/build-injected-image.sh rename to docker/scripts/polkadot-parachain_build-injected-image.sh index b8bb0dd7dd2c11fe6bd7a755b8abe8b8c9ff0af6..bb6909dd3b7b78763763509c9dc6a324d84ca347 100755 --- a/cumulus/docker/scripts/build-injected-image.sh +++ b/docker/scripts/polkadot-parachain_build-injected-image.sh @@ -6,5 +6,5 @@ IMAGE_NAME=${IMAGE_NAME:-polkadot-parachain} docker build --no-cache \ --build-arg IMAGE_NAME=$IMAGE_NAME \ -t $OWNER/$IMAGE_NAME \ - -f ./docker/injected.Dockerfile \ + -f ./docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile \ . && docker images diff --git a/docker/scripts/polkadot/build-injected.sh b/docker/scripts/polkadot/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..7cc6db43a54a6b1a54fa4e917ed82185d28e93d0 --- /dev/null +++ b/docker/scripts/polkadot/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot,polkadot-execute-worker,polkadot-prepare-worker +export ARTIFACTS_FOLDER=$1 + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/polkadot/test-build.sh b/docker/scripts/polkadot/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..d2d904561cb559d0dbb6d62e94cefa8d01213a06 --- /dev/null +++ b/docker/scripts/polkadot/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot -c \ + 'cp "$(which polkadot)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/docker/staking-miner/staking-miner_builder.Dockerfile b/docker/staking-miner/staking-miner_builder.Dockerfile deleted file mode 100644 index a1932095fd4cab0130009ebf00953fbf09955142..0000000000000000000000000000000000000000 --- a/docker/staking-miner/staking-miner_builder.Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM paritytech/ci-linux:production as builder - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" -ARG PROFILE=release - -LABEL description="This is the build stage. Here we create the binary." - -WORKDIR /app -COPY . /app -RUN cargo build --locked --$PROFILE --package staking-miner - -# ===== SECOND STAGE ====== - -FROM docker.io/library/ubuntu:20.04 -LABEL description="This is the 2nd stage: a very small image where we copy the binary." -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -ARG PROFILE=release -COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin - -RUN useradd -u 1000 -U -s /bin/sh miner && \ - rm -rf /usr/bin /usr/sbin - -# show backtraces -ENV RUST_BACKTRACE 1 - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/staking-miner/staking-miner_injected.Dockerfile b/docker/staking-miner/staking-miner_injected.Dockerfile deleted file mode 100644 index 4901ab4a3736ea6fa1622ae6b8bb3216e95bb1b5..0000000000000000000000000000000000000000 --- a/docker/staking-miner/staking-miner_injected.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - useradd -u 1000 -U -s /bin/sh miner - -# add binary to docker image -COPY ./staking-miner /usr/local/bin - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/test-parachain-collator.dockerfile b/docker/test-parachain-collator.dockerfile deleted file mode 100644 index 9c2d8fbe5818fe241da70a401968aa45a39fbf44..0000000000000000000000000000000000000000 --- a/docker/test-parachain-collator.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile -FROM docker.io/paritytech/ci-linux:production as builder - -WORKDIR /cumulus -COPY . /cumulus - -RUN cargo build --release --locked -p polkadot-parachain - -# the collator stage is normally built once, cached, and then ignored, but can -# be specified with the --target build flag. This adds some extra tooling to the -# image, which is required for a launcher script. The script simply adds two -# arguments to the list passed in: -# -# --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/PEER_ID -# -# with the appropriate ip and ID for both Alice and Bob -FROM debian:buster-slim as collator -RUN apt-get update && apt-get install jq curl bash -y && \ - curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \ - chmod +x /wait-for-it.sh && \ - curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ - apt-get install -y nodejs && \ - npm install --global yarn && \ - yarn global add @polkadot/api-cli@0.10.0-beta.14 -COPY --from=builder \ - /paritytech/cumulus/target/release/polkadot-parachain /usr/bin -COPY ./docker/scripts/inject_bootnodes.sh /usr/bin -CMD ["/usr/bin/inject_bootnodes.sh"] -COPY ./docker/scripts/healthcheck.sh /usr/bin/ -HEALTHCHECK --interval=300s --timeout=75s --start-period=30s --retries=3 \ - CMD ["/usr/bin/healthcheck.sh"] - -# the runtime stage is normally built once, cached, and ignored, but can be -# specified with the --target build flag. This just preserves one of the builder's -# outputs, which can then be moved into a volume at runtime -FROM debian:buster-slim as runtime -COPY --from=builder \ - /paritytech/cumulus/target/release/wbuild/cumulus-test-parachain-runtime/cumulus_test_parachain_runtime.compact.wasm \ - /var/opt/ -CMD ["cp", "-v", "/var/opt/cumulus_test_parachain_runtime.compact.wasm", "/runtime/"] - -FROM debian:buster-slim -COPY --from=builder \ - /paritytech/cumulus/target/release/polkadot-parachain /usr/bin - -CMD ["/usr/bin/polkadot-parachain"] diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 5e892625fb77e7b6dc2784d315414b0191f4ba0a..245369b702012317c7fdc02d48cc792f1386b848 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -32,30 +32,30 @@ If it is an urgent fix with no large change to logic, then it may be merged afte contributor has reviewed it well and approved the review once CI is complete. No PR should be merged until all reviews' comments are addressed. -### Labels: +### Labels The set of labels and their description can be found [here](https://paritytech.github.io/labels/doc_polkadot-sdk.html). -### Process: +### Process 1. Please use our [Pull Request Template](./PULL_REQUEST_TEMPLATE.md) and make sure all relevant information is reflected in your PR. 2. Please tag each PR with minimum one `T*` label. The respective `T*` labels should signal the component that was changed, they are also used by downstream users to track changes and to include these changes properly into their own releases. -3. If your’re still working on your PR, please submit as “Draft”. Once a PR is ready for review change +3. If you’re still working on your PR, please submit as “Draft”. Once a PR is ready for review change the status to “Open”, so that the maintainers get to review your PR. Generally PRs should sit for 48 hours in order to garner feedback. It may be merged before if all relevant parties had a look at it. 4. If you’re introducing a major change, that might impact the documentation please add the label `T13-documentation`. The docs team will get in touch. 5. If your PR changes files in these paths: - `polkadot` : '^runtime/polkadot' - `polkadot` : '^runtime/kusama' - `polkadot` : '^primitives/src/' - `polkadot` : '^runtime/common' - `substrate` : '^frame/' - `substrate` : '^primitives/' + `polkadot` : `^runtime/polkadot` + `polkadot` : `^runtime/kusama` + `polkadot` : `^primitives/src/` + `polkadot` : `^runtime/common` + `substrate` : `^frame/` + `substrate` : `^primitives/` It should be added to the [security audit board](https://github.com/orgs/paritytech/projects/103) and will need to undergo an audit before merge. @@ -67,7 +67,7 @@ to change the code to make it work/compile. It should also mention potential storage migrations and if they require some special setup aside adding it to the list of migrations in the runtime. -## Reviewing pull requests: +## Reviewing pull requests When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in: @@ -80,18 +80,36 @@ Reviews should finish with approval unless there are issues that would result in The reviewers are also responsible to check: -a) if a changelog is necessary and attached - -b) the quality of information in the changelog file - -c) the PR has an impact on docs - -d) that the docs team was included in the review process of a docs update +1. if a changelog is necessary and attached +1. the quality of information in the changelog file +1. the PR has an impact on docs +1. that the docs team was included in the review process of a docs update **Reviews may not be used as an effective veto for a PR because**: 1. There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix. 2. It does not fit well with some other contributors' longer-term vision for the project. +## Documentation + +All Pull Requests must contain proper title & description. + +Some Pull Requests can be exempt of `prdoc` documentation, those +must be labelled with +[`R0-silent`](https://github.com/paritytech/labels/blob/main/ruled_labels/specs_polkadot-sdk.yaml#L89-L91). + +Non "silent" PRs must come with documentation in the form of a `.prdoc` file. +A `.prdoc` documentation is made of a text file (YAML) named `/prdoc/pr_NNNN.prdoc` where `NNNN` is the PR number. +For convenience, those file can also contain a short description/title: `/prdoc/pr_NNNN_pr-foobar.prdoc`. + +The CI automation checks for the presence and validity of a `prdoc` in the `/prdoc` folder. +Those files need to comply with a specific [schema](https://github.com/paritytech/prdoc/blob/master/schema_user.json). It +is highly recommended to [make your editor aware](https://github.com/paritytech/prdoc#schemas) of the schema as it is +self-described and will assist you in writing correct content. + +This schema is also embedded in the +[prdoc](https://github.com/paritytech/prdoc) utility that can also be used to generate and check the validity of a +`prdoc` locally. + ## Helping out We use [labels](https://github.com/paritytech/polkadot-sdk/labels) to manage PRs and issues and communicate @@ -125,4 +143,4 @@ UI tests are used for macros to ensure that the output of a macro doesn’t chan These UI tests are sensible to any changes in the macro generated code or to switching the rust stable version. The tests are only run when the `RUN_UI_TESTS` environment variable is set. So, when the CI is for example complaining about failing UI tests and it is expected that they fail these tests need to be executed locally. -To simplify the updating of the UI test ouput there is the `.maintain/update-rust-stable +To simplify the updating of the UI test output there is the `.maintain/update-rust-stable diff --git a/docs/DOCUMENTATION_GUIDELINE.md b/docs/DOCUMENTATION_GUIDELINES.md similarity index 50% rename from docs/DOCUMENTATION_GUIDELINE.md rename to docs/DOCUMENTATION_GUIDELINES.md index 8236173295902bc414cb992f26ef2068fd0ab66c..dbb4298d50fe7fa3435eff53be368fe9b8ba5d61 100644 --- a/docs/DOCUMENTATION_GUIDELINE.md +++ b/docs/DOCUMENTATION_GUIDELINES.md @@ -1,11 +1,10 @@ # Substrate Documentation Guidelines -This document is focused on documenting parts of substrate that relate to its -external API. The list of such crates can be found in [CODEOWNERS](./CODEOWNERS). -Search for the crates auto-assigned to the `docs-audit` team. +This document is focused on documenting parts of Substrate that relate to its external API. The list of such crates can +be found in [CODEOWNERS](./CODEOWNERS). Search for the crates auto-assigned to the `docs-audit` team. -These crates are used by external developers and need thorough documentation. -They are the most concerned with FRAME development. +These crates are used by external developers and need thorough documentation. They are the most concerned with FRAME +development. - [Substrate Documentation Guidelines](#substrate-documentation-guidelines) - [General/Non-Pallet Crates](#generalnon-pallet-crates) @@ -35,22 +34,19 @@ First, consider the case for all such crates, except for those that are pallets. The first question is, what should you document? Use this filter: 1. In the crates assigned to `docs-audit` in [CODEOWNERS](./CODEOWNERS), -2. All `pub` items need to be documented. If not `pub`, it doesn't appear in the -rust-docs, and is not public facing. +2. All `pub` items need to be documented. If not `pub`, it doesn't appear in the rust-docs, and is not public facing. - * Within `pub` items, sometimes they are only `pub` to be used by another - internal crate, and you can foresee that this won't be used by anyone else. - These need **not** be documented thoroughly. + - Within `pub` items, sometimes they are only `pub` to be used by another internal crate, and you can foresee that + this won't be used by anyone else. These need **not** be documented thoroughly. - * Reminder: `trait` items are public by definition if the trait is public. + - Reminder: `trait` items are public by definition if the trait is public. 3. All public modules (`mod`) should have reasonable module-level documentation (`//!`). #### Rust Docs vs. Code Comments -Note that anything starting with `///` is an external rust-doc, and everything -starting with `//` does not appear in the rust-docs. -It's important to not confuse the two in your documentation. +Note that anything starting with `///` is an external rust-doc, and everything starting with `//` does not appear in the +rust-docs. It's important to not confuse the two in your documentation. ```rust /// Computes the square root of the input, returning `Ok(_)` if successful. @@ -73,100 +69,88 @@ pub fn sqrt(x: u32) -> Result { There are good sources to look into: - [Rust Documentation Guide](https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html) -- [Documentation in Rust Book](https://doc.rust-lang.org/book/ch14-02-publishing-to-crates-io.html#making-useful-documentation-comments) -- [Guide on Writing Documentation for a Rust Crate](https://blog.guillaume-gomez.fr/articles/2020-03-12+Guide+on+how+to+write+documentation+for+a+Rust+crate) - -As mentioned [here](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/documentation.html#writing-documentation-comments) and [here](https://blog.guillaume-gomez.fr/articles/2020-03-12+Guide+on+how+to+write+documentation+for+a+Rust+crate), -always start with a **single sentence** demonstrating what is documented. All additional -documentation should be added *after a newline*. Strive to make the first sentence succinct -and short.The reason for this is the first paragraph of docs about an item (everything -before the first newline) is used as the excerpt that rust doc displays about -this item when it appears in tables, such as the table listing all functions in -a module. If this excerpt is too long, the module docs will be very difficult -to read. - -About [special sections](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/documentation.html#special-sections), we will most likely not need to think about panic and safety in any runtime related code. Our code is never `unsafe`, and will (almost) never panic. - -Use `# Examples as much as possible. These are great ways to further -demonstrate what your APIs are doing, and add free test coverage. As an -additional benefit, any code in rust-docs is treated as an "integration tests", -not unit tests, which tests your crate in a different way than unit tests. So, -it is both a win for "more documentation" and a win for "more test coverage". - -You can also consider having an `# Error` section optionally. Of course, this -only applies if there is a `Result` being returned, and if the `Error` variants -are overly complicated. - -Strive to include correct links to other items in your written docs as much as -possible. In other words, avoid \`some_func\` and instead use \[\`some_fund\`\]. -Read more about how to correctly use links in your rust-docs -[here](https://doc.rust-lang.org/rustdoc/write-documentation/linking-to-items-by-name.html#valid-links) -and [here](https://rust-lang.github.io/rfcs/1946-intra-rustdoc-links.html#additions-to-the-documentation-syntax). -Strive to include correct links to other items in your written docs as much as -possible. In other words, avoid `` `some_func` `` and instead use -``[`some_func`]``. - -> While you are linking, you might become conscious of the fact that you are -in need of linking to (too many) foreign items in order to explain your API. -This is leaning more towards API-Design rather than documentation, but it is a -warning that the subject API might be slightly wrong. For example, most "glue" -traits[^1] in `frame/support` should be designed and documented without making -hard assumptions about particular pallets that implement them. +- [Documentation in Rust + Book](https://doc.rust-lang.org/book/ch14-02-publishing-to-crates-io.html#making-useful-documentation-comments) +- [Guide on Writing Documentation for a Rust + Crate](https://blog.guillaume-gomez.fr/articles/2020-03-12+Guide+on+how+to+write+documentation+for+a+Rust+crate) + +As mentioned +[here](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/documentation.html#writing-documentation-comments) +and [here](https://blog.guillaume-gomez.fr/articles/2020-03-12+Guide+on+how+to+write+documentation+for+a+Rust+crate), +always start with a **single sentence** demonstrating what is documented. All additional documentation should be added +*after a newline*. Strive to make the first sentence succinct and short.The reason for this is the first paragraph of +docs about an item (everything before the first newline) is used as the excerpt that rust doc displays about this item +when it appears in tables, such as the table listing all functions in a module. If this excerpt is too long, the module +docs will be very difficult to read. + +About [special +sections](https://web.mit.edu/rust-lang_v1.25/arch/amd64_ubuntu1404/share/doc/rust/html/book/first-edition/documentation.html#special-sections), +we will most likely not need to think about panic and safety in any runtime related code. Our code is never `unsafe`, +and will (almost) never panic. + +Use `# Examples as much as possible. These are great ways to further demonstrate what your APIs are doing, and add free +test coverage. As an additional benefit, any code in rust-docs is treated as an "integration tests", not unit tests, +which tests your crate in a different way than unit tests. So, it is both a win for "more documentation" and a win for +"more test coverage". + +You can also consider having an `# Error` section optionally. Of course, this only applies if there is a `Result` being +returned, and if the `Error` variants are overly complicated. + +Strive to include correct links to other items in your written docs as much as possible. In other words, avoid +\`some_func\` and instead use \[\`some_fund\`\]. Read more about how to correctly use links in your rust-docs +[here](https://doc.rust-lang.org/rustdoc/write-documentation/linking-to-items-by-name.html#valid-links) and +[here](https://rust-lang.github.io/rfcs/1946-intra-rustdoc-links.html#additions-to-the-documentation-syntax). Strive to +include correct links to other items in your written docs as much as possible. In other words, avoid `` `some_func` `` +and instead use ``[`some_func`]``. + +> While you are linking, you might become conscious of the fact that you are in need of linking to (too many) foreign +items in order to explain your API. This is leaning more towards API-Design rather than documentation, but it is a +warning that the subject API might be slightly wrong. For example, most "glue" traits[^1] in `frame/support` should be +designed and documented without making hard assumptions about particular pallets that implement them. --- #### TLDR -0. Have the goal of enforcing `#![deny(missing_docs)]` mentally, even if it is -not enforced by the compiler 🙈. -1. Start with a single, clear and concise sentence. Follow up with more context, -after a newline, if needed. +0. Have the goal of enforcing `#![deny(missing_docs)]` mentally, even if it is not enforced by the compiler 🙈. +1. Start with a single, clear and concise sentence. Follow up with more context, after a newline, if needed. 2. Use examples as much as reasonably possible. 3. Use links as much as possible. -4. Think about context. If you are explaining a lot of foreign topics while -documenting a trait that should not explicitly depend on them, you have likely -not designed it properly. +4. Think about context. If you are explaining a lot of foreign topics while documenting a trait that should not +explicitly depend on them, you have likely not designed it properly. --- #### Proc-Macros -Note that there are special considerations when documenting proc macros. Doc -links will appear to function _within_ your proc macro crate, but often will no -longer function when these proc macros are re-exported elsewhere in your -project. The exception is doc links to _other proc macros_ which will function -just fine if they are also being re-exported. It is also often necessary to -disambiguate between a proc macro and a function of the same name, which can be -done using the `macro@my_macro_name` syntax in your link. Read more about how to -correctly use links in your rust-docs [here](https://doc.rust-lang.org/rustdoc/write-documentation/linking-to-items-by-name.html#valid-links) -and [here](https://rust-lang.github.io/rfcs/1946-intra-rustdoc-links.html#additions-to-the-documentation-syntax). +Note that there are special considerations when documenting proc macros. Doc links will appear to function *within* your +proc macro crate, but often will no longer function when these proc macros are re-exported elsewhere in your project. +The exception is doc links to *other proc macros* which will function just fine if they are also being re-exported. It +is also often necessary to disambiguate between a proc macro and a function of the same name, which can be done using +the `macro@my_macro_name` syntax in your link. Read more about how to correctly use links in your rust-docs +[here](https://doc.rust-lang.org/rustdoc/write-documentation/linking-to-items-by-name.html#valid-links) and +[here](https://rust-lang.github.io/rfcs/1946-intra-rustdoc-links.html#additions-to-the-documentation-syntax). --- ### Other Guidelines -The above five guidelines must always be reasonably respected in the -documentation. +The above five guidelines must always be reasonably respected in the documentation. -The following are a set of notes that may not necessarily hold in all -circumstances: +The following are a set of notes that may not necessarily hold in all circumstances: --- #### Document Through Code -You should make sure that your code is properly-named and well-organized so that -your code functions as a form of documentation. However, within the complexity -of our projects in Polkadot/Substrate that is not enough. Particularly, things -like examples, errors and panics cannot be documented only through properly- -named and well-organized code. +You should make sure that your code is properly-named and well-organized so that your code functions as a form of +documentation. However, within the complexity of our projects in Polkadot/Substrate that is not enough. Particularly, +things like examples, errors and panics cannot be documented only through properly- named and well-organized code. -> Our north star is self-documenting code that also happens to be well-documented -and littered with examples. +> Our north star is self-documenting code that also happens to be well-documented and littered with examples. -* Your written documents should *complement* the code, not *repeat* it. As an -example, a documentation on top of a code example should never look like the -following: +- Your written documents should *complement* the code, not *repeat* it. As an example, a documentation on top of a code +example should never look like the following: ```rust /// Sends request and handles the response. @@ -175,15 +159,14 @@ following: } ``` -In the above example, the documentation has added no useful information not -already contained within the properly-named trait and is redundant. +In the above example, the documentation has added no useful information not already contained within the properly-named +trait and is redundant. --- #### Formatting Matters -The way you format your documents (newlines, heading and so on) makes a -difference. Consider the below examples: +The way you format your documents (newlines, heading and so on) makes a difference. Consider the below examples: ```rust /// This function works with input u32 x and multiplies it by two. If @@ -206,30 +189,28 @@ fn multiply_by_2(x: u32) -> u32 { .. } // More efficiency can be achieved if we improve this via such and such. fn multiply_by_2(x: u32) -> u32 { .. } ``` -They are both roughly conveying the same set of facts, but one is easier to -follow because it was formatted cleanly. Especially for traits and types that -you can foresee will be seen and used a lot, try and write a well formatted + +They are both roughly conveying the same set of facts, but one is easier to follow because it was formatted cleanly. +Especially for traits and types that you can foresee will be seen and used a lot, try and write a well formatted version. -Similarly, make sure your comments are wrapped at 100 characters line-width (as -defined by our [`rustfmt.toml`](../rustfmt.toml)), no **more and no less**! The -more is fixed by `rustfmt` and our CI, but if you (for some unknown reason) -wrap your lines at 59 characters, it will pass the CI, and it will not look good -🫣. Consider using a plugin like [rewrap](https://marketplace.visualstudio.com/items?itemName=stkb.rewrap) (for Visual Studio Code) to properly do this. +Similarly, make sure your comments are wrapped at 100 characters line-width (as defined by our +[`rustfmt.toml`](../rustfmt.toml)), no **more and no less**! The more is fixed by `rustfmt` and our CI, but if you (for +some unknown reason) wrap your lines at 59 characters, it will pass the CI, and it will not look good 🫣. Consider using +a plugin like [rewrap](https://marketplace.visualstudio.com/items?itemName=stkb.rewrap) (for Visual Studio Code) to +properly do this. [^1]: Those that help two pallets talk to each other. --- - ## Pallet Crates -The guidelines so far have been general in nature, and are applicable to crates -that are pallets and crates that're not pallets. +The guidelines so far have been general in nature, and are applicable to crates that are pallets and crates that're not +pallets. -The following is relevant to how to document parts of a crate that is a pallet. -See [`pallet-fast-unstake`](../frame/fast-unstake/src/lib.rs) as one example of -adhering these guidelines. +The following is relevant to how to document parts of a crate that is a pallet. See +[`pallet-fast-unstake`](../frame/fast-unstake/src/lib.rs) as one example of adhering these guidelines. --- @@ -242,6 +223,14 @@ For the top-level pallet docs, consider the following template: //! //! . //! +//! ## Pallet API +//! +//! +//! +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, including its +//! configuration trait, dispatchables, storage items, events and errors. +//! //! ## Overview //! //! @@ -252,15 +241,12 @@ For the top-level pallet docs, consider the following template: //! //! ### Example //! -//! . +//! //! -//! ## Pallet API -//! -//! -//! -//! See the [`pallet`] module for more information about the interfaces this pallet exposes, including its configuration trait, dispatchables, storage items, events and errors. -//! -//! +//! //! //! This section can most often be left as-is. //! @@ -268,7 +254,8 @@ For the top-level pallet docs, consider the following template: //! //! //! -//! +//! //! //! ### Design Goals (optional) //! @@ -276,31 +263,30 @@ For the top-level pallet docs, consider the following template: //! //! ### Design (optional) //! -//! +//! //! //! ### Terminology (optional) //! -//! +//! ``` - -This template's details (heading 3s and beyond) are left flexible, and at the -discretion of the developer to make the best final choice about. For example, -you might want to include `### Terminology` or not. Moreover, you might find it +This template's details (heading 3s and beyond) are left flexible, and at the discretion of the developer to make the +best final choice about. For example, you might want to include `### Terminology` or not. Moreover, you might find it more useful to include it in `## Overview`. -Nonetheless, the high level flow of going from the most high level explanation -to the most low level explanation is important to follow. +Nonetheless, the high level flow of going from the most high level explanation to the most low level explanation is +important to follow. -As a rule of thumb, the Heading 2s (`##`) in this template can be considered a -strict rule, while the Heading 3s (`###`) and beyond are flexible. +As a rule of thumb, the Heading 2s (`##`) in this template can be considered a strict rule, while the Heading 3s (`###`) +and beyond are flexible. --- #### Polkadot and Substrate -Optionally, in order to demonstrate the relation between the two, you can start -the pallet documentation with: +Optionally, in order to demonstrate the relation between the two, you can start the pallet documentation with: ``` //! > Made with *Substrate*, for *Polkadot*. @@ -331,7 +317,8 @@ For each dispatchable (`fn` item inside `#[pallet::call]`), consider the followi /// /// ## Errors (optional) /// -/// +/// /// /// ## Events (optional) /// @@ -339,29 +326,26 @@ For each dispatchable (`fn` item inside `#[pallet::call]`), consider the followi pub fn name_of_dispatchable(origin: OriginFor, ...) -> DispatchResult {} ``` -Consider the fact that these docs will be part of the metadata of the associated dispatchable, and might be used by wallets and explorers. +Consider the fact that these docs will be part of the metadata of the associated dispatchable, and might be used by +wallets and explorers. --- ### Storage Items -1. If a map-like type is being used, always note the choice of your hashers as -private code docs (`// Hasher X chosen because ...`). Recall that this is not -relevant information to external people, so it must be documented as `//`. +1. If a map-like type is being used, always note the choice of your hashers as private code docs (`// Hasher X chosen +because ...`). Recall that this is not relevant information to external people, so it must be documented as `//`. -2. Consider explaining the crypto-economics of how a deposit is being taken in -return of the storage being used. +2. Consider explaining the crypto-economics of how a deposit is being taken in return of the storage being used. -3. Consider explaining why it is safe for the storage item to be unbounded, if -`#[pallet::unbounded]` or `#[pallet::without_storage_info]` is being used. +3. Consider explaining why it is safe for the storage item to be unbounded, if `#[pallet::unbounded]` or +`#[pallet::without_storage_info]` is being used. --- ### Errors and Events -Consider the fact that, similar to dispatchables, these docs will be part of -the metadata of the associated event/error, and might be used by wallets and -explorers. +Consider the fact that, similar to dispatchables, these docs will be part of the metadata of the associated event/error, +and might be used by wallets and explorers. -Specifically for `error`, explain why the error has happened, and what can be -done in order to avoid it. +Specifically for `error`, explain why the error has happened, and what can be done in order to avoid it. diff --git a/docs/PULL_REQUEST_TEMPLATE.md b/docs/PULL_REQUEST_TEMPLATE.md index 1d0ef338c10b1acd1e27d3c265d6e7a35ebceb8a..284ad15afc9e6c5b6ffe5f666d5a5bfa43da550b 100644 --- a/docs/PULL_REQUEST_TEMPLATE.md +++ b/docs/PULL_REQUEST_TEMPLATE.md @@ -2,25 +2,27 @@ ✄ ----------------------------------------------------------------------------- -Thank you for your Pull Request! 🙏 Please make sure it follows the contribution -guidelines outlined in [this document](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CONTRIBUTING.md) and fill out the -sections below. Once you're ready to submit your PR for review, please delete -this section and leave only the text under the "Description" heading. - +Thank you for your Pull Request! 🙏 Please make sure it follows the contribution guidelines outlined in +[this document](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CONTRIBUTING.md) and fill +out the sections below. Once you're ready to submit your PR for review, please +delete this section and leave only the text under the "Description" heading. # Description -*Please include a summary of the changes and the related issue. Please also include relevant motivation and context, including:* +*Please include a summary of the changes and the related issue. Please also include relevant motivation and context, +including:* - What does this PR do? - Why are these changes needed? - How were these changes implemented and what do they affect? -*Use [Github semantic linking](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) to address any open issues this PR relates to or closes.* +*Use [Github semantic +linking](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) +to address any open issues this PR relates to or closes.* -Fixes # (issue number, *if applicable*) +Fixes # (issue number, *if applicable*) -Closes # (issue number, *if applicable*) +Closes # (issue number, *if applicable*) Polkadot companion: (*if applicable*) @@ -29,10 +31,12 @@ Cumulus companion: (*if applicable*) # Checklist - [ ] My PR includes a detailed description as outlined in the "Description" section above -- [ ] My PR follows the [labeling requirements](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CONTRIBUTING.md#process) of this project (at minimum one label for `T` required) +- [ ] My PR follows the [labeling requirements](CONTRIBUTING.md#Process) of this project (at minimum one label for `T` + required) - [ ] I have made corresponding changes to the documentation (if applicable) - [ ] I have added tests that prove my fix is effective or that my feature works (if applicable) -- [ ] If this PR alters any external APIs or interfaces used by Polkadot, the corresponding Polkadot PR is ready as well as the corresponding Cumulus PR (optional) +- [ ] If this PR alters any external APIs or interfaces used by Polkadot, the corresponding Polkadot PR is ready as well + as the corresponding Cumulus PR (optional) You can remove the "Checklist" section once all have been checked. Thank you for your contribution! diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.md index eb3399880f534c472bffad54d5cc3b13556f65c1..a9ac4a5910b3547c44612313bbc9745e02febf1d 100644 --- a/docs/STYLE_GUIDE.md +++ b/docs/STYLE_GUIDE.md @@ -2,19 +2,19 @@ title: Style Guide for Rust in the Polkadot-SDK --- -Where possible these styles are enforced by settings in `rustfmt.toml` so if you run `cargo fmt` +Where possible these styles are enforced by settings in `rustfmt.toml` so if you run `cargo fmt` then you will adhere to most of these style guidelines automatically. # Formatting -- Indent using tabs. -- Lines should be longer than 100 characters long only in exceptional circumstances and certainly +- Indent using tabs. +- Lines should be longer than 100 characters long only in exceptional circumstances and certainly no longer than 120. For this purpose, tabs are considered 4 characters wide. -- Indent levels should be greater than 5 only in exceptional circumstances and certainly no +- Indent levels should be greater than 5 only in exceptional circumstances and certainly no greater than 8. If they are greater than 5, then consider using `let` or auxiliary functions in order to strip out complex inline expressions. -- Never have spaces on a line prior to a non-whitespace character -- Follow-on lines are only ever a single indent from the original line. +- Never have spaces on a line prior to a non-whitespace character +- Follow-on lines are only ever a single indent from the original line. ```rust fn calculation(some_long_variable_a: i8, some_long_variable_b: i8) -> bool { @@ -25,7 +25,7 @@ fn calculation(some_long_variable_a: i8, some_long_variable_b: i8) -> bool { } ``` -- Indent level should follow open parens/brackets, but should be collapsed to the smallest number +- Indent level should follow open parens/brackets, but should be collapsed to the smallest number of levels actually used: ```rust @@ -45,8 +45,8 @@ fn calculate( } ``` -- `where` is indented, and its items are indented one further. -- Argument lists or function invocations that are too long to fit on one line are indented +- `where` is indented, and its items are indented one further. +- Argument lists or function invocations that are too long to fit on one line are indented similarly to code blocks, and once one param is indented in such a way, all others should be, too. Run-on parameter lists are also acceptable for single-line run-ons of basic function calls. @@ -92,7 +92,7 @@ fn foo(really_long_parameter_name_1: SomeLongTypeName, really_long_parameter_nam } ``` -- Always end last item of a multi-line comma-delimited set with `,` when legal: +- Always end last item of a multi-line comma-delimited set with `,` when legal: ```rust struct Point { @@ -104,7 +104,7 @@ struct Point { enum Meal { Breakfast, Lunch, Dinner }; ``` -- Avoid trailing `;`s where unneeded. +- Avoid trailing `;`s where unneeded. ```rust if condition { @@ -112,8 +112,8 @@ if condition { } ``` -- `match` arms may be either blocks or have a trailing `,` but not both. -- Blocks should not be used unnecessarily. +- `match` arms may be either blocks or have a trailing `,` but not both. +- Blocks should not be used unnecessarily. ```rust match meal { @@ -126,7 +126,7 @@ match meal { # Style -- Panickers require explicit proofs they don't trigger. Calling `unwrap` is discouraged. The +- Panickers require explicit proofs they don't trigger. Calling `unwrap` is discouraged. The exception to this rule is test code. Avoiding panickers by restructuring code is preferred if feasible. @@ -139,14 +139,14 @@ let mut target_path = ); ``` -- Unsafe code requires explicit proofs just as panickers do. When introducing unsafe code, +- Unsafe code requires explicit proofs just as panickers do. When introducing unsafe code, consider trade-offs between efficiency on one hand and reliability, maintenance costs, and security on the other. Here is a list of questions that may help evaluating the trade-off while preparing or reviewing a PR: - - how much more performant or compact the resulting code will be using unsafe code, - - how likely is it that invariants could be violated, - - are issues stemming from the use of unsafe code caught by existing tests/tooling, - - what are the consequences if the problems slip into production. + - how much more performant or compact the resulting code will be using unsafe code, + - how likely is it that invariants could be violated, + - are issues stemming from the use of unsafe code caught by existing tests/tooling, + - what are the consequences if the problems slip into production. # Manifest Formatting @@ -177,4 +177,4 @@ default = [ # Comments go here as well ;) "std", ] -``` \ No newline at end of file +``` diff --git a/cumulus/docs/container.md b/docs/container.md similarity index 55% rename from cumulus/docs/container.md rename to docs/container.md index 63575f37a59b93fc31d8d09030b1d97153566a42..afd3b27957c259501a85df76f978ba9f3af2db2b 100644 --- a/cumulus/docs/container.md +++ b/docs/container.md @@ -1,19 +1,22 @@ -## Using Containers +# Using Containers -Using containers via **Podman** or **Docker** brings benefit, whether it is to build a container image or -run a node while keeping a minimum footprint on your local system. +Using containers via **Podman** or **Docker** brings benefit, whether it is to build a container image or run a node +while keeping a minimum footprint on your local system. -This document mentions using `podman` or `docker`. Those are usually interchangeable and it is encouraged using preferably **Podman**. If you have podman installed and want to use all the commands mentioned below, you can simply create an alias with `alias docker=podman`. +This document mentions using `podman` or `docker`. Those are usually interchangeable and it is encouraged using +preferably **Podman**. If you have podman installed and want to use all the commands mentioned below, you can simply +create an alias with `alias docker=podman`. There are a few options to build a node within a container and inject a binary inside an image. -### Parity built container image +## Parity built container image Parity builds and publishes a container image that can be found as `docker.io/parity/polkadot-parachain`. -### Parity CI image +## Parity CI image -Parity maintains and uses internally a generic "CI" image that can be used as a base to build binaries: [Parity CI container image](https://github.com/paritytech/scripts/tree/master/dockerfiles/ci-linux): +Parity maintains and uses internally a generic "CI" image that can be used as a base to build binaries: [Parity CI +container image](https://github.com/paritytech/scripts/tree/master/dockerfiles/ci-linux): The command below allows building a Linux binary without having to even install Rust or any dependency locally: @@ -29,24 +32,27 @@ sudo chown -R $(id -u):$(id -g) target/ If you want to reproduce other steps of CI process you can use the following [guide](https://github.com/paritytech/scripts#gitlab-ci-for-building-docker-images). -### Injected image +## Injected image -Injecting a binary inside a base image is the quickest option to get a working container image. This only works if you were able to build a Linux binary, either locally, or using a container as described above. +Injecting a binary inside a base image is the quickest option to get a working container image. This only works if you +were able to build a Linux binary, either locally, or using a container as described above. -After building a Linux binary ()`polkadot-parachain`) with cargo or with Parity CI image as documented above, the following command allows producing a new container image where the compiled binary is injected: +After building a Linux binary ()`polkadot-parachain`) with cargo or with Parity CI image as documented above, the +following command allows producing a new container image where the compiled binary is injected: ```bash ./docker/scripts/build-injected-image.sh ``` -### Container build +## Container build -Alternatively, you can build an image with a builder pattern. This options takes a while but offers a simple method for anyone to get a working container image without requiring any of the Rust toolchain installed locally. +Alternatively, you can build an image with a builder pattern. This options takes a while but offers a simple method for +anyone to get a working container image without requiring any of the Rust toolchain installed locally. ```bash docker build \ --tag $OWNER/$IMAGE_NAME \ - --file ./docker/polkadot-parachain_builder.Containerfile . + --file ./docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile . ``` You may then run your new container: diff --git a/polkadot/doc/docker.md b/docs/docker.md similarity index 56% rename from polkadot/doc/docker.md rename to docs/docker.md index f20c2d001edd4b9a75261573b7955e5146b28002..53619ca1a9717c500f0ac8ee6e63291018d96806 100644 --- a/polkadot/doc/docker.md +++ b/docs/docker.md @@ -1,6 +1,8 @@ # Using Containers -The following commands should work no matter if you use Docker or Podman. In general, Podman is recommended. All commands are "engine neutral" so you can use the container engine of your choice while still being able to copy/paste the commands below. +The following commands should work no matter if you use Docker or Podman. In general, Podman is recommended. All +commands are "engine neutral" so you can use the container engine of your choice while still being able to copy/paste +the commands below. Let's start defining Podman as our engine: ``` @@ -14,11 +16,15 @@ ENGINE=docker ## The easiest way -The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use the latest official release of the Polkadot binary, pulled from our Debian package. +The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use +the latest official release of the Polkadot binary, pulled from our Debian package. -**_The following examples are running on westend chain and without SSL. They can be used to quick start and learn how Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do not expose RPC and WS ports, if they are not correctly configured._** +**_The following examples are running on westend chain and without SSL. They can be used to quick start and learn how +Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do +not expose RPC and WS ports, if they are not correctly configured._** -Let's first check the version we have. The first time you run this command, the Polkadot docker image will be downloaded. This takes a bit of time and bandwidth, be patient: +Let's first check the version we have. The first time you run this command, the Polkadot docker image will be +downloaded. This takes a bit of time and bandwidth, be patient: ```bash $ENGINE run --rm -it parity/polkadot:latest --version @@ -32,11 +38,14 @@ $ENGINE run --rm -it parity/polkadot:latest --chain westend --name "PolkaDocker" ## Examples -Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local directory to the Polkadot user that is used by the container. +Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot +ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local +directory to the Polkadot user that is used by the container. Set user id 1000 and group id 1000, by running `chown 1000.1000 /my/local/folder -R` if you use a bind mount. -To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. +To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to +connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. ```bash $ENGINE run -d -p 30333:30333 -p 9933:9933 \ @@ -82,7 +91,8 @@ services: ] ``` -With following `docker-compose.yml` you can set up a node and use polkadot-js-apps as the front end on port 80. After starting the node use a browser and enter your Docker host IP in the URL field: __ +With following `docker-compose.yml` you can set up a node and use `polkadot-js-apps` as the front end on port 80. After +starting the node use a browser and enter your Docker host IP in the URL field: __ ```bash version: '2' @@ -117,12 +127,14 @@ services: Chain syncing will utilize all available memory and CPU power your server has to offer, which can lead to crashing. -If running on a low resource VPS, use `--memory` and `--cpus` to limit the resources used. E.g. To allow a maximum of 512MB memory and 50% of 1 CPU, use `--cpus=".5" --memory="512m"`. Read more about limiting a container's resources [here](https://docs.docker.com/config/containers/resource_constraints). +If running on a low resource VPS, use `--memory` and `--cpus` to limit the resources used. E.g. To allow a maximum of +512MB memory and 50% of 1 CPU, use `--cpus=".5" --memory="512m"`. Read more about limiting a container's resources +[here](https://docs.docker.com/config/containers/resource_constraints). ## Build your own image -There are 3 options to build a polkadot container image: +There are 3 options to build a Polkadot container image: - using the builder image - using the injected "Debian" image - using the generic injected image @@ -131,27 +143,31 @@ There are 3 options to build a polkadot container image: To get up and running with the smallest footprint on your system, you may use an existing Polkadot Container image. -You may also build a polkadot container image yourself (it takes a while...) using the container specs `scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile`. +You may also build a Polkadot container image yourself (it takes a while...) using the container specs +`docker/dockerfiles/polkadot/polkadot_builder.Dockerfile`. ### Debian injected -The Debian injected image is how the official polkadot container image is produced. It relies on the Debian package that is published upon each release. The Debian injected image is usually available a few minutes after a new release is published. -It has the benefit of relying on the GPG signatures embedded in the Debian package. +The Debian injected image is how the official Polkadot container image is produced. It relies on the Debian package that +is published upon each release. The Debian injected image is usually available a few minutes after a new release is +published. It has the benefit of relying on the GPG signatures embedded in the Debian package. ### Generic injected -For simple testing purposes, the easiest option for polkadot and also random binaries, is to use the `binary_injected.Dockerfile` container spec. This option is less secure since the injected binary is not checked at all but it has the benefit to be simple. This option requires to already have a valid `polkadot` binary, compiled for Linux. +For simple testing purposes, the easiest option for Polkadot and also random binaries, is to use the +`binary_injected.Dockerfile` container spec. This option is less secure since the injected binary is not checked at all +but it has the benefit to be simple. This option requires to already have a valid `polkadot` binary, compiled for Linux. This binary is then simply copied inside the `parity/base-bin` image. ## Reporting issues -If you run into issues with Polkadot when using docker, please run the following command -(replace the tag with the appropriate one if you do not use latest): +If you run into issues with Polkadot when using docker, please run the following command (replace the tag with the +appropriate one if you do not use latest): ```bash $ENGINE run --rm -it parity/polkadot:latest --version ``` -This will show you the Polkadot version as well as the git commit ref that was used to build your container. -You can now paste the version information in a [new issue](https://github.com/paritytech/polkadot/issues/new/choose). +This will show you the Polkadot version as well as the git commit ref that was used to build your container. You can now +paste the version information in a [new issue](https://github.com/paritytech/polkadot/issues/new/choose). diff --git a/docs/markdown_linting.md b/docs/markdown_linting.md new file mode 100644 index 0000000000000000000000000000000000000000..d916b86ba54489265d59568a798c309d27d655b6 --- /dev/null +++ b/docs/markdown_linting.md @@ -0,0 +1,20 @@ +# Markdown linting + +Since the introduction of [PR #1309](https://github.com/paritytech/polkadot-sdk/pull/1309), the markdown +files in this repository are checked by a linter for formatting and consistency. + +The linter used is [`markdownlint`](https://github.com/DavidAnson/markdownlint) and can be installed locally on your +machine. It can also be setup as [pre-commit hook](https://github.com/igorshubovych/markdownlint-cli#use-with-pre-commit) +to ensure that your markdown is passing all the tests. + +The rules in place are defined +[here](https://github.com/paritytech/polkadot-sdk/blob/master/.github/.markdownlint.yaml). + +You may run `markdownlint` locally using: +``` +markdownlint --config .github/.markdownlint.yaml --ignore target . +``` + +There are also plugins for your favorite editor, that can ensure that most +of the rules will pass and fix typical issues (such as trailing spaces, +missing eof new line, long lines, etc...) diff --git a/polkadot/.github/CODEOWNERS b/polkadot/.github/CODEOWNERS deleted file mode 100644 index a92dc0bb006cb377d2c4a6b0075fca882b820647..0000000000000000000000000000000000000000 --- a/polkadot/.github/CODEOWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# CI -/.github/ @paritytech/ci @chevdor -/scripts/ci/ @paritytech/ci @chevdor -/.gitlab-ci.yml @paritytech/ci -# lingua.dic is not managed by CI team -/scripts/ci/gitlab/lingua.dic diff --git a/polkadot/.github/ISSUE_TEMPLATE/bug_report.md b/polkadot/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index c2214ab7d93276beb1422f043992ddc620b44f61..0000000000000000000000000000000000000000 --- a/polkadot/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -- It would help if you submit info about the system you are running, e.g.: operating system, kernel version, amount of available memory and swap, etc. -- Logs could be very helpful. If possible, submit the whole log. Please format it as ```code blocks```. -- Describe the role your node plays, e.g. validator, full node or light client. -- Any command-line options were passed? diff --git a/polkadot/.github/ISSUE_TEMPLATE/release.md b/polkadot/.github/ISSUE_TEMPLATE/release.md deleted file mode 100644 index 37b422a9b3ecceef31a6bb4f06ca37c6e46151aa..0000000000000000000000000000000000000000 --- a/polkadot/.github/ISSUE_TEMPLATE/release.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: Release issue template -about: Tracking issue for new releases -title: Polkadot {{ env.VERSION }} Release checklist ---- -# Release Checklist - -This is the release checklist for Polkadot {{ env.VERSION }}. **All** following -checks should be completed before publishing a new release of the -Polkadot/Kusama/Westend/Rococo runtime or client. The current release candidate can be -checked out with `git checkout release-{{ env.VERSION }}` - -### Runtime Releases - -These checks should be performed on the codebase prior to forking to a release- -candidate branch. - -- [ ] Verify [`spec_version`](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#spec-version) has been incremented since the - last release for any native runtimes from any existing use on public - (non-private) networks. If the runtime was published (release or pre-release), either - the `spec_version` or `impl` must be bumped. -- [ ] Verify previously [completed migrations](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#old-migrations-removed) are - removed for any public (non-private/test) networks. -- [ ] Verify pallet and [extrinsic ordering](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#extrinsic-ordering) has stayed - the same. Bump `transaction_version` if not. -- [ ] Verify new extrinsics have been correctly whitelisted/blacklisted for - [proxy filters](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#proxy-filtering). -- [ ] Verify [benchmarks](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#benchmarks) have been updated for any modified - runtime logic. - -The following checks can be performed after we have forked off to the release- -candidate branch or started an additional release candidate branch (rc-2, rc-3, etc) - -- [ ] Verify [new migrations](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#new-migrations) complete successfully, and the - runtime state is correctly updated for any public (non-private/test) - networks. -- [ ] Verify [Polkadot JS API](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#polkadot-js) are up to date with the latest - runtime changes. -- [ ] Check with the Signer's team to make sure metadata update QR are lined up -- [ ] Push runtime upgrade to Westend and verify network stability. - -### All Releases - -- [ ] Check that the new client versions have [run on the network](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#burn-in) - without issue for 12+ hours on >75% of our validator nodes. -- [ ] Check that a draft release has been created at - https://github.com/paritytech/polkadot/releases with relevant [release - notes](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#release-notes) -- [ ] Check that [build artifacts](https://github.com/paritytech/polkadot/blob/master/doc/release-checklist.md#build-artifacts) have been added to the - draft-release -- [ ] Check that all items listed in the [milestone](https://github.com/paritytech/polkadot/milestones) are included in the release. -- [ ] Ensure that no `freenotes` were added into the release branch after the latest generated RC diff --git a/polkadot/.github/workflows/release-40_publish-rc-image.yml b/polkadot/.github/workflows/release-40_publish-rc-image.yml deleted file mode 100644 index 3d91c5b8c682b85d3ae937ff0b98690394b41114..0000000000000000000000000000000000000000 --- a/polkadot/.github/workflows/release-40_publish-rc-image.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Release - Publish RC Container image -# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277 - -on: - workflow_dispatch: - inputs: - release_id: - description: | - Release ID. - You can find it using the command: - curl -s \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ - jq '.[] | { name: .name, id: .id }' - required: true - type: string - registry: - description: "Container registry" - required: true - type: string - default: docker.io - owner: - description: Owner of the container image repo - required: true - type: string - default: parity - -env: - RELEASE_ID: ${{ inputs.release_id }} - ENGINE: docker - REGISTRY: ${{ inputs.registry }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} - REPO: ${{ github.repository }} - -jobs: - fetch-artifacts: - runs-on: ubuntu-latest - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Fetch all artifacts - run: | - . ./scripts/ci/common/lib.sh - fetch_release_artifacts - - - name: Cache the artifacts - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - path: | - ./release-artifacts/**/* - - build-container: - runs-on: ubuntu-latest - needs: fetch-artifacts - - strategy: - matrix: - binary: ["polkadot", "staking-miner"] - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Get artifacts from cache - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - fail-on-cache-miss: true - path: | - ./release-artifacts/**/* - - - name: Check sha256 ${{ matrix.binary }} - working-directory: ./release-artifacts - run: | - . ../scripts/ci/common/lib.sh - - echo "Checking binary ${{ matrix.binary }}" - check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR" - - - name: Check GPG ${{ matrix.binary }} - working-directory: ./release-artifacts - run: | - . ../scripts/ci/common/lib.sh - import_gpg_keys - check_gpg ${{ matrix.binary }} - - - name: Fetch commit and tag - id: fetch_refs - run: | - release=release-${{ inputs.release_id }} && \ - echo "release=${release}" >> $GITHUB_OUTPUT - - commit=$(git rev-parse --short HEAD) && \ - echo "commit=${commit}" >> $GITHUB_OUTPUT - - tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ - [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ - echo "No tag, doing without" - - - name: Build Injected Container image for ${{ matrix.binary }} - env: - BIN_FOLDER: ./release-artifacts - BINARY: ${{ matrix.binary }} - TAGS: ${{join(steps.fetch_refs.outputs.*, ',')}} - run: | - echo "Building container for ${{ matrix.binary }}" - ./scripts/ci/dockerfiles/build-injected.sh - - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push Container image for ${{ matrix.binary }} - id: docker_push - env: - BINARY: ${{ matrix.binary }} - run: | - $ENGINE images | grep ${BINARY} - $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} - - - name: Check version for the published image for ${{ matrix.binary }} - env: - BINARY: ${{ matrix.binary }} - RELEASE_TAG: ${{ steps.fetch_refs.outputs.release }} - run: | - echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" - $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version diff --git a/polkadot/.github/workflows/release-50_publish-docker-release.yml b/polkadot/.github/workflows/release-50_publish-docker-release.yml deleted file mode 100644 index 81e5caa718f3e251f8eeded4210d2d4448ecb036..0000000000000000000000000000000000000000 --- a/polkadot/.github/workflows/release-50_publish-docker-release.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Release - Publish Docker image for new releases - -on: - release: - types: - - published - -jobs: - main: - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70 # v2.1.0 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v4 - with: - push: true - file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile - tags: | - parity/polkadot:latest - parity/polkadot:${{ github.event.release.tag_name }} - build-args: | - POLKADOT_VERSION=${{ github.event.release.tag_name }} - VCS_REF=${{ github.ref }} - BUILD_DATE=${{ github.event.release.published_at }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/polkadot/.github/workflows/release-51_publish-docker-manual.yml b/polkadot/.github/workflows/release-51_publish-docker-manual.yml deleted file mode 100644 index 919769f8700d1ac70f0d831a5f6645aff86f42f2..0000000000000000000000000000000000000000 --- a/polkadot/.github/workflows/release-51_publish-docker-manual.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Release - Publish Docker image (manual dispatch) - -on: - workflow_dispatch: - inputs: - version: - description: version to build/release - default: v0.9.18 - required: true - date: - description: release date of version - default: "2022-02-23T19:11:58Z" - required: true - -jobs: - main: - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70 # v2.1.0 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v4 - with: - push: true - file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile - tags: | - parity/polkadot:latest - parity/polkadot:${{ github.event.inputs.version }} - build-args: | - POLKADOT_VERSION=${{ github.event.inputs.version }} - VCS_REF=${{ github.ref }} - BUILD_DATE=${{ github.event.inputs.date }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/polkadot/Cargo.toml b/polkadot/Cargo.toml index f173557a0ffba1fb2b696cfb7a98cf665bac0d51..aacc6ad405cca23ea2e1cb1bd7122137177c8f0b 100644 --- a/polkadot/Cargo.toml +++ b/polkadot/Cargo.toml @@ -18,18 +18,15 @@ rust-version = "1.64.0" readme = "README.md" authors.workspace = true edition.workspace = true -version = "1.0.0" +version = "1.1.0" +default-run = "polkadot" [dependencies] color-eyre = { version = "0.6.1", default-features = false } tikv-jemallocator = { version = "0.5.0", optional = true } # Crates in our workspace, defined as dependencies so we can pass them feature flags. -polkadot-cli = { path = "cli", features = [ - "kusama-native", - "westend-native", - "rococo-native", -] } +polkadot-cli = { path = "cli", features = [ "westend-native", "rococo-native" ] } polkadot-node-core-pvf = { path = "node/core/pvf" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } diff --git a/polkadot/README.md b/polkadot/README.md index 8f0809c9fc7a8b833948e49f1db5a0b0488821a6..93e93cfba0ee54aac2b5b61d8bb0685ee10f7bb0 100644 --- a/polkadot/README.md +++ b/polkadot/README.md @@ -22,7 +22,7 @@ Installation from the Debian repository will create a `systemd` service that can Polkadot node. This is disabled by default, and can be started by running `systemctl start polkadot` on demand (use `systemctl enable polkadot` to make it auto-start after reboot). By default, it will run as the `polkadot` user. Command-line flags passed to the binary can be customized by editing -`/etc/default/polkadot`. This file will not be overwritten on updating polkadot. You may also just +`/etc/default/polkadot`. This file will not be overwritten on updating Polkadot. You may also just run the node directly from the command-line. ### Debian-based (Debian, Ubuntu) @@ -128,7 +128,7 @@ Connect to the global Polkadot Mainnet network by running: You can see your node on [telemetry] (set a custom name with `--name "my custom name"`). -[telemetry]: https://telemetry.polkadot.io/#list/Polkadot +[telemetry](https://telemetry.polkadot.io/#list/Polkadot): https://telemetry.polkadot.io/#list/Polkadot ### Connect to the "Kusama" Canary Network @@ -140,7 +140,7 @@ Connect to the global Kusama canary network by running: You can see your node on [telemetry] (set a custom name with `--name "my custom name"`). -[telemetry]: https://telemetry.polkadot.io/#list/Kusama +[telemetry](https://telemetry.polkadot.io/#list/Kusama): https://telemetry.polkadot.io/#list/Kusama ### Connect to the Westend Testnet @@ -152,7 +152,7 @@ Connect to the global Westend testnet by running: You can see your node on [telemetry] (set a custom name with `--name "my custom name"`). -[telemetry]: https://telemetry.polkadot.io/#list/Westend +[telemetry](https://telemetry.polkadot.io/#list/Westend): https://telemetry.polkadot.io/#list/Westend ### Obtaining DOTs diff --git a/polkadot/RELEASE.md b/polkadot/RELEASE.md deleted file mode 100644 index 937c1e6e515fba0c523fec02b845365934a105fe..0000000000000000000000000000000000000000 --- a/polkadot/RELEASE.md +++ /dev/null @@ -1,57 +0,0 @@ -Polkadot Release Process ------------------------- - -### Branches -* release-candidate branch: The branch used for staging of the next release. - Named like `release-v0.8.26` - -### Notes -* The release-candidate branch *must* be made in the paritytech/polkadot repo in -order for release automation to work correctly -* Any new pushes/merges to the release-candidate branch (for example, -refs/heads/release-v0.8.26) will result in the rc index being bumped (e.g., v0.8.26-rc1 -to v0.8.26-rc2) and new wasms built. - -### Release workflow - -Below are the steps of the release workflow. Steps prefixed with NOACTION are -automated and require no human action. - -1. To initiate the release process: - 1. branch master off to a release candidate branch: - - `git checkout master; git pull; git checkout -b release-v0.8.26` - 2. In the [substrate](https://github.com/paritytech/substrate) repo, check out the commit used by polkadot (this can be found using the following command in the *polkadot* repo: `grep 'paritytech/substrate' Cargo.lock | grep -E '[0-9a-f]{40}' | sort | uniq ` - 3. Branch off this **substrate** commit into its own branch: `git branch -b polkadot-v0.8.26; git push origin refs/heads/polkadot-v0.8.26` - 4. In the **polkadot** repository, use [diener](https://github.com/bkchr/diener/) to switch to this branch: `diener update --branch "polkadot-v0.8.26" --substrate`. Update Cargo.lock (to do this, you can run `cargo build` and then ctrl+c once it finishes fetching and begins compiling) - 5. Push the **polkadot** `release-v0.8.26` branch to Github: `git push origin refs/heads/release-v0.8.26` -2. NOACTION: The current HEAD of the release-candidate branch is tagged `v0.8.26-rc1` -3. NOACTION: A draft release and runtime WASMs are created for this - release-candidate automatically. A link to the draft release will be linked in - the internal polkadot matrix channel. -4. NOACTION: A new Github issue is created containing a checklist of manual - steps to be completed before we are confident with the release. This will be - linked in Matrix. -5. Complete the steps in the issue created in step 4, signing them off as - completed -6. (optional) If a fix is required to the release-candidate: - 1. Merge the fix with `master` first - 2. Cherry-pick the commit from `master` to `release-v0.8.26`, fixing any - merge conflicts. Try to avoid unnecessarily bumping crates. - 3. Push the release-candidate branch to Github - this is now the new release- - candidate - 4. Depending on the cherry-picked changes, it may be necessary to perform some - or all of the manual tests again. - 5. If there are **substrate** changes required, these should be cherry-picked to the substrate `polkadot-v0.8.26` branch and pushed, and the version of substrate used in **polkadot** updated using `cargo update -p sp-io` -7. Once happy with the release-candidate, tag the current top commit in the release candidate branch and push to Github: `git tag -s -m 'v0.8.26' v0.8.26; git push --tags` -9. NOACTION: The HEAD of the `release` branch will be tagged with `v0.8.26`, - and a final draft release will be created on Github. - -### Security releases - -Occasionally there may be changes that need to be made to the most recently -released version of Polkadot, without taking *every* change to `master` since -the last release. For example, in the event of a security vulnerability being -found, where releasing a fixed version is a matter of some expediency. In cases -like this, the fix should first be merged with master, cherry-picked to a branch -forked from `release`, tested, and then finally merged with `release`. A -sensible versioning scheme for changes like this is `vX.Y.Z-1`. diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index bcb4f2ac130863e790710ee716be819ad4081bb1..53961c90a2a817812f77e5432f25ba4cf12b4385 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "polkadot-cli" description = "Polkadot Relay-chain Client Node" -version = "1.0.0" +version = "1.1.0" authors.workspace = true edition.workspace = true license.workspace = true @@ -15,9 +15,9 @@ wasm-opt = false crate-type = ["cdylib", "rlib"] [dependencies] -clap = { version = "4.4.2", features = ["derive"], optional = true } +clap = { version = "4.4.4", features = ["derive"], optional = true } log = "0.4.17" -thiserror = "1.0.31" +thiserror = "1.0.48" futures = "0.3.21" pyro = { package = "pyroscope", version = "0.5.3", optional = true } pyroscope_pprofrs = { version = "0.2", optional = true } @@ -67,11 +67,7 @@ fast-runtime = [ "service/fast-runtime" ] pyroscope = [ "pyro", "pyroscope_pprofrs" ] hostperfcheck = [ "polkadot-performance-test" ] -# Configure the native runtimes to use. Polkadot is enabled by default. -# -# Validators require the native runtime currently -polkadot-native = [ "service/polkadot-native" ] -kusama-native = [ "service/kusama-native" ] +# Configure the native runtimes to use. westend-native = [ "service/westend-native" ] rococo-native = [ "service/rococo-native" ] diff --git a/polkadot/cli/src/cli.rs b/polkadot/cli/src/cli.rs index 66205902b79dfd26e2ccb3a615c52c7ad0d72738..aaf8f170576075414a00aba17f39c042d9463a98 100644 --- a/polkadot/cli/src/cli.rs +++ b/polkadot/cli/src/cli.rs @@ -19,8 +19,15 @@ use clap::Parser; use std::path::PathBuf; -/// The version of the node. The passed-in version of the workers should match this. -pub const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); +/// The version of the node. +/// +/// This is the version that is used for versioning this node binary. +/// By default the `minor` version is bumped in every release. `Major` or `patch` releases are only +/// expected in very rare cases. +/// +/// The worker binaries associated to the node binary should ensure that they are using the same +/// version as the main node that started them. +pub const NODE_VERSION: &'static str = "1.1.0"; #[allow(missing_docs)] #[derive(Debug, Parser)] diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index a2a00d0ebd3f87939f94a1f22b2201d6e74d683d..19437ce875308cb71894909ab4df047cdbdaa130 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -34,12 +34,6 @@ pub use polkadot_performance_test::PerfCheckError; #[cfg(feature = "pyroscope")] use pyroscope_pprofrs::{pprof_backend, PprofConfig}; -impl From for Error { - fn from(s: String) -> Self { - Self::Other(s) - } -} - type Result = std::result::Result; fn get_exec_name() -> Option { @@ -55,7 +49,8 @@ impl SubstrateCli for Cli { } fn impl_version() -> String { - NODE_VERSION.into() + let commit_hash = env!("SUBSTRATE_CLI_COMMIT_HASH"); + format!("{NODE_VERSION}-{commit_hash}") } fn description() -> String { @@ -91,29 +86,20 @@ impl SubstrateCli for Cli { }; Ok(match id { "kusama" => Box::new(service::chain_spec::kusama_config()?), - #[cfg(feature = "kusama-native")] - "kusama-dev" => Box::new(service::chain_spec::kusama_development_config()?), - #[cfg(feature = "kusama-native")] - "kusama-local" => Box::new(service::chain_spec::kusama_local_testnet_config()?), - #[cfg(feature = "kusama-native")] - "kusama-staging" => Box::new(service::chain_spec::kusama_staging_testnet_config()?), - #[cfg(not(feature = "kusama-native"))] name if name.starts_with("kusama-") && !name.ends_with(".json") => - Err(format!("`{}` only supported with `kusama-native` feature enabled.", name))?, + Err(format!("`{name}` is not supported anymore as the kusama native runtime no longer part of the node."))?, "polkadot" => Box::new(service::chain_spec::polkadot_config()?), - #[cfg(feature = "polkadot-native")] - "polkadot-dev" | "dev" => Box::new(service::chain_spec::polkadot_development_config()?), - #[cfg(feature = "polkadot-native")] - "polkadot-local" => Box::new(service::chain_spec::polkadot_local_testnet_config()?), + name if name.starts_with("polkadot-") && !name.ends_with(".json") => + Err(format!("`{name}` is not supported anymore as the polkadot native runtime no longer part of the node."))?, "rococo" => Box::new(service::chain_spec::rococo_config()?), #[cfg(feature = "rococo-native")] - "rococo-dev" => Box::new(service::chain_spec::rococo_development_config()?), + "dev" | "rococo-dev" => Box::new(service::chain_spec::rococo_development_config()?), #[cfg(feature = "rococo-native")] "rococo-local" => Box::new(service::chain_spec::rococo_local_testnet_config()?), #[cfg(feature = "rococo-native")] "rococo-staging" => Box::new(service::chain_spec::rococo_staging_testnet_config()?), #[cfg(not(feature = "rococo-native"))] - name if name.starts_with("rococo-") && !name.ends_with(".json") => + name if name.starts_with("rococo-") && !name.ends_with(".json") || name == "dev" => Err(format!("`{}` only supported with `rococo-native` feature enabled.", name))?, "westend" => Box::new(service::chain_spec::westend_config()?), #[cfg(feature = "westend-native")] @@ -145,7 +131,7 @@ impl SubstrateCli for Cli { path => { let path = std::path::PathBuf::from(path); - let chain_spec = Box::new(service::PolkadotChainSpec::from_json_file(path.clone())?) + let chain_spec = Box::new(service::GenericChainSpec::from_json_file(path.clone())?) as Box; // When `force_*` is given or the file name starts with the name of one of the known @@ -157,7 +143,7 @@ impl SubstrateCli for Cli { { Box::new(service::RococoChainSpec::from_json_file(path)?) } else if self.run.force_kusama || chain_spec.is_kusama() { - Box::new(service::KusamaChainSpec::from_json_file(path)?) + Box::new(service::GenericChainSpec::from_json_file(path)?) } else if self.run.force_westend || chain_spec.is_westend() { Box::new(service::WestendChainSpec::from_json_file(path)?) } else { @@ -181,17 +167,6 @@ fn set_default_ss58_version(spec: &Box) { sp_core::crypto::set_default_ss58_version(ss58_version); } -const DEV_ONLY_ERROR_PATTERN: &'static str = - "can only use subcommand with --chain [polkadot-dev, kusama-dev, westend-dev, rococo-dev, wococo-dev], got "; - -fn ensure_dev(spec: &Box) -> std::result::Result<(), String> { - if spec.is_dev() { - Ok(()) - } else { - Err(format!("{}{}", DEV_ONLY_ERROR_PATTERN, spec.id())) - } -} - /// Runs performance checks. /// Should only be used in release build since the check would take too much time otherwise. fn host_perf_check() -> Result<()> { @@ -470,8 +445,7 @@ pub fn run() -> Result<()> { cmd.run(client.clone()).map_err(Error::SubstrateCli) }), // These commands are very similar and can be handled in nearly the same way. - BenchmarkCmd::Extrinsic(_) | BenchmarkCmd::Overhead(_) => { - ensure_dev(chain_spec).map_err(Error::Other)?; + BenchmarkCmd::Extrinsic(_) | BenchmarkCmd::Overhead(_) => runner.sync_run(|mut config| { let (client, _, _, _) = service::new_chain_ops(&mut config, None)?; let header = client.header(client.info().genesis_hash).unwrap().unwrap(); @@ -507,11 +481,9 @@ pub fn run() -> Result<()> { .map_err(Error::SubstrateCli), _ => unreachable!("Ensured by the outside match; qed"), } - }) - }, + }), BenchmarkCmd::Pallet(cmd) => { set_default_ss58_version(chain_spec); - ensure_dev(chain_spec).map_err(Error::Other)?; if cfg!(feature = "runtime-benchmarks") { runner.sync_run(|config| { diff --git a/polkadot/cli/src/error.rs b/polkadot/cli/src/error.rs index a4591e2508c9796764d09296405bdb05fb834a3c..62ee4d139074e3a4a15ec16ec0d4d24a918a05a9 100644 --- a/polkadot/cli/src/error.rs +++ b/polkadot/cli/src/error.rs @@ -58,3 +58,9 @@ pub enum Error { #[error("This subcommand is only available when compiled with `{feature}`")] FeatureNotEnabled { feature: &'static str }, } + +impl From for Error { + fn from(s: String) -> Self { + Self::Other(s) + } +} diff --git a/polkadot/doc/release-checklist.md b/polkadot/doc/release-checklist.md deleted file mode 100644 index 4be7c9bcd5df87e8a183f1bfbea43b83cc754eb0..0000000000000000000000000000000000000000 --- a/polkadot/doc/release-checklist.md +++ /dev/null @@ -1,98 +0,0 @@ - -## Notes - -### Burn In - -Ensure that Parity DevOps has run the new release on Westend, Kusama, and -Polkadot validators for at least 12 hours prior to publishing the release. - -### Build Artifacts - -Add any necessary assets to the release. They should include: - -- Linux binary -- GPG signature of the Linux binary -- SHA256 of binary -- Source code -- Wasm binaries of any runtimes - -### Release notes - -The release notes should list: - -- The priority of the release (i.e., how quickly users should upgrade) - this is - based on the max priority of any *client* changes. -- Which native runtimes and their versions are included -- The proposal hashes of the runtimes as built with - [srtool](https://gitlab.com/chevdor/srtool) -- Any changes in this release that are still awaiting audit - -The release notes may also list: - -- Free text at the beginning of the notes mentioning anything important - regarding this release -- Notable changes (those labelled with B[1-9]-* labels) separated into sections - -### Spec Version - -A runtime upgrade must bump the spec number. This may follow a pattern with the -client release (e.g. runtime v12 corresponds to v0.8.12, even if the current -runtime is not v11). - -### Old Migrations Removed - -Any previous `on_runtime_upgrade` functions from old upgrades must be removed -to prevent them from executing a second time. The `on_runtime_upgrade` function -can be found in `runtime//src/lib.rs`. - -### New Migrations - -Ensure that any migrations that are required due to storage or logic changes -are included in the `on_runtime_upgrade` function of the appropriate pallets. - -### Extrinsic Ordering - -Offline signing libraries depend on a consistent ordering of call indices and -functions. Compare the metadata of the current and new runtimes and ensure that -the `module index, call index` tuples map to the same set of functions. In case -of a breaking change, increase `transaction_version`. - -To verify the order has not changed, you may manually start the following [Github Action](https://github.com/paritytech/polkadot/actions/workflows/extrinsic-ordering-check-from-bin.yml). It takes around a minute to run and will produce the report as artifact you need to manually check. - -The things to look for in the output are lines like: - - `[Identity] idx 28 -> 25 (calls 15)` - indicates the index for `Identity` has changed - - `[+] Society, Recovery` - indicates the new version includes 2 additional modules/pallets. - - If no indices have changed, every modules line should look something like `[Identity] idx 25 (calls 15)` - -Note: Adding new functions to the runtime does not constitute a breaking change -as long as the indexes did not change. - -### Proxy Filtering - -The runtime contains proxy filters that map proxy types to allowable calls. If -the new runtime contains any new calls, verify that the proxy filters are up to -date to include them. - -### Benchmarks - -There are three benchmarking machines reserved for updating the weights at -release-time. To initialise a benchmark run for each production runtime -(westend, kusama, polkadot): -* Go to https://gitlab.parity.io/parity/polkadot/-/pipelines?page=1&scope=branches&ref=master -* Click the link to the last pipeline run for master -* Start each of the manual jobs: - * 'update_westend_weights' - * 'update_polkadot_weights' - * 'update_kusama_weights' -* When these jobs have completed (it takes a few hours), a git PATCH file will - be available to download as an artifact. -* On your local machine, branch off master -* Download the patch file and apply it to your branch with `git patch patchfile.patch` -* Commit the changes to your branch and submit a PR against master -* The weights should be (Currently manually) checked to make sure there are no - big outliers (i.e., twice or half the weight). - -### Polkadot JS - -Ensure that a release of [Polkadot JS API]() contains any new types or -interfaces necessary to interact with the new runtime. diff --git a/polkadot/doc/shell-completion.md b/polkadot/doc/shell-completion.md index 9c53cf43a10ff7ddc4f438872a6cdd773a51fff6..9761c153d5be307cfed590f421a12530d34864e3 100644 --- a/polkadot/doc/shell-completion.md +++ b/polkadot/doc/shell-completion.md @@ -1,6 +1,7 @@ # Shell completion -The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system. +The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script +matching you build and system. Assuming you built a release version using `cargo build --release` and use `bash` run the following: @@ -30,7 +31,8 @@ source $HOME/.bash_profile ## Update -When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current binary: +When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current +binary: ```bash COMPL_DIR=$HOME/.completion diff --git a/polkadot/doc/testing.md b/polkadot/doc/testing.md index 78ad77e0e0fda7230cbe4f7f6a6890ed3362c68f..1045303baf0df1e757cefa374ca2396b99984504 100644 --- a/polkadot/doc/testing.md +++ b/polkadot/doc/testing.md @@ -4,7 +4,7 @@ Automated testing is an essential tool to assure correctness. ## Scopes -The testing strategy for polkadot is 4-fold: +The testing strategy for Polkadot is 4-fold: ### Unit testing (1) @@ -16,18 +16,15 @@ There are two variants of integration tests: #### Subsystem tests (2) -One particular subsystem (subsystem under test) interacts with a -mocked overseer that is made to assert incoming and outgoing messages -of the subsystem under test. -This is largely present today, but has some fragmentation in the evolved -integration test implementation. A `proc-macro`/`macro_rules` would allow -for more consistent implementation and structure. +One particular subsystem (subsystem under test) interacts with a mocked overseer that is made to assert incoming and +outgoing messages of the subsystem under test. This is largely present today, but has some fragmentation in the evolved +integration test implementation. A `proc-macro`/`macro_rules` would allow for more consistent implementation and +structure. #### Behavior tests (3) -Launching small scale networks, with multiple adversarial nodes without any further tooling required. -This should include tests around the thresholds in order to evaluate the error handling once certain -assumed invariants fail. +Launching small scale networks, with multiple adversarial nodes without any further tooling required. This should +include tests around the thresholds in order to evaluate the error handling once certain assumed invariants fail. For this purpose based on `AllSubsystems` and `proc-macro` `AllSubsystemsGen`. @@ -35,18 +32,14 @@ This assumes a simplistic test runtime. #### Testing at scale (4) -Launching many nodes with configurable network speed and node features in a cluster of nodes. -At this scale the [Simnet][simnet] comes into play which launches a full cluster of nodes. -The scale is handled by spawning a kubernetes cluster and the meta description -is covered by [Gurke][Gurke]. -Asserts are made using Grafana rules, based on the existing prometheus metrics. This can -be extended by adding an additional service translating `jaeger` spans into addition -prometheus avoiding additional polkadot source changes. +Launching many nodes with configurable network speed and node features in a cluster of nodes. At this scale the +[Simnet][simnet] comes into play which launches a full cluster of nodes. The scale is handled by spawning a kubernetes +cluster and the meta description is covered by [Gurke][Gurke]. Asserts are made using Grafana rules, based on the +existing prometheus metrics. This can be extended by adding an additional service translating `jaeger` spans into +addition prometheus avoiding additional Polkadot source changes. -_Behavior tests_ and _testing at scale_ have naturally soft boundary. -The most significant difference is the presence of a real network and -the number of nodes, since a single host often not capable to run -multiple nodes at once. +_Behavior tests_ and _testing at scale_ have naturally soft boundary. The most significant difference is the presence of +a real network and the number of nodes, since a single host often not capable to run multiple nodes at once. --- @@ -54,8 +47,8 @@ multiple nodes at once. Coverage gives a _hint_ of the actually covered source lines by tests and test applications. -The state of the art is currently [tarpaulin][tarpaulin] which unfortunately yields a -lot of false negatives. Lines that are in fact covered, marked as uncovered due to a mere linebreak in a statement can cause these artifacts. This leads to +The state of the art is currently [tarpaulin][tarpaulin] which unfortunately yields a lot of false negatives. Lines that +are in fact covered, marked as uncovered due to a mere linebreak in a statement can cause these artifacts. This leads to lower coverage percentages than there actually is. Since late 2020 rust has gained [MIR based coverage tooling]( @@ -97,9 +90,11 @@ The test coverage in `lcov` can the be published to . bash <(curl -s https://codecov.io/bash) -f lcov.info ``` -or just printed as part of the PR using a github action i.e. [`jest-lcov-reporter`](https://github.com/marketplace/actions/jest-lcov-reporter). +or just printed as part of the PR using a github action i.e. +[`jest-lcov-reporter`](https://github.com/marketplace/actions/jest-lcov-reporter). -For full examples on how to use [`grcov` /w polkadot specifics see the github repo](https://github.com/mozilla/grcov#coverallscodecov-output). +For full examples on how to use [`grcov` /w Polkadot specifics see the github +repo](https://github.com/mozilla/grcov#coverallscodecov-output). ## Fuzzing @@ -109,11 +104,12 @@ Currently implemented fuzzing targets: * `erasure-coding` -The tooling of choice here is `honggfuzz-rs` as it allows _fastest_ coverage according to "some paper" which is a positive feature when run as part of PRs. +The tooling of choice here is `honggfuzz-rs` as it allows _fastest_ coverage according to "some paper" which is a +positive feature when run as part of PRs. -Fuzzing is generally not applicable for data secured by cryptographic hashes or signatures. Either the input has to be specifically crafted, such that the discarded input -percentage stays in an acceptable range. -System level fuzzing is hence simply not feasible due to the amount of state that is required. +Fuzzing is generally not applicable for data secured by cryptographic hashes or signatures. Either the input has to be +specifically crafted, such that the discarded input percentage stays in an acceptable range. System level fuzzing is +hence simply not feasible due to the amount of state that is required. Other candidates to implement fuzzing are: @@ -128,14 +124,17 @@ There are various ways of performance metrics. * cache hits/misses w/ `iai` harness or `criterion-perf` * `coz` a performance based compiler -Most of them are standard tools to aid in the creation of statistical tests regarding change in time of certain unit tests. +Most of them are standard tools to aid in the creation of statistical tests regarding change in time of certain unit +tests. -`coz` is meant for runtime. In our case, the system is far too large to yield a sufficient number of measurements in finite time. -An alternative approach could be to record incoming package streams per subsystem and store dumps of them, which in return could be replayed repeatedly at an -accelerated speed, with which enough metrics could be obtained to yield -information on which areas would improve the metrics. -This unfortunately will not yield much information, since most if not all of the subsystem code is linear based on the input to generate one or multiple output messages, it is unlikely to get any useful metrics without mocking a sufficiently large part of the other subsystem which overlaps with [#Integration tests] which is unfortunately not repeatable as of now. -As such the effort gain seems low and this is not pursued at the current time. +`coz` is meant for runtime. In our case, the system is far too large to yield a sufficient number of measurements in +finite time. An alternative approach could be to record incoming package streams per subsystem and store dumps of them, +which in return could be replayed repeatedly at an accelerated speed, with which enough metrics could be obtained to +yield information on which areas would improve the metrics. This unfortunately will not yield much information, since +most if not all of the subsystem code is linear based on the input to generate one or multiple output messages, it is +unlikely to get any useful metrics without mocking a sufficiently large part of the other subsystem which overlaps with +[#Integration tests] which is unfortunately not repeatable as of now. As such the effort gain seems low and this is not +pursued at the current time. ## Writing small scope integration tests with preconfigured workers @@ -152,29 +151,25 @@ Requirements: ### Goals -The main goals are is to allow creating a test node which -exhibits a certain behavior by utilizing a subset of _wrapped_ or _replaced_ subsystems easily. -The runtime must not matter at all for these tests and should be simplistic. -The execution must be fast, this mostly means to assure a close to zero network latency as -well as shorting the block time and epoch times down to a few `100ms` and a few dozend blocks per epoch. +The main goals are is to allow creating a test node which exhibits a certain behavior by utilizing a subset of _wrapped_ +or _replaced_ subsystems easily. The runtime must not matter at all for these tests and should be simplistic. The +execution must be fast, this mostly means to assure a close to zero network latency as well as shorting the block time +and epoch times down to a few `100ms` and a few dozend blocks per epoch. ### Approach #### MVP -A simple small scale builder pattern would suffice for stage one implementation of allowing to -replace individual subsystems. -An alternative would be to harness the existing `AllSubsystems` type -and replace the subsystems as needed. +A simple small scale builder pattern would suffice for stage one implementation of allowing to replace individual +subsystems. An alternative would be to harness the existing `AllSubsystems` type and replace the subsystems as needed. #### Full `proc-macro` implementation -`Overseer` is a common pattern. -It could be extracted as `proc` macro and generative `proc-macro`. -This would replace the `AllSubsystems` type as well as implicitly create -the `AllMessages` enum as `AllSubsystemsGen` does today. +`Overseer` is a common pattern. It could be extracted as `proc` macro and generative `proc-macro`. This would replace +the `AllSubsystems` type as well as implicitly create the `AllMessages` enum as `AllSubsystemsGen` does today. -The implementation is yet to be completed, see the [implementation PR](https://github.com/paritytech/polkadot/pull/2962) for details. +The implementation is yet to be completed, see the [implementation PR](https://github.com/paritytech/polkadot/pull/2962) +for details. ##### Declare an overseer implementation @@ -233,19 +228,16 @@ fn main() -> eyre::Result<()> { #### Simnet -Spawn a kubernetes cluster based on a meta description using [Gurke] with the -[Simnet] scripts. +Spawn a kubernetes cluster based on a meta description using [Gurke] with the [Simnet] scripts. -Coordinated attacks of multiple nodes or subsystems must be made possible via -a side-channel, that is out of scope for this document. +Coordinated attacks of multiple nodes or subsystems must be made possible via a side-channel, that is out of scope for +this document. -The individual node configurations are done as targets with a particular -builder configuration. +The individual node configurations are done as targets with a particular builder configuration. #### Behavior tests w/o Simnet -Commonly this will require multiple nodes, and most machines are limited to -running two or three nodes concurrently. +Commonly this will require multiple nodes, and most machines are limited to running two or three nodes concurrently. Hence, this is not the common case and is just an implementation _idea_. ```rust diff --git a/polkadot/erasure-coding/Cargo.toml b/polkadot/erasure-coding/Cargo.toml index f74a880388255033739235b27821db11fd7ea9a7..d07b77ec4ddf5574bf29cef0113c1dc1ebf8b32d 100644 --- a/polkadot/erasure-coding/Cargo.toml +++ b/polkadot/erasure-coding/Cargo.toml @@ -12,7 +12,7 @@ novelpoly = { package = "reed-solomon-novelpoly", version = "1.0.0" } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["std", "derive"] } sp-core = { path = "../../substrate/primitives/core" } sp-trie = { path = "../../substrate/primitives/trie" } -thiserror = "1.0.31" +thiserror = "1.0.48" [dev-dependencies] criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support"] } diff --git a/polkadot/erasure-coding/benches/README.md b/polkadot/erasure-coding/benches/README.md index e643643229e882bee30c5a6c727e239a17794c02..94fca5400c610636831c08b5282d6d47f4199878 100644 --- a/polkadot/erasure-coding/benches/README.md +++ b/polkadot/erasure-coding/benches/README.md @@ -1,10 +1,10 @@ -### Run benches +# Run benches ``` -$ cd erasure-coding # ensure you are in the right directory -$ cargo bench +cd erasure-coding # ensure you are in the right directory +cargo bench ``` -### `scaling_with_validators` +## `scaling_with_validators` This benchmark evaluates the performance of constructing the chunks and the erasure root from PoV and reconstructing the PoV from chunks. You can see the results of running this bench on 5950x below. diff --git a/polkadot/grafana/README.md b/polkadot/grafana/README.md index 73c5b6feacf2e1b51f19f991343382089208cd4b..7350001bfa1fcebe367799cc36426522c4cf4f97 100644 --- a/polkadot/grafana/README.md +++ b/polkadot/grafana/README.md @@ -1,35 +1,35 @@ # Do I need this ? -Polkadot nodes collect and produce Prometheus metrics and logs. These include health, performance and debug -information such as last finalized block, height of the chain, and many other deeper implementation details -of the Polkadot/Substrate node subsystems. These are crucial pieces of information that one needs to successfully +Polkadot nodes collect and produce Prometheus metrics and logs. These include health, performance and debug +information such as last finalized block, height of the chain, and many other deeper implementation details +of the Polkadot/Substrate node subsystems. These are crucial pieces of information that one needs to successfully monitor the liveliness and performance of a network and its validators. # How does it work ? -Just import the dashboard JSON files from this folder in your Grafana installation. All dashboards are grouped in +Just import the dashboard JSON files from this folder in your Grafana installation. All dashboards are grouped in folder percategory (like for example `parachains`). The files have been created by Grafana export functionality and follow the data model specified [here](https://grafana.com/docs/grafana/latest/dashboards/json-model/). -We aim to keep the dashboards here in sync with the implementation, except dashboards for development and +We aim to keep the dashboards here in sync with the implementation, except dashboards for development and testing. # Contributing -**Your contributions are most welcome!** +**Your contributions are most welcome!** Please make sure to follow the following design guidelines: - Add a new entry in this file and describe the usecase and key metrics -- Ensure proper names and descriptions for dashboard panels and add relevant documentation when needed. -This is very important as not all users have similar depth of understanding of the implementation +- Ensure proper names and descriptions for dashboard panels and add relevant documentation when needed. +This is very important as not all users have similar depth of understanding of the implementation - Have labels for axis - All values have proper units of measurement - A crisp and clear color scheme is used # Prerequisites -Before you continue make sure you have Grafana set up, or otherwise follow this -[guide](https://wiki.polkadot.network/docs/maintain-guides-how-to-monitor-your-node). +Before you continue make sure you have Grafana set up, or otherwise follow this +[guide](https://wiki.polkadot.network/docs/maintain-guides-how-to-monitor-your-node). You might also need to [setup Loki](https://grafana.com/go/webinar/loki-getting-started/). @@ -44,7 +44,7 @@ This section is a list of dashboards, their use case as well as the key metrics ## Node Versions -Useful for monitoring versions and logs of validator nodes. Includes time series panels that +Useful for monitoring versions and logs of validator nodes. Includes time series panels that track node warning and error log rates. These can be further investigated in Grafana Loki. Requires Loki for log aggregation and querying. @@ -64,30 +64,30 @@ It includes panels covering key subsystems of the parachain node side implementa - Disputes coordinator - Chain selection -It is important to note that this dashboard applies only for validator nodes. The prometheus -queries assume the `instance` label value contains the string `validator` only for validator nodes. +It is important to note that this dashboard applies only for validator nodes. The prometheus +queries assume the `instance` label value contains the string `validator` only for validator nodes. [Dashboard JSON](parachains/status.json) ### Key liveliness indicators - **Relay chain finality lag**. How far behind finality is compared to the current best block. By design, GRANDPA never finalizes past last 2 blocks, so this value is always >=2 blocks. -- **Approval checking finality lag**. The distance (in blocks) between the chain head and the last block -on which Approval voting is happening. The block is generally the highest approved ancestor of the head +- **Approval checking finality lag**. The distance (in blocks) between the chain head and the last block +on which Approval voting is happening. The block is generally the highest approved ancestor of the head block and the metric is computed during relay chain selection. -- **Disputes finality lag**. How far behind the chain head is the last approved and non disputed block. -This value is always higher than approval checking lag as it further restricts finality to only undisputed +- **Disputes finality lag**. How far behind the chain head is the last approved and non disputed block. +This value is always higher than approval checking lag as it further restricts finality to only undisputed chains. -- **PVF preparation and execution time**. Each parachain has it's own PVF (parachain validation function): -a wasm blob that is executed by validators during backing, approval checking and disputing. The PVF -preparation time refers to the time it takes for the PVF wasm to be compiled. This step is done once and -then result cached. PVF execution will use the resulting artifact to execute the PVF for a given candidate. -PVFs are expected to have a limited execution time to ensure there is enough time left for the parachain +- **PVF preparation and execution time**. Each parachain has it's own PVF (parachain validation function): +a wasm blob that is executed by validators during backing, approval checking and disputing. The PVF +preparation time refers to the time it takes for the PVF wasm to be compiled. This step is done once and +then result cached. PVF execution will use the resulting artifact to execute the PVF for a given candidate. +PVFs are expected to have a limited execution time to ensure there is enough time left for the parachain block to be included in the relay block. -- **Time to recover and check candidate**. This is part of approval voting and covers the time it takes +- **Time to recover and check candidate**. This is part of approval voting and covers the time it takes to recover the candidate block available data from other validators, check it (includes PVF execution time) and issue statement or initiate dispute. -- **Assignment delay tranches**. Approval voting is designed such that validators assigned to check a specific -candidate are split up into equal delay tranches (0.5 seconds each). All validators checks are ordered by the delay -tranche index. Early tranches of validators have the opportunity to check the candidate first before later tranches +- **Assignment delay tranches**. Approval voting is designed such that validators assigned to check a specific +candidate are split up into equal delay tranches (0.5 seconds each). All validators checks are ordered by the delay +tranche index. Early tranches of validators have the opportunity to check the candidate first before later tranches that act as as backups in case of no shows. diff --git a/polkadot/node/collation-generation/Cargo.toml b/polkadot/node/collation-generation/Cargo.toml index e2870dc2cc8a78d48fe3e35b920dbe93a21178e0..b110540140f958b0624738c38fde3aad417bc85e 100644 --- a/polkadot/node/collation-generation/Cargo.toml +++ b/polkadot/node/collation-generation/Cargo.toml @@ -15,7 +15,7 @@ polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } sp-core = { path = "../../../substrate/primitives/core" } sp-maybe-compressed-blob = { path = "../../../substrate/primitives/maybe-compressed-blob" } -thiserror = "1.0.31" +thiserror = "1.0.48" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["bit-vec", "derive"] } [dev-dependencies] diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 307d9947a961619cc2eca8a3562221103db66a73..acad0d1fa4e4f5c6286fde17181d582298ed2922 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -16,7 +16,7 @@ merlin = "2.0" schnorrkel = "0.9.1" kvdb = "0.13.0" derive_more = "0.99.17" -thiserror = "1.0.31" +thiserror = "1.0.48" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 0087f8e1435047dfc8e558c033c45d7d9dcf1c9d..ddef736feab7c9cdc5b79e3508efef47e7bf76af 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -2350,7 +2350,7 @@ async fn process_wakeup( match get_extended_session_info( session_info_provider, ctx.sender(), - block_entry.parent_hash(), + block_entry.block_hash(), block_entry.session(), ) .await diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 3c7b74b26fbed50d2c860ac885bf3e8ecea1665d..0b98f28fbbf0439cd0e9eee4aae537a6956a7cab 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use self::test_helpers::mock::new_leaf; use super::*; use polkadot_node_primitives::{ approval::{ @@ -26,7 +27,7 @@ use polkadot_node_subsystem::{ messages::{ AllMessages, ApprovalVotingMessage, AssignmentCheckResult, AvailabilityRecoveryMessage, }, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, + ActiveLeavesUpdate, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt; @@ -777,12 +778,7 @@ async fn import_block( overseer_send( overseer, FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( - ActivatedLeaf { - hash: *new_head, - number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }, + new_leaf(*new_head, number), ))), ) .await; diff --git a/polkadot/node/core/av-store/Cargo.toml b/polkadot/node/core/av-store/Cargo.toml index efbbb27754e2471e3631330da7b720fb37f2efec..955fe37d7c39e0cc4a840fe7bdd8e2a98be364db 100644 --- a/polkadot/node/core/av-store/Cargo.toml +++ b/polkadot/node/core/av-store/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true futures = "0.3.21" futures-timer = "3.0.2" kvdb = "0.13.0" -thiserror = "1.0.31" +thiserror = "1.0.48" gum = { package = "tracing-gum", path = "../../gum" } bitvec = "1.0.0" diff --git a/polkadot/node/core/av-store/src/tests.rs b/polkadot/node/core/av-store/src/tests.rs index dbccf1401582270cadab9c4078dacc864f31a2d1..652bf2a3fda4822fdae0020bcd48993ed8e771ed 100644 --- a/polkadot/node/core/av-store/src/tests.rs +++ b/polkadot/node/core/av-store/src/tests.rs @@ -19,14 +19,14 @@ use super::*; use assert_matches::assert_matches; use futures::{channel::oneshot, executor, future, Future}; +use self::test_helpers::mock::new_leaf; use ::test_helpers::TestCandidateBuilder; use parking_lot::Mutex; use polkadot_node_primitives::{AvailableData, BlockData, PoV, Proof}; use polkadot_node_subsystem::{ errors::RuntimeApiError, - jaeger, messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, + ActiveLeavesUpdate, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{database::Database, TimeoutExt}; @@ -219,16 +219,11 @@ fn runtime_api_error_does_not_stop_the_subsystem() { let store = test_store(); test_harness(TestState::default(), store, |mut virtual_overseer| async move { - let new_leaf = Hash::repeat_byte(0x01); + let a_leaf = Hash::repeat_byte(0x01); overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: new_leaf, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(a_leaf, 1))), ) .await; @@ -246,7 +241,7 @@ fn runtime_api_error_does_not_stop_the_subsystem() { relay_parent, tx, )) => { - assert_eq!(relay_parent, new_leaf); + assert_eq!(relay_parent, a_leaf); tx.send(Ok(Some(header))).unwrap(); } ); @@ -258,7 +253,7 @@ fn runtime_api_error_does_not_stop_the_subsystem() { relay_parent, RuntimeApiRequest::CandidateEvents(tx), )) => { - assert_eq!(relay_parent, new_leaf); + assert_eq!(relay_parent, a_leaf); #[derive(Debug)] struct FauxError; impl std::error::Error for FauxError {} @@ -741,7 +736,7 @@ fn stored_data_kept_until_finalized() { available_data, ); - let new_leaf = import_leaf( + let a_leaf = import_leaf( &mut virtual_overseer, parent, block_number, @@ -764,7 +759,7 @@ fn stored_data_kept_until_finalized() { overseer_signal( &mut virtual_overseer, - OverseerSignal::BlockFinalized(new_leaf, block_number), + OverseerSignal::BlockFinalized(a_leaf, block_number), ) .await; @@ -849,16 +844,11 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() { extrinsics_root: Hash::zero(), digest: Default::default(), }; - let new_leaf = Hash::repeat_byte(4); + let a_leaf = Hash::repeat_byte(4); overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: new_leaf, - number: 4, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(a_leaf, 4))), ) .await; @@ -868,7 +858,7 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() { relay_parent, tx, )) => { - assert_eq!(relay_parent, new_leaf); + assert_eq!(relay_parent, a_leaf); tx.send(Ok(Some(header))).unwrap(); } ); @@ -886,7 +876,7 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() { k, response_channel: tx, }) => { - assert_eq!(hash, new_leaf); + assert_eq!(hash, a_leaf); assert_eq!(k, 2); let _ = tx.send(Ok(vec![ Hash::repeat_byte(3), @@ -1166,16 +1156,11 @@ async fn import_leaf( extrinsics_root: Hash::zero(), digest: Default::default(), }; - let new_leaf = header.hash(); + let a_leaf = header.hash(); overseer_signal( virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: new_leaf, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf(a_leaf, 1))), ) .await; @@ -1185,7 +1170,7 @@ async fn import_leaf( relay_parent, tx, )) => { - assert_eq!(relay_parent, new_leaf); + assert_eq!(relay_parent, a_leaf); tx.send(Ok(Some(header))).unwrap(); } ); @@ -1196,7 +1181,7 @@ async fn import_leaf( relay_parent, RuntimeApiRequest::CandidateEvents(tx), )) => { - assert_eq!(relay_parent, new_leaf); + assert_eq!(relay_parent, a_leaf); tx.send(Ok(events)).unwrap(); } ); @@ -1212,7 +1197,7 @@ async fn import_leaf( } ); - new_leaf + a_leaf } #[test] diff --git a/polkadot/node/core/backing/Cargo.toml b/polkadot/node/core/backing/Cargo.toml index 0005f6f6a30d5aad66f2982fa14f844e6c4a4851..e7e6358e8a46ea546fe1f97d4e01a094db7fce63 100644 --- a/polkadot/node/core/backing/Cargo.toml +++ b/polkadot/node/core/backing/Cargo.toml @@ -16,7 +16,7 @@ erasure-coding = { package = "polkadot-erasure-coding", path = "../../../erasure statement-table = { package = "polkadot-statement-table", path = "../../../statement-table" } bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } gum = { package = "tracing-gum", path = "../../gum" } -thiserror = "1.0.31" +thiserror = "1.0.48" fatality = "0.0.6" [dev-dependencies] diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index f5ea827d254c6d95c59c8dac848461dd5a9eac9e..4c2fd6becb424033a13d1d1f5b1653cf811c48ad 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -14,22 +14,22 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use self::test_helpers::mock::new_leaf; use super::*; use ::test_helpers::{ dummy_candidate_receipt_bad_sig, dummy_collator, dummy_collator_signature, - dummy_committed_candidate_receipt, dummy_hash, + dummy_committed_candidate_receipt, dummy_hash, validator_pubkeys, }; use assert_matches::assert_matches; use futures::{future, Future}; use polkadot_node_primitives::{BlockData, InvalidCandidate, SignedFullStatement, Statement}; use polkadot_node_subsystem::{ errors::RuntimeApiError, - jaeger, messages::{ AllMessages, CollatorProtocolMessage, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, - ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, LeafStatus, OverseerSignal, TimeoutExt, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, TimeoutExt, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{ @@ -48,10 +48,6 @@ mod prospective_parachains; const ASYNC_BACKING_DISABLED_ERROR: RuntimeApiError = RuntimeApiError::NotSupported { runtime_api_name: "test-runtime" }; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - fn table_statement_to_primitive(statement: TableStatement) -> Statement { match statement { TableStatement::Seconded(committed_candidate_receipt) => @@ -234,12 +230,7 @@ async fn test_startup(virtual_overseer: &mut VirtualOverseer, test_state: &TestS // Start work on some new parent. virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( - ActivatedLeaf { - hash: test_state.relay_parent, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }, + new_leaf(test_state.relay_parent, 1), )))) .await; @@ -304,6 +295,81 @@ async fn test_startup(virtual_overseer: &mut VirtualOverseer, test_state: &TestS ); } +async fn assert_validation_requests( + virtual_overseer: &mut VirtualOverseer, + validation_code: ValidationCode, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) + ) if hash == validation_code.hash() => { + tx.send(Ok(Some(validation_code))).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) + ) => { + tx.send(Ok(1u32.into())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) + ) if sess_idx == 1 => { + tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); +} + +async fn assert_validate_from_exhaustive( + virtual_overseer: &mut VirtualOverseer, + pvd: &PersistedValidationData, + pov: &PoV, + validation_code: &ValidationCode, + candidate: &CommittedCandidateReceipt, + expected_head_data: &HeadData, + result_validation_data: PersistedValidationData, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive( + _pvd, + _validation_code, + candidate_receipt, + _pov, + _, + timeout, + tx, + ), + ) if _pvd == *pvd && + _validation_code == *validation_code && + *_pov == *pov && &candidate_receipt.descriptor == candidate.descriptor() && + timeout == PvfExecTimeoutKind::Backing && + candidate.commitments.hash() == candidate_receipt.commitments_hash => + { + tx.send(Ok(ValidationResult::Valid( + CandidateCommitments { + head_data: expected_head_data.clone(), + horizontal_messages: Default::default(), + upward_messages: Default::default(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, + result_validation_data, + ))) + .unwrap(); + } + ); +} + // Test that a `CandidateBackingMessage::Second` issues validation work // and in case validation is successful issues a `StatementDistributionMessage`. #[test] @@ -339,65 +405,18 @@ fn backing_second_works() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive( - _pvd, - _validation_code, - candidate_receipt, - _pov, - _, - timeout, - tx, - ), - ) if _pvd == pvd && - _validation_code == validation_code && - *_pov == pov && &candidate_receipt.descriptor == candidate.descriptor() && - timeout == PvfExecTimeoutKind::Backing && - candidate.commitments.hash() == candidate_receipt.commitments_hash => - { - tx.send(Ok(ValidationResult::Valid( - CandidateCommitments { - head_data: expected_head_data.clone(), - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, - test_state.validation_data.clone(), - ))) - .unwrap(); - } - ); + assert_validate_from_exhaustive( + &mut virtual_overseer, + &pvd, + &pov, + &validation_code, + &candidate, + expected_head_data, + test_state.validation_data.clone(), + ) + .await; assert_matches!( virtual_overseer.recv().await, @@ -504,32 +523,7 @@ fn backing_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Sending a `Statement::Seconded` for our assignment will start // validation process. The first thing requested is the PoV. @@ -707,32 +701,7 @@ fn backing_works_while_validation_ongoing() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Sending a `Statement::Seconded` for our assignment will start // validation process. The first thing requested is PoV from the @@ -898,32 +867,7 @@ fn backing_misbehavior_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1103,32 +1047,7 @@ fn backing_dont_second_invalid() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_a.hash() => { - tx.send(Ok(Some(validation_code_a.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_a.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1168,32 +1087,7 @@ fn backing_dont_second_invalid() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_b.hash() => { - tx.send(Ok(Some(validation_code_b.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_b.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1305,32 +1199,7 @@ fn backing_second_after_first_fails_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. assert_matches!( @@ -1413,32 +1282,7 @@ fn backing_second_after_first_fails_works() { // triggered on the prev step. virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_to_second.hash() => { - tx.send(Ok(Some(validation_code_to_second.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_to_second.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1499,32 +1343,7 @@ fn backing_works_after_failed_validation() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. assert_matches!( @@ -1727,32 +1546,7 @@ fn retry_works() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. // We cancel - should mean retry on next backing statement. @@ -1815,32 +1609,7 @@ fn retry_works() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_c.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -2031,65 +1800,18 @@ fn cannot_second_multiple_candidates_per_parent() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive( - _pvd, - _validation_code, - candidate_receipt, - _pov, - _, - timeout, - tx, - ), - ) if _pvd == pvd && - _validation_code == validation_code && - *_pov == pov && &candidate_receipt.descriptor == candidate.descriptor() && - timeout == PvfExecTimeoutKind::Backing && - candidate.commitments.hash() == candidate_receipt.commitments_hash => - { - tx.send(Ok(ValidationResult::Valid( - CandidateCommitments { - head_data: expected_head_data.clone(), - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, - test_state.validation_data.clone(), - ))) - .unwrap(); - } - ); + assert_validate_from_exhaustive( + &mut virtual_overseer, + &pvd, + &pov, + &validation_code, + &candidate, + expected_head_data, + test_state.validation_data.clone(), + ) + .await; assert_matches!( virtual_overseer.recv().await, @@ -2136,32 +1858,7 @@ fn cannot_second_multiple_candidates_per_parent() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; // The validation is still requested. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index 07e2a3a89bb66a573b00e59e05aaa96eae38ae72..14f720b721f2612e668fecf295232d1a34a766f3 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -18,7 +18,7 @@ use polkadot_node_subsystem::{ messages::{ChainApiMessage, FragmentTreeMembership}, - TimeoutExt, + ActivatedLeaf, TimeoutExt, }; use polkadot_primitives::{vstaging as vstaging_primitives, BlockNumber, Header, OccupiedCore}; @@ -208,32 +208,7 @@ async fn assert_validate_seconded_candidate( expected_head_data: &HeadData, fetch_pov: bool, ) { - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if parent == relay_parent && hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(virtual_overseer, validation_code.clone()).await; if fetch_pov { assert_matches!( @@ -346,12 +321,7 @@ fn seconding_sanity_check_allowed() { // `a` is grandparent of `b`. let leaf_a_hash = Hash::from_low_u64_be(130); let leaf_a_parent = get_parent_hash(leaf_a_hash); - let activated = ActivatedLeaf { - hash: leaf_a_hash, - number: LEAF_A_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_a_hash, LEAF_A_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_A_BLOCK_NUMBER - LEAF_A_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -359,12 +329,7 @@ fn seconding_sanity_check_allowed() { const LEAF_B_ANCESTRY_LEN: BlockNumber = 4; let leaf_b_hash = Hash::from_low_u64_be(128); - let activated = ActivatedLeaf { - hash: leaf_b_hash, - number: LEAF_B_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_b_hash, LEAF_B_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_B_BLOCK_NUMBER - LEAF_B_ANCESTRY_LEN)]; let test_leaf_b = TestLeaf { activated, min_relay_parents }; @@ -503,24 +468,14 @@ fn seconding_sanity_check_disallowed() { // `a` is grandparent of `b`. let leaf_a_hash = Hash::from_low_u64_be(130); let leaf_a_parent = get_parent_hash(leaf_a_hash); - let activated = ActivatedLeaf { - hash: leaf_a_hash, - number: LEAF_A_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_a_hash, LEAF_A_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_A_BLOCK_NUMBER - LEAF_A_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; const LEAF_B_BLOCK_NUMBER: BlockNumber = LEAF_A_BLOCK_NUMBER + 2; const LEAF_B_ANCESTRY_LEN: BlockNumber = 4; - let activated = ActivatedLeaf { - hash: leaf_b_hash, - number: LEAF_B_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_b_hash, LEAF_B_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_B_BLOCK_NUMBER - LEAF_B_ANCESTRY_LEN)]; let test_leaf_b = TestLeaf { activated, min_relay_parents }; @@ -722,12 +677,7 @@ fn prospective_parachains_reject_candidate() { let leaf_a_hash = Hash::from_low_u64_be(130); let leaf_a_parent = get_parent_hash(leaf_a_hash); - let activated = ActivatedLeaf { - hash: leaf_a_hash, - number: LEAF_A_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_a_hash, LEAF_A_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_A_BLOCK_NUMBER - LEAF_A_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -905,12 +855,7 @@ fn second_multiple_candidates_per_relay_parent() { let leaf_hash = Hash::from_low_u64_be(130); let leaf_parent = get_parent_hash(leaf_hash); let leaf_grandparent = get_parent_hash(leaf_parent); - let activated = ActivatedLeaf { - hash: leaf_hash, - number: LEAF_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_hash, LEAF_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_BLOCK_NUMBER - LEAF_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -1046,12 +991,7 @@ fn backing_works() { let leaf_hash = Hash::from_low_u64_be(130); let leaf_parent = get_parent_hash(leaf_hash); - let activated = ActivatedLeaf { - hash: leaf_hash, - number: LEAF_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_hash, LEAF_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_BLOCK_NUMBER - LEAF_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -1212,12 +1152,7 @@ fn concurrent_dependent_candidates() { let leaf_hash = Hash::from_low_u64_be(130); let leaf_parent = get_parent_hash(leaf_hash); let leaf_grandparent = get_parent_hash(leaf_parent); - let activated = ActivatedLeaf { - hash: leaf_hash, - number: LEAF_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_hash, LEAF_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_BLOCK_NUMBER - LEAF_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; @@ -1458,12 +1393,7 @@ fn seconding_sanity_check_occupy_same_depth() { let leaf_hash = Hash::from_low_u64_be(130); let leaf_parent = get_parent_hash(leaf_hash); - let activated = ActivatedLeaf { - hash: leaf_hash, - number: LEAF_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_hash, LEAF_BLOCK_NUMBER); let min_block_number = LEAF_BLOCK_NUMBER - LEAF_ANCESTRY_LEN; let min_relay_parents = vec![(para_id_a, min_block_number), (para_id_b, min_block_number)]; @@ -1617,12 +1547,7 @@ fn occupied_core_assignment() { let leaf_a_hash = Hash::from_low_u64_be(130); let leaf_a_parent = get_parent_hash(leaf_a_hash); - let activated = ActivatedLeaf { - hash: leaf_a_hash, - number: LEAF_A_BLOCK_NUMBER, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf_a_hash, LEAF_A_BLOCK_NUMBER); let min_relay_parents = vec![(para_id, LEAF_A_BLOCK_NUMBER - LEAF_A_ANCESTRY_LEN)]; let test_leaf_a = TestLeaf { activated, min_relay_parents }; diff --git a/polkadot/node/core/bitfield-signing/Cargo.toml b/polkadot/node/core/bitfield-signing/Cargo.toml index c2df6cc709e6aafc4387a76e28fc719dac0324d4..de38d18d970623dd6dc14fdcecb5dd4447232481 100644 --- a/polkadot/node/core/bitfield-signing/Cargo.toml +++ b/polkadot/node/core/bitfield-signing/Cargo.toml @@ -13,7 +13,7 @@ polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } sp-keystore = { path = "../../../../substrate/primitives/keystore" } wasm-timer = "0.2.5" -thiserror = "1.0.31" +thiserror = "1.0.48" [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } diff --git a/polkadot/node/core/chain-selection/Cargo.toml b/polkadot/node/core/chain-selection/Cargo.toml index 57ba7908315bf9ca2abcab083a1ff223025ad6b7..7678379870e0371de28eeb3d3807b90820504b87 100644 --- a/polkadot/node/core/chain-selection/Cargo.toml +++ b/polkadot/node/core/chain-selection/Cargo.toml @@ -15,7 +15,7 @@ polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } kvdb = "0.13.0" -thiserror = "1.0.31" +thiserror = "1.0.48" parity-scale-codec = "3.6.1" [dev-dependencies] diff --git a/polkadot/node/core/chain-selection/src/tests.rs b/polkadot/node/core/chain-selection/src/tests.rs index c04f9aaf66063d6f9d7e16d8b2a4e020c0c75b70..cf021c0efeb0638b5fe1db4ef5dbe9dd4c7766f6 100644 --- a/polkadot/node/core/chain-selection/src/tests.rs +++ b/polkadot/node/core/chain-selection/src/tests.rs @@ -35,11 +35,10 @@ use parity_scale_codec::Encode; use parking_lot::Mutex; use sp_core::testing::TaskExecutor; -use polkadot_node_subsystem::{ - jaeger, messages::AllMessages, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, -}; +use polkadot_node_subsystem::{messages::AllMessages, ActiveLeavesUpdate}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{BlakeTwo256, ConsensusLog, HashT}; +use test_helpers::mock::new_leaf; #[derive(Default)] struct TestBackendInner { @@ -367,12 +366,10 @@ async fn import_blocks_into( let hash = header.hash(); virtual_overseer .send( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( hash, - number: header.number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })) + header.number, + ))) .into(), ) .await; @@ -425,12 +422,10 @@ async fn import_all_blocks_into( let (_, write_rx) = backend.await_next_write(); virtual_overseer .send( - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: head_hash, - number: head.number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })) + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + head_hash, + head.number, + ))) .into(), ) .await; diff --git a/polkadot/node/core/dispute-coordinator/Cargo.toml b/polkadot/node/core/dispute-coordinator/Cargo.toml index c49bd507127dda023f947d7a75b3934d6b3f837e..6061c52b7e810e75cd810dd76287ea1337166ed2 100644 --- a/polkadot/node/core/dispute-coordinator/Cargo.toml +++ b/polkadot/node/core/dispute-coordinator/Cargo.toml @@ -10,7 +10,7 @@ futures = "0.3.21" gum = { package = "tracing-gum", path = "../../gum" } parity-scale-codec = "3.6.1" kvdb = "0.13.0" -thiserror = "1.0.31" +thiserror = "1.0.48" schnellru = "0.2.1" fatality = "0.0.6" diff --git a/polkadot/node/core/dispute-coordinator/src/initialized.rs b/polkadot/node/core/dispute-coordinator/src/initialized.rs index 2d8e15064e4736776864b3a2ecfb8b37960002ef..9cd544a8c5362ea25c30d1ed1fc21c38bd24458e 100644 --- a/polkadot/node/core/dispute-coordinator/src/initialized.rs +++ b/polkadot/node/core/dispute-coordinator/src/initialized.rs @@ -301,6 +301,13 @@ impl Initialized { self.participation.process_active_leaves_update(ctx, &update).await?; if let Some(new_leaf) = update.activated { + gum::trace!( + target: LOG_TARGET, + leaf_hash = ?new_leaf.hash, + block_number = new_leaf.number, + "Processing ActivatedLeaf" + ); + let session_idx = self.runtime_info.get_session_index_for_child(ctx.sender(), new_leaf.hash).await; @@ -308,6 +315,8 @@ impl Initialized { Ok(session_idx) if self.gaps_in_cache || session_idx > self.highest_session_seen => { + // Pin the block from the new session. + self.runtime_info.pin_block(session_idx, new_leaf.unpin_handle); // Fetch the last `DISPUTE_WINDOW` number of sessions unless there are no gaps // in cache and we are not missing too many `SessionInfo`s let mut lower_bound = session_idx.saturating_sub(DISPUTE_WINDOW.get() - 1); @@ -387,26 +396,28 @@ impl Initialized { "Processing unapplied validator slashes", ); + let pinned_hash = self.runtime_info.get_block_in_session(session_index); let inclusions = self.scraper.get_blocks_including_candidate(&candidate_hash); - if inclusions.is_empty() { + if pinned_hash.is_none() && inclusions.is_empty() { gum::info!( target: LOG_TARGET, - "Couldn't find inclusion parent for an unapplied slash", + ?session_index, + "Couldn't find blocks in the session for an unapplied slash", ); return } - // Find the first inclusion parent that we can use + // Find a relay block that we can use // to generate key ownership proof on. - // We use inclusion parents because of the proper session index. + // We use inclusion parents as a fallback. let mut key_ownership_proofs = Vec::new(); let mut dispute_proofs = Vec::new(); - for (_height, inclusion_parent) in inclusions { + let blocks_in_the_session = + pinned_hash.into_iter().chain(inclusions.into_iter().map(|(_n, h)| h)); + for hash in blocks_in_the_session { for (validator_index, validator_id) in pending.keys.iter() { - let res = - key_ownership_proof(ctx.sender(), inclusion_parent, validator_id.clone()) - .await; + let res = key_ownership_proof(ctx.sender(), hash, validator_id.clone()).await; match res { Ok(Some(key_ownership_proof)) => { diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 5a3c4be90aa0107a245a16e97c303a568a083851..35d07b411e8f644ed5491a2e6344fd8f99a71c60 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -260,6 +260,13 @@ impl Participation { req: ParticipationRequest, recent_head: Hash, ) -> FatalResult<()> { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Forking participation" + ); + let participation_timer = self.metrics.time_participation(); if self.running_participations.insert(*req.candidate_hash()) { let sender = ctx.sender().clone(); @@ -314,12 +321,24 @@ async fn participate( }, Ok(Ok(data)) => data, Ok(Err(RecoveryError::Invalid)) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Invalid availability data during participation" + ); // the available data was recovered but it is invalid, therefore we'll // vote negatively for the candidate dispute send_result(&mut result_sender, req, ParticipationOutcome::Invalid).await; return }, Ok(Err(RecoveryError::Unavailable)) | Ok(Err(RecoveryError::ChannelClosed)) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Can't fetch availability data in participation" + ); send_result(&mut result_sender, req, ParticipationOutcome::Unavailable).await; return }, diff --git a/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs index 1105135a747f2a6fb16c5f1cf8e21c6ef9c0271b..d9e86def168c97812fb7bcfb462b49ef7c75548d 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs @@ -277,6 +277,11 @@ impl Queues { match self.priority.entry(comparator) { Entry::Occupied(_) => req.discard_timer(), Entry::Vacant(vac) => { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + "Added to priority participation queue" + ); vac.insert(req); }, } @@ -295,6 +300,11 @@ impl Queues { match self.best_effort.entry(comparator) { Entry::Occupied(_) => req.discard_timer(), Entry::Vacant(vac) => { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + "Added to best effort participation queue" + ); vac.insert(req); }, } diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index a2e8e88cb674a3e760b88d49c5409188203008d0..ee6b950720e5f672d2e33a1f22e367b08d829d6f 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -28,15 +28,14 @@ use ::test_helpers::{ use parity_scale_codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ - jaeger, messages::{ AllMessages, ChainApiMessage, DisputeCoordinatorMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SpawnGlue, + ActiveLeavesUpdate, SpawnGlue, }; use polkadot_node_subsystem_test_helpers::{ - make_subsystem_context, TestSubsystemContext, TestSubsystemContextHandle, + make_subsystem_context, mock::new_leaf, TestSubsystemContext, TestSubsystemContextHandle, }; use polkadot_primitives::{ BlakeTwo256, CandidateCommitments, HashT, Header, PersistedValidationData, ValidationCode, @@ -100,12 +99,7 @@ async fn activate_leaf( participation .process_active_leaves_update( ctx, - &ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: block_hash, - span: Arc::new(jaeger::Span::Disabled), - number: block_number, - status: LeafStatus::Fresh, - }), + &ActiveLeavesUpdate::start_work(new_leaf(block_hash, block_number)), ) .await } diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs b/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs index b1870164cb88cadaad7019ca0be5a0f23f5a3454..c7c9c519916e7c8081df0b48554642f0c2c0ac02 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs @@ -120,18 +120,13 @@ impl ScrapedCandidates { // Removes all candidates up to a given height. The candidates at the block height are NOT // removed. - pub fn remove_up_to_height(&mut self, height: &BlockNumber) -> HashSet { - let mut candidates_modified: HashSet = HashSet::new(); + pub fn remove_up_to_height(&mut self, height: &BlockNumber) { let not_stale = self.candidates_by_block_number.split_off(&height); let stale = std::mem::take(&mut self.candidates_by_block_number); self.candidates_by_block_number = not_stale; - for candidates in stale.values() { - for c in candidates { - self.candidates.remove(c); - candidates_modified.insert(*c); - } + for candidate in stale.values().flatten() { + self.candidates.remove(candidate); } - candidates_modified } pub fn insert(&mut self, block_number: BlockNumber, candidate_hash: CandidateHash) { diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs b/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs index 67434bca85d418b0dccd7b96173935e72d7942cc..fdf39cff0f2ef9b44f9f3d047cbab484a93ce5c5 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::collections::{BTreeMap, HashSet}; +use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use futures::channel::oneshot; use schnellru::{ByLength, LruMap}; @@ -78,49 +78,64 @@ impl ScrapedUpdates { /// concluding against a candidate. Each candidate hash maps to a number of /// block heights, which in turn map to vectors of blocks at those heights. pub struct Inclusions { - inclusions_inner: BTreeMap>>, + inclusions_inner: BTreeMap>>, + /// Keeps track at which block number a candidate was inserted. Used in `remove_up_to_height`. + /// Without this tracking we won't be able to remove all candidates before block X. + candidates_by_block_number: BTreeMap>, } impl Inclusions { pub fn new() -> Self { - Self { inclusions_inner: BTreeMap::new() } + Self { inclusions_inner: BTreeMap::new(), candidates_by_block_number: BTreeMap::new() } } - // Add parent block to the vector which has CandidateHash as an outer key and - // BlockNumber as an inner key pub fn insert( &mut self, candidate_hash: CandidateHash, block_number: BlockNumber, block_hash: Hash, ) { - if let Some(blocks_including) = self.inclusions_inner.get_mut(&candidate_hash) { - if let Some(blocks_at_height) = blocks_including.get_mut(&block_number) { - blocks_at_height.push(block_hash); - } else { - blocks_including.insert(block_number, Vec::from([block_hash])); - } - } else { - let mut blocks_including: BTreeMap> = BTreeMap::new(); - blocks_including.insert(block_number, Vec::from([block_hash])); - self.inclusions_inner.insert(candidate_hash, blocks_including); - } + self.inclusions_inner + .entry(candidate_hash) + .or_default() + .entry(block_number) + .or_default() + .insert(block_hash); + + self.candidates_by_block_number + .entry(block_number) + .or_default() + .insert(candidate_hash); } - pub fn remove_up_to_height( - &mut self, - height: &BlockNumber, - candidates_modified: HashSet, - ) { - for candidate in candidates_modified { - if let Some(blocks_including) = self.inclusions_inner.get_mut(&candidate) { - // Returns everything after the given key, including the key. This works because the - // blocks are sorted in ascending order. - *blocks_including = blocks_including.split_off(height); + /// Removes all candidates up to a given height. + /// + /// The candidates at the block height are NOT removed. + pub fn remove_up_to_height(&mut self, height: &BlockNumber) { + let not_stale = self.candidates_by_block_number.split_off(&height); + let stale = std::mem::take(&mut self.candidates_by_block_number); + self.candidates_by_block_number = not_stale; + + for candidate in stale.into_values().flatten() { + match self.inclusions_inner.entry(candidate) { + Entry::Vacant(_) => { + // Rare case where same candidate was present on multiple heights, but all are + // pruned at the same time. This candidate was already pruned in the previous + // occurence so it is skipped now. + }, + Entry::Occupied(mut e) => { + let mut blocks_including = std::mem::take(e.get_mut()); + // Returns everything after the given key, including the key. This works because + // the blocks are sorted in ascending order. + blocks_including = blocks_including.split_off(&height); + if blocks_including.is_empty() { + e.remove_entry(); + } else { + *e.get_mut() = blocks_including; + } + }, } } - self.inclusions_inner - .retain(|_, blocks_including| blocks_including.keys().len() > 0); } pub fn get(&self, candidate: &CandidateHash) -> Vec<(BlockNumber, Hash)> { @@ -134,6 +149,10 @@ impl Inclusions { } inclusions_as_vec } + + pub fn contains(&self, candidate: &CandidateHash) -> bool { + self.inclusions_inner.get(candidate).is_some() + } } /// Chain scraper @@ -159,20 +178,20 @@ impl Inclusions { /// another precaution to have their `CandidateReceipts` available in case a dispute is raised on /// them, pub struct ChainScraper { - /// All candidates we have seen included, which not yet have been finalized. - included_candidates: candidates::ScrapedCandidates, - /// All candidates we have seen backed + /// All candidates we have seen backed. backed_candidates: candidates::ScrapedCandidates, - /// Latest relay blocks observed by the provider. - /// - /// We assume that ancestors of cached blocks are already processed, i.e. we have saved - /// corresponding included candidates. - last_observed_blocks: LruMap, + /// Maps included candidate hashes to one or more relay block heights and hashes. /// These correspond to all the relay blocks which marked a candidate as included, /// and are needed to apply reversions in case a dispute is concluded against the /// candidate. inclusions: Inclusions, + + /// Latest relay blocks observed by the provider. + /// + /// This is used to avoid redundant scraping of ancestry. We assume that ancestors of cached + /// blocks are already processed, i.e. we have saved corresponding included candidates. + last_observed_blocks: LruMap, } impl ChainScraper { @@ -194,10 +213,9 @@ impl ChainScraper { Sender: overseer::DisputeCoordinatorSenderTrait, { let mut s = Self { - included_candidates: candidates::ScrapedCandidates::new(), backed_candidates: candidates::ScrapedCandidates::new(), - last_observed_blocks: LruMap::new(ByLength::new(LRU_OBSERVED_BLOCKS_CAPACITY)), inclusions: Inclusions::new(), + last_observed_blocks: LruMap::new(ByLength::new(LRU_OBSERVED_BLOCKS_CAPACITY)), }; let update = ActiveLeavesUpdate { activated: Some(initial_head), deactivated: Default::default() }; @@ -207,7 +225,7 @@ impl ChainScraper { /// Check whether we have seen a candidate included on any chain. pub fn is_candidate_included(&self, candidate_hash: &CandidateHash) -> bool { - self.included_candidates.contains(candidate_hash) + self.inclusions.contains(candidate_hash) } /// Check whether the candidate is backed @@ -298,9 +316,7 @@ impl ChainScraper { { Some(key_to_prune) => { self.backed_candidates.remove_up_to_height(&key_to_prune); - let candidates_modified = - self.included_candidates.remove_up_to_height(&key_to_prune); - self.inclusions.remove_up_to_height(&key_to_prune, candidates_modified); + self.inclusions.remove_up_to_height(&key_to_prune); }, None => { // Nothing to prune. We are still in the beginning of the chain and there are not @@ -337,7 +353,6 @@ impl ChainScraper { ?block_number, "Processing included event" ); - self.included_candidates.insert(block_number, candidate_hash); self.inclusions.insert(candidate_hash, block_number, block_hash); included_receipts.push(receipt); }, diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs b/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs index 3dd58a060d709459bcf612a7958df3165eec4cb2..748f9a16f493e2bce191091638771aa5be57fa9f 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use assert_matches::assert_matches; @@ -25,15 +25,15 @@ use sp_core::testing::TaskExecutor; use ::test_helpers::{dummy_collator, dummy_collator_signature, dummy_hash}; use polkadot_node_primitives::DISPUTE_CANDIDATE_LIFETIME_AFTER_FINALIZATION; use polkadot_node_subsystem::{ - jaeger, messages::{ AllMessages, ChainApiMessage, DisputeCoordinatorMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SpawnGlue, + ActivatedLeaf, ActiveLeavesUpdate, SpawnGlue, }; use polkadot_node_subsystem_test_helpers::{ - make_subsystem_context, TestSubsystemContext, TestSubsystemContextHandle, TestSubsystemSender, + make_subsystem_context, mock::new_leaf, TestSubsystemContext, TestSubsystemContextHandle, + TestSubsystemSender, }; use polkadot_node_subsystem_util::{reexports::SubsystemContext, TimeoutExt}; use polkadot_primitives::{ @@ -41,7 +41,7 @@ use polkadot_primitives::{ GroupIndex, Hash, HashT, HeadData, Id as ParaId, }; -use crate::LOG_TARGET; +use crate::{scraping::Inclusions, LOG_TARGET}; use super::ChainScraper; @@ -141,12 +141,7 @@ fn make_candidate_receipt(relay_parent: Hash) -> CandidateReceipt { /// Get a dummy `ActivatedLeaf` for a given block number. fn get_activated_leaf(n: BlockNumber) -> ActivatedLeaf { - ActivatedLeaf { - hash: get_block_number_hash(n), - number: n, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - } + new_leaf(get_block_number_hash(n), n) } /// Get a dummy relay parent hash for dummy block number. @@ -154,6 +149,11 @@ fn get_block_number_hash(n: BlockNumber) -> Hash { BlakeTwo256::hash(&n.encode()) } +// Creates a dummy relay chain block hash with the convention of hash(b). +fn get_relay_block_hash(height: BlockNumber, fork: u32) -> Hash { + BlakeTwo256::hash(&format!("b_{}_{}", height, fork).encode()) +} + /// Get a dummy event that corresponds to candidate inclusion for the given block number. fn get_backed_and_included_candidate_events(block_number: BlockNumber) -> Vec { let candidate_receipt = make_candidate_receipt(get_block_number_hash(block_number)); @@ -667,3 +667,594 @@ fn inclusions_per_candidate_properly_adds_and_prunes() { assert!(scraper.get_blocks_including_candidate(&candidate.hash()).len() == 0); }); } + +// ----- Inclusions tests ----- + +#[test] +fn inclusions_initialization() { + let inclusions = Inclusions::new(); + + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} +#[test] +fn inclusions_insertion() { + let mut inclusions = Inclusions::new(); + let candidate_receipt = make_candidate_receipt(get_magic_candidate_hash()); + let candidate_hash = candidate_receipt.hash(); + let block_number = 0; + let block_hash = get_block_number_hash(block_number); + + inclusions.insert(candidate_hash, block_number, block_hash); + + // Check inclusions_inner + assert!(inclusions.inclusions_inner.len() == 1, "Expected inclusions_inner to have length 1"); + assert!( + inclusions.inclusions_inner.contains_key(&candidate_hash), + "Expected candidate_hash to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate_hash).unwrap(); + assert!(inner_map.len() == 1, "Expected inner_map to have length 1"); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate_hash in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert!(hash_set.len() == 1, "Expected hash_map to have length 1"); + assert!( + hash_set.contains(&block_hash), + "Expected block_hash to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.len() == 1, + "Expected candidates_by_block_number to have length 1" + ); + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert!( + candidate_set.len() == 1, + "Expected candidate_set to have length 1 for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate_hash), + "Expected candidate_hash to be present for the block_number in candidates_by_block_number" + ); +} + +#[test] +fn inclusions_get() { + let mut inclusions = Inclusions::new(); + let candidate_receipt = make_candidate_receipt(get_magic_candidate_hash()); + let candidate_hash = candidate_receipt.hash(); + + // Insert the candidate with multiple block numbers and block hashes + let block_numbers = [0, 1, 2]; + let block_hashes: Vec<_> = + block_numbers.iter().map(|&num| get_block_number_hash(num)).collect(); + + for (&block_number, &block_hash) in block_numbers.iter().zip(&block_hashes) { + inclusions.insert(candidate_hash, block_number, block_hash); + } + + // Call the get method for that candidate + let result = inclusions.get(&candidate_hash); + + // Verify that the method returns the correct list of block numbers and hashes associated with + // that candidate + assert_eq!( + result.len(), + block_numbers.len(), + "Expected the same number of results as inserted block numbers" + ); + + for (&block_number, &block_hash) in block_numbers.iter().zip(&block_hashes) { + assert!( + result.contains(&(block_number, block_hash)), + "Expected to find ({}, {}) in the result", + block_number, + block_hash + ); + } +} + +#[test] +fn inclusions_duplicate_insertion_same_height_and_block() { + let mut inclusions = Inclusions::new(); + + // Insert a candidate + let candidate1 = make_candidate_receipt(get_magic_candidate_hash()).hash(); + let block_number = 0; + let block_hash = get_block_number_hash(block_number); + + // Insert the candidate once + inclusions.insert(candidate1, block_number, block_hash); + + // Insert the same candidate again at the same height and block + inclusions.insert(candidate1, block_number, block_hash); + + // Check inclusions_inner + assert!( + inclusions.inclusions_inner.contains_key(&candidate1), + "Expected candidate1 to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate1).unwrap(); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate1 in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert_eq!( + hash_set.len(), + 1, + "Expected only one block_hash for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash), + "Expected block_hash to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert_eq!( + candidate_set.len(), + 1, + "Expected only one candidate for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate1), + "Expected candidate1 to be present for the block_number in candidates_by_block_number" + ); +} + +#[test] +fn test_duplicate_insertion_same_height_different_blocks() { + let mut inclusions = Inclusions::new(); + + // Insert a candidate + let candidate1 = make_candidate_receipt(get_magic_candidate_hash()).hash(); + let block_number = 0; + let block_hash1 = BlakeTwo256::hash(&"b1".encode()); + let block_hash2 = BlakeTwo256::hash(&"b2".encode()); // Different block hash for the same height + inclusions.insert(candidate1, block_number, block_hash1); + inclusions.insert(candidate1, block_number, block_hash2); + + // Check inclusions_inner + assert!( + inclusions.inclusions_inner.contains_key(&candidate1), + "Expected candidate1 to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate1).unwrap(); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate1 in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert_eq!( + hash_set.len(), + 2, + "Expected two block_hashes for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash1), + "Expected block_hash1 to be present for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash2), + "Expected block_hash2 to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert_eq!( + candidate_set.len(), + 1, + "Expected only one candidate for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate1), + "Expected candidate1 to be present for the block_number in candidates_by_block_number" + ); +} + +// ----- Inclusions removal tests ----- +// inclusions_removal_null_case +// +// inclusions_removal_one_candidate_one_height_one_branch +// +// inclusions_removal_one_candidate_one_height_multi_branch +// inclusions_removal_one_candidate_multi_height_one_branch +// inclusions_removal_multi_candidate_one_height_one_branch +// +// inclusions_removal_multi_candidate_multi_height_one_branch +// inclusions_removal_one_candidate_multi_height_multi_branch +// inclusions_removal_multi_candidate_one_height_multi_branch +// +// inclusions_removal_multi_candidate_multi_height_multi_branch +#[test] +fn inclusions_removal_null_case() { + let mut inclusions = Inclusions::new(); + let height = 5; + + // Ensure both maps are empty before the operation + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); + + inclusions.remove_up_to_height(&height); + + // Ensure both maps remain empty after the operation + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_one_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 + // C1 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_one_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 + // C1 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.inclusions_inner.get(&candidate1).unwrap().get(&0).unwrap().len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_multi_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 1 2 3 4 + // C1 0 0 + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 3, get_relay_block_hash(3, 0)); + + // No prune case + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 20 (overshot) + inclusions.remove_up_to_height(&20); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_one_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 + // C1 0 + // C2 0 + // C3 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 1 2 3 + // C1 0 0 + // C2 0 + // C3 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 2, get_relay_block_hash(2, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(!inclusions.contains(&candidate2), "Expected candidate2 to be removed"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(!inclusions.contains(&candidate2), "Expected candidate2 to be removed"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_multi_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 1 2 + // C1 0 0&1 1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 3); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_one_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 + // C1 0 + // C2 0&1 + // C3 1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + let candidate4 = make_candidate_receipt(BlakeTwo256::hash(&"c4".encode())).hash(); + + // B 0 1 2 + // C1 0&1 0 0 //shouldn't get pruned as long as one of the forks need it + // C2 1 1 + // C3 0 1 + // C4 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate2, 2, get_relay_block_hash(2, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.contains(&candidate4), "Expected candidate4 to remain"); + assert!(inclusions.inclusions_inner.len() == 4); + assert!(inclusions.candidates_by_block_number.len() == 3); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.contains(&candidate4), "Expected candidate4 to remain"); + assert!(inclusions.inclusions_inner.len() == 4); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(!inclusions.contains(&candidate3), "Expected candidate3 to be removed"); + assert!(!inclusions.contains(&candidate4), "Expected candidate4 to be removed"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_multi_branch_multi_height_prune() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + let candidate4 = make_candidate_receipt(BlakeTwo256::hash(&"c4".encode())).hash(); + + // B 0 1 2 + // C1 0&1 0 0 + // C2 1 1 + // C3 0 1 + // C4 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate2, 2, get_relay_block_hash(2, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 1)); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(!inclusions.contains(&candidate3), "Expected candidate3 to be removed"); + assert!(!inclusions.contains(&candidate4), "Expected candidate4 to be removed"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 20 + inclusions.remove_up_to_height(&20); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} diff --git a/polkadot/node/core/dispute-coordinator/src/tests.rs b/polkadot/node/core/dispute-coordinator/src/tests.rs index 8d6a2e10396286c605df620b0d89776cc7045bba..9254c2a851cea502ed8daccf434a68f48863de94 100644 --- a/polkadot/node/core/dispute-coordinator/src/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/tests.rs @@ -54,12 +54,11 @@ use sp_keystore::{Keystore, KeystorePtr}; use ::test_helpers::{dummy_candidate_receipt_bad_sig, dummy_digest, dummy_hash}; use polkadot_node_primitives::{Timestamp, ACTIVE_DURATION_SECS}; use polkadot_node_subsystem::{ - jaeger, messages::{AllMessages, BlockDescription, RuntimeApiMessage, RuntimeApiRequest}, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, + ActiveLeavesUpdate, }; use polkadot_node_subsystem_test_helpers::{ - make_buffered_subsystem_context, TestSubsystemContextHandle, + make_buffered_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, }; use polkadot_primitives::{ ApprovalVote, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, @@ -276,12 +275,7 @@ impl TestState { gum::debug!(?block_number, "Activating block in activate_leaf_at_session."); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: block_hash, - span: Arc::new(jaeger::Span::Disabled), - number: block_number, - status: LeafStatus::Fresh, - }), + ActiveLeavesUpdate::start_work(new_leaf(block_hash, block_number)), ))) .await; @@ -449,12 +443,7 @@ impl TestState { ); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: *leaf, - number: n as u32, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }), + ActiveLeavesUpdate::start_work(new_leaf(*leaf, n as u32)), ))) .await; diff --git a/polkadot/node/core/parachains-inherent/Cargo.toml b/polkadot/node/core/parachains-inherent/Cargo.toml index 515d70dad8276dbb31e759f0784c1c543ac4a1d1..18d91dcfb565abcc0011777d18b881966013701a 100644 --- a/polkadot/node/core/parachains-inherent/Cargo.toml +++ b/polkadot/node/core/parachains-inherent/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../../gum" } -thiserror = "1.0.31" +thiserror = "1.0.48" async-trait = "0.1.57" polkadot-node-subsystem = { path = "../../subsystem" } polkadot-overseer = { path = "../../overseer" } diff --git a/polkadot/node/core/prospective-parachains/Cargo.toml b/polkadot/node/core/prospective-parachains/Cargo.toml index 9fa17ec0c15440bea25c6a7bb0dba12d94a62319..77a59d87f3fa77a42347550e7538d21b8ceda5b0 100644 --- a/polkadot/node/core/prospective-parachains/Cargo.toml +++ b/polkadot/node/core/prospective-parachains/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true futures = "0.3.19" gum = { package = "tracing-gum", path = "../../gum" } parity-scale-codec = "3.6.4" -thiserror = "1.0.30" +thiserror = "1.0.48" fatality = "0.0.6" bitvec = "1" diff --git a/polkadot/node/core/prospective-parachains/src/tests.rs b/polkadot/node/core/prospective-parachains/src/tests.rs index 9fc77624b979919d5e28cb91936c009b6b6e801e..eb12ea4537f745c2241865c3d3af7a1d4150105b 100644 --- a/polkadot/node/core/prospective-parachains/src/tests.rs +++ b/polkadot/node/core/prospective-parachains/src/tests.rs @@ -24,7 +24,6 @@ use polkadot_node_subsystem::{ }, }; use polkadot_node_subsystem_test_helpers as test_helpers; -use polkadot_node_subsystem_types::{jaeger, ActivatedLeaf, LeafStatus}; use polkadot_primitives::{ vstaging::{AsyncBackingParams, BackingState, Constraints, InboundHrmpLimitations}, CommittedCandidateReceipt, HeadData, Header, PersistedValidationData, ScheduledCore, @@ -32,6 +31,7 @@ use polkadot_primitives::{ }; use polkadot_primitives_test_helpers::make_candidate; use std::sync::Arc; +use test_helpers::mock::new_leaf; const ALLOWED_ANCESTRY_LEN: u32 = 3; const ASYNC_BACKING_PARAMETERS: AsyncBackingParams = @@ -197,12 +197,7 @@ async fn activate_leaf_with_params( ) { let TestLeaf { number, hash, .. } = leaf; - let activated = ActivatedLeaf { - hash: *hash, - number: *number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(*hash, *number); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( @@ -497,12 +492,7 @@ fn should_do_no_work_if_async_backing_disabled_for_leaf() { // Start work on some new parent. virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash, 1)), ))) .await; @@ -1318,12 +1308,7 @@ fn correctly_updates_leaves() { .await; // Activate a leaf and remove one at the same time. - let activated = ActivatedLeaf { - hash: leaf_c.hash, - number: leaf_c.number, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }; + let activated = new_leaf(leaf_c.hash, leaf_c.number); let update = ActiveLeavesUpdate { activated: Some(activated), deactivated: [leaf_b.hash][..].into(), @@ -1349,12 +1334,7 @@ fn correctly_updates_leaves() { .await; // Activate and deactivate the same leaf. - let activated = ActivatedLeaf { - hash: leaf_a.hash, - number: leaf_a.number, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }; + let activated = new_leaf(leaf_a.hash, leaf_a.number); let update = ActiveLeavesUpdate { activated: Some(activated), deactivated: [leaf_a.hash][..].into(), @@ -1578,12 +1558,7 @@ fn uses_ancestry_only_within_session() { vec![Hash::repeat_byte(4), Hash::repeat_byte(3), Hash::repeat_byte(2)]; let session_change_hash = Hash::repeat_byte(3); - let activated = ActivatedLeaf { - hash, - number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(hash, number); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( diff --git a/polkadot/node/core/provisioner/Cargo.toml b/polkadot/node/core/provisioner/Cargo.toml index dc791417166820fc73e81d681f883ea263af7e89..05ea92caa976d99c9274313821cfdce212d74b4b 100644 --- a/polkadot/node/core/provisioner/Cargo.toml +++ b/polkadot/node/core/provisioner/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } futures = "0.3.21" gum = { package = "tracing-gum", path = "../../gum" } -thiserror = "1.0.31" +thiserror = "1.0.48" polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem = { path = "../../subsystem" } diff --git a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/tests.rs b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/tests.rs index 2fdeadb2f4f0e532be1e460507d428a07e1cbc1a..f6c49e52eeb981d6a6a22a6d9a85301933d3fde4 100644 --- a/polkadot/node/core/provisioner/src/disputes/prioritized_selection/tests.rs +++ b/polkadot/node/core/provisioner/src/disputes/prioritized_selection/tests.rs @@ -24,12 +24,11 @@ use polkadot_node_primitives::{CandidateVotes, DisputeStatus, ACTIVE_DURATION_SE use polkadot_node_subsystem::messages::{ AllMessages, DisputeCoordinatorMessage, RuntimeApiMessage, RuntimeApiRequest, }; -use polkadot_node_subsystem_test_helpers::TestSubsystemSender; +use polkadot_node_subsystem_test_helpers::{mock::new_leaf, TestSubsystemSender}; use polkadot_primitives::{ CandidateHash, DisputeState, InvalidDisputeStatementKind, SessionIndex, ValidDisputeStatementKind, ValidatorSignature, }; -use std::sync::Arc; use test_helpers; // @@ -353,12 +352,7 @@ async fn mock_overseer( } fn leaf() -> ActivatedLeaf { - ActivatedLeaf { - hash: Hash::repeat_byte(0xAA), - number: 0xAA, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - } + new_leaf(Hash::repeat_byte(0xAA), 0xAA) } struct TestDisputes { diff --git a/polkadot/node/core/pvf-checker/Cargo.toml b/polkadot/node/core/pvf-checker/Cargo.toml index 783ac19009a3fd7c7020a4bdefe3f8eedbd1c71b..0326a20e5a52e7a07fe96a8c721af1c6ac86e5ef 100644 --- a/polkadot/node/core/pvf-checker/Cargo.toml +++ b/polkadot/node/core/pvf-checker/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true [dependencies] futures = "0.3.21" -thiserror = "1.0.31" +thiserror = "1.0.48" gum = { package = "tracing-gum", path = "../../gum" } polkadot-node-primitives = { path = "../../primitives" } diff --git a/polkadot/node/core/pvf-checker/src/tests.rs b/polkadot/node/core/pvf-checker/src/tests.rs index d1daa7a5813586675e85b4416516a41cd2db0ac9..4ac6e5eba31d7a98b24b4acc69d645147f107bc8 100644 --- a/polkadot/node/core/pvf-checker/src/tests.rs +++ b/polkadot/node/core/pvf-checker/src/tests.rs @@ -14,17 +14,18 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use ::test_helpers::{dummy_digest, dummy_hash}; +use ::test_helpers::{dummy_digest, dummy_hash, validator_pubkeys}; use futures::{channel::oneshot, future::BoxFuture, prelude::*}; use polkadot_node_subsystem::{ - jaeger, messages::{ AllMessages, CandidateValidationMessage, PreCheckOutcome, PvfCheckerMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, LeafStatus, OverseerSignal, RuntimeApiError, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, RuntimeApiError, +}; +use polkadot_node_subsystem_test_helpers::{ + make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, }; -use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; use polkadot_primitives::{ BlockNumber, Hash, Header, PvfCheckStatement, SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, @@ -93,10 +94,6 @@ struct TestState { const OUR_VALIDATOR: Sr25519Keyring = Sr25519Keyring::Alice; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - impl TestState { fn new() -> Self { // Initialize the default session 1. No validators are present there. @@ -195,12 +192,7 @@ impl TestState { }, ); - Some(ActivatedLeaf { - hash: activated_leaf.block_hash, - span: Arc::new(jaeger::Span::Disabled), - number: activated_leaf.block_number, - status: LeafStatus::Fresh, - }) + Some(new_leaf(activated_leaf.block_hash, activated_leaf.block_number)) } else { None }; diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index a265139d95c01ef3d2ac2334d2570fdab4d83994..478d1952d9d9168080cdd14917b25cd051495a60 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -6,11 +6,6 @@ authors.workspace = true edition.workspace = true license.workspace = true -[[bin]] -name = "puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] always-assert = "0.1" futures = "0.3.21" @@ -35,7 +30,6 @@ polkadot-primitives = { path = "../../../primitives" } sp-core = { path = "../../../../substrate/primitives/core" } sp-wasm-interface = { path = "../../../../substrate/primitives/wasm-interface" } sp-maybe-compressed-blob = { path = "../../../../substrate/primitives/maybe-compressed-blob" } -sp-tracing = { path = "../../../../substrate/primitives/tracing", optional = true } polkadot-node-core-pvf-prepare-worker = { path = "prepare-worker", optional = true } polkadot-node-core-pvf-execute-worker = { path = "execute-worker", optional = true } @@ -44,7 +38,7 @@ substrate-build-script-utils = { path = "../../../../substrate/utils/build-scrip [dev-dependencies] assert_matches = "1.4.0" -hex-literal = "0.3.4" +hex-literal = "0.4.1" polkadot-node-core-pvf-common = { path = "common", features = ["test-utils"] } # For the puppet worker, depend on ourselves with the test-utils feature. polkadot-node-core-pvf = { path = ".", features = ["test-utils"] } @@ -56,9 +50,7 @@ halt = { package = "test-parachain-halt", path = "../../../parachain/test-parach ci-only-tests = [] jemalloc-allocator = [ "polkadot-node-core-pvf-common/jemalloc-allocator" ] # This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. test-utils = [ "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", - "sp-tracing", ] diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs index 40e540bb3f7eb165dddee7fc2394ec53b147ef4a..bcdf882f300c09c14c54e6f5b3cbc131f600c3a4 100644 --- a/polkadot/node/core/pvf/common/src/worker/mod.rs +++ b/polkadot/node/core/pvf/common/src/worker/mod.rs @@ -33,7 +33,7 @@ use tokio::{io, net::UnixStream, runtime::Runtime}; /// spawning the desired worker. #[macro_export] macro_rules! decl_worker_main { - ($expected_command:expr, $entrypoint:expr, $worker_version:expr) => { + ($expected_command:expr, $entrypoint:expr, $worker_version:expr $(,)*) => { fn print_help(expected_command: &str) { println!("{} {}", expected_command, $worker_version); println!(); @@ -60,6 +60,10 @@ macro_rules! decl_worker_main { println!("{}", $worker_version); return }, + "test-sleep" => { + std::thread::sleep(std::time::Duration::from_secs(5)); + return + }, subcommand => { // Must be passed for compatibility with the single-binary test workers. if subcommand != $expected_command { diff --git a/polkadot/node/core/pvf/prepare-worker/src/memory_stats.rs b/polkadot/node/core/pvf/prepare-worker/src/memory_stats.rs index 7904dfa9cb8857d34c1d0f84040caec3692e16bf..c70ff56fc84dbb00439d0c461179425809749739 100644 --- a/polkadot/node/core/pvf/prepare-worker/src/memory_stats.rs +++ b/polkadot/node/core/pvf/prepare-worker/src/memory_stats.rs @@ -151,7 +151,7 @@ pub mod memory_tracker { /// Module for dealing with the `ru_maxrss` (peak resident memory) stat from `getrusage`. /// /// NOTE: `getrusage` with the `RUSAGE_THREAD` parameter is only supported on Linux. `RUSAGE_SELF` -/// works on MacOS, but we need to get the max rss only for the preparation thread. Gettng it for +/// works on MacOS, but we need to get the max rss only for the preparation thread. Getting it for /// the current process would conflate the stats of previous jobs run by the process. #[cfg(target_os = "linux")] pub mod max_rss_stat { diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index c3a7a4613139ef121bc445a7957a50afb1495865..0e4f2444adf72ce01c4444024f589e79653d0e86 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -100,10 +100,6 @@ mod worker_intf; #[cfg(feature = "test-utils")] pub mod testing; -// Used by `decl_puppet_worker_main!`. -#[cfg(feature = "test-utils")] -pub use sp_tracing; - pub use error::{InvalidCandidate, ValidationError}; pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; pub use metrics::Metrics; @@ -117,11 +113,5 @@ pub use polkadot_node_core_pvf_common::{ pvf::PvfPrepData, }; -// Re-export worker entrypoints. -#[cfg(feature = "test-utils")] -pub use polkadot_node_core_pvf_execute_worker::worker_entrypoint as execute_worker_entrypoint; -#[cfg(feature = "test-utils")] -pub use polkadot_node_core_pvf_prepare_worker::worker_entrypoint as prepare_worker_entrypoint; - /// The log target for this crate. pub const LOG_TARGET: &str = "parachain::pvf"; diff --git a/polkadot/node/core/pvf/src/testing.rs b/polkadot/node/core/pvf/src/testing.rs index 980a28c01566ce7e5b79af5200989b399af654c3..4301afc3cc7ea6e736a351470b39019b74f5f16b 100644 --- a/polkadot/node/core/pvf/src/testing.rs +++ b/polkadot/node/core/pvf/src/testing.rs @@ -47,45 +47,3 @@ pub fn validate_candidate( Ok(result) } - -/// Use this macro to declare a `fn main() {}` that will check the arguments and dispatch them to -/// the appropriate worker, making the executable that can be used for spawning workers. -#[macro_export] -macro_rules! decl_puppet_worker_main { - () => { - fn main() { - $crate::sp_tracing::try_init_simple(); - - let args = std::env::args().collect::>(); - if args.len() == 1 { - panic!("wrong number of arguments"); - } - - let entrypoint = match args[1].as_ref() { - "exit" => { - std::process::exit(1); - }, - "sleep" => { - std::thread::sleep(std::time::Duration::from_secs(5)); - return - }, - "prepare-worker" => $crate::prepare_worker_entrypoint, - "execute-worker" => $crate::execute_worker_entrypoint, - other => panic!("unknown subcommand: {}", other), - }; - - let mut node_version = None; - let mut socket_path: &str = ""; - - for i in (2..args.len()).step_by(2) { - match args[i].as_ref() { - "--socket-path" => socket_path = args[i + 1].as_str(), - "--node-impl-version" => node_version = Some(args[i + 1].as_str()), - arg => panic!("Unexpected argument found: {}", arg), - } - } - - entrypoint(&socket_path, node_version, None); - } - }; -} diff --git a/polkadot/node/core/pvf/tests/README.md b/polkadot/node/core/pvf/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..27385e190250df9baa0d14cea7afc9e00f973a98 --- /dev/null +++ b/polkadot/node/core/pvf/tests/README.md @@ -0,0 +1,9 @@ +# PVF host integration tests + +## Testing + +Before running these tests, make sure the worker binaries are built first. This can be done with: + +```sh +cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker +``` diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 12d87ee4426226b887317ab995cfefdf4ab55b6c..dc8f00098ec5ebe55fa319116d23b308c094ffd9 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -33,7 +33,6 @@ use tokio::sync::Mutex; mod adder; mod worker_common; -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_puppet_worker"); const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3); const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(3); @@ -51,10 +50,20 @@ impl TestHost { where F: FnOnce(&mut Config), { + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + let mut prepare_worker_path = workers_path.clone(); + prepare_worker_path.push("polkadot-prepare-worker"); + let mut execute_worker_path = workers_path.clone(); + execute_worker_path.push("polkadot-execute-worker"); let cache_dir = tempfile::tempdir().unwrap(); - let program_path = std::path::PathBuf::from(PUPPET_EXE); - let mut config = - Config::new(cache_dir.path().to_owned(), None, program_path.clone(), program_path); + let mut config = Config::new( + cache_dir.path().to_owned(), + None, + prepare_worker_path, + execute_worker_path, + ); f(&mut config); let (host, task) = start(config, Metrics::default()); let _ = tokio::task::spawn(task); diff --git a/polkadot/node/core/pvf/tests/it/worker_common.rs b/polkadot/node/core/pvf/tests/it/worker_common.rs index a3bf552e894a5321b6755ccb3f70dc2385df5bf1..875ae79af09732fd14ebabbc1c711549c43f46fa 100644 --- a/polkadot/node/core/pvf/tests/it/worker_common.rs +++ b/polkadot/node/core/pvf/tests/it/worker_common.rs @@ -14,26 +14,41 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::time::Duration; - use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr}; +use std::time::Duration; -use crate::PUPPET_EXE; +fn worker_path(name: &str) -> std::path::PathBuf { + let mut worker_path = std::env::current_exe().unwrap(); + worker_path.pop(); + worker_path.pop(); + worker_path.push(name); + worker_path +} // Test spawning a program that immediately exits with a failure code. #[tokio::test] async fn spawn_immediate_exit() { - let result = - spawn_with_program_path("integration-test", PUPPET_EXE, &["exit"], Duration::from_secs(2)) - .await; + // There's no explicit `exit` subcommand in the worker; it will panic on an unknown + // subcommand anyway + let result = spawn_with_program_path( + "integration-test", + worker_path("polkadot-prepare-worker"), + &["exit"], + Duration::from_secs(2), + ) + .await; assert!(matches!(result, Err(SpawnErr::AcceptTimeout))); } #[tokio::test] async fn spawn_timeout() { - let result = - spawn_with_program_path("integration-test", PUPPET_EXE, &["sleep"], Duration::from_secs(2)) - .await; + let result = spawn_with_program_path( + "integration-test", + worker_path("polkadot-execute-worker"), + &["test-sleep"], + Duration::from_secs(2), + ) + .await; assert!(matches!(result, Err(SpawnErr::AcceptTimeout))); } @@ -41,7 +56,7 @@ async fn spawn_timeout() { async fn should_connect() { let _ = spawn_with_program_path( "integration-test", - PUPPET_EXE, + worker_path("polkadot-prepare-worker"), &["prepare-worker"], Duration::from_secs(2), ) diff --git a/polkadot/node/gum/README.md b/polkadot/node/gum/README.md index 739ce3066ecb33bcb4dcc4143917f02465400e85..aed576351326bc97a02f55099730c1fabb3a11b1 100644 --- a/polkadot/node/gum/README.md +++ b/polkadot/node/gum/README.md @@ -52,6 +52,6 @@ when providing to any of the log macros (`warn!`, `info!`, etc.). The crate has to be used throughout the entire codebase to work consistently, to disambiguate, the prefix `gum::` is used. -Feature parity with `tracing::{warn!,..}` is not desired. We want consistency +Feature Parity with `tracing::{warn!,..}` is not desired. We want consistency more than anything. All currently used features _are_ supported with _gum_ as well. diff --git a/polkadot/node/gum/proc-macro/Cargo.toml b/polkadot/node/gum/proc-macro/Cargo.toml index ec6b817476a9a7c2fa61fc80804dc6361a58d623..83d064cadbed34f7f4103a87b68eb9059d55af5a 100644 --- a/polkadot/node/gum/proc-macro/Cargo.toml +++ b/polkadot/node/gum/proc-macro/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "2.0.31", features = ["full", "extra-traits"] } +syn = { version = "2.0.37", features = ["full", "extra-traits"] } quote = "1.0.28" proc-macro2 = "1.0.56" proc-macro-crate = "1.1.3" diff --git a/polkadot/node/jaeger/Cargo.toml b/polkadot/node/jaeger/Cargo.toml index 7b4b5e1c8bce1e588cba50c768f4440f2d1e6487..fcfbbaec611ef22ae593ad05bcb058a2316a0e9b 100644 --- a/polkadot/node/jaeger/Cargo.toml +++ b/polkadot/node/jaeger/Cargo.toml @@ -14,7 +14,7 @@ polkadot-primitives = { path = "../../primitives" } polkadot-node-primitives = { path = "../primitives" } sc-network = { path = "../../../substrate/client/network" } sp-core = { path = "../../../substrate/primitives/core" } -thiserror = "1.0.31" +thiserror = "1.0.48" tokio = "1.24.2" log = "0.4.17" parity-scale-codec = { version = "3.6.1", default-features = false } diff --git a/polkadot/node/malus/Cargo.toml b/polkadot/node/malus/Cargo.toml index 56053b8814a704d8ca49f25fc13189e3f8f839d8..42dd4af73c130bb75fd4a58b736ab9944b02fb2e 100644 --- a/polkadot/node/malus/Cargo.toml +++ b/polkadot/node/malus/Cargo.toml @@ -26,7 +26,7 @@ path = "../../src/bin/prepare-worker.rs" doc = false [dependencies] -polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } +polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "westend-native" ] } polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-node-subsystem-types = { path = "../subsystem-types" } @@ -40,7 +40,7 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { path = "../../../substrate/primitives/keystore" } sp-core = { path = "../../../substrate/primitives/core" } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } diff --git a/polkadot/node/malus/README.md b/polkadot/node/malus/README.md index fb4bb0cd272fa7b89db995ebaaefa41f6a1ac37c..e0c7893a75319ed40b6bde633309f209dac91225 100644 --- a/polkadot/node/malus/README.md +++ b/polkadot/node/malus/README.md @@ -18,7 +18,7 @@ defined in the [(DSL[(**D**omain **S**pecific **L**anguage)]) doc](https://parit ## Usage -> Assumes you already gained permissiones, ping in element @javier:matrix.parity.io to get access. +> Assumes you already gained permissiones, ping in element `@javier:matrix.parity.io` to get access. > and you have cloned the [zombienet][zombienet] repo. To launch a test case in the development cluster use (e.g. for the ./node/malus/integrationtests/0001-dispute-valid-block.toml): @@ -48,7 +48,7 @@ This will also teardown the namespace after completion. ## Container Image Building Note In order to build the container image you need to have the latest changes from -polkadot and substrate master branches. +Polkadot and Substrate master branches. ```sh pwd # run this from the current dir diff --git a/polkadot/node/malus/src/interceptor.rs b/polkadot/node/malus/src/interceptor.rs index cbf39bccd16027e9b0dbe59745ed2dc28b283968..04ee0905deeb0758f222fce2cc6d3075b5a6000f 100644 --- a/polkadot/node/malus/src/interceptor.rs +++ b/polkadot/node/malus/src/interceptor.rs @@ -47,12 +47,20 @@ where Some(msg) } - /// Modify outgoing messages. + /// Specifies if we need to replace some outgoing message with another (potentially empty) + /// message + fn need_intercept_outgoing( + &self, + _msg: &::OutgoingMessages, + ) -> bool { + false + } + /// Send modified message instead of the original one fn intercept_outgoing( &self, - msg: ::OutgoingMessages, + _msg: &::OutgoingMessages, ) -> Option<::OutgoingMessages> { - Some(msg) + None } } @@ -66,7 +74,7 @@ pub struct InterceptedSender { #[async_trait::async_trait] impl overseer::SubsystemSender for InterceptedSender where - OutgoingMessage: overseer::AssociateOutgoing + Send + 'static, + OutgoingMessage: overseer::AssociateOutgoing + Send + 'static + TryFrom, Sender: overseer::SubsystemSender + overseer::SubsystemSender< < @@ -78,17 +86,48 @@ where < >::Message as overseer::AssociateOutgoing >::OutgoingMessages: - From, + From + Send + Sync, + >::Error: std::fmt::Debug, { async fn send_message(&mut self, msg: OutgoingMessage) { let msg = < <>::Message as overseer::AssociateOutgoing >::OutgoingMessages as From>::from(msg); - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(&msg) { + self.inner.send_message(msg).await; + } + } + else { self.inner.send_message(msg).await; } } + fn try_send_message(&mut self, msg: OutgoingMessage) -> Result<(), TrySendError> { + let msg = < + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(real_msg) = self.message_filter.intercept_outgoing(&msg) { + let orig_msg : OutgoingMessage = msg.into().try_into().expect("must be able to recover the original message"); + self.inner.try_send_message(real_msg).map_err(|e| { + match e { + TrySendError::Full(_) => TrySendError::Full(orig_msg), + TrySendError::Closed(_) => TrySendError::Closed(orig_msg), + } + }) + } + else { + // No message to send after intercepting + Ok(()) + } + } + else { + let orig_msg : OutgoingMessage = msg.into().try_into().expect("must be able to recover the original message"); + self.inner.try_send_message(orig_msg) + } + } + async fn send_messages(&mut self, msgs: T) where T: IntoIterator + Send, @@ -101,9 +140,14 @@ where fn send_unbounded_message(&mut self, msg: OutgoingMessage) { let msg = < - <>::Message as overseer::AssociateOutgoing - >::OutgoingMessages as From>::from(msg); - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(&msg) { + self.inner.send_unbounded_message(msg); + } + } + else { self.inner.send_unbounded_message(msg); } } diff --git a/polkadot/node/malus/src/variants/common.rs b/polkadot/node/malus/src/variants/common.rs index bc5f6f92aedbc3e4b0aa26bf17b22013d54256ba..365b2f16ac21b67480b2dfc1c49b6092511c998d 100644 --- a/polkadot/node/malus/src/variants/common.rs +++ b/polkadot/node/malus/src/variants/common.rs @@ -498,11 +498,4 @@ where msg => Some(msg), } } - - fn intercept_outgoing( - &self, - msg: overseer::CandidateValidationOutgoingMessages, - ) -> Option { - Some(msg) - } } diff --git a/polkadot/node/metrics/Cargo.toml b/polkadot/node/metrics/Cargo.toml index d497fa7607a16bb95f23550bfaf82c69ba062659..e13ae63199ff0960af100a34451c14090b23e764 100644 --- a/polkadot/node/metrics/Cargo.toml +++ b/polkadot/node/metrics/Cargo.toml @@ -11,8 +11,7 @@ futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } - +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } # Both `sc-service` and `sc-cli` are required by runtime metrics `logger_hook()`. sc-service = { path = "../../../substrate/client/service" } sc-cli = { path = "../../../substrate/client/cli" } diff --git a/polkadot/node/metrics/README.md b/polkadot/node/metrics/README.md index cc88884f21422317b4e5ecb02b9f0c8555a523f7..0cf57006f67a97f34ecf1dfdd7738ceff7e55943 100644 --- a/polkadot/node/metrics/README.md +++ b/polkadot/node/metrics/README.md @@ -1,4 +1,4 @@ -# polkadot-node-metrics +# `polkadot-node-metrics` ## Testing diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml index 581192e9560fbd803edbebb379b10a73686e2352..c3c7aa4e0ea517b68e6e69d7aef45870d94efb90 100644 --- a/polkadot/node/network/availability-distribution/Cargo.toml +++ b/polkadot/node/network/availability-distribution/Cargo.toml @@ -17,7 +17,7 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-node-primitives = { path = "../../primitives" } sp-core = { path = "../../../../substrate/primitives/core", features = ["std"] } sp-keystore = { path = "../../../../substrate/primitives/keystore" } -thiserror = "1.0.31" +thiserror = "1.0.48" rand = "0.8.5" derive_more = "0.99.17" schnellru = "0.2.1" diff --git a/polkadot/node/network/availability-distribution/src/requester/tests.rs b/polkadot/node/network/availability-distribution/src/requester/tests.rs index 5f7e4c36f063431bc08a6c43e4ed7acbf4a8612c..c4252b4e439e8a58a5ed54e2039999fb71dfde60 100644 --- a/polkadot/node/network/availability-distribution/src/requester/tests.rs +++ b/polkadot/node/network/availability-distribution/src/requester/tests.rs @@ -16,12 +16,13 @@ use std::collections::HashMap; -use std::{future::Future, sync::Arc}; +use std::future::Future; use futures::FutureExt; use polkadot_node_network_protocol::jaeger; use polkadot_node_primitives::{BlockData, ErasureChunk, PoV}; +use polkadot_node_subsystem_test_helpers::mock::new_leaf; use polkadot_node_subsystem_util::runtime::RuntimeInfo; use polkadot_primitives::{ BlockNumber, CoreState, ExecutorParams, GroupIndex, Hash, Id as ParaId, ScheduledCore, @@ -34,7 +35,7 @@ use polkadot_node_subsystem::{ AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage, ChainApiMessage, NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, SpawnGlue, + ActiveLeavesUpdate, SpawnGlue, }; use polkadot_node_subsystem_test_helpers::{ make_subsystem_context, mock::make_ferdie_keystore, TestSubsystemContext, @@ -205,12 +206,7 @@ fn check_ancestry_lookup_in_same_session() { let spans: HashMap = HashMap::new(); let block_number = 1; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: Vec::new().into(), }; @@ -225,12 +221,7 @@ fn check_ancestry_lookup_in_same_session() { let block_number = 2; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: Vec::new().into(), }; @@ -252,12 +243,7 @@ fn check_ancestry_lookup_in_same_session() { // part of ancestry. let block_number = 2 + Requester::LEAF_ANCESTRY_LEN_WITHIN_SESSION; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: test_state.relay_chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: vec![chain[1], chain[2]].into(), }; requester @@ -292,12 +278,7 @@ fn check_ancestry_lookup_in_different_sessions() { let spans: HashMap = HashMap::new(); let block_number = 3; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: Vec::new().into(), }; @@ -310,12 +291,7 @@ fn check_ancestry_lookup_in_different_sessions() { let block_number = 4; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: vec![chain[1], chain[2], chain[3]].into(), }; @@ -328,12 +304,7 @@ fn check_ancestry_lookup_in_different_sessions() { let block_number = 5; let update = ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: chain[block_number], - number: block_number as u32, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(chain[block_number], block_number as u32)), deactivated: vec![chain[4]].into(), }; diff --git a/polkadot/node/network/availability-distribution/src/tests/state.rs b/polkadot/node/network/availability-distribution/src/tests/state.rs index ecde44788c25def8c4565c15ec57ae2a81951e6e..101d917c0db5bbd572b86647a03c4b7b2bedf969 100644 --- a/polkadot/node/network/availability-distribution/src/tests/state.rs +++ b/polkadot/node/network/availability-distribution/src/tests/state.rs @@ -16,7 +16,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::Arc, time::Duration, }; @@ -34,9 +33,8 @@ use sc_network::{config as netconfig, config::RequestResponseConfig, IfDisconnec use sp_core::{testing::TaskExecutor, traits::SpawnNamed}; use sp_keystore::KeystorePtr; -use polkadot_node_network_protocol::{ - jaeger, - request_response::{v1, IncomingRequest, OutgoingRequest, Requests}, +use polkadot_node_network_protocol::request_response::{ + v1, IncomingRequest, OutgoingRequest, Requests, }; use polkadot_node_primitives::ErasureChunk; use polkadot_node_subsystem::{ @@ -44,14 +42,14 @@ use polkadot_node_subsystem::{ AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage, ChainApiMessage, NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, LeafStatus, OverseerSignal, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{ CandidateHash, CoreState, ExecutorParams, GroupIndex, Hash, Id as ParaId, ScheduledCore, SessionInfo, ValidatorIndex, }; -use test_helpers::mock::make_ferdie_keystore; +use test_helpers::mock::{make_ferdie_keystore, new_leaf}; use super::mock::{make_session_info, OccupiedCoreBuilder}; use crate::LOG_TARGET; @@ -175,12 +173,7 @@ impl TestState { .iter() .zip(advanced) .map(|(old, new)| ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: *new, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + activated: Some(new_leaf(*new, 1)), deactivated: vec![*old].into(), }) .collect::>() diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index bf95fb1e9f4c79d3c1c2daa6acc761a86b61eacf..42c3abef547b93e4187ed92764a91b4415602f06 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -10,7 +10,8 @@ futures = "0.3.21" schnellru = "0.2.1" rand = "0.8.5" fatality = "0.0.6" -thiserror = "1.0.31" +thiserror = "1.0.48" +async-trait = "0.1.73" gum = { package = "tracing-gum", path = "../../gum" } polkadot-erasure-coding = { path = "../../../erasure-coding" } diff --git a/polkadot/node/network/availability-recovery/src/lib.rs b/polkadot/node/network/availability-recovery/src/lib.rs index 99f42f4bf9fe668132cb020dd3e74d4faca34c46..e2146981da926d2c7027c902ba5be4716382f4da 100644 --- a/polkadot/node/network/availability-recovery/src/lib.rs +++ b/polkadot/node/network/availability-recovery/src/lib.rs @@ -23,11 +23,10 @@ use std::{ iter::Iterator, num::NonZeroUsize, pin::Pin, - time::Duration, }; use futures::{ - channel::oneshot::{self, channel}, + channel::oneshot, future::{Future, FutureExt, RemoteHandle}, pin_mut, prelude::*, @@ -35,77 +34,55 @@ use futures::{ stream::{FuturesUnordered, StreamExt}, task::{Context, Poll}, }; -use rand::seq::SliceRandom; use schnellru::{ByLength, LruMap}; +use task::{FetchChunks, FetchChunksParams, FetchFull, FetchFullParams}; use fatality::Nested; use polkadot_erasure_coding::{ branch_hash, branches, obtain_chunks_v1, recovery_threshold, Error as ErasureEncodingError, }; -#[cfg(not(test))] -use polkadot_node_network_protocol::request_response::CHUNK_REQUEST_TIMEOUT; +use task::{RecoveryParams, RecoveryStrategy, RecoveryTask}; + use polkadot_node_network_protocol::{ - request_response::{ - self as req_res, outgoing::RequestError, v1 as request_v1, IncomingRequestReceiver, - OutgoingRequest, Recipient, Requests, - }, - IfDisconnected, UnifiedReputationChange as Rep, + request_response::{v1 as request_v1, IncomingRequestReceiver}, + UnifiedReputationChange as Rep, }; use polkadot_node_primitives::{AvailableData, ErasureChunk}; use polkadot_node_subsystem::{ errors::RecoveryError, jaeger, - messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage, NetworkBridgeTxMessage}, - overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, - SubsystemResult, + messages::{AvailabilityRecoveryMessage, AvailabilityStoreMessage}, + overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, + SubsystemContext, SubsystemError, SubsystemResult, }; use polkadot_node_subsystem_util::request_session_info; use polkadot_primitives::{ - AuthorityDiscoveryId, BlakeTwo256, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, - Hash, HashT, IndexedVec, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, + BlakeTwo256, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HashT, + SessionIndex, SessionInfo, ValidatorIndex, }; mod error; mod futures_undead; mod metrics; +mod task; use metrics::Metrics; -use futures_undead::FuturesUndead; -use sc_network::{OutboundFailure, RequestFailure}; - #[cfg(test)] mod tests; const LOG_TARGET: &str = "parachain::availability-recovery"; -// How many parallel recovery tasks should be running at once. -const N_PARALLEL: usize = 50; - // Size of the LRU cache where we keep recovered data. const LRU_SIZE: u32 = 16; const COST_INVALID_REQUEST: Rep = Rep::CostMajor("Peer sent unparsable request"); -/// Time after which we consider a request to have failed -/// -/// and we should try more peers. Note in theory the request times out at the network level, -/// measurements have shown, that in practice requests might actually take longer to fail in -/// certain occasions. (The very least, authority discovery is not part of the timeout.) -/// -/// For the time being this value is the same as the timeout on the networking layer, but as this -/// timeout is more soft than the networking one, it might make sense to pick different values as -/// well. -#[cfg(not(test))] -const TIMEOUT_START_NEW_REQUESTS: Duration = CHUNK_REQUEST_TIMEOUT; -#[cfg(test)] -const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); - /// PoV size limit in bytes for which prefer fetching from backers. const SMALL_POV_LIMIT: usize = 128 * 1024; #[derive(Clone, PartialEq)] /// The strategy we use to recover the PoV. -pub enum RecoveryStrategy { +pub enum RecoveryStrategyKind { /// We always try the backing group first, then fallback to validator chunks. BackersFirstAlways, /// We try the backing group first if PoV size is lower than specified, then fallback to @@ -113,101 +90,25 @@ pub enum RecoveryStrategy { BackersFirstIfSizeLower(usize), /// We always recover using validator chunks. ChunksAlways, - /// Do not request data from the availability store. - /// This is the useful for nodes where the - /// availability-store subsystem is not expected to run, - /// such as collators. - BypassAvailabilityStore, } -impl RecoveryStrategy { - /// Returns true if the strategy needs backing group index. - pub fn needs_backing_group(&self) -> bool { - match self { - RecoveryStrategy::BackersFirstAlways | RecoveryStrategy::BackersFirstIfSizeLower(_) => - true, - _ => false, - } - } - - /// Returns the PoV size limit in bytes for `BackersFirstIfSizeLower` strategy, otherwise - /// `None`. - pub fn pov_size_limit(&self) -> Option { - match *self { - RecoveryStrategy::BackersFirstIfSizeLower(limit) => Some(limit), - _ => None, - } - } -} /// The Availability Recovery Subsystem. pub struct AvailabilityRecoverySubsystem { /// PoV recovery strategy to use. - recovery_strategy: RecoveryStrategy, + recovery_strategy_kind: RecoveryStrategyKind, + // If this is true, do not request data from the availability store. + /// This is the useful for nodes where the + /// availability-store subsystem is not expected to run, + /// such as collators. + bypass_availability_store: bool, /// Receiver for available data requests. req_receiver: IncomingRequestReceiver, /// Metrics for this subsystem. metrics: Metrics, } -struct RequestFromBackers { - // a random shuffling of the validators from the backing group which indicates the order - // in which we connect to them and request the chunk. - shuffled_backers: Vec, - // channel to the erasure task handler. - erasure_task_tx: futures::channel::mpsc::Sender, -} - -struct RequestChunksFromValidators { - /// How many request have been unsuccessful so far. - error_count: usize, - /// Total number of responses that have been received. - /// - /// including failed ones. - total_received_responses: usize, - /// a random shuffling of the validators which indicates the order in which we connect to the - /// validators and request the chunk from them. - shuffling: VecDeque, - /// Chunks received so far. - received_chunks: HashMap, - /// Pending chunk requests with soft timeout. - requesting_chunks: FuturesUndead, (ValidatorIndex, RequestError)>>, - // channel to the erasure task handler. - erasure_task_tx: futures::channel::mpsc::Sender, -} - -struct RecoveryParams { - /// Discovery ids of `validators`. - validator_authority_keys: Vec, - - /// Validators relevant to this `RecoveryTask`. - validators: IndexedVec, - - /// The number of pieces needed. - threshold: usize, - - /// A hash of the relevant candidate. - candidate_hash: CandidateHash, - - /// The root of the erasure encoding of the para block. - erasure_root: Hash, - - /// Metrics to report - metrics: Metrics, - - /// Do not request data from availability-store - bypass_availability_store: bool, -} - -/// Source the availability data either by means -/// of direct request response protocol to -/// backers (a.k.a. fast-path), or recover from chunks. -enum Source { - RequestFromBackers(RequestFromBackers), - RequestChunks(RequestChunksFromValidators), -} - /// Expensive erasure coding computations that we want to run on a blocking thread. -enum ErasureTask { +pub enum ErasureTask { /// Reconstructs `AvailableData` from chunks given `n_validators`. Reconstruct( usize, @@ -219,486 +120,6 @@ enum ErasureTask { Reencode(usize, Hash, AvailableData, oneshot::Sender>), } -/// A stateful reconstruction of availability data in reference to -/// a candidate hash. -struct RecoveryTask { - sender: Sender, - - /// The parameters of the recovery process. - params: RecoveryParams, - - /// The source to obtain the availability data from. - source: Source, - - // channel to the erasure task handler. - erasure_task_tx: futures::channel::mpsc::Sender, -} - -impl RequestFromBackers { - fn new( - mut backers: Vec, - erasure_task_tx: futures::channel::mpsc::Sender, - ) -> Self { - backers.shuffle(&mut rand::thread_rng()); - - RequestFromBackers { shuffled_backers: backers, erasure_task_tx } - } - - // Run this phase to completion. - async fn run( - &mut self, - params: &RecoveryParams, - sender: &mut impl overseer::AvailabilityRecoverySenderTrait, - ) -> Result { - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - erasure_root = ?params.erasure_root, - "Requesting from backers", - ); - loop { - // Pop the next backer, and proceed to next phase if we're out. - let validator_index = - self.shuffled_backers.pop().ok_or_else(|| RecoveryError::Unavailable)?; - - // Request data. - let (req, response) = OutgoingRequest::new( - Recipient::Authority( - params.validator_authority_keys[validator_index.0 as usize].clone(), - ), - req_res::v1::AvailableDataFetchingRequest { candidate_hash: params.candidate_hash }, - ); - - sender - .send_message(NetworkBridgeTxMessage::SendRequests( - vec![Requests::AvailableDataFetchingV1(req)], - IfDisconnected::ImmediateError, - )) - .await; - - match response.await { - Ok(req_res::v1::AvailableDataFetchingResponse::AvailableData(data)) => { - let (reencode_tx, reencode_rx) = channel(); - self.erasure_task_tx - .send(ErasureTask::Reencode( - params.validators.len(), - params.erasure_root, - data, - reencode_tx, - )) - .await - .map_err(|_| RecoveryError::ChannelClosed)?; - - let reencode_response = - reencode_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; - - if let Some(data) = reencode_response { - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - "Received full data", - ); - - return Ok(data) - } else { - gum::debug!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - ?validator_index, - "Invalid data response", - ); - - // it doesn't help to report the peer with req/res. - } - }, - Ok(req_res::v1::AvailableDataFetchingResponse::NoSuchData) => {}, - Err(e) => gum::debug!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - ?validator_index, - err = ?e, - "Error fetching full available data." - ), - } - } - } -} - -impl RequestChunksFromValidators { - fn new( - n_validators: u32, - erasure_task_tx: futures::channel::mpsc::Sender, - ) -> Self { - let mut shuffling: Vec<_> = (0..n_validators).map(ValidatorIndex).collect(); - shuffling.shuffle(&mut rand::thread_rng()); - - RequestChunksFromValidators { - error_count: 0, - total_received_responses: 0, - shuffling: shuffling.into(), - received_chunks: HashMap::new(), - requesting_chunks: FuturesUndead::new(), - erasure_task_tx, - } - } - - fn is_unavailable(&self, params: &RecoveryParams) -> bool { - is_unavailable( - self.chunk_count(), - self.requesting_chunks.total_len(), - self.shuffling.len(), - params.threshold, - ) - } - - fn chunk_count(&self) -> usize { - self.received_chunks.len() - } - - fn insert_chunk(&mut self, validator_index: ValidatorIndex, chunk: ErasureChunk) { - self.received_chunks.insert(validator_index, chunk); - } - - fn can_conclude(&self, params: &RecoveryParams) -> bool { - self.chunk_count() >= params.threshold || self.is_unavailable(params) - } - - /// Desired number of parallel requests. - /// - /// For the given threshold (total required number of chunks) get the desired number of - /// requests we want to have running in parallel at this time. - fn get_desired_request_count(&self, threshold: usize) -> usize { - // Upper bound for parallel requests. - // We want to limit this, so requests can be processed within the timeout and we limit the - // following feedback loop: - // 1. Requests fail due to timeout - // 2. We request more chunks to make up for it - // 3. Bandwidth is spread out even more, so we get even more timeouts - // 4. We request more chunks to make up for it ... - let max_requests_boundary = std::cmp::min(N_PARALLEL, threshold); - // How many chunks are still needed? - let remaining_chunks = threshold.saturating_sub(self.chunk_count()); - // What is the current error rate, so we can make up for it? - let inv_error_rate = - self.total_received_responses.checked_div(self.error_count).unwrap_or(0); - // Actual number of requests we want to have in flight in parallel: - std::cmp::min( - max_requests_boundary, - remaining_chunks + remaining_chunks.checked_div(inv_error_rate).unwrap_or(0), - ) - } - - async fn launch_parallel_requests( - &mut self, - params: &RecoveryParams, - sender: &mut Sender, - ) where - Sender: overseer::AvailabilityRecoverySenderTrait, - { - let num_requests = self.get_desired_request_count(params.threshold); - let candidate_hash = ¶ms.candidate_hash; - let already_requesting_count = self.requesting_chunks.len(); - - gum::debug!( - target: LOG_TARGET, - ?candidate_hash, - ?num_requests, - error_count= ?self.error_count, - total_received = ?self.total_received_responses, - threshold = ?params.threshold, - ?already_requesting_count, - "Requesting availability chunks for a candidate", - ); - let mut requests = Vec::with_capacity(num_requests - already_requesting_count); - - while self.requesting_chunks.len() < num_requests { - if let Some(validator_index) = self.shuffling.pop_back() { - let validator = params.validator_authority_keys[validator_index.0 as usize].clone(); - gum::trace!( - target: LOG_TARGET, - ?validator, - ?validator_index, - ?candidate_hash, - "Requesting chunk", - ); - - // Request data. - let raw_request = req_res::v1::ChunkFetchingRequest { - candidate_hash: params.candidate_hash, - index: validator_index, - }; - - let (req, res) = OutgoingRequest::new(Recipient::Authority(validator), raw_request); - requests.push(Requests::ChunkFetchingV1(req)); - - params.metrics.on_chunk_request_issued(); - let timer = params.metrics.time_chunk_request(); - - self.requesting_chunks.push(Box::pin(async move { - let _timer = timer; - match res.await { - Ok(req_res::v1::ChunkFetchingResponse::Chunk(chunk)) => - Ok(Some(chunk.recombine_into_chunk(&raw_request))), - Ok(req_res::v1::ChunkFetchingResponse::NoSuchChunk) => Ok(None), - Err(e) => Err((validator_index, e)), - } - })); - } else { - break - } - } - - sender - .send_message(NetworkBridgeTxMessage::SendRequests( - requests, - IfDisconnected::TryConnect, - )) - .await; - } - - /// Wait for a sufficient amount of chunks to reconstruct according to the provided `params`. - async fn wait_for_chunks(&mut self, params: &RecoveryParams) { - let metrics = ¶ms.metrics; - - // Wait for all current requests to conclude or time-out, or until we reach enough chunks. - // We also declare requests undead, once `TIMEOUT_START_NEW_REQUESTS` is reached and will - // return in that case for `launch_parallel_requests` to fill up slots again. - while let Some(request_result) = - self.requesting_chunks.next_with_timeout(TIMEOUT_START_NEW_REQUESTS).await - { - self.total_received_responses += 1; - - match request_result { - Ok(Some(chunk)) => - if is_chunk_valid(params, &chunk) { - metrics.on_chunk_request_succeeded(); - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - validator_index = ?chunk.index, - "Received valid chunk", - ); - self.insert_chunk(chunk.index, chunk); - } else { - metrics.on_chunk_request_invalid(); - self.error_count += 1; - }, - Ok(None) => { - metrics.on_chunk_request_no_such_chunk(); - self.error_count += 1; - }, - Err((validator_index, e)) => { - self.error_count += 1; - - gum::trace!( - target: LOG_TARGET, - candidate_hash= ?params.candidate_hash, - err = ?e, - ?validator_index, - "Failure requesting chunk", - ); - - match e { - RequestError::InvalidResponse(_) => { - metrics.on_chunk_request_invalid(); - - gum::debug!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - err = ?e, - ?validator_index, - "Chunk fetching response was invalid", - ); - }, - RequestError::NetworkError(err) => { - // No debug logs on general network errors - that became very spammy - // occasionally. - if let RequestFailure::Network(OutboundFailure::Timeout) = err { - metrics.on_chunk_request_timeout(); - } else { - metrics.on_chunk_request_error(); - } - - self.shuffling.push_front(validator_index); - }, - RequestError::Canceled(_) => { - metrics.on_chunk_request_error(); - - self.shuffling.push_front(validator_index); - }, - } - }, - } - - // Stop waiting for requests when we either can already recover the data - // or have gotten firm 'No' responses from enough validators. - if self.can_conclude(params) { - gum::debug!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - received_chunks_count = ?self.chunk_count(), - requested_chunks_count = ?self.requesting_chunks.len(), - threshold = ?params.threshold, - "Can conclude availability for a candidate", - ); - break - } - } - } - - async fn run( - &mut self, - params: &RecoveryParams, - sender: &mut Sender, - ) -> Result - where - Sender: overseer::AvailabilityRecoverySenderTrait, - { - let metrics = ¶ms.metrics; - - // First query the store for any chunks we've got. - if !params.bypass_availability_store { - let (tx, rx) = oneshot::channel(); - sender - .send_message(AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx)) - .await; - - match rx.await { - Ok(chunks) => { - // This should either be length 1 or 0. If we had the whole data, - // we wouldn't have reached this stage. - let chunk_indices: Vec<_> = chunks.iter().map(|c| c.index).collect(); - self.shuffling.retain(|i| !chunk_indices.contains(i)); - - for chunk in chunks { - if is_chunk_valid(params, &chunk) { - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - validator_index = ?chunk.index, - "Found valid chunk on disk" - ); - self.insert_chunk(chunk.index, chunk); - } else { - gum::error!( - target: LOG_TARGET, - "Loaded invalid chunk from disk! Disk/Db corruption _very_ likely - please fix ASAP!" - ); - }; - } - }, - Err(oneshot::Canceled) => { - gum::warn!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - "Failed to reach the availability store" - ); - }, - } - } - - let _recovery_timer = metrics.time_full_recovery(); - - loop { - if self.is_unavailable(¶ms) { - gum::debug!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - erasure_root = ?params.erasure_root, - received = %self.chunk_count(), - requesting = %self.requesting_chunks.len(), - total_requesting = %self.requesting_chunks.total_len(), - n_validators = %params.validators.len(), - "Data recovery is not possible", - ); - - metrics.on_recovery_failed(); - - return Err(RecoveryError::Unavailable) - } - - self.launch_parallel_requests(params, sender).await; - self.wait_for_chunks(params).await; - - // If received_chunks has more than threshold entries, attempt to recover the data. - // If that fails, or a re-encoding of it doesn't match the expected erasure root, - // return Err(RecoveryError::Invalid) - if self.chunk_count() >= params.threshold { - let recovery_duration = metrics.time_erasure_recovery(); - - // Send request to reconstruct available data from chunks. - let (avilable_data_tx, available_data_rx) = channel(); - self.erasure_task_tx - .send(ErasureTask::Reconstruct( - params.validators.len(), - std::mem::take(&mut self.received_chunks), - avilable_data_tx, - )) - .await - .map_err(|_| RecoveryError::ChannelClosed)?; - - let available_data_response = - available_data_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; - - return match available_data_response { - Ok(data) => { - // Send request to re-encode the chunks and check merkle root. - let (reencode_tx, reencode_rx) = channel(); - self.erasure_task_tx - .send(ErasureTask::Reencode( - params.validators.len(), - params.erasure_root, - data, - reencode_tx, - )) - .await - .map_err(|_| RecoveryError::ChannelClosed)?; - - let reencode_response = - reencode_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; - - if let Some(data) = reencode_response { - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - erasure_root = ?params.erasure_root, - "Data recovery complete", - ); - metrics.on_recovery_succeeded(); - - Ok(data) - } else { - recovery_duration.map(|rd| rd.stop_and_discard()); - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - erasure_root = ?params.erasure_root, - "Data recovery - root mismatch", - ); - metrics.on_recovery_invalid(); - - Err(RecoveryError::Invalid) - } - }, - Err(err) => { - recovery_duration.map(|rd| rd.stop_and_discard()); - gum::trace!( - target: LOG_TARGET, - candidate_hash = ?params.candidate_hash, - erasure_root = ?params.erasure_root, - ?err, - "Data recovery error ", - ); - metrics.on_recovery_invalid(); - - Err(RecoveryError::Invalid) - }, - } - } - } - } -} - const fn is_unavailable( received_chunks: usize, requesting_chunks: usize, @@ -777,60 +198,6 @@ fn reconstructed_data_matches_root( branches.root() == *expected_root } -impl RecoveryTask -where - Sender: overseer::AvailabilityRecoverySenderTrait, -{ - async fn run(mut self) -> Result { - // First just see if we have the data available locally. - if !self.params.bypass_availability_store { - let (tx, rx) = oneshot::channel(); - self.sender - .send_message(AvailabilityStoreMessage::QueryAvailableData( - self.params.candidate_hash, - tx, - )) - .await; - - match rx.await { - Ok(Some(data)) => return Ok(data), - Ok(None) => {}, - Err(oneshot::Canceled) => { - gum::warn!( - target: LOG_TARGET, - candidate_hash = ?self.params.candidate_hash, - "Failed to reach the availability store", - ) - }, - } - } - - self.params.metrics.on_recovery_started(); - - loop { - // These only fail if we cannot reach the underlying subsystem, which case there is - // nothing meaningful we can do. - match self.source { - Source::RequestFromBackers(ref mut from_backers) => { - match from_backers.run(&self.params, &mut self.sender).await { - Ok(data) => break Ok(data), - Err(RecoveryError::Invalid) => break Err(RecoveryError::Invalid), - Err(RecoveryError::ChannelClosed) => - break Err(RecoveryError::ChannelClosed), - Err(RecoveryError::Unavailable) => - self.source = Source::RequestChunks(RequestChunksFromValidators::new( - self.params.validators.len() as _, - self.erasure_task_tx.clone(), - )), - } - }, - Source::RequestChunks(ref mut from_all) => - break from_all.run(&self.params, &mut self.sender).await, - } - } - } -} - /// Accumulate all awaiting sides for some particular `AvailableData`. struct RecoveryHandle { candidate_hash: CandidateHash, @@ -973,65 +340,23 @@ async fn launch_recovery_task( ctx: &mut Context, session_info: SessionInfo, receipt: CandidateReceipt, - mut backing_group: Option, response_sender: oneshot::Sender>, metrics: &Metrics, - recovery_strategy: &RecoveryStrategy, - erasure_task_tx: futures::channel::mpsc::Sender, + recovery_strategies: VecDeque::Sender>>>, + bypass_availability_store: bool, ) -> error::Result<()> { let candidate_hash = receipt.hash(); let params = RecoveryParams { validator_authority_keys: session_info.discovery_keys.clone(), - validators: session_info.validators.clone(), + n_validators: session_info.validators.len(), threshold: recovery_threshold(session_info.validators.len())?, candidate_hash, erasure_root: receipt.descriptor.erasure_root, metrics: metrics.clone(), - bypass_availability_store: recovery_strategy == &RecoveryStrategy::BypassAvailabilityStore, + bypass_availability_store, }; - if let Some(small_pov_limit) = recovery_strategy.pov_size_limit() { - // Get our own chunk size to get an estimate of the PoV size. - let chunk_size: Result, error::Error> = - query_chunk_size(ctx, candidate_hash).await; - if let Ok(Some(chunk_size)) = chunk_size { - let pov_size_estimate = chunk_size.saturating_mul(session_info.validators.len()) / 3; - let prefer_backing_group = pov_size_estimate < small_pov_limit; - - gum::trace!( - target: LOG_TARGET, - ?candidate_hash, - pov_size_estimate, - small_pov_limit, - enabled = prefer_backing_group, - "Prefer fetch from backing group", - ); - - backing_group = backing_group.filter(|_| { - // We keep the backing group only if `1/3` of chunks sum up to less than - // `small_pov_limit`. - prefer_backing_group - }); - } - } - - let phase = backing_group - .and_then(|g| session_info.validator_groups.get(g)) - .map(|group| { - Source::RequestFromBackers(RequestFromBackers::new( - group.clone(), - erasure_task_tx.clone(), - )) - }) - .unwrap_or_else(|| { - Source::RequestChunks(RequestChunksFromValidators::new( - params.validators.len() as _, - erasure_task_tx.clone(), - )) - }); - - let recovery_task = - RecoveryTask { sender: ctx.sender().clone(), params, source: phase, erasure_task_tx }; + let recovery_task = RecoveryTask::new(ctx.sender().clone(), params, recovery_strategies); let (remote, remote_handle) = recovery_task.run().remote_handle(); @@ -1062,8 +387,9 @@ async fn handle_recover( backing_group: Option, response_sender: oneshot::Sender>, metrics: &Metrics, - recovery_strategy: &RecoveryStrategy, erasure_task_tx: futures::channel::mpsc::Sender, + recovery_strategy_kind: RecoveryStrategyKind, + bypass_availability_store: bool, ) -> error::Result<()> { let candidate_hash = receipt.hash(); @@ -1098,19 +424,71 @@ async fn handle_recover( let _span = span.child("session-info-ctx-received"); match session_info { - Some(session_info) => + Some(session_info) => { + let mut recovery_strategies: VecDeque< + Box::Sender>>, + > = VecDeque::with_capacity(2); + + if let Some(backing_group) = backing_group { + if let Some(backing_validators) = session_info.validator_groups.get(backing_group) { + let mut small_pov_size = true; + + if let RecoveryStrategyKind::BackersFirstIfSizeLower(small_pov_limit) = + recovery_strategy_kind + { + // Get our own chunk size to get an estimate of the PoV size. + let chunk_size: Result, error::Error> = + query_chunk_size(ctx, candidate_hash).await; + if let Ok(Some(chunk_size)) = chunk_size { + let pov_size_estimate = + chunk_size.saturating_mul(session_info.validators.len()) / 3; + small_pov_size = pov_size_estimate < small_pov_limit; + + gum::trace!( + target: LOG_TARGET, + ?candidate_hash, + pov_size_estimate, + small_pov_limit, + enabled = small_pov_size, + "Prefer fetch from backing group", + ); + } else { + // we have a POV limit but were not able to query the chunk size, so + // don't use the backing group. + small_pov_size = false; + } + }; + + match (&recovery_strategy_kind, small_pov_size) { + (RecoveryStrategyKind::BackersFirstAlways, _) | + (RecoveryStrategyKind::BackersFirstIfSizeLower(_), true) => recovery_strategies.push_back( + Box::new(FetchFull::new(FetchFullParams { + validators: backing_validators.to_vec(), + erasure_task_tx: erasure_task_tx.clone(), + })), + ), + _ => {}, + }; + } + } + + recovery_strategies.push_back(Box::new(FetchChunks::new(FetchChunksParams { + n_validators: session_info.validators.len(), + erasure_task_tx, + }))); + launch_recovery_task( state, ctx, session_info, receipt, - backing_group, response_sender, metrics, - recovery_strategy, - erasure_task_tx, + recovery_strategies, + bypass_availability_store, ) - .await, + .await + }, None => { gum::warn!(target: LOG_TARGET, "SessionInfo is `None` at {:?}", state.live_block); response_sender @@ -1155,7 +533,12 @@ impl AvailabilityRecoverySubsystem { req_receiver: IncomingRequestReceiver, metrics: Metrics, ) -> Self { - Self { recovery_strategy: RecoveryStrategy::BypassAvailabilityStore, req_receiver, metrics } + Self { + recovery_strategy_kind: RecoveryStrategyKind::BackersFirstIfSizeLower(SMALL_POV_LIMIT), + bypass_availability_store: true, + req_receiver, + metrics, + } } /// Create a new instance of `AvailabilityRecoverySubsystem` which starts with a fast path to @@ -1164,7 +547,12 @@ impl AvailabilityRecoverySubsystem { req_receiver: IncomingRequestReceiver, metrics: Metrics, ) -> Self { - Self { recovery_strategy: RecoveryStrategy::BackersFirstAlways, req_receiver, metrics } + Self { + recovery_strategy_kind: RecoveryStrategyKind::BackersFirstAlways, + bypass_availability_store: false, + req_receiver, + metrics, + } } /// Create a new instance of `AvailabilityRecoverySubsystem` which requests only chunks @@ -1172,7 +560,12 @@ impl AvailabilityRecoverySubsystem { req_receiver: IncomingRequestReceiver, metrics: Metrics, ) -> Self { - Self { recovery_strategy: RecoveryStrategy::ChunksAlways, req_receiver, metrics } + Self { + recovery_strategy_kind: RecoveryStrategyKind::ChunksAlways, + bypass_availability_store: false, + req_receiver, + metrics, + } } /// Create a new instance of `AvailabilityRecoverySubsystem` which requests chunks if PoV is @@ -1182,7 +575,8 @@ impl AvailabilityRecoverySubsystem { metrics: Metrics, ) -> Self { Self { - recovery_strategy: RecoveryStrategy::BackersFirstIfSizeLower(SMALL_POV_LIMIT), + recovery_strategy_kind: RecoveryStrategyKind::BackersFirstIfSizeLower(SMALL_POV_LIMIT), + bypass_availability_store: false, req_receiver, metrics, } @@ -1190,7 +584,8 @@ impl AvailabilityRecoverySubsystem { async fn run(self, mut ctx: Context) -> SubsystemResult<()> { let mut state = State::default(); - let Self { recovery_strategy, mut req_receiver, metrics } = self; + let Self { mut req_receiver, metrics, recovery_strategy_kind, bypass_availability_store } = + self; let (erasure_task_tx, erasure_task_rx) = futures::channel::mpsc::channel(16); let mut erasure_task_rx = erasure_task_rx.fuse(); @@ -1275,11 +670,12 @@ impl AvailabilityRecoverySubsystem { &mut ctx, receipt, session_index, - maybe_backing_group.filter(|_| recovery_strategy.needs_backing_group()), + maybe_backing_group, response_sender, &metrics, - &recovery_strategy, erasure_task_tx.clone(), + recovery_strategy_kind.clone(), + bypass_availability_store ).await { gum::warn!( target: LOG_TARGET, @@ -1295,7 +691,7 @@ impl AvailabilityRecoverySubsystem { in_req = recv_req => { match in_req.into_nested().map_err(|fatal| SubsystemError::with_origin("availability-recovery", fatal))? { Ok(req) => { - if recovery_strategy == RecoveryStrategy::BypassAvailabilityStore { + if bypass_availability_store { gum::debug!( target: LOG_TARGET, "Skipping request to availability-store.", diff --git a/polkadot/node/network/availability-recovery/src/task.rs b/polkadot/node/network/availability-recovery/src/task.rs new file mode 100644 index 0000000000000000000000000000000000000000..d5bc2da84944a3403f9d4a3bf9d5a11b0e772f40 --- /dev/null +++ b/polkadot/node/network/availability-recovery/src/task.rs @@ -0,0 +1,830 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Recovery task and associated strategies. + +#![warn(missing_docs)] + +use crate::{ + futures_undead::FuturesUndead, is_chunk_valid, is_unavailable, metrics::Metrics, ErasureTask, + LOG_TARGET, +}; +use futures::{channel::oneshot, SinkExt}; +#[cfg(not(test))] +use polkadot_node_network_protocol::request_response::CHUNK_REQUEST_TIMEOUT; +use polkadot_node_network_protocol::request_response::{ + self as req_res, outgoing::RequestError, OutgoingRequest, Recipient, Requests, +}; +use polkadot_node_primitives::{AvailableData, ErasureChunk}; +use polkadot_node_subsystem::{ + messages::{AvailabilityStoreMessage, NetworkBridgeTxMessage}, + overseer, RecoveryError, +}; +use polkadot_primitives::{AuthorityDiscoveryId, CandidateHash, Hash, ValidatorIndex}; +use rand::seq::SliceRandom; +use sc_network::{IfDisconnected, OutboundFailure, RequestFailure}; +use std::{ + collections::{HashMap, VecDeque}, + time::Duration, +}; + +// How many parallel recovery tasks should be running at once. +const N_PARALLEL: usize = 50; + +/// Time after which we consider a request to have failed +/// +/// and we should try more peers. Note in theory the request times out at the network level, +/// measurements have shown, that in practice requests might actually take longer to fail in +/// certain occasions. (The very least, authority discovery is not part of the timeout.) +/// +/// For the time being this value is the same as the timeout on the networking layer, but as this +/// timeout is more soft than the networking one, it might make sense to pick different values as +/// well. +#[cfg(not(test))] +const TIMEOUT_START_NEW_REQUESTS: Duration = CHUNK_REQUEST_TIMEOUT; +#[cfg(test)] +const TIMEOUT_START_NEW_REQUESTS: Duration = Duration::from_millis(100); + +#[async_trait::async_trait] +/// Common trait for runnable recovery strategies. +pub trait RecoveryStrategy: Send { + /// Main entry point of the strategy. + async fn run( + &mut self, + state: &mut State, + sender: &mut Sender, + common_params: &RecoveryParams, + ) -> Result; + + /// Return the name of the strategy for logging purposes. + fn display_name(&self) -> &'static str; +} + +/// Recovery parameters common to all strategies in a `RecoveryTask`. +pub struct RecoveryParams { + /// Discovery ids of `validators`. + pub validator_authority_keys: Vec, + + /// Number of validators. + pub n_validators: usize, + + /// The number of chunks needed. + pub threshold: usize, + + /// A hash of the relevant candidate. + pub candidate_hash: CandidateHash, + + /// The root of the erasure encoding of the candidate. + pub erasure_root: Hash, + + /// Metrics to report. + pub metrics: Metrics, + + /// Do not request data from availability-store. Useful for collators. + pub bypass_availability_store: bool, +} + +/// Intermediate/common data that must be passed between `RecoveryStrategy`s belonging to the +/// same `RecoveryTask`. +pub struct State { + /// Chunks received so far. + received_chunks: HashMap, +} + +impl State { + fn new() -> Self { + Self { received_chunks: HashMap::new() } + } + + fn insert_chunk(&mut self, validator: ValidatorIndex, chunk: ErasureChunk) { + self.received_chunks.insert(validator, chunk); + } + + fn chunk_count(&self) -> usize { + self.received_chunks.len() + } + + /// Retrieve the local chunks held in the av-store (either 0 or 1). + async fn populate_from_av_store( + &mut self, + params: &RecoveryParams, + sender: &mut Sender, + ) -> Vec { + let (tx, rx) = oneshot::channel(); + sender + .send_message(AvailabilityStoreMessage::QueryAllChunks(params.candidate_hash, tx)) + .await; + + match rx.await { + Ok(chunks) => { + // This should either be length 1 or 0. If we had the whole data, + // we wouldn't have reached this stage. + let chunk_indices: Vec<_> = chunks.iter().map(|c| c.index).collect(); + + for chunk in chunks { + if is_chunk_valid(params, &chunk) { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?params.candidate_hash, + validator_index = ?chunk.index, + "Found valid chunk on disk" + ); + self.insert_chunk(chunk.index, chunk); + } else { + gum::error!( + target: LOG_TARGET, + "Loaded invalid chunk from disk! Disk/Db corruption _very_ likely - please fix ASAP!" + ); + }; + } + + chunk_indices + }, + Err(oneshot::Canceled) => { + gum::warn!( + target: LOG_TARGET, + candidate_hash = ?params.candidate_hash, + "Failed to reach the availability store" + ); + + vec![] + }, + } + } + + /// Launch chunk requests in parallel, according to the parameters. + async fn launch_parallel_chunk_requests( + &mut self, + params: &RecoveryParams, + sender: &mut Sender, + desired_requests_count: usize, + validators: &mut VecDeque, + requesting_chunks: &mut FuturesUndead< + Result, (ValidatorIndex, RequestError)>, + >, + ) where + Sender: overseer::AvailabilityRecoverySenderTrait, + { + let candidate_hash = ¶ms.candidate_hash; + let already_requesting_count = requesting_chunks.len(); + + let mut requests = Vec::with_capacity(desired_requests_count - already_requesting_count); + + while requesting_chunks.len() < desired_requests_count { + if let Some(validator_index) = validators.pop_back() { + let validator = params.validator_authority_keys[validator_index.0 as usize].clone(); + gum::trace!( + target: LOG_TARGET, + ?validator, + ?validator_index, + ?candidate_hash, + "Requesting chunk", + ); + + // Request data. + let raw_request = req_res::v1::ChunkFetchingRequest { + candidate_hash: params.candidate_hash, + index: validator_index, + }; + + let (req, res) = OutgoingRequest::new(Recipient::Authority(validator), raw_request); + requests.push(Requests::ChunkFetchingV1(req)); + + params.metrics.on_chunk_request_issued(); + let timer = params.metrics.time_chunk_request(); + + requesting_chunks.push(Box::pin(async move { + let _timer = timer; + match res.await { + Ok(req_res::v1::ChunkFetchingResponse::Chunk(chunk)) => + Ok(Some(chunk.recombine_into_chunk(&raw_request))), + Ok(req_res::v1::ChunkFetchingResponse::NoSuchChunk) => Ok(None), + Err(e) => Err((validator_index, e)), + } + })); + } else { + break + } + } + + sender + .send_message(NetworkBridgeTxMessage::SendRequests( + requests, + IfDisconnected::TryConnect, + )) + .await; + } + + /// Wait for a sufficient amount of chunks to reconstruct according to the provided `params`. + async fn wait_for_chunks( + &mut self, + params: &RecoveryParams, + validators: &mut VecDeque, + requesting_chunks: &mut FuturesUndead< + Result, (ValidatorIndex, RequestError)>, + >, + can_conclude: impl Fn(usize, usize, usize, &RecoveryParams, usize) -> bool, + ) -> (usize, usize) { + let metrics = ¶ms.metrics; + + let mut total_received_responses = 0; + let mut error_count = 0; + + // Wait for all current requests to conclude or time-out, or until we reach enough chunks. + // We also declare requests undead, once `TIMEOUT_START_NEW_REQUESTS` is reached and will + // return in that case for `launch_parallel_requests` to fill up slots again. + while let Some(request_result) = + requesting_chunks.next_with_timeout(TIMEOUT_START_NEW_REQUESTS).await + { + total_received_responses += 1; + + match request_result { + Ok(Some(chunk)) => + if is_chunk_valid(params, &chunk) { + metrics.on_chunk_request_succeeded(); + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?params.candidate_hash, + validator_index = ?chunk.index, + "Received valid chunk", + ); + self.insert_chunk(chunk.index, chunk); + } else { + metrics.on_chunk_request_invalid(); + error_count += 1; + }, + Ok(None) => { + metrics.on_chunk_request_no_such_chunk(); + error_count += 1; + }, + Err((validator_index, e)) => { + error_count += 1; + + gum::trace!( + target: LOG_TARGET, + candidate_hash= ?params.candidate_hash, + err = ?e, + ?validator_index, + "Failure requesting chunk", + ); + + match e { + RequestError::InvalidResponse(_) => { + metrics.on_chunk_request_invalid(); + + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?params.candidate_hash, + err = ?e, + ?validator_index, + "Chunk fetching response was invalid", + ); + }, + RequestError::NetworkError(err) => { + // No debug logs on general network errors - that became very spammy + // occasionally. + if let RequestFailure::Network(OutboundFailure::Timeout) = err { + metrics.on_chunk_request_timeout(); + } else { + metrics.on_chunk_request_error(); + } + + validators.push_front(validator_index); + }, + RequestError::Canceled(_) => { + metrics.on_chunk_request_error(); + + validators.push_front(validator_index); + }, + } + }, + } + + // Stop waiting for requests when we either can already recover the data + // or have gotten firm 'No' responses from enough validators. + if can_conclude( + validators.len(), + requesting_chunks.total_len(), + self.chunk_count(), + params, + error_count, + ) { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?params.candidate_hash, + received_chunks_count = ?self.chunk_count(), + requested_chunks_count = ?requesting_chunks.len(), + threshold = ?params.threshold, + "Can conclude availability for a candidate", + ); + break + } + } + + (total_received_responses, error_count) + } +} + +/// A stateful reconstruction of availability data in reference to +/// a candidate hash. +pub struct RecoveryTask { + sender: Sender, + params: RecoveryParams, + strategies: VecDeque>>, + state: State, +} + +impl RecoveryTask +where + Sender: overseer::AvailabilityRecoverySenderTrait, +{ + /// Instantiate a new recovery task. + pub fn new( + sender: Sender, + params: RecoveryParams, + strategies: VecDeque>>, + ) -> Self { + Self { sender, params, strategies, state: State::new() } + } + + async fn in_availability_store(&mut self) -> Option { + if !self.params.bypass_availability_store { + let (tx, rx) = oneshot::channel(); + self.sender + .send_message(AvailabilityStoreMessage::QueryAvailableData( + self.params.candidate_hash, + tx, + )) + .await; + + match rx.await { + Ok(Some(data)) => return Some(data), + Ok(None) => {}, + Err(oneshot::Canceled) => { + gum::warn!( + target: LOG_TARGET, + candidate_hash = ?self.params.candidate_hash, + "Failed to reach the availability store", + ) + }, + } + } + + None + } + + /// Run this recovery task to completion. It will loop through the configured strategies + /// in-order and return whenever the first one recovers the full `AvailableData`. + pub async fn run(mut self) -> Result { + if let Some(data) = self.in_availability_store().await { + return Ok(data) + } + + self.params.metrics.on_recovery_started(); + + let _timer = self.params.metrics.time_full_recovery(); + + while let Some(mut current_strategy) = self.strategies.pop_front() { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?self.params.candidate_hash, + "Starting `{}` strategy", + current_strategy.display_name(), + ); + + let res = current_strategy.run(&mut self.state, &mut self.sender, &self.params).await; + + match res { + Err(RecoveryError::Unavailable) => + if self.strategies.front().is_some() { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?self.params.candidate_hash, + "Recovery strategy `{}` did not conclude. Trying the next one.", + current_strategy.display_name(), + ); + continue + }, + Err(err) => { + match &err { + RecoveryError::Invalid => self.params.metrics.on_recovery_invalid(), + _ => self.params.metrics.on_recovery_failed(), + } + return Err(err) + }, + Ok(data) => { + self.params.metrics.on_recovery_succeeded(); + return Ok(data) + }, + } + } + + // We have no other strategies to try. + gum::warn!( + target: LOG_TARGET, + candidate_hash = ?self.params.candidate_hash, + "Recovery of available data failed.", + ); + self.params.metrics.on_recovery_failed(); + + Err(RecoveryError::Unavailable) + } +} + +/// `RecoveryStrategy` that sequentially tries to fetch the full `AvailableData` from +/// already-connected validators in the configured validator set. +pub struct FetchFull { + params: FetchFullParams, +} + +pub struct FetchFullParams { + /// Validators that will be used for fetching the data. + pub validators: Vec, + /// Channel to the erasure task handler. + pub erasure_task_tx: futures::channel::mpsc::Sender, +} + +impl FetchFull { + /// Create a new `FetchFull` recovery strategy. + pub fn new(mut params: FetchFullParams) -> Self { + params.validators.shuffle(&mut rand::thread_rng()); + Self { params } + } +} + +#[async_trait::async_trait] +impl RecoveryStrategy for FetchFull { + fn display_name(&self) -> &'static str { + "Full recovery from backers" + } + + async fn run( + &mut self, + _: &mut State, + sender: &mut Sender, + common_params: &RecoveryParams, + ) -> Result { + loop { + // Pop the next validator, and proceed to next fetch_chunks_task if we're out. + let validator_index = + self.params.validators.pop().ok_or_else(|| RecoveryError::Unavailable)?; + + // Request data. + let (req, response) = OutgoingRequest::new( + Recipient::Authority( + common_params.validator_authority_keys[validator_index.0 as usize].clone(), + ), + req_res::v1::AvailableDataFetchingRequest { + candidate_hash: common_params.candidate_hash, + }, + ); + + sender + .send_message(NetworkBridgeTxMessage::SendRequests( + vec![Requests::AvailableDataFetchingV1(req)], + IfDisconnected::ImmediateError, + )) + .await; + + match response.await { + Ok(req_res::v1::AvailableDataFetchingResponse::AvailableData(data)) => { + let (reencode_tx, reencode_rx) = oneshot::channel(); + self.params + .erasure_task_tx + .send(ErasureTask::Reencode( + common_params.n_validators, + common_params.erasure_root, + data, + reencode_tx, + )) + .await + .map_err(|_| RecoveryError::ChannelClosed)?; + + let reencode_response = + reencode_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; + + if let Some(data) = reencode_response { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + "Received full data", + ); + + return Ok(data) + } else { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + ?validator_index, + "Invalid data response", + ); + + // it doesn't help to report the peer with req/res. + } + }, + Ok(req_res::v1::AvailableDataFetchingResponse::NoSuchData) => {}, + Err(e) => gum::debug!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + ?validator_index, + err = ?e, + "Error fetching full available data." + ), + } + } + } +} + +/// `RecoveryStrategy` that requests chunks from validators, in parallel. +pub struct FetchChunks { + /// How many requests have been unsuccessful so far. + error_count: usize, + /// Total number of responses that have been received, including failed ones. + total_received_responses: usize, + /// Collection of in-flight requests. + requesting_chunks: FuturesUndead, (ValidatorIndex, RequestError)>>, + /// A random shuffling of the validators which indicates the order in which we connect to the + /// validators and request the chunk from them. + validators: VecDeque, + /// Channel to the erasure task handler. + erasure_task_tx: futures::channel::mpsc::Sender, +} + +/// Parameters specific to the `FetchChunks` strategy. +pub struct FetchChunksParams { + /// Total number of validators. + pub n_validators: usize, + /// Channel to the erasure task handler. + pub erasure_task_tx: futures::channel::mpsc::Sender, +} + +impl FetchChunks { + /// Instantiate a new strategy. + pub fn new(params: FetchChunksParams) -> Self { + let mut shuffling: Vec<_> = (0..params.n_validators) + .map(|i| ValidatorIndex(i.try_into().expect("number of validators must fit in a u32"))) + .collect(); + shuffling.shuffle(&mut rand::thread_rng()); + + Self { + error_count: 0, + total_received_responses: 0, + requesting_chunks: FuturesUndead::new(), + validators: shuffling.into(), + erasure_task_tx: params.erasure_task_tx, + } + } + + fn is_unavailable( + unrequested_validators: usize, + in_flight_requests: usize, + chunk_count: usize, + threshold: usize, + ) -> bool { + is_unavailable(chunk_count, in_flight_requests, unrequested_validators, threshold) + } + + /// Desired number of parallel requests. + /// + /// For the given threshold (total required number of chunks) get the desired number of + /// requests we want to have running in parallel at this time. + fn get_desired_request_count(&self, chunk_count: usize, threshold: usize) -> usize { + // Upper bound for parallel requests. + // We want to limit this, so requests can be processed within the timeout and we limit the + // following feedback loop: + // 1. Requests fail due to timeout + // 2. We request more chunks to make up for it + // 3. Bandwidth is spread out even more, so we get even more timeouts + // 4. We request more chunks to make up for it ... + let max_requests_boundary = std::cmp::min(N_PARALLEL, threshold); + // How many chunks are still needed? + let remaining_chunks = threshold.saturating_sub(chunk_count); + // What is the current error rate, so we can make up for it? + let inv_error_rate = + self.total_received_responses.checked_div(self.error_count).unwrap_or(0); + // Actual number of requests we want to have in flight in parallel: + std::cmp::min( + max_requests_boundary, + remaining_chunks + remaining_chunks.checked_div(inv_error_rate).unwrap_or(0), + ) + } + + async fn attempt_recovery( + &mut self, + state: &mut State, + common_params: &RecoveryParams, + ) -> Result { + let recovery_duration = common_params.metrics.time_erasure_recovery(); + + // Send request to reconstruct available data from chunks. + let (avilable_data_tx, available_data_rx) = oneshot::channel(); + self.erasure_task_tx + .send(ErasureTask::Reconstruct( + common_params.n_validators, + // Safe to leave an empty vec in place, as we're stopping the recovery process if + // this reconstruct fails. + std::mem::take(&mut state.received_chunks), + avilable_data_tx, + )) + .await + .map_err(|_| RecoveryError::ChannelClosed)?; + + let available_data_response = + available_data_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; + + match available_data_response { + Ok(data) => { + // Send request to re-encode the chunks and check merkle root. + let (reencode_tx, reencode_rx) = oneshot::channel(); + self.erasure_task_tx + .send(ErasureTask::Reencode( + common_params.n_validators, + common_params.erasure_root, + data, + reencode_tx, + )) + .await + .map_err(|_| RecoveryError::ChannelClosed)?; + + let reencode_response = + reencode_rx.await.map_err(|_| RecoveryError::ChannelClosed)?; + + if let Some(data) = reencode_response { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + erasure_root = ?common_params.erasure_root, + "Data recovery from chunks complete", + ); + + Ok(data) + } else { + recovery_duration.map(|rd| rd.stop_and_discard()); + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + erasure_root = ?common_params.erasure_root, + "Data recovery error - root mismatch", + ); + + Err(RecoveryError::Invalid) + } + }, + Err(err) => { + recovery_duration.map(|rd| rd.stop_and_discard()); + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + erasure_root = ?common_params.erasure_root, + ?err, + "Data recovery error ", + ); + + Err(RecoveryError::Invalid) + }, + } + } +} + +#[async_trait::async_trait] +impl RecoveryStrategy for FetchChunks { + fn display_name(&self) -> &'static str { + "Fetch chunks" + } + + async fn run( + &mut self, + state: &mut State, + sender: &mut Sender, + common_params: &RecoveryParams, + ) -> Result { + // First query the store for any chunks we've got. + if !common_params.bypass_availability_store { + let local_chunk_indices = state.populate_from_av_store(common_params, sender).await; + self.validators.retain(|i| !local_chunk_indices.contains(i)); + } + + // No need to query the validators that have the chunks we already received. + self.validators.retain(|i| !state.received_chunks.contains_key(i)); + + loop { + // If received_chunks has more than threshold entries, attempt to recover the data. + // If that fails, or a re-encoding of it doesn't match the expected erasure root, + // return Err(RecoveryError::Invalid). + // Do this before requesting any chunks because we may have enough of them coming from + // past RecoveryStrategies. + if state.chunk_count() >= common_params.threshold { + return self.attempt_recovery(state, common_params).await + } + + if Self::is_unavailable( + self.validators.len(), + self.requesting_chunks.total_len(), + state.chunk_count(), + common_params.threshold, + ) { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?common_params.candidate_hash, + erasure_root = ?common_params.erasure_root, + received = %state.chunk_count(), + requesting = %self.requesting_chunks.len(), + total_requesting = %self.requesting_chunks.total_len(), + n_validators = %common_params.n_validators, + "Data recovery from chunks is not possible", + ); + + return Err(RecoveryError::Unavailable) + } + + let desired_requests_count = + self.get_desired_request_count(state.chunk_count(), common_params.threshold); + let already_requesting_count = self.requesting_chunks.len(); + gum::debug!( + target: LOG_TARGET, + ?common_params.candidate_hash, + ?desired_requests_count, + error_count= ?self.error_count, + total_received = ?self.total_received_responses, + threshold = ?common_params.threshold, + ?already_requesting_count, + "Requesting availability chunks for a candidate", + ); + state + .launch_parallel_chunk_requests( + common_params, + sender, + desired_requests_count, + &mut self.validators, + &mut self.requesting_chunks, + ) + .await; + + let (total_responses, error_count) = state + .wait_for_chunks( + common_params, + &mut self.validators, + &mut self.requesting_chunks, + |unrequested_validators, reqs, chunk_count, params, _error_count| { + chunk_count >= params.threshold || + Self::is_unavailable( + unrequested_validators, + reqs, + chunk_count, + params.threshold, + ) + }, + ) + .await; + + self.total_received_responses += total_responses; + self.error_count += error_count; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use polkadot_erasure_coding::recovery_threshold; + + #[test] + fn parallel_request_calculation_works_as_expected() { + let num_validators = 100; + let threshold = recovery_threshold(num_validators).unwrap(); + let (erasure_task_tx, _erasure_task_rx) = futures::channel::mpsc::channel(16); + + let mut fetch_chunks_task = + FetchChunks::new(FetchChunksParams { n_validators: 100, erasure_task_tx }); + assert_eq!(fetch_chunks_task.get_desired_request_count(0, threshold), threshold); + fetch_chunks_task.error_count = 1; + fetch_chunks_task.total_received_responses = 1; + // We saturate at threshold (34): + assert_eq!(fetch_chunks_task.get_desired_request_count(0, threshold), threshold); + + fetch_chunks_task.total_received_responses = 2; + // With given error rate - still saturating: + assert_eq!(fetch_chunks_task.get_desired_request_count(1, threshold), threshold); + fetch_chunks_task.total_received_responses += 8; + // error rate: 1/10 + // remaining chunks needed: threshold (34) - 9 + // expected: 24 * (1+ 1/10) = (next greater integer) = 27 + assert_eq!(fetch_chunks_task.get_desired_request_count(9, threshold), 27); + fetch_chunks_task.error_count = 0; + // With error count zero - we should fetch exactly as needed: + assert_eq!(fetch_chunks_task.get_desired_request_count(10, threshold), threshold - 10); + } +} diff --git a/polkadot/node/network/availability-recovery/src/tests.rs b/polkadot/node/network/availability-recovery/src/tests.rs index de923f5967e5fd919d76ede2fc38e69d25f160e7..63ccf0e94f91ebad2a8cba3158d6db24159683c0 100644 --- a/polkadot/node/network/availability-recovery/src/tests.rs +++ b/polkadot/node/network/availability-recovery/src/tests.rs @@ -21,20 +21,22 @@ use futures::{executor, future}; use futures_timer::Delay; use parity_scale_codec::Encode; -use polkadot_node_network_protocol::request_response::{IncomingRequest, ReqProtocolNames}; +use polkadot_node_network_protocol::request_response::{ + self as req_res, IncomingRequest, Recipient, ReqProtocolNames, Requests, +}; use super::*; -use sc_network::config::RequestResponseConfig; +use sc_network::{config::RequestResponseConfig, IfDisconnected, OutboundFailure, RequestFailure}; use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks}; use polkadot_node_primitives::{BlockData, PoV, Proof}; -use polkadot_node_subsystem::{ - jaeger, - messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, - ActivatedLeaf, LeafStatus, +use polkadot_node_subsystem::messages::{ + AllMessages, NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest, +}; +use polkadot_node_subsystem_test_helpers::{ + make_subsystem_context, mock::new_leaf, TestSubsystemContextHandle, }; -use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::{ AuthorityDiscoveryId, Hash, HeadData, IndexedVec, PersistedValidationData, ValidatorId, @@ -206,7 +208,7 @@ use sp_keyring::Sr25519Keyring; enum Has { No, Yes, - NetworkError(sc_network::RequestFailure), + NetworkError(RequestFailure), /// Make request not return at all, instead the sender is returned from the function. /// /// Note, if you use `DoesNotReturn` you have to keep the returned senders alive, otherwise the @@ -216,7 +218,7 @@ enum Has { impl Has { fn timeout() -> Self { - Has::NetworkError(sc_network::RequestFailure::Network(sc_network::OutboundFailure::Timeout)) + Has::NetworkError(RequestFailure::Network(OutboundFailure::Timeout)) } } @@ -395,7 +397,7 @@ impl TestState { candidate_hash: CandidateHash, virtual_overseer: &mut VirtualOverseer, who_has: impl Fn(usize) -> Has, - ) -> Vec, sc_network::RequestFailure>>> { + ) -> Vec, RequestFailure>>> { let mut senders = Vec::new(); for _ in 0..self.validators.len() { // Receive a request for a chunk. @@ -561,12 +563,10 @@ fn availability_is_recovered_from_chunks_if_no_group_provided() { test_harness_fast_path(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -647,12 +647,10 @@ fn availability_is_recovered_from_chunks_even_if_backing_group_supplied_if_chunk test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -733,12 +731,10 @@ fn bad_merkle_path_leads_to_recovery_error() { test_harness_fast_path(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -791,12 +787,10 @@ fn wrong_chunk_index_leads_to_recovery_error() { test_harness_fast_path(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -865,12 +859,10 @@ fn invalid_erasure_coding_leads_to_invalid_error() { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -914,12 +906,10 @@ fn fast_path_backing_group_recovers() { test_harness_fast_path(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -964,12 +954,10 @@ fn recovers_from_only_chunks_if_pov_large() { test_harness_chunks_if_pov_large(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1026,7 +1014,7 @@ fn recovers_from_only_chunks_if_pov_large() { AvailabilityRecoveryMessage::RecoverAvailableData( new_candidate.clone(), test_state.session_index, - None, + Some(GroupIndex(0)), tx, ), ) @@ -1068,12 +1056,10 @@ fn fast_path_backing_group_recovers_if_pov_small() { test_harness_chunks_if_pov_large(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1127,12 +1113,10 @@ fn no_answers_in_fast_path_causes_chunk_requests() { test_harness_fast_path(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1189,12 +1173,10 @@ fn task_canceled_when_receivers_dropped() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1231,12 +1213,10 @@ fn chunks_retry_until_all_nodes_respond() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1292,12 +1272,10 @@ fn not_returning_requests_wont_stall_retrieval() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1364,12 +1342,10 @@ fn all_not_returning_requests_still_recovers_on_return() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1441,12 +1417,10 @@ fn returns_early_if_we_have_the_data() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1478,12 +1452,10 @@ fn does_not_query_local_validator() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1537,12 +1509,10 @@ fn invalid_local_chunk_is_ignored() { test_harness_chunks_only(|mut virtual_overseer, req_cfg| async move { overseer_signal( &mut virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.current, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.current, + 1, + ))), ) .await; @@ -1580,36 +1550,3 @@ fn invalid_local_chunk_is_ignored() { (virtual_overseer, req_cfg) }); } - -#[test] -fn parallel_request_calculation_works_as_expected() { - let num_validators = 100; - let threshold = recovery_threshold(num_validators).unwrap(); - let (erasure_task_tx, _erasure_task_rx) = futures::channel::mpsc::channel(16); - - let mut phase = RequestChunksFromValidators::new(100, erasure_task_tx); - assert_eq!(phase.get_desired_request_count(threshold), threshold); - phase.error_count = 1; - phase.total_received_responses = 1; - // We saturate at threshold (34): - assert_eq!(phase.get_desired_request_count(threshold), threshold); - - let dummy_chunk = - ErasureChunk { chunk: Vec::new(), index: ValidatorIndex(0), proof: Proof::dummy_proof() }; - phase.insert_chunk(ValidatorIndex(0), dummy_chunk.clone()); - phase.total_received_responses = 2; - // With given error rate - still saturating: - assert_eq!(phase.get_desired_request_count(threshold), threshold); - for i in 1..9 { - phase.insert_chunk(ValidatorIndex(i), dummy_chunk.clone()); - } - phase.total_received_responses += 8; - // error rate: 1/10 - // remaining chunks needed: threshold (34) - 9 - // expected: 24 * (1+ 1/10) = (next greater integer) = 27 - assert_eq!(phase.get_desired_request_count(threshold), 27); - phase.insert_chunk(ValidatorIndex(9), dummy_chunk.clone()); - phase.error_count = 0; - // With error count zero - we should fetch exactly as needed: - assert_eq!(phase.get_desired_request_count(threshold), threshold - phase.chunk_count()); -} diff --git a/polkadot/node/network/bridge/src/metrics.rs b/polkadot/node/network/bridge/src/metrics.rs index bb90daad56761d24d5d68995580e893a4d773e8c..083a2a71aa0f8ad05c16822feabd0e6528f48a31 100644 --- a/polkadot/node/network/bridge/src/metrics.rs +++ b/polkadot/node/network/bridge/src/metrics.rs @@ -105,9 +105,27 @@ impl Metrics { pub fn on_report_event(&self) { if let Some(metrics) = self.0.as_ref() { + self.on_message("report_peer"); metrics.report_events.inc() } } + + pub fn on_message(&self, message_type: &'static str) { + if let Some(metrics) = self.0.as_ref() { + metrics.messages_sent.with_label_values(&[message_type]).inc() + } + } + + pub fn on_delayed_rx_queue(&self, queue_size: usize) { + if let Some(metrics) = self.0.as_ref() { + metrics.rx_delayed_processing.observe(queue_size as f64); + } + } + pub fn time_delayed_rx_events( + &self, + ) -> Option { + self.0.as_ref().map(|metrics| metrics.rx_delayed_processing_time.start_timer()) + } } #[derive(Clone)] @@ -123,6 +141,13 @@ pub(crate) struct MetricsInner { bytes_received: prometheus::CounterVec, bytes_sent: prometheus::CounterVec, + + messages_sent: prometheus::CounterVec, + // The reason why a `Histogram` is used to track a queue size is that + // we need not only an average size of the queue (that will be 0 normally), but + // we also need a dynamics for this queue size in case of messages delays. + rx_delayed_processing: prometheus::Histogram, + rx_delayed_processing_time: prometheus::Histogram, } impl metrics::Metrics for Metrics { @@ -217,6 +242,34 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + messages_sent: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_messages_sent_total", + "The number of messages sent via network bridge", + ), + &["type"] + )?, + registry, + )?, + rx_delayed_processing: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_network_bridge_rx_delayed", + "Number of events being delayed while broadcasting from the network bridge", + ).buckets(vec![0.0, 1.0, 2.0, 8.0, 16.0]), + )?, + registry, + )?, + rx_delayed_processing_time: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_network_bridge_rx_delayed_time", + "Time spent for waiting of the delayed events", + ), + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 4f21212dcb64a76e25396de94b35204d53682aaa..823e1254612f8cae6980c518ffd848f0b61f2e88 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -61,6 +61,7 @@ pub(crate) fn send_message( let message = { let encoded = message.encode(); metrics.on_notification_sent(peer_set, version, encoded.len(), peers.len()); + metrics.on_message(std::any::type_name::()); encoded }; diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 51d248ca2d49fbe5d6b2cf3b246f42cfb849fe19..82c67061d9a5053e6c531c82c560f0c124470647 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -20,7 +20,7 @@ use super::*; use always_assert::never; use bytes::Bytes; -use futures::stream::BoxStream; +use futures::stream::{BoxStream, StreamExt}; use parity_scale_codec::{Decode, DecodeAll}; use sc_network::Event as NetworkEvent; @@ -244,6 +244,7 @@ where NetworkBridgeEvent::PeerViewChange(peer, View::default()), ], &mut sender, + &metrics, ) .await; @@ -352,6 +353,7 @@ where dispatch_validation_event_to_all( NetworkBridgeEvent::PeerDisconnected(peer), &mut sender, + &metrics, ) .await, PeerSet::Collation => @@ -490,7 +492,7 @@ where network_service.report_peer(remote, report.into()); } - dispatch_validation_events_to_all(events, &mut sender).await; + dispatch_validation_events_to_all(events, &mut sender, &metrics).await; } if !c_messages.is_empty() { @@ -992,8 +994,9 @@ fn send_collation_message_vstaging( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, + metrics: &Metrics, ) { - dispatch_validation_events_to_all(std::iter::once(event), ctx).await + dispatch_validation_events_to_all(std::iter::once(event), ctx, metrics).await } async fn dispatch_collation_event_to_all( @@ -1041,6 +1044,7 @@ fn dispatch_collation_event_to_all_unbounded( async fn dispatch_validation_events_to_all( events: I, sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + _metrics: &Metrics, ) where I: IntoIterator>, I::IntoIter: Send, diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index 88a4b247fdc6f6d7d24d4facceea7a558b53c31c..127f46e0fa37390a440125f8d567c6ff47e53aef 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -16,8 +16,9 @@ use super::*; use futures::{channel::oneshot, executor, stream::BoxStream}; +use overseer::jaeger; use polkadot_node_network_protocol::{self as net_protocol, OurView}; -use polkadot_node_subsystem::{messages::NetworkBridgeEvent, ActivatedLeaf}; +use polkadot_node_subsystem::messages::NetworkBridgeEvent; use assert_matches::assert_matches; use async_trait::async_trait; @@ -36,15 +37,14 @@ use polkadot_node_network_protocol::{ view, ObservedRole, Versioned, }; use polkadot_node_subsystem::{ - jaeger, messages::{ AllMessages, ApprovalDistributionMessage, BitfieldDistributionMessage, GossipSupportMessage, StatementDistributionMessage, }, - ActiveLeavesUpdate, FromOrchestra, LeafStatus, OverseerSignal, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, }; use polkadot_node_subsystem_test_helpers::{ - SingleItemSink, SingleItemStream, TestSubsystemContextHandle, + mock::new_leaf, SingleItemSink, SingleItemStream, TestSubsystemContextHandle, }; use polkadot_node_subsystem_util::metered; use polkadot_primitives::{AuthorityDiscoveryId, CandidateHash, Hash}; @@ -427,12 +427,7 @@ fn send_our_view_upon_connection() { let head = Hash::repeat_byte(1); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: head, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(head, 1)), ))) .await; @@ -514,12 +509,7 @@ fn sends_view_updates_to_peers() { virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -585,12 +575,7 @@ fn do_not_send_view_update_until_synced() { virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -601,12 +586,7 @@ fn do_not_send_view_update_until_synced() { virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_b, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_b, 1)), ))) .await; @@ -672,12 +652,7 @@ fn do_not_send_view_update_when_only_finalized_block_changed() { // This should trigger the view update to our peers. virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -895,12 +870,7 @@ fn peer_disconnect_from_just_one_peerset() { virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -1132,12 +1102,7 @@ fn sent_views_include_finalized_number_update() { .await; virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_b, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_b, 1)), ))) .await; @@ -1211,12 +1176,7 @@ fn our_view_updates_decreasing_order_and_limited_to_max() { // get the correct view. virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash, - number: i as _, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash, i as _)), ))) .await; } @@ -1265,12 +1225,7 @@ fn network_protocol_versioning_view_update() { let head = Hash::repeat_byte(1); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: head, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(head, 1)), ))) .await; diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 1b386ce1239bd2e3204aa6871dd3f40309eda9c1..7fa1149593cab977edbb321a96a5e8268859476c 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -33,6 +33,7 @@ use polkadot_node_subsystem::{ /// /// To be passed to [`FullNetworkConfiguration::add_notification_protocol`](). pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; +use polkadot_node_network_protocol::request_response::Requests; use sc_network::ReputationChange; use crate::validator_discovery; @@ -290,6 +291,20 @@ where ); for req in reqs { + match req { + Requests::ChunkFetchingV1(_) => metrics.on_message("chunk_fetching_v1"), + Requests::AvailableDataFetchingV1(_) => + metrics.on_message("available_data_fetching_v1"), + Requests::CollationFetchingV1(_) => metrics.on_message("collation_fetching_v1"), + Requests::CollationFetchingVStaging(_) => + metrics.on_message("collation_fetching_vstaging"), + Requests::PoVFetchingV1(_) => metrics.on_message("pov_fetching_v1"), + Requests::DisputeSendingV1(_) => metrics.on_message("dispute_sending_v1"), + Requests::StatementFetchingV1(_) => metrics.on_message("statement_fetching_v1"), + Requests::AttestedCandidateVStaging(_) => + metrics.on_message("attested_candidate_vstaging"), + } + network_service .start_request( &mut authority_discovery_service, diff --git a/polkadot/node/network/collator-protocol/Cargo.toml b/polkadot/node/network/collator-protocol/Cargo.toml index ac6284b305bd9a0af82f9317cd65a1d59418b1e3..e5328cf16629f4c7bdb2180e90fbe888dbfb8f42 100644 --- a/polkadot/node/network/collator-protocol/Cargo.toml +++ b/polkadot/node/network/collator-protocol/Cargo.toml @@ -21,7 +21,7 @@ polkadot-node-primitives = { path = "../../primitives" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-node-subsystem = { path = "../../subsystem" } fatality = "0.0.6" -thiserror = "1.0.31" +thiserror = "1.0.48" tokio-util = "0.7.1" [dev-dependencies] diff --git a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs index 96978f39a5328bc3f2c6935c1598a6f5c0ea85a0..ad2ab99568c8cdc3676984ae3fad43e87ccf4374 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/mod.rs @@ -926,16 +926,42 @@ async fn handle_incoming_peer_message( target: LOG_TARGET, ?statement, ?origin, - "received a valid `CollationSeconded`", + "received a valid `CollationSeconded`, forwarding result to collator", ); let _ = sender.send(CollationSecondedSignal { statement, relay_parent }); } else { - gum::debug!( - target: LOG_TARGET, - candidate_hash = ?&statement.payload().candidate_hash(), - ?origin, - "received an unexpected `CollationSeconded`: unknown statement", - ); + // Checking whether the `CollationSeconded` statement is unexpected + let relay_parent = match state.per_relay_parent.get(&relay_parent) { + Some(per_relay_parent) => per_relay_parent, + None => { + gum::debug!( + target: LOG_TARGET, + candidate_relay_parent = %relay_parent, + candidate_hash = ?&statement.payload().candidate_hash(), + "Seconded statement relay parent is out of our view", + ); + return Ok(()) + }, + }; + match relay_parent.collations.get(&statement.payload().candidate_hash()) { + Some(_) => { + // We've seen this collation before, so a seconded statement is expected + gum::trace!( + target: LOG_TARGET, + ?statement, + ?origin, + "received a valid `CollationSeconded`", + ); + }, + None => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?&statement.payload().candidate_hash(), + ?origin, + "received an unexpected `CollationSeconded`: unknown statement", + ); + }, + } } } }, diff --git a/polkadot/node/network/collator-protocol/src/collator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/collator_side/tests/mod.rs index 8b28730c90125b3b6c2788b700389ddb2738e0b7..b452c84c2cd8f1091a3c5f0e2c34f3e40f4b2a0b 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/tests/mod.rs @@ -40,7 +40,7 @@ use polkadot_node_subsystem::{ errors::RuntimeApiError, jaeger, messages::{AllMessages, ReportPeerMessage, RuntimeApiMessage, RuntimeApiRequest}, - ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, + ActiveLeavesUpdate, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt}; @@ -49,6 +49,7 @@ use polkadot_primitives::{ ScheduledCore, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, }; use polkadot_primitives_test_helpers::TestCandidateBuilder; +use test_helpers::mock::new_leaf; mod prospective_parachains; @@ -310,12 +311,10 @@ async fn setup_system(virtual_overseer: &mut VirtualOverseer, test_state: &TestS overseer_signal( virtual_overseer, - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: test_state.relay_parent, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + test_state.relay_parent, + 1, + ))), ) .await; diff --git a/polkadot/node/network/dispute-distribution/Cargo.toml b/polkadot/node/network/dispute-distribution/Cargo.toml index ece89f34c88d5bcbcf26b07d403232c5e6bef525..5d8e245d289a7637f9c64c069f6038ab41946c49 100644 --- a/polkadot/node/network/dispute-distribution/Cargo.toml +++ b/polkadot/node/network/dispute-distribution/Cargo.toml @@ -20,7 +20,7 @@ polkadot-node-primitives = { path = "../../primitives" } sc-network = { path = "../../../../substrate/client/network" } sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto" } sp-keystore = { path = "../../../../substrate/primitives/keystore" } -thiserror = "1.0.31" +thiserror = "1.0.48" fatality = "0.0.6" schnellru = "0.2.1" indexmap = "1.9.1" diff --git a/polkadot/node/network/dispute-distribution/src/tests/mod.rs b/polkadot/node/network/dispute-distribution/src/tests/mod.rs index f53b9c0dd4b57a0bd65e49b1c1896999eb9e1d23..96f045cbf769219e737b4366fb0c96201c9c3b6e 100644 --- a/polkadot/node/network/dispute-distribution/src/tests/mod.rs +++ b/polkadot/node/network/dispute-distribution/src/tests/mod.rs @@ -19,7 +19,6 @@ use std::{ collections::HashSet, - sync::Arc, task::Poll, time::{Duration, Instant}, }; @@ -51,10 +50,11 @@ use polkadot_node_subsystem::{ AllMessages, DisputeCoordinatorMessage, DisputeDistributionMessage, ImportStatementsResult, NetworkBridgeTxMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, ActiveLeavesUpdate, FromOrchestra, LeafStatus, OverseerSignal, Span, + ActiveLeavesUpdate, FromOrchestra, OverseerSignal, }; use polkadot_node_subsystem_test_helpers::{ - mock::make_ferdie_keystore, subsystem_test_harness, TestSubsystemContextHandle, + mock::{make_ferdie_keystore, new_leaf}, + subsystem_test_harness, TestSubsystemContextHandle, }; use polkadot_primitives::{ AuthorityDiscoveryId, CandidateHash, CandidateReceipt, ExecutorParams, Hash, SessionIndex, @@ -735,12 +735,7 @@ async fn activate_leaf( ) { handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: activate, - number: 10, - status: LeafStatus::Fresh, - span: Arc::new(Span::Disabled), - }), + activated: Some(new_leaf(activate, 10)), deactivated: deactivate.into_iter().collect(), }))) .await; diff --git a/polkadot/node/network/gossip-support/src/tests.rs b/polkadot/node/network/gossip-support/src/tests.rs index 20f550514157d7c542805fcc26db06b672ce8194..2e909bb0a67433e667ae87d679005091fe505810 100644 --- a/polkadot/node/network/gossip-support/src/tests.rs +++ b/polkadot/node/network/gossip-support/src/tests.rs @@ -16,7 +16,7 @@ //! Unit tests for Gossip Support Subsystem. -use std::{collections::HashSet, sync::Arc, time::Duration}; +use std::{collections::HashSet, time::Duration}; use assert_matches::assert_matches; use async_trait::async_trait; @@ -30,15 +30,11 @@ use sp_core::crypto::Pair as PairT; use sp_keyring::Sr25519Keyring; use polkadot_node_network_protocol::grid_topology::{SessionGridTopology, TopologyPeerInfo}; -use polkadot_node_subsystem::{ - jaeger, - messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}, - ActivatedLeaf, LeafStatus, -}; +use polkadot_node_subsystem::messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest}; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::TimeoutExt as _; use polkadot_primitives::{GroupIndex, IndexedVec}; -use test_helpers::mock::make_ferdie_keystore; +use test_helpers::mock::{make_ferdie_keystore, new_leaf}; use super::*; @@ -196,12 +192,7 @@ fn test_harness, AD: AuthorityDiscovery>( const TIMEOUT: Duration = Duration::from_millis(100); async fn overseer_signal_active_leaves(overseer: &mut VirtualOverseer, leaf: Hash) { - let leaf = ActivatedLeaf { - hash: leaf, - number: 0xdeadcafe, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let leaf = new_leaf(leaf, 0xdeadcafe); overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( leaf, diff --git a/polkadot/node/network/protocol/Cargo.toml b/polkadot/node/network/protocol/Cargo.toml index 2a56f197b854789aaf7244a94739538f1c8e5117..c33b9eae3252606e8d2fcbd954a0e180f4a47acb 100644 --- a/polkadot/node/network/protocol/Cargo.toml +++ b/polkadot/node/network/protocol/Cargo.toml @@ -18,7 +18,7 @@ sc-network = { path = "../../../../substrate/client/network" } sc-authority-discovery = { path = "../../../../substrate/client/authority-discovery" } strum = { version = "0.24", features = ["derive"] } futures = "0.3.21" -thiserror = "1.0.31" +thiserror = "1.0.48" fatality = "0.0.6" rand = "0.8" derive_more = "0.99" diff --git a/polkadot/node/network/statement-distribution/Cargo.toml b/polkadot/node/network/statement-distribution/Cargo.toml index 5ff1ba9de02ba380c83facd840303fbb0b770c73..bf516e7b7ba9b8b89dbea715bef9bf6af2ff5d36 100644 --- a/polkadot/node/network/statement-distribution/Cargo.toml +++ b/polkadot/node/network/statement-distribution/Cargo.toml @@ -21,7 +21,7 @@ polkadot-node-network-protocol = { path = "../protocol" } arrayvec = "0.7.4" indexmap = "1.9.1" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } -thiserror = "1.0.31" +thiserror = "1.0.48" fatality = "0.0.6" bitvec = "1" diff --git a/polkadot/node/network/statement-distribution/src/legacy_v1/tests.rs b/polkadot/node/network/statement-distribution/src/legacy_v1/tests.rs index 63ac1febde56a7bd53a2acacd38f5ae40d8e2913..17a66a9ff792ffddca4c5e8a750b5f7bdcf205f0 100644 --- a/polkadot/node/network/statement-distribution/src/legacy_v1/tests.rs +++ b/polkadot/node/network/statement-distribution/src/legacy_v1/tests.rs @@ -36,13 +36,12 @@ use polkadot_node_primitives::{ SignedFullStatementWithPVD, Statement, UncheckedSignedFullStatement, }; use polkadot_node_subsystem::{ - jaeger, messages::{ network_bridge_event, AllMessages, ReportPeerMessage, RuntimeApiMessage, RuntimeApiRequest, }, - ActivatedLeaf, LeafStatus, RuntimeApiError, + RuntimeApiError, }; -use polkadot_node_subsystem_test_helpers::mock::make_ferdie_keystore; +use polkadot_node_subsystem_test_helpers::mock::{make_ferdie_keystore, new_leaf}; use polkadot_primitives::{ ExecutorParams, GroupIndex, Hash, HeadData, Id as ParaId, IndexedVec, SessionInfo, ValidationCode, @@ -787,12 +786,7 @@ fn receiving_from_one_sends_to_another_and_to_candidate_backing() { // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -1032,12 +1026,7 @@ fn receiving_large_statement_from_one_sends_to_another_and_to_candidate_backing( // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -1567,12 +1556,7 @@ fn delay_reputation_changes() { // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -2052,12 +2036,7 @@ fn share_prioritizes_backing_group() { // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -2379,12 +2358,7 @@ fn peer_cant_flood_with_large_statements() { // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: hash_a, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(hash_a, 1)), ))) .await; @@ -2609,12 +2583,7 @@ fn handle_multiple_seconded_statements() { // register our active heads. handle .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves( - ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: relay_parent_hash, - number: 1, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }), + ActiveLeavesUpdate::start_work(new_leaf(relay_parent_hash, 1)), ))) .await; diff --git a/polkadot/node/network/statement-distribution/src/vstaging/tests/mod.rs b/polkadot/node/network/statement-distribution/src/vstaging/tests/mod.rs index c5a4d14d2c7a5c13920bbc753429442d95eb3c5e..48ceebb1949bfa2889986554b9583cbf33cd72aa 100644 --- a/polkadot/node/network/statement-distribution/src/vstaging/tests/mod.rs +++ b/polkadot/node/network/statement-distribution/src/vstaging/tests/mod.rs @@ -30,7 +30,6 @@ use polkadot_node_subsystem::messages::{ RuntimeApiMessage, RuntimeApiRequest, }; use polkadot_node_subsystem_test_helpers as test_helpers; -use polkadot_node_subsystem_types::{jaeger, ActivatedLeaf, LeafStatus}; use polkadot_node_subsystem_util::TimeoutExt; use polkadot_primitives::vstaging::{ AssignmentPair, AsyncBackingParams, BlockNumber, CommittedCandidateReceipt, CoreState, @@ -46,6 +45,7 @@ use assert_matches::assert_matches; use futures::Future; use parity_scale_codec::Encode; use rand::{Rng, SeedableRng}; +use test_helpers::mock::new_leaf; use std::sync::Arc; @@ -358,12 +358,7 @@ async fn activate_leaf( test_state: &TestState, is_new_session: bool, ) { - let activated = ActivatedLeaf { - hash: leaf.hash, - number: leaf.number, - status: LeafStatus::Fresh, - span: Arc::new(jaeger::Span::Disabled), - }; + let activated = new_leaf(leaf.hash, leaf.number); virtual_overseer .send(FromOrchestra::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work( diff --git a/polkadot/node/overseer/Cargo.toml b/polkadot/node/overseer/Cargo.toml index 2ebe2e9e0743ac2936511ad991c0a6a97bded75e..5d41407ef83a832351ff29a4026352afbb21bf5c 100644 --- a/polkadot/node/overseer/Cargo.toml +++ b/polkadot/node/overseer/Cargo.toml @@ -16,7 +16,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } -orchestra = "0.0.5" +orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } gum = { package = "tracing-gum", path = "../gum" } schnellru = "0.2.1" sp-core = { path = "../../../substrate/primitives/core" } @@ -24,19 +24,20 @@ async-trait = "0.1.57" tikv-jemalloc-ctl = { version = "0.5.0", optional = true } [dev-dependencies] -metered = { package = "prioritized-metered-channel", version = "0.2.0" } - +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } sp-core = { path = "../../../substrate/primitives/core" } futures = { version = "0.3.21", features = ["thread-pool"] } femme = "2.2.1" assert_matches = "1.4.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } +node-test-helpers = { package = "polkadot-node-subsystem-test-helpers", path = "../subsystem-test-helpers" } [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" [features] -default = [] -expand = [ "orchestra/expand" ] +default = [ "futures_channel" ] dotgraph = [ "orchestra/dotgraph" ] +expand = [ "orchestra/expand" ] +futures_channel = [ "metered/futures_channel", "orchestra/futures_channel" ] jemalloc-allocator = [ "dep:tikv-jemalloc-ctl" ] diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index b8643982323cdc0c495a192c2de49140193ef88a..84d5d19c3b93c708110fc35d2f124b94984f8a86 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -87,7 +87,7 @@ use polkadot_node_subsystem_types::messages::{ pub use polkadot_node_subsystem_types::{ errors::{SubsystemError, SubsystemResult}, jaeger, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus, OverseerSignal, - RuntimeApiSubsystemClient, + RuntimeApiSubsystemClient, UnpinHandle, }; pub mod metrics; @@ -107,7 +107,7 @@ pub use orchestra::{ contextbounds, orchestra, subsystem, FromOrchestra, MapSubsystem, MessagePacket, OrchestraError as OverseerError, SignalsReceived, Spawner, Subsystem, SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, SubsystemMeterReadouts, SubsystemMeters, - SubsystemSender, TimeoutExt, ToOrchestra, + SubsystemSender, TimeoutExt, ToOrchestra, TrySendError, }; /// Store 2 days worth of blocks, not accounting for forks, @@ -245,23 +245,35 @@ impl Handle { /// `HeaderBackend::block_number_from_id()`. #[derive(Debug, Clone)] pub struct BlockInfo { - /// hash of the block. + /// Hash of the block. pub hash: Hash, - /// hash of the parent block. + /// Hash of the parent block. pub parent_hash: Hash, - /// block's number. + /// Block's number. pub number: BlockNumber, + /// A handle to unpin the block on drop. + pub unpin_handle: UnpinHandle, } impl From> for BlockInfo { fn from(n: BlockImportNotification) -> Self { - BlockInfo { hash: n.hash, parent_hash: n.header.parent_hash, number: n.header.number } + let hash = n.hash; + let parent_hash = n.header.parent_hash; + let number = n.header.number; + let unpin_handle = n.into_unpin_handle(); + + BlockInfo { hash, parent_hash, number, unpin_handle } } } impl From> for BlockInfo { fn from(n: FinalityNotification) -> Self { - BlockInfo { hash: n.hash, parent_hash: n.header.parent_hash, number: n.header.number } + let hash = n.hash; + let parent_hash = n.header.parent_hash; + let number = n.header.number; + let unpin_handle = n.into_unpin_handle(); + + BlockInfo { hash, parent_hash, number, unpin_handle } } } @@ -792,6 +804,7 @@ where hash: block.hash, number: block.number, status, + unpin_handle: block.unpin_handle, span, }), None => ActiveLeavesUpdate::default(), diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 22d9bf0a708d578a21eb53331c07a1c74630fbfb..c17613fb7ea5dc3d12f04a6d3a9162b7de4dbbe6 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -19,15 +19,14 @@ use futures::{executor, pending, pin_mut, poll, select, stream, FutureExt}; use std::{collections::HashMap, sync::atomic, task::Poll}; use ::test_helpers::{dummy_candidate_descriptor, dummy_candidate_receipt, dummy_hash}; +use node_test_helpers::mock::{dummy_unpin_handle, new_leaf}; use polkadot_node_network_protocol::{PeerId, UnifiedReputationChange}; use polkadot_node_primitives::{ BlockData, CollationGenerationConfig, CollationResult, DisputeMessage, InvalidDisputeVote, PoV, UncheckedDisputeMessage, ValidDisputeVote, }; -use polkadot_node_subsystem_types::{ - jaeger, - messages::{NetworkBridgeEvent, ReportPeerMessage, RuntimeApiRequest}, - ActivatedLeaf, LeafStatus, +use polkadot_node_subsystem_types::messages::{ + NetworkBridgeEvent, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, @@ -99,7 +98,7 @@ where if c < 10 { let candidate_receipt = CandidateReceipt { descriptor: dummy_candidate_descriptor(dummy_hash()), - commitments_hash: Hash::zero(), + commitments_hash: dummy_hash(), }; let (tx, _) = oneshot::channel(); @@ -216,11 +215,20 @@ fn overseer_metrics_work() { executor::block_on(async move { let first_block_hash = [1; 32].into(); let second_block_hash = [2; 32].into(); + let unpin_handle = dummy_unpin_handle(dummy_hash()); - let first_block = - BlockInfo { hash: first_block_hash, parent_hash: [0; 32].into(), number: 1 }; - let second_block = - BlockInfo { hash: second_block_hash, parent_hash: first_block_hash, number: 2 }; + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + unpin_handle: unpin_handle.clone(), + }; let registry = prometheus::Registry::new(); let (overseer, handle) = @@ -368,11 +376,20 @@ fn overseer_start_stop_works() { executor::block_on(async move { let first_block_hash = [1; 32].into(); let second_block_hash = [2; 32].into(); + let unpin_handle = dummy_unpin_handle(dummy_hash()); - let first_block = - BlockInfo { hash: first_block_hash, parent_hash: [0; 32].into(), number: 1 }; - let second_block = - BlockInfo { hash: second_block_hash, parent_hash: first_block_hash, number: 2 }; + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: first_block_hash, + number: 2, + unpin_handle: unpin_handle.clone(), + }; let (tx_5, mut rx_5) = metered::channel(64); let (tx_6, mut rx_6) = metered::channel(64); @@ -396,21 +413,11 @@ fn overseer_start_stop_works() { let expected_heartbeats = vec![ OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }), + activated: Some(new_leaf(first_block_hash, 1)), deactivated: Default::default(), }), OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }), + activated: Some(new_leaf(second_block_hash, 2)), deactivated: [first_block_hash].as_ref().into(), }), ]; @@ -456,13 +463,26 @@ fn overseer_finalize_works() { let first_block_hash = [1; 32].into(); let second_block_hash = [2; 32].into(); let third_block_hash = [3; 32].into(); + let unpin_handle = dummy_unpin_handle(dummy_hash()); - let first_block = - BlockInfo { hash: first_block_hash, parent_hash: [0; 32].into(), number: 1 }; - let second_block = - BlockInfo { hash: second_block_hash, parent_hash: [42; 32].into(), number: 2 }; - let third_block = - BlockInfo { hash: third_block_hash, parent_hash: second_block_hash, number: 3 }; + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: [42; 32].into(), + number: 2, + unpin_handle: unpin_handle.clone(), + }; + let third_block = BlockInfo { + hash: third_block_hash, + parent_hash: second_block_hash, + number: 3, + unpin_handle: unpin_handle.clone(), + }; let (tx_5, mut rx_5) = metered::channel(64); let (tx_6, mut rx_6) = metered::channel(64); @@ -492,21 +512,11 @@ fn overseer_finalize_works() { let expected_heartbeats = vec![ OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }), + activated: Some(new_leaf(first_block_hash, 1)), deactivated: Default::default(), }), OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { - activated: Some(ActivatedLeaf { - hash: second_block_hash, - number: 2, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - }), + activated: Some(new_leaf(second_block_hash, 2)), deactivated: Default::default(), }), OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { @@ -563,11 +573,20 @@ fn overseer_finalize_leaf_preserves_it() { executor::block_on(async move { let first_block_hash = [1; 32].into(); let second_block_hash = [2; 32].into(); + let unpin_handle = dummy_unpin_handle(dummy_hash()); - let first_block = - BlockInfo { hash: first_block_hash, parent_hash: [0; 32].into(), number: 1 }; - let second_block = - BlockInfo { hash: second_block_hash, parent_hash: [42; 32].into(), number: 1 }; + let first_block = BlockInfo { + hash: first_block_hash, + parent_hash: [0; 32].into(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; + let second_block = BlockInfo { + hash: second_block_hash, + parent_hash: [42; 32].into(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; let (tx_5, mut rx_5) = metered::channel(64); let (tx_6, mut rx_6) = metered::channel(64); @@ -595,18 +614,14 @@ fn overseer_finalize_leaf_preserves_it() { handle.block_finalized(first_block).await; let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: first_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - })), - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: second_block_hash, - number: 1, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + first_block_hash, + 1, + ))), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + second_block_hash, + 2, + ))), OverseerSignal::ActiveLeaves(ActiveLeavesUpdate { deactivated: [second_block_hash].as_ref().into(), ..Default::default() @@ -657,11 +672,21 @@ fn do_not_send_empty_leaves_update_on_block_finalization() { let spawner = sp_core::testing::TaskExecutor::new(); executor::block_on(async move { - let imported_block = - BlockInfo { hash: Hash::random(), parent_hash: Hash::random(), number: 1 }; + let unpin_handle = dummy_unpin_handle(dummy_hash()); - let finalized_block = - BlockInfo { hash: Hash::random(), parent_hash: Hash::random(), number: 1 }; + let imported_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; + + let finalized_block = BlockInfo { + hash: Hash::random(), + parent_hash: Hash::random(), + number: 1, + unpin_handle: unpin_handle.clone(), + }; let (tx_5, mut rx_5) = metered::channel(64); @@ -682,12 +707,10 @@ fn do_not_send_empty_leaves_update_on_block_finalization() { handle.block_imported(imported_block.clone()).await; let expected_heartbeats = vec![ - OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf { - hash: imported_block.hash, - number: imported_block.number, - span: Arc::new(jaeger::Span::Disabled), - status: LeafStatus::Fresh, - })), + OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(new_leaf( + imported_block.hash, + imported_block.number, + ))), OverseerSignal::BlockFinalized(finalized_block.hash, 1), ]; @@ -952,11 +975,13 @@ fn overseer_all_subsystems_receive_signals_and_messages() { pin_mut!(overseer_fut); // send a signal to each subsystem + let unpin_handle = dummy_unpin_handle(dummy_hash()); handle .block_imported(BlockInfo { hash: Default::default(), parent_hash: Default::default(), number: Default::default(), + unpin_handle: unpin_handle.clone(), }) .await; @@ -1049,6 +1074,7 @@ fn overseer_all_subsystems_receive_signals_and_messages() { #[test] fn context_holds_onto_message_until_enough_signals_received() { + const CHANNEL_CAPACITY: usize = 64; let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); diff --git a/polkadot/node/primitives/Cargo.toml b/polkadot/node/primitives/Cargo.toml index ef03c02f7bc67d55e4ffb9f61c0d4a3c8ecf4897..55dfa67387094198c2126b79b5a31abc23dc2b4e 100644 --- a/polkadot/node/primitives/Cargo.toml +++ b/polkadot/node/primitives/Cargo.toml @@ -19,7 +19,7 @@ sp-maybe-compressed-blob = { path = "../../../substrate/primitives/maybe-compres sp-runtime = { path = "../../../substrate/primitives/runtime" } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } schnorrkel = "0.9.1" -thiserror = "1.0.31" +thiserror = "1.0.48" serde = { version = "1.0.188", features = ["derive"] } [target.'cfg(not(target_os = "unknown"))'.dependencies] diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 90881aa051a0f0952c0af71a6879c165567b5008..ee092e27733267419a47ee1553d6998be3228ca8 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -78,8 +78,8 @@ futures = "0.3.21" hex-literal = "0.4.1" gum = { package = "tracing-gum", path = "../gum" } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.96" -thiserror = "1.0.31" +serde_json = "1.0.107" +thiserror = "1.0.48" kvdb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } parity-db = { version = "0.4.8", optional = true } @@ -103,17 +103,12 @@ polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-runtime-parachains = { path = "../../runtime/parachains" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-runtime-common = { path = "../../runtime/common" } # Polkadot Runtime Constants -polkadot-runtime-constants = { path = "../../runtime/polkadot/constants", optional = true } -kusama-runtime-constants = { path = "../../runtime/kusama/constants", optional = true } rococo-runtime-constants = { path = "../../runtime/rococo/constants", optional = true } westend-runtime-constants = { path = "../../runtime/westend/constants", optional = true } # Polkadot Runtimes -polkadot-runtime = { path = "../../runtime/polkadot", optional = true } -kusama-runtime = { package = "staging-kusama-runtime", path = "../../runtime/kusama", optional = true } westend-runtime = { path = "../../runtime/westend", optional = true } rococo-runtime = { path = "../../runtime/rococo", optional = true } @@ -183,11 +178,7 @@ full-node = [ "polkadot-statement-distribution", ] -# Configure the native runtimes to use. Polkadot is enabled by default. -# -# Validators require the native runtime currently -polkadot-native = [ "polkadot-runtime", "polkadot-runtime-constants" ] -kusama-native = [ "kusama-runtime", "kusama-runtime-constants" ] +# Configure the native runtimes to use. westend-native = [ "westend-runtime", "westend-runtime-constants" ] rococo-native = [ "rococo-runtime", "rococo-runtime-constants" ] @@ -196,15 +187,12 @@ runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", - "kusama-runtime?/runtime-benchmarks", "pallet-babe/runtime-benchmarks", "pallet-im-online/runtime-benchmarks", "pallet-staking/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", - "polkadot-runtime-common/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", - "polkadot-runtime?/runtime-benchmarks", "polkadot-test-client/runtime-benchmarks", "rococo-runtime?/runtime-benchmarks", "sc-client-db/runtime-benchmarks", @@ -215,30 +203,23 @@ runtime-benchmarks = [ try-runtime = [ "frame-support/try-runtime", "frame-system/try-runtime", - "kusama-runtime?/try-runtime", "pallet-babe/try-runtime", "pallet-im-online/try-runtime", "pallet-staking/try-runtime", "pallet-transaction-payment/try-runtime", - "polkadot-runtime-common/try-runtime", "polkadot-runtime-parachains/try-runtime", - "polkadot-runtime?/try-runtime", "rococo-runtime?/try-runtime", "sp-runtime/try-runtime", "westend-runtime?/try-runtime", ] fast-runtime = [ - "kusama-runtime?/fast-runtime", - "polkadot-runtime?/fast-runtime", "rococo-runtime?/fast-runtime", "westend-runtime?/fast-runtime", ] malus = [ "full-node" ] runtime-metrics = [ - "kusama-runtime?/runtime-metrics", "polkadot-runtime-parachains/runtime-metrics", - "polkadot-runtime?/runtime-metrics", "rococo-runtime?/runtime-metrics", "westend-runtime?/runtime-metrics", ] diff --git a/polkadot/node/service/README.adoc b/polkadot/node/service/README.adoc deleted file mode 100644 index 2196d5467806cd63751c89c6ab63674c554a5e87..0000000000000000000000000000000000000000 --- a/polkadot/node/service/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot Service - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/node/service/src/benchmarking.rs b/polkadot/node/service/src/benchmarking.rs index cfe1c873c0554b4a75d1cf6fefa389a5f56f22bd..400daf1aee3448f841f9030f24b78a0029fc0830 100644 --- a/polkadot/node/service/src/benchmarking.rs +++ b/polkadot/node/service/src/benchmarking.rs @@ -34,36 +34,8 @@ macro_rules! identify_chain { $generic_code:expr $(,)* ) => { match $chain { - Chain::Polkadot => { - #[cfg(feature = "polkadot-native")] - { - use polkadot_runtime as runtime; - - let call = $generic_code; - - Ok(polkadot_sign_call(call, $nonce, $current_block, $period, $genesis, $signer)) - } - - #[cfg(not(feature = "polkadot-native"))] - { - Err("`polkadot-native` feature not enabled") - } - }, - Chain::Kusama => { - #[cfg(feature = "kusama-native")] - { - use kusama_runtime as runtime; - - let call = $generic_code; - - Ok(kusama_sign_call(call, $nonce, $current_block, $period, $genesis, $signer)) - } - - #[cfg(not(feature = "kusama-native"))] - { - Err("`kusama-native` feature not enabled") - } - }, + Chain::Polkadot => Err("Polkadot runtimes are currently not supported"), + Chain::Kusama => Err("Kusama runtimes are currently not supported"), Chain::Rococo => { #[cfg(feature = "rococo-native")] { @@ -91,16 +63,18 @@ macro_rules! identify_chain { #[cfg(not(feature = "westend-native"))] { - let _ = $nonce; - let _ = $current_block; - let _ = $period; - let _ = $genesis; - let _ = $signer; - Err("`westend-native` feature not enabled") } }, - Chain::Unknown => Err("Unknown chain"), + Chain::Unknown => { + let _ = $nonce; + let _ = $current_block; + let _ = $period; + let _ = $genesis; + let _ = $signer; + + Err("Unknown chain") + }, } }; } @@ -130,10 +104,8 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder { } fn build(&self, nonce: u32) -> std::result::Result { - let period = polkadot_runtime_common::BlockHashCount::get() - .checked_next_power_of_two() - .map(|c| c / 2) - .unwrap_or(2) as u64; + // We apply the extrinsic directly, so let's take some random period. + let period = 128; let genesis = self.client.usage_info().chain.best_hash; let signer = Sr25519Keyring::Bob.pair(); let current_block = 0; @@ -181,10 +153,8 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { fn build(&self, nonce: u32) -> std::result::Result { let signer = Sr25519Keyring::Bob.pair(); - let period = polkadot_runtime_common::BlockHashCount::get() - .checked_next_power_of_two() - .map(|c| c / 2) - .unwrap_or(2) as u64; + // We apply the extrinsic directly, so let's take some random period. + let period = 128; let genesis = self.client.usage_info().chain.best_hash; let current_block = 0; let _dest = self.dest.clone(); @@ -206,60 +176,6 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder { } } -#[cfg(feature = "polkadot-native")] -fn polkadot_sign_call( - call: polkadot_runtime::RuntimeCall, - nonce: u32, - current_block: u64, - period: u64, - genesis: sp_core::H256, - acc: sp_core::sr25519::Pair, -) -> OpaqueExtrinsic { - use codec::Encode; - use polkadot_runtime as runtime; - use sp_core::Pair; - - let extra: runtime::SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(sp_runtime::generic::Era::mortal( - period, - current_block, - )), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(0), - polkadot_runtime_common::claims::PrevalidateAttests::::new(), - ); - - let payload = runtime::SignedPayload::from_raw( - call.clone(), - extra.clone(), - ( - (), - runtime::VERSION.spec_version, - runtime::VERSION.transaction_version, - genesis, - genesis, - (), - (), - (), - (), - ), - ); - - let signature = payload.using_encoded(|p| acc.sign(p)); - runtime::UncheckedExtrinsic::new_signed( - call, - sp_runtime::AccountId32::from(acc.public()).into(), - polkadot_core_primitives::Signature::Sr25519(signature.clone()), - extra, - ) - .into() -} - #[cfg(feature = "westend-native")] fn westend_sign_call( call: westend_runtime::RuntimeCall, @@ -312,58 +228,6 @@ fn westend_sign_call( .into() } -#[cfg(feature = "kusama-native")] -fn kusama_sign_call( - call: kusama_runtime::RuntimeCall, - nonce: u32, - current_block: u64, - period: u64, - genesis: sp_core::H256, - acc: sp_core::sr25519::Pair, -) -> OpaqueExtrinsic { - use codec::Encode; - use kusama_runtime as runtime; - use sp_core::Pair; - - let extra: runtime::SignedExtra = ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(sp_runtime::generic::Era::mortal( - period, - current_block, - )), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(0), - ); - - let payload = runtime::SignedPayload::from_raw( - call.clone(), - extra.clone(), - ( - (), - runtime::VERSION.spec_version, - runtime::VERSION.transaction_version, - genesis, - genesis, - (), - (), - (), - ), - ); - - let signature = payload.using_encoded(|p| acc.sign(p)); - runtime::UncheckedExtrinsic::new_signed( - call, - sp_runtime::AccountId32::from(acc.public()).into(), - polkadot_core_primitives::Signature::Sr25519(signature.clone()), - extra, - ) - .into() -} - #[cfg(feature = "rococo-native")] fn rococo_sign_call( call: rococo_runtime::RuntimeCall, diff --git a/polkadot/node/service/src/chain_spec.rs b/polkadot/node/service/src/chain_spec.rs index 1e5aaa807b8178989700902f0a7a80b1b1fe1d76..7fd9ce61c95eeafb51dcec4dfaadcede46e3843c 100644 --- a/polkadot/node/service/src/chain_spec.rs +++ b/polkadot/node/service/src/chain_spec.rs @@ -18,22 +18,10 @@ use beefy_primitives::ecdsa_crypto::AuthorityId as BeefyId; use grandpa::AuthorityId as GrandpaId; -#[cfg(feature = "kusama-native")] -use kusama_runtime as kusama; -#[cfg(feature = "kusama-native")] -use kusama_runtime_constants::currency::UNITS as KSM; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", -))] +#[cfg(feature = "westend-native")] use pallet_staking::Forcing; use polkadot_primitives::{AccountId, AccountPublic, AssignmentId, ValidatorId}; -#[cfg(feature = "polkadot-native")] -use polkadot_runtime as polkadot; -#[cfg(feature = "polkadot-native")] -use polkadot_runtime_constants::currency::UNITS as DOT; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; use sp_consensus_babe::AuthorityId as BabeId; @@ -42,48 +30,27 @@ use rococo_runtime as rococo; #[cfg(feature = "rococo-native")] use rococo_runtime_constants::currency::UNITS as ROC; use sc_chain_spec::ChainSpecExtension; -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", - feature = "rococo-native" -))] +#[cfg(any(feature = "westend-native", feature = "rococo-native"))] use sc_chain_spec::ChainType; use serde::{Deserialize, Serialize}; use sp_core::{sr25519, Pair, Public}; use sp_runtime::traits::IdentifyAccount; -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", -))] +#[cfg(feature = "westend-native")] use sp_runtime::Perbill; -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", - feature = "rococo-native" -))] +#[cfg(any(feature = "westend-native", feature = "rococo-native"))] use telemetry::TelemetryEndpoints; #[cfg(feature = "westend-native")] use westend_runtime as westend; #[cfg(feature = "westend-native")] use westend_runtime_constants::currency::UNITS as WND; -#[cfg(feature = "kusama-native")] -const KUSAMA_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; #[cfg(feature = "westend-native")] const WESTEND_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; #[cfg(feature = "rococo-native")] const ROCOCO_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; #[cfg(feature = "rococo-native")] const VERSI_STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/"; -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", - feature = "rococo-native" -))] +#[cfg(any(feature = "westend-native", feature = "rococo-native"))] const DEFAULT_PROTOCOL_ID: &str = "dot"; /// Node `ChainSpec` extensions. @@ -103,25 +70,8 @@ pub struct Extensions { pub light_sync_state: sc_sync_state_rpc::LightSyncStateExtension, } -/// The `ChainSpec` parameterized for the polkadot runtime. -#[cfg(feature = "polkadot-native")] -pub type PolkadotChainSpec = service::GenericChainSpec; - -// Dummy chain spec, in case when we don't have the native runtime. -pub type DummyChainSpec = service::GenericChainSpec<(), Extensions>; - -// Dummy chain spec, but that is fine when we don't have the native runtime. -#[cfg(not(feature = "polkadot-native"))] -pub type PolkadotChainSpec = DummyChainSpec; - -/// The `ChainSpec` parameterized for the kusama runtime. -#[cfg(feature = "kusama-native")] -pub type KusamaChainSpec = service::GenericChainSpec; - -/// The `ChainSpec` parameterized for the kusama runtime. -// Dummy chain spec, but that is fine when we don't have the native runtime. -#[cfg(not(feature = "kusama-native"))] -pub type KusamaChainSpec = DummyChainSpec; +// Generic chain spec, in case when we don't have the native runtime. +pub type GenericChainSpec = service::GenericChainSpec<(), Extensions>; /// The `ChainSpec` parameterized for the westend runtime. #[cfg(feature = "westend-native")] @@ -130,7 +80,7 @@ pub type WestendChainSpec = service::GenericChainSpec Result { - PolkadotChainSpec::from_json_bytes(&include_bytes!("../chain-specs/polkadot.json")[..]) +pub fn polkadot_config() -> Result { + GenericChainSpec::from_json_bytes(&include_bytes!("../chain-specs/polkadot.json")[..]) } -pub fn kusama_config() -> Result { - KusamaChainSpec::from_json_bytes(&include_bytes!("../chain-specs/kusama.json")[..]) +pub fn kusama_config() -> Result { + GenericChainSpec::from_json_bytes(&include_bytes!("../chain-specs/kusama.json")[..]) } pub fn westend_config() -> Result { @@ -192,12 +142,7 @@ pub fn wococo_config() -> Result { } /// The default parachains host configuration. -#[cfg(any( - feature = "rococo-native", - feature = "kusama-native", - feature = "westend-native", - feature = "polkadot-native" -))] +#[cfg(any(feature = "rococo-native", feature = "westend-native",))] fn default_parachains_host_configuration( ) -> polkadot_runtime_parachains::configuration::HostConfiguration { @@ -236,57 +181,12 @@ fn default_parachains_host_configuration( } } -#[cfg(any( - feature = "rococo-native", - feature = "kusama-native", - feature = "westend-native", - feature = "polkadot-native" -))] +#[cfg(any(feature = "rococo-native", feature = "westend-native",))] #[test] fn default_parachains_host_configuration_is_consistent() { default_parachains_host_configuration().panic_if_not_consistent(); } -#[cfg(feature = "polkadot-native")] -fn polkadot_session_keys( - babe: BabeId, - grandpa: GrandpaId, - im_online: ImOnlineId, - para_validator: ValidatorId, - para_assignment: AssignmentId, - authority_discovery: AuthorityDiscoveryId, -) -> polkadot::SessionKeys { - polkadot::SessionKeys { - babe, - grandpa, - im_online, - para_validator, - para_assignment, - authority_discovery, - } -} - -#[cfg(feature = "kusama-native")] -fn kusama_session_keys( - babe: BabeId, - grandpa: GrandpaId, - im_online: ImOnlineId, - para_validator: ValidatorId, - para_assignment: AssignmentId, - authority_discovery: AuthorityDiscoveryId, - beefy: BeefyId, -) -> kusama::SessionKeys { - kusama::SessionKeys { - babe, - grandpa, - im_online, - para_validator, - para_assignment, - authority_discovery, - beefy, - } -} - #[cfg(feature = "westend-native")] fn westend_session_keys( babe: BabeId, @@ -539,214 +439,6 @@ fn westend_staging_testnet_config_genesis(wasm_binary: &[u8]) -> westend::Runtim } } -#[cfg(feature = "kusama-native")] -fn kusama_staging_testnet_config_genesis(wasm_binary: &[u8]) -> kusama::RuntimeGenesisConfig { - use hex_literal::hex; - use sp_core::crypto::UncheckedInto; - - // Following keys are used in genesis config for development chains. - // DO NOT use them in production chains as the secret seed is public. - // - // SECRET_SEED="explain impose opinion genius bar parrot erupt panther surround best expire - // album" subkey inspect -n kusama "$SECRET_SEED" - let endowed_accounts = vec![ - // FLN5cfhF7VCGJYefjPQJR2V6WwbfRmb9ozTwLAzBNeQQG6y - hex!["7a0fe424217ed176da7abf12e08198db0d0949298e1372c80a1930cb6dc21d3e"].into(), - ]; - - // SECRET=$SECRET_SEED ./scripts/prepare-test-net.sh 4 - let initial_authorities: Vec<( - AccountId, - AccountId, - BabeId, - GrandpaId, - ImOnlineId, - ValidatorId, - AssignmentId, - AuthorityDiscoveryId, - BeefyId, - )> = vec![ - ( - //5D5EsvSJf3KR3WHeZNG8rETdW6homig1cGHezspFt1P4o7sL - hex!["2ca4a9582244a3356a0d96e59d71f7e4d12aa88bca6d46f360ef11f6487cab1f"].into(), - //5Ev6RixvmK62UQE2PW19MPdLsYT4Nomwj85HKPdbnRECbDYh - hex!["7e237806f642b7f45f70ec45fbc41034516c8e5561bae2a62cd287129e1d0712"].into(), - //5GbjzK1uYVo6v1SaYhTeK3dbYy2GN9X4K5iwRkHEQ9eLS3We - hex!["c89cb7afc47ec0b5aac5824e5338a62959c92978167d3f841491836746e70b3d"] - .unchecked_into(), - //5GFz3YFW8QzEUsWhRjJzvDP7e5X5tPf5U12vUw32R8oJVgqb - hex!["b98b200021a608148f9817aeb553596b6968a5aa61b6d320c522f520ecc9cf9c"] - .unchecked_into(), - //5GzaFD8YsqnP5FYe5ijA9M4LQvzU9TPJmnBGdpuoqEvR1gQC - hex!["da0690438c0dd7a9aa26e03c9f1deaa58ba2b88d0bec0954b06478632164a401"] - .unchecked_into(), - //5CkZPtNy61PtbJpLqnjNFmbi1qukGkFdqFr5GKduSEthJ1cd - hex!["1e6554d35f6f17a37176c71801426204d6df400a1869114e4f00564b35d31150"] - .unchecked_into(), - //5CodnwweaYA1zB4QhdP4YVYFWnuZHY6W7zkN1NCRqJ9wZhap - hex!["20bddf09b1d0a2d93bafeb87fe19eb5bd59950c174f23a141a6d99736a5e700d"] - .unchecked_into(), - //5E7TSvNAP6QeJNeckdvYvADpHsx7v6aHXtGoQv5R2N1V3hEB - hex!["5a91b2546f1aac1c388eb0739c83e42d9972884d74360200ce32b7595bc65a04"] - .unchecked_into(), - //5GsoKeoM2HmjXPsdCua4oPu3Ms1Jgu4HbSnB81Lisa2tBFZp - hex!["02fd1e7e8455ab888ad054bbec7bc19409e6b1a5bb0300feefc6b58e60efae7e85"] - .unchecked_into(), - ), - ( - //5HMtKQuL2GQ7YvLBTh3vqFJEpkZW19sQh2X2mcUzAwBAe885 - hex!["ea478deab0ebfbeab7342febc236a9f1af5129ca0083fa25e6b0cf6a998d8354"].into(), - //5EFD5pLC3w5NFEcmQ6rGw9dUZ6fTSjWJemsvJZuaj7Qmq2WT - hex!["607b4e88129804eca8cd6fa26cbe2dd36667130e2a061050b08d9015871f4263"].into(), - //5DFztsnvC9hN85j5AP116atcnzFhAxnbzPodEp1AsYq1LYXu - hex!["34d949c39fae5801ba328ac6d0ddc76e469b7d5a4372a4a0d94f6aad6f9c1600"] - .unchecked_into(), - //5EZJNJ4j1eEEwCWusg7nYsZxTYBwoTH2drszxRqgMBTgNxMW - hex!["6e47830dcfc1f2b53a1b5db3f76702fc2760c1cc119119aceb00a57ec6658465"] - .unchecked_into(), - //5Dts3SrgDQMY9XCzKeQrxYSTh5MphPek994qkDCDk5c4neeF - hex!["50f6ef6326cd61ac500f167493e435f1204ce1d66ad18024bc5810d09673785e"] - .unchecked_into(), - //5DMKT99825TvA8F1yCQvE1ZcKTqg8T8Ad1KEjN6EuVpz4E6w - hex!["38e7fb2f6a1dcec73d93b07a0dc7cff1f9a9cc32cde8eb1e6ea1782f5316b431"] - .unchecked_into(), - //5EestuSehdMsWsBZ1hXCVo5YQiYiTPJwtV281x5fjUVtaqtP - hex!["72889a7b6ada28c3bd05a5a7298437f01d6d3270559768d16275efaf11864c0a"] - .unchecked_into(), - //5FNd5EabUbcReXEPwY9aASJMwSqyiic9w1Qt23YxNXj3dzbi - hex!["925f03f6211c68377987b0f78cd02aa882ad1fa9cc00c01fe6ce68e14c23340d"] - .unchecked_into(), - //5DxhuqfovpooTn8yH7WJGFjYw3pQxSEN9y9kvYUiGguHAj9D - hex!["030e77039e470ccdec7fe23dbc41c66f1c187ec8345e8919d3dc1250d975c3ce82"] - .unchecked_into(), - ), - ( - //5DAiYTKQ5KxwLncfNoTAH58dXBk2oDcQxtAXyDwMdKGLpGeY - hex!["30d203d942c1d056245b51e466a50b684f172a37c1cdde678f5346a0b3dbcd52"].into(), - //5Dq778qqNiAsjdF4qLVdkSBR8SftJKU35nyeBnkztRgniVhV - hex!["4e194bbafeec45647b2679e6b615b2a879d2e74fe706921930509ab3c9dbb22d"].into(), - //5E6iENoE1tXJUd7PkopQ8uqejg6xhPpqAnsVjS3hAQHWK1tm - hex!["5a0037b6bfc5e879ba5ef480ac29c59a12873854159686899082f41950ffd472"] - .unchecked_into(), - //5F8Dtgoc5dCaLAGYtaDqQUDg91fPQUynd497Fvhor8SYMdXp - hex!["87638aef8ab75db093150a6677c0919292ff66fc17f9f006a71fd0618415e164"] - .unchecked_into(), - //5EKsYx6Wj1Qg7LLc12U2YRjRUFmHa4Q3rNSoGZaP1ofS54km - hex!["6409c85a1125fa456b9dc6e85408a6d931aa8e04f48511c87fc147d1c103e902"] - .unchecked_into(), - //5H3UQy1NhCUUq3getmSEG8R1capY7Uy8JtKJz68UABmD9UxS - hex!["dc3cab0f94fa974cba826984f23dd4dc77ade20f25d935af5f07b85518da8044"] - .unchecked_into(), - //5DstCjokShCt9NppNnAcjg2nS4M5PKY3etn2BoFkZzMhQJ3w - hex!["50379866eb62e5c8aac31133efc4a1723e964a8e30c93c3ce2e7758bd03eb776"] - .unchecked_into(), - //5E4SCbSqUWKC4NVRCkMkJEnXCaVRiNQbSHL4upRB1ffd1Mk1 - hex!["5843c339c39d2c308bfb1841cd10beecfa157580492db05b66db8553e8d6512c"] - .unchecked_into(), - //5HNoMQ1PL3m7eBhp24FZxZUBtz4eh3AiwWq8i8jXLCRpJHsu - hex!["03c81d4e72cbdb96a7e6aad76830ae783b0b4650dc19703dde96866d8894dc921f"] - .unchecked_into(), - ), - ( - //5FNnjg8hXcPVLKASA69bPbooatacxcWNqkQAyXZfFiXi7T8r - hex!["927f8b12a0fa7185077353d9f6b4fe6bc6cd9682bd498642fa3801280909711a"].into(), - //5GipjBdL3rbex9qyxMinZpJYQbobbwk1ctbZp6B2mh3H25c6 - hex!["ce03638cd1e8496793b0540ba23370034511ea5d08837deb17f6c4d905b8d017"].into(), - //5GByn4uRpwmPe4i4MA4PjTQ8HXuycdue8HMWDhZ7vbU4WR9R - hex!["b67d3ed42ab1fcf3fcd7dee99bd6963bc22058ee22bcfddddb776492e85bd76e"] - .unchecked_into(), - //5GnZZ1rs7RE1jwPiyw1kts4JqaxnML5SdsWMuHV9TqCcuPWj - hex!["d0dd492b1a33d2f06a9aa7213e1aaa41d8820a6b56e95cd2462129b446574014"] - .unchecked_into(), - //5GKEKSAa3gbitHhvu5gm4f7q942azCVGDNhrw3hnsGPEMzyg - hex!["bc04e9764e23330b9f4e6922aa6437f87f3dd17b8590825e824724ae89d4ac51"] - .unchecked_into(), - //5H6QLnsfU7sAQ5ZACs9bPivsn9CXrqqwxhq4KKyoquZb5mVW - hex!["de78b26966c08357d66f7f56e7dcac7e4beb16aa0b74939290a42b3f5949bc36"] - .unchecked_into(), - //5FUUeYiAvFfXfB5yZLNkis2ZDy9T3CBLBPC6SwXFriGEjH5f - hex!["96d61fe92a50a79944ea93e3afc0a95a328773878e774cf8c8fbe8eba81cd95c"] - .unchecked_into(), - //5DLkWtgJahWG99cMcQxtftW9W14oduySyQi6hdhav7w3BiKq - hex!["38791c68ee472b94105c66cf150387979c49175062a687d1a1509119cfdc9e0c"] - .unchecked_into(), - //5Cjm1c3Jwt5jp6AaN2XfnncgZcswAmyfJn1buHEUaPauXAKK - hex!["025185a88886008267d27797fc74e34241e3aa8da767fafc9dd3ae5a59546802bb"] - .unchecked_into(), - ), - ]; - - const ENDOWMENT: u128 = 1_000_000 * KSM; - const STASH: u128 = 100 * KSM; - - kusama::RuntimeGenesisConfig { - system: kusama::SystemConfig { code: wasm_binary.to_vec(), ..Default::default() }, - balances: kusama::BalancesConfig { - balances: endowed_accounts - .iter() - .map(|k: &AccountId| (k.clone(), ENDOWMENT)) - .chain(initial_authorities.iter().map(|x| (x.0.clone(), STASH))) - .collect(), - }, - beefy: Default::default(), - indices: kusama::IndicesConfig { indices: vec![] }, - session: kusama::SessionConfig { - keys: initial_authorities - .iter() - .map(|x| { - ( - x.0.clone(), - x.0.clone(), - kusama_session_keys( - x.2.clone(), - x.3.clone(), - x.4.clone(), - x.5.clone(), - x.6.clone(), - x.7.clone(), - x.8.clone(), - ), - ) - }) - .collect::>(), - }, - staking: kusama::StakingConfig { - validator_count: 50, - minimum_validator_count: 4, - stakers: initial_authorities - .iter() - .map(|x| (x.0.clone(), x.0.clone(), STASH, kusama::StakerStatus::Validator)) - .collect(), - invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), - force_era: Forcing::ForceNone, - slash_reward_fraction: Perbill::from_percent(10), - ..Default::default() - }, - babe: kusama::BabeConfig { - authorities: Default::default(), - epoch_config: Some(kusama::BABE_GENESIS_EPOCH_CONFIG), - ..Default::default() - }, - grandpa: Default::default(), - im_online: Default::default(), - authority_discovery: kusama::AuthorityDiscoveryConfig { - keys: vec![], - ..Default::default() - }, - claims: kusama::ClaimsConfig { claims: vec![], vesting: vec![] }, - vesting: kusama::VestingConfig { vesting: vec![] }, - treasury: Default::default(), - hrmp: Default::default(), - configuration: kusama::ConfigurationConfig { - config: default_parachains_host_configuration(), - }, - paras: Default::default(), - xcm_pallet: Default::default(), - nomination_pools: Default::default(), - nis_counterpart_balances: Default::default(), - } -} - #[cfg(feature = "rococo-native")] fn rococo_staging_testnet_config_genesis( wasm_binary: &[u8], @@ -1062,39 +754,6 @@ fn rococo_staging_testnet_config_genesis( } } -/// Returns the properties for the [`PolkadotChainSpec`]. -pub fn polkadot_chain_spec_properties() -> serde_json::map::Map { - serde_json::json!({ - "tokenDecimals": 10, - }) - .as_object() - .expect("Map given; qed") - .clone() -} - -/// Staging testnet config. -#[cfg(feature = "kusama-native")] -pub fn kusama_staging_testnet_config() -> Result { - let wasm_binary = kusama::WASM_BINARY.ok_or("Kusama development wasm not available")?; - let boot_nodes = vec![]; - - Ok(KusamaChainSpec::from_genesis( - "Kusama Staging Testnet", - "kusama_staging_testnet", - ChainType::Live, - move || kusama_staging_testnet_config_genesis(wasm_binary), - boot_nodes, - Some( - TelemetryEndpoints::new(vec![(KUSAMA_STAGING_TELEMETRY_URL.to_string(), 0)]) - .expect("Kusama Staging telemetry url is valid; qed"), - ), - Some(DEFAULT_PROTOCOL_ID), - None, - None, - Default::default(), - )) -} - /// Westend staging testnet config. #[cfg(feature = "westend-native")] pub fn westend_staging_testnet_config() -> Result { @@ -1239,12 +898,7 @@ pub fn get_authority_keys_from_seed_no_beefy( ) } -#[cfg(any( - feature = "polkadot-native", - feature = "kusama-native", - feature = "westend-native", - feature = "rococo-native" -))] +#[cfg(any(feature = "westend-native", feature = "rococo-native"))] fn testnet_accounts() -> Vec { vec![ get_account_id_from_seed::("Alice"), @@ -1262,176 +916,6 @@ fn testnet_accounts() -> Vec { ] } -/// Helper function to create polkadot `RuntimeGenesisConfig` for testing -#[cfg(feature = "polkadot-native")] -pub fn polkadot_testnet_genesis( - wasm_binary: &[u8], - initial_authorities: Vec<( - AccountId, - AccountId, - BabeId, - GrandpaId, - ImOnlineId, - ValidatorId, - AssignmentId, - AuthorityDiscoveryId, - )>, - _root_key: AccountId, - endowed_accounts: Option>, -) -> polkadot::RuntimeGenesisConfig { - let endowed_accounts: Vec = endowed_accounts.unwrap_or_else(testnet_accounts); - - const ENDOWMENT: u128 = 1_000_000 * DOT; - const STASH: u128 = 100 * DOT; - - polkadot::RuntimeGenesisConfig { - system: polkadot::SystemConfig { code: wasm_binary.to_vec(), ..Default::default() }, - indices: polkadot::IndicesConfig { indices: vec![] }, - balances: polkadot::BalancesConfig { - balances: endowed_accounts.iter().map(|k| (k.clone(), ENDOWMENT)).collect(), - }, - session: polkadot::SessionConfig { - keys: initial_authorities - .iter() - .map(|x| { - ( - x.0.clone(), - x.0.clone(), - polkadot_session_keys( - x.2.clone(), - x.3.clone(), - x.4.clone(), - x.5.clone(), - x.6.clone(), - x.7.clone(), - ), - ) - }) - .collect::>(), - }, - staking: polkadot::StakingConfig { - minimum_validator_count: 1, - validator_count: initial_authorities.len() as u32, - stakers: initial_authorities - .iter() - .map(|x| (x.0.clone(), x.0.clone(), STASH, polkadot::StakerStatus::Validator)) - .collect(), - invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), - force_era: Forcing::NotForcing, - slash_reward_fraction: Perbill::from_percent(10), - ..Default::default() - }, - babe: polkadot::BabeConfig { - authorities: Default::default(), - epoch_config: Some(polkadot::BABE_GENESIS_EPOCH_CONFIG), - ..Default::default() - }, - grandpa: Default::default(), - im_online: Default::default(), - authority_discovery: polkadot::AuthorityDiscoveryConfig { - keys: vec![], - ..Default::default() - }, - claims: polkadot::ClaimsConfig { claims: vec![], vesting: vec![] }, - vesting: polkadot::VestingConfig { vesting: vec![] }, - treasury: Default::default(), - hrmp: Default::default(), - configuration: polkadot::ConfigurationConfig { - config: default_parachains_host_configuration(), - }, - paras: Default::default(), - xcm_pallet: Default::default(), - nomination_pools: Default::default(), - } -} - -/// Helper function to create kusama `RuntimeGenesisConfig` for testing -#[cfg(feature = "kusama-native")] -pub fn kusama_testnet_genesis( - wasm_binary: &[u8], - initial_authorities: Vec<( - AccountId, - AccountId, - BabeId, - GrandpaId, - ImOnlineId, - ValidatorId, - AssignmentId, - AuthorityDiscoveryId, - BeefyId, - )>, - _root_key: AccountId, - endowed_accounts: Option>, -) -> kusama::RuntimeGenesisConfig { - let endowed_accounts: Vec = endowed_accounts.unwrap_or_else(testnet_accounts); - - const ENDOWMENT: u128 = 1_000_000 * KSM; - const STASH: u128 = 100 * KSM; - - kusama::RuntimeGenesisConfig { - system: kusama::SystemConfig { code: wasm_binary.to_vec(), ..Default::default() }, - indices: kusama::IndicesConfig { indices: vec![] }, - balances: kusama::BalancesConfig { - balances: endowed_accounts.iter().map(|k| (k.clone(), ENDOWMENT)).collect(), - }, - beefy: Default::default(), - session: kusama::SessionConfig { - keys: initial_authorities - .iter() - .map(|x| { - ( - x.0.clone(), - x.0.clone(), - kusama_session_keys( - x.2.clone(), - x.3.clone(), - x.4.clone(), - x.5.clone(), - x.6.clone(), - x.7.clone(), - x.8.clone(), - ), - ) - }) - .collect::>(), - }, - staking: kusama::StakingConfig { - minimum_validator_count: 1, - validator_count: initial_authorities.len() as u32, - stakers: initial_authorities - .iter() - .map(|x| (x.0.clone(), x.0.clone(), STASH, kusama::StakerStatus::Validator)) - .collect(), - invulnerables: initial_authorities.iter().map(|x| x.0.clone()).collect(), - force_era: Forcing::NotForcing, - slash_reward_fraction: Perbill::from_percent(10), - ..Default::default() - }, - babe: kusama::BabeConfig { - authorities: Default::default(), - epoch_config: Some(kusama::BABE_GENESIS_EPOCH_CONFIG), - ..Default::default() - }, - grandpa: Default::default(), - im_online: Default::default(), - authority_discovery: kusama::AuthorityDiscoveryConfig { - keys: vec![], - ..Default::default() - }, - claims: kusama::ClaimsConfig { claims: vec![], vesting: vec![] }, - vesting: kusama::VestingConfig { vesting: vec![] }, - treasury: Default::default(), - hrmp: Default::default(), - configuration: kusama::ConfigurationConfig { - config: default_parachains_host_configuration(), - }, - paras: Default::default(), - xcm_pallet: Default::default(), - nomination_pools: Default::default(), - nis_counterpart_balances: Default::default(), - } -} - /// Helper function to create westend `RuntimeGenesisConfig` for testing #[cfg(feature = "westend-native")] pub fn westend_testnet_genesis( @@ -1612,26 +1096,6 @@ pub fn rococo_testnet_genesis( } } -#[cfg(feature = "polkadot-native")] -fn polkadot_development_config_genesis(wasm_binary: &[u8]) -> polkadot::RuntimeGenesisConfig { - polkadot_testnet_genesis( - wasm_binary, - vec![get_authority_keys_from_seed_no_beefy("Alice")], - get_account_id_from_seed::("Alice"), - None, - ) -} - -#[cfg(feature = "kusama-native")] -fn kusama_development_config_genesis(wasm_binary: &[u8]) -> kusama::RuntimeGenesisConfig { - kusama_testnet_genesis( - wasm_binary, - vec![get_authority_keys_from_seed("Alice")], - get_account_id_from_seed::("Alice"), - None, - ) -} - #[cfg(feature = "westend-native")] fn westend_development_config_genesis(wasm_binary: &[u8]) -> westend::RuntimeGenesisConfig { westend_testnet_genesis( @@ -1652,44 +1116,6 @@ fn rococo_development_config_genesis(wasm_binary: &[u8]) -> rococo_runtime::Runt ) } -/// Polkadot development config (single validator Alice) -#[cfg(feature = "polkadot-native")] -pub fn polkadot_development_config() -> Result { - let wasm_binary = polkadot::WASM_BINARY.ok_or("Polkadot development wasm not available")?; - - Ok(PolkadotChainSpec::from_genesis( - "Development", - "polkadot_dev", - ChainType::Development, - move || polkadot_development_config_genesis(wasm_binary), - vec![], - None, - Some(DEFAULT_PROTOCOL_ID), - None, - Some(polkadot_chain_spec_properties()), - Default::default(), - )) -} - -/// Kusama development config (single validator Alice) -#[cfg(feature = "kusama-native")] -pub fn kusama_development_config() -> Result { - let wasm_binary = kusama::WASM_BINARY.ok_or("Kusama development wasm not available")?; - - Ok(KusamaChainSpec::from_genesis( - "Development", - "kusama_dev", - ChainType::Development, - move || kusama_development_config_genesis(wasm_binary), - vec![], - None, - Some(DEFAULT_PROTOCOL_ID), - None, - None, - Default::default(), - )) -} - /// Westend development config (single validator Alice) #[cfg(feature = "westend-native")] pub fn westend_development_config() -> Result { @@ -1779,67 +1205,6 @@ pub fn wococo_development_config() -> Result { )) } -#[cfg(feature = "polkadot-native")] -fn polkadot_local_testnet_genesis(wasm_binary: &[u8]) -> polkadot::RuntimeGenesisConfig { - polkadot_testnet_genesis( - wasm_binary, - vec![ - get_authority_keys_from_seed_no_beefy("Alice"), - get_authority_keys_from_seed_no_beefy("Bob"), - ], - get_account_id_from_seed::("Alice"), - None, - ) -} - -/// Polkadot local testnet config (multivalidator Alice + Bob) -#[cfg(feature = "polkadot-native")] -pub fn polkadot_local_testnet_config() -> Result { - let wasm_binary = polkadot::WASM_BINARY.ok_or("Polkadot development wasm not available")?; - - Ok(PolkadotChainSpec::from_genesis( - "Local Testnet", - "local_testnet", - ChainType::Local, - move || polkadot_local_testnet_genesis(wasm_binary), - vec![], - None, - Some(DEFAULT_PROTOCOL_ID), - None, - Some(polkadot_chain_spec_properties()), - Default::default(), - )) -} - -#[cfg(feature = "kusama-native")] -fn kusama_local_testnet_genesis(wasm_binary: &[u8]) -> kusama::RuntimeGenesisConfig { - kusama_testnet_genesis( - wasm_binary, - vec![get_authority_keys_from_seed("Alice"), get_authority_keys_from_seed("Bob")], - get_account_id_from_seed::("Alice"), - None, - ) -} - -/// Kusama local testnet config (multivalidator Alice + Bob) -#[cfg(feature = "kusama-native")] -pub fn kusama_local_testnet_config() -> Result { - let wasm_binary = kusama::WASM_BINARY.ok_or("Kusama development wasm not available")?; - - Ok(KusamaChainSpec::from_genesis( - "Kusama Local Testnet", - "kusama_local_testnet", - ChainType::Local, - move || kusama_local_testnet_genesis(wasm_binary), - vec![], - None, - Some(DEFAULT_PROTOCOL_ID), - None, - None, - Default::default(), - )) -} - #[cfg(feature = "westend-native")] fn westend_local_testnet_genesis(wasm_binary: &[u8]) -> westend::RuntimeGenesisConfig { westend_testnet_genesis( diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 95c887947c98f577ecc5bf144f90ccac1af5ebbf..5286631fbbb5c495ba66cbaa80ad6a583ac9f852 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -84,7 +84,7 @@ use telemetry::TelemetryWorker; #[cfg(feature = "full-node")] use telemetry::{Telemetry, TelemetryWorkerHandle}; -pub use chain_spec::{KusamaChainSpec, PolkadotChainSpec, RococoChainSpec, WestendChainSpec}; +pub use chain_spec::{GenericChainSpec, RococoChainSpec, WestendChainSpec}; pub use consensus_common::{Proposal, SelectChain}; use frame_benchmarking_cli::SUBSTRATE_REFERENCE_HARDWARE; use mmr_gadget::MmrGadget; @@ -104,10 +104,6 @@ pub use sp_runtime::{ traits::{self as runtime_traits, BlakeTwo256, Block as BlockT, Header as HeaderT, NumberFor}, }; -#[cfg(feature = "kusama-native")] -pub use {kusama_runtime, kusama_runtime_constants}; -#[cfg(feature = "polkadot-native")] -pub use {polkadot_runtime, polkadot_runtime_constants}; #[cfg(feature = "rococo-native")] pub use {rococo_runtime, rococo_runtime_constants}; #[cfg(feature = "westend-native")] @@ -732,7 +728,7 @@ pub fn new_full( }: NewFullParams, ) -> Result { use polkadot_node_network_protocol::request_response::IncomingRequest; - use sc_network_common::sync::warp::WarpSyncParams; + use sc_network_sync::warp::WarpSyncParams; let is_offchain_indexing_enabled = config.offchain_worker.indexing_enabled; let role = config.role.clone(); @@ -894,6 +890,7 @@ pub fn new_full( import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; if config.offchain_worker.enabled { diff --git a/polkadot/node/subsystem-test-helpers/Cargo.toml b/polkadot/node/subsystem-test-helpers/Cargo.toml index 98b0d182a611989387eea5de5f0e56b9213b89d7..9087ca11f5d22ee5307fbc3877c88e1be5a72e27 100644 --- a/polkadot/node/subsystem-test-helpers/Cargo.toml +++ b/polkadot/node/subsystem-test-helpers/Cargo.toml @@ -14,11 +14,10 @@ parking_lot = "0.12.0" polkadot-node-subsystem = { path = "../subsystem" } polkadot-node-subsystem-util = { path = "../subsystem-util" } polkadot-primitives = { path = "../../primitives" } +sc-client-api = { path = "../../../substrate/client/api" } +sc-utils = { path = "../../../substrate/client/utils" } sp-core = { path = "../../../substrate/primitives/core" } sp-keystore = { path = "../../../substrate/primitives/keystore" } sc-keystore = { path = "../../../substrate/client/keystore" } sp-keyring = { path = "../../../substrate/primitives/keyring" } sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } - -[dev-dependencies] -polkadot-overseer = { path = "../overseer" } diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index 6f0c3016c7aeed7b9cddba123c7cd22013a50304..3f92513498c4129f418690946c2a2e2ac85605cc 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -20,7 +20,7 @@ use polkadot_node_subsystem::{ messages::AllMessages, overseer, FromOrchestra, OverseerSignal, SpawnGlue, SpawnedSubsystem, - SubsystemError, SubsystemResult, + SubsystemError, SubsystemResult, TrySendError, }; use polkadot_node_subsystem_util::TimeoutExt; @@ -160,6 +160,14 @@ where self.tx.send(msg.into()).await.expect("test overseer no longer live"); } + fn try_send_message( + &mut self, + msg: OutgoingMessage, + ) -> Result<(), TrySendError> { + self.tx.unbounded_send(msg.into()).expect("test overseer no longer live"); + Ok(()) + } + async fn send_messages(&mut self, msgs: I) where I: IntoIterator + Send, @@ -435,42 +443,6 @@ impl Future for Yield { #[cfg(test)] mod tests { use super::*; - use futures::executor::block_on; - use polkadot_node_subsystem::messages::CollatorProtocolMessage; - use polkadot_overseer::{dummy::dummy_overseer_builder, Handle, HeadSupportsParachains}; - use polkadot_primitives::Hash; - use sp_core::traits::SpawnNamed; - - struct AlwaysSupportsParachains; - - #[async_trait::async_trait] - impl HeadSupportsParachains for AlwaysSupportsParachains { - async fn head_supports_parachains(&self, _head: &Hash) -> bool { - true - } - } - - #[test] - fn forward_subsystem_works() { - let spawner = sp_core::testing::TaskExecutor::new(); - let (tx, rx) = mpsc::channel(2); - let (overseer, handle) = - dummy_overseer_builder(spawner.clone(), AlwaysSupportsParachains, None) - .unwrap() - .replace_collator_protocol(|_| ForwardSubsystem(tx)) - .build() - .unwrap(); - - let mut handle = Handle::new(handle); - - spawner.spawn("overseer", None, overseer.run().then(|_| async { () }).boxed()); - - block_on(handle.send_msg_anon(CollatorProtocolMessage::CollateOn(Default::default()))); - assert!(matches!( - block_on(rx.into_future()).0.unwrap(), - CollatorProtocolMessage::CollateOn(_) - )); - } #[test] fn macro_arbitrary_order() { diff --git a/polkadot/node/subsystem-test-helpers/src/mock.rs b/polkadot/node/subsystem-test-helpers/src/mock.rs index 04695983d1d533a6bb57ed12dd60254e0a56d5cb..35d74e27c9c1c237e6cb71382bae9c1cd730ccd6 100644 --- a/polkadot/node/subsystem-test-helpers/src/mock.rs +++ b/polkadot/node/subsystem-test-helpers/src/mock.rs @@ -16,12 +16,15 @@ use std::sync::Arc; +use polkadot_node_subsystem::{jaeger, ActivatedLeaf, LeafStatus}; +use sc_client_api::UnpinHandle; use sc_keystore::LocalKeystore; +use sc_utils::mpsc::tracing_unbounded; use sp_application_crypto::AppCrypto; use sp_keyring::Sr25519Keyring; use sp_keystore::{Keystore, KeystorePtr}; -use polkadot_primitives::{AuthorityDiscoveryId, ValidatorId}; +use polkadot_primitives::{AuthorityDiscoveryId, Block, BlockNumber, Hash, ValidatorId}; /// Get mock keystore with `Ferdie` key. pub fn make_ferdie_keystore() -> KeystorePtr { @@ -40,3 +43,20 @@ pub fn make_ferdie_keystore() -> KeystorePtr { .expect("Insert key into keystore"); keystore } + +/// Create a meaningless unpin handle for a block. +pub fn dummy_unpin_handle(block: Hash) -> UnpinHandle { + let (dummy_sink, _) = tracing_unbounded("Expect Chaos", 69); + UnpinHandle::new(block, dummy_sink) +} + +/// Create a new leaf with the given hash and number. +pub fn new_leaf(hash: Hash, number: BlockNumber) -> ActivatedLeaf { + ActivatedLeaf { + hash, + number, + status: LeafStatus::Fresh, + unpin_handle: dummy_unpin_handle(hash), + span: Arc::new(jaeger::Span::Disabled), + } +} diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index 7ca3a0faf31abdf0e77760cdbfa040130baf3109..a1c00cb0652e2538c590d1c3892eed8a168e1014 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -14,13 +14,14 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } -orchestra = "0.0.5" +orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } sc-network = { path = "../../../substrate/client/network" } sp-api = { path = "../../../substrate/primitives/api" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } sp-authority-discovery = { path = "../../../substrate/primitives/authority-discovery" } +sc-client-api = { path = "../../../substrate/client/api" } sc-transaction-pool-api = { path = "../../../substrate/client/transaction-pool/api" } smallvec = "1.8.0" substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" } -thiserror = "1.0.31" +thiserror = "1.0.48" async-trait = "0.1.57" diff --git a/polkadot/node/subsystem-types/src/lib.rs b/polkadot/node/subsystem-types/src/lib.rs index f438a09592c1d22ad310c03b4b8386cb41b5a612..02651ace1e5b32829804c62cd2d192c9a0e35fd3 100644 --- a/polkadot/node/subsystem-types/src/lib.rs +++ b/polkadot/node/subsystem-types/src/lib.rs @@ -22,10 +22,19 @@ #![warn(missing_docs)] +use smallvec::SmallVec; use std::{fmt, sync::Arc}; -pub use polkadot_primitives::{BlockNumber, Hash}; -use smallvec::SmallVec; +pub use polkadot_primitives::{Block, BlockNumber, Hash}; + +/// Keeps the state of a specific block pinned in memory while the handle is alive. +/// +/// The handle is reference counted and once the last is dropped, the +/// block is unpinned. +/// +/// This is useful for runtime API calls to blocks that are +/// racing against finality, e.g. for slashing purposes. +pub type UnpinHandle = sc_client_api::UnpinHandle; pub mod errors; pub mod messages; @@ -80,6 +89,8 @@ pub struct ActivatedLeaf { pub number: BlockNumber, /// The status of the leaf. pub status: LeafStatus, + /// A handle to unpin the block on drop. + pub unpin_handle: UnpinHandle, /// An associated [`jaeger::Span`]. /// /// NOTE: Each span should only be kept active as long as the leaf is considered active and diff --git a/polkadot/node/subsystem-util/Cargo.toml b/polkadot/node/subsystem-util/Cargo.toml index d243a90a2bd628590edf7fa945b5796ee5f23747..d9364e2c2c0f15158b861ef0e8c44121a7e2d650 100644 --- a/polkadot/node/subsystem-util/Cargo.toml +++ b/polkadot/node/subsystem-util/Cargo.toml @@ -15,24 +15,26 @@ parity-scale-codec = { version = "3.6.1", default-features = false, features = [ parking_lot = "0.11.2" pin-project = "1.0.9" rand = "0.8.5" -thiserror = "1.0.31" +thiserror = "1.0.48" fatality = "0.0.6" gum = { package = "tracing-gum", path = "../gum" } derive_more = "0.99.17" schnellru = "0.2.1" polkadot-node-subsystem = { path = "../subsystem" } +polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-jaeger = { path = "../jaeger" } polkadot-node-metrics = { path = "../metrics" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-node-primitives = { path = "../primitives" } polkadot-overseer = { path = "../overseer" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } sp-core = { path = "../../../substrate/primitives/core" } sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } sp-keystore = { path = "../../../substrate/primitives/keystore" } +sc-client-api = { path = "../../../substrate/client/api" } kvdb = "0.13.0" parity-db = { version = "0.4.8"} diff --git a/polkadot/node/subsystem-util/src/runtime/mod.rs b/polkadot/node/subsystem-util/src/runtime/mod.rs index fc767faa763c475be46773f27f0be9bf2f39e3e6..c078b17d217517c3cdb7fcccbfd590adf128bb3f 100644 --- a/polkadot/node/subsystem-util/src/runtime/mod.rs +++ b/polkadot/node/subsystem-util/src/runtime/mod.rs @@ -28,6 +28,7 @@ use polkadot_node_subsystem::{ messages::{RuntimeApiMessage, RuntimeApiRequest}, overseer, SubsystemSender, }; +use polkadot_node_subsystem_types::UnpinHandle; use polkadot_primitives::{ vstaging, CandidateEvent, CandidateHash, CoreState, EncodeAs, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, IndexedVec, OccupiedCore, ScrapedOnChainVotes, SessionIndex, @@ -75,6 +76,10 @@ pub struct RuntimeInfo { /// Look up cached sessions by `SessionIndex`. session_info_cache: LruMap, + /// Unpin handle of *some* block in the session. + /// Only blocks pinned explicitly by `pin_block` are stored here. + pinned_blocks: LruMap, + /// Key store for determining whether we are a validator and what `ValidatorIndex` we have. keystore: Option, } @@ -120,6 +125,7 @@ impl RuntimeInfo { Self { session_index_cache: LruMap::new(ByLength::new(cfg.session_cache_lru_size.max(10))), session_info_cache: LruMap::new(ByLength::new(cfg.session_cache_lru_size)), + pinned_blocks: LruMap::new(ByLength::new(cfg.session_cache_lru_size)), keystore: cfg.keystore, } } @@ -145,6 +151,17 @@ impl RuntimeInfo { } } + /// Pin a given block in the given session if none are pinned in that session. + /// Unpinning will happen automatically when LRU cache grows over the limit. + pub fn pin_block(&mut self, session_index: SessionIndex, unpin_handle: UnpinHandle) { + self.pinned_blocks.get_or_insert(session_index, || unpin_handle); + } + + /// Get the hash of a pinned block for the given session index, if any. + pub fn get_block_in_session(&self, session_index: SessionIndex) -> Option { + self.pinned_blocks.peek(&session_index).map(|h| h.hash()) + } + /// Get `ExtendedSessionInfo` by relay parent hash. pub async fn get_session_info<'a, Sender>( &'a mut self, diff --git a/polkadot/node/test/performance-test/Cargo.toml b/polkadot/node/test/performance-test/Cargo.toml index 98b67615a6f253e69e547568f7a1606fb4678682..5747ac88b1e4f7dc95f80706c4dd749ecfd78f8a 100644 --- a/polkadot/node/test/performance-test/Cargo.toml +++ b/polkadot/node/test/performance-test/Cargo.toml @@ -7,7 +7,7 @@ edition.workspace = true license.workspace = true [dependencies] -thiserror = "1.0.31" +thiserror = "1.0.48" quote = "1.0.28" env_logger = "0.9" log = "0.4" diff --git a/polkadot/node/test/service/Cargo.toml b/polkadot/node/test/service/Cargo.toml index d730a601d5a765c7d9a90c97b5d4fbb1005d727f..1924418a48565e12fa5f1b628943f604ba9c4116 100644 --- a/polkadot/node/test/service/Cargo.toml +++ b/polkadot/node/test/service/Cargo.toml @@ -11,6 +11,7 @@ futures = "0.3.21" hex = "0.4.3" gum = { package = "tracing-gum", path = "../../gum" } rand = "0.8.5" +serde_json = "1.0.106" tempfile = "3.2.0" tokio = "1.24.2" @@ -58,7 +59,7 @@ substrate-test-client = { path = "../../../../substrate/test-utils/client" } [dev-dependencies] pallet-balances = { path = "../../../../substrate/frame/balances", default-features = false } -serde_json = "1.0.96" +serde_json = "1.0.107" substrate-test-utils = { path = "../../../../substrate/test-utils" } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/node/test/service/README.md b/polkadot/node/test/service/README.md index 2fdee46a7f932010eeab1c30a7766dfb4049fbc3..b0240588c44cef277067dd1513a213fc3ed02d92 100644 --- a/polkadot/node/test/service/README.md +++ b/polkadot/node/test/service/README.md @@ -1,4 +1,4 @@ -# polkadot-test-service +# `polkadot-test-service` ## Testing diff --git a/polkadot/node/test/service/src/chain_spec.rs b/polkadot/node/test/service/src/chain_spec.rs index 9aadd7d203c0d6ba6197a0b7c55c20172dcd6deb..bedb1250b2a9853a7bcf96867b717cee92f272c5 100644 --- a/polkadot/node/test/service/src/chain_spec.rs +++ b/polkadot/node/test/service/src/chain_spec.rs @@ -20,9 +20,7 @@ use babe_primitives::AuthorityId as BabeId; use grandpa::AuthorityId as GrandpaId; use pallet_staking::Forcing; use polkadot_primitives::{AccountId, AssignmentId, ValidatorId, MAX_CODE_SIZE, MAX_POV_SIZE}; -use polkadot_service::chain_spec::{ - get_account_id_from_seed, get_from_seed, polkadot_chain_spec_properties, Extensions, -}; +use polkadot_service::chain_spec::{get_account_id_from_seed, get_from_seed, Extensions}; use polkadot_test_runtime::BABE_GENESIS_EPOCH_CONFIG; use sc_chain_spec::{ChainSpec, ChainType}; use sp_authority_discovery::AuthorityId as AuthorityDiscoveryId; @@ -36,6 +34,16 @@ const DEFAULT_PROTOCOL_ID: &str = "dot"; pub type PolkadotChainSpec = sc_service::GenericChainSpec; +/// Returns the properties for the [`PolkadotChainSpec`]. +pub fn polkadot_chain_spec_properties() -> serde_json::map::Map { + serde_json::json!({ + "tokenDecimals": 10, + }) + .as_object() + .expect("Map given; qed") + .clone() +} + /// Local testnet config (multivalidator Alice + Bob) pub fn polkadot_local_testnet_config() -> PolkadotChainSpec { PolkadotChainSpec::from_genesis( diff --git a/polkadot/node/zombienet-backchannel/Cargo.toml b/polkadot/node/zombienet-backchannel/Cargo.toml index 3d59a4a3cddc1a4def3ea788d435a67ab80f909a..9bf56b550bbcd7803806bf37d2a40b9ac5a92e01 100644 --- a/polkadot/node/zombienet-backchannel/Cargo.toml +++ b/polkadot/node/zombienet-backchannel/Cargo.toml @@ -16,7 +16,7 @@ futures-util = "0.3.23" lazy_static = "1.4.0" parity-scale-codec = { version = "3.6.1", features = ["derive"] } reqwest = { version = "0.11", features = ["rustls-tls"], default-features = false } -thiserror = "1.0.31" +thiserror = "1.0.48" gum = { package = "tracing-gum", path = "../gum" } serde = { version = "1.0", features = ["derive"] } serde_json = "1" diff --git a/polkadot/parachain/README.adoc b/polkadot/parachain/README.adoc deleted file mode 100644 index 8650919e64ec45b9c3ceb5aee36bb3112c06a678..0000000000000000000000000000000000000000 --- a/polkadot/parachain/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot Parachain - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/parachain/src/primitives.rs b/polkadot/parachain/src/primitives.rs index 5cea9d3bbf4e827a632a103132626a39c623c5c2..5f77810f5c235adffc5bde8c377b1058ef7a8644 100644 --- a/polkadot/parachain/src/primitives.rs +++ b/polkadot/parachain/src/primitives.rs @@ -199,13 +199,11 @@ impl From for Id { } } -const USER_INDEX_START: u32 = 1000; +// System parachain ID is considered `< 2000`. +const SYSTEM_INDEX_END: u32 = 1999; const PUBLIC_INDEX_START: u32 = 2000; -/// The ID of the first user (non-system) parachain. -pub const LOWEST_USER_ID: Id = Id(USER_INDEX_START); - -/// The ID of the first publicly registerable parachain. +/// The ID of the first publicly registrable parachain. pub const LOWEST_PUBLIC_ID: Id = Id(PUBLIC_INDEX_START); impl Id { @@ -223,7 +221,7 @@ pub trait IsSystem { impl IsSystem for Id { fn is_system(&self) -> bool { - self.0 < USER_INDEX_START + self.0 <= SYSTEM_INDEX_END } } diff --git a/polkadot/parachain/test-parachains/README.md b/polkadot/parachain/test-parachains/README.md index 2c708bd543185744506bf559baa18d83003bb7d3..e679d40591b393d7bedfdf4492a985a00da130be 100644 --- a/polkadot/parachain/test-parachains/README.md +++ b/polkadot/parachain/test-parachains/README.md @@ -1,3 +1,4 @@ # Test Parachains -Each parachain consists of three parts: a `#![no_std]` library with the main execution logic, a WASM crate which wraps this logic, and a collator node. +Each parachain consists of three parts: a `#![no_std]` library with the main execution logic, a WASM crate which wraps +this logic, and a collator node. diff --git a/polkadot/parachain/test-parachains/adder/collator/Cargo.toml b/polkadot/parachain/test-parachains/adder/collator/Cargo.toml index 7079ab73270482ccd053933d5aeb1cf1864db9d0..fcbba9bbe21220362d8428f3d4e988b298003112 100644 --- a/polkadot/parachain/test-parachains/adder/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/adder/collator/Cargo.toml @@ -11,14 +11,9 @@ license.workspace = true name = "adder-collator" path = "src/main.rs" -[[bin]] -name = "adder_collator_puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" log = "0.4.17" @@ -33,25 +28,17 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" } sc-cli = { path = "../../../../../substrate/client/cli" } sp-core = { path = "../../../../../substrate/primitives/core" } sc-service = { path = "../../../../../substrate/client/service" } -# This one is tricky. Even though it is not used directly by the collator, we still need it for the -# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be -# a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true } [dev-dependencies] polkadot-parachain-primitives = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } +polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] } substrate-test-utils = { path = "../../../../../substrate/test-utils" } sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } -# For the puppet worker, depend on ourselves with the test-utils feature. -test-parachain-adder-collator = { path = "", features = ["test-utils"] } tokio = { version = "1.24.2", features = ["macros"] } [features] network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] -# This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. -test-utils = [ "polkadot-node-core-pvf/test-utils" ] diff --git a/polkadot/parachain/test-parachains/adder/collator/README.md b/polkadot/parachain/test-parachains/adder/collator/README.md index a1378544c386f9fb3edb1f3be3e9675a4c61acc7..dd737627c9ad9ca443131b91707b23ad9ccb84ee 100644 --- a/polkadot/parachain/test-parachains/adder/collator/README.md +++ b/polkadot/parachain/test-parachains/adder/collator/README.md @@ -19,7 +19,7 @@ Next start the collator that will collate for the adder parachain: cargo run --release -p test-parachain-adder-collator -- --tmp --chain rococo-local --port 50553 ``` -The last step is to register the parachain using polkadot-js. The parachain id is +The last step is to register the parachain using `polkadot-js`. The parachain id is 100. The genesis state and the validation code are printed at startup by the collator. To do this automatically, run `scripts/adder-collator.sh`. diff --git a/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs b/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs deleted file mode 100644 index 7f93519d845400684a8e3a044ea5ecac50566ac5..0000000000000000000000000000000000000000 --- a/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs b/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs index 6b481f961a429c56912848668f3afdd260e56914..85abf8bf36b978bb33aae5115711443da5968a79 100644 --- a/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs +++ b/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs @@ -17,8 +17,6 @@ //! Integration test that ensures that we can build and include parachain //! blocks of the adder parachain. -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being // enabled. @@ -41,8 +39,12 @@ async fn collating_using_adder_collator() { true, ); + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone())); let bob_config = polkadot_test_service::node_config( || {}, @@ -53,7 +55,7 @@ async fn collating_using_adder_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path)); let collator = test_parachain_adder_collator::Collator::new(); diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 0f1fd60a90001f3934f184ea96920c41fcd15bc7..3fbed4046bdedb2a8ed44745662e2c17132a698c 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -11,15 +11,10 @@ publish = false name = "undying-collator" path = "src/main.rs" -[[bin]] -name = "undying_collator_puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } -clap = { version = "4.4.2", features = ["derive"] } -futures = "0.3.19" +clap = { version = "4.4.4", features = ["derive"] } +futures = "0.3.21" futures-timer = "3.0.2" log = "0.4.17" @@ -33,24 +28,14 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" } sc-cli = { path = "../../../../../substrate/client/cli" } sp-core = { path = "../../../../../substrate/primitives/core" } sc-service = { path = "../../../../../substrate/client/service" } -# This one is tricky. Even though it is not used directly by the collator, we still need it for the -# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be -# a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true } [dev-dependencies] polkadot-parachain-primitives = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } -# For the puppet worker, depend on ourselves with the test-utils feature. -test-parachain-undying-collator = { path = "", features = ["test-utils"] } +polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] } substrate-test-utils = { path = "../../../../../substrate/test-utils" } sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } - -[features] -# This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. -test-utils = [ "polkadot-node-core-pvf/test-utils" ] diff --git a/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs b/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs deleted file mode 100644 index 7f93519d845400684a8e3a044ea5ecac50566ac5..0000000000000000000000000000000000000000 --- a/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs b/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs index a98a7ff6eefc48f913d5e17f4a0c750a81c8eafd..8be535b9bb4cc1275b59a9a3782ef352ac60d643 100644 --- a/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs +++ b/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs @@ -17,8 +17,6 @@ //! Integration test that ensures that we can build and include parachain //! blocks of the `Undying` parachain. -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_undying_collator_puppet_worker"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being // enabled. #[tokio::test(flavor = "multi_thread")] @@ -40,8 +38,12 @@ async fn collating_using_undying_collator() { true, ); + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone())); let bob_config = polkadot_test_service::node_config( || {}, @@ -52,7 +54,7 @@ async fn collating_using_undying_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path)); let collator = test_parachain_undying_collator::Collator::new(1_000, 1); diff --git a/polkadot/primitives/README.adoc b/polkadot/primitives/README.adoc deleted file mode 100644 index 0e5c9412f002902a0a00970b08f746ffdfe5cc77..0000000000000000000000000000000000000000 --- a/polkadot/primitives/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot primitives - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/primitives/src/lib.rs b/polkadot/primitives/src/lib.rs index 729908cc12ba0d471a2f0fbc93f972e575f1f220..9121b3790858339352d04c7a4f83eb2800929940 100644 --- a/polkadot/primitives/src/lib.rs +++ b/polkadot/primitives/src/lib.rs @@ -41,8 +41,8 @@ pub use v5::{ BackedCandidate, Balance, BlakeTwo256, Block, BlockId, BlockNumber, CandidateCommitments, CandidateDescriptor, CandidateEvent, CandidateHash, CandidateIndex, CandidateReceipt, CheckedDisputeStatementSet, CheckedMultiDisputeStatementSet, CollatorId, CollatorSignature, - CommittedCandidateReceipt, CompactStatement, ConsensusLog, CoreIndex, CoreOccupied, CoreState, - DisputeState, DisputeStatement, DisputeStatementSet, DownwardMessage, EncodeAs, ExecutorParam, + CommittedCandidateReceipt, CompactStatement, ConsensusLog, CoreIndex, CoreState, DisputeState, + DisputeStatement, DisputeStatementSet, DownwardMessage, EncodeAs, ExecutorParam, ExecutorParams, ExecutorParamsHash, ExplicitDisputeStatement, GroupIndex, GroupRotationInfo, Hash, HashT, HeadData, Header, HrmpChannelId, Id, InboundDownwardMessage, InboundHrmpMessage, IndexedVec, InherentData, InvalidDisputeStatementKind, Moment, MultiDisputeStatementSet, Nonce, diff --git a/polkadot/primitives/src/v5/mod.rs b/polkadot/primitives/src/v5/mod.rs index eed4cc2b36ba6b4214a3fcfc048edcc0120fc998..81743225403d6159fd86a48e5d8e7fa38d84d06a 100644 --- a/polkadot/primitives/src/v5/mod.rs +++ b/polkadot/primitives/src/v5/mod.rs @@ -44,7 +44,7 @@ pub use polkadot_core_primitives::v2::{ // Export some polkadot-parachain primitives pub use polkadot_parachain_primitives::primitives::{ HeadData, HorizontalMessages, HrmpChannelId, Id, UpwardMessage, UpwardMessages, ValidationCode, - ValidationCodeHash, LOWEST_PUBLIC_ID, LOWEST_USER_ID, + ValidationCodeHash, LOWEST_PUBLIC_ID, }; use serde::{Deserialize, Serialize}; @@ -830,60 +830,6 @@ pub struct ParathreadEntry { pub retries: u32, } -/// An assignment for a parachain scheduled to be backed and included in a relay chain block. -#[derive(Clone, Encode, Decode, PartialEq, TypeInfo, RuntimeDebug)] -pub struct Assignment { - /// Assignment's ParaId - pub para_id: Id, -} - -impl Assignment { - /// Create a new `Assignment`. - pub fn new(para_id: Id) -> Self { - Self { para_id } - } -} - -/// An entry tracking a paras -#[derive(Clone, Encode, Decode, TypeInfo, PartialEq, RuntimeDebug)] -pub struct ParasEntry { - /// The `Assignment` - pub assignment: Assignment, - /// The number of times the entry has timed out in availability. - pub availability_timeouts: u32, - /// The block height where this entry becomes invalid. - pub ttl: N, -} - -impl ParasEntry { - /// Return `Id` from the underlying `Assignment`. - pub fn para_id(&self) -> Id { - self.assignment.para_id - } - - /// Create a new `ParasEntry`. - pub fn new(assignment: Assignment, now: N) -> Self { - ParasEntry { assignment, availability_timeouts: 0, ttl: now } - } -} - -/// What is occupying a specific availability core. -#[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] -#[cfg_attr(feature = "std", derive(PartialEq))] -pub enum CoreOccupied { - /// The core is not occupied. - Free, - /// A paras. - Paras(ParasEntry), -} - -impl CoreOccupied { - /// Is core free? - pub fn is_free(&self) -> bool { - matches!(self, Self::Free) - } -} - /// A helper data-type for tracking validator-group rotations. #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] #[cfg_attr(feature = "std", derive(PartialEq))] diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index 8ee5f180d34290b8077f8b68eed73dadc81062dc..d532d6ff57f4a34c53d7099d011f1ca68bf6cf5f 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -159,6 +159,11 @@ pub fn dummy_pvd(parent_head: HeadData, relay_parent_number: u32) -> PersistedVa } } +/// Creates a meaningless signature +pub fn dummy_signature() -> polkadot_primitives::ValidatorSignature { + sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]) +} + /// Create a meaningless candidate, returning its receipt and PVD. pub fn make_candidate( relay_parent_hash: Hash, @@ -244,6 +249,11 @@ pub fn resign_candidate_descriptor_with_collator>( descriptor.signature = signature; } +/// Extracts validators's public keus (`ValidatorId`) from `Sr25519Keyring` +pub fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + /// Builder for `CandidateReceipt`. pub struct TestCandidateBuilder { pub para_id: ParaId, @@ -298,7 +308,3 @@ impl rand::RngCore for AlwaysZeroRng { Ok(()) } } - -pub fn dummy_signature() -> polkadot_primitives::ValidatorSignature { - sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]) -} diff --git a/polkadot/roadmap/implementers-guide/src/README.md b/polkadot/roadmap/implementers-guide/src/README.md index 9b6189992773b02e97cd63f5db51843f74d799cb..6673c533ece43a8bea6e648efcf81894f4b7d6f6 100644 --- a/polkadot/roadmap/implementers-guide/src/README.md +++ b/polkadot/roadmap/implementers-guide/src/README.md @@ -1,5 +1,11 @@ # Preamble -This document aims to describe the purpose, functionality, and implementation of the host for Polkadot's _parachains_ functionality - that is, the software which provides security and advancement for constituent parachains. It is not for the implementer of a specific parachain but rather for the implementer of the Parachain Host. In practice, this is for the implementers of Polkadot in general. +This document aims to describe the purpose, functionality, and implementation of the host for Polkadot's _parachains_ +functionality - that is, the software which provides security and advancement for constituent parachains. It is not for +the implementer of a specific parachain but rather for the implementer of the Parachain Host. In practice, this is for +the implementers of Polkadot in general. -There are a number of other documents describing the research in more detail. All referenced documents will be linked here and should be read alongside this document for the best understanding of the full picture. However, this is the only document which aims to describe key aspects of Polkadot's particular instantiation of much of that research down to low-level technical details and software architecture. +There are a number of other documents describing the research in more detail. All referenced documents will be linked +here and should be read alongside this document for the best understanding of the full picture. However, this is the +only document which aims to describe key aspects of Polkadot's particular instantiation of much of that research down to +low-level technical details and software architecture. diff --git a/polkadot/roadmap/implementers-guide/src/SUMMARY.md b/polkadot/roadmap/implementers-guide/src/SUMMARY.md index e997d4d77ad1073f9799ada6cbdd79ffd63753c7..bb19390c7af4d422dfe62252a7153392c8ef534d 100644 --- a/polkadot/roadmap/implementers-guide/src/SUMMARY.md +++ b/polkadot/roadmap/implementers-guide/src/SUMMARY.md @@ -71,16 +71,16 @@ - [Chain Selection Request](node/utility/chain-selection.md) - [PVF Pre-Checking](node/utility/pvf-prechecker.md) - [Data Structures and Types](types/README.md) - - [Candidate](types/candidate.md) - - [Backing](types/backing.md) - - [Availability](types/availability.md) - - [Overseer and Subsystem Protocol](types/overseer-protocol.md) - - [Runtime](types/runtime.md) - - [Messages](types/messages.md) - - [Network](types/network.md) - - [Approvals](types/approval.md) - - [Disputes](types/disputes.md) - - [PVF Pre-checking](types/pvf-prechecking.md) + - [Candidate](types/candidate.md) + - [Backing](types/backing.md) + - [Availability](types/availability.md) + - [Overseer and Subsystem Protocol](types/overseer-protocol.md) + - [Runtime](types/runtime.md) + - [Messages](types/messages.md) + - [Network](types/network.md) + - [Approvals](types/approval.md) + - [Disputes](types/disputes.md) + - [PVF Pre-checking](types/pvf-prechecking.md) [Glossary](glossary.md) [Further Reading](further-reading.md) diff --git a/polkadot/roadmap/implementers-guide/src/architecture.md b/polkadot/roadmap/implementers-guide/src/architecture.md index 0c192a5b980d76f82ddbb6f5c0978dd3ccb5b68b..b75270662005005f4b0a1102b4d95b3d9af2dab0 100644 --- a/polkadot/roadmap/implementers-guide/src/architecture.md +++ b/polkadot/roadmap/implementers-guide/src/architecture.md @@ -1,8 +1,13 @@ # Architecture Overview -This section aims to describe, at a high level, the code architecture and subsystems involved in the implementation of an individual Parachain Host. It also illuminates certain subtleties and challenges faced in the design and implementation of those subsystems. +This section aims to describe, at a high level, the code architecture and subsystems involved in the implementation of +an individual Parachain Host. It also illuminates certain subtleties and challenges faced in the design and +implementation of those subsystems. -To recap, Polkadot includes a blockchain known as the relay-chain. A blockchain is a Directed Acyclic Graph (DAG) of state transitions, where every block can be considered to be the head of a linked-list (known as a "chain" or "fork") with a cumulative state which is determined by applying the state transition of each block in turn. All paths through the DAG terminate at the Genesis Block. In fact, the blockchain is a tree, since each block can have only one parent. +To recap, Polkadot includes a blockchain known as the relay-chain. A blockchain is a Directed Acyclic Graph (DAG) of +state transitions, where every block can be considered to be the head of a linked-list (known as a "chain" or "fork") +with a cumulative state which is determined by applying the state transition of each block in turn. All paths through +the DAG terminate at the Genesis Block. In fact, the blockchain is a tree, since each block can have only one parent. ```dot process digraph { @@ -22,16 +27,25 @@ digraph { } ``` -A blockchain network is comprised of nodes. These nodes each have a view of many different forks of a blockchain and must decide which forks to follow and what actions to take based on the forks of the chain that they are aware of. +A blockchain network is comprised of nodes. These nodes each have a view of many different forks of a blockchain and +must decide which forks to follow and what actions to take based on the forks of the chain that they are aware of. -So in specifying an architecture to carry out the functionality of a Parachain Host, we have to answer two categories of questions: +So in specifying an architecture to carry out the functionality of a Parachain Host, we have to answer two categories of +questions: -1. What is the state-transition function of the blockchain? What is necessary for a transition to be considered valid, and what information is carried within the implicit state of a block? -1. Being aware of various forks of the blockchain as well as global private state such as a view of the current time, what behaviors should a node undertake? What information should a node extract from the state of which forks, and how should that information be used? +1. What is the state-transition function of the blockchain? What is necessary for a transition to be considered valid, + and what information is carried within the implicit state of a block? +1. Being aware of various forks of the blockchain as well as global private state such as a view of the current time, + what behaviors should a node undertake? What information should a node extract from the state of which forks, and how + should that information be used? -The first category of questions will be addressed by the Runtime, which defines the state-transition logic of the chain. Runtime logic only has to focus on the perspective of one chain, as each state has only a single parent state. +The first category of questions will be addressed by the Runtime, which defines the state-transition logic of the chain. +Runtime logic only has to focus on the perspective of one chain, as each state has only a single parent state. -The second category of questions addressed by Node-side behavior. Node-side behavior defines all activities that a node undertakes, given its view of the blockchain/block-DAG. Node-side behavior can take into account all or many of the forks of the blockchain, and only conditionally undertake certain activities based on which forks it is aware of, as well as the state of the head of those forks. +The second category of questions addressed by Node-side behavior. Node-side behavior defines all activities that a node +undertakes, given its view of the blockchain/block-DAG. Node-side behavior can take into account all or many of the +forks of the blockchain, and only conditionally undertake certain activities based on which forks it is aware of, as +well as the state of the head of those forks. ```dot process digraph G { @@ -46,7 +60,13 @@ digraph G { ``` -It is also helpful to divide Node-side behavior into two further categories: Networking and Core. Networking behaviors relate to how information is distributed between nodes. Core behaviors relate to internal work that a specific node does. These two categories of behavior often interact, but can be heavily abstracted from each other. Core behaviors care that information is distributed and received, but not the internal details of how distribution and receipt function. Networking behaviors act on requests for distribution or fetching of information, but are not concerned with how the information is used afterwards. This allows us to create clean boundaries between Core and Networking activities, improving the modularity of the code. +It is also helpful to divide Node-side behavior into two further categories: Networking and Core. Networking behaviors +relate to how information is distributed between nodes. Core behaviors relate to internal work that a specific node +does. These two categories of behavior often interact, but can be heavily abstracted from each other. Core behaviors +care that information is distributed and received, but not the internal details of how distribution and receipt +function. Networking behaviors act on requests for distribution or fetching of information, but are not concerned with +how the information is used afterwards. This allows us to create clean boundaries between Core and Networking +activities, improving the modularity of the code. ```text ___________________ ____________________ @@ -65,8 +85,18 @@ It is also helpful to divide Node-side behavior into two further categories: Net ``` -Node-side behavior is split up into various subsystems. Subsystems are long-lived workers that perform a particular category of work. Subsystems can communicate with each other, and do so via an [Overseer](node/overseer.md) that prevents race conditions. - -Runtime logic is divided up into Modules and APIs. Modules encapsulate particular behavior of the system. Modules consist of storage, routines, and entry-points. Routines are invoked by entry points, by other modules, upon block initialization or closing. Routines can read and alter the storage of the module. Entry-points are the means by which new information is introduced to a module and can limit the origins (user, root, parachain) that they accept being called by. Each block in the blockchain contains a set of Extrinsics. Each extrinsic targets a a specific entry point to trigger and which data should be passed to it. Runtime APIs provide a means for Node-side behavior to extract meaningful information from the state of a single fork. - -These two aspects of the implementation are heavily dependent on each other. The Runtime depends on Node-side behavior to author blocks, and to include Extrinsics which trigger the correct entry points. The Node-side behavior relies on Runtime APIs to extract information necessary to determine which actions to take. +Node-side behavior is split up into various subsystems. Subsystems are long-lived workers that perform a particular +category of work. Subsystems can communicate with each other, and do so via an [Overseer](node/overseer.md) that +prevents race conditions. + +Runtime logic is divided up into Modules and APIs. Modules encapsulate particular behavior of the system. Modules +consist of storage, routines, and entry-points. Routines are invoked by entry points, by other modules, upon block +initialization or closing. Routines can read and alter the storage of the module. Entry-points are the means by which +new information is introduced to a module and can limit the origins (user, root, parachain) that they accept being +called by. Each block in the blockchain contains a set of Extrinsics. Each extrinsic targets a a specific entry point to +trigger and which data should be passed to it. Runtime APIs provide a means for Node-side behavior to extract meaningful +information from the state of a single fork. + +These two aspects of the implementation are heavily dependent on each other. The Runtime depends on Node-side behavior +to author blocks, and to include Extrinsics which trigger the correct entry points. The Node-side behavior relies on +Runtime APIs to extract information necessary to determine which actions to take. diff --git a/polkadot/roadmap/implementers-guide/src/disputes-flow.md b/polkadot/roadmap/implementers-guide/src/disputes-flow.md index a325b2ce727276fbb8b91e0a11760365bd56cdc1..f9fd8dcce351e2b7a4461c05c35a647a69cfb4fe 100644 --- a/polkadot/roadmap/implementers-guide/src/disputes-flow.md +++ b/polkadot/roadmap/implementers-guide/src/disputes-flow.md @@ -70,10 +70,12 @@ stateDiagram-v2 ## Conditional formulation -The set of validators eligible to vote consists of -the validators that had duty at the time of backing, plus backing votes by the backing validators. +The set of validators eligible to vote consists of the validators that had duty at the time of backing, plus backing +votes by the backing validators. -If a validator receives an initial dispute message (a set of votes where there are at least two opposing votes contained), and the PoV or Code are hence not reconstructable from local storage, that validator must request the required data from its peers. +If a validator receives an initial dispute message (a set of votes where there are at least two opposing votes +contained), and the PoV or Code are hence not reconstructable from local storage, that validator must request the +required data from its peers. The dispute availability message must contain code, persisted validation data, and the proof of validity. @@ -81,9 +83,11 @@ Only peers that already voted shall be queried for the dispute availability data The peer to be queried for disputes data, must be picked at random. -A validator must retain code, persisted validation data and PoV until a block, that contains the dispute resolution, is finalized - plus an additional 24 hours. +A validator must retain code, persisted validation data and PoV until a block, that contains the dispute resolution, is +finalized - plus an additional 24 hours. -Dispute availability gossip must continue beyond the dispute resolution, until the post resolution timeout expired (equiv to the timeout until which additional late votes are accepted). +Dispute availability gossip must continue beyond the dispute resolution, until the post resolution timeout expired +(equiv to the timeout until which additional late votes are accepted). Remote disputes are disputes that are in relation to a chain that is not part of the local validators active heads. @@ -93,32 +97,42 @@ Persisted votes stay persisted for `N` sessions, and are cleaned up on a per ses Votes must be queryable by a particular validator, identified by its signing key. -Votes must be queryable by a particular validator, identified by a session index and the validator index valid in that session. +Votes must be queryable by a particular validator, identified by a session index and the validator index valid in that +session. If there exists a negative and a positive vote for a particular block, a dispute is detected. If a dispute is detected, all currently available votes for that block must be gossiped. -If an incoming dispute vote is detected, a validator must cast their own vote. The vote is determined by validating the PoV with the Code at the time of backing the block in question. +If an incoming dispute vote is detected, a validator must cast their own vote. The vote is determined by validating the +PoV with the Code at the time of backing the block in question. If the validator was also a backer of the block, validation and casting an additional vote should be skipped. -If the count of votes pro or cons regarding the disputed block, reaches the required ⅔ supermajority (including the backing votes), the conclusion must be recorded on chain and the voters on the loosing and no-shows being slashed appropriately. +If the count of votes pro or cons regarding the disputed block, reaches the required ⅔ supermajority (including the +backing votes), the conclusion must be recorded on chain and the voters on the loosing and no-shows being slashed +appropriately. -If a block is found invalid by a dispute resolution, it must be blacklisted to avoid resync or further build on that chain if other chains are available (to be detailed in the grandpa fork choice rule). +If a block is found invalid by a dispute resolution, it must be blacklisted to avoid resync or further build on that +chain if other chains are available (to be detailed in the grandpa fork choice rule). A dispute accepts Votes after the dispute is resolved, for 1 day. -If a vote is received, after the dispute is resolved, the vote shall still be recorded in the state root, albeit yielding less reward. +If a vote is received, after the dispute is resolved, the vote shall still be recorded in the state root, albeit +yielding less reward. Recording in the state root might happen batched, at timeout expiry. -If a new active head/chain appears, and the dispute resolution was not recorded on that chain yet, the dispute resolution or open dispute must be recorded / transplanted to that chain as well, since the disputes must be present on all chains to make sure the offender is punished. +If a new active head/chain appears, and the dispute resolution was not recorded on that chain yet, the dispute +resolution or open dispute must be recorded / transplanted to that chain as well, since the disputes must be present on +all chains to make sure the offender is punished. -If a validator votes in two opposing ways, this composes of a double vote like in other cases (backing, approval voting). +If a validator votes in two opposing ways, this composes of a double vote like in other cases (backing, approval +voting). If a dispute is not resolved within due time, all validators are to be slashed for a small amount. If a dispute is not resolved within due time, governance mode shall be entered for manual resolution. -If a validator unexpectedly restarts, the dispute shall be continued with the state based on votes being cast and being present in persistent storage. +If a validator unexpectedly restarts, the dispute shall be continued with the state based on votes being cast and being +present in persistent storage. diff --git a/polkadot/roadmap/implementers-guide/src/glossary.md b/polkadot/roadmap/implementers-guide/src/glossary.md index a036ccdd668c54ff89933189238757d9b65632b1..b2365ba51c5ce80fd0d60b53c592b1037f540a52 100644 --- a/polkadot/roadmap/implementers-guide/src/glossary.md +++ b/polkadot/roadmap/implementers-guide/src/glossary.md @@ -2,46 +2,72 @@ Here you can find definitions of a bunch of jargon, usually specific to the Polkadot project. -- **Approval Checker:** A validator who randomly self-selects so to perform validity checks on a parablock which is pending approval. -- **BABE:** (Blind Assignment for Blockchain Extension). The algorithm validators use to safely extend the Relay Chain. See [the Polkadot wiki][0] for more information. -- **Backable Candidate:** A Parachain Candidate which is backed by a majority of validators assigned to a given parachain. +- **Approval Checker:** A validator who randomly self-selects so to perform validity checks on a parablock which is + pending approval. +- **BABE:** (Blind Assignment for Blockchain Extension). The algorithm validators use to safely extend the Relay Chain. + See [the Polkadot wiki][0] for more information. +- **Backable Candidate:** A Parachain Candidate which is backed by a majority of validators assigned to a given + parachain. - **Backed Candidate:** A Backable Candidate noted in a relay-chain block - **Backing:** A set of statements proving that a Parachain Candidate is backable. - **Collator:** A node who generates Proofs-of-Validity (PoV) for blocks of a specific parachain. -- **DMP:** (Downward Message Passing). Message passing from the relay-chain to a parachain. Also there is a runtime parachains module with the same name. -- **DMQ:** (Downward Message Queue). A message queue for messages from the relay-chain down to a parachain. A parachain has -exactly one downward message queue. -- **Extrinsic:** An element of a relay-chain block which triggers a specific entry-point of a runtime module with given arguments. -- **GRANDPA:** (Ghost-based Recursive ANcestor Deriving Prefix Agreement). The algorithm validators use to guarantee finality of the Relay Chain. -- **HRMP:** (Horizontally Relay-routed Message Passing). A mechanism for message passing between parachains (hence horizontal) that leverages the relay-chain storage. Predates XCMP. Also there is a runtime parachains module with the same name. -- **Inclusion Pipeline:** The set of steps taken to carry a Parachain Candidate from authoring, to backing, to availability and full inclusion in an active fork of its parachain. +- **DMP:** (Downward Message Passing). Message passing from the relay-chain to a parachain. Also there is a runtime + parachains module with the same name. +- **DMQ:** (Downward Message Queue). A message queue for messages from the relay-chain down to a parachain. A parachain +has exactly one downward message queue. +- **Extrinsic:** An element of a relay-chain block which triggers a specific entry-point of a runtime module with given + arguments. +- **GRANDPA:** (Ghost-based Recursive ANcestor Deriving Prefix Agreement). The algorithm validators use to guarantee + finality of the Relay Chain. +- **HRMP:** (Horizontally Relay-routed Message Passing). A mechanism for message passing between parachains (hence + horizontal) that leverages the relay-chain storage. Predates XCMP. Also there is a runtime parachains module with the + same name. +- **Inclusion Pipeline:** The set of steps taken to carry a Parachain Candidate from authoring, to backing, to + availability and full inclusion in an active fork of its parachain. - **Module:** A component of the Runtime logic, encapsulating storage, routines, and entry-points. - **Module Entry Point:** A recipient of new information presented to the Runtime. This may trigger routines. -- **Module Routine:** A piece of code executed within a module by block initialization, closing, or upon an entry point being triggered. This may execute computation, and read or write storage. -- **MQC:** (Message Queue Chain). A cryptographic data structure that resembles an append-only linked list which doesn't store original values but only their hashes. The whole structure is described by a single hash, referred as a "head". When a value is appended, it's contents hashed with the previous head creating a hash that becomes a new head. -- **Node:** A participant in the Polkadot network, who follows the protocols of communication and connection to other nodes. Nodes form a peer-to-peer network topology without a central authority. +- **Module Routine:** A piece of code executed within a module by block initialization, closing, or upon an entry point + being triggered. This may execute computation, and read or write storage. +- **MQC:** (Message Queue Chain). A cryptographic data structure that resembles an append-only linked list which doesn't + store original values but only their hashes. The whole structure is described by a single hash, referred as a "head". + When a value is appended, it's contents hashed with the previous head creating a hash that becomes a new head. +- **Node:** A participant in the Polkadot network, who follows the protocols of communication and connection to other + nodes. Nodes form a peer-to-peer network topology without a central authority. - **Parachain Candidate, or Candidate:** A proposed block for inclusion into a parachain. - **Parablock:** A block in a parachain. - **Parachain:** A constituent chain secured by the Relay Chain's validators. -- **Parachain Validators:** A subset of validators assigned during a period of time to back candidates for a specific parachain +- **Parachain Validators:** A subset of validators assigned during a period of time to back candidates for a specific + parachain - **On-demand parachain:** A parachain which is scheduled on a pay-as-you-go basis. -- **Lease holding parachain:** A parachain possessing an active slot lease. The lease holder is assigned a single availability core for the duration of the lease, granting consistent blockspace scheduling at the rate 1 parablock per relay block. +- **Lease holding parachain:** A parachain possessing an active slot lease. The lease holder is assigned a single + availability core for the duration of the lease, granting consistent blockspace scheduling at the rate 1 parablock per + relay block. - **PDK (Parachain Development Kit):** A toolset that allows one to develop a parachain. Cumulus is a PDK. -- **Preimage:** In our context, if `H(X) = Y` where `H` is a hash function and `Y` is the hash, then `X` is the hash preimage. -- **Proof-of-Validity (PoV):** A stateless-client proof that a parachain candidate is valid, with respect to some validation function. +- **Preimage:** In our context, if `H(X) = Y` where `H` is a hash function and `Y` is the hash, then `X` is the hash + preimage. +- **Proof-of-Validity (PoV):** A stateless-client proof that a parachain candidate is valid, with respect to some + validation function. - **PVF:** Parachain Validation Function. The validation code that is run by validators on parachains. -- **PVF Prechecking:** This is the process of initially checking the PVF when it is first added. We attempt preparation of the PVF and make sure it succeeds within a given timeout, plus some additional checks. -- **PVF Preparation:** This is the process of preparing the WASM blob and includes both prevalidation and compilation. As there is no prevalidation right now, preparation just consists of compilation. -- **Relay Parent:** A block in the relay chain, referred to in a context where work is being done in the context of the state at this block. +- **PVF Prechecking:** This is the process of initially checking the PVF when it is first added. We attempt preparation + of the PVF and make sure it succeeds within a given timeout, plus some additional checks. +- **PVF Preparation:** This is the process of preparing the WASM blob and includes both prevalidation and compilation. + As there is no prevalidation right now, preparation just consists of compilation. +- **Relay Parent:** A block in the relay chain, referred to in a context where work is being done in the context of the + state at this block. - **Runtime:** The relay-chain state machine. - **Runtime Module:** See Module. -- **Runtime API:** A means for the node-side behavior to access structured information based on the state of a fork of the blockchain. +- **Runtime API:** A means for the node-side behavior to access structured information based on the state of a fork of + the blockchain. - **Subsystem:** A long-running task which is responsible for carrying out a particular category of work. - **UMP:** (Upward Message Passing) A vertical message passing mechanism from a parachain to the relay chain. -- **Validator:** Specially-selected node in the network who is responsible for validating parachain blocks and issuing attestations about their validity. +- **Validator:** Specially-selected node in the network who is responsible for validating parachain blocks and issuing + attestations about their validity. - **Validation Function:** A piece of Wasm code that describes the state-transition function of a parachain. -- **VMP:** (Vertical Message Passing) A family of mechanisms that are responsible for message exchange between the relay chain and parachains. -- **XCMP:** (Cross-Chain Message Passing) A type of horizontal message passing (i.e. between parachains) that allows secure message passing directly between parachains and has minimal resource requirements from the relay chain, thus highly scalable. +- **VMP:** (Vertical Message Passing) A family of mechanisms that are responsible for message exchange between the relay + chain and parachains. +- **XCMP:** (Cross-Chain Message Passing) A type of horizontal message passing (i.e. between parachains) that allows + secure message passing directly between parachains and has minimal resource requirements from the relay chain, thus + highly scalable. ## See Also diff --git a/polkadot/roadmap/implementers-guide/src/messaging.md b/polkadot/roadmap/implementers-guide/src/messaging.md index edc810e034154278e53e78b1e563dcbf299b1613..11cd3565e8ede8ee5e47b5b54c78cc0b627146e8 100644 --- a/polkadot/roadmap/implementers-guide/src/messaging.md +++ b/polkadot/roadmap/implementers-guide/src/messaging.md @@ -1,9 +1,9 @@ # Messaging Overview -The Polkadot Host has a few mechanisms that are responsible for message passing. They can be generally divided -on two categories: Horizontal and Vertical. Horizontal Message Passing (HMP) refers to mechanisms -that are responsible for exchanging messages between parachains. Vertical Message Passing (VMP) is -used for communication between the relay chain and parachains. +The Polkadot Host has a few mechanisms that are responsible for message passing. They can be generally divided on two +categories: Horizontal and Vertical. Horizontal Message Passing (HMP) refers to mechanisms that are responsible for +exchanging messages between parachains. Vertical Message Passing (VMP) is used for communication between the relay chain +and parachains. ## Vertical Message Passing @@ -19,35 +19,34 @@ digraph { Downward Message Passing (DMP) is a mechanism for delivering messages to parachains from the relay chain. -Each parachain has its own queue that stores all pending inbound downward messages. A parachain -doesn't have to process all messages at once, however, there are rules as to how the downward message queue -should be processed. Currently, at least one message must be consumed per candidate if the queue is not empty. -The downward message queue doesn't have a cap on its size and it is up to the relay-chain to put mechanisms -that prevent spamming in place. +Each parachain has its own queue that stores all pending inbound downward messages. A parachain doesn't have to process +all messages at once, however, there are rules as to how the downward message queue should be processed. Currently, at +least one message must be consumed per candidate if the queue is not empty. The downward message queue doesn't have a +cap on its size and it is up to the relay-chain to put mechanisms that prevent spamming in place. -Upward Message Passing (UMP) is a mechanism responsible for delivering messages in the opposite direction: -from a parachain up to the relay chain. Upward messages are essentially byte blobs. However, they are interpreted -by the relay-chain according to the XCM standard. +Upward Message Passing (UMP) is a mechanism responsible for delivering messages in the opposite direction: from a +parachain up to the relay chain. Upward messages are essentially byte blobs. However, they are interpreted by the +relay-chain according to the XCM standard. -The XCM standard is a common vocabulary of messages. The XCM standard doesn't require a particular interpretation of -a message. However, the parachains host (e.g. Polkadot) guarantees certain semantics for those. +The XCM standard is a common vocabulary of messages. The XCM standard doesn't require a particular interpretation of a +message. However, the parachains host (e.g. Polkadot) guarantees certain semantics for those. -Moreover, while most XCM messages are handled by the on-chain XCM interpreter, some of the messages are special -cased. Specifically, those messages can be checked during the acceptance criteria and thus invalid -messages would lead to rejecting the candidate itself. +Moreover, while most XCM messages are handled by the on-chain XCM interpreter, some of the messages are special cased. +Specifically, those messages can be checked during the acceptance criteria and thus invalid messages would lead to +rejecting the candidate itself. -One kind of such a message is `Xcm::Transact`. This upward message can be seen as a way for a parachain -to execute arbitrary entrypoints on the relay-chain. `Xcm::Transact` messages resemble regular extrinsics with the exception that they -originate from a parachain. +One kind of such a message is `Xcm::Transact`. This upward message can be seen as a way for a parachain to execute +arbitrary entrypoints on the relay-chain. `Xcm::Transact` messages resemble regular extrinsics with the exception that +they originate from a parachain. -The payload of `Xcm::Transact` messages is referred as to `Dispatchable`. When a candidate with such a message is enacted -the dispatchables are put into a queue corresponding to the parachain. There can be only so many dispatchables in that queue at once. -The weight that processing of the dispatchables can consume is limited by a preconfigured value. Therefore, it is possible -that some dispatchables will be left for later blocks. To make the dispatching more fair, the queues are processed turn-by-turn -in a round robin fashion. +The payload of `Xcm::Transact` messages is referred as to `Dispatchable`. When a candidate with such a message is +enacted the dispatchables are put into a queue corresponding to the parachain. There can be only so many dispatchables +in that queue at once. The weight that processing of the dispatchables can consume is limited by a preconfigured value. +Therefore, it is possible that some dispatchables will be left for later blocks. To make the dispatching more fair, the +queues are processed turn-by-turn in a round robin fashion. -The second category of special cased XCM messages are for horizontal messaging channel management, -namely messages meant to request opening and closing HRMP channels (HRMP will be described below). +The second category of special cased XCM messages are for horizontal messaging channel management, namely messages meant +to request opening and closing HRMP channels (HRMP will be described below). ## Horizontal Message Passing @@ -77,29 +76,28 @@ The most important member of this family is XCMP. > ℹ️ XCMP is currently under construction and details are subject for change. -XCMP is a message passing mechanism between parachains that require minimal involvement of the relay chain. -The relay chain provides means for sending parachains to authenticate messages sent to recipient parachains. +XCMP is a message passing mechanism between parachains that require minimal involvement of the relay chain. The relay +chain provides means for sending parachains to authenticate messages sent to recipient parachains. -Semantically communication occurs through so called channels. A channel is unidirectional and it has -two endpoints, for sender and for recipient. A channel can be opened only if the both parties agree -and closed unilaterally. +Semantically communication occurs through so called channels. A channel is unidirectional and it has two endpoints, for +sender and for recipient. A channel can be opened only if the both parties agree and closed unilaterally. -Only the channel metadata is stored on the relay-chain in a very compact form: all messages and their -contents sent by the sender parachain are encoded using only one root hash. This root is referred as -MQC head. +Only the channel metadata is stored on the relay-chain in a very compact form: all messages and their contents sent by +the sender parachain are encoded using only one root hash. This root is referred as MQC head. -The authenticity of the messages must be proven using that root hash to the receiving party at the -candidate authoring time. The proof stems from the relay parent storage that contains the root hash of the channel. -Since not all messages are required to be processed by the receiver's candidate, only the processed -messages are supplied (i.e. preimages), rest are provided as hashes. +The authenticity of the messages must be proven using that root hash to the receiving party at the candidate authoring +time. The proof stems from the relay parent storage that contains the root hash of the channel. Since not all messages +are required to be processed by the receiver's candidate, only the processed messages are supplied (i.e. preimages), +rest are provided as hashes. -Further details can be found at the official repository for the -[Cross-Consensus Message Format (XCM)](https://github.com/paritytech/xcm-format/blob/master/README.md), as well as -at the [W3F research website](https://research.web3.foundation/en/latest/polkadot/XCMP.html) and -[this blogpost](https://medium.com/web3foundation/polkadots-messaging-scheme-b1ec560908b7). +Further details can be found at the official repository for the [Cross-Consensus Message Format +(XCM)](https://github.com/paritytech/xcm-format/blob/master/README.md), as well as at the [W3F research +website](https://research.web3.foundation/en/latest/polkadot/XCMP.html) and [this +blogpost](https://medium.com/web3foundation/polkadots-messaging-scheme-b1ec560908b7). -HRMP (Horizontally Relay-routed Message Passing) is a stop gap that predates XCMP. Semantically, it mimics XCMP's interface. -The crucial difference from XCMP though is that all the messages are stored in the relay-chain storage. That makes -things simple but at the same time that makes HRMP more demanding in terms of resources thus making it more expensive. +HRMP (Horizontally Relay-routed Message Passing) is a stop gap that predates XCMP. Semantically, it mimics XCMP's +interface. The crucial difference from XCMP though is that all the messages are stored in the relay-chain storage. That +makes things simple but at the same time that makes HRMP more demanding in terms of resources thus making it more +expensive. Once XCMP is available we expect to retire HRMP. diff --git a/polkadot/roadmap/implementers-guide/src/node/README.md b/polkadot/roadmap/implementers-guide/src/node/README.md index edd72d2335b5941971bfc8b9534a4f70b94762d8..c67f2b0b82ee565c139f7126d1243cbed1a3c51a 100644 --- a/polkadot/roadmap/implementers-guide/src/node/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/README.md @@ -2,30 +2,49 @@ ## Design Goals -* Modularity: Components of the system should be as self-contained as possible. Communication boundaries between components should be well-defined and mockable. This is key to creating testable, easily reviewable code. -* Minimizing side effects: Components of the system should aim to minimize side effects and to communicate with other components via message-passing. -* Operational Safety: The software will be managing signing keys where conflicting messages can lead to large amounts of value to be slashed. Care should be taken to ensure that no messages are signed incorrectly or in conflict with each other. +* Modularity: Components of the system should be as self-contained as possible. Communication boundaries between + components should be well-defined and mockable. This is key to creating testable, easily reviewable code. +* Minimizing side effects: Components of the system should aim to minimize side effects and to communicate with other + components via message-passing. +* Operational Safety: The software will be managing signing keys where conflicting messages can lead to large amounts of + value to be slashed. Care should be taken to ensure that no messages are signed incorrectly or in conflict with each + other. -The architecture of the node-side behavior aims to embody the Rust principles of ownership and message-passing to create clean, isolatable code. Each resource should have a single owner, with minimal sharing where unavoidable. +The architecture of the node-side behavior aims to embody the Rust principles of ownership and message-passing to create +clean, isolatable code. Each resource should have a single owner, with minimal sharing where unavoidable. -Many operations that need to be carried out involve the network, which is asynchronous. This asynchrony affects all core subsystems that rely on the network as well. The approach of hierarchical state machines is well-suited to this kind of environment. +Many operations that need to be carried out involve the network, which is asynchronous. This asynchrony affects all core +subsystems that rely on the network as well. The approach of hierarchical state machines is well-suited to this kind of +environment. We introduce ## Components The node architecture consists of the following components: - * The Overseer (and subsystems): A hierarchy of state machines where an overseer supervises subsystems. Subsystems can contain their own internal hierarchy of jobs. This is elaborated on in the next section on Subsystems. + * The Overseer (and subsystems): A hierarchy of state machines where an overseer supervises subsystems. Subsystems can + contain their own internal hierarchy of jobs. This is elaborated on in the next section on Subsystems. * A block proposer: Logic triggered by the consensus algorithm of the chain when the node should author a block. - * A GRANDPA voting rule: A strategy for selecting chains to vote on in the GRANDPA algorithm to ensure that only valid parachain candidates appear in finalized relay-chain blocks. + * A GRANDPA voting rule: A strategy for selecting chains to vote on in the GRANDPA algorithm to ensure that only valid + parachain candidates appear in finalized relay-chain blocks. ## Assumptions -The Node-side code comes with a set of assumptions that we build upon. These assumptions encompass most of the fundamental blockchain functionality. +The Node-side code comes with a set of assumptions that we build upon. These assumptions encompass most of the +fundamental blockchain functionality. We assume the following constraints regarding provided basic functionality: * The underlying **consensus** algorithm, whether it is BABE or SASSAFRAS is implemented. - * There is a **chain synchronization** protocol which will search for and download the longest available chains at all times. - * The **state** of all blocks at the head of the chain is available. There may be **state pruning** such that state of the last `k` blocks behind the last finalized block are available, as well as the state of all their descendants. This assumption implies that the state of all active leaves and their last `k` ancestors are all available. The underlying implementation is expected to support `k` of a few hundred blocks, but we reduce this to a very conservative `k=5` for our purposes. - * There is an underlying **networking** framework which provides **peer discovery** services which will provide us with peers and will not create "loopback" connections to our own node. The number of peers we will have is assumed to be bounded at 1000. - * There is a **transaction pool** and a **transaction propagation** mechanism which maintains a set of current transactions and distributes to connected peers. Current transactions are those which are not outdated relative to some "best" fork of the chain, which is part of the active heads, and have not been included in the best fork. + * There is a **chain synchronization** protocol which will search for and download the longest available chains at all + times. + * The **state** of all blocks at the head of the chain is available. There may be **state pruning** such that state of + the last `k` blocks behind the last finalized block are available, as well as the state of all their descendants. + This assumption implies that the state of all active leaves and their last `k` ancestors are all available. The + underlying implementation is expected to support `k` of a few hundred blocks, but we reduce this to a very + conservative `k=5` for our purposes. + * There is an underlying **networking** framework which provides **peer discovery** services which will provide us + with peers and will not create "loopback" connections to our own node. The number of peers we will have is assumed + to be bounded at 1000. + * There is a **transaction pool** and a **transaction propagation** mechanism which maintains a set of current + transactions and distributes to connected peers. Current transactions are those which are not outdated relative to + some "best" fork of the chain, which is part of the active heads, and have not been included in the best fork. diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/README.md b/polkadot/roadmap/implementers-guide/src/node/approval/README.md index 1f65173e16bb0c3ffd47d72ccf674820dc252148..ae9f46674e54699c8fa90a5851b77b67d3bbe330 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/README.md @@ -2,6 +2,9 @@ The approval subsystems implement the node-side of the [Approval Protocol](../../protocol-approval.md). -We make a divide between the [assignment/voting logic](approval-voting.md) and the [distribution logic](approval-distribution.md) that distributes assignment certifications and approval votes. The logic in the assignment and voting also informs the GRANDPA voting rule on how to vote. +We make a divide between the [assignment/voting logic](approval-voting.md) and the [distribution +logic](approval-distribution.md) that distributes assignment certifications and approval votes. The logic in the +assignment and voting also informs the GRANDPA voting rule on how to vote. -These subsystems are intended to flag issues and begin participating in live disputes. Dispute subsystems also track all observed votes (backing, approval, and dispute-specific) by all validators on all candidates. +These subsystems are intended to flag issues and begin participating in live disputes. Dispute subsystems also track all +observed votes (backing, approval, and dispute-specific) by all validators on all candidates. diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-distribution.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-distribution.md index 81c98afa16bf25439a59ae42ae1d3ace96a5558c..ce71de6f76b8f8550e815849ba80002f890a9b85 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-distribution.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-distribution.md @@ -2,50 +2,73 @@ A subsystem for the distribution of assignments and approvals for approval checks on candidates over the network. -The [Approval Voting](approval-voting.md) subsystem is responsible for active participation in a protocol designed to select a sufficient number of validators to check each and every candidate which appears in the relay chain. Statements of participation in this checking process are divided into two kinds: - - **Assignments** indicate that validators have been selected to do checking - - **Approvals** indicate that validators have checked and found the candidate satisfactory. - -The [Approval Voting](approval-voting.md) subsystem handles all the issuing and tallying of this protocol, but this subsystem is responsible for the disbursal of statements among the validator-set. - -The inclusion pipeline of candidates concludes after availability, and only after inclusion do candidates actually get pushed into the approval checking pipeline. As such, this protocol deals with the candidates _made available by_ particular blocks, as opposed to the candidates which actually appear within those blocks, which are the candidates _backed by_ those blocks. Unless stated otherwise, whenever we reference a candidate partially by block hash, we are referring to the set of candidates _made available by_ those blocks. - -We implement this protocol as a gossip protocol, and like other parachain-related gossip protocols our primary concerns are about ensuring fast message propagation while maintaining an upper bound on the number of messages any given node must store at any time. - -Approval messages should always follow assignments, so we need to be able to discern two pieces of information based on our [View](../../types/network.md#universal-types): +The [Approval Voting](approval-voting.md) subsystem is responsible for active participation in a protocol designed to +select a sufficient number of validators to check each and every candidate which appears in the relay chain. Statements +of participation in this checking process are divided into two kinds: + * **Assignments** indicate that validators have been selected to do checking + * **Approvals** indicate that validators have checked and found the candidate satisfactory. + +The [Approval Voting](approval-voting.md) subsystem handles all the issuing and tallying of this protocol, but this +subsystem is responsible for the disbursal of statements among the validator-set. + +The inclusion pipeline of candidates concludes after availability, and only after inclusion do candidates actually get +pushed into the approval checking pipeline. As such, this protocol deals with the candidates _made available by_ +particular blocks, as opposed to the candidates which actually appear within those blocks, which are the candidates +_backed by_ those blocks. Unless stated otherwise, whenever we reference a candidate partially by block hash, we are +referring to the set of candidates _made available by_ those blocks. + +We implement this protocol as a gossip protocol, and like other parachain-related gossip protocols our primary concerns +are about ensuring fast message propagation while maintaining an upper bound on the number of messages any given node +must store at any time. + +Approval messages should always follow assignments, so we need to be able to discern two pieces of information based on +our [View](../../types/network.md#universal-types): 1. Is a particular assignment relevant under a given `View`? 2. Is a particular approval relevant to any assignment in a set? -For our own local view, these two queries must not yield false negatives. When applied to our peers' views, it is acceptable for them to yield false negatives. The reason for that is that our peers' views may be beyond ours, and we are not capable of fully evaluating them. Once we have caught up, we can check again for false negatives to continue distributing. +For our own local view, these two queries must not yield false negatives. When applied to our peers' views, it is +acceptable for them to yield false negatives. The reason for that is that our peers' views may be beyond ours, and we +are not capable of fully evaluating them. Once we have caught up, we can check again for false negatives to continue +distributing. -For assignments, what we need to be checking is whether we are aware of the (block, candidate) pair that the assignment references. For approvals, we need to be aware of an assignment by the same validator which references the candidate being approved. +For assignments, what we need to be checking is whether we are aware of the (block, candidate) pair that the assignment +references. For approvals, we need to be aware of an assignment by the same validator which references the candidate +being approved. -However, awareness on its own of a (block, candidate) pair would imply that even ancient candidates all the way back to the genesis are relevant. We are actually not interested in anything before finality. +However, awareness on its own of a (block, candidate) pair would imply that even ancient candidates all the way back to +the genesis are relevant. We are actually not interested in anything before finality. -We gossip assignments along a grid topology produced by the [Gossip Support Subsystem](../utility/gossip-support.md) and also to a few random peers. The first time we accept an assignment or approval, regardless of the source, which originates from a validator peer in a shared dimension of the grid, we propagate the message to validator peers in the unshared dimension as well as a few random peers. +We gossip assignments along a grid topology produced by the [Gossip Support Subsystem](../utility/gossip-support.md) and +also to a few random peers. The first time we accept an assignment or approval, regardless of the source, which +originates from a validator peer in a shared dimension of the grid, we propagate the message to validator peers in the +unshared dimension as well as a few random peers. -But, in case these mechanisms don't work on their own, we need to trade bandwidth for protocol liveness by introducing aggression. +But, in case these mechanisms don't work on their own, we need to trade bandwidth for protocol liveness by introducing +aggression. Aggression has 3 levels: - Aggression Level 0: The basic behaviors described above. - Aggression Level 1: The originator of a message sends to all peers. Other peers follow the rules above. - Aggression Level 2: All peers send all messages to all their row and column neighbors. This means that each validator will, on average, receive each message approximately 2*sqrt(n) times. +* Aggression Level 0: The basic behaviors described above. +* Aggression Level 1: The originator of a message sends to all peers. Other peers follow the rules above. +* Aggression Level 2: All peers send all messages to all their row and column neighbors. This means that each validator + will, on average, receive each message approximately 2*sqrt(n) times. -These aggression levels are chosen based on how long a block has taken to finalize: assignments and approvals related to the unfinalized block will be propagated with more aggression. In particular, it's only the earliest unfinalized blocks that aggression should be applied to, because descendants may be unfinalized only by virtue of being descendants. +These aggression levels are chosen based on how long a block has taken to finalize: assignments and approvals related to +the unfinalized block will be propagated with more aggression. In particular, it's only the earliest unfinalized blocks +that aggression should be applied to, because descendants may be unfinalized only by virtue of being descendants. ## Protocol Input: - - `ApprovalDistributionMessage::NewBlocks` - - `ApprovalDistributionMessage::DistributeAssignment` - - `ApprovalDistributionMessage::DistributeApproval` - - `ApprovalDistributionMessage::NetworkBridgeUpdate` - - `OverseerSignal::BlockFinalized` + * `ApprovalDistributionMessage::NewBlocks` + * `ApprovalDistributionMessage::DistributeAssignment` + * `ApprovalDistributionMessage::DistributeApproval` + * `ApprovalDistributionMessage::NetworkBridgeUpdate` + * `OverseerSignal::BlockFinalized` Output: - - `ApprovalVotingMessage::CheckAndImportAssignment` - - `ApprovalVotingMessage::CheckAndImportApproval` - - `NetworkBridgeMessage::SendValidationMessage::ApprovalDistribution` + * `ApprovalVotingMessage::CheckAndImportAssignment` + * `ApprovalVotingMessage::CheckAndImportApproval` + * `NetworkBridgeMessage::SendValidationMessage::ApprovalDistribution` ## Functionality @@ -134,28 +157,37 @@ Iterate over every `BlockEntry` and remove `PeerId` from it. #### `NetworkBridgeEvent::OurViewChange` -Remove entries in `pending_known` for all hashes not present in the view. -Ensure a vector is present in `pending_known` for each hash in the view that does not have an entry in `blocks`. +Remove entries in `pending_known` for all hashes not present in the view. Ensure a vector is present in `pending_known` +for each hash in the view that does not have an entry in `blocks`. #### `NetworkBridgeEvent::PeerViewChange` Invoke `unify_with_peer(peer, view)` to catch them up to messages we have. -We also need to use the `view.finalized_number` to remove the `PeerId` from any blocks that it won't be wanting information about anymore. Note that we have to be on guard for peers doing crazy stuff like jumping their `finalized_number` forward 10 trillion blocks to try and get us stuck in a loop for ages. +We also need to use the `view.finalized_number` to remove the `PeerId` from any blocks that it won't be wanting +information about anymore. Note that we have to be on guard for peers doing crazy stuff like jumping their +`finalized_number` forward 10 trillion blocks to try and get us stuck in a loop for ages. -One of the safeguards we can implement is to reject view updates from peers where the new `finalized_number` is less than the previous. +One of the safeguards we can implement is to reject view updates from peers where the new `finalized_number` is less +than the previous. -We augment that by defining `constrain(x)` to output the x bounded by the first and last numbers in `state.blocks_by_number`. +We augment that by defining `constrain(x)` to output the x bounded by the first and last numbers in +`state.blocks_by_number`. -From there, we can loop backwards from `constrain(view.finalized_number)` until `constrain(last_view.finalized_number)` is reached, removing the `PeerId` from all `BlockEntry`s referenced at that height. We can break the loop early if we ever exit the bound supplied by the first block in `state.blocks_by_number`. +From there, we can loop backwards from `constrain(view.finalized_number)` until `constrain(last_view.finalized_number)` +is reached, removing the `PeerId` from all `BlockEntry`s referenced at that height. We can break the loop early if we +ever exit the bound supplied by the first block in `state.blocks_by_number`. #### `NetworkBridgeEvent::PeerMessage` -If the block hash referenced by the message exists in `pending_known`, add it to the vector of pending messages and return. +If the block hash referenced by the message exists in `pending_known`, add it to the vector of pending messages and +return. -If the message is of type `ApprovalDistributionV1Message::Assignment(assignment_cert, claimed_index)`, then call `import_and_circulate_assignment(MessageSource::Peer(sender), assignment_cert, claimed_index)` +If the message is of type `ApprovalDistributionV1Message::Assignment(assignment_cert, claimed_index)`, then call +`import_and_circulate_assignment(MessageSource::Peer(sender), assignment_cert, claimed_index)` -If the message is of type `ApprovalDistributionV1Message::Approval(approval_vote)`, then call `import_and_circulate_approval(MessageSource::Peer(sender), approval_vote)` +If the message is of type `ApprovalDistributionV1Message::Approval(approval_vote)`, then call +`import_and_circulate_approval(MessageSource::Peer(sender), approval_vote)` ### Subsystem Updates @@ -164,7 +196,8 @@ If the message is of type `ApprovalDistributionV1Message::Approval(approval_vote Create `BlockEntry` and `CandidateEntries` for all blocks. For all entries in `pending_known`: - * If there is now an entry under `blocks` for the block hash, drain all messages and import with `import_and_circulate_assignment` and `import_and_circulate_approval`. + * If there is now an entry under `blocks` for the block hash, drain all messages and import with + `import_and_circulate_assignment` and `import_and_circulate_approval`. For all peers: * Compute `view_intersection` as the intersection of the peer's view blocks with the hashes of the new blocks. @@ -180,7 +213,8 @@ Call `import_and_circulate_approval` with `MessageSource::Local`. #### `OverseerSignal::BlockFinalized` -Prune all lists from `blocks_by_number` with number less than or equal to `finalized_number`. Prune all the `BlockEntry`s referenced by those lists. +Prune all lists from `blocks_by_number` with number less than or equal to `finalized_number`. Prune all the +`BlockEntry`s referenced by those lists. ### Utility @@ -192,9 +226,14 @@ enum MessageSource { } ``` -#### `import_and_circulate_assignment(source: MessageSource, assignment: IndirectAssignmentCert, claimed_candidate_index: CandidateIndex)` +#### `import_and_circulate_assignment(...)` -Imports an assignment cert referenced by block hash and candidate index. As a postcondition, if the cert is valid, it will have distributed the cert to all peers who have the block in their view, with the exclusion of the peer referenced by the `MessageSource`. +`import_and_circulate_assignment(source: MessageSource, assignment: IndirectAssignmentCert, claimed_candidate_index: +CandidateIndex)` + +Imports an assignment cert referenced by block hash and candidate index. As a postcondition, if the cert is valid, it +will have distributed the cert to all peers who have the block in their view, with the exclusion of the peer referenced +by the `MessageSource`. We maintain a few invariants: * we only send an assignment to a peer after we add its fingerprint to our knowledge @@ -202,61 +241,84 @@ We maintain a few invariants: The algorithm is the following: - * Load the `BlockEntry` using `assignment.block_hash`. If it does not exist, report the source if it is `MessageSource::Peer` and return. + * Load the `BlockEntry` using `assignment.block_hash`. If it does not exist, report the source if it is + `MessageSource::Peer` and return. * Compute a fingerprint for the `assignment` using `claimed_candidate_index`. * If the source is `MessageSource::Peer(sender)`: - * check if `peer` appears under `known_by` and whether the fingerprint is in the knowledge of the peer. If the peer does not know the block, report for providing data out-of-view and proceed. If the peer does know the block and the `sent` knowledge contains the fingerprint, report for providing replicate data and return, otherwise, insert into the `received` knowledge and return. - * If the message fingerprint appears under the `BlockEntry`'s `Knowledge`, give the peer a small positive reputation boost, - add the fingerprint to the peer's knowledge only if it knows about the block and return. - Note that we must do this after checking for out-of-view and if the peers knows about the block to avoid being spammed. - If we did this check earlier, a peer could provide data out-of-view repeatedly and be rewarded for it. + * check if `peer` appears under `known_by` and whether the fingerprint is in the knowledge of the peer. If the peer + does not know the block, report for providing data out-of-view and proceed. If the peer does know the block and + the `sent` knowledge contains the fingerprint, report for providing replicate data and return, otherwise, insert + into the `received` knowledge and return. + * If the message fingerprint appears under the `BlockEntry`'s `Knowledge`, give the peer a small positive reputation + boost, add the fingerprint to the peer's knowledge only if it knows about the block and return. Note that we must do + this after checking for out-of-view and if the peers knows about the block to avoid being spammed. If we did this + check earlier, a peer could provide data out-of-view repeatedly and be rewarded for it. * Dispatch `ApprovalVotingMessage::CheckAndImportAssignment(assignment)` and wait for the response. * If the result is `AssignmentCheckResult::Accepted` * If the vote was accepted but not duplicate, give the peer a positive reputation boost - * add the fingerprint to both our and the peer's knowledge in the `BlockEntry`. Note that we only doing this after making sure we have the right fingerprint. - * If the result is `AssignmentCheckResult::AcceptedDuplicate`, add the fingerprint to the peer's knowledge if it knows about the block and return. + * add the fingerprint to both our and the peer's knowledge in the `BlockEntry`. Note that we only doing this after + making sure we have the right fingerprint. + * If the result is `AssignmentCheckResult::AcceptedDuplicate`, add the fingerprint to the peer's knowledge if it + knows about the block and return. * If the result is `AssignmentCheckResult::TooFarInFuture`, mildly punish the peer and return. * If the result is `AssignmentCheckResult::Bad`, punish the peer and return. * If the source is `MessageSource::Local(CandidateIndex)` * check if the fingerprint appears under the `BlockEntry's` knowledge. If not, add it. - * Load the candidate entry for the given candidate index. It should exist unless there is a logic error in the approval voting subsystem. - * Set the approval state for the validator index to `ApprovalState::Assigned` unless the approval state is set already. This should not happen as long as the approval voting subsystem instructs us to ignore duplicate assignments. - * Dispatch a `ApprovalDistributionV1Message::Assignment(assignment, candidate_index)` to all peers in the `BlockEntry`'s `known_by` set, excluding the peer in the `source`, if `source` has kind `MessageSource::Peer`. Add the fingerprint of the assignment to the knowledge of each peer. + * Load the candidate entry for the given candidate index. It should exist unless there is a logic error in the + approval voting subsystem. + * Set the approval state for the validator index to `ApprovalState::Assigned` unless the approval state is set + already. This should not happen as long as the approval voting subsystem instructs us to ignore duplicate + assignments. + * Dispatch a `ApprovalDistributionV1Message::Assignment(assignment, candidate_index)` to all peers in the + `BlockEntry`'s `known_by` set, excluding the peer in the `source`, if `source` has kind `MessageSource::Peer`. Add + the fingerprint of the assignment to the knowledge of each peer. #### `import_and_circulate_approval(source: MessageSource, approval: IndirectSignedApprovalVote)` Imports an approval signature referenced by block hash and candidate index: - * Load the `BlockEntry` using `approval.block_hash` and the candidate entry using `approval.candidate_entry`. If either does not exist, report the source if it is `MessageSource::Peer` and return. + * Load the `BlockEntry` using `approval.block_hash` and the candidate entry using `approval.candidate_entry`. If + either does not exist, report the source if it is `MessageSource::Peer` and return. * Compute a fingerprint for the approval. - * Compute a fingerprint for the corresponding assignment. If the `BlockEntry`'s knowledge does not contain that fingerprint, then report the source if it is `MessageSource::Peer` and return. All references to a fingerprint after this refer to the approval's, not the assignment's. + * Compute a fingerprint for the corresponding assignment. If the `BlockEntry`'s knowledge does not contain that + fingerprint, then report the source if it is `MessageSource::Peer` and return. All references to a fingerprint after + this refer to the approval's, not the assignment's. * If the source is `MessageSource::Peer(sender)`: - * check if `peer` appears under `known_by` and whether the fingerprint is in the knowledge of the peer. If the peer does not know the block, report for providing data out-of-view and proceed. If the peer does know the block and the `sent` knowledge contains the fingerprint, report for providing replicate data and return, otherwise, insert into the `received` knowledge and return. - * If the message fingerprint appears under the `BlockEntry`'s `Knowledge`, give the peer a small positive reputation boost, - add the fingerprint to the peer's knowledge only if it knows about the block and return. - Note that we must do this after checking for out-of-view to avoid being spammed. If we did this check earlier, a peer could provide data out-of-view repeatedly and be rewarded for it. + * check if `peer` appears under `known_by` and whether the fingerprint is in the knowledge of the peer. If the peer + does not know the block, report for providing data out-of-view and proceed. If the peer does know the block and + the `sent` knowledge contains the fingerprint, report for providing replicate data and return, otherwise, insert + into the `received` knowledge and return. + * If the message fingerprint appears under the `BlockEntry`'s `Knowledge`, give the peer a small positive reputation + boost, add the fingerprint to the peer's knowledge only if it knows about the block and return. Note that we must do + this after checking for out-of-view to avoid being spammed. If we did this check earlier, a peer could provide data + out-of-view repeatedly and be rewarded for it. * Dispatch `ApprovalVotingMessage::CheckAndImportApproval(approval)` and wait for the response. * If the result is `VoteCheckResult::Accepted(())`: * Give the peer a positive reputation boost and add the fingerprint to both our and the peer's knowledge. * If the result is `VoteCheckResult::Bad`: * Report the peer and return. - * Load the candidate entry for the given candidate index. It should exist unless there is a logic error in the approval voting subsystem. - * Set the approval state for the validator index to `ApprovalState::Approved`. It should already be in the `Assigned` state as our `BlockEntry` knowledge contains a fingerprint for the assignment. - * Dispatch a `ApprovalDistributionV1Message::Approval(approval)` to all peers in the `BlockEntry`'s `known_by` set, excluding the peer in the `source`, if `source` has kind `MessageSource::Peer`. Add the fingerprint of the assignment to the knowledge of each peer. Note that this obeys the politeness conditions: + * Load the candidate entry for the given candidate index. It should exist unless there is a logic error in the + approval voting subsystem. + * Set the approval state for the validator index to `ApprovalState::Approved`. It should already be in the `Assigned` + state as our `BlockEntry` knowledge contains a fingerprint for the assignment. + * Dispatch a `ApprovalDistributionV1Message::Approval(approval)` to all peers in the `BlockEntry`'s `known_by` set, + excluding the peer in the `source`, if `source` has kind `MessageSource::Peer`. Add the fingerprint of the + assignment to the knowledge of each peer. Note that this obeys the politeness conditions: * We guarantee elsewhere that all peers within `known_by` are aware of all assignments relative to the block. * We've checked that this specific approval has a corresponding assignment within the `BlockEntry`. * Thus, all peers are aware of the assignment or have a message to them in-flight which will make them so. - -#### `unify_with_peer(peer: PeerId, view)`: +#### `unify_with_peer(peer: PeerId, view)` 1. Initialize a set `missing_knowledge = {}` For each block in the view: - 2. Load the `BlockEntry` for the block. If the block is unknown, or the number is less than or equal to the view's finalized number go to step 6. - 3. Inspect the `known_by` set of the `BlockEntry`. If the peer already knows all assignments/approvals, go to step 6. - 4. Add the peer to `known_by` and add the hash and missing knowledge of the block to `missing_knowledge`. - 5. Return to step 2 with the ancestor of the block. - -6. For each block in `missing_knowledge`, send all assignments and approvals for all candidates in those blocks to the peer. + 1. Load the `BlockEntry` for the block. If the block is unknown, or the number is less than or equal to the view's + finalized number go to step 6. + 1. Inspect the `known_by` set of the `BlockEntry`. If the peer already knows all assignments/approvals, go to step 6. + 1. Add the peer to `known_by` and add the hash and missing knowledge of the block to `missing_knowledge`. + 1. Return to step 2 with the ancestor of the block. + +1. For each block in `missing_knowledge`, send all assignments and approvals for all candidates in those blocks to the + peer. diff --git a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md index 88744e50cf7956b6629a1f8b97895eae13287012..6da2f5d9b883a08877928bf6e7c2507ec1f38604 100644 --- a/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md +++ b/polkadot/roadmap/implementers-guide/src/node/approval/approval-voting.md @@ -1,35 +1,61 @@ # Approval Voting -Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the aims of this subsystem. - -Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to indicate intent to check a candidate. Upon successfully checking, they broadcast an approval vote. If a validator doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being prevented from recovering or validating the block data and that more validators should self-select to check the candidate. This is known as a "no-show". - -The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of `DelayTranche`s, which measure the number of ticks elapsed since a block was produced. We track metadata for all un-finalized but included candidates. We compute our local assignments to check each candidate, as well as which `DelayTranche` those assignments may be minimally triggered at. As the same candidate may appear in more than one block, we must produce our potential assignments for each (Block, Candidate) pair. The timing loop is based on waiting for assignments to become no-shows or waiting to broadcast and begin our own assignment to check. - -Another main component of this subsystem is the logic for determining when a (Block, Candidate) pair has been approved and when to broadcast and trigger our own assignment. Once a (Block, Candidate) pair has been approved, we mark a corresponding bit in the `BlockEntry` that indicates the candidate has been approved under the block. When we trigger our own assignment, we broadcast it via Approval Distribution, begin fetching the data from Availability Recovery, and then pass it through to the Candidate Validation. Once these steps are successful, we issue our approval vote. If any of these steps fail, we don't issue any vote and will "no-show" from the perspective of other validators in addition a dispute is raised via the dispute-coordinator, by sending `IssueLocalStatement`. - -Where this all fits into Polkadot is via block finality. Our goal is to not finalize any block containing a candidate that is not approved. We provide a hook for a custom GRANDPA voting rule - GRANDPA makes requests of the form (target, minimum) consisting of a target block (i.e. longest chain) that it would like to finalize, and a minimum block which, due to the rules of GRANDPA, must be voted on. The minimum is typically the last finalized block, but may be beyond it, in the case of having a last-round-estimate beyond the last finalized. Thus, our goal is to inform GRANDPA of some block between target and minimum which we believe can be finalized safely. We do this by iterating backwards from the target to the minimum and finding the longest continuous chain from minimum where all candidates included by those blocks have been approved. +Reading the [section on the approval protocol](../../protocol-approval.md) will likely be necessary to understand the +aims of this subsystem. + +Approval votes are split into two parts: Assignments and Approvals. Validators first broadcast their assignment to +indicate intent to check a candidate. Upon successfully checking, they broadcast an approval vote. If a validator +doesn't broadcast their approval vote shortly after issuing an assignment, this is an indication that they are being +prevented from recovering or validating the block data and that more validators should self-select to check the +candidate. This is known as a "no-show". + +The core of this subsystem is a Tick-based timer loop, where Ticks are 500ms. We also reason about time in terms of +`DelayTranche`s, which measure the number of ticks elapsed since a block was produced. We track metadata for all +un-finalized but included candidates. We compute our local assignments to check each candidate, as well as which +`DelayTranche` those assignments may be minimally triggered at. As the same candidate may appear in more than one block, +we must produce our potential assignments for each (Block, Candidate) pair. The timing loop is based on waiting for +assignments to become no-shows or waiting to broadcast and begin our own assignment to check. + +Another main component of this subsystem is the logic for determining when a (Block, Candidate) pair has been approved +and when to broadcast and trigger our own assignment. Once a (Block, Candidate) pair has been approved, we mark a +corresponding bit in the `BlockEntry` that indicates the candidate has been approved under the block. When we trigger +our own assignment, we broadcast it via Approval Distribution, begin fetching the data from Availability Recovery, and +then pass it through to the Candidate Validation. Once these steps are successful, we issue our approval vote. If any of +these steps fail, we don't issue any vote and will "no-show" from the perspective of other validators in addition a +dispute is raised via the dispute-coordinator, by sending `IssueLocalStatement`. + +Where this all fits into Polkadot is via block finality. Our goal is to not finalize any block containing a candidate +that is not approved. We provide a hook for a custom GRANDPA voting rule - GRANDPA makes requests of the form (target, +minimum) consisting of a target block (i.e. longest chain) that it would like to finalize, and a minimum block which, +due to the rules of GRANDPA, must be voted on. The minimum is typically the last finalized block, but may be beyond it, +in the case of having a last-round-estimate beyond the last finalized. Thus, our goal is to inform GRANDPA of some block +between target and minimum which we believe can be finalized safely. We do this by iterating backwards from the target +to the minimum and finding the longest continuous chain from minimum where all candidates included by those blocks have +been approved. ## Protocol Input: - - `ApprovalVotingMessage::CheckAndImportAssignment` - - `ApprovalVotingMessage::CheckAndImportApproval` - - `ApprovalVotingMessage::ApprovedAncestor` + * `ApprovalVotingMessage::CheckAndImportAssignment` + * `ApprovalVotingMessage::CheckAndImportApproval` + * `ApprovalVotingMessage::ApprovedAncestor` Output: - - `ApprovalDistributionMessage::DistributeAssignment` - - `ApprovalDistributionMessage::DistributeApproval` - - `RuntimeApiMessage::Request` - - `ChainApiMessage` - - `AvailabilityRecoveryMessage::Recover` - - `CandidateExecutionMessage::ValidateFromExhaustive` + * `ApprovalDistributionMessage::DistributeAssignment` + * `ApprovalDistributionMessage::DistributeApproval` + * `RuntimeApiMessage::Request` + * `ChainApiMessage` + * `AvailabilityRecoveryMessage::Recover` + * `CandidateExecutionMessage::ValidateFromExhaustive` ## Functionality -The approval voting subsystem is responsible for casting votes and determining approval of candidates and as a result, blocks. +The approval voting subsystem is responsible for casting votes and determining approval of candidates and as a result, +blocks. -This subsystem wraps a database which is used to store metadata about unfinalized blocks and the candidates within them. Candidates may appear in multiple blocks, and assignment criteria are chosen differently based on the hash of the block they appear in. +This subsystem wraps a database which is used to store metadata about unfinalized blocks and the candidates within them. +Candidates may appear in multiple blocks, and assignment criteria are chosen differently based on the hash of the block +they appear in. ## Database Schema @@ -150,16 +176,22 @@ struct State { } ``` -This guide section makes no explicit references to writes to or reads from disk. Instead, it handles them implicitly, with the understanding that updates to block, candidate, and approval entries are persisted to disk. +This guide section makes no explicit references to writes to or reads from disk. Instead, it handles them implicitly, +with the understanding that updates to block, candidate, and approval entries are persisted to disk. [`SessionInfo`](../../runtime/session_info.md) -On start-up, we clear everything currently stored by the database. This is done by loading the `StoredBlockRange`, iterating through each block number, iterating through each block hash, and iterating through each candidate referenced by each block. Although this is `O(o*n*p)`, we don't expect to have more than a few unfinalized blocks at any time and in extreme cases, a few thousand. The clearing operation should be relatively fast as a result. +On start-up, we clear everything currently stored by the database. This is done by loading the `StoredBlockRange`, +iterating through each block number, iterating through each block hash, and iterating through each candidate referenced +by each block. Although this is `O(o*n*p)`, we don't expect to have more than a few unfinalized blocks at any time and +in extreme cases, a few thousand. The clearing operation should be relatively fast as a result. Main loop: * Each iteration, select over all of - * The next `Tick` in `wakeups`: trigger `wakeup_process` for each `(Hash, Hash)` pair scheduled under the `Tick` and then remove all entries under the `Tick`. - * The next message from the overseer: handle the message as described in the [Incoming Messages section](#incoming-messages) + * The next `Tick` in `wakeups`: trigger `wakeup_process` for each `(Hash, Hash)` pair scheduled under the `Tick` and + then remove all entries under the `Tick`. + * The next message from the overseer: handle the message as described in the [Incoming Messages + section](#incoming-messages) * The next approval vote request from `background_rx` * If this is an `ApprovalVoteRequest`, [Issue an approval vote](#issue-approval-vote). @@ -167,41 +199,84 @@ Main loop: #### `OverseerSignal::BlockFinalized` -On receiving an `OverseerSignal::BlockFinalized(h)`, we fetch the block number `b` of that block from the `ChainApi` subsystem. We update our `StoredBlockRange` to begin at `b+1`. Additionally, we remove all block entries and candidates referenced by them up to and including `b`. Lastly, we prune out all descendants of `h` transitively: when we remove a `BlockEntry` with number `b` that is not equal to `h`, we recursively delete all the `BlockEntry`s referenced as children. We remove the `block_assignments` entry for the block hash and if `block_assignments` is now empty, remove the `CandidateEntry`. We also update each of the `BlockNumber -> Vec` keys in the database to reflect the blocks at that height, clearing if empty. +On receiving an `OverseerSignal::BlockFinalized(h)`, we fetch the block number `b` of that block from the `ChainApi` +subsystem. We update our `StoredBlockRange` to begin at `b+1`. Additionally, we remove all block entries and candidates +referenced by them up to and including `b`. Lastly, we prune out all descendants of `h` transitively: when we remove a +`BlockEntry` with number `b` that is not equal to `h`, we recursively delete all the `BlockEntry`s referenced as +children. We remove the `block_assignments` entry for the block hash and if `block_assignments` is now empty, remove the +`CandidateEntry`. We also update each of the `BlockNumber -> Vec` keys in the database to reflect the blocks at +that height, clearing if empty. #### `OverseerSignal::ActiveLeavesUpdate` On receiving an `OverseerSignal::ActiveLeavesUpdate(update)`: - * We determine the set of new blocks that were not in our previous view. This is done by querying the ancestry of all new items in the view and contrasting against the stored `BlockNumber`s. Typically, there will be only one new block. We fetch the headers and information on these blocks from the `ChainApi` subsystem. Stale leaves in the update can be ignored. + * We determine the set of new blocks that were not in our previous view. This is done by querying the ancestry of all + new items in the view and contrasting against the stored `BlockNumber`s. Typically, there will be only one new + block. We fetch the headers and information on these blocks from the `ChainApi` subsystem. Stale leaves in the + update can be ignored. * We update the `StoredBlockRange` and the `BlockNumber` maps. - * We use the `RuntimeApiSubsystem` to determine information about these blocks. It is generally safe to assume that runtime state is available for recent, unfinalized blocks. In the case that it isn't, it means that we are catching up to the head of the chain and needn't worry about assignments to those blocks anyway, as the security assumption of the protocol tolerates nodes being temporarily offline or out-of-date. - * We fetch the set of candidates included by each block by dispatching a `RuntimeApiRequest::CandidateEvents` and checking the `CandidateIncluded` events. - * We fetch the session of the block by dispatching a `session_index_for_child` request with the parent-hash of the block. - * If the `session index - APPROVAL_SESSIONS > state.earliest_session`, then bump `state.earliest_sessions` to that amount and prune earlier sessions. - * If the session isn't in our `state.session_info`, load the session info for it and for all sessions since the earliest-session, including the earliest-session, if that is missing. And it can be, just after pruning, if we've done a big jump forward, as is the case when we've just finished chain synchronization. + * We use the `RuntimeApiSubsystem` to determine information about these blocks. It is generally safe to assume that + runtime state is available for recent, unfinalized blocks. In the case that it isn't, it means that we are catching + up to the head of the chain and needn't worry about assignments to those blocks anyway, as the security assumption + of the protocol tolerates nodes being temporarily offline or out-of-date. + * We fetch the set of candidates included by each block by dispatching a `RuntimeApiRequest::CandidateEvents` and + checking the `CandidateIncluded` events. + * We fetch the session of the block by dispatching a `session_index_for_child` request with the parent-hash of the + block. + * If the `session index - APPROVAL_SESSIONS > state.earliest_session`, then bump `state.earliest_sessions` to that + amount and prune earlier sessions. + * If the session isn't in our `state.session_info`, load the session info for it and for all sessions since the + earliest-session, including the earliest-session, if that is missing. And it can be, just after pruning, if we've + done a big jump forward, as is the case when we've just finished chain synchronization. * If any of the runtime API calls fail, we just warn and skip the block. - * We use the `RuntimeApiSubsystem` to determine the set of candidates included in these blocks and use BABE logic to determine the slot number and VRF of the blocks. - * We also note how late we appear to have received the block. We create a `BlockEntry` for each block and a `CandidateEntry` for each candidate obtained from `CandidateIncluded` events after making a `RuntimeApiRequest::CandidateEvents` request. - * For each candidate, if the amount of needed approvals is more than the validators remaining after the backing group of the candidate is subtracted, then the candidate is insta-approved as approval would be impossible otherwise. If all candidates in the block are insta-approved, or there are no candidates in the block, then the block is insta-approved. If the block is insta-approved, a [`ChainSelectionMessage::Approved`][CSM] should be sent for the block. - * Ensure that the `CandidateEntry` contains a `block_assignments` entry for the block, with the correct backing group set. + * We use the `RuntimeApiSubsystem` to determine the set of candidates included in these blocks and use BABE logic to + determine the slot number and VRF of the blocks. + * We also note how late we appear to have received the block. We create a `BlockEntry` for each block and a + `CandidateEntry` for each candidate obtained from `CandidateIncluded` events after making a + `RuntimeApiRequest::CandidateEvents` request. + * For each candidate, if the amount of needed approvals is more than the validators remaining after the backing group + of the candidate is subtracted, then the candidate is insta-approved as approval would be impossible otherwise. If + all candidates in the block are insta-approved, or there are no candidates in the block, then the block is + insta-approved. If the block is insta-approved, a [`ChainSelectionMessage::Approved`][CSM] should be sent for the + block. + * Ensure that the `CandidateEntry` contains a `block_assignments` entry for the block, with the correct backing group + set. * If a validator in this session, compute and assign `our_assignment` for the `block_assignments` * Only if not a member of the backing group. - * Run `RelayVRFModulo` and `RelayVRFDelay` according to the [the approvals protocol section](../../protocol-approval.md#assignment-criteria). Ensure that the assigned core derived from the output is covered by the auxiliary signature aggregated in the `VRFPRoof`. - * [Handle Wakeup](#handle-wakeup) for each new candidate in each new block - this will automatically broadcast a 0-tranche assignment, kick off approval work, and schedule the next delay. + * Run `RelayVRFModulo` and `RelayVRFDelay` according to the [the approvals protocol + section](../../protocol-approval.md#assignment-criteria). Ensure that the assigned core derived from the output is + covered by the auxiliary signature aggregated in the `VRFPRoof`. + * [Handle Wakeup](#handle-wakeup) for each new candidate in each new block - this will automatically broadcast a + 0-tranche assignment, kick off approval work, and schedule the next delay. * Dispatch an `ApprovalDistributionMessage::NewBlocks` with the meta information filled out for each new block. #### `ApprovalVotingMessage::CheckAndImportAssignment` -On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we check the assignment cert against the block entry. The cert itself contains information necessary to determine the candidate that is being assigned-to. In detail: - * Load the `BlockEntry` for the relay-parent referenced by the message. If there is none, return `AssignmentCheckResult::Bad`. +On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we check the assignment cert against the block +entry. The cert itself contains information necessary to determine the candidate that is being assigned-to. In detail: + * Load the `BlockEntry` for the relay-parent referenced by the message. If there is none, return + `AssignmentCheckResult::Bad`. * Fetch the `SessionInfo` for the session of the block * Determine the assignment key of the validator based on that. - * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return `AssignmentCheckResult::Bad` if missing. + * Determine the claimed core index by looking up the candidate with given index in `block_entry.candidates`. Return + `AssignmentCheckResult::Bad` if missing. * Check the assignment cert - * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. - * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % (session_info.n_delay_tranches + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. - * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary signature. + * If the cert kind is `RelayVRFModulo`, then the certificate is valid as long as `sample < + session_info.relay_vrf_samples` and the VRF is valid for the validator's key with the input + `block_entry.relay_vrf_story ++ sample.encode()` as described with [the approvals protocol + section](../../protocol-approval.md#assignment-criteria). We set `core_index = vrf.make_bytes().to_u32() % + session_info.n_cores`. If the `BlockEntry` causes inclusion of a candidate at `core_index`, then this is a valid + assignment for the candidate at `core_index` and has delay tranche 0. Otherwise, it can be ignored. + * If the cert kind is `RelayVRFDelay`, then we check if the VRF is valid for the validator's key with the input + `block_entry.relay_vrf_story ++ cert.core_index.encode()` as described in [the approvals protocol + section](../../protocol-approval.md#assignment-criteria). The cert can be ignored if the block did not cause + inclusion of a candidate on that core index. Otherwise, this is a valid assignment for the included candidate. The + delay tranche for the assignment is determined by reducing `(vrf.make_bytes().to_u64() % + (session_info.n_delay_tranches + + session_info.zeroth_delay_tranche_width)).saturating_sub(session_info.zeroth_delay_tranche_width)`. + * We also check that the core index derived by the output is covered by the `VRFProof` by means of an auxiliary + signature. * If the delay tranche is too far in the future, return `AssignmentCheckResult::TooFarInFuture`. * Import the assignment. * Load the candidate in question and access the `approval_entry` for the block hash the cert references. @@ -217,32 +292,41 @@ On receiving a `ApprovalVotingMessage::CheckAndImportAssignment` message, we che On receiving a `CheckAndImportApproval(indirect_approval_vote, response_channel)` message: * Fetch the `BlockEntry` from the indirect approval vote's `block_hash`. If none, return `ApprovalCheckResult::Bad`. - * Fetch the `CandidateEntry` from the indirect approval vote's `candidate_index`. If the block did not trigger inclusion of enough candidates, return `ApprovalCheckResult::Bad`. - * Construct a `SignedApprovalVote` using the candidate hash and check against the validator's approval key, based on the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. + * Fetch the `CandidateEntry` from the indirect approval vote's `candidate_index`. If the block did not trigger + inclusion of enough candidates, return `ApprovalCheckResult::Bad`. + * Construct a `SignedApprovalVote` using the candidate hash and check against the validator's approval key, based on + the session info of the block. If invalid or no such validator, return `ApprovalCheckResult::Bad`. * Send `ApprovalCheckResult::Accepted` * [Import the checked approval vote](#import-checked-approval) #### `ApprovalVotingMessage::ApprovedAncestor` On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: - * Iterate over the ancestry of the hash all the way back to block number given, starting from the provided block hash. Load the `CandidateHash`es from each block entry. + * Iterate over the ancestry of the hash all the way back to block number given, starting from the provided block hash. + Load the `CandidateHash`es from each block entry. * Keep track of an `all_approved_max: Option<(Hash, BlockNumber, Vec<(Hash, Vec))>`. - * For each block hash encountered, load the `BlockEntry` associated. If any are not found, return `None` on the response channel and conclude. - * If the block entry's `approval_bitfield` has all bits set to 1 and `all_approved_max == None`, set `all_approved_max = Some((current_hash, current_number))`. + * For each block hash encountered, load the `BlockEntry` associated. If any are not found, return `None` on the + response channel and conclude. + * If the block entry's `approval_bitfield` has all bits set to 1 and `all_approved_max == None`, set `all_approved_max + = Some((current_hash, current_number))`. * If the block entry's `approval_bitfield` has any 0 bits, set `all_approved_max = None`. - * If `all_approved_max` is `Some`, push the current block hash and candidate hashes onto the list of blocks and candidates `all_approved_max`. + * If `all_approved_max` is `Some`, push the current block hash and candidate hashes onto the list of blocks and + candidates `all_approved_max`. * After iterating all ancestry, return `all_approved_max`. ### Updates and Auxiliary Logic #### Import Checked Approval - * Import an approval vote which we can assume to have passed signature checks and correspond to an imported assignment. + * Import an approval vote which we can assume to have passed signature checks and correspond to an imported + assignment. * Requires `(BlockEntry, CandidateEntry, ValidatorIndex)` * Set the corresponding bit of the `approvals` bitfield in the `CandidateEntry` to `1`. If already `1`, return. - * Checks the approval state of a candidate under a specific block, and updates the block and candidate entries accordingly. + * Checks the approval state of a candidate under a specific block, and updates the block and candidate entries + accordingly. * Checks the `ApprovalEntry` for the block. * [determine the tranches to inspect](#determine-required-tranches) of the candidate, - * [the candidate is approved under the block](#check-approval), set the corresponding bit in the `block_entry.approved_bitfield`. + * [the candidate is approved under the block](#check-approval), set the corresponding bit in the + `block_entry.approved_bitfield`. * If the block is now fully approved and was not before, send a [`ChainSelectionMessage::Approved`][CSM]. * Otherwise, [schedule a wakeup of the candidate](#schedule-wakeup) * If the approval vote originates locally, set the `our_approval_sig` in the candidate entry. @@ -250,13 +334,19 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Handling Wakeup * Handle a previously-scheduled wakeup of a candidate under a specific block. * Requires `(relay_block, candidate_hash)` - * Load the `BlockEntry` and `CandidateEntry` from disk. If either is not present, this may have lost a race with finality and can be ignored. Also load the `ApprovalEntry` for the block and candidate. + * Load the `BlockEntry` and `CandidateEntry` from disk. If either is not present, this may have lost a race with + finality and can be ignored. Also load the `ApprovalEntry` for the block and candidate. * [determine the `RequiredTranches` of the candidate](#determine-required-tranches). * Determine if we should trigger our assignment. * If we've already triggered or `OurAssignment` is `None`, we do not trigger. - * If we have `RequiredTranches::All`, then we trigger if the candidate is [not approved](#check-approval). We have no next wakeup as we assume that other validators are doing the same and we will be implicitly woken up by handling new votes. - * If we have `RequiredTranches::Pending { considered, next_no_show, uncovered, maximum_broadcast, clock_drift }`, then we trigger if our assignment's tranche is less than or equal to `maximum_broadcast` and the current tick, with `clock_drift` applied, is at least the tick of our tranche. - * If we have `RequiredTranches::Exact { .. }` then we do not trigger, because this value indicates that no new assignments are needed at the moment. + * If we have `RequiredTranches::All`, then we trigger if the candidate is [not approved](#check-approval). We have + no next wakeup as we assume that other validators are doing the same and we will be implicitly woken up by + handling new votes. + * If we have `RequiredTranches::Pending { considered, next_no_show, uncovered, maximum_broadcast, clock_drift }`, + then we trigger if our assignment's tranche is less than or equal to `maximum_broadcast` and the current tick, + with `clock_drift` applied, is at least the tick of our tranche. + * If we have `RequiredTranches::Exact { .. }` then we do not trigger, because this value indicates that no new + assignments are needed at the moment. * If we should trigger our assignment * Import the assignment to the `ApprovalEntry` * Broadcast on network with an `ApprovalDistributionMessage::DistributeAssignment`. @@ -265,26 +355,39 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Schedule Wakeup - * Requires `(approval_entry, candidate_entry)` which effectively denotes a `(Block Hash, Candidate Hash)` pair - the candidate, along with the block it appears in. + * Requires `(approval_entry, candidate_entry)` which effectively denotes a `(Block Hash, Candidate Hash)` pair - the + candidate, along with the block it appears in. * Also requires `RequiredTranches` * If the `approval_entry` is approved, this doesn't need to be woken up again. - * If `RequiredTranches::All` - no wakeup. We assume other incoming votes will trigger wakeup and potentially re-schedule. - * If `RequiredTranches::Pending { considered, next_no_show, uncovered, maximum_broadcast, clock_drift }` - schedule at the lesser of the next no-show tick, or the tick, offset positively by `clock_drift` of the next non-empty tranche we are aware of after `considered`, including any tranche containing our own unbroadcast assignment. This can lead to no wakeup in the case that we have already broadcast our assignment and there are no pending no-shows; that is, we have approval votes for every assignment we've received that is not already a no-show. In this case, we will be re-triggered by other validators broadcasting their assignments. - * If `RequiredTranches::Exact { next_no_show, latest_assignment_tick, .. }` - set a wakeup for the earlier of the next no-show tick or the latest assignment tick + `APPROVAL_DELAY`. + * If `RequiredTranches::All` - no wakeup. We assume other incoming votes will trigger wakeup and potentially + re-schedule. + * If `RequiredTranches::Pending { considered, next_no_show, uncovered, maximum_broadcast, clock_drift }` - schedule at + the lesser of the next no-show tick, or the tick, offset positively by `clock_drift` of the next non-empty tranche + we are aware of after `considered`, including any tranche containing our own unbroadcast assignment. This can lead + to no wakeup in the case that we have already broadcast our assignment and there are no pending no-shows; that is, + we have approval votes for every assignment we've received that is not already a no-show. In this case, we will be + re-triggered by other validators broadcasting their assignments. + * If `RequiredTranches::Exact { next_no_show, latest_assignment_tick, .. }` - set a wakeup for the earlier of the next + no-show tick or the latest assignment tick + `APPROVAL_DELAY`. #### Launch Approval Work * Requires `(SessionIndex, SessionInfo, CandidateReceipt, ValidatorIndex, backing_group, block_hash, candidate_index)` * Extract the public key of the `ValidatorIndex` from the `SessionInfo` for the session. -* Issue an `AvailabilityRecoveryMessage::RecoverAvailableData(candidate, session_index, Some(backing_group), response_sender)` -* Load the historical validation code of the parachain by dispatching a `RuntimeApiRequest::ValidationCodeByHash(descriptor.validation_code_hash)` against the state of `block_hash`. +* Issue an `AvailabilityRecoveryMessage::RecoverAvailableData(candidate, session_index, Some(backing_group), + response_sender)` +* Load the historical validation code of the parachain by dispatching a + `RuntimeApiRequest::ValidationCodeByHash(descriptor.validation_code_hash)` against the state of `block_hash`. * Spawn a background task with a clone of `background_tx` * Wait for the available data - * Issue a `CandidateValidationMessage::ValidateFromExhaustive` message with `APPROVAL_EXECUTION_TIMEOUT` as the timeout parameter. + * Issue a `CandidateValidationMessage::ValidateFromExhaustive` message with `APPROVAL_EXECUTION_TIMEOUT` as the + timeout parameter. * Wait for the result of validation * Check that the result of validation, if valid, matches the commitments in the receipt. * If valid, issue a message on `background_tx` detailing the request. - * If any of the data, the candidate, or the commitments are invalid, issue on `background_tx` a [`DisputeCoordinatorMessage::IssueLocalStatement`](../../types/overseer-protocol.md#dispute-coordinator-message) with `valid = false` to initiate a dispute. + * If any of the data, the candidate, or the commitments are invalid, issue on `background_tx` a + [`DisputeCoordinatorMessage::IssueLocalStatement`](../../types/overseer-protocol.md#dispute-coordinator-message) + with `valid = false` to initiate a dispute. #### Issue Approval Vote * Fetch the block entry and candidate entry. Ignore if `None` - we've probably just lost a race with finality. @@ -297,14 +400,22 @@ On receiving an `ApprovedAncestor(Hash, BlockNumber, response_channel)`: #### Determine Required Tranches -This logic is for inspecting an approval entry that tracks the assignments received, along with information on which assignments have corresponding approval votes. Inspection also involves the current time and expected requirements and is used to help the higher-level code determine the following: +This logic is for inspecting an approval entry that tracks the assignments received, along with information on which +assignments have corresponding approval votes. Inspection also involves the current time and expected requirements and +is used to help the higher-level code determine the following: * Whether to broadcast the local assignment * Whether to check that the candidate entry has been completely approved. - * If the candidate is waiting on approval, when to schedule the next wakeup of the `(candidate, block)` pair at a point where the state machine could be advanced. + * If the candidate is waiting on approval, when to schedule the next wakeup of the `(candidate, block)` pair at a + point where the state machine could be advanced. -These routines are pure functions which only depend on the environmental state. The expectation is that this determination is re-run every time we attempt to update an approval entry: either when we trigger a wakeup to advance the state machine based on a no-show or our own broadcast, or when we receive further assignments or approvals from the network. +These routines are pure functions which only depend on the environmental state. The expectation is that this +determination is re-run every time we attempt to update an approval entry: either when we trigger a wakeup to advance +the state machine based on a no-show or our own broadcast, or when we receive further assignments or approvals from the +network. -Thus it may be that at some point in time, we consider that tranches 0..X is required to be considered, but as we receive more information, we might require fewer tranches. Or votes that we perceived to be missing and require replacement are filled in and change our view. +Thus it may be that at some point in time, we consider that tranches 0..X is required to be considered, but as we +receive more information, we might require fewer tranches. Or votes that we perceived to be missing and require +replacement are filled in and change our view. Requires `(approval_entry, approvals_received, tranche_now, block_tick, no_show_duration, needed_approvals)` @@ -327,7 +438,8 @@ enum RequiredTranches { /// as though it is `clock_drift` ticks earlier. clock_drift: Tick, }, - // An exact number of required tranches and a number of no-shows. This indicates that the amount of `needed_approvals` are assigned and additionally all no-shows are covered. + // An exact number of required tranches and a number of no-shows. This indicates that the amount of `needed_approvals` + // are assigned and additionally all no-shows are covered. Exact { /// The tranche to inspect up to. needed: DelayTranche, @@ -345,13 +457,21 @@ enum RequiredTranches { **Clock-drift and Tranche-taking** -Our vote-counting procedure depends heavily on how we interpret time based on the presence of no-shows - assignments which have no corresponding approval after some time. +Our vote-counting procedure depends heavily on how we interpret time based on the presence of no-shows - assignments +which have no corresponding approval after some time. We have this is because of how we handle no-shows: we keep track of the depth of no-shows we are covering. -As an example: there may be initial no-shows in tranche 0. It'll take `no_show_duration` ticks before those are considered no-shows. Then, we don't want to immediately take `no_show_duration` more tranches. Instead, we want to take one tranche for each uncovered no-show. However, as we take those tranches, there may be further no-shows. Since these depth-1 no-shows should have only been triggered after the depth-0 no-shows were already known to be no-shows, we need to discount the local clock by `no_show_duration` to see whether these should be considered no-shows or not. There may be malicious parties who broadcast their assignment earlier than they were meant to, who shouldn't be counted as instant no-shows. We continue onwards to cover all depth-1 no-shows which may lead to depth-2 no-shows and so on. +As an example: there may be initial no-shows in tranche 0. It'll take `no_show_duration` ticks before those are +considered no-shows. Then, we don't want to immediately take `no_show_duration` more tranches. Instead, we want to take +one tranche for each uncovered no-show. However, as we take those tranches, there may be further no-shows. Since these +depth-1 no-shows should have only been triggered after the depth-0 no-shows were already known to be no-shows, we need +to discount the local clock by `no_show_duration` to see whether these should be considered no-shows or not. There may +be malicious parties who broadcast their assignment earlier than they were meant to, who shouldn't be counted as instant +no-shows. We continue onwards to cover all depth-1 no-shows which may lead to depth-2 no-shows and so on. -Likewise, when considering how many tranches to take, the no-show depth should be used to apply a depth-discount or clock drift to the `tranche_now`. +Likewise, when considering how many tranches to take, the no-show depth should be used to apply a depth-discount or +clock drift to the `tranche_now`. **Procedure** @@ -360,21 +480,35 @@ Likewise, when considering how many tranches to take, the no-show depth should b * Take tranches up to `tranche_now - clock_drift` until all needed assignments are met. * Keep track of the `next_no_show` according to the clock drift, as we go. * Keep track of the `last_assignment_tick` as we go. - * If running out of tranches before then, return `Pending { considered, next_no_show, maximum_broadcast, clock_drift }` + * If running out of tranches before then, return `Pending { considered, next_no_show, maximum_broadcast, clock_drift + }` * If there are no no-shows, return `Exact { needed, tolerated_missing, next_no_show, last_assignment_tick }` - * `maximum_broadcast` is either `DelayTranche::max_value()` at tranche 0 or otherwise by the last considered tranche + the number of uncovered no-shows at this point. - * If there are no-shows, return to the beginning, incrementing `depth` and attempting to cover the number of no-shows. Each no-show must be covered by a non-empty tranche, which are tranches that have at least one assignment. Each non-empty tranche covers exactly one no-show. - * If at any point, it seems that all validators are required, do an early return with `RequiredTranches::All` which indicates that everyone should broadcast. + * `maximum_broadcast` is either `DelayTranche::max_value()` at tranche 0 or otherwise by the last considered tranche + + the number of uncovered no-shows at this point. + * If there are no-shows, return to the beginning, incrementing `depth` and attempting to cover the number of no-shows. + Each no-show must be covered by a non-empty tranche, which are tranches that have at least one assignment. Each + non-empty tranche covers exactly one no-show. + * If at any point, it seems that all validators are required, do an early return with `RequiredTranches::All` which + indicates that everyone should broadcast. #### Check Approval * Check whether a candidate is approved under a particular block. * Requires `(block_entry, candidate_entry, approval_entry, n_tranches)` - * If we have `3 * n_approvals > n_validators`, return true. This is because any set with f+1 validators must have at least one honest validator, who has approved the candidate. + * If we have `3 * n_approvals > n_validators`, return true. This is because any set with f+1 validators must have at + least one honest validator, who has approved the candidate. * If `n_tranches` is `RequiredTranches::Pending`, return false * If `n_tranches` is `RequiredTranches::All`, return false. - * If `n_tranches` is `RequiredTranches::Exact { tranche, tolerated_missing, latest_assignment_tick, .. }`, then we return whether all assigned validators up to `tranche` less `tolerated_missing` have approved and `latest_assignment_tick + APPROVAL_DELAY >= tick_now`. - * e.g. if we had 5 tranches and 1 tolerated missing, we would accept only if all but 1 of assigned validators in tranches 0..=5 have approved. In that example, we also accept all validators in tranches 0..=5 having approved, but that would indicate that the `RequiredTranches` value was incorrectly constructed, so it is not realistic. `tolerated_missing` actually represents covered no-shows. If there are more missing approvals than there are tolerated missing, that indicates that there are some assignments which are not yet no-shows, but may become no-shows, and we should wait for the validators to either approve or become no-shows. - * e.g. If the above passes and the `latest_assignment_tick` was 5 and the current tick was 6, then we'd return false. + * If `n_tranches` is `RequiredTranches::Exact { tranche, tolerated_missing, latest_assignment_tick, .. }`, then we + return whether all assigned validators up to `tranche` less `tolerated_missing` have approved and + `latest_assignment_tick + APPROVAL_DELAY >= tick_now`. + * e.g. if we had 5 tranches and 1 tolerated missing, we would accept only if all but 1 of assigned validators in + tranches 0..=5 have approved. In that example, we also accept all validators in tranches 0..=5 having approved, + but that would indicate that the `RequiredTranches` value was incorrectly constructed, so it is not realistic. + `tolerated_missing` actually represents covered no-shows. If there are more missing approvals than there are + tolerated missing, that indicates that there are some assignments which are not yet no-shows, but may become + no-shows, and we should wait for the validators to either approve or become no-shows. + * e.g. If the above passes and the `latest_assignment_tick` was 5 and the current tick was 6, then we'd return + false. ### Time diff --git a/polkadot/roadmap/implementers-guide/src/node/availability/README.md b/polkadot/roadmap/implementers-guide/src/node/availability/README.md index 76bd6467e178917b396173be1d59aca32e5b3930..2fcfd58de9e886a21eebe033cda09e22c9f26cd6 100644 --- a/polkadot/roadmap/implementers-guide/src/node/availability/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/availability/README.md @@ -1,3 +1,7 @@ # Availability Subsystems -The availability subsystems are responsible for ensuring that Proofs of Validity of backed candidates are widely available within the validator set, without requiring every node to retain a full copy. They accomplish this by broadly distributing erasure-coded chunks of the PoV, keeping track of which validator has which chunk by means of signed bitfields. They are also responsible for reassembling a complete PoV when required, e.g. when an approval checker needs to validate a parachain block. +The availability subsystems are responsible for ensuring that Proofs of Validity of backed candidates are widely +available within the validator set, without requiring every node to retain a full copy. They accomplish this by broadly +distributing erasure-coded chunks of the PoV, keeping track of which validator has which chunk by means of signed +bitfields. They are also responsible for reassembling a complete PoV when required, e.g. when an approval checker needs +to validate a parachain block. diff --git a/polkadot/roadmap/implementers-guide/src/node/availability/availability-distribution.md b/polkadot/roadmap/implementers-guide/src/node/availability/availability-distribution.md index 1760a2ee7df4ef44d33ce514f02c34a1620083f2..931d82d529b9b690c57a47acb0dc1cf01914f207 100644 --- a/polkadot/roadmap/implementers-guide/src/node/availability/availability-distribution.md +++ b/polkadot/roadmap/implementers-guide/src/node/availability/availability-distribution.md @@ -1,31 +1,26 @@ # Availability Distribution -This subsystem is responsible for distribution availability data to peers. -Availability data are chunks, `PoV`s and `AvailableData` (which is `PoV` + -`PersistedValidationData`). It does so via request response protocols. +This subsystem is responsible for distribution availability data to peers. Availability data are chunks, `PoV`s and +`AvailableData` (which is `PoV` + `PersistedValidationData`). It does so via request response protocols. In particular this subsystem is responsible for: -- Respond to network requests requesting availability data by querying the - [Availability Store](../utility/availability-store.md). -- Request chunks from backing validators to put them in the local `Availability - Store` whenever we find an occupied core on any fresh leaf, - this is to ensure availability by at least 2/3+ of all validators, this - happens after a candidate is backed. -- Fetch `PoV` from validators, when requested via `FetchPoV` message from - backing (`pov_requester` module). +- Respond to network requests requesting availability data by querying the [Availability + Store](../utility/availability-store.md). +- Request chunks from backing validators to put them in the local `Availability Store` whenever we find an occupied core + on any fresh leaf, this is to ensure availability by at least 2/3+ of all validators, this happens after a candidate + is backed. +- Fetch `PoV` from validators, when requested via `FetchPoV` message from backing (`pov_requester` module). -The backing subsystem is responsible of making available data available in the -local `Availability Store` upon validation. This subsystem will serve any -network requests by querying that store. +The backing subsystem is responsible of making available data available in the local `Availability Store` upon +validation. This subsystem will serve any network requests by querying that store. ## Protocol -This subsystem does not handle any peer set messages, but the `pov_requester` -does connect to validators of the same backing group on the validation peer -set, to ensure fast propagation of statements between those validators and for -ensuring already established connections for requesting `PoV`s. Other than that -this subsystem drives request/response protocols. +This subsystem does not handle any peer set messages, but the `pov_requester` does connect to validators of the same +backing group on the validation peer set, to ensure fast propagation of statements between those validators and for +ensuring already established connections for requesting `PoV`s. Other than that this subsystem drives request/response +protocols. Input: @@ -48,51 +43,42 @@ Output: ### PoV Requester -The PoV requester in the `pov_requester` module takes care of staying connected -to validators of the current backing group of this very validator on the `Validation` -peer set and it will handle `FetchPoV` requests by issuing network requests to -those validators. It will check the hash of the received `PoV`, but will not do any -further validation. That needs to be done by the original `FetchPoV` sender -(backing subsystem). +The PoV requester in the `pov_requester` module takes care of staying connected to validators of the current backing +group of this very validator on the `Validation` peer set and it will handle `FetchPoV` requests by issuing network +requests to those validators. It will check the hash of the received `PoV`, but will not do any further validation. That +needs to be done by the original `FetchPoV` sender (backing subsystem). ### Chunk Requester -After a candidate is backed, the availability of the PoV block must be confirmed -by 2/3+ of all validators. The chunk requester is responsible of making that -availability a reality. +After a candidate is backed, the availability of the PoV block must be confirmed by 2/3+ of all validators. The chunk +requester is responsible of making that availability a reality. -It does that by querying checking occupied cores for all active leaves. For each -occupied core it will spawn a task fetching the erasure chunk which has the -`ValidatorIndex` of the node. For this an `ChunkFetchingRequest` is issued, via -substrate's generic request/response protocol. +It does that by querying checking occupied cores for all active leaves. For each occupied core it will spawn a task +fetching the erasure chunk which has the `ValidatorIndex` of the node. For this an `ChunkFetchingRequest` is issued, via +Substrate's generic request/response protocol. -The spawned task will start trying to fetch the chunk from validators in -responsible group of the occupied core, in a random order. For ensuring that we -use already open TCP connections wherever possible, the requester maintains a -cache and preserves that random order for the entire session. +The spawned task will start trying to fetch the chunk from validators in responsible group of the occupied core, in a +random order. For ensuring that we use already open TCP connections wherever possible, the requester maintains a cache +and preserves that random order for the entire session. -Note however that, because not all validators in a group have to be actual -backers, not all of them are required to have the needed chunk. This in turn -could lead to low throughput, as we have to wait for fetches to fail, -before reaching a validator finally having our chunk. We do rank back validators -not delivering our chunk, but as backers could vary from block to block on a -perfectly legitimate basis, this is still not ideal. See issues [2509](https://github.com/paritytech/polkadot/issues/2509) and [2512](https://github.com/paritytech/polkadot/issues/2512) -for more information. +Note however that, because not all validators in a group have to be actual backers, not all of them are required to have +the needed chunk. This in turn could lead to low throughput, as we have to wait for fetches to fail, before reaching a +validator finally having our chunk. We do rank back validators not delivering our chunk, but as backers could vary from +block to block on a perfectly legitimate basis, this is still not ideal. See issues +[2509](https://github.com/paritytech/polkadot/issues/2509) and +[2512](https://github.com/paritytech/polkadot/issues/2512) for more information. -The current implementation also only fetches chunks for occupied cores in blocks -in active leaves. This means though, if active leaves skips a block or we are -particularly slow in fetching our chunk, we might not fetch our chunk if -availability reached 2/3 fast enough (slot becomes free). This is not desirable -as we would like as many validators as possible to have their chunk. See this -[issue](https://github.com/paritytech/polkadot/issues/2513) for more details. +The current implementation also only fetches chunks for occupied cores in blocks in active leaves. This means though, if +active leaves skips a block or we are particularly slow in fetching our chunk, we might not fetch our chunk if +availability reached 2/3 fast enough (slot becomes free). This is not desirable as we would like as many validators as +possible to have their chunk. See this [issue](https://github.com/paritytech/polkadot/issues/2513) for more details. ### Serving -On the other side the subsystem will listen for incoming `ChunkFetchingRequest`s -and `PoVFetchingRequest`s from the network bridge and will respond to queries, -by looking the requested chunks and `PoV`s up in the availability store, this -happens in the `responder` module. +On the other side the subsystem will listen for incoming `ChunkFetchingRequest`s and `PoVFetchingRequest`s from the +network bridge and will respond to queries, by looking the requested chunks and `PoV`s up in the availability store, +this happens in the `responder` module. -We rely on the backing subsystem to make available data available locally in the -`Availability Store` after it has validated it. +We rely on the backing subsystem to make available data available locally in the `Availability Store` after it has +validated it. diff --git a/polkadot/roadmap/implementers-guide/src/node/availability/availability-recovery.md b/polkadot/roadmap/implementers-guide/src/node/availability/availability-recovery.md index ac733c1ce9186c182140e42dfb822006b7aac33b..e3bb14db3a55499dfc1546494c8e83d5e6cb6ed7 100644 --- a/polkadot/roadmap/implementers-guide/src/node/availability/availability-recovery.md +++ b/polkadot/roadmap/implementers-guide/src/node/availability/availability-recovery.md @@ -1,8 +1,13 @@ # Availability Recovery -This subsystem is the inverse of the [Availability Distribution](availability-distribution.md) subsystem: validators will serve the availability chunks kept in the availability store to nodes who connect to them. And the subsystem will also implement the other side: the logic for nodes to connect to validators, request availability pieces, and reconstruct the `AvailableData`. +This subsystem is the inverse of the [Availability Distribution](availability-distribution.md) subsystem: validators +will serve the availability chunks kept in the availability store to nodes who connect to them. And the subsystem will +also implement the other side: the logic for nodes to connect to validators, request availability pieces, and +reconstruct the `AvailableData`. -This version of the availability recovery subsystem is based off of direct connections to validators. In order to recover any given `AvailableData`, we must recover at least `f + 1` pieces from validators of the session. Thus, we will connect to and query randomly chosen validators until we have received `f + 1` pieces. +This version of the availability recovery subsystem is based off of direct connections to validators. In order to +recover any given `AvailableData`, we must recover at least `f + 1` pieces from validators of the session. Thus, we will +connect to and query randomly chosen validators until we have received `f + 1` pieces. ## Protocol @@ -10,18 +15,20 @@ This version of the availability recovery subsystem is based off of direct conne Input: -- `NetworkBridgeUpdate(update)` -- `AvailabilityRecoveryMessage::RecoverAvailableData(candidate, session, backing_group, response)` +* `NetworkBridgeUpdate(update)` +* `AvailabilityRecoveryMessage::RecoverAvailableData(candidate, session, backing_group, response)` Output: -- `NetworkBridge::SendValidationMessage` -- `NetworkBridge::ReportPeer` -- `AvailabilityStore::QueryChunk` +* `NetworkBridge::SendValidationMessage` +* `NetworkBridge::ReportPeer` +* `AvailabilityStore::QueryChunk` ## Functionality -We hold a state which tracks the currently ongoing recovery tasks, as well as which request IDs correspond to which task. A recovery task is a structure encapsulating all recovery tasks with the network necessary to recover the available data in respect to one candidate. +We hold a state which tracks the currently ongoing recovery tasks, as well as which request IDs correspond to which +task. A recovery task is a structure encapsulating all recovery tasks with the network necessary to recover the +available data in respect to one candidate. ```rust struct State { @@ -87,17 +94,22 @@ On `Conclude`, shut down the subsystem. 1. Check the `availability_lru` for the candidate and return the data if so. 1. Check if there is already an recovery handle for the request. If so, add the response handle to it. -1. Otherwise, load the session info for the given session under the state of `live_block_hash`, and initiate a recovery task with *`launch_recovery_task`*. Add a recovery handle to the state and add the response channel to it. +1. Otherwise, load the session info for the given session under the state of `live_block_hash`, and initiate a recovery + task with *`launch_recovery_task`*. Add a recovery handle to the state and add the response channel to it. 1. If the session info is not available, return `RecoveryError::Unavailable` on the response channel. ### Recovery logic #### `launch_recovery_task(session_index, session_info, candidate_receipt, candidate_hash, Option)` -1. Compute the threshold from the session info. It should be `f + 1`, where `n = 3f + k`, where `k in {1, 2, 3}`, and `n` is the number of validators. -1. Set the various fields of `RecoveryParams` based on the validator lists in `session_info` and information about the candidate. -1. If the `backing_group_index` is `Some`, start in the `RequestFromBackers` phase with a shuffling of the backing group validator indices and a `None` requesting value. -1. Otherwise, start in the `RequestChunksFromValidators` source with `received_chunks`,`requesting_chunks`, and `next_shuffling` all empty. +1. Compute the threshold from the session info. It should be `f + 1`, where `n = 3f + k`, where `k in {1, 2, 3}`, and + `n` is the number of validators. +1. Set the various fields of `RecoveryParams` based on the validator lists in `session_info` and information about the + candidate. +1. If the `backing_group_index` is `Some`, start in the `RequestFromBackers` phase with a shuffling of the backing group + validator indices and a `None` requesting value. +1. Otherwise, start in the `RequestChunksFromValidators` source with `received_chunks`,`requesting_chunks`, and + `next_shuffling` all empty. 1. Set the `to_subsystems` sender to be equal to a clone of the `SubsystemContext`'s sender. 1. Initialize `received_chunks` to an empty set, as well as `requesting_chunks`. @@ -115,19 +127,24 @@ const N_PARALLEL: usize = 50; * Loop: * If the `requesting_pov` is `Some`, poll for updates on it. If it concludes, set `requesting_pov` to `None`. * If the `requesting_pov` is `None`, take the next backer off the `shuffled_backers`. - * If the backer is `Some`, issue a `NetworkBridgeMessage::Requests` with a network request for the `AvailableData` and wait for the response. + * If the backer is `Some`, issue a `NetworkBridgeMessage::Requests` with a network request for the + `AvailableData` and wait for the response. * If it concludes with a `None` result, return to beginning. * If it concludes with available data, attempt a re-encoding. * If it has the correct erasure-root, break and issue a `Ok(available_data)`. * If it has an incorrect erasure-root, return to beginning. * Send the result to each member of `awaiting`. - * If the backer is `None`, set the source to `RequestChunksFromValidators` with a random shuffling of validators and empty `received_chunks`, and `requesting_chunks` and break the loop. + * If the backer is `None`, set the source to `RequestChunksFromValidators` with a random shuffling of validators + and empty `received_chunks`, and `requesting_chunks` and break the loop. * If the task contains `RequestChunksFromValidators`: - * Request `AvailabilityStoreMessage::QueryAllChunks`. For each chunk that exists, add it to `received_chunks` and remote the validator from `shuffling`. + * Request `AvailabilityStoreMessage::QueryAllChunks`. For each chunk that exists, add it to `received_chunks` and + remote the validator from `shuffling`. * Loop: - * If `received_chunks + requesting_chunks + shuffling` lengths are less than the threshold, break and return `Err(Unavailable)`. - * Poll for new updates from `requesting_chunks`. Check merkle proofs of any received chunks. If the request simply fails due to network issues, insert into the front of `shuffling` to be retried. + * If `received_chunks + requesting_chunks + shuffling` lengths are less than the threshold, break and return + `Err(Unavailable)`. + * Poll for new updates from `requesting_chunks`. Check merkle proofs of any received chunks. If the request simply + fails due to network issues, insert into the front of `shuffling` to be retried. * If `received_chunks` has more than `threshold` entries, attempt to recover the data. * If that fails, return `Err(RecoveryError::Invalid)` * If correct: @@ -135,5 +152,6 @@ const N_PARALLEL: usize = 50; * break and issue `Ok(available_data)` * Send the result to each member of `awaiting`. * While there are fewer than `N_PARALLEL` entries in `requesting_chunks`, - * Pop the next item from `shuffling`. If it's empty and `requesting_chunks` is empty, return `Err(RecoveryError::Unavailable)`. + * Pop the next item from `shuffling`. If it's empty and `requesting_chunks` is empty, return + `Err(RecoveryError::Unavailable)`. * Issue a `NetworkBridgeMessage::Requests` and wait for the response in `requesting_chunks`. diff --git a/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-distribution.md b/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-distribution.md index 53bd8a1bced61f5783bfeee937786c9b4741d80e..097f2e6353843d4949c2f41088815bc4cf6e5c38 100644 --- a/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-distribution.md +++ b/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-distribution.md @@ -1,34 +1,40 @@ # Bitfield Distribution -Validators vote on the availability of a backed candidate by issuing signed bitfields, where each bit corresponds to a single candidate. These bitfields can be used to compactly determine which backed candidates are available or not based on a 2/3+ quorum. +Validators vote on the availability of a backed candidate by issuing signed bitfields, where each bit corresponds to a +single candidate. These bitfields can be used to compactly determine which backed candidates are available or not based +on a 2/3+ quorum. ## Protocol `PeerSet`: `Validation` -Input: -[`BitfieldDistributionMessage`](../../types/overseer-protocol.md#bitfield-distribution-message) which are gossiped to all peers, no matter if validator or not. +Input: [`BitfieldDistributionMessage`](../../types/overseer-protocol.md#bitfield-distribution-message) which are +gossiped to all peers, no matter if validator or not. Output: -- `NetworkBridge::SendValidationMessage([PeerId], message)` gossip a verified incoming bitfield on to interested subsystems within this validator node. -- `NetworkBridge::ReportPeer(PeerId, cost_or_benefit)` improve or penalize the reputation of peers based on the messages that are received relative to the current view. -- `ProvisionerMessage::ProvisionableData(ProvisionableData::Bitfield(relay_parent, SignedAvailabilityBitfield))` pass - on the bitfield to the other submodules via the overseer. +- `NetworkBridge::SendValidationMessage([PeerId], message)` gossip a verified incoming bitfield on to interested + subsystems within this validator node. +- `NetworkBridge::ReportPeer(PeerId, cost_or_benefit)` improve or penalize the reputation of peers based on the messages + that are received relative to the current view. +- `ProvisionerMessage::ProvisionableData(ProvisionableData::Bitfield(relay_parent, SignedAvailabilityBitfield))` pass on + the bitfield to the other submodules via the overseer. ## Functionality This is implemented as a gossip system. -It is necessary to track peer connection, view change, and disconnection events, in order to maintain an index of which peers are interested in which relay parent bitfields. +It is necessary to track peer connection, view change, and disconnection events, in order to maintain an index of which +peers are interested in which relay parent bitfields. -Before gossiping incoming bitfields, they must be checked to be signed by one of the validators -of the validator set relevant to the current relay parent. -Only accept bitfields relevant to our current view and only distribute bitfields to other peers when relevant to their most recent view. -Accept and distribute only one bitfield per validator. +Before gossiping incoming bitfields, they must be checked to be signed by one of the validators of the validator set +relevant to the current relay parent. Only accept bitfields relevant to our current view and only distribute bitfields +to other peers when relevant to their most recent view. Accept and distribute only one bitfield per validator. -When receiving a bitfield either from the network or from a `DistributeBitfield` message, forward it along to the block authorship (provisioning) subsystem for potential inclusion in a block. +When receiving a bitfield either from the network or from a `DistributeBitfield` message, forward it along to the block +authorship (provisioning) subsystem for potential inclusion in a block. -Peers connecting after a set of valid bitfield gossip messages was received, those messages must be cached and sent upon connection of new peers or re-connecting peers. +Peers connecting after a set of valid bitfield gossip messages was received, those messages must be cached and sent upon +connection of new peers or re-connecting peers. diff --git a/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-signing.md b/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-signing.md index 08cbe528473ea4fd7099cedf15b58a6995a07226..a5875e0a8055037b0f31b32a531d418fa49cc815 100644 --- a/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-signing.md +++ b/polkadot/roadmap/implementers-guide/src/node/availability/bitfield-signing.md @@ -1,12 +1,15 @@ # Bitfield Signing -Validators vote on the availability of a backed candidate by issuing signed bitfields, where each bit corresponds to a single candidate. These bitfields can be used to compactly determine which backed candidates are available or not based on a 2/3+ quorum. +Validators vote on the availability of a backed candidate by issuing signed bitfields, where each bit corresponds to a +single candidate. These bitfields can be used to compactly determine which backed candidates are available or not based +on a 2/3+ quorum. ## Protocol Input: -There is no dedicated input mechanism for bitfield signing. Instead, Bitfield Signing produces a bitfield representing the current state of availability on `StartWork`. +There is no dedicated input mechanism for bitfield signing. Instead, Bitfield Signing produces a bitfield representing +the current state of availability on `StartWork`. Output: @@ -15,15 +18,20 @@ Output: ## Functionality -Upon receipt of an `ActiveLeavesUpdate`, launch bitfield signing job for each `activated` head referring to a fresh leaf. Stop the job for each `deactivated` head. +Upon receipt of an `ActiveLeavesUpdate`, launch bitfield signing job for each `activated` head referring to a fresh +leaf. Stop the job for each `deactivated` head. ## Bitfield Signing Job -Localized to a specific relay-parent `r` -If not running as a validator, do nothing. +Localized to a specific relay-parent `r` If not running as a validator, do nothing. -- For each fresh leaf, begin by waiting a fixed period of time so availability distribution has the chance to make candidates available. -- Determine our validator index `i`, the set of backed candidates pending availability in `r`, and which bit of the bitfield each corresponds to. -- Start with an empty bitfield. For each bit in the bitfield, if there is a candidate pending availability, query the [Availability Store](../utility/availability-store.md) for whether we have the availability chunk for our validator index. The `OccupiedCore` struct contains the candidate hash so the full candidate does not need to be fetched from runtime. +- For each fresh leaf, begin by waiting a fixed period of time so availability distribution has the chance to make + candidates available. +- Determine our validator index `i`, the set of backed candidates pending availability in `r`, and which bit of the + bitfield each corresponds to. +- Start with an empty bitfield. For each bit in the bitfield, if there is a candidate pending availability, query the + [Availability Store](../utility/availability-store.md) for whether we have the availability chunk for our validator + index. The `OccupiedCore` struct contains the candidate hash so the full candidate does not need to be fetched from + runtime. - For all chunks we have, set the corresponding bit in the bitfield. - Sign the bitfield and dispatch a `BitfieldDistribution::DistributeBitfield` message. diff --git a/polkadot/roadmap/implementers-guide/src/node/backing/README.md b/polkadot/roadmap/implementers-guide/src/node/backing/README.md index ac47abefb0d043e3daaf6d2c3ba9a7e19c3b9413..d2a77ded6b763326331307b4bcefb5b9aa78336e 100644 --- a/polkadot/roadmap/implementers-guide/src/node/backing/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/backing/README.md @@ -1,10 +1,15 @@ # Backing Subsystems -The backing subsystems, when conceived as a black box, receive an arbitrary quantity of parablock candidates and associated proofs of validity from arbitrary untrusted collators. From these, they produce a bounded quantity of backable candidates which relay chain block authors may choose to include in a subsequent block. +The backing subsystems, when conceived as a black box, receive an arbitrary quantity of parablock candidates and +associated proofs of validity from arbitrary untrusted collators. From these, they produce a bounded quantity of +backable candidates which relay chain block authors may choose to include in a subsequent block. In broad strokes, the flow operates like this: - **Candidate Selection** winnows the field of parablock candidates, selecting up to one of them to second. -- **Candidate Backing** ensures that a seconding candidate is valid, then generates the appropriate `Statement`. It also keeps track of which candidates have received the backing of a quorum of other validators. -- **Statement Distribution** is the networking component which ensures that all validators receive each others' statements. -- **PoV Distribution** is the networking component which ensures that validators considering a candidate can get the appropriate PoV. +- **Candidate Backing** ensures that a seconding candidate is valid, then generates the appropriate `Statement`. It also + keeps track of which candidates have received the backing of a quorum of other validators. +- **Statement Distribution** is the networking component which ensures that all validators receive each others' + statements. +- **PoV Distribution** is the networking component which ensures that validators considering a candidate can get the + appropriate PoV. diff --git a/polkadot/roadmap/implementers-guide/src/node/backing/candidate-backing.md b/polkadot/roadmap/implementers-guide/src/node/backing/candidate-backing.md index 0eee0cc532ef17c922e03e0b2fb299313702ffb3..31f8423fe27b275452b84bc8d2486e10b81e1908 100644 --- a/polkadot/roadmap/implementers-guide/src/node/backing/candidate-backing.md +++ b/polkadot/roadmap/implementers-guide/src/node/backing/candidate-backing.md @@ -1,12 +1,20 @@ # Candidate Backing -The Candidate Backing subsystem ensures every parablock considered for relay block inclusion has been seconded by at least one validator, and approved by a quorum. Parablocks for which not enough validators will assert correctness are discarded. If the block later proves invalid, the initial backers are slashable; this gives polkadot a rational threat model during subsequent stages. +The Candidate Backing subsystem ensures every parablock considered for relay block inclusion has been seconded by at +least one validator, and approved by a quorum. Parablocks for which not enough validators will assert correctness are +discarded. If the block later proves invalid, the initial backers are slashable; this gives Polkadot a rational threat +model during subsequent stages. -Its role is to produce backable candidates for inclusion in new relay-chain blocks. It does so by issuing signed [`Statement`s][Statement] and tracking received statements signed by other validators. Once enough statements are received, they can be combined into backing for specific candidates. +Its role is to produce backable candidates for inclusion in new relay-chain blocks. It does so by issuing signed +[`Statement`s][Statement] and tracking received statements signed by other validators. Once enough statements are +received, they can be combined into backing for specific candidates. -Note that though the candidate backing subsystem attempts to produce as many backable candidates as possible, it does _not_ attempt to choose a single authoritative one. The choice of which actually gets included is ultimately up to the block author, by whatever metrics it may use; those are opaque to this subsystem. +Note that though the candidate backing subsystem attempts to produce as many backable candidates as possible, it does +_not_ attempt to choose a single authoritative one. The choice of which actually gets included is ultimately up to the +block author, by whatever metrics it may use; those are opaque to this subsystem. -Once a sufficient quorum has agreed that a candidate is valid, this subsystem notifies the [Provisioner][PV], which in turn engages block production mechanisms to include the parablock. +Once a sufficient quorum has agreed that a candidate is valid, this subsystem notifies the [Provisioner][PV], which in +turn engages block production mechanisms to include the parablock. ## Protocol @@ -14,33 +22,49 @@ Input: [`CandidateBackingMessage`][CBM] Output: -- [`CandidateValidationMessage`][CVM] -- [`RuntimeApiMessage`][RAM] -- [`CollatorProtocolMessage`][CPM] -- [`ProvisionerMessage`][PM] -- [`AvailabilityDistributionMessage`][ADM] -- [`StatementDistributionMessage`][SDM] +* [`CandidateValidationMessage`][CVM] +* [`RuntimeApiMessage`][RAM] +* [`CollatorProtocolMessage`][CPM] +* [`ProvisionerMessage`][PM] +* [`AvailabilityDistributionMessage`][ADM] +* [`StatementDistributionMessage`][SDM] ## Functionality -The [Collator Protocol][CP] subsystem is the primary source of non-overseer messages into this subsystem. That subsystem generates appropriate [`CandidateBackingMessage`s][CBM] and passes them to this subsystem. +The [Collator Protocol][CP] subsystem is the primary source of non-overseer messages into this subsystem. That subsystem +generates appropriate [`CandidateBackingMessage`s][CBM] and passes them to this subsystem. -This subsystem requests validation from the [Candidate Validation][CV] and generates an appropriate [`Statement`][Statement]. All `Statement`s are then passed on to the [Statement Distribution][SD] subsystem to be gossiped to peers. When [Candidate Validation][CV] decides that a candidate is invalid, and it was recommended to us to second by our own [Collator Protocol][CP] subsystem, a message is sent to the [Collator Protocol][CP] subsystem with the candidate's hash so that the collator which recommended it can be penalized. +This subsystem requests validation from the [Candidate Validation][CV] and generates an appropriate +[`Statement`][Statement]. All `Statement`s are then passed on to the [Statement Distribution][SD] subsystem to be +gossiped to peers. When [Candidate Validation][CV] decides that a candidate is invalid, and it was recommended to us to +second by our own [Collator Protocol][CP] subsystem, a message is sent to the [Collator Protocol][CP] subsystem with the +candidate's hash so that the collator which recommended it can be penalized. -The subsystem should maintain a set of handles to Candidate Backing Jobs that are currently live, as well as the relay-parent to which they correspond. +The subsystem should maintain a set of handles to Candidate Backing Jobs that are currently live, as well as the +relay-parent to which they correspond. ### On Overseer Signal * If the signal is an [`OverseerSignal`][OverseerSignal]`::ActiveLeavesUpdate`: - * spawn a Candidate Backing Job for each `activated` head referring to a fresh leaf, storing a bidirectional channel with the Candidate Backing Job in the set of handles. + * spawn a Candidate Backing Job for each `activated` head referring to a fresh leaf, storing a bidirectional channel + with the Candidate Backing Job in the set of handles. * cease the Candidate Backing Job for each `deactivated` head, if any. -* If the signal is an [`OverseerSignal`][OverseerSignal]`::Conclude`: Forward conclude messages to all jobs, wait a small amount of time for them to join, and then exit. +* If the signal is an [`OverseerSignal`][OverseerSignal]`::Conclude`: Forward conclude messages to all jobs, wait a + small amount of time for them to join, and then exit. ### On Receiving `CandidateBackingMessage` -* If the message is a [`CandidateBackingMessage`][CBM]`::GetBackedCandidates`, get all backable candidates from the statement table and send them back. -* If the message is a [`CandidateBackingMessage`][CBM]`::Second`, sign and dispatch a `Seconded` statement only if we have not seconded any other candidate and have not signed a `Valid` statement for the requested candidate. Signing both a `Seconded` and `Valid` message is a double-voting misbehavior with a heavy penalty, and this could occur if another validator has seconded the same candidate and we've received their message before the internal seconding request. -* If the message is a [`CandidateBackingMessage`][CBM]`::Statement`, count the statement to the quorum. If the statement in the message is `Seconded` and it contains a candidate that belongs to our assignment, request the corresponding `PoV` from the backing node via `AvailabilityDistribution` and launch validation. Issue our own `Valid` or `Invalid` statement as a result. +* If the message is a [`CandidateBackingMessage`][CBM]`::GetBackedCandidates`, get all backable candidates from the + statement table and send them back. +* If the message is a [`CandidateBackingMessage`][CBM]`::Second`, sign and dispatch a `Seconded` statement only if we + have not seconded any other candidate and have not signed a `Valid` statement for the requested candidate. Signing + both a `Seconded` and `Valid` message is a double-voting misbehavior with a heavy penalty, and this could occur if + another validator has seconded the same candidate and we've received their message before the internal seconding + request. +* If the message is a [`CandidateBackingMessage`][CBM]`::Statement`, count the statement to the quorum. If the statement + in the message is `Seconded` and it contains a candidate that belongs to our assignment, request the corresponding + `PoV` from the backing node via `AvailabilityDistribution` and launch validation. Issue our own `Valid` or `Invalid` + statement as a result. If the seconding node did not provide us with the `PoV` we will retry fetching from other backing validators. @@ -51,19 +75,25 @@ If the seconding node did not provide us with the `PoV` we will retry fetching f > * Allow inclusion of _old_ parachain candidates validated by _current_ validators. > * Allow inclusion of _old_ parachain candidates validated by _old_ validators. > -> This will probably blur the lines between jobs, will probably require inter-job communication and a short-term memory of recently backable, but not backed candidates. +> This will probably blur the lines between jobs, will probably require inter-job communication and a short-term memory +> of recently backable, but not backed candidates. ## Candidate Backing Job -The Candidate Backing Job represents the work a node does for backing candidates with respect to a particular relay-parent. +The Candidate Backing Job represents the work a node does for backing candidates with respect to a particular +relay-parent. -The goal of a Candidate Backing Job is to produce as many backable candidates as possible. This is done via signed [`Statement`s][STMT] by validators. If a candidate receives a majority of supporting Statements from the Parachain Validators currently assigned, then that candidate is considered backable. +The goal of a Candidate Backing Job is to produce as many backable candidates as possible. This is done via signed +[`Statement`s][STMT] by validators. If a candidate receives a majority of supporting Statements from the Parachain +Validators currently assigned, then that candidate is considered backable. ### On Startup -* Fetch current validator set, validator -> parachain assignments from [`Runtime API`][RA] subsystem using [`RuntimeApiRequest::Validators`][RAM] and [`RuntimeApiRequest::ValidatorGroups`][RAM] +* Fetch current validator set, validator -> parachain assignments from [`Runtime API`][RA] subsystem using + [`RuntimeApiRequest::Validators`][RAM] and [`RuntimeApiRequest::ValidatorGroups`][RAM] * Determine if the node controls a key in the current validator set. Call this the local key if so. -* If the local key exists, extract the parachain head and validation function from the [`Runtime API`][RA] for the parachain the local key is assigned to by issuing a [`RuntimeApiRequest::Validators`][RAM] +* If the local key exists, extract the parachain head and validation function from the [`Runtime API`][RA] for the + parachain the local key is assigned to by issuing a [`RuntimeApiRequest::Validators`][RAM] * Issue a [`RuntimeApiRequest::SigningContext`][RAM] message to get a context that will later be used upon signing. ### On Receiving New Candidate Backing Message @@ -91,15 +121,17 @@ match msg { } ``` -Add `Seconded` statements and `Valid` statements to a quorum. If the quorum reaches a pre-defined threshold, send a [`ProvisionerMessage`][PM]`::ProvisionableData(ProvisionableData::BackedCandidate(CandidateReceipt))` message. -`Invalid` statements that conflict with already witnessed `Seconded` and `Valid` statements for the given candidate, statements that are double-votes, self-contradictions and so on, should result in issuing a [`ProvisionerMessage`][PM]`::MisbehaviorReport` message for each newly detected case of this kind. +Add `Seconded` statements and `Valid` statements to a quorum. If the quorum reaches a pre-defined threshold, send a +[`ProvisionerMessage`][PM]`::ProvisionableData(ProvisionableData::BackedCandidate(CandidateReceipt))` message. `Invalid` +statements that conflict with already witnessed `Seconded` and `Valid` statements for the given candidate, statements +that are double-votes, self-contradictions and so on, should result in issuing a +[`ProvisionerMessage`][PM]`::MisbehaviorReport` message for each newly detected case of this kind. -Backing does not need to concern itself with providing statements to the dispute -coordinator as the dispute coordinator scrapes them from chain. This way the -import is batched and contains only statements that actually made it on some +Backing does not need to concern itself with providing statements to the dispute coordinator as the dispute coordinator +scrapes them from chain. This way the import is batched and contains only statements that actually made it on some chain. -### Validating Candidates. +### Validating Candidates ```rust fn spawn_validation_work(candidate, parachain head, validation function) { @@ -119,14 +151,16 @@ fn spawn_validation_work(candidate, parachain head, validation function) { ### Fetch PoV Block -Create a `(sender, receiver)` pair. -Dispatch a [`AvailabilityDistributionMessage`][ADM]`::FetchPoV{ validator_index, pov_hash, candidate_hash, tx, } and listen on the passed receiver for a response. Availability distribution will send the request to the validator specified by `validator_index`, which might not be serving it for whatever reasons, therefore we need to retry with other backing validators in that case. +Create a `(sender, receiver)` pair. Dispatch a [`AvailabilityDistributionMessage`][ADM]`::FetchPoV{ validator_index, +pov_hash, candidate_hash, tx, }` and listen on the passed receiver for a response. Availability distribution will send +the request to the validator specified by `validator_index`, which might not be serving it for whatever reasons, +therefore we need to retry with other backing validators in that case. ### Validate PoV Block -Create a `(sender, receiver)` pair. -Dispatch a `CandidateValidationMessage::Validate(validation function, candidate, pov, BACKING_EXECUTION_TIMEOUT, sender)` and listen on the receiver for a response. +Create a `(sender, receiver)` pair. Dispatch a `CandidateValidationMessage::Validate(validation function, candidate, +pov, BACKING_EXECUTION_TIMEOUT, sender)` and listen on the receiver for a response. ### Distribute Signed Statement diff --git a/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution-legacy.md b/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution-legacy.md index 5cbc875d8a733ebd224056b1658570c345b9b7b7..e10a55010b91ca1dab6d3cf09aed409530c2bba5 100644 --- a/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution-legacy.md +++ b/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution-legacy.md @@ -1,18 +1,16 @@ # Statement Distribution (Legacy) -This describes the legacy, backwards-compatible version of the Statement -Distribution subsystem. +This describes the legacy, backwards-compatible version of the Statement Distribution subsystem. -**Note:** All the V1 (legacy) code was extracted out to a `legacy_v1` module of -the `statement-distribution` crate, which doesn't alter any logic. V2 (new -protocol) peers also run `legacy_v1` and communicate with V1 peers using V1 -messages and with V2 peers using V2 messages. Once the runtime upgrade goes -through on all networks, this `legacy_v1` code will no longer be triggered and -will be vestigial and can be removed. +**Note:** All the V1 (legacy) code was extracted out to a `legacy_v1` module of the `statement-distribution` crate, +which doesn't alter any logic. V2 (new protocol) peers also run `legacy_v1` and communicate with V1 peers using V1 +messages and with V2 peers using V2 messages. Once the runtime upgrade goes through on all networks, this `legacy_v1` +code will no longer be triggered and will be vestigial and can be removed. ## Overview -The Statement Distribution Subsystem is responsible for distributing statements about seconded candidates between validators. +The Statement Distribution Subsystem is responsible for distributing statements about seconded candidates between +validators. ## Protocol @@ -31,89 +29,133 @@ Output: ## Functionality -Implemented as a gossip protocol. Handles updates to our view and peers' views. Neighbor packets are used to inform peers which chain heads we are interested in data for. +Implemented as a gossip protocol. Handles updates to our view and peers' views. Neighbor packets are used to inform +peers which chain heads we are interested in data for. -The Statement Distribution Subsystem is responsible for distributing signed statements that we have generated and for forwarding statements generated by other validators. It also detects a variety of Validator misbehaviors for reporting to the [Provisioner Subsystem](../utility/provisioner.md). During the Backing stage of the inclusion pipeline, Statement Distribution is the main point of contact with peer nodes. On receiving a signed statement from a peer in the same backing group, assuming the peer receipt state machine is in an appropriate state, it sends the Candidate Receipt to the [Candidate Backing subsystem](candidate-backing.md) to handle the validator's statement. On receiving `StatementDistributionMessage::Share` we make sure to send messages to our backing group in addition to random other peers, to ensure a fast backing process and getting all statements quickly for distribution. +The Statement Distribution Subsystem is responsible for distributing signed statements that we have generated and for +forwarding statements generated by other validators. It also detects a variety of Validator misbehaviors for reporting +to the [Provisioner Subsystem](../utility/provisioner.md). During the Backing stage of the inclusion pipeline, Statement +Distribution is the main point of contact with peer nodes. On receiving a signed statement from a peer in the same +backing group, assuming the peer receipt state machine is in an appropriate state, it sends the Candidate Receipt to the +[Candidate Backing subsystem](candidate-backing.md) to handle the validator's statement. On receiving +`StatementDistributionMessage::Share` we make sure to send messages to our backing group in addition to random other +peers, to ensure a fast backing process and getting all statements quickly for distribution. -This subsystem tracks equivocating validators and stops accepting information from them. It establishes a data-dependency order: +This subsystem tracks equivocating validators and stops accepting information from them. It establishes a +data-dependency order: - In order to receive a `Seconded` message we have the corresponding chain head in our view - In order to receive a `Valid` message we must have received the corresponding `Seconded` message. -And respect this data-dependency order from our peers by respecting their views. This subsystem is responsible for checking message signatures. +And respect this data-dependency order from our peers by respecting their views. This subsystem is responsible for +checking message signatures. The Statement Distribution subsystem sends statements to peer nodes. ## Peer Receipt State Machine -There is a very simple state machine which governs which messages we are willing to receive from peers. Not depicted in the state machine: on initial receipt of any [`SignedFullStatement`](../../types/backing.md#signed-statement-type), validate that the provided signature does in fact sign the included data. Note that each individual parablock candidate gets its own instance of this state machine; it is perfectly legal to receive a `Valid(X)` before a `Seconded(Y)`, as long as a `Seconded(X)` has been received. +There is a very simple state machine which governs which messages we are willing to receive from peers. Not depicted in +the state machine: on initial receipt of any [`SignedFullStatement`](../../types/backing.md#signed-statement-type), +validate that the provided signature does in fact sign the included data. Note that each individual parablock candidate +gets its own instance of this state machine; it is perfectly legal to receive a `Valid(X)` before a `Seconded(Y)`, as +long as a `Seconded(X)` has been received. -A: Initial State. Receive `SignedFullStatement(Statement::Second)`: extract `Statement`, forward to Candidate Backing, proceed to B. Receive any other `SignedFullStatement` variant: drop it. +A: Initial State. Receive `SignedFullStatement(Statement::Second)`: extract `Statement`, forward to Candidate Backing, +proceed to B. Receive any other `SignedFullStatement` variant: drop it. -B: Receive any `SignedFullStatement`: check signature and determine whether the statement is new to us. if new, forward to Candidate Backing and circulate to other peers. Receive `OverseerMessage::StopWork`: proceed to C. +B: Receive any `SignedFullStatement`: check signature and determine whether the statement is new to us. if new, forward +to Candidate Backing and circulate to other peers. Receive `OverseerMessage::StopWork`: proceed to C. C: Receive any message for this block: drop it. -For large statements (see below), we also keep track of the total received large -statements per peer and have a hard limit on that number for flood protection. -This is necessary as in the current code we only forward statements once we have -all the data, therefore flood protection for large statement is a bit more -subtle. This will become an obsolete problem once [off chain code -upgrades](https://github.com/paritytech/polkadot/issues/2979) are implemented. +For large statements (see below), we also keep track of the total received large statements per peer and have a hard +limit on that number for flood protection. This is necessary as in the current code we only forward statements once we +have all the data, therefore flood protection for large statement is a bit more subtle. This will become an obsolete +problem once [off chain code upgrades](https://github.com/paritytech/polkadot/issues/2979) are implemented. ## Peer Knowledge Tracking -The peer receipt state machine implies that for parsimony of network resources, we should model the knowledge of our peers, and help them out. For example, let's consider a case with peers A, B, and C, validators X and Y, and candidate M. A sends us a `Statement::Second(M)` signed by X. We've double-checked it, and it's valid. While we're checking it, we receive a copy of X's `Statement::Second(M)` from `B`, along with a `Statement::Valid(M)` signed by Y. +The peer receipt state machine implies that for parsimony of network resources, we should model the knowledge of our +peers, and help them out. For example, let's consider a case with peers A, B, and C, validators X and Y, and candidate +M. A sends us a `Statement::Second(M)` signed by X. We've double-checked it, and it's valid. While we're checking it, we +receive a copy of X's `Statement::Second(M)` from `B`, along with a `Statement::Valid(M)` signed by Y. -Our response to A is just the `Statement::Valid(M)` signed by Y. However, we haven't heard anything about this from C. Therefore, we send it everything we have: first a copy of X's `Statement::Second`, then Y's `Statement::Valid`. +Our response to A is just the `Statement::Valid(M)` signed by Y. However, we haven't heard anything about this from C. +Therefore, we send it everything we have: first a copy of X's `Statement::Second`, then Y's `Statement::Valid`. -This system implies a certain level of duplication of messages--we received X's `Statement::Second` from both our peers, and C may experience the same--but it minimizes the degree to which messages are simply dropped. +This system implies a certain level of duplication of messages--we received X's `Statement::Second` from both our peers, +and C may experience the same--but it minimizes the degree to which messages are simply dropped. And respect this data-dependency order from our peers. This subsystem is responsible for checking message signatures. -No jobs. We follow view changes from the [`NetworkBridge`](../utility/network-bridge.md), which in turn is updated by the overseer. +No jobs. We follow view changes from the [`NetworkBridge`](../utility/network-bridge.md), which in turn is updated by +the overseer. ## Equivocations and Flood Protection -An equivocation is a double-vote by a validator. The [Candidate Backing](candidate-backing.md) Subsystem is better-suited than this one to detect equivocations as it adds votes to quorum trackers. +An equivocation is a double-vote by a validator. The [Candidate Backing](candidate-backing.md) Subsystem is +better-suited than this one to detect equivocations as it adds votes to quorum trackers. -At this level, we are primarily concerned about flood-protection, and to some extent, detecting equivocations is a part of that. In particular, we are interested in detecting equivocations of `Seconded` statements. Since every other statement is dependent on `Seconded` statements, ensuring that we only ever hold a bounded number of `Seconded` statements is sufficient for flood-protection. +At this level, we are primarily concerned about flood-protection, and to some extent, detecting equivocations is a part +of that. In particular, we are interested in detecting equivocations of `Seconded` statements. Since every other +statement is dependent on `Seconded` statements, ensuring that we only ever hold a bounded number of `Seconded` +statements is sufficient for flood-protection. -The simple approach is to say that we only receive up to two `Seconded` statements per validator per chain head. However, the marginal cost of equivocation, conditional on having already equivocated, is close to 0, since a single double-vote offence is counted as all double-vote offences for a particular chain-head. Even if it were not, there is some amount of equivocations that can be done such that the marginal cost of issuing further equivocations is close to 0, as there would be an amount of equivocations necessary to be completely and totally obliterated by the slashing algorithm. We fear the validator with nothing left to lose. +The simple approach is to say that we only receive up to two `Seconded` statements per validator per chain head. +However, the marginal cost of equivocation, conditional on having already equivocated, is close to 0, since a single +double-vote offence is counted as all double-vote offences for a particular chain-head. Even if it were not, there is +some amount of equivocations that can be done such that the marginal cost of issuing further equivocations is close to +0, as there would be an amount of equivocations necessary to be completely and totally obliterated by the slashing +algorithm. We fear the validator with nothing left to lose. With that in mind, this simple approach has a caveat worth digging deeper into. -First: We may be aware of two equivocated `Seconded` statements issued by a validator. A totally honest peer of ours can also be aware of one or two different `Seconded` statements issued by the same validator. And yet another peer may be aware of one or two _more_ `Seconded` statements. And so on. This interacts badly with pre-emptive sending logic. Upon sending a `Seconded` statement to a peer, we will want to pre-emptively follow up with all statements relative to that candidate. Waiting for acknowledgment introduces latency at every hop, so that is best avoided. What can happen is that upon receipt of the `Seconded` statement, the peer will discard it as it falls beyond the bound of 2 that it is allowed to store. It cannot store anything in memory about discarded candidates as that would introduce a DoS vector. Then, the peer would receive from us all of the statements pertaining to that candidate, which, from its perspective, would be undesired - they are data-dependent on the `Seconded` statement we sent them, but they have erased all record of that from their memory. Upon receiving a potential flood of undesired statements, this 100% honest peer may choose to disconnect from us. In this way, an adversary may be able to partition the network with careful distribution of equivocated `Seconded` statements. - -The fix is to track, per-peer, the hashes of up to 4 candidates per validator (per relay-parent) that the peer is aware of. It is 4 because we may send them 2 and they may send us 2 different ones. We track the data that they are aware of as the union of things we have sent them and things they have sent us. If we receive a 1st or 2nd `Seconded` statement from a peer, we note it in the peer's known candidates even if we do disregard the data locally. And then, upon receipt of any data dependent on that statement, we do not reduce that peer's standing in our eyes, as the data was not undesired. - -There is another caveat to the fix: we don't want to allow the peer to flood us because it has set things up in a way that it knows we will drop all of its traffic. -We also track how many statements we have received per peer, per candidate, and per chain-head. This is any statement concerning a particular candidate: `Seconded`, `Valid`, or `Invalid`. If we ever receive a statement from a peer which would push any of these counters beyond twice the amount of validators at the chain-head, we begin to lower the peer's standing and eventually disconnect. This bound is a massive overestimate and could be reduced to twice the number of validators in the corresponding validator group. It is worth noting that the goal at the time of writing is to ensure any finite bound on the amount of stored data, as any equivocation results in a large slash. +First: We may be aware of two equivocated `Seconded` statements issued by a validator. A totally honest peer of ours can +also be aware of one or two different `Seconded` statements issued by the same validator. And yet another peer may be +aware of one or two _more_ `Seconded` statements. And so on. This interacts badly with pre-emptive sending logic. Upon +sending a `Seconded` statement to a peer, we will want to pre-emptively follow up with all statements relative to that +candidate. Waiting for acknowledgment introduces latency at every hop, so that is best avoided. What can happen is that +upon receipt of the `Seconded` statement, the peer will discard it as it falls beyond the bound of 2 that it is allowed +to store. It cannot store anything in memory about discarded candidates as that would introduce a DoS vector. Then, the +peer would receive from us all of the statements pertaining to that candidate, which, from its perspective, would be +undesired - they are data-dependent on the `Seconded` statement we sent them, but they have erased all record of that +from their memory. Upon receiving a potential flood of undesired statements, this 100% honest peer may choose to +disconnect from us. In this way, an adversary may be able to partition the network with careful distribution of +equivocated `Seconded` statements. + +The fix is to track, per-peer, the hashes of up to 4 candidates per validator (per relay-parent) that the peer is aware +of. It is 4 because we may send them 2 and they may send us 2 different ones. We track the data that they are aware of +as the union of things we have sent them and things they have sent us. If we receive a 1st or 2nd `Seconded` statement +from a peer, we note it in the peer's known candidates even if we do disregard the data locally. And then, upon receipt +of any data dependent on that statement, we do not reduce that peer's standing in our eyes, as the data was not +undesired. + +There is another caveat to the fix: we don't want to allow the peer to flood us because it has set things up in a way +that it knows we will drop all of its traffic. We also track how many statements we have received per peer, per +candidate, and per chain-head. This is any statement concerning a particular candidate: `Seconded`, `Valid`, or +`Invalid`. If we ever receive a statement from a peer which would push any of these counters beyond twice the amount of +validators at the chain-head, we begin to lower the peer's standing and eventually disconnect. This bound is a massive +overestimate and could be reduced to twice the number of validators in the corresponding validator group. It is worth +noting that the goal at the time of writing is to ensure any finite bound on the amount of stored data, as any +equivocation results in a large slash. ## Large statements -Seconded statements can become quite large on parachain runtime upgrades for -example. For this reason, there exists a `LargeStatement` constructor for the -`StatementDistributionMessage` wire message, which only contains light metadata -of a statement. The actual candidate data is not included. This message type is -used whenever a message is deemed large. The receiver of such a message needs to -request the actual payload via request/response by means of a +Seconded statements can become quite large on parachain runtime upgrades for example. For this reason, there exists a +`LargeStatement` constructor for the `StatementDistributionMessage` wire message, which only contains light metadata of +a statement. The actual candidate data is not included. This message type is used whenever a message is deemed large. +The receiver of such a message needs to request the actual payload via request/response by means of a `StatementFetchingV1` request. -This is necessary as distribution of a large payload (mega bytes) via gossip -would make the network collapse and timely distribution of statements would no -longer be possible. By using request/response it is ensured that each peer only -transferes large data once. We only take good care to detect an overloaded -peer early and immediately move on to a different peer for fetching the data. -This mechanism should result in a good load distribution and therefore a rather +This is necessary as distribution of a large payload (mega bytes) via gossip would make the network collapse and timely +distribution of statements would no longer be possible. By using request/response it is ensured that each peer only +transferes large data once. We only take good care to detect an overloaded peer early and immediately move on to a +different peer for fetching the data. This mechanism should result in a good load distribution and therefore a rather optimal distribution path. -With these optimizations, distribution of payloads in the size of up to 3 to 4 -MB should work with Kusama validator specifications. For scaling up even more, -runtime upgrades and message passing should be done off chain at some point. +With these optimizations, distribution of payloads in the size of up to 3 to 4 MB should work with Kusama validator +specifications. For scaling up even more, runtime upgrades and message passing should be done off chain at some point. -Flood protection considerations: For making DoS attacks slightly harder on this -subsystem, nodes will only respond to large statement requests, when they -previously notified that peer via gossip about that statement. So, it is not -possible to DoS nodes at scale, by requesting candidate data over and over -again. +Flood protection considerations: For making DoS attacks slightly harder on this subsystem, nodes will only respond to +large statement requests, when they previously notified that peer via gossip about that statement. So, it is not +possible to DoS nodes at scale, by requesting candidate data over and over again. diff --git a/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution.md b/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution.md index 2e014284821063075f08d2b624a387a9a4879a12..24f2785f8294d32ff3a890f2781d13be3a232d4e 100644 --- a/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution.md +++ b/polkadot/roadmap/implementers-guide/src/node/backing/statement-distribution.md @@ -1,158 +1,127 @@ # Statement Distribution -This subsystem is responsible for distributing signed statements that we have generated and forwarding statements generated by our peers. Received candidate receipts and statements are passed to the [Candidate Backing subsystem](candidate-backing.md) to handle producing local statements. On receiving `StatementDistributionMessage::Share`, this subsystem distributes the message across the network with redundency to ensure a fast backing process. +This subsystem is responsible for distributing signed statements that we have generated and forwarding statements +generated by our peers. Received candidate receipts and statements are passed to the [Candidate Backing +subsystem](candidate-backing.md) to handle producing local statements. On receiving +`StatementDistributionMessage::Share`, this subsystem distributes the message across the network with redundency to +ensure a fast backing process. ## Overview -**Goal:** every well-connected node is aware of every next potential parachain -block. +**Goal:** every well-connected node is aware of every next potential parachain block. Validators can either: - receive parachain block from collator, check block, and gossip statement. -- receive statements from other validators, check the parachain block if it - originated within their own group, gossip forward statement if valid. - -Validators must have statements, candidates, and persisted validation from all -other validators. This is because we need to store statements from validators -who've checked the candidate on the relay chain, so we know who to hold -accountable in case of disputes. Any validator can be selected as the next -relay-chain block author, and this is not revealed in advance for security -reasons. As a result, all validators must have a up to date view of all possible -parachain candidates + backing statements that could be placed on-chain in the -next block. - -[This blog post](https://polkadot.network/blog/polkadot-v1-0-sharding-and-economic-security) -puts it another way: "Validators who aren't assigned to the parachain still -listen for the attestations [statements] because whichever validator ends up -being the author of the relay-chain block needs to bundle up attested parachain -blocks for several parachains and place them into the relay-chain block." - -Backing-group quorum (that is, enough backing group votes) must be reached -before the block author will consider the candidate. Therefore, validators need -to consider _all_ seconded candidates within their own group, because that's -what they're assigned to work on. Validators only need to consider _backable_ -candidates from other groups. This informs the design of the statement -distribution protocol to have separate phases for in-group and out-group -distribution, respectively called "cluster" and "grid" mode (see below). +- receive statements from other validators, check the parachain block if it originated within their own group, gossip + forward statement if valid. + +Validators must have statements, candidates, and persisted validation from all other validators. This is because we need +to store statements from validators who've checked the candidate on the relay chain, so we know who to hold accountable +in case of disputes. Any validator can be selected as the next relay-chain block author, and this is not revealed in +advance for security reasons. As a result, all validators must have a up to date view of all possible parachain +candidates + backing statements that could be placed on-chain in the next block. + +[This blog post](https://polkadot.network/blog/polkadot-v1-0-sharding-and-economic-security) puts it another way: +"Validators who aren't assigned to the parachain still listen for the attestations [statements] because whichever +validator ends up being the author of the relay-chain block needs to bundle up attested parachain blocks for several +parachains and place them into the relay-chain block." + +Backing-group quorum (that is, enough backing group votes) must be reached before the block author will consider the +candidate. Therefore, validators need to consider _all_ seconded candidates within their own group, because that's what +they're assigned to work on. Validators only need to consider _backable_ candidates from other groups. This informs the +design of the statement distribution protocol to have separate phases for in-group and out-group distribution, +respectively called "cluster" and "grid" mode (see below). ### With Async Backing -Asynchronous backing changes the runtime to accept parachain candidates from a -certain allowed range of historic relay-parents. These candidates must be backed -by the group assigned to the parachain as-of their corresponding relay parents. +Asynchronous backing changes the runtime to accept parachain candidates from a certain allowed range of historic +relay-parents. These candidates must be backed by the group assigned to the parachain as-of their corresponding relay +parents. ## Protocol -To address the concern of dealing with large numbers of spam candidates or -statements, the overall design approach is to combine a focused "clustering" -protocol for legitimate fresh candidates with a broad-distribution "grid" -protocol to quickly get backed candidates into the hands of many validators. -Validators do not eagerly send each other heavy `CommittedCandidateReceipt`, -but instead request these lazily through request/response protocols. +To address the concern of dealing with large numbers of spam candidates or statements, the overall design approach is to +combine a focused "clustering" protocol for legitimate fresh candidates with a broad-distribution "grid" protocol to +quickly get backed candidates into the hands of many validators. Validators do not eagerly send each other heavy +`CommittedCandidateReceipt`, but instead request these lazily through request/response protocols. A high-level description of the protocol follows: ### Messages -Nodes can send each other a few kinds of messages: `Statement`, -`BackedCandidateManifest`, `BackedCandidateAcknowledgement`. +Nodes can send each other a few kinds of messages: `Statement`, `BackedCandidateManifest`, +`BackedCandidateAcknowledgement`. -- `Statement` messages contain only a signed compact statement, without full - candidate info. -- `BackedCandidateManifest` messages advertise a description of a backed - candidate and stored statements. -- `BackedCandidateAcknowledgement` messages acknowledge that a backed candidate - is fully known. +- `Statement` messages contain only a signed compact statement, without full candidate info. +- `BackedCandidateManifest` messages advertise a description of a backed candidate and stored statements. +- `BackedCandidateAcknowledgement` messages acknowledge that a backed candidate is fully known. ### Request/response protocol -Nodes can request the full `CommittedCandidateReceipt` and -`PersistedValidationData`, along with statements, over a request/response -protocol. This is the `AttestedCandidateRequest`; the response is -`AttestedCandidateResponse`. +Nodes can request the full `CommittedCandidateReceipt` and `PersistedValidationData`, along with statements, over a +request/response protocol. This is the `AttestedCandidateRequest`; the response is `AttestedCandidateResponse`. ### Importability and the Hypothetical Frontier -The **prospective parachains** subsystem maintains prospective "fragment trees" -which can be used to determine whether a particular parachain candidate could -possibly be included in the future. Candidates which either are within a -fragment tree or _would be_ part of a fragment tree if accepted are said to be -in the "hypothetical frontier". +The **prospective parachains** subsystem maintains prospective "fragment trees" which can be used to determine whether a +particular parachain candidate could possibly be included in the future. Candidates which either are within a fragment +tree or _would be_ part of a fragment tree if accepted are said to be in the "hypothetical frontier". -The **statement-distribution** subsystem keeps track of all candidates, and -updates its knowledge of the hypothetical frontier based on events such as new -relay parents, new confirmed candidates, and newly backed candidates. +The **statement-distribution** subsystem keeps track of all candidates, and updates its knowledge of the hypothetical +frontier based on events such as new relay parents, new confirmed candidates, and newly backed candidates. -We only consider statements as "importable" when the corresponding candidate is -part of the hypothetical frontier, and only send "importable" statements to the -backing subsystem itself. +We only consider statements as "importable" when the corresponding candidate is part of the hypothetical frontier, and +only send "importable" statements to the backing subsystem itself. ### Cluster Mode -- Validator nodes are partitioned into groups (with some exceptions), and - validators within a group at a relay-parent can send each other `Statement` - messages for any candidates within that group and based on that relay-parent. +- Validator nodes are partitioned into groups (with some exceptions), and validators within a group at a relay-parent + can send each other `Statement` messages for any candidates within that group and based on that relay-parent. - This is referred to as the "cluster" mode. - - Right now these are the same as backing groups, though "cluster" - specifically refers to the set of nodes communicating with each other in the - first phase of distribution. + - Right now these are the same as backing groups, though "cluster" specifically refers to the set of nodes + communicating with each other in the first phase of distribution. - `Seconded` statements must be sent before `Valid` statements. -- `Seconded` statements may only be sent to other members of the group when the - candidate is fully known by the local validator. - - "Fully known" means the validator has the full `CommittedCandidateReceipt` - and `PersistedValidationData`, which it receives on request from other - validators or from a collator. - - The reason for this is that sending a statement (which is always a - `CompactStatement` carrying nothing but a hash and signature) to the - cluster, is also a signal that the sending node is available to request the - candidate from. - - This makes the protocol easier to reason about, while also reducing network - messages about candidates that don't really exist. -- Validators in a cluster receiving messages about unknown candidates request - the candidate (and statements) from other cluster members which have it. +- `Seconded` statements may only be sent to other members of the group when the candidate is fully known by the local + validator. + - "Fully known" means the validator has the full `CommittedCandidateReceipt` and `PersistedValidationData`, which it + receives on request from other validators or from a collator. + - The reason for this is that sending a statement (which is always a `CompactStatement` carrying nothing but a hash + and signature) to the cluster, is also a signal that the sending node is available to request the candidate from. + - This makes the protocol easier to reason about, while also reducing network messages about candidates that don't + really exist. +- Validators in a cluster receiving messages about unknown candidates request the candidate (and statements) from other + cluster members which have it. - Spam considerations - - The maximum depth of candidates allowed in asynchronous backing determines - the maximum amount of `Seconded` statements originating from a validator V - which each validator in a cluster may send to others. This bounds the number - of candidates. - - There is a small number of validators in each group, which further limits - the amount of candidates. -- We accept candidates which don't fit in the fragment trees of any relay - parents. - - "Accept" means "attempt to request and store in memory until useful or - expired". - - We listen to prospective parachains subsystem to learn of new additions to - the fragment trees. + - The maximum depth of candidates allowed in asynchronous backing determines the maximum amount of `Seconded` + statements originating from a validator V which each validator in a cluster may send to others. This bounds the + number of candidates. + - There is a small number of validators in each group, which further limits the amount of candidates. +- We accept candidates which don't fit in the fragment trees of any relay parents. + - "Accept" means "attempt to request and store in memory until useful or expired". + - We listen to prospective parachains subsystem to learn of new additions to the fragment trees. - Use this to attempt to import the candidate later. ### Grid Mode -- Every consensus session provides randomness and a fixed validator set, which - is used to build a redundant grid topology. - - It's redundant in the sense that there are 2 paths from every node to every - other node. See "Grid Topology" section for more details. -- This grid topology is used to create a sending path from each validator group - to every validator. -- When a node observes a candidate as backed, it sends a - `BackedCandidateManifest` to their "receiving" nodes. +- Every consensus session provides randomness and a fixed validator set, which is used to build a redundant grid + topology. + - It's redundant in the sense that there are 2 paths from every node to every other node. See "Grid Topology" section + for more details. +- This grid topology is used to create a sending path from each validator group to every validator. +- When a node observes a candidate as backed, it sends a `BackedCandidateManifest` to their "receiving" nodes. - If receiving nodes don't yet know the candidate, they request it. -- Once they know the candidate, they respond with a - `BackedCandidateAcknowledgement`. -- Once two nodes perform a manifest/acknowledgement exchange, they can send - `Statement` messages directly to each other for any new statements they might - need. - - This limits the amount of statements we'd have to deal with w.r.t. - candidates that don't really exist. See "Manifest Exchange" section. -- There are limitations on the number of candidates that can be advertised by - each peer, similar to those in the cluster. Validators do not request - candidates which exceed these limitations. -- Validators request candidates as soon as they are advertised, but do not - import the statements until the candidate is part of the hypothetical - frontier, and do not re-advertise or acknowledge until the candidate is - considered both backable and part of the hypothetical frontier. -- Note that requesting is not an implicit acknowledgement, and an explicit - acknowledgement must be sent upon receipt. +- Once they know the candidate, they respond with a `BackedCandidateAcknowledgement`. +- Once two nodes perform a manifest/acknowledgement exchange, they can send `Statement` messages directly to each other + for any new statements they might need. + - This limits the amount of statements we'd have to deal with w.r.t. candidates that don't really exist. See "Manifest + Exchange" section. +- There are limitations on the number of candidates that can be advertised by each peer, similar to those in the + cluster. Validators do not request candidates which exceed these limitations. +- Validators request candidates as soon as they are advertised, but do not import the statements until the candidate is + part of the hypothetical frontier, and do not re-advertise or acknowledge until the candidate is considered both + backable and part of the hypothetical frontier. +- Note that requesting is not an implicit acknowledgement, and an explicit acknowledgement must be sent upon receipt. ## Messages @@ -161,27 +130,23 @@ backing subsystem itself. - `ActiveLeaves` - Notification of a change in the set of active leaves. - `StatementDistributionMessage::Share` - - Notification of a locally-originating statement. That is, this statement - comes from our node and should be distributed to other nodes. - - Sent by the Backing Subsystem after it successfully imports a - locally-originating statement. + - Notification of a locally-originating statement. That is, this statement comes from our node and should be + distributed to other nodes. + - Sent by the Backing Subsystem after it successfully imports a locally-originating statement. - `StatementDistributionMessage::Backed` - - Notification of a candidate being backed (received enough validity votes - from the backing group). - - Sent by the Backing Subsystem after it successfully imports a statement for - the first time and after sending ~Share~. + - Notification of a candidate being backed (received enough validity votes from the backing group). + - Sent by the Backing Subsystem after it successfully imports a statement for the first time and after sending + ~Share~. - `StatementDistributionMessage::NetworkBridgeUpdate` - See next section. #### Network bridge events - v1 compatibility - - Messages for the v1 protocol are routed to the legacy statement - distribution. + - Messages for the v1 protocol are routed to the legacy statement distribution. - `Statement` - Notification of a signed statement. - - Sent by a peer's Statement Distribution subsystem when circulating - statements. + - Sent by a peer's Statement Distribution subsystem when circulating statements. - `BackedCandidateManifest` - Notification of a backed candidate being known by the sending node. - For the candidate being requested by the receiving node if needed. @@ -196,26 +161,23 @@ backing subsystem itself. ### Outgoing - `NetworkBridgeTxMessage::SendValidationMessages` - - Sends a peer all pending messages / acknowledgements / statements for a - relay parent, either through the cluster or the grid. + - Sends a peer all pending messages / acknowledgements / statements for a relay parent, either through the cluster or + the grid. - `NetworkBridgeTxMessage::SendValidationMessage` - - Circulates a compact statement to all peers who need it, either through the - cluster or the grid. + - Circulates a compact statement to all peers who need it, either through the cluster or the grid. - `NetworkBridgeTxMessage::ReportPeer` - Reports a peer (either good or bad). - `CandidateBackingMessage::Statement` - Note a validator's statement about a particular candidate. - `ProspectiveParachainsMessage::GetHypotheticalFrontier` - - Gets the hypothetical frontier membership of candidates under active leaves' - fragment trees. + - Gets the hypothetical frontier membership of candidates under active leaves' fragment trees. - `NetworkBridgeTxMessage::SendRequests` - Sends requests, initiating the request/response protocol. ## Request/Response -We also have a request/response protocol because validators do not eagerly send -each other heavy `CommittedCandidateReceipt`, but instead need to request these -lazily. +We also have a request/response protocol because validators do not eagerly send each other heavy +`CommittedCandidateReceipt`, but instead need to request these lazily. ### Protocol @@ -225,16 +187,13 @@ lazily. - Done as needed, when handling incoming manifests/statements. - `RequestManager::dispatch_requests` sends any queued-up requests. - Calls `RequestManager::next_request` to completion. - - Creates the `OutgoingRequest`, saves the receiver in - `RequestManager::pending_responses`. - - Does nothing if we have more responses pending than the limit of parallel - requests. + - Creates the `OutgoingRequest`, saves the receiver in `RequestManager::pending_responses`. + - Does nothing if we have more responses pending than the limit of parallel requests. 2. Peer - Requests come in on a peer on the `IncomingRequestReceiver`. - - Runs in a background responder task which feeds requests to `answer_request` - through `MuxedMessage`. + - Runs in a background responder task which feeds requests to `answer_request` through `MuxedMessage`. - This responder task has a limit on the number of parallel requests. - `answer_request` on the peer takes the request and sends a response. - Does this using the response sender on the request. @@ -243,8 +202,7 @@ lazily. - `receive_response` on the original validator yields a response. - Response was sent on the request's response sender. - - Uses `RequestManager::await_incoming` to await on pending responses in an - unordered fashion. + - Uses `RequestManager::await_incoming` to await on pending responses in an unordered fashion. - Runs on the `MuxedMessage` receiver. - `handle_response` handles the response. @@ -265,25 +223,23 @@ lazily. ## Manifests -A manifest is a message about a known backed candidate, along with a description -of the statements backing it. It can be one of two kinds: +A manifest is a message about a known backed candidate, along with a description of the statements backing it. It can be +one of two kinds: -- `Full`: Contains information about the candidate and should be sent to peers - who may not have the candidate yet. This is also called an `Announcement`. -- `Acknowledgement`: Omits information implicit in the candidate, and should be - sent to peers which are guaranteed to have the candidate already. +- `Full`: Contains information about the candidate and should be sent to peers who may not have the candidate yet. This + is also called an `Announcement`. +- `Acknowledgement`: Omits information implicit in the candidate, and should be sent to peers which are guaranteed to + have the candidate already. ### Manifest Exchange -Manifest exchange is when a receiving node received a `Full` manifest and -replied with an `Acknowledgement`. It indicates that both nodes know the -candidate as valid and backed. This allows the nodes to send `Statement` -messages directly to each other for any new statements. +Manifest exchange is when a receiving node received a `Full` manifest and replied with an `Acknowledgement`. It +indicates that both nodes know the candidate as valid and backed. This allows the nodes to send `Statement` messages +directly to each other for any new statements. -Why? This limits the amount of statements we'd have to deal with w.r.t. -candidates that don't really exist. Limiting out-of-group statement distribution -between peers to only candidates that both peers agree are backed and exist -ensures we only have to store statements about real candidates. +Why? This limits the amount of statements we'd have to deal with w.r.t. candidates that don't really exist. Limiting +out-of-group statement distribution between peers to only candidates that both peers agree are backed and exist ensures +we only have to store statements about real candidates. In practice, manifest exchange means that one of three things have happened: @@ -291,36 +247,31 @@ In practice, manifest exchange means that one of three things have happened: - We announced, they acknowledged. - We announced, they announced. -Concerning the last case, note that it is possible for two nodes to have each -other in their sending set. Consider: +Concerning the last case, note that it is possible for two nodes to have each other in their sending set. Consider: ``` 1 2 3 4 ``` -If validators 2 and 4 are in group B, then there is a path `2->1->3` and -`4->3->1`. Therefore, 1 and 3 might send each other manifests for the same -candidate at the same time, without having seen the other's yet. This also -counts as a manifest exchange, but is only allowed to occur in this way. +If validators 2 and 4 are in group B, then there is a path `2->1->3` and `4->3->1`. Therefore, 1 and 3 might send each +other manifests for the same candidate at the same time, without having seen the other's yet. This also counts as a +manifest exchange, but is only allowed to occur in this way. -After the exchange is complete, we update pending statements. Pending statements -are those we know locally that the remote node does not. +After the exchange is complete, we update pending statements. Pending statements are those we know locally that the +remote node does not. #### Alternative Paths Through The Topology -Nodes should send a `BackedCandidateAcknowledgement(CandidateHash, -StatementFilter)` notification to any peer which has sent a manifest, and the -candidate has been acquired by other means. This keeps alternative paths through -the topology open, which allows nodes to receive additional statements that come -later, but not after the candidate has been posted on-chain. +Nodes should send a `BackedCandidateAcknowledgement(CandidateHash, StatementFilter)` notification to any peer which has +sent a manifest, and the candidate has been acquired by other means. This keeps alternative paths through the topology +open, which allows nodes to receive additional statements that come later, but not after the candidate has been posted +on-chain. -This is mostly about the limitation that the runtime has no way for block -authors to post statements that come after the parablock is posted on-chain and -ensure those validators still get rewarded. Technically, we only need enough -statements to back the candidate and the manifest + request will provide that. -But more statements might come shortly afterwards, and we want those to end up -on-chain as well to ensure all validators in the group are rewarded. +This is mostly about the limitation that the runtime has no way for block authors to post statements that come after the +parablock is posted on-chain and ensure those validators still get rewarded. Technically, we only need enough statements +to back the candidate and the manifest + request will provide that. But more statements might come shortly afterwards, +and we want those to end up on-chain as well to ensure all validators in the group are rewarded. For clarity, here is the full timeline: @@ -333,52 +284,42 @@ For clarity, here is the full timeline: ## Cluster Module -The cluster module provides direct distribution of unbacked candidates within a -group. By utilizing this initial phase of propagating only within -clusters/groups, we bound the number of `Seconded` messages per validator per -relay-parent, helping us prevent spam. Validators can try to circumvent this, -but they would only consume a few KB of memory and it is trivially slashable on -chain. +The cluster module provides direct distribution of unbacked candidates within a group. By utilizing this initial phase +of propagating only within clusters/groups, we bound the number of `Seconded` messages per validator per relay-parent, +helping us prevent spam. Validators can try to circumvent this, but they would only consume a few KB of memory and it is +trivially slashable on chain. -The cluster module determines whether to accept/reject messages from other -validators in the same group. It keeps track of what we have sent to other -validators in the group, and pending statements. For the full protocol, see -"Protocol". +The cluster module determines whether to accept/reject messages from other validators in the same group. It keeps track +of what we have sent to other validators in the group, and pending statements. For the full protocol, see "Protocol". ## Grid Module -The grid module provides distribution of backed candidates and late statements -outside the backing group. For the full protocol, see the "Protocol" section. +The grid module provides distribution of backed candidates and late statements outside the backing group. For the full +protocol, see the "Protocol" section. ### Grid Topology -For distributing outside our cluster (aka backing group) we use a 2D grid -topology. This limits the amount of peers we send messages to, and handles -view updates. +For distributing outside our cluster (aka backing group) we use a 2D grid topology. This limits the amount of peers we +send messages to, and handles view updates. The basic operation of the grid topology is that: -- A validator producing a message sends it to its row-neighbors and its - column-neighbors. -- A validator receiving a message originating from one of its row-neighbors - sends it to its column-neighbors. -- A validator receiving a message originating from one of its column-neighbors - sends it to its row-neighbors. +- A validator producing a message sends it to its row-neighbors and its column-neighbors. +- A validator receiving a message originating from one of its row-neighbors sends it to its column-neighbors. +- A validator receiving a message originating from one of its column-neighbors sends it to its row-neighbors. -This grid approach defines 2 unique paths for every validator to reach every -other validator in at most 2 hops, providing redundancy. +This grid approach defines 2 unique paths for every validator to reach every other validator in at most 2 hops, +providing redundancy. Propagation follows these rules: -- Each node has a receiving set and a sending set. These are different for each - group. That is, if a node receives a candidate from group A, it checks if it - is allowed to receive from that node for candidates from group A. +- Each node has a receiving set and a sending set. These are different for each group. That is, if a node receives a + candidate from group A, it checks if it is allowed to receive from that node for candidates from group A. - For groups that we are in, receive from nobody and send to our X/Y peers. - For groups that we are not part of: - - We receive from any validator in the group we share a slice with and send to - the corresponding X/Y slice in the other dimension. - - For any validators we don't share a slice with, we receive from the nodes - which share a slice with them. + - We receive from any validator in the group we share a slice with and send to the corresponding X/Y slice in the + other dimension. + - For any validators we don't share a slice with, we receive from the nodes which share a slice with them. ### Example @@ -391,81 +332,63 @@ For size 11, the matrix would be: 9 10 ``` -e.g. for index 10, the neighbors would be 1, 4, 7, 9 -- these are the nodes we -could directly communicate with (e.g. either send to or receive from). - -Now, which of these neighbors can 10 receive from? Recall that the -sending/receiving sets for 10 would be different for different groups. Here are -some hypothetical scenarios: - -- **Scenario 1:** 9 belongs to group A but not 10. Here, 10 can directly receive - candidates from group A from 9. 10 would propagate them to the nodes in {1, 4, - 7} that are not in A. -- **Scenario 2:** 6 is in group A instead of 9, and 7 is not in group A. 10 can - receive group A messages from 7 or 9. 10 will try to relay these messages, but - 7 and 9 together should have already propagated the message to all x/y - peers of 10. If so, then 10 will just receive acknowledgements in reply rather - than requests. -- **Scenario 3:** 10 itself is in group A. 10 would not receive candidates from - this group from any other nodes through the grid. It would itself send such - candidates to all its neighbors that are not in A. +e.g. for index 10, the neighbors would be 1, 4, 7, 9 -- these are the nodes we could directly communicate with (e.g. +either send to or receive from). + +Now, which of these neighbors can 10 receive from? Recall that the sending/receiving sets for 10 would be different for +different groups. Here are some hypothetical scenarios: + +- **Scenario 1:** 9 belongs to group A but not 10. Here, 10 can directly receive candidates from group A from 9. 10 + would propagate them to the nodes in {1, 4, 7} that are not in A. +- **Scenario 2:** 6 is in group A instead of 9, and 7 is not in group A. 10 can receive group A messages from 7 or 9. 10 + will try to relay these messages, but 7 and 9 together should have already propagated the message to all x/y peers of + 10. If so, then 10 will just receive acknowledgements in reply rather than requests. +- **Scenario 3:** 10 itself is in group A. 10 would not receive candidates from this group from any other nodes through + the grid. It would itself send such candidates to all its neighbors that are not in A. ### Seconding Limit -The seconding limit is a per-validator limit. Before asynchronous backing, we -had a rule that every validator was only allowed to second one candidate per -relay parent. With asynchronous backing, we have a 'maximum depth' which makes -it possible to second multiple candidates per relay parent. The seconding limit -is set to `max depth + 1` to set an upper bound on candidates entering the -system. +The seconding limit is a per-validator limit. Before asynchronous backing, we had a rule that every validator was only +allowed to second one candidate per relay parent. With asynchronous backing, we have a 'maximum depth' which makes it +possible to second multiple candidates per relay parent. The seconding limit is set to `max depth + 1` to set an upper +bound on candidates entering the system. ## Candidates Module -The candidates module provides a tracker for all known candidates in the view, -whether they are confirmed or not, and how peers have advertised the candidates. -What is a confirmed candidate? It is a candidate for which we have the full -receipt and the persisted validation data. This module gets confirmed candidates -from two sources: +The candidates module provides a tracker for all known candidates in the view, whether they are confirmed or not, and +how peers have advertised the candidates. What is a confirmed candidate? It is a candidate for which we have the full +receipt and the persisted validation data. This module gets confirmed candidates from two sources: -- It can be that a validator fetched a collation directly from the collator and - validated it. -- The first time a validator gets an announcement for an unknown candidate, it - will send a request for the candidate. Upon receiving a response and - validating it (see `UnhandledResponse::validate_response`), it will mark the - candidate as confirmed. +- It can be that a validator fetched a collation directly from the collator and validated it. +- The first time a validator gets an announcement for an unknown candidate, it will send a request for the candidate. + Upon receiving a response and validating it (see `UnhandledResponse::validate_response`), it will mark the candidate + as confirmed. ## Requests Module -The requests module provides a manager for pending requests for candidate data, -as well as pending responses. See "Request/Response Protocol" for a high-level -description of the flow. See module-docs for full details. +The requests module provides a manager for pending requests for candidate data, as well as pending responses. See +"Request/Response Protocol" for a high-level description of the flow. See module-docs for full details. ## Glossary -- **Acknowledgement:** A partial manifest sent to a validator that already has the - candidate to inform them that the sending node also knows the candidate. - Concludes a manifest exchange. -- **Announcement:** A full manifest indicating that a backed candidate is known by - the sending node. Initiates a manifest exchange. +- **Acknowledgement:** A partial manifest sent to a validator that already has the candidate to inform them that the + sending node also knows the candidate. Concludes a manifest exchange. +- **Announcement:** A full manifest indicating that a backed candidate is known by the sending node. Initiates a + manifest exchange. - **Attestation:** See "Statement". - **Backable vs. Backed:** - - Note that we sometimes use "backed" to refer to candidates that are - "backable", but not yet backed on chain. - - **Backed** should technically mean that the parablock candidate and its - backing statements have been added to a relay chain block. - - **Backable** is when the necessary backing statements have been acquired but - those statements and the parablock candidate haven't been backed in a relay - chain block yet. -- **Fragment tree:** A parachain fragment not referenced by the relay-chain. - It is a tree of prospective parachain blocks. -- **Manifest:** A message about a known backed candidate, along with a - description of the statements backing it. There are two kinds of manifest, - `Acknowledgement` and `Announcement`. See "Manifests" section. + - Note that we sometimes use "backed" to refer to candidates that are "backable", but not yet backed on chain. + - **Backed** should technically mean that the parablock candidate and its backing statements have been added to a + relay chain block. + - **Backable** is when the necessary backing statements have been acquired but those statements and the parablock + candidate haven't been backed in a relay chain block yet. +- **Fragment tree:** A parachain fragment not referenced by the relay-chain. It is a tree of prospective parachain + blocks. +- **Manifest:** A message about a known backed candidate, along with a description of the statements backing it. There + are two kinds of manifest, `Acknowledgement` and `Announcement`. See "Manifests" section. - **Peer:** Another validator that a validator is connected to. -- **Request/response:** A protocol used to lazily request and receive heavy - candidate data when needed. -- **Reputation:** Tracks reputation of peers. Applies annoyance cost and good - behavior benefits. +- **Request/response:** A protocol used to lazily request and receive heavy candidate data when needed. +- **Reputation:** Tracks reputation of peers. Applies annoyance cost and good behavior benefits. - **Statement:** Signed statements that can be made about parachain candidates. - **Seconded:** Proposal of a parachain candidate. Implicit validity vote. - **Valid:** States that a parachain candidate is valid. @@ -474,6 +397,5 @@ description of the flow. See module-docs for full details. - **Explicit view** / **immediate view** - The view a peer has of the relay chain heads and highest finalized block. - **Implicit view** - - Derived from the immediate view. Composed of active leaves and minimum - relay-parents allowed for candidates of various parachains at those - leaves. + - Derived from the immediate view. Composed of active leaves and minimum relay-parents allowed for candidates of + various parachains at those leaves. diff --git a/polkadot/roadmap/implementers-guide/src/node/collators/README.md b/polkadot/roadmap/implementers-guide/src/node/collators/README.md index 3642e415efaba54621eccf98f781e17567f2502d..09edd0c119f36218a9aa2879175eb5cb154f3eb5 100644 --- a/polkadot/roadmap/implementers-guide/src/node/collators/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/collators/README.md @@ -1,6 +1,8 @@ # Collators -Collators are special nodes which bridge a parachain to the relay chain. They are simultaneously full nodes of the parachain, and at least light clients of the relay chain. Their overall contribution to the system is the generation of Proofs of Validity for parachain candidates. +Collators are special nodes which bridge a parachain to the relay chain. They are simultaneously full nodes of the +parachain, and at least light clients of the relay chain. Their overall contribution to the system is the generation of +Proofs of Validity for parachain candidates. -The **Collation Generation** subsystem triggers collators to produce collations -and then forwards them to **Collator Protocol** to circulate to validators. +The **Collation Generation** subsystem triggers collators to produce collations and then forwards them to **Collator +Protocol** to circulate to validators. diff --git a/polkadot/roadmap/implementers-guide/src/node/collators/collation-generation.md b/polkadot/roadmap/implementers-guide/src/node/collators/collation-generation.md index 9053ea40f89e8f2407f98d8e2a5b0826135c13f1..05148357f753d6e64f95ed30ec5af278eb7e47bd 100644 --- a/polkadot/roadmap/implementers-guide/src/node/collators/collation-generation.md +++ b/polkadot/roadmap/implementers-guide/src/node/collators/collation-generation.md @@ -1,17 +1,18 @@ # Collation Generation -The collation generation subsystem is executed on collator nodes and produces candidates to be distributed to validators. If configured to produce collations for a para, it produces collations and then feeds them to the [Collator Protocol][CP] subsystem, which handles the networking. +The collation generation subsystem is executed on collator nodes and produces candidates to be distributed to +validators. If configured to produce collations for a para, it produces collations and then feeds them to the [Collator +Protocol][CP] subsystem, which handles the networking. ## Protocol Collation generation for Parachains currently works in the following way: -1. A new relay chain block is imported. -2. The collation generation subsystem checks if the core associated to - the parachain is free and if yes, continues. -3. Collation generation calls our collator callback, if present, to generate a PoV. If none exists, do nothing. -4. Authoring logic determines if the current node should build a PoV. -5. Build new PoV and give it back to collation generation. +1. A new relay chain block is imported. +2. The collation generation subsystem checks if the core associated to the parachain is free and if yes, continues. +3. Collation generation calls our collator callback, if present, to generate a PoV. If none exists, do nothing. +4. Authoring logic determines if the current node should build a PoV. +5. Build new PoV and give it back to collation generation. ## Messages @@ -22,8 +23,7 @@ Collation generation for Parachains currently works in the following way: - Triggers collation generation procedure outlined in "Protocol" section. - `CollationGenerationMessage::Initialize` - Initializes the subsystem. Carries a config. - - No more than one initialization message should ever be sent to the collation - generation subsystem. + - No more than one initialization message should ever be sent to the collation generation subsystem. - Sent by a collator to initialize this subsystem. - `CollationGenerationMessage::SubmitCollation` - If the subsystem isn't initialized or the relay-parent is too old to be relevant, ignore the message. @@ -37,7 +37,9 @@ Collation generation for Parachains currently works in the following way: ## Functionality -The process of generating a collation for a parachain is very parachain-specific. As such, the details of how to do so are left beyond the scope of this description. The subsystem should be implemented as an abstract wrapper, which is aware of this configuration: +The process of generating a collation for a parachain is very parachain-specific. As such, the details of how to do so +are left beyond the scope of this description. The subsystem should be implemented as an abstract wrapper, which is +aware of this configuration: ```rust /// The output of a collator. @@ -117,30 +119,24 @@ The configuration should be optional, to allow for the case where the node is no - **Collation (output of a collator)** - - Contains the PoV (proof to verify the state transition of the - parachain) and other data. + - Contains the PoV (proof to verify the state transition of the parachain) and other data. - **Collation result** - - Contains the collation, and an optional result sender for a - collation-seconded signal. + - Contains the collation, and an optional result sender for a collation-seconded signal. - **Collation seconded signal** - - The signal that is returned when a collation was seconded by a - validator. + - The signal that is returned when a collation was seconded by a validator. - **Collation function** - - Called with the relay chain block the parablock will be built on top - of. + - Called with the relay chain block the parablock will be built on top of. - Called with the validation data. - - Provides information about the state of the parachain on the relay - chain. + - Provides information about the state of the parachain on the relay chain. - **Collation generation config** - - Contains collator's authentication key, optional collator function, and - parachain ID. + - Contains collator's authentication key, optional collator function, and parachain ID. [CP]: collator-protocol.md diff --git a/polkadot/roadmap/implementers-guide/src/node/collators/collator-protocol.md b/polkadot/roadmap/implementers-guide/src/node/collators/collator-protocol.md index 09265a5348475b9f277a0ae5379ce9af3e15da55..1fed671170c7c42f6de97dead0e53200ef5d675f 100644 --- a/polkadot/roadmap/implementers-guide/src/node/collators/collator-protocol.md +++ b/polkadot/roadmap/implementers-guide/src/node/collators/collator-protocol.md @@ -1,16 +1,25 @@ # Collator Protocol -The Collator Protocol implements the network protocol by which collators and validators communicate. It is used by collators to distribute collations to validators and used by validators to accept collations by collators. +The Collator Protocol implements the network protocol by which collators and validators communicate. It is used by +collators to distribute collations to validators and used by validators to accept collations by collators. -Collator-to-Validator networking is more difficult than Validator-to-Validator networking because the set of possible collators for any given para is unbounded, unlike the validator set. Validator-to-Validator networking protocols can easily be implemented as gossip because the data can be bounded, and validators can authenticate each other by their `PeerId`s for the purposes of instantiating and accepting connections. +Collator-to-Validator networking is more difficult than Validator-to-Validator networking because the set of possible +collators for any given para is unbounded, unlike the validator set. Validator-to-Validator networking protocols can +easily be implemented as gossip because the data can be bounded, and validators can authenticate each other by their +`PeerId`s for the purposes of instantiating and accepting connections. -Since, at least at the level of the para abstraction, the collator-set for any given para is unbounded, validators need to make sure that they are receiving connections from capable and honest collators and that their bandwidth and time are not being wasted by attackers. Communicating across this trust-boundary is the most difficult part of this subsystem. +Since, at least at the level of the para abstraction, the collator-set for any given para is unbounded, validators need +to make sure that they are receiving connections from capable and honest collators and that their bandwidth and time are +not being wasted by attackers. Communicating across this trust-boundary is the most difficult part of this subsystem. -Validation of candidates is a heavy task, and furthermore, the [`PoV`][PoV] itself is a large piece of data. Empirically, `PoV`s are on the order of 10MB. +Validation of candidates is a heavy task, and furthermore, the [`PoV`][PoV] itself is a large piece of data. +Empirically, `PoV`s are on the order of 10MB. > TODO: note the incremental validation function Ximin proposes at https://github.com/paritytech/polkadot/issues/1348 -As this network protocol serves as a bridge between collators and validators, it communicates primarily with one subsystem on behalf of each. As a collator, this will receive messages from the [`CollationGeneration`][CG] subsystem. As a validator, this will communicate only with the [`CandidateBacking`][CB]. +As this network protocol serves as a bridge between collators and validators, it communicates primarily with one +subsystem on behalf of each. As a collator, this will receive messages from the [`CollationGeneration`][CG] subsystem. +As a validator, this will communicate only with the [`CandidateBacking`][CB]. ## Protocol @@ -18,9 +27,9 @@ Input: [`CollatorProtocolMessage`][CPM] Output: -- [`RuntimeApiMessage`][RAM] -- [`NetworkBridgeMessage`][NBM] -- [`CandidateBackingMessage`][CBM] +* [`RuntimeApiMessage`][RAM] +* [`NetworkBridgeMessage`][NBM] +* [`CandidateBackingMessage`][CBM] ## Functionality @@ -28,7 +37,8 @@ This network protocol uses the `Collation` peer-set of the [`NetworkBridge`][NB] It uses the [`CollatorProtocolV1Message`](../../types/network.md#collator-protocol) as its `WireMessage` -Since this protocol functions both for validators and collators, it is easiest to go through the protocol actions for each of them separately. +Since this protocol functions both for validators and collators, it is easiest to go through the protocol actions for +each of them separately. Validators and collators. ```dot process @@ -47,24 +57,44 @@ digraph { ### Collators -It is assumed that collators are only collating on a single parachain. Collations are generated by the [Collation Generation][CG] subsystem. We will keep up to one local collation per relay-parent, based on `DistributeCollation` messages. If the para is not scheduled on any core, at the relay-parent, or the relay-parent isn't in the active-leaves set, we ignore the message as it must be invalid in that case - although this indicates a logic error elsewhere in the node. +It is assumed that collators are only collating on a single parachain. Collations are generated by the [Collation +Generation][CG] subsystem. We will keep up to one local collation per relay-parent, based on `DistributeCollation` +messages. If the para is not scheduled on any core, at the relay-parent, or the relay-parent isn't in the active-leaves +set, we ignore the message as it must be invalid in that case - although this indicates a logic error elsewhere in the +node. -We keep track of the Para ID we are collating on as a collator. This starts as `None`, and is updated with each `CollateOn` message received. If the `ParaId` of a collation requested to be distributed does not match the one we expect, we ignore the message. +We keep track of the Para ID we are collating on as a collator. This starts as `None`, and is updated with each +`CollateOn` message received. If the `ParaId` of a collation requested to be distributed does not match the one we +expect, we ignore the message. As with most other subsystems, we track the active leaves set by following `ActiveLeavesUpdate` signals. -For the purposes of actually distributing a collation, we need to be connected to the validators who are interested in collations on that `ParaId` at this point in time. We assume that there is a discovery API for connecting to a set of validators. +For the purposes of actually distributing a collation, we need to be connected to the validators who are interested in +collations on that `ParaId` at this point in time. We assume that there is a discovery API for connecting to a set of +validators. -As seen in the [Scheduler Module][SCH] of the runtime, validator groups are fixed for an entire session and their rotations across cores are predictable. Collators will want to do these things when attempting to distribute collations at a given relay-parent: +As seen in the [Scheduler Module][SCH] of the runtime, validator groups are fixed for an entire session and their +rotations across cores are predictable. Collators will want to do these things when attempting to distribute collations +at a given relay-parent: * Determine which core the para collated-on is assigned to. * Determine the group on that core. - * Issue a discovery request for the validators of the current group with[`NetworkBridgeMessage`][NBM]`::ConnectToValidators`. - -Once connected to the relevant peers for the current group assigned to the core (transitively, the para), advertise the collation to any of them which advertise the relay-parent in their view (as provided by the [Network Bridge][NB]). If any respond with a request for the full collation, provide it. However, we only send one collation at a time per relay parent, other requests need to wait. This is done to reduce the bandwidth requirements of a collator and also increases the chance to fully send the collation to at least one validator. From the point where one validator has received the collation and seconded it, it will also start to share this collation with other validators in its backing group. Upon receiving a view update from any of these peers which includes a relay-parent for which we have a collation that they will find relevant, advertise the collation to them if we haven't already. + * Issue a discovery request for the validators of the current group + with[`NetworkBridgeMessage`][NBM]`::ConnectToValidators`. + +Once connected to the relevant peers for the current group assigned to the core (transitively, the para), advertise the +collation to any of them which advertise the relay-parent in their view (as provided by the [Network Bridge][NB]). If +any respond with a request for the full collation, provide it. However, we only send one collation at a time per relay +parent, other requests need to wait. This is done to reduce the bandwidth requirements of a collator and also increases +the chance to fully send the collation to at least one validator. From the point where one validator has received the +collation and seconded it, it will also start to share this collation with other validators in its backing group. Upon +receiving a view update from any of these peers which includes a relay-parent for which we have a collation that they +will find relevant, advertise the collation to them if we haven't already. ### Validators -On the validator side of the protocol, validators need to accept incoming connections from collators. They should keep some peer slots open for accepting new speculative connections from collators and should disconnect from collators who are not relevant. +On the validator side of the protocol, validators need to accept incoming connections from collators. They should keep +some peer slots open for accepting new speculative connections from collators and should disconnect from collators who +are not relevant. ```dot process digraph G { @@ -98,32 +128,62 @@ digraph G { } ``` -When peers connect to us, they can `Declare` that they represent a collator with given public key and intend to collate on a specific para ID. Once they've declared that, and we checked their signature, they can begin to send advertisements of collations. The peers should not send us any advertisements for collations that are on a relay-parent outside of our view or for a para outside of the one they've declared. +When peers connect to us, they can `Declare` that they represent a collator with given public key and intend to collate +on a specific para ID. Once they've declared that, and we checked their signature, they can begin to send advertisements +of collations. The peers should not send us any advertisements for collations that are on a relay-parent outside of our +view or for a para outside of the one they've declared. -The protocol tracks advertisements received and the source of the advertisement. The advertisement source is the `PeerId` of the peer who sent the message. We accept one advertisement per collator per source per relay-parent. +The protocol tracks advertisements received and the source of the advertisement. The advertisement source is the +`PeerId` of the peer who sent the message. We accept one advertisement per collator per source per relay-parent. -As a validator, we will handle requests from other subsystems to fetch a collation on a specific `ParaId` and relay-parent. These requests are made with the request response protocol `CollationFetchingRequest` request. To do so, we need to first check if we have already gathered a collation on that `ParaId` and relay-parent. If not, we need to select one of the advertisements and issue a request for it. If we've already issued a request, we shouldn't issue another one until the first has returned. +As a validator, we will handle requests from other subsystems to fetch a collation on a specific `ParaId` and +relay-parent. These requests are made with the request response protocol `CollationFetchingRequest` request. To do so, +we need to first check if we have already gathered a collation on that `ParaId` and relay-parent. If not, we need to +select one of the advertisements and issue a request for it. If we've already issued a request, we shouldn't issue +another one until the first has returned. -When acting on an advertisement, we issue a `Requests::CollationFetchingV1`. However, we only request one collation at a time per relay parent. This reduces the bandwidth requirements and as we can second only one candidate per relay parent, the others are probably not required anyway. If the request times out, we need to note the collator as being unreliable and reduce its priority relative to other collators. +When acting on an advertisement, we issue a `Requests::CollationFetchingV1`. However, we only request one collation at a +time per relay parent. This reduces the bandwidth requirements and as we can second only one candidate per relay parent, +the others are probably not required anyway. If the request times out, we need to note the collator as being unreliable +and reduce its priority relative to other collators. -As a validator, once the collation has been fetched some other subsystem will inspect and do deeper validation of the collation. The subsystem will report to this subsystem with a [`CollatorProtocolMessage`][CPM]`::ReportCollator`. In that case, if we are connected directly to the collator, we apply a cost to the `PeerId` associated with the collator and potentially disconnect or blacklist it. If the collation is seconded, we notify the collator and apply a benefit to the `PeerId` associated with the collator. +As a validator, once the collation has been fetched some other subsystem will inspect and do deeper validation of the +collation. The subsystem will report to this subsystem with a [`CollatorProtocolMessage`][CPM]`::ReportCollator`. In +that case, if we are connected directly to the collator, we apply a cost to the `PeerId` associated with the collator +and potentially disconnect or blacklist it. If the collation is seconded, we notify the collator and apply a benefit to +the `PeerId` associated with the collator. ### Interaction with [Candidate Backing][CB] -As collators advertise the availability, a validator will simply second the first valid parablock candidate per relay head by sending a [`CandidateBackingMessage`][CBM]`::Second`. Note that this message contains the relay parent of the advertised collation, the candidate receipt and the [PoV][PoV]. +As collators advertise the availability, a validator will simply second the first valid parablock candidate per relay +head by sending a [`CandidateBackingMessage`][CBM]`::Second`. Note that this message contains the relay parent of the +advertised collation, the candidate receipt and the [PoV][PoV]. -Subsequently, once a valid parablock candidate has been seconded, the [`CandidateBacking`][CB] subsystem will send a [`CollatorProtocolMessage`][CPM]`::Seconded`, which will trigger this subsystem to notify the collator at the `PeerId` that first advertised the parablock on the seconded relay head of their successful seconding. +Subsequently, once a valid parablock candidate has been seconded, the [`CandidateBacking`][CB] subsystem will send a +[`CollatorProtocolMessage`][CPM]`::Seconded`, which will trigger this subsystem to notify the collator at the `PeerId` +that first advertised the parablock on the seconded relay head of their successful seconding. ## Future Work Several approaches have been discussed, but all have some issues: -- The current approach is very straightforward. However, that protocol is vulnerable to a single collator which, as an attack or simply through chance, gets its block candidate to the node more often than its fair share of the time. -- If collators produce blocks via Aura, BABE or in future Sassafras, it may be possible to choose an "Official" collator for the round, but it may be tricky to ensure that the PVF logic is enforced at collator leader election. -- We could use relay-chain BABE randomness to generate some delay `D` on the order of 1 second, +- 1 second. The collator would then second the first valid parablock which arrives after `D`, or in case none has arrived by `2*D`, the last valid parablock which has arrived. This makes it very hard for a collator to game the system to always get its block nominated, but it reduces the maximum throughput of the system by introducing delay into an already tight schedule. -- A variation of that scheme would be to have a fixed acceptance window `D` for parablock candidates and keep track of count `C`: the number of parablock candidates received. At the end of the period `D`, we choose a random number I in the range `[0, C)` and second the block at Index I. Its drawback is the same: it must wait the full `D` period before seconding any of its received candidates, reducing throughput. -- In order to protect against DoS attacks, it may be prudent to run throw out collations from collators that have behaved poorly (whether recently or historically) and subsequently only verify the PoV for the most suitable of collations. +* The current approach is very straightforward. However, that protocol is vulnerable to a single collator which, as an + attack or simply through chance, gets its block candidate to the node more often than its fair share of the time. +* If collators produce blocks via Aura, BABE or in future Sassafras, it may be possible to choose an "Official" collator + for the round, but it may be tricky to ensure that the PVF logic is enforced at collator leader election. +* We could use relay-chain BABE randomness to generate some delay `D` on the order of 1 second, +* 1 second. The + collator would then second the first valid parablock which arrives after `D`, or in case none has arrived by `2*D`, + the last valid parablock which has arrived. This makes it very hard for a collator to game the system to always get + its block nominated, but it reduces the maximum throughput of the system by introducing delay into an already tight + schedule. +* A variation of that scheme would be to have a fixed acceptance window `D` for parablock candidates and keep track of + count `C`: the number of parablock candidates received. At the end of the period `D`, we choose a random number I in + the range `[0, C)` and second the block at Index I. Its drawback is the same: it must wait the full `D` period before + seconding any of its received candidates, reducing throughput. +* In order to protect against DoS attacks, it may be prudent to run throw out collations from collators that have + behaved poorly (whether recently or historically) and subsequently only verify the PoV for the most suitable of + collations. [CB]: ../backing/candidate-backing.md [CBM]: ../../types/overseer-protocol.md#candidate-backing-mesage diff --git a/polkadot/roadmap/implementers-guide/src/node/disputes/README.md b/polkadot/roadmap/implementers-guide/src/node/disputes/README.md index a6e126b1534b8548f2f971dbaee095908f6db3c9..36f497114a7315572161f6d3b1caf760437cbe6a 100644 --- a/polkadot/roadmap/implementers-guide/src/node/disputes/README.md +++ b/polkadot/roadmap/implementers-guide/src/node/disputes/README.md @@ -4,12 +4,12 @@ If approval voting finds an invalid candidate, a dispute is raised. The disputes subsystems are concerned with the following: 1. Disputes can be raised -2. Disputes (votes) get propagated to all other validators -3. Votes get recorded as necessary -3. Nodes will participate in disputes in a sensible fashion -4. Finality is stopped while a candidate is being disputed on chain -5. Chains can be reverted in case a dispute concludes invalid -6. Votes are provided to the provisioner for importing on chain, in order for +1. Disputes (votes) get propagated to all other validators +1. Votes get recorded as necessary +1. Nodes will participate in disputes in a sensible fashion +1. Finality is stopped while a candidate is being disputed on chain +1. Chains can be reverted in case a dispute concludes invalid +1. Votes are provided to the provisioner for importing on chain, in order for slashing to work. The dispute-coordinator subsystem interfaces with the provisioner and chain diff --git a/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-coordinator.md b/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-coordinator.md index f8bfe6506aa5c01b3a2624eafec07a7bfb53938c..daba416e2639edbd93431a9f65a13120d52d6d25 100644 --- a/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-coordinator.md +++ b/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-coordinator.md @@ -1,86 +1,64 @@ # Dispute Coordinator -The coordinator is the central subsystem of the node-side components which -participate in disputes. It wraps a database, which is used to track statements -observed by _all_ validators over some window of sessions. Votes older than this +The coordinator is the central subsystem of the node-side components which participate in disputes. It wraps a database, +which is used to track statements observed by _all_ validators over some window of sessions. Votes older than this session window are pruned. In particular the dispute-coordinator is responsible for: -- Ensuring that the node is able to raise a dispute in case an invalid candidate - is found during approval checking. -- Ensuring that backing and approval votes will be recorded on chain. With these - votes on chain we can be certain that appropriate targets for slashing will be - available for concluded disputes. Also, scraping these votes during a dispute +- Ensuring that the node is able to raise a dispute in case an invalid candidate is found during approval checking. +- Ensuring that backing and approval votes will be recorded on chain. With these votes on chain we can be certain that + appropriate targets for slashing will be available for concluded disputes. Also, scraping these votes during a dispute is necessary for critical spam prevention measures. - Ensuring backing votes will never get overridden by explicit votes. -- Coordinating actual participation in a dispute, ensuring that the node - participates in any justified dispute in a way that ensures resolution of - disputes on the network even in the case of many disputes raised (flood/DoS - scenario). -- Ensuring disputes resolve, even for candidates on abandoned forks as much as - reasonably possible, to rule out "free tries" and thus guarantee our gambler's - ruin property. -- Providing an API for chain selection, so we can prevent finalization of any - chain which has included candidates for which a dispute is either ongoing or - concluded invalid and avoid building on chains with an included invalid +- Coordinating actual participation in a dispute, ensuring that the node participates in any justified dispute in a way + that ensures resolution of disputes on the network even in the case of many disputes raised (flood/DoS scenario). +- Ensuring disputes resolve, even for candidates on abandoned forks as much as reasonably possible, to rule out "free + tries" and thus guarantee our gambler's ruin property. +- Providing an API for chain selection, so we can prevent finalization of any chain which has included candidates for + which a dispute is either ongoing or concluded invalid and avoid building on chains with an included invalid candidate. -- Providing an API for retrieving (resolved) disputes, including all votes, both - implicit (approval, backing) and explicit dispute votes. So validators can get - rewarded/slashed accordingly. +- Providing an API for retrieving (resolved) disputes, including all votes, both implicit (approval, backing) and + explicit dispute votes. So validators can get rewarded/slashed accordingly. ## Ensuring That Disputes Can Be Raised -If a candidate turns out invalid in approval checking, the `approval-voting` -subsystem will try to issue a dispute. For this, it will send a message -`DisputeCoordinatorMessage::IssueLocalStatement` to the dispute coordinator, -indicating to cast an explicit invalid vote. It is the responsibility of the -dispute coordinator on reception of such a message to create and sign that -explicit invalid vote and trigger a dispute if none for that candidate is -already ongoing. - -In order to raise a dispute, a node has to be able to provide two opposing votes. -Given that the reason of the backing phase is to have validators with skin in -the game, the opposing valid vote will very likely be a backing vote. It could -also be some already cast approval vote, but the significant point here is: As -long as we have backing votes available, any node will be able to raise a -dispute. - -Therefore a vital responsibility of the dispute coordinator is to make sure -backing votes are available for all candidates that might still get disputed. To -accomplish this task in an efficient way the dispute-coordinator relies on chain -scraping. Whenever a candidate gets backed on chain, we record in chain storage -the backing votes imported in that block. This way, given the chain state for a -given relay chain block, we can retrieve via a provided runtime API the backing -votes imported by that block. The dispute coordinator makes sure to query those -votes for any non finalized blocks: In case of missed blocks, it will do chain -traversal as necessary. +If a candidate turns out invalid in approval checking, the `approval-voting` subsystem will try to issue a dispute. For +this, it will send a message `DisputeCoordinatorMessage::IssueLocalStatement` to the dispute coordinator, indicating to +cast an explicit invalid vote. It is the responsibility of the dispute coordinator on reception of such a message to +create and sign that explicit invalid vote and trigger a dispute if none for that candidate is already ongoing. + +In order to raise a dispute, a node has to be able to provide two opposing votes. Given that the reason of the backing +phase is to have validators with skin in the game, the opposing valid vote will very likely be a backing vote. It could +also be some already cast approval vote, but the significant point here is: As long as we have backing votes available, +any node will be able to raise a dispute. + +Therefore a vital responsibility of the dispute coordinator is to make sure backing votes are available for all +candidates that might still get disputed. To accomplish this task in an efficient way the dispute-coordinator relies on +chain scraping. Whenever a candidate gets backed on chain, we record in chain storage the backing votes imported in that +block. This way, given the chain state for a given relay chain block, we can retrieve via a provided runtime API the +backing votes imported by that block. The dispute coordinator makes sure to query those votes for any non finalized +blocks: In case of missed blocks, it will do chain traversal as necessary. Relying on chain scraping is very efficient for two reasons: -1. Votes are already batched. We import all available backing votes for a - candidate all at once. If instead we imported votes from candidate-backing as - they came along, we would import each vote individually which is - inefficient in the current dispute coordinator implementation (quadratic - complexity). -2. We also import less votes in total, as we avoid importing statements for - candidates that never got successfully backed on any chain. - -It also is secure, because disputes are only ever raised in the approval voting -phase. A node only starts the approval process after it has seen a candidate -included on some chain, for that to happen it must have been backed previously. -Therefore backing votes are available at that point in time. Signals are -processed first, so even if a block is skipped and we only start importing -backing votes on the including block, we will have seen the backing votes by the -time we process messages from approval voting. - -In summary, for making it possible for a dispute to be raised, recording of -backing votes from chain is sufficient and efficient. In particular there is no -need to preemptively import approval votes, which has shown to be a very +1. Votes are already batched. We import all available backing votes for a candidate all at once. If instead we imported + votes from candidate-backing as they came along, we would import each vote individually which is inefficient in the + current dispute coordinator implementation (quadratic complexity). +2. We also import less votes in total, as we avoid importing statements for candidates that never got successfully + backed on any chain. + +It also is secure, because disputes are only ever raised in the approval voting phase. A node only starts the approval +process after it has seen a candidate included on some chain, for that to happen it must have been backed previously. +Therefore backing votes are available at that point in time. Signals are processed first, so even if a block is skipped +and we only start importing backing votes on the including block, we will have seen the backing votes by the time we +process messages from approval voting. + +In summary, for making it possible for a dispute to be raised, recording of backing votes from chain is sufficient and +efficient. In particular there is no need to preemptively import approval votes, which has shown to be a very inefficient process. (Quadratic complexity adds up, with 35 votes in total per candidate) -Approval votes are very relevant nonetheless as we are going to see in the next -section. +Approval votes are very relevant nonetheless as we are going to see in the next section. ## Ensuring approval votes will be recorded @@ -88,521 +66,402 @@ section. Only votes recorded by the dispute coordinator will be considered for slashing. -While there is no need to record approval votes in the dispute coordinator -preemptively, we make some effort to have any in approval-voting received -approval votes recorded when a dispute actually happens: - -This is not required for concluding the dispute, as nodes send their own vote -anyway (either explicit valid or their existing approval-vote). What nodes can -do though, is participating in approval-voting, casting a vote, but later when a -dispute is raised reconsider their vote and send an explicit invalid vote. If -they managed to only have that one recorded, then they could avoid a slash. - -This is not a problem for our basic security assumptions: The backers are the -ones to be supposed to have skin in the game, so we are not too woried about -colluding approval voters getting away slash free as the gambler's ruin property -is maintained anyway. There is however a separate problem, from colluding -approval-voters, that is "lazy" approval voters. If it were easy and reliable -for approval-voters to reconsider their vote, in case of an actual dispute, then -they don't have a direct incentive (apart from playing a part in securing the -network) to properly run the validation function at all - they could just always -vote "valid" totally risk free. (While they would alwasy risk a slash by voting -invalid.) - - -So we do want to fetch approval votes from approval-voting. Importing votes is -most efficient when batched. At the same time approval voting and disputes are -running concurrently so approval votes are expected to trickle in still, when a +While there is no need to record approval votes in the dispute coordinator preemptively, we make some effort to have any +in approval-voting received approval votes recorded when a dispute actually happens: + +This is not required for concluding the dispute, as nodes send their own vote anyway (either explicit valid or their +existing approval-vote). What nodes can do though, is participating in approval-voting, casting a vote, but later when a +dispute is raised reconsider their vote and send an explicit invalid vote. If they managed to only have that one +recorded, then they could avoid a slash. + +This is not a problem for our basic security assumptions: The backers are the ones to be supposed to have skin in the +game, so we are not too woried about colluding approval voters getting away slash free as the gambler's ruin property is +maintained anyway. There is however a separate problem, from colluding approval-voters, that is "lazy" approval voters. +If it were easy and reliable for approval-voters to reconsider their vote, in case of an actual dispute, then they don't +have a direct incentive (apart from playing a part in securing the network) to properly run the validation function at +all - they could just always vote "valid" totally risk free. (While they would alwasy risk a slash by voting invalid.) + + +So we do want to fetch approval votes from approval-voting. Importing votes is most efficient when batched. At the same +time approval voting and disputes are running concurrently so approval votes are expected to trickle in still, when a dispute is already ongoing. Hence, we have the following requirements for importing approval votes: -1. Only import them when there is a dispute, because otherwise we are - wasting lots of resources _always_ for the exceptional case of a dispute. +1. Only import them when there is a dispute, because otherwise we are wasting lots of resources _always_ for the + exceptional case of a dispute. 2. Import votes batched when possible, to avoid quadratic import complexity. -3. Take into account that approval voting is still ongoing, while a dispute is - already running. - -With a design where approval voting sends votes to the dispute-coordinator by -itself, we would need to make approval voting aware of ongoing disputes and once -it is aware it could start sending all already existing votes batched and -trickling in votes as they come. The problem with this is, that it adds some -unnecessary complexity to approval-voting and also we might still import most of -the votes unbatched one-by-one, depending on what point in time the dispute was +3. Take into account that approval voting is still ongoing, while a dispute is already running. + +With a design where approval voting sends votes to the dispute-coordinator by itself, we would need to make approval +voting aware of ongoing disputes and once it is aware it could start sending all already existing votes batched and +trickling in votes as they come. The problem with this is, that it adds some unnecessary complexity to approval-voting +and also we might still import most of the votes unbatched one-by-one, depending on what point in time the dispute was raised. -Instead of the dispute coordinator informing approval-voting of an ongoing -dispute for it to begin forwarding votes to the dispute coordinator, it makes -more sense for the dispute-coordinator to just ask approval-voting for votes of -candidates in dispute. This way, the dispute coordinator can also pick the best -time for maximizing the number of votes in the batch. - -Now the question remains, when should the dispute coordinator ask -approval-voting for votes? - -In fact for slashing it is only relevant to have them once the dispute -concluded, so we can query approval voting the moment the dispute concludes! -Two concerns that come to mind, are easily addressed: - -1. Timing: We would like to rely as little as possible on implementation details - of approval voting. In particular, if the dispute is ongoing for a long time, - do we have any guarantees that approval votes are kept around long enough by - approval voting? Will approval votes still be present by the time the - dispute concludes in all cases? The answer is nuanced, but in general we - cannot rely on it. The problem is first, that finalization and - approval-voting is an off-chain process so there is no global consensus: As - soon as at least f+1 honest (f=n/3, where n is the number of - validators/nodes) nodes have seen the dispute conclude, finalization will - take place and approval votes will be cleared. This would still be fine, if - we had some guarantees that those honest nodes will be able to include those - votes in a block. This guarantee does not exist unfortunately, we will - discuss the problem and solutions in more detail [below][#Ensuring Chain Import]. - - The second problem is that approval-voting will abandon votes as soon as a - chain can no longer be finalized (some other/better fork already has been). - This second problem can somehow be mitigated by also importing votes as soon - as a dispute is detected, but not fully resolved. It is still inherently - racy. The good thing is, this should be good enough: We are worried about - lazy approval checkers, the system does not need to be perfect. It should be - enough if there is some risk of getting caught. -2. We are not worried about the dispute not concluding, as nodes will always - send their own vote, regardless of it being an explict or an already existing - approval-vote. - -Conclusion: As long as we make sure, if our own approval vote gets imported -(which would prevent dispute participation) to also distribute it via -dispute-distribution, disputes can conclude. To mitigate raciness with -approval-voting deleting votes we will import approval votes twice during a -dispute: Once when it is raised, to make as sure as possible to see approval -votes also for abandoned forks and second when the dispute concludes, to -maximize the amount of potentially malicious approval votes to be recorded. The -raciness obviously is not fully resolved by this, but this is fine as argued -above. +Instead of the dispute coordinator informing approval-voting of an ongoing dispute for it to begin forwarding votes to +the dispute coordinator, it makes more sense for the dispute-coordinator to just ask approval-voting for votes of +candidates in dispute. This way, the dispute coordinator can also pick the best time for maximizing the number of votes +in the batch. + +Now the question remains, when should the dispute coordinator ask approval-voting for votes? + +In fact for slashing it is only relevant to have them once the dispute concluded, so we can query approval voting the +moment the dispute concludes! Two concerns that come to mind, are easily addressed: + +1. Timing: We would like to rely as little as possible on implementation details of approval voting. In particular, if + the dispute is ongoing for a long time, do we have any guarantees that approval votes are kept around long enough by + approval voting? Will approval votes still be present by the time the dispute concludes in all cases? The answer is + nuanced, but in general we cannot rely on it. The problem is first, that finalization and approval-voting is an + off-chain process so there is no global consensus: As soon as at least f+1 honest (f=n/3, where n is the number of + validators/nodes) nodes have seen the dispute conclude, finalization will take place and approval votes will be + cleared. This would still be fine, if we had some guarantees that those honest nodes will be able to include those + votes in a block. This guarantee does not exist unfortunately, we will discuss the problem and solutions in more + detail [below][#Ensuring Chain Import]. + + The second problem is that approval-voting will abandon votes as soon as a chain can no longer be finalized (some + other/better fork already has been). This second problem can somehow be mitigated by also importing votes as soon as + a dispute is detected, but not fully resolved. It is still inherently racy. The good thing is, this should be good + enough: We are worried about lazy approval checkers, the system does not need to be perfect. It should be enough if + there is some risk of getting caught. +2. We are not worried about the dispute not concluding, as nodes will always send their own vote, regardless of it being + an explict or an already existing approval-vote. + +Conclusion: As long as we make sure, if our own approval vote gets imported (which would prevent dispute participation) +to also distribute it via dispute-distribution, disputes can conclude. To mitigate raciness with approval-voting +deleting votes we will import approval votes twice during a dispute: Once when it is raised, to make as sure as possible +to see approval votes also for abandoned forks and second when the dispute concludes, to maximize the amount of +potentially malicious approval votes to be recorded. The raciness obviously is not fully resolved by this, but this is +fine as argued above. Ensuring vote import on chain is covered in the next section. -What we don't care about is that honest approval-voters will likely validate -twice, once in approval voting and once via dispute-participation. Avoiding that -does not really seem worthwhile though, as disputes are for one exceptional, so -a little wasted effort won't affect everyday performance - second, even with -eager importing of approval votes, those doubled work is still present as -disputes and approvals are racing. Every time participation is faster than -approval, a node would do double work. +What we don't care about is that honest approval-voters will likely validate twice, once in approval voting and once via +dispute-participation. Avoiding that does not really seem worthwhile though, as disputes are for one exceptional, so a +little wasted effort won't affect everyday performance - second, even with eager importing of approval votes, those +doubled work is still present as disputes and approvals are racing. Every time participation is faster than approval, a +node would do double work. ### Ensuring Chain Import -While in the previous section we discussed means for nodes to ensure relevant -votes are recorded so lazy approval checkers get slashed properly, it is crucial -to also discuss the actual chain import. Only if we guarantee that recorded votes -will get imported on chain (on all potential chains really) we will succeed -in executing slashes. Particularly we need to make sure backing votes end up on -chain consistently. - -Dispute distribution will make sure all explicit dispute votes get distributed -among nodes which includes current block producers (current authority set) which -is an important property: If the dispute carries on across an era change, we -need to ensure that the new validator set will learn about any disputes and -their votes, so they can put that information on chain. Dispute-distribution -luckily has this property and always sends votes to the current authority set. -The issue is, for dispute-distribution, nodes send only their own explicit (or -in some cases their approval vote) in addition to some opposing vote. This -guarantees that at least some backing or approval vote will be present at the -block producer, but we don't have a 100% guarantee to have votes for all -backers, even less for approval checkers. - -Reason for backing votes: While backing votes will be present on at least some -chain, that does not mean that any such chain is still considered for block -production in the current set - they might only exist on an already abandoned -fork. This means a block producer that just joined the set, might not have seen -any of them. - -For approvals it is even more tricky and less necessary: Approval voting together -with finalization is a completely off-chain process therefore those protocols -don't care about block production at all. Approval votes only have a guarantee of -being propagated between the nodes that are responsible for finalizing the -concerned blocks. This implies that on an era change the current authority set, -will not necessarily get informed about any approval votes for the previous era. -Hence even if all validators of the previous era successfully recorded all approval -votes in the dispute coordinator, they won't get a chance to put them on chain, -hence they won't be considered for slashing. - -It is important to note, that the essential properties of the system still hold: -Dispute-distribution will distribute at _least one_ "valid" vote to the current -authority set, hence at least one node will get slashed in case of outcome -"invalid". Also in reality the validator set is rarely exchanged 100%, therefore -in practice some validators in the current authority set will overlap with the -ones in the previous set and will be able to record votes on chain. - -Still, for maximum accountability we need to make sure a previous authority set -can communicate votes to the next one, regardless of any chain: This is yet to -be implemented see section "Resiliency" in dispute-distribution and +While in the previous section we discussed means for nodes to ensure relevant votes are recorded so lazy approval +checkers get slashed properly, it is crucial to also discuss the actual chain import. Only if we guarantee that recorded +votes will get imported on chain (on all potential chains really) we will succeed in executing slashes. Particularly we +need to make sure backing votes end up on chain consistently. + +Dispute distribution will make sure all explicit dispute votes get distributed among nodes which includes current block +producers (current authority set) which is an important property: If the dispute carries on across an era change, we +need to ensure that the new validator set will learn about any disputes and their votes, so they can put that +information on chain. Dispute-distribution luckily has this property and always sends votes to the current authority +set. The issue is, for dispute-distribution, nodes send only their own explicit (or in some cases their approval vote) +in addition to some opposing vote. This guarantees that at least some backing or approval vote will be present at the +block producer, but we don't have a 100% guarantee to have votes for all backers, even less for approval checkers. + +Reason for backing votes: While backing votes will be present on at least some chain, that does not mean that any such +chain is still considered for block production in the current set - they might only exist on an already abandoned fork. +This means a block producer that just joined the set, might not have seen any of them. + +For approvals it is even more tricky and less necessary: Approval voting together with finalization is a completely +off-chain process therefore those protocols don't care about block production at all. Approval votes only have a +guarantee of being propagated between the nodes that are responsible for finalizing the concerned blocks. This implies +that on an era change the current authority set, will not necessarily get informed about any approval votes for the +previous era. Hence even if all validators of the previous era successfully recorded all approval votes in the dispute +coordinator, they won't get a chance to put them on chain, hence they won't be considered for slashing. + +It is important to note, that the essential properties of the system still hold: Dispute-distribution will distribute at +_least one_ "valid" vote to the current authority set, hence at least one node will get slashed in case of outcome +"invalid". Also in reality the validator set is rarely exchanged 100%, therefore in practice some validators in the +current authority set will overlap with the ones in the previous set and will be able to record votes on chain. + +Still, for maximum accountability we need to make sure a previous authority set can communicate votes to the next one, +regardless of any chain: This is yet to be implemented see section "Resiliency" in dispute-distribution and [this](https://github.com/paritytech/polkadot/issues/3398) ticket. ## Coordinating Actual Dispute Participation -Once the dispute coordinator learns about a dispute, it is its responsibility to -make sure the local node participates in that dispute. +Once the dispute coordinator learns about a dispute, it is its responsibility to make sure the local node participates +in that dispute. -The dispute coordinator learns about a dispute by importing votes from either -chain scraping or from dispute-distribution. If it finds opposing votes (always -the case when coming from dispute-distribution), it records the presence of a -dispute. Then, in case it does not find any local vote for that dispute already, -it needs to trigger participation in the dispute (see previous section for -considerations when the found local vote is an approval vote). +The dispute coordinator learns about a dispute by importing votes from either chain scraping or from +dispute-distribution. If it finds opposing votes (always the case when coming from dispute-distribution), it records the +presence of a dispute. Then, in case it does not find any local vote for that dispute already, it needs to trigger +participation in the dispute (see previous section for considerations when the found local vote is an approval vote). -Participation means, recovering availability and re-evaluating the POV. The -result of that validation (either valid or invalid) will be the node's vote on -that dispute: Either explicit "invalid" or "valid". The dispute coordinator will -inform `dispute-distribution` about our vote and `dispute-distribution` will make -sure that our vote gets distributed to all other validators. +Participation means, recovering availability and re-evaluating the POV. The result of that validation (either valid or +invalid) will be the node's vote on that dispute: Either explicit "invalid" or "valid". The dispute coordinator will +inform `dispute-distribution` about our vote and `dispute-distribution` will make sure that our vote gets distributed to +all other validators. -Nothing ever is that easy though. We can not blindly import anything that comes -along and trigger participation no matter what. +Nothing ever is that easy though. We can not blindly import anything that comes along and trigger participation no +matter what. ### Spam Considerations -In Polkadot's security model, it is important that attempts to attack the system -result in a slash of the offenders. Therefore we need to make sure that this -slash is actually happening. Attackers could try to prevent the slashing from -taking place, by overwhelming validators with disputes in such a way that no -single dispute ever concludes, because nodes are busy processing newly incoming -ones. Other attacks are imaginable as well, like raising disputes for candidates -that don't exist, just filling up everyone's disk slowly or worse making nodes -try to participate, which will result in lots of network requests for recovering -availability. - -The last point brings up a significant consideration in general: Disputes are -about escalation: Every node will suddenly want to check, instead of only a few. -A single message will trigger the whole network to start significant amount of -work and will cause lots of network traffic and messages. Hence the -dispute system is very susceptible to being a brutal amplifier for DoS attacks, -resulting in DoS attacks to become very easy and cheap, if we are not careful. - -One counter measure we are taking is making raising of disputes a costly thing: -If you raise a dispute, because you claim a candidate is invalid, although it is -in fact valid - you will get slashed, hence you pay for consuming those -resources. The issue is: This only works if the dispute concerns a candidate -that actually exists! - -If a node raises a dispute for a candidate that never got included (became -available) on any chain, then the dispute can never conclude, hence nobody gets -slashed. It makes sense to point out that this is less bad than it might sound -at first, as trying to participate in a dispute for a non existing candidate is -"relatively" cheap. Each node will send out a few hundred tiny request messages -for availability chunks, which all will end up in a tiny response "NoSuchChunk" -and then no participation will actually happen as there is nothing to -participate. Malicious nodes could provide chunks, which would make things more -costly, but at the full expense of the attackers bandwidth - no amplification -here. I am bringing that up for completeness only: Triggering a thousand nodes -to send out a thousand tiny network messages by just sending out a single -garbage message, is still a significant amplification and is nothing to ignore - -this could absolutely be used to cause harm! +In Polkadot's security model, it is important that attempts to attack the system result in a slash of the offenders. +Therefore we need to make sure that this slash is actually happening. Attackers could try to prevent the slashing from +taking place, by overwhelming validators with disputes in such a way that no single dispute ever concludes, because +nodes are busy processing newly incoming ones. Other attacks are imaginable as well, like raising disputes for +candidates that don't exist, just filling up everyone's disk slowly or worse making nodes try to participate, which will +result in lots of network requests for recovering availability. + +The last point brings up a significant consideration in general: Disputes are about escalation: Every node will suddenly +want to check, instead of only a few. A single message will trigger the whole network to start significant amount of +work and will cause lots of network traffic and messages. Hence the dispute system is very susceptible to being a brutal +amplifier for DoS attacks, resulting in DoS attacks to become very easy and cheap, if we are not careful. + +One counter measure we are taking is making raising of disputes a costly thing: If you raise a dispute, because you +claim a candidate is invalid, although it is in fact valid - you will get slashed, hence you pay for consuming those +resources. The issue is: This only works if the dispute concerns a candidate that actually exists! + +If a node raises a dispute for a candidate that never got included (became available) on any chain, then the dispute can +never conclude, hence nobody gets slashed. It makes sense to point out that this is less bad than it might sound at +first, as trying to participate in a dispute for a non existing candidate is "relatively" cheap. Each node will send out +a few hundred tiny request messages for availability chunks, which all will end up in a tiny response "NoSuchChunk" and +then no participation will actually happen as there is nothing to participate. Malicious nodes could provide chunks, +which would make things more costly, but at the full expense of the attackers bandwidth - no amplification here. I am +bringing that up for completeness only: Triggering a thousand nodes to send out a thousand tiny network messages by just +sending out a single garbage message, is still a significant amplification and is nothing to ignore - this could +absolutely be used to cause harm! ### Participation -As explained, just blindly participating in any "dispute" that comes along is -not a good idea. First we would like to make sure the dispute is actually -genuine, to prevent cheap DoS attacks. Secondly, in case of genuine disputes, we -would like to conclude one after the other, in contrast to -processing all at the same time, slowing down progress on all of them, bringing -individual processing to a complete halt in the worst case (nodes get overwhelmed -at some stage in the pipeline). - -To ensure to only spend significant work on genuine disputes, we only trigger -participation at all on any _vote import_ if any of the following holds true: - -- We saw the disputed candidate included in some not yet finalized block on at - least one fork of the chain. -- We have seen the disputed candidate backed in some not yet finalized block on - at least one fork of the chain. This ensures the candidate is at least not - completely made up and there has been some effort already flown into that - candidate. Generally speaking a dispute shouldn't be raised for a candidate - which is backed but is not yet included. Disputes are raised during approval - checking. We participate on such disputes as a precaution - maybe we haven't - seen the `CandidateIncluded` event yet? -- The dispute is already confirmed: Meaning that 1/3+1 nodes already - participated, as this suggests in our threat model that there was at least one - honest node that already voted, so the dispute must be genuine. - -Note: A node might be out of sync with the chain and we might only learn about a -block, including a candidate, after we learned about the dispute. This means, we -have to re-evaluate participation decisions on block import! - -With this, nodes won't waste significant resources on completely made up -candidates. The next step is to process dispute participation in a (globally) -ordered fashion. Meaning a majority of validators should arrive at at least -roughly at the same ordering of participation, for disputes to get resolved one -after another. This order is only relevant if there are lots of disputes, so we -obviously only need to worry about order if participations start queuing up. - -We treat participation for candidates that we have seen included with priority -and put them on a priority queue which sorts participation based on the block -number of the relay parent of the candidate and for candidates with the same -relay parent height further by the `CandidateHash`. This ordering is globally -unique and also prioritizes older candidates. - -The latter property makes sense, because if an older candidate turns out invalid, -we can roll back the full chain at once. If we resolved earlier disputes first -and they turned out invalid as well, we might need to roll back a couple of -times instead of just once to the oldest offender. This is obviously a good -idea, in particular it makes it impossible for an attacker to prevent rolling -back a very old candidate, by keeping raising disputes for newer candidates. - -For candidates we have not seen included, but we know are backed (thanks to -chain scraping) or we have seen a dispute with 1/3+1 participation (confirmed -dispute) on them - we put participation on a best-effort queue. It has got the -same ordering as the priority one - by block heights of the relay parent, older -blocks are with priority. There is a possibility not to be able to obtain the -block number of the parent when we are inserting the dispute in the queue. To -account for races, we will promote any existing participation request to the -priority queue once we learn about an including block. NOTE: this is still work -in progress and is tracked by [this +As explained, just blindly participating in any "dispute" that comes along is not a good idea. First we would like to +make sure the dispute is actually genuine, to prevent cheap DoS attacks. Secondly, in case of genuine disputes, we would +like to conclude one after the other, in contrast to processing all at the same time, slowing down progress on all of +them, bringing individual processing to a complete halt in the worst case (nodes get overwhelmed at some stage in the +pipeline). + +To ensure to only spend significant work on genuine disputes, we only trigger participation at all on any _vote import_ +if any of the following holds true: + +- We saw the disputed candidate included in some not yet finalized block on at least one fork of the chain. +- We have seen the disputed candidate backed in some not yet finalized block on at least one fork of the chain. This + ensures the candidate is at least not completely made up and there has been some effort already flown into that + candidate. Generally speaking a dispute shouldn't be raised for a candidate which is backed but is not yet included. + Disputes are raised during approval checking. We participate on such disputes as a precaution - maybe we haven't seen + the `CandidateIncluded` event yet? +- The dispute is already confirmed: Meaning that 1/3+1 nodes already participated, as this suggests in our threat model + that there was at least one honest node that already voted, so the dispute must be genuine. + +Note: A node might be out of sync with the chain and we might only learn about a block, including a candidate, after we +learned about the dispute. This means, we have to re-evaluate participation decisions on block import! + +With this, nodes won't waste significant resources on completely made up candidates. The next step is to process dispute +participation in a (globally) ordered fashion. Meaning a majority of validators should arrive at at least roughly at the +same ordering of participation, for disputes to get resolved one after another. This order is only relevant if there are +lots of disputes, so we obviously only need to worry about order if participations start queuing up. + +We treat participation for candidates that we have seen included with priority and put them on a priority queue which +sorts participation based on the block number of the relay parent of the candidate and for candidates with the same +relay parent height further by the `CandidateHash`. This ordering is globally unique and also prioritizes older +candidates. + +The latter property makes sense, because if an older candidate turns out invalid, we can roll back the full chain at +once. If we resolved earlier disputes first and they turned out invalid as well, we might need to roll back a couple of +times instead of just once to the oldest offender. This is obviously a good idea, in particular it makes it impossible +for an attacker to prevent rolling back a very old candidate, by keeping raising disputes for newer candidates. + +For candidates we have not seen included, but we know are backed (thanks to chain scraping) or we have seen a dispute +with 1/3+1 participation (confirmed dispute) on them - we put participation on a best-effort queue. It has got the same +ordering as the priority one - by block heights of the relay parent, older blocks are with priority. There is a +possibility not to be able to obtain the block number of the parent when we are inserting the dispute in the queue. To +account for races, we will promote any existing participation request to the priority queue once we learn about an +including block. NOTE: this is still work in progress and is tracked by [this issue](https://github.com/paritytech/polkadot/issues/5875). ### Abandoned Forks -Finalization: As mentioned we care about included and backed candidates on any -non-finalized chain, given that any disputed chain will not get finalized, we -don't need to care about finalized blocks, but what about forks that fall behind -the finalized chain in terms of block number? For those we would still like to -be able to participate in any raised disputes, otherwise attackers might be able -to avoid a slash if they manage to create a better fork after they learned about -the approval checkers. Therefore we do care about those forks even after they -have fallen behind the finalized chain. - -For simplicity we also care about the actual finalized chain (not just forks) up -to a certain depth. We do have to limit the depth, because otherwise we open a -DoS vector again. The depth (into the finalized chain) should be oriented on the -approval-voting execution timeout, in particular it should be significantly -larger. Otherwise by the time the execution is allowed to finish, we already -dropped information about those candidates and the dispute could not conclude. +Finalization: As mentioned we care about included and backed candidates on any non-finalized chain, given that any +disputed chain will not get finalized, we don't need to care about finalized blocks, but what about forks that fall +behind the finalized chain in terms of block number? For those we would still like to be able to participate in any +raised disputes, otherwise attackers might be able to avoid a slash if they manage to create a better fork after they +learned about the approval checkers. Therefore we do care about those forks even after they have fallen behind the +finalized chain. + +For simplicity we also care about the actual finalized chain (not just forks) up to a certain depth. We do have to limit +the depth, because otherwise we open a DoS vector again. The depth (into the finalized chain) should be oriented on the +approval-voting execution timeout, in particular it should be significantly larger. Otherwise by the time the execution +is allowed to finish, we already dropped information about those candidates and the dispute could not conclude. ## Import ### Spam Considerations -In the last section we looked at how to treat queuing participations to -handle heavy dispute load well. This already ensures, that honest nodes won't -amplify cheap DoS attacks. There is one minor issue remaining: Even if we delay -participation until we have some confirmation of the authenticity of the -dispute, we should also not blindly import all votes arriving into the database -as this might be used to just slowly fill up disk space, until the node is no -longer functional. This leads to our last protection mechanism at the dispute -coordinator level (dispute-distribution also has its own), which is spam slots. -For each import containing an invalid vote, where we don't know whether it might -be spam or not we increment a counter for each signing participant of explicit -`invalid` votes. - -What votes do we treat as a potential spam? A vote will increase a spam slot if -and only if all of the following conditions are satisfied: - -* the candidate under dispute was not seen included nor backed on any chain -* the dispute is not confirmed -* we haven't cast a vote for the dispute - -Whenever any vote on a dispute is imported these conditions are checked. If the -dispute is found not to be potential spam, then spam slots for the disputed candidate hash are cleared. This decrements the spam count for every validator +In the last section we looked at how to treat queuing participations to handle heavy dispute load well. This already +ensures, that honest nodes won't amplify cheap DoS attacks. There is one minor issue remaining: Even if we delay +participation until we have some confirmation of the authenticity of the dispute, we should also not blindly import all +votes arriving into the database as this might be used to just slowly fill up disk space, until the node is no longer +functional. This leads to our last protection mechanism at the dispute coordinator level (dispute-distribution also has +its own), which is spam slots. For each import containing an invalid vote, where we don't know whether it might be spam +or not we increment a counter for each signing participant of explicit `invalid` votes. + +What votes do we treat as a potential spam? A vote will increase a spam slot if and only if all of the following +conditions are satisfied: + +- the candidate under dispute was not seen included nor backed on any chain +- the dispute is not confirmed +- we haven't cast a vote for the dispute + +Whenever any vote on a dispute is imported these conditions are checked. If the dispute is found not to be potential +spam, then spam slots for the disputed candidate hash are cleared. This decrements the spam count for every validator which had voted invalid. -To keep spam slots from filling up unnecessarily we want to clear spam slots -whenever a candidate is seen to be backed or included. Fortunately this behavior -is acheived by clearing slots on vote import as described above. Because on chain -backing votes are processed when a block backing the disputed candidate is discovered, spam slots are cleared for every backed candidate. Included -candidates have also been seen as backed on the same fork, so decrementing spam -slots is handled in that case as well. - -The reason this works is because we only need to worry about actual dispute -votes. Import of backing votes are already rate limited and concern only real -candidates. For approval votes a similar argument holds (if they come from -approval-voting), but we also don't import them until a dispute already -concluded. For actual dispute votes we need two opposing votes, so there must be -an explicit `invalid` vote in the import. Only a third of the validators can be -malicious, so spam disk usage is limited to `2*vote_size*n/3*NUM_SPAM_SLOTS`, with -`n` being the number of validators. +To keep spam slots from filling up unnecessarily we want to clear spam slots whenever a candidate is seen to be backed +or included. Fortunately this behavior is acheived by clearing slots on vote import as described above. Because on chain +backing votes are processed when a block backing the disputed candidate is discovered, spam slots are cleared for every +backed candidate. Included candidates have also been seen as backed on the same fork, so decrementing spam slots is +handled in that case as well. + +The reason this works is because we only need to worry about actual dispute votes. Import of backing votes are already +rate limited and concern only real candidates. For approval votes a similar argument holds (if they come from +approval-voting), but we also don't import them until a dispute already concluded. For actual dispute votes we need two +opposing votes, so there must be an explicit `invalid` vote in the import. Only a third of the validators can be +malicious, so spam disk usage is limited to `2*vote_size*n/3*NUM_SPAM_SLOTS`, with `n` being the number of validators. ### Backing Votes -Backing votes are in some way special. For starters they are the only valid -votes that are guaranteed to exist for any valid dispute to be raised. Second -they are the only votes that commit to a shorter execution timeout -`BACKING_EXECUTION_TIMEOUT`, compared to a more lenient timeout used in approval -voting. To account properly for execution time variance across machines, -slashing might treat backing votes differently (more aggressively) than other -voting `valid` votes. Hence in import we shall never override a backing vote -with another valid vote. They can not be assumed to be interchangeable. +Backing votes are in some way special. For starters they are the only valid votes that are guaranteed to exist for any +valid dispute to be raised. Second they are the only votes that commit to a shorter execution timeout +`BACKING_EXECUTION_TIMEOUT`, compared to a more lenient timeout used in approval voting. To account properly for +execution time variance across machines, slashing might treat backing votes differently (more aggressively) than other +voting `valid` votes. Hence in import we shall never override a backing vote with another valid vote. They can not be +assumed to be interchangeable. ## Attacks & Considerations -The following attacks on the priority queue and best-effort queues are -considered in above design. +The following attacks on the priority queue and best-effort queues are considered in above design. ### Priority Queue -On the priority queue, we will only queue participations for candidates we have -seen included on any chain. Any attack attempt would start with a candidate -included on some chain, but an attacker could try to only reveal the including -relay chain blocks to just some honest validators and stop as soon as it learns -that some honest validator would have a relevant approval assignment. +On the priority queue, we will only queue participations for candidates we have seen included on any chain. Any attack +attempt would start with a candidate included on some chain, but an attacker could try to only reveal the including +relay chain blocks to just some honest validators and stop as soon as it learns that some honest validator would have a +relevant approval assignment. -Without revealing the including block to any honest validator, we don't really -have an attack yet. Once the block is revealed though, the above is actually -very hard. Each honest validator will re-distribute the block it just learned -about. This means an attacker would need to pull of a targeted DoS attack, which -allows the validator to send its assignment, but prevents it from forwarding and -sharing the relay chain block. +Without revealing the including block to any honest validator, we don't really have an attack yet. Once the block is +revealed though, the above is actually very hard. Each honest validator will re-distribute the block it just learned +about. This means an attacker would need to pull of a targeted DoS attack, which allows the validator to send its +assignment, but prevents it from forwarding and sharing the relay chain block. -This sounds already hard enough, provided that we also start participation if -we learned about an including block after the dispute has been raised already -(we need to update participation queues on new leaves), but to be even safer -we choose to have an additional best-effort queue. +This sounds already hard enough, provided that we also start participation if we learned about an including block after +the dispute has been raised already (we need to update participation queues on new leaves), but to be even safer we +choose to have an additional best-effort queue. ### Best-Effort Queue -While attacking the priority queue is already pretty hard, attacking the -best-effort queue is even harder. For a candidate to be a threat, it has to be -included on some chain. For it to be included, it has to have been backed before -and at least n/3 honest nodes must have seen that block, so availability -(inclusion) can be reached. Making a full third of the nodes not further -propagate a block, while at the same time allowing them to fetch chunks, sign -and distribute bitfields seems almost infeasible and even if accomplished, those -nodes would be enough to confirm a dispute and we have not even touched the -above fact that in addition, for an attack, the following including block must -be shared with honest validators as well. - -It is worth mentioning that a successful attack on the priority queue as -outlined above is already outside of our threat model, as it assumes n/3 -malicious nodes + additionally malfunctioning/DoSed nodes. Even more so for -attacks on the best-effort queue, as our threat model only allows for n/3 -malicious _or_ malfunctioning nodes in total. It would therefore be a valid -decision to ditch the best-effort queue, if it proves to become a burden or -creates other issues. - -One issue we should not be worried about though is spam. For abusing best-effort -for spam, the following scenario would be necessary: - -An attacker controls a backing group: The attacker can then have candidates -backed and choose to not provide chunks. This should come at a cost to miss out -on rewards for backing, so is not free. At the same time it is rate limited, as -a backing group can only back so many candidates legitimately. (~ 1 per slot): - -1. They have to wait until a malicious actor becomes block producer (for causing - additional forks via equivocation for example). +While attacking the priority queue is already pretty hard, attacking the best-effort queue is even harder. For a +candidate to be a threat, it has to be included on some chain. For it to be included, it has to have been backed before +and at least n/3 honest nodes must have seen that block, so availability (inclusion) can be reached. Making a full third +of the nodes not further propagate a block, while at the same time allowing them to fetch chunks, sign and distribute +bitfields seems almost infeasible and even if accomplished, those nodes would be enough to confirm a dispute and we have +not even touched the above fact that in addition, for an attack, the following including block must be shared with +honest validators as well. + +It is worth mentioning that a successful attack on the priority queue as outlined above is already outside of our threat +model, as it assumes n/3 malicious nodes + additionally malfunctioning/DoSed nodes. Even more so for attacks on the +best-effort queue, as our threat model only allows for n/3 malicious _or_ malfunctioning nodes in total. It would +therefore be a valid decision to ditch the best-effort queue, if it proves to become a burden or creates other issues. + +One issue we should not be worried about though is spam. For abusing best-effort for spam, the following scenario would +be necessary: + +An attacker controls a backing group: The attacker can then have candidates backed and choose to not provide chunks. +This should come at a cost to miss out on rewards for backing, so is not free. At the same time it is rate limited, as a +backing group can only back so many candidates legitimately. (~ 1 per slot): + +1. They have to wait until a malicious actor becomes block producer (for causing additional forks via equivocation for + example). 2. Forks are possible, but if caused by equivocation also not free. -3. For each fork the attacker has to wait until the candidate times out, for - backing another one. +3. For each fork the attacker has to wait until the candidate times out, for backing another one. -Assuming there can only be a handful of forks, 2) together with 3) the candidate -timeout restriction, frequency should indeed be in the ballpark of once per -slot. Scaling linearly in the number of controlled backing groups, so two groups +Assuming there can only be a handful of forks, 2) together with 3) the candidate timeout restriction, frequency should +indeed be in the ballpark of once per slot. Scaling linearly in the number of controlled backing groups, so two groups would mean 2 backings per slot, ... -So by this reasoning an attacker could only do very limited harm and at the same -time will have to pay some price for it (it will miss out on rewards). Overall -the work done by the network might even be in the same ballpark as if actors -just behaved honestly: +So by this reasoning an attacker could only do very limited harm and at the same time will have to pay some price for it +(it will miss out on rewards). Overall the work done by the network might even be in the same ballpark as if actors just +behaved honestly: 1. Validators would have fetched chunks 2. Approval checkers would have done approval checks -While because of the attack (backing, not providing chunks and afterwards -disputing the candidate), the work for 1000 validators would be: +While because of the attack (backing, not providing chunks and afterwards disputing the candidate), the work for 1000 +validators would be: -All validators sending out ~ 1000 tiny requests over already established -connections, with also tiny (byte) responses. +All validators sending out ~ 1000 tiny requests over already established connections, with also tiny (byte) responses. -This means around a million requests, while in the honest case it would be ~ -10000 (30 approval checkers x330) - where each request triggers a response in -the range of kilobytes. Hence network load alone will likely be higher in the -honest case than in the DoS attempt case, which would mean the DoS attempt -actually reduces load, while also costing rewards. +This means around a million requests, while in the honest case it would be ~ 10000 (30 approval checkers x330) - where +each request triggers a response in the range of kilobytes. Hence network load alone will likely be higher in the honest +case than in the DoS attempt case, which would mean the DoS attempt actually reduces load, while also costing rewards. -In the worst case this can happen multiple times, as we would retry that on -every vote import. The effect would still be in the same ballpark as honest -behavior though and can also be mitigated by chilling repeated availability -recovery requests for example. +In the worst case this can happen multiple times, as we would retry that on every vote import. The effect would still be +in the same ballpark as honest behavior though and can also be mitigated by chilling repeated availability recovery +requests for example. ## Out of Scope ### No Disputes for Non Included Candidates -We only ever care about disputes for candidates that have been included on at -least some chain (became available). This is because the availability system was -designed for precisely that: Only with inclusion (availability) we have -guarantees about the candidate to actually be available. Because only then we -have guarantees that malicious backers can be reliably checked and slashed. Also, by design non included candidates do not pose any threat to the system. - -One could think of an (additional) dispute system to make it possible to dispute -any candidate that has been proposed by a validator, no matter whether it got -successfully included or even backed. Unfortunately, it would be very brittle -(no availability) and also spam protection would be way harder than for the -disputes handled by the dispute-coordinator. In fact, all the spam handling -strategies described above would simply be unavailable. - -It is worth thinking about who could actually raise such disputes anyway: -Approval checkers certainly not, as they will only ever check once availability -succeeded. The only other nodes that meaningfully could/would are honest backing -nodes or collators. For collators spam considerations would be even worse as -there can be an unlimited number of them and we can not charge them for spam, so -trying to handle disputes raised by collators would be even more complex. For -honest backers: It actually makes more sense for them to wait until availability -is reached as well, as only then they have guarantees that other nodes will be -able to check. If they disputed before, all nodes would need to recover the data +We only ever care about disputes for candidates that have been included on at least some chain (became available). This +is because the availability system was designed for precisely that: Only with inclusion (availability) we have +guarantees about the candidate to actually be available. Because only then we have guarantees that malicious backers can +be reliably checked and slashed. Also, by design non included candidates do not pose any threat to the system. + +One could think of an (additional) dispute system to make it possible to dispute any candidate that has been proposed by +a validator, no matter whether it got successfully included or even backed. Unfortunately, it would be very brittle (no +availability) and also spam protection would be way harder than for the disputes handled by the dispute-coordinator. In +fact, all the spam handling strategies described above would simply be unavailable. + +It is worth thinking about who could actually raise such disputes anyway: Approval checkers certainly not, as they will +only ever check once availability succeeded. The only other nodes that meaningfully could/would are honest backing nodes +or collators. For collators spam considerations would be even worse as there can be an unlimited number of them and we +can not charge them for spam, so trying to handle disputes raised by collators would be even more complex. For honest +backers: It actually makes more sense for them to wait until availability is reached as well, as only then they have +guarantees that other nodes will be able to check. If they disputed before, all nodes would need to recover the data from them, so they would be an easy DoS target. -In summary: The availability system was designed for raising disputes in a -meaningful and secure way after availability was reached. Trying to raise -disputes before does not meaningfully contribute to the systems security/might -even weaken it as attackers are warned before availability is reached, while at -the same time adding signficant amount of complexity. We therefore punt on such -disputes and concentrate on disputes the system was designed to handle. +In summary: The availability system was designed for raising disputes in a meaningful and secure way after availability +was reached. Trying to raise disputes before does not meaningfully contribute to the systems security/might even weaken +it as attackers are warned before availability is reached, while at the same time adding signficant amount of +complexity. We therefore punt on such disputes and concentrate on disputes the system was designed to handle. ### No Disputes for Already Finalized Blocks -Note that by above rules in the `Participation` section, we will not participate -in disputes concerning a candidate in an already finalized block. This is -because, disputing an already finalized block is simply too late and therefore -of little value. Once finalized, bridges have already processed the block for -example, so we have to assume the damage is already done. Governance has to step -in and fix what can be fixed. +Note that by above rules in the `Participation` section, we will not participate in disputes concerning a candidate in +an already finalized block. This is because, disputing an already finalized block is simply too late and therefore of +little value. Once finalized, bridges have already processed the block for example, so we have to assume the damage is +already done. Governance has to step in and fix what can be fixed. -Making disputes for already finalized blocks possible would only provide two -features: +Making disputes for already finalized blocks possible would only provide two features: 1. We can at least still slash attackers. -2. We can freeze the chain to some governance only mode, in an attempt to - minimize potential harm done. +2. We can freeze the chain to some governance only mode, in an attempt to minimize potential harm done. -Both seem kind of worthwhile, although as argued above, it is likely that there -is not too much that can be done in 2 and we would likely only ending up DoSing -the whole system without much we can do. 1 can also be achieved via governance +Both seem kind of worthwhile, although as argued above, it is likely that there is not too much that can be done in 2 +and we would likely only ending up DoSing the whole system without much we can do. 1 can also be achieved via governance mechanisms. -In any case, our focus should be making as sure as reasonably possible that any -potentially invalid block does not get finalized in the first place. Not -allowing disputing already finalized blocks actually helps a great deal with -this goal as it massively reduces the amount of candidates that can be disputed. - -This makes attempts to overwhelm the system with disputes significantly harder -and counter measures way easier. We can limit inclusion for example (as -suggested [here](https://github.com/paritytech/polkadot/issues/5898) in case of -high dispute load. Another measure we have at our disposal is that on finality -lag block production will slow down, implicitly reducing the rate of new -candidates that can be disputed. Hence, the cutting-off of the unlimited -candidate supply of already finalized blocks, guarantees the necessary DoS -protection and ensures we can have measures in place to keep up with processing -of disputes. - -If we allowed participation for disputes for already finalized candidates, the -above spam protection mechanisms would be insufficient/relying 100% on full and -quick disabling of spamming validators. +In any case, our focus should be making as sure as reasonably possible that any potentially invalid block does not get +finalized in the first place. Not allowing disputing already finalized blocks actually helps a great deal with this goal +as it massively reduces the amount of candidates that can be disputed. + +This makes attempts to overwhelm the system with disputes significantly harder and counter measures way easier. We can +limit inclusion for example (as suggested [here](https://github.com/paritytech/polkadot/issues/5898) in case of high +dispute load. Another measure we have at our disposal is that on finality lag block production will slow down, +implicitly reducing the rate of new candidates that can be disputed. Hence, the cutting-off of the unlimited candidate +supply of already finalized blocks, guarantees the necessary DoS protection and ensures we can have measures in place to +keep up with processing of disputes. + +If we allowed participation for disputes for already finalized candidates, the above spam protection mechanisms would be +insufficient/relying 100% on full and quick disabling of spamming validators. ## Database Schema We use an underlying Key-Value database where we assume we have the following operations available: - * `write(key, value)` - * `read(key) -> Option` - * `iter_with_prefix(prefix) -> Iterator<(key, value)>` - gives all keys and values in - lexicographical order where the key starts with `prefix`. + - `write(key, value)` + - `read(key) -> Option` + - `iter_with_prefix(prefix) -> Iterator<(key, value)>` - gives all keys and values in lexicographical order where the + key starts with `prefix`. We use this database to encode the following schema: @@ -612,8 +471,8 @@ We use this database to encode the following schema: "earliest-session" -> Option ``` -The meta information that we track per-candidate is defined as the `CandidateVotes` struct. -This draws on the [dispute statement types][DisputeTypes] +The meta information that we track per-candidate is defined as the `CandidateVotes` struct. This draws on the [dispute +statement types][DisputeTypes] ```rust /// Tracked votes on candidates, for the purposes of dispute resolution. @@ -659,8 +518,7 @@ Output: ## Functionality -This assumes a constant `DISPUTE_WINDOW: SessionWindowSize`. This should correspond to at least 1 -day. +This assumes a constant `DISPUTE_WINDOW: SessionWindowSize`. This should correspond to at least 1 day. Ephemeral in-memory state: @@ -684,42 +542,36 @@ struct State { ### On startup -When the subsystem is initialised it waits for a new leaf (message -`OverseerSignal::ActiveLeaves`). The leaf is used to initialise a -`RollingSessionWindow` instance (contains leaf hash and `DISPUTE_WINDOW` which -is a constant). +When the subsystem is initialised it waits for a new leaf (message `OverseerSignal::ActiveLeaves`). The leaf is used to +initialise a `RollingSessionWindow` instance (contains leaf hash and `DISPUTE_WINDOW` which is a constant). -Next the active disputes are loaded from the DB and initialize spam slots -accordingly, then for each loaded dispute, we either send a -`DisputeDistribution::SendDispute` if there is a local vote from us available or -if there is none and participation is in order, we push the dispute to -participation. +Next the active disputes are loaded from the DB and initialize spam slots accordingly, then for each loaded dispute, we +either send a `DisputeDistribution::SendDispute` if there is a local vote from us available or if there is none and +participation is in order, we push the dispute to participation. ### The main loop -Just after the subsystem initialisation the main loop (`fn run_until_error()`) runs until -`OverseerSignal::Conclude` signal is received. Before executing the actual main loop the leaf and -the participations, obtained during startup are enqueued for processing. If there is capacity (the -number of running participations is less than `MAX_PARALLEL_PARTICIPATIONS`) participation jobs are -started (`func participate`). Finally the component waits for messages from Overseer. The behaviour -on each message is described in the following subsections. +Just after the subsystem initialisation the main loop (`fn run_until_error()`) runs until `OverseerSignal::Conclude` +signal is received. Before executing the actual main loop the leaf and the participations, obtained during startup are +enqueued for processing. If there is capacity (the number of running participations is less than +`MAX_PARALLEL_PARTICIPATIONS`) participation jobs are started (`func participate`). Finally the component waits for +messages from Overseer. The behaviour on each message is described in the following subsections. ### On `OverseerSignal::ActiveLeaves` -Initiates processing via the `Participation` module and updates the internal state of the subsystem. -More concretely: +Initiates processing via the `Participation` module and updates the internal state of the subsystem. More concretely: -* Passes the `ActiveLeavesUpdate` message to the ordering provider. -* Updates the session info cache. -* Updates `self.highest_session`. -* Prunes old spam slots in case the session window has advanced. -* Scrapes on chain votes. +- Passes the `ActiveLeavesUpdate` message to the ordering provider. +- Updates the session info cache. +- Updates `self.highest_session`. +- Prunes old spam slots in case the session window has advanced. +- Scrapes on chain votes. ### On `MuxedMessage::Participation` -This message is sent from `Participatuion` module and indicates a processed dispute participation. -It's the result of the processing job initiated with `OverseerSignal::ActiveLeaves`. The subsystem -issues a `DisputeMessage` with the result. +This message is sent from `Participatuion` module and indicates a processed dispute participation. It's the result of +the processing job initiated with `OverseerSignal::ActiveLeaves`. The subsystem issues a `DisputeMessage` with the +result. ### On `OverseerSignal::Conclude` @@ -731,14 +583,13 @@ Performs cleanup of the finalized candidate. ### On `DisputeCoordinatorMessage::ImportStatements` -Import statements by validators are processed in `fn handle_import_statements()`. The function has -got three main responsibilities: -* Initiate participation in disputes and sending out of any existing own - approval vote in case of a raised dispute. -* Persist all fresh votes in the database. Fresh votes in this context means votes that are not - already processed by the node. -* Spam protection on all invalid (`DisputeStatement::Invalid`) votes. Please check the SpamSlots - section for details on how spam protection works. +Import statements by validators are processed in `fn handle_import_statements()`. The function has got three main +responsibilities: +- Initiate participation in disputes and sending out of any existing own approval vote in case of a raised dispute. +- Persist all fresh votes in the database. Fresh votes in this context means votes that are not already processed by the + node. +- Spam protection on all invalid (`DisputeStatement::Invalid`) votes. Please check the SpamSlots section for details on + how spam protection works. ### On `DisputeCoordinatorMessage::RecentDisputes` @@ -750,38 +601,35 @@ Returns all recent disputes concluded within the last `ACTIVE_DURATION_SECS` . ### On `DisputeCoordinatorMessage::QueryCandidateVotes` -Loads `candidate-votes` for every `(SessionIndex, CandidateHash)` in the input query and returns -data within each `CandidateVote`. If a particular `candidate-vote` is missing, that particular -request is omitted from the response. +Loads `candidate-votes` for every `(SessionIndex, CandidateHash)` in the input query and returns data within each +`CandidateVote`. If a particular `candidate-vote` is missing, that particular request is omitted from the response. ### On `DisputeCoordinatorMessage::IssueLocalStatement` Executes `fn issue_local_statement()` which performs the following operations: -* Deconstruct into parts `{ session_index, candidate_hash, candidate_receipt, is_valid }`. -* Construct a [`DisputeStatement`][DisputeStatement] based on `Valid` or `Invalid`, depending on the - parameterization of this routine. -* Sign the statement with each key in the `SessionInfo`'s list of parachain validation keys which is - present in the keystore, except those whose indices appear in `voted_indices`. This will typically - just be one key, but this does provide some future-proofing for situations where the same node may - run on behalf multiple validators. At the time of writing, this is not a use-case we support as - other subsystems do not invariably provide this guarantee. -* Write statement to DB. -* Send a `DisputeDistributionMessage::SendDispute` message to get the vote distributed to other - validators. +- Deconstruct into parts `{ session_index, candidate_hash, candidate_receipt, is_valid }`. +- Construct a [`DisputeStatement`][DisputeStatement] based on `Valid` or `Invalid`, depending on the parameterization of + this routine. +- Sign the statement with each key in the `SessionInfo`'s list of parachain validation keys which is present in the + keystore, except those whose indices appear in `voted_indices`. This will typically just be one key, but this does + provide some future-proofing for situations where the same node may run on behalf multiple validators. At the time of + writing, this is not a use-case we support as other subsystems do not invariably provide this guarantee. +- Write statement to DB. +- Send a `DisputeDistributionMessage::SendDispute` message to get the vote distributed to other validators. ### On `DisputeCoordinatorMessage::DetermineUndisputedChain` Executes `fn determine_undisputed_chain()` which performs the following: -* Load `"recent-disputes"`. -* Deconstruct into parts `{ base_number, block_descriptions, rx }` -* Starting from the beginning of `block_descriptions`: +- Load `"recent-disputes"`. +- Deconstruct into parts `{ base_number, block_descriptions, rx }` +- Starting from the beginning of `block_descriptions`: 1. Check the `RecentDisputes` for a dispute of each candidate in the block description. 1. If there is a dispute which is active or concluded negative, exit the loop. -* For the highest index `i` reached in the `block_descriptions`, send `(base_number + i + 1, - block_hash)` on the channel, unless `i` is 0, in which case `None` should be sent. The - `block_hash` is determined by inspecting `block_descriptions[i]`. +- For the highest index `i` reached in the `block_descriptions`, send `(base_number + i + 1, block_hash)` on the + channel, unless `i` is 0, in which case `None` should be sent. The `block_hash` is determined by inspecting + `block_descriptions[i]`. [DisputeTypes]: ../../types/disputes.md [DisputeStatement]: ../../types/disputes.md#disputestatement diff --git a/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-distribution.md b/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-distribution.md index 3a45f53c45d7905a5c0b9ff3fe1a8ae299b211f9..4547e02352ec1c44eecd81c6e03d1b92bb1e26f1 100644 --- a/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-distribution.md +++ b/polkadot/roadmap/implementers-guide/src/node/disputes/dispute-distribution.md @@ -202,8 +202,8 @@ the dispute-coordinator already knows about the dispute. Goal 3 and 4 are obviously very related and both can easily be solved via rate limiting as we shall see below. Rate limits should already be implemented at the -substrate level, but [are not](https://github.com/paritytech/substrate/issues/7750) -at the time of writing. But even if they were, the enforced substrate limits would +Substrate level, but [are not](https://github.com/paritytech/substrate/issues/7750) +at the time of writing. But even if they were, the enforced Substrate limits would likely not be configurable and thus would still be to high for our needs as we can rely on the following observations: @@ -282,10 +282,10 @@ well, we will do the following: to assume this is concerning a new dispute. 2. We open a batch and start collecting incoming messages for that candidate, instead of immediately forwarding. -4. We keep collecting votes in the batch until we receive less than +3. We keep collecting votes in the batch until we receive less than `MIN_KEEP_BATCH_ALIVE_VOTES` unique votes in the last `BATCH_COLLECTING_INTERVAL`. This is important to accommodate for goal 5 and also 3. -5. We send the whole batch to the dispute-coordinator. +4. We send the whole batch to the dispute-coordinator. This together with rate limiting explained above ensures we will be able to process valid disputes: We can limit the number of simultaneous existing batches @@ -312,8 +312,8 @@ of attackers, each has 10 messages per second, all are needed to maintain the batches in memory. Therefore we have a hard cap of around 330 (number of malicious nodes) open batches. Each can be filled with number of malicious actor's votes. So 330 batches with each 330 votes: Let's assume approximately 100 -bytes per signature/vote. This results in a worst case memory usage of 330 * 330 -* 100 ~= 10 MiB. +bytes per signature/vote. This results in a worst case memory usage of +`330 * 330 * 100 ~= 10 MiB`. For 10_000 validators, we are already in the Gigabyte range, which means that with a validator set that large we might want to be more strict with the rate limit or diff --git a/polkadot/roadmap/implementers-guide/src/node/grandpa-voting-rule.md b/polkadot/roadmap/implementers-guide/src/node/grandpa-voting-rule.md index 5e608ccfd62e7d37a9a1c7cdc45130b5a535e9c3..6997dc99f80eb40301c090c1d2dc33782faa9a7f 100644 --- a/polkadot/roadmap/implementers-guide/src/node/grandpa-voting-rule.md +++ b/polkadot/roadmap/implementers-guide/src/node/grandpa-voting-rule.md @@ -1,10 +1,25 @@ # GRANDPA Voting Rule -Specifics on the motivation and types of constraints we apply to the GRANDPA voting logic as well as the definitions of **viable** and **finalizable** blocks can be found in the [Chain Selection Protocol](../protocol-chain-selection.md) section. -The subsystem which provides us with viable leaves is the [Chain Selection Subsystem](utility/chain-selection.md). +Specifics on the motivation and types of constraints we apply to the GRANDPA voting logic as well as the definitions of +**viable** and **finalizable** blocks can be found in the [Chain Selection Protocol](../protocol-chain-selection.md) +section. The subsystem which provides us with viable leaves is the [Chain Selection +Subsystem](utility/chain-selection.md). -GRANDPA's regular voting rule is for each validator to select the longest chain they are aware of. GRANDPA proceeds in rounds, collecting information from all online validators and determines the blocks that a supermajority of validators all have in common with each other. +GRANDPA's regular voting rule is for each validator to select the longest chain they are aware of. GRANDPA proceeds in +rounds, collecting information from all online validators and determines the blocks that a supermajority of validators +all have in common with each other. -The low-level GRANDPA logic will provide us with a **required block**. We can find the best leaf containing that block in its chain with the [`ChainSelectionMessage::BestLeafContaining`](../types/overseer-protocol.md#chain-selection-message). If the result is `None`, then we will simply cast a vote on the required block. +The low-level GRANDPA logic will provide us with a **required block**. We can find the best leaf containing that block +in its chain with the +[`ChainSelectionMessage::BestLeafContaining`](../types/overseer-protocol.md#chain-selection-message). If the result is +`None`, then we will simply cast a vote on the required block. -The **viable** leaves provided from the chain selection subsystem are not necessarily **finalizable**, so we need to perform further work to discover the finalizable ancestor of the block. The first constraint is to avoid voting on any unapproved block. The highest approved ancestor of a given block can be determined by querying the Approval Voting subsystem via the [`ApprovalVotingMessage::ApprovedAncestor`](../types/overseer-protocol.md#approval-voting) message. If the response is `Some`, we continue and apply the second constraint. The second constraint is to avoid voting on any block containing a candidate undergoing an active dispute. The list of block hashes and candidates returned from `ApprovedAncestor` should be reversed, and passed to the [`DisputeCoordinatorMessage::DetermineUndisputedChain`](../types/overseer-protocol.md#dispute-coordinator-message) to determine the **finalizable** block which will be our eventual vote. +The **viable** leaves provided from the chain selection subsystem are not necessarily **finalizable**, so we need to +perform further work to discover the finalizable ancestor of the block. The first constraint is to avoid voting on any +unapproved block. The highest approved ancestor of a given block can be determined by querying the Approval Voting +subsystem via the [`ApprovalVotingMessage::ApprovedAncestor`](../types/overseer-protocol.md#approval-voting) message. If +the response is `Some`, we continue and apply the second constraint. The second constraint is to avoid voting on any +block containing a candidate undergoing an active dispute. The list of block hashes and candidates returned from +`ApprovedAncestor` should be reversed, and passed to the +[`DisputeCoordinatorMessage::DetermineUndisputedChain`](../types/overseer-protocol.md#dispute-coordinator-message) to +determine the **finalizable** block which will be our eventual vote. diff --git a/polkadot/roadmap/implementers-guide/src/node/overseer.md b/polkadot/roadmap/implementers-guide/src/node/overseer.md index 21300d9098a2949b21b5ac8f9efd54cd5dd37472..53a1153081013b684d0cab419fb3eaa0be5a15c1 100644 --- a/polkadot/roadmap/implementers-guide/src/node/overseer.md +++ b/polkadot/roadmap/implementers-guide/src/node/overseer.md @@ -24,27 +24,44 @@ The hierarchy of subsystems: ``` -The overseer determines work to do based on block import events and block finalization events. It does this by keeping track of the set of relay-parents for which work is currently being done. This is known as the "active leaves" set. It determines an initial set of active leaves on startup based on the data on-disk, and uses events about blockchain import to update the active leaves. Updates lead to [`OverseerSignal`](../types/overseer-protocol.md#overseer-signal)`::ActiveLeavesUpdate` being sent according to new relay-parents, as well as relay-parents to stop considering. Block import events inform the overseer of leaves that no longer need to be built on, now that they have children, and inform us to begin building on those children. Block finalization events inform us when we can stop focusing on blocks that appear to have been orphaned. - -The overseer is also responsible for tracking the freshness of active leaves. Leaves are fresh when they're encountered for the first time, and stale when they're encountered for subsequent times. This can occur after chain reversions or when the fork-choice rule abandons some chain. This distinction is used to manage **Reversion Safety**. Consensus messages are often localized to a specific relay-parent, and it is often a misbehavior to equivocate or sign two conflicting messages. When reverting the chain, we may begin work on a leaf that subsystems have already signed messages for. Subsystems which need to account for reversion safety should avoid performing work on stale leaves. +The overseer determines work to do based on block import events and block finalization events. It does this by keeping +track of the set of relay-parents for which work is currently being done. This is known as the "active leaves" set. It +determines an initial set of active leaves on startup based on the data on-disk, and uses events about blockchain import +to update the active leaves. Updates lead to +[`OverseerSignal`](../types/overseer-protocol.md#overseer-signal)`::ActiveLeavesUpdate` being sent according to new +relay-parents, as well as relay-parents to stop considering. Block import events inform the overseer of leaves that no +longer need to be built on, now that they have children, and inform us to begin building on those children. Block +finalization events inform us when we can stop focusing on blocks that appear to have been orphaned. + +The overseer is also responsible for tracking the freshness of active leaves. Leaves are fresh when they're encountered +for the first time, and stale when they're encountered for subsequent times. This can occur after chain reversions or +when the fork-choice rule abandons some chain. This distinction is used to manage **Reversion Safety**. Consensus +messages are often localized to a specific relay-parent, and it is often a misbehavior to equivocate or sign two +conflicting messages. When reverting the chain, we may begin work on a leaf that subsystems have already signed messages +for. Subsystems which need to account for reversion safety should avoid performing work on stale leaves. The overseer's logic can be described with these functions: ## On Startup * Start all subsystems -* Determine all blocks of the blockchain that should be built on. This should typically be the head of the best fork of the chain we are aware of. Sometimes add recent forks as well. +* Determine all blocks of the blockchain that should be built on. This should typically be the head of the best fork of + the chain we are aware of. Sometimes add recent forks as well. * Send an `OverseerSignal::ActiveLeavesUpdate` to all subsystems with `activated` containing each of these blocks. * Begin listening for block import and finality events ## On Block Import Event -* Apply the block import event to the active leaves. A new block should lead to its addition to the active leaves set and its parent being deactivated. -* Mark any stale leaves as stale. The overseer should track all leaves it activates to determine whether leaves are fresh or stale. -* Send an `OverseerSignal::ActiveLeavesUpdate` message to all subsystems containing all activated and deactivated leaves. +* Apply the block import event to the active leaves. A new block should lead to its addition to the active leaves set + and its parent being deactivated. +* Mark any stale leaves as stale. The overseer should track all leaves it activates to determine whether leaves are + fresh or stale. +* Send an `OverseerSignal::ActiveLeavesUpdate` message to all subsystems containing all activated and deactivated + leaves. * Ensure all `ActiveLeavesUpdate` messages are flushed before resuming activity as a message router. -> TODO: in the future, we may want to avoid building on too many sibling blocks at once. the notion of a "preferred head" among many competing sibling blocks would imply changes in our "active leaves" update rules here +> TODO: in the future, we may want to avoid building on too many sibling blocks at once. the notion of a "preferred +> head" among many competing sibling blocks would imply changes in our "active leaves" update rules here ## On Finalization Event @@ -54,11 +71,16 @@ The overseer's logic can be described with these functions: ## On Subsystem Failure -Subsystems are essential tasks meant to run as long as the node does. Subsystems can spawn ephemeral work in the form of jobs, but the subsystems themselves should not go down. If a subsystem goes down, it will be because of a critical error that should take the entire node down as well. +Subsystems are essential tasks meant to run as long as the node does. Subsystems can spawn ephemeral work in the form of +jobs, but the subsystems themselves should not go down. If a subsystem goes down, it will be because of a critical error +that should take the entire node down as well. ## Communication Between Subsystems -When a subsystem wants to communicate with another subsystem, or, more typically, a job within a subsystem wants to communicate with its counterpart under another subsystem, that communication must happen via the overseer. Consider this example where a job on subsystem A wants to send a message to its counterpart under subsystem B. This is a realistic scenario, where you can imagine that both jobs correspond to work under the same relay-parent. +When a subsystem wants to communicate with another subsystem, or, more typically, a job within a subsystem wants to +communicate with its counterpart under another subsystem, that communication must happen via the overseer. Consider this +example where a job on subsystem A wants to send a message to its counterpart under subsystem B. This is a realistic +scenario, where you can imagine that both jobs correspond to work under the same relay-parent. ```text +--------+ +--------+ @@ -78,21 +100,48 @@ When a subsystem wants to communicate with another subsystem, or, more typically +------------------------------+ ``` -First, the subsystem that spawned a job is responsible for handling the first step of the communication. The overseer is not aware of the hierarchy of tasks within any given subsystem and is only responsible for subsystem-to-subsystem communication. So the sending subsystem must pass on the message via the overseer to the receiving subsystem, in such a way that the receiving subsystem can further address the communication to one of its internal tasks, if necessary. - -This communication prevents a certain class of race conditions. When the Overseer determines that it is time for subsystems to begin working on top of a particular relay-parent, it will dispatch a `ActiveLeavesUpdate` message to all subsystems to do so, and those messages will be handled asynchronously by those subsystems. Some subsystems will receive those messsages before others, and it is important that a message sent by subsystem A after receiving `ActiveLeavesUpdate` message will arrive at subsystem B after its `ActiveLeavesUpdate` message. If subsystem A maintained an independent channel with subsystem B to communicate, it would be possible for subsystem B to handle the side message before the `ActiveLeavesUpdate` message, but it wouldn't have any logical course of action to take with the side message - leading to it being discarded or improperly handled. Well-architectured state machines should have a single source of inputs, so that is what we do here. - -One exception is reasonable to make for responses to requests. A request should be made via the overseer in order to ensure that it arrives after any relevant `ActiveLeavesUpdate` message. A subsystem issuing a request as a result of a `ActiveLeavesUpdate` message can safely receive the response via a side-channel for two reasons: - -1. It's impossible for a request to be answered before it arrives, it is provable that any response to a request obeys the same ordering constraint. -1. The request was sent as a result of handling a `ActiveLeavesUpdate` message. Then there is no possible future in which the `ActiveLeavesUpdate` message has not been handled upon the receipt of the response. - -So as a single exception to the rule that all communication must happen via the overseer we allow the receipt of responses to requests via a side-channel, which may be established for that purpose. This simplifies any cases where the outside world desires to make a request to a subsystem, as the outside world can then establish a side-channel to receive the response on. - -It's important to note that the overseer is not aware of the internals of subsystems, and this extends to the jobs that they spawn. The overseer isn't aware of the existence or definition of those jobs, and is only aware of the outer subsystems with which it interacts. This gives subsystem implementations leeway to define internal jobs as they see fit, and to wrap a more complex hierarchy of state machines than having a single layer of jobs for relay-parent-based work. Likewise, subsystems aren't required to spawn jobs. Certain types of subsystems, such as those for shared storage or networking resources, won't perform block-based work but would still benefit from being on the Overseer's message bus. These subsystems can just ignore the overseer's signals for block-based work. - -Furthermore, the protocols by which subsystems communicate with each other should be well-defined irrespective of the implementation of the subsystem. In other words, their interface should be distinct from their implementation. This will prevent subsystems from accessing aspects of each other that are beyond the scope of the communication boundary. +First, the subsystem that spawned a job is responsible for handling the first step of the communication. The overseer is +not aware of the hierarchy of tasks within any given subsystem and is only responsible for subsystem-to-subsystem +communication. So the sending subsystem must pass on the message via the overseer to the receiving subsystem, in such a +way that the receiving subsystem can further address the communication to one of its internal tasks, if necessary. + +This communication prevents a certain class of race conditions. When the Overseer determines that it is time for +subsystems to begin working on top of a particular relay-parent, it will dispatch a `ActiveLeavesUpdate` message to all +subsystems to do so, and those messages will be handled asynchronously by those subsystems. Some subsystems will receive +those messsages before others, and it is important that a message sent by subsystem A after receiving +`ActiveLeavesUpdate` message will arrive at subsystem B after its `ActiveLeavesUpdate` message. If subsystem A +maintained an independent channel with subsystem B to communicate, it would be possible for subsystem B to handle the +side message before the `ActiveLeavesUpdate` message, but it wouldn't have any logical course of action to take with the +side message - leading to it being discarded or improperly handled. Well-architectured state machines should have a +single source of inputs, so that is what we do here. + +One exception is reasonable to make for responses to requests. A request should be made via the overseer in order to +ensure that it arrives after any relevant `ActiveLeavesUpdate` message. A subsystem issuing a request as a result of a +`ActiveLeavesUpdate` message can safely receive the response via a side-channel for two reasons: + +1. It's impossible for a request to be answered before it arrives, it is provable that any response to a request obeys + the same ordering constraint. +1. The request was sent as a result of handling a `ActiveLeavesUpdate` message. Then there is no possible future in + which the `ActiveLeavesUpdate` message has not been handled upon the receipt of the response. + +So as a single exception to the rule that all communication must happen via the overseer we allow the receipt of +responses to requests via a side-channel, which may be established for that purpose. This simplifies any cases where the +outside world desires to make a request to a subsystem, as the outside world can then establish a side-channel to +receive the response on. + +It's important to note that the overseer is not aware of the internals of subsystems, and this extends to the jobs that +they spawn. The overseer isn't aware of the existence or definition of those jobs, and is only aware of the outer +subsystems with which it interacts. This gives subsystem implementations leeway to define internal jobs as they see fit, +and to wrap a more complex hierarchy of state machines than having a single layer of jobs for relay-parent-based work. +Likewise, subsystems aren't required to spawn jobs. Certain types of subsystems, such as those for shared storage or +networking resources, won't perform block-based work but would still benefit from being on the Overseer's message bus. +These subsystems can just ignore the overseer's signals for block-based work. + +Furthermore, the protocols by which subsystems communicate with each other should be well-defined irrespective of the +implementation of the subsystem. In other words, their interface should be distinct from their implementation. This will +prevent subsystems from accessing aspects of each other that are beyond the scope of the communication boundary. ## On shutdown -Send an `OverseerSignal::Conclude` message to each subsystem and wait some time for them to conclude before hard-exiting. +Send an `OverseerSignal::Conclude` message to each subsystem and wait some time for them to conclude before +hard-exiting. diff --git a/polkadot/roadmap/implementers-guide/src/node/subsystems-and-jobs.md b/polkadot/roadmap/implementers-guide/src/node/subsystems-and-jobs.md index 6e3b4cd2d166b820a1084fe1e423886d7d6dea38..a3ca7347eb63efedb787c90a23c8e5b95138bcdb 100644 --- a/polkadot/roadmap/implementers-guide/src/node/subsystems-and-jobs.md +++ b/polkadot/roadmap/implementers-guide/src/node/subsystems-and-jobs.md @@ -1,25 +1,66 @@ # Subsystems and Jobs -In this section we define the notions of Subsystems and Jobs. These are guidelines for how we will employ an architecture of hierarchical state machines. We'll have a top-level state machine which oversees the next level of state machines which oversee another layer of state machines and so on. The next sections will lay out these guidelines for what we've called subsystems and jobs, since this model applies to many of the tasks that the Node-side behavior needs to encompass, but these are only guidelines and some Subsystems may have deeper hierarchies internally. - -Subsystems are long-lived worker tasks that are in charge of performing some particular kind of work. All subsystems can communicate with each other via a well-defined protocol. Subsystems can't generally communicate directly, but must coordinate communication through an [Overseer](overseer.md), which is responsible for relaying messages, handling subsystem failures, and dispatching work signals. - -Most work that happens on the Node-side is related to building on top of a specific relay-chain block, which is contextually known as the "relay parent". We call it the relay parent to explicitly denote that it is a block in the relay chain and not on a parachain. We refer to the parent because when we are in the process of building a new block, we don't know what that new block is going to be. The parent block is our only stable point of reference, even though it is usually only useful when it is not yet a parent but in fact a leaf of the block-DAG expected to soon become a parent (because validators are authoring on top of it). Furthermore, we are assuming a forkful blockchain-extension protocol, which means that there may be multiple possible children of the relay-parent. Even if the relay parent has multiple children blocks, the parent of those children is the same, and the context in which those children is authored should be the same. The parent block is the best and most stable reference to use for defining the scope of work items and messages, and is typically referred to by its cryptographic hash. - -Since this goal of determining when to start and conclude work relative to a specific relay-parent is common to most, if not all subsystems, it is logically the job of the Overseer to distribute those signals as opposed to each subsystem duplicating that effort, potentially being out of synchronization with each other. Subsystem A should be able to expect that subsystem B is working on the same relay-parents as it is. One of the Overseer's tasks is to provide this heartbeat, or synchronized rhythm, to the system. - -The work that subsystems spawn to be done on a specific relay-parent is known as a job. Subsystems should set up and tear down jobs according to the signals received from the overseer. Subsystems may share or cache state between jobs. - -Subsystems must be robust to spurious exits. The outputs of the set of subsystems as a whole comprises of signed messages and data committed to disk. Care must be taken to avoid issuing messages that are not substantiated. Since subsystems need to be safe under spurious exits, it is the expected behavior that an `OverseerSignal::Conclude` can just lead to breaking the loop and exiting directly as opposed to waiting for everything to shut down gracefully. +In this section we define the notions of Subsystems and Jobs. These are +guidelines for how we will employ an architecture of hierarchical state +machines. We'll have a top-level state machine which oversees the next level of +state machines which oversee another layer of state machines and so on. The next +sections will lay out these guidelines for what we've called subsystems and +jobs, since this model applies to many of the tasks that the Node-side behavior +needs to encompass, but these are only guidelines and some Subsystems may have +deeper hierarchies internally. + +Subsystems are long-lived worker tasks that are in charge of performing some +particular kind of work. All subsystems can communicate with each other via a +well-defined protocol. Subsystems can't generally communicate directly, but must +coordinate communication through an [Overseer](overseer.md), which is +responsible for relaying messages, handling subsystem failures, and dispatching +work signals. + +Most work that happens on the Node-side is related to building on top of a +specific relay-chain block, which is contextually known as the "relay parent". +We call it the relay parent to explicitly denote that it is a block in the relay +chain and not on a parachain. We refer to the parent because when we are in the +process of building a new block, we don't know what that new block is going to +be. The parent block is our only stable point of reference, even though it is +usually only useful when it is not yet a parent but in fact a leaf of the +block-DAG expected to soon become a parent (because validators are authoring on +top of it). Furthermore, we are assuming a forkful blockchain-extension +protocol, which means that there may be multiple possible children of the +relay-parent. Even if the relay parent has multiple children blocks, the parent +of those children is the same, and the context in which those children is +authored should be the same. The parent block is the best and most stable +reference to use for defining the scope of work items and messages, and is +typically referred to by its cryptographic hash. + +Since this goal of determining when to start and conclude work relative to a +specific relay-parent is common to most, if not all subsystems, it is logically +the job of the Overseer to distribute those signals as opposed to each subsystem +duplicating that effort, potentially being out of synchronization with each +other. Subsystem A should be able to expect that subsystem B is working on the +same relay-parents as it is. One of the Overseer's tasks is to provide this +heartbeat, or synchronized rhythm, to the system. + +The work that subsystems spawn to be done on a specific relay-parent is known as +a job. Subsystems should set up and tear down jobs according to the signals +received from the overseer. Subsystems may share or cache state between jobs. + +Subsystems must be robust to spurious exits. The outputs of the set of +subsystems as a whole comprises of signed messages and data committed to disk. +Care must be taken to avoid issuing messages that are not substantiated. Since +subsystems need to be safe under spurious exits, it is the expected behavior +that an `OverseerSignal::Conclude` can just lead to breaking the loop and +exiting directly as opposed to waiting for everything to shut down gracefully. ## Subsystem Message Traffic Which subsystems send messages to which other subsystems. -**Note**: This diagram omits the overseer for simplicity. In fact, all messages are relayed via the overseer. +**Note**: This diagram omits the overseer for simplicity. In fact, all messages +are relayed via the overseer. -**Note**: Messages with a filled diamond arrowhead ("♦") include a `oneshot::Sender` which communicates a response from the recipient. -Messages with an open triangle arrowhead ("Δ") do not include a return sender. +**Note**: Messages with a filled diamond arrowhead ("♦") include a +`oneshot::Sender` which communicates a response from the recipient. Messages +with an open triangle arrowhead ("Δ") do not include a return sender. ```dot process digraph { @@ -125,14 +166,17 @@ digraph { ## The Path to Inclusion (Node Side) -Let's contextualize that diagram a bit by following a parachain block from its creation through finalization. -Parachains can use completely arbitrary processes to generate blocks. The relay chain doesn't know or care about -the details; each parachain just needs to provide a [collator](collators/collation-generation.md). +Let's contextualize that diagram a bit by following a parachain block from its +creation through finalization. Parachains can use completely arbitrary processes +to generate blocks. The relay chain doesn't know or care about the details; each +parachain just needs to provide a [collator](collators/collation-generation.md). -**Note**: Inter-subsystem communications are relayed via the overseer, but that step is omitted here for brevity. +**Note**: Inter-subsystem communications are relayed via the overseer, but that +step is omitted here for brevity. -**Note**: Dashed lines indicate a request/response cycle, where the response is communicated asynchronously via -a oneshot channel. Adjacent dashed lines may be processed in parallel. +**Note**: Dashed lines indicate a request/response cycle, where the response is +communicated asynchronously via a oneshot channel. Adjacent dashed lines may be +processed in parallel. ```mermaid sequenceDiagram @@ -156,11 +200,13 @@ sequenceDiagram end ``` -The `DistributeCollation` messages that `CollationGeneration` sends to the `CollatorProtocol` contains -two items: a `CandidateReceipt` and `PoV`. The `CollatorProtocol` is then responsible for distributing -that collation to interested validators. However, not all potential collations are of interest. The -`CandidateSelection` subsystem is responsible for determining which collations are interesting, before -`CollatorProtocol` actually fetches the collation. +The `DistributeCollation` messages that `CollationGeneration` sends to the +`CollatorProtocol` contains two items: a `CandidateReceipt` and `PoV`. The +`CollatorProtocol` is then responsible for distributing that collation to +interested validators. However, not all potential collations are of interest. +The `CandidateSelection` subsystem is responsible for determining which +collations are interesting, before `CollatorProtocol` actually fetches the +collation. ```mermaid sequenceDiagram @@ -205,10 +251,11 @@ sequenceDiagram end ``` -Assuming we hit the happy path, flow continues with `CandidateSelection` receiving a `(candidate_receipt, pov)` as -the return value from its -`FetchCollation` request. The only time `CandidateSelection` actively requests a collation is when -it hasn't yet seconded one for some `relay_parent`, and is ready to second. +Assuming we hit the happy path, flow continues with `CandidateSelection` +receiving a `(candidate_receipt, pov)` as the return value from its +`FetchCollation` request. The only time `CandidateSelection` actively requests a +collation is when it hasn't yet seconded one for some `relay_parent`, and is +ready to second. ```mermaid sequenceDiagram @@ -243,15 +290,17 @@ sequenceDiagram end ``` -At this point, you'll see that control flows in two directions: to `StatementDistribution` to distribute -the `SignedStatement`, and to `PoVDistribution` to distribute the `PoV`. However, that's largely a mirage: -while the initial implementation distributes `PoV`s by gossip, that's inefficient, and will be replaced -with a system which fetches `PoV`s only when actually necessary. +At this point, you'll see that control flows in two directions: to +`StatementDistribution` to distribute the `SignedStatement`, and to +`PoVDistribution` to distribute the `PoV`. However, that's largely a mirage: +while the initial implementation distributes `PoV`s by gossip, that's +inefficient, and will be replaced with a system which fetches `PoV`s only when +actually necessary. > TODO: figure out more precisely the current status and plans; write them up -Therefore, we'll follow the `SignedStatement`. The `StatementDistribution` subsystem is largely concerned -with implementing a gossip protocol: +Therefore, we'll follow the `SignedStatement`. The `StatementDistribution` +subsystem is largely concerned with implementing a gossip protocol: ```mermaid sequenceDiagram @@ -278,8 +327,8 @@ sequenceDiagram end ``` -But who are these `Listener`s who've asked to be notified about incoming `SignedStatement`s? -Nobody, as yet. +But who are these `Listener`s who've asked to be notified about incoming +`SignedStatement`s? Nobody, as yet. Let's pick back up with the PoV Distribution subsystem. @@ -305,11 +354,13 @@ sequenceDiagram Note over PD,NB: On receipt of a network PoV, PovDistribution forwards it to each Listener.
It also penalizes bad gossipers. ``` -Unlike in the case of `StatementDistribution`, there is another subsystem which in various circumstances -already registers a listener to be notified when a new `PoV` arrives: `CandidateBacking`. Note that this -is the second time that `CandidateBacking` has gotten involved. The first instance was from the perspective -of the validator choosing to second a candidate via its `CandidateSelection` subsystem. This time, it's -from the perspective of some other validator, being informed that this foreign `PoV` has been received. +Unlike in the case of `StatementDistribution`, there is another subsystem which +in various circumstances already registers a listener to be notified when a new +`PoV` arrives: `CandidateBacking`. Note that this is the second time that +`CandidateBacking` has gotten involved. The first instance was from the +perspective of the validator choosing to second a candidate via its +`CandidateSelection` subsystem. This time, it's from the perspective of some +other validator, being informed that this foreign `PoV` has been received. ```mermaid sequenceDiagram @@ -326,10 +377,11 @@ sequenceDiagram CB ->> AS: StoreAvailableData ``` -At this point, things have gone a bit nonlinear. Let's pick up the thread again with `BitfieldSigning`. As -the `Overseer` activates each relay parent, it starts a `BitfieldSigningJob` which operates on an extremely -simple metric: after creation, it immediately goes to sleep for 1.5 seconds. On waking, it records the state -of the world pertaining to availability at that moment. +At this point, things have gone a bit nonlinear. Let's pick up the thread again +with `BitfieldSigning`. As the `Overseer` activates each relay parent, it starts +a `BitfieldSigningJob` which operates on an extremely simple metric: after +creation, it immediately goes to sleep for 1.5 seconds. On waking, it records +the state of the world pertaining to availability at that moment. ```mermaid sequenceDiagram @@ -350,9 +402,10 @@ sequenceDiagram end ``` -`BitfieldDistribution` is, like the other `*Distribution` subsystems, primarily interested in implementing -a peer-to-peer gossip network propagating its particular messages. However, it also serves as an essential -relay passing the message along. +`BitfieldDistribution` is, like the other `*Distribution` subsystems, primarily +interested in implementing a peer-to-peer gossip network propagating its +particular messages. However, it also serves as an essential relay passing the +message along. ```mermaid sequenceDiagram @@ -366,12 +419,14 @@ sequenceDiagram BD ->> NB: SendValidationMessage::BitfieldDistribution::Bitfield ``` -We've now seen the message flow to the `Provisioner`: both `CandidateBacking` and `BitfieldDistribution` -contribute provisionable data. Now, let's look at that subsystem. +We've now seen the message flow to the `Provisioner`: both `CandidateBacking` +and `BitfieldDistribution` contribute provisionable data. Now, let's look at +that subsystem. -Much like the `BitfieldSigning` subsystem, the `Provisioner` creates a new job for each newly-activated -leaf, and starts a timer. Unlike `BitfieldSigning`, we won't depict that part of the process, because -the `Provisioner` also has other things going on. +Much like the `BitfieldSigning` subsystem, the `Provisioner` creates a new job +for each newly-activated leaf, and starts a timer. Unlike `BitfieldSigning`, we +won't depict that part of the process, because the `Provisioner` also has other +things going on. ```mermaid sequenceDiagram @@ -411,8 +466,9 @@ sequenceDiagram end ``` -In principle, any arbitrary subsystem could send a `RequestInherentData` to the `Provisioner`. In practice, -only the `ParachainsInherentDataProvider` does so. +In principle, any arbitrary subsystem could send a `RequestInherentData` to the +`Provisioner`. In practice, only the `ParachainsInherentDataProvider` does so. -The tuple `(SignedAvailabilityBitfields, BackedCandidates, ParentHeader)` is injected by the `ParachainsInherentDataProvider` -into the inherent data. From that point on, control passes from the node to the runtime. +The tuple `(SignedAvailabilityBitfields, BackedCandidates, ParentHeader)` is +injected by the `ParachainsInherentDataProvider` into the inherent data. From +that point on, control passes from the node to the runtime. diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/availability-store.md b/polkadot/roadmap/implementers-guide/src/node/utility/availability-store.md index bd61455934e43080502382c54b2c3c6b5afb85ce..71d3f32452314dec6c4687a5081b27d00a77eb83 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/availability-store.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/availability-store.md @@ -9,13 +9,20 @@ The two data types: For each of these data we have pruning rules that determine how long we need to keep that data available. -PoV hypothetically only need to be kept around until the block where the data was made fully available is finalized. However, disputes can revert finality, so we need to be a bit more conservative and we add a delay. We should keep the PoV until a block that finalized availability of it has been finalized for 1 day + 1 hour. +PoV hypothetically only need to be kept around until the block where the data was made fully available is finalized. +However, disputes can revert finality, so we need to be a bit more conservative and we add a delay. We should keep the +PoV until a block that finalized availability of it has been finalized for 1 day + 1 hour. -Availability chunks need to be kept available until the dispute period for the corresponding candidate has ended. We can accomplish this by using the same criterion as the above. This gives us a pruning condition of the block finalizing availability of the chunk being final for 1 day + 1 hour. +Availability chunks need to be kept available until the dispute period for the corresponding candidate has ended. We can +accomplish this by using the same criterion as the above. This gives us a pruning condition of the block finalizing +availability of the chunk being final for 1 day + 1 hour. -There is also the case where a validator commits to make a PoV available, but the corresponding candidate is never backed. In this case, we keep the PoV available for 1 hour. +There is also the case where a validator commits to make a PoV available, but the corresponding candidate is never +backed. In this case, we keep the PoV available for 1 hour. -There may be multiple competing blocks all ending the availability phase for a particular candidate. Until finality, it will be unclear which of those is actually the canonical chain, so the pruning records for PoVs and Availability chunks should keep track of all such blocks. +There may be multiple competing blocks all ending the availability phase for a particular candidate. Until finality, it +will be unclear which of those is actually the canonical chain, so the pruning records for PoVs and Availability chunks +should keep track of all such blocks. ## Lifetime of the block data and chunks in storage @@ -44,7 +51,8 @@ We use an underlying Key-Value database where we assume we have the following op - `write(key, value)` - `read(key) -> Option` -- `iter_with_prefix(prefix) -> Iterator<(key, value)>` - gives all keys and values in lexicographical order where the key starts with `prefix`. +- `iter_with_prefix(prefix) -> Iterator<(key, value)>` - gives all keys and values in lexicographical order where the + key starts with `prefix`. We use this database to encode the following schema: @@ -57,7 +65,8 @@ We use this database to encode the following schema: ("prune_by_time", Timestamp, CandidateHash) -> Option<()> ``` -Timestamps are the wall-clock seconds since Unix epoch. Timestamps and block numbers are both encoded as big-endian so lexicographic order is ascending. +Timestamps are the wall-clock seconds since Unix epoch. Timestamps and block numbers are both encoded as big-endian so +lexicographic order is ascending. The meta information that we track per-candidate is defined as the `CandidateMeta` struct @@ -80,9 +89,12 @@ enum State { } ``` -We maintain the invariant that if a candidate has a meta entry, its available data exists on disk if `data_available` is true. All chunks mentioned in the meta entry are available. +We maintain the invariant that if a candidate has a meta entry, its available data exists on disk if `data_available` is +true. All chunks mentioned in the meta entry are available. -Additionally, there is exactly one `prune_by_time` entry which holds the candidate hash unless the state is `Unfinalized`. There may be zero, one, or many "unfinalized" keys with the given candidate, and this will correspond to the `state` of the meta entry. +Additionally, there is exactly one `prune_by_time` entry which holds the candidate hash unless the state is +`Unfinalized`. There may be zero, one, or many "unfinalized" keys with the given candidate, and this will correspond to +the `state` of the meta entry. ## Protocol @@ -96,9 +108,15 @@ Output: For each head in the `activated` list: -- Load all ancestors of the head back to the finalized block so we don't miss anything if import notifications are missed. If a `StoreChunk` message is received for a candidate which has no entry, then we will prematurely lose the data. -- Note any new candidates backed in the head. Update the `CandidateMeta` for each. If the `CandidateMeta` does not exist, create it as `Unavailable` with the current timestamp. Register a `"prune_by_time"` entry based on the current timestamp + 1 hour. -- Note any new candidate included in the head. Update the `CandidateMeta` for each, performing a transition from `Unavailable` to `Unfinalized` if necessary. That includes removing the `"prune_by_time"` entry. Add the head hash and number to the state, if unfinalized. Add an `"unfinalized"` entry for the block and candidate. +- Load all ancestors of the head back to the finalized block so we don't miss anything if import notifications are + missed. If a `StoreChunk` message is received for a candidate which has no entry, then we will prematurely lose the + data. +- Note any new candidates backed in the head. Update the `CandidateMeta` for each. If the `CandidateMeta` does not + exist, create it as `Unavailable` with the current timestamp. Register a `"prune_by_time"` entry based on the current + timestamp + 1 hour. +- Note any new candidate included in the head. Update the `CandidateMeta` for each, performing a transition from + `Unavailable` to `Unfinalized` if necessary. That includes removing the `"prune_by_time"` entry. Add the head hash and + number to the state, if unfinalized. Add an `"unfinalized"` entry for the block and candidate. - The `CandidateEvent` runtime API can be used for this purpose. On `OverseerSignal::BlockFinalized(finalized)` events: @@ -106,17 +124,22 @@ On `OverseerSignal::BlockFinalized(finalized)` events: - for each key in `iter_with_prefix("unfinalized")` - Stop if the key is beyond `("unfinalized, finalized)` - For each block number f that we encounter, load the finalized hash for that block. - - The state of each `CandidateMeta` we encounter here must be `Unfinalized`, since we loaded the candidate from an `"unfinalized"` key. + - The state of each `CandidateMeta` we encounter here must be `Unfinalized`, since we loaded the candidate from an + `"unfinalized"` key. - For each candidate that we encounter under `f` and the finalized block hash, - - Update the `CandidateMeta` to have `State::Finalized`. Remove all `"unfinalized"` entries from the old `Unfinalized` state. + - Update the `CandidateMeta` to have `State::Finalized`. Remove all `"unfinalized"` entries from the old + `Unfinalized` state. - Register a `"prune_by_time"` entry for the candidate based on the current time + 1 day + 1 hour. - For each candidate that we encounter under `f` which is not under the finalized block hash, - Remove all entries under `f` in the `Unfinalized` state. - - If the `CandidateMeta` has state `Unfinalized` with an empty list of blocks, downgrade to `Unavailable` and re-schedule pruning under the timestamp + 1 hour. We do not prune here as the candidate still may be included in a descendant of the finalized chain. + - If the `CandidateMeta` has state `Unfinalized` with an empty list of blocks, downgrade to `Unavailable` and + re-schedule pruning under the timestamp + 1 hour. We do not prune here as the candidate still may be included in + a descendant of the finalized chain. - Remove all `"unfinalized"` keys under `f`. - Update `last_finalized` = finalized. - This is roughly `O(n * m)` where n is the number of blocks finalized since the last update, and `m` is the number of parachains. + This is roughly `O(n * m)` where n is the number of blocks finalized since the last update, and `m` is the number of + parachains. On `QueryAvailableData` message: @@ -139,7 +162,8 @@ On `QueryChunk` message: On `QueryAllChunks` message: - Query `("meta", candidate_hash)`. If `None`, send an empty response and return. -- For all `1` bits in the `chunks_stored`, query `("chunk", candidate_hash, index)`. Ignore but warn on errors, and return a vector of all loaded chunks. +- For all `1` bits in the `chunks_stored`, query `("chunk", candidate_hash, index)`. Ignore but warn on errors, and + return a vector of all loaded chunks. On `QueryChunkAvailability` message: @@ -149,14 +173,17 @@ On `QueryChunkAvailability` message: On `StoreChunk` message: -- If there is a `CandidateMeta` under the candidate hash, set the bit of the erasure-chunk in the `chunks_stored` bitfield to `1`. If it was not `1` already, write the chunk under `("chunk", candidate_hash, chunk_index)`. +- If there is a `CandidateMeta` under the candidate hash, set the bit of the erasure-chunk in the `chunks_stored` + bitfield to `1`. If it was not `1` already, write the chunk under `("chunk", candidate_hash, chunk_index)`. This is `O(n)` in the size of the chunk. On `StoreAvailableData` message: -- Compute the erasure root of the available data and compare it with `expected_erasure_root`. Return `StoreAvailableDataError::InvalidErasureRoot` on mismatch. -- If there is no `CandidateMeta` under the candidate hash, create it with `State::Unavailable(now)`. Load the `CandidateMeta` otherwise. +- Compute the erasure root of the available data and compare it with `expected_erasure_root`. Return + `StoreAvailableDataError::InvalidErasureRoot` on mismatch. +- If there is no `CandidateMeta` under the candidate hash, create it with `State::Unavailable(now)`. Load the + `CandidateMeta` otherwise. - Store `data` under `("available", candidate_hash)` and set `data_available` to true. - Store each chunk under `("chunk", candidate_hash, index)` and set every bit in `chunks_stored` to `1`. @@ -172,12 +199,13 @@ Every 5 minutes, run a pruning routine: - For each erasure chunk bit set, remove `("chunk", candidate_hash, bit_index)`. - If `data_available`, remove `("available", candidate_hash)` - This is O(n * m) in the amount of candidates and average size of the data stored. This is probably the most expensive operation but does not need - to be run very often. + This is O(n * m) in the amount of candidates and average size of the data stored. This is probably the most expensive + operation but does not need to be run very often. ## Basic scenarios to test -Basically we need to test the correctness of data flow through state FSMs described earlier. These tests obviously assume that some mocking of time is happening. +Basically we need to test the correctness of data flow through state FSMs described earlier. These tests obviously +assume that some mocking of time is happening. - Stored data that is never included pruned in necessary timeout - A block (and/or a chunk) is added to the store. diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md b/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md index 4a1d02be556091b2e223996493bffadb7fd43476..376fa187de1bf4b18849ba8c8b134352fbe21630 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/candidate-validation.md @@ -2,7 +2,8 @@ This subsystem is responsible for handling candidate validation requests. It is a simple request/response server. -A variety of subsystems want to know if a parachain block candidate is valid. None of them care about the detailed mechanics of how a candidate gets validated, just the results. This subsystem handles those details. +A variety of subsystems want to know if a parachain block candidate is valid. None of them care about the detailed +mechanics of how a candidate gets validated, just the results. This subsystem handles those details. ## Protocol @@ -12,35 +13,53 @@ Output: Validation result via the provided response side-channel. ## Functionality -This subsystem groups the requests it handles in two categories: *candidate validation* and *PVF pre-checking*. +This subsystem groups the requests it handles in two categories: *candidate validation* and *PVF pre-checking*. -The first category can be further subdivided in two request types: one which draws out validation data from the state, and another which accepts all validation data exhaustively. Validation returns three possible outcomes on the response channel: the candidate is valid, the candidate is invalid, or an internal error occurred. +The first category can be further subdivided in two request types: one which draws out validation data from the state, +and another which accepts all validation data exhaustively. Validation returns three possible outcomes on the response +channel: the candidate is valid, the candidate is invalid, or an internal error occurred. -Parachain candidates are validated against their validation function: A piece of Wasm code that describes the state-transition of the parachain. Validation function execution is not metered. This means that an execution which is an infinite loop or simply takes too long must be forcibly exited by some other means. For this reason, we recommend dispatching candidate validation to be done on subprocesses which can be killed if they time-out. +Parachain candidates are validated against their validation function: A piece of Wasm code that describes the +state-transition of the parachain. Validation function execution is not metered. This means that an execution which is +an infinite loop or simply takes too long must be forcibly exited by some other means. For this reason, we recommend +dispatching candidate validation to be done on subprocesses which can be killed if they time-out. -Upon receiving a validation request, the first thing the candidate validation subsystem should do is make sure it has all the necessary parameters to the validation function. These are: +Upon receiving a validation request, the first thing the candidate validation subsystem should do is make sure it has +all the necessary parameters to the validation function. These are: * The Validation Function itself. * The [`CandidateDescriptor`](../../types/candidate.md#candidatedescriptor). * The [`ValidationData`](../../types/candidate.md#validationdata). * The [`PoV`](../../types/availability.md#proofofvalidity). -The second category is for PVF pre-checking. This is primarly used by the [PVF pre-checker](pvf-prechecker.md) subsystem. +The second category is for PVF pre-checking. This is primarly used by the [PVF pre-checker](pvf-prechecker.md) +subsystem. ### Determining Parameters For a [`CandidateValidationMessage`][CVM]`::ValidateFromExhaustive`, these parameters are exhaustively provided. -For a [`CandidateValidationMessage`][CVM]`::ValidateFromChainState`, some more work needs to be done. Due to the uncertainty of Availability Cores (implemented in the [`Scheduler`](../../runtime/scheduler.md) module of the runtime), a candidate at a particular relay-parent and for a particular para may have two different valid validation-data to be executed under depending on what is assumed to happen if the para is occupying a core at the onset of the new block. This is encoded as an `OccupiedCoreAssumption` in the runtime API. +For a [`CandidateValidationMessage`][CVM]`::ValidateFromChainState`, some more work needs to be done. Due to the +uncertainty of Availability Cores (implemented in the [`Scheduler`](../../runtime/scheduler.md) module of the runtime), +a candidate at a particular relay-parent and for a particular para may have two different valid validation-data to be +executed under depending on what is assumed to happen if the para is occupying a core at the onset of the new block. +This is encoded as an `OccupiedCoreAssumption` in the runtime API. -The way that we can determine which assumption the candidate is meant to be executed under is simply to do an exhaustive check of both possibilities based on the state of the relay-parent. First we fetch the validation data under the assumption that the block occupying becomes available. If the `validation_data_hash` of the `CandidateDescriptor` matches this validation data, we use that. Otherwise, if the `validation_data_hash` matches the validation data fetched under the `TimedOut` assumption, we use that. Otherwise, we return a `ValidationResult::Invalid` response and conclude. +The way that we can determine which assumption the candidate is meant to be executed under is simply to do an exhaustive +check of both possibilities based on the state of the relay-parent. First we fetch the validation data under the +assumption that the block occupying becomes available. If the `validation_data_hash` of the `CandidateDescriptor` +matches this validation data, we use that. Otherwise, if the `validation_data_hash` matches the validation data fetched +under the `TimedOut` assumption, we use that. Otherwise, we return a `ValidationResult::Invalid` response and conclude. -Then, we can fetch the validation code from the runtime based on which type of candidate this is. This gives us all the parameters. The descriptor and PoV come from the request itself, and the other parameters have been derived from the state. +Then, we can fetch the validation code from the runtime based on which type of candidate this is. This gives us all the +parameters. The descriptor and PoV come from the request itself, and the other parameters have been derived from the +state. > TODO: This would be a great place for caching to avoid making lots of runtime requests. That would need a job, though. ### Execution of the Parachain Wasm -Once we have all parameters, we can spin up a background task to perform the validation in a way that doesn't hold up the entire event loop. Before invoking the validation function itself, this should first do some basic checks: +Once we have all parameters, we can spin up a background task to perform the validation in a way that doesn't hold up +the entire event loop. Before invoking the validation function itself, this should first do some basic checks: * The collator signature is valid * The PoV provided matches the `pov_hash` field of the descriptor @@ -48,6 +67,8 @@ For more details please see [PVF Host and Workers](pvf-host-and-workers.md). ### Checking Validation Outputs -If we can assume the presence of the relay-chain state (that is, during processing [`CandidateValidationMessage`][CVM]`::ValidateFromChainState`) we can run all the checks that the relay-chain would run at the inclusion time thus confirming that the candidate will be accepted. +If we can assume the presence of the relay-chain state (that is, during processing +[`CandidateValidationMessage`][CVM]`::ValidateFromChainState`) we can run all the checks that the relay-chain would run +at the inclusion time thus confirming that the candidate will be accepted. [CVM]: ../../types/overseer-protocol.md#validationrequesttype diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/chain-api.md b/polkadot/roadmap/implementers-guide/src/node/utility/chain-api.md index e9ef9b5695bc386ca9f0bc493b151fd6f0dff892..aab4f2d5cb5625d4a5399217ca979b9d6a6670aa 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/chain-api.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/chain-api.md @@ -1,6 +1,7 @@ # Chain API -The Chain API subsystem is responsible for providing a single point of access to chain state data via a set of pre-determined queries. +The Chain API subsystem is responsible for providing a single point of access to chain state data via a set of +pre-determined queries. ## Protocol @@ -10,7 +11,8 @@ Output: None ## Functionality -On receipt of `ChainApiMessage`, answer the request and provide the response to the side-channel embedded within the request. +On receipt of `ChainApiMessage`, answer the request and provide the response to the side-channel embedded within the +request. Currently, the following requests are supported: * Block hash to number diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/chain-selection.md b/polkadot/roadmap/implementers-guide/src/node/utility/chain-selection.md index 640691e559615b3f703535aa82658a3c6c9693d9..1d1358b228b61295e7b41d4256b5fa387ebd9af2 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/chain-selection.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/chain-selection.md @@ -1,8 +1,12 @@ # Chain Selection Subsystem -This subsystem implements the necessary metadata for the implementation of the [chain selection](../../protocol-chain-selection.md) portion of the protocol. +This subsystem implements the necessary metadata for the implementation of the [chain +selection](../../protocol-chain-selection.md) portion of the protocol. -The subsystem wraps a database component which maintains a view of the unfinalized chain and records the properties of each block: whether the block is **viable**, whether it is **stagnant**, and whether it is **reverted**. It should also maintain an updated set of active leaves in accordance with this view, which should be cheap to query. Leaves are ordered descending first by weight and then by block number. +The subsystem wraps a database component which maintains a view of the unfinalized chain and records the properties of +each block: whether the block is **viable**, whether it is **stagnant**, and whether it is **reverted**. It should also +maintain an updated set of active leaves in accordance with this view, which should be cheap to query. Leaves are +ordered descending first by weight and then by block number. This subsystem needs to update its information on the unfinalized chain: * On every leaf-activated signal @@ -11,32 +15,47 @@ This subsystem needs to update its information on the unfinalized chain: * On every `ChainSelectionMessage::RevertBlocks` * Periodically, to detect stagnation. -Simple implementations of these updates do `O(n_unfinalized_blocks)` disk operations. If the amount of unfinalized blocks is relatively small, the updates should not take very much time. However, in cases where there are hundreds or thousands of unfinalized blocks the naive implementations of these update algorithms would have to be replaced with more sophisticated versions. +Simple implementations of these updates do `O(n_unfinalized_blocks)` disk operations. If the amount of unfinalized +blocks is relatively small, the updates should not take very much time. However, in cases where there are hundreds or +thousands of unfinalized blocks the naive implementations of these update algorithms would have to be replaced with more +sophisticated versions. -### `OverseerSignal::ActiveLeavesUpdate` +## `OverseerSignal::ActiveLeavesUpdate` -Determine all new blocks implicitly referenced by any new active leaves and add them to the view. Update the set of viable leaves accordingly. The weights of imported blocks can be determined by the [`ChainApiMessage::BlockWeight`](../../types/overseer-protocol.md#chain-api-message). +Determine all new blocks implicitly referenced by any new active leaves and add them to the view. Update the set of +viable leaves accordingly. The weights of imported blocks can be determined by the +[`ChainApiMessage::BlockWeight`](../../types/overseer-protocol.md#chain-api-message). -### `OverseerSignal::BlockFinalized` +## `OverseerSignal::BlockFinalized` -Delete data for all orphaned chains and update all metadata descending from the new finalized block accordingly, along with the set of viable leaves. Note that finalizing a **reverted** or **stagnant** block means that the descendants of those blocks may lose that status because the definitions of those properties don't include the finalized chain. Update the set of viable leaves accordingly. +Delete data for all orphaned chains and update all metadata descending from the new finalized block accordingly, along +with the set of viable leaves. Note that finalizing a **reverted** or **stagnant** block means that the descendants of +those blocks may lose that status because the definitions of those properties don't include the finalized chain. Update +the set of viable leaves accordingly. -### `ChainSelectionMessage::Approved` +## `ChainSelectionMessage::Approved` -Update the approval status of the referenced block. If the block was stagnant and thus non-viable and is now viable, then the metadata of all of its descendants needs to be updated as well, as they may no longer be stagnant either. Update the set of viable leaves accordingly. +Update the approval status of the referenced block. If the block was stagnant and thus non-viable and is now viable, +then the metadata of all of its descendants needs to be updated as well, as they may no longer be stagnant either. +Update the set of viable leaves accordingly. -### `ChainSelectionMessage::Leaves` +## `ChainSelectionMessage::Leaves` -Gets all leaves of the chain, i.e. block hashes that are suitable to build upon and have no suitable children. Supplies the leaves in descending order by score. +Gets all leaves of the chain, i.e. block hashes that are suitable to build upon and have no suitable children. Supplies +the leaves in descending order by score. -### `ChainSelectionMessage::BestLeafContaining` +## `ChainSelectionMessage::BestLeafContaining` -If the required block is unknown or not viable, then return `None`. Iterate over all leaves in order of descending weight, returning the first leaf containing the required block in its chain, and `None` otherwise. +If the required block is unknown or not viable, then return `None`. Iterate over all leaves in order of descending +weight, returning the first leaf containing the required block in its chain, and `None` otherwise. -### `ChainSelectionMessage::RevertBlocks` -This message indicates that a dispute has concluded against a parachain block candidate. The message passes along a vector containing the block number and block hash of each block where the disputed candidate was included. The passed blocks will be marked as reverted, and their descendants will be marked as non-viable. +## `ChainSelectionMessage::RevertBlocks` +This message indicates that a dispute has concluded against a parachain block candidate. The message passes along a +vector containing the block number and block hash of each block where the disputed candidate was included. The passed +blocks will be marked as reverted, and their descendants will be marked as non-viable. -### Periodically +## Periodically -Detect stagnant blocks and apply the stagnant definition to all descendants. Update the set of viable leaves accordingly. +Detect stagnant blocks and apply the stagnant definition to all descendants. Update the set of viable leaves +accordingly. diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/network-bridge.md b/polkadot/roadmap/implementers-guide/src/node/utility/network-bridge.md index 3245772d9d8d180fef6f7e8b7374ea03c34b72c3..d2506fbe83e3ff15e786fddafa83c28bc7b63ac2 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/network-bridge.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/network-bridge.md @@ -1,30 +1,43 @@ # Network Bridge -One of the main features of the overseer/subsystem duality is to avoid shared ownership of resources and to communicate via message-passing. However, implementing each networking subsystem as its own network protocol brings a fair share of challenges. - -The most notable challenge is coordinating and eliminating race conditions of peer connection and disconnection events. If we have many network protocols that peers are supposed to be connected on, it is difficult to enforce that a peer is indeed connected on all of them or the order in which those protocols receive notifications that peers have connected. This becomes especially difficult when attempting to share peer state across protocols. All of the Parachain-Host's gossip protocols eliminate DoS with a data-dependency on current chain heads. However, it is inefficient and confusing to implement the logic for tracking our current chain heads as well as our peers' on each of those subsystems. Having one subsystem for tracking this shared state and distributing it to the others is an improvement in architecture and efficiency. - -One other piece of shared state to track is peer reputation. When peers are found to have provided value or cost, we adjust their reputation accordingly. - -So in short, this Subsystem acts as a bridge between an actual network component and a subsystem's protocol. The implementation of the underlying network component is beyond the scope of this module. We make certain assumptions about the network component: - * The network allows registering of protocols and multiple versions of each protocol. - * The network handles version negotiation of protocols with peers and only connects the peer on the highest version of the protocol. - * Each protocol has its own peer-set, although there may be some overlap. - * The network provides peer-set management utilities for discovering the peer-IDs of validators and a means of dialing peers with given IDs. - - -The network bridge makes use of the peer-set feature, but is not generic over peer-set. Instead, it exposes two peer-sets that event producers can attach to: `Validation` and `Collation`. More information can be found on the documentation of the [`NetworkBridgeMessage`][NBM]. +One of the main features of the overseer/subsystem duality is to avoid shared ownership of resources and to communicate +via message-passing. However, implementing each networking subsystem as its own network protocol brings a fair share of +challenges. + +The most notable challenge is coordinating and eliminating race conditions of peer connection and disconnection events. +If we have many network protocols that peers are supposed to be connected on, it is difficult to enforce that a peer is +indeed connected on all of them or the order in which those protocols receive notifications that peers have connected. +This becomes especially difficult when attempting to share peer state across protocols. All of the Parachain-Host's +gossip protocols eliminate DoS with a data-dependency on current chain heads. However, it is inefficient and confusing +to implement the logic for tracking our current chain heads as well as our peers' on each of those subsystems. Having +one subsystem for tracking this shared state and distributing it to the others is an improvement in architecture and +efficiency. + +One other piece of shared state to track is peer reputation. When peers are found to have provided value or cost, we +adjust their reputation accordingly. + +So in short, this Subsystem acts as a bridge between an actual network component and a subsystem's protocol. The +implementation of the underlying network component is beyond the scope of this module. We make certain assumptions about +the network component: + - The network allows registering of protocols and multiple versions of each protocol. + - The network handles version negotiation of protocols with peers and only connects the peer on the highest version of + the protocol. + - Each protocol has its own peer-set, although there may be some overlap. + - The network provides peer-set management utilities for discovering the peer-IDs of validators and a means of dialing + peers with given IDs. + +The network bridge makes use of the peer-set feature, but is not generic over peer-set. Instead, it exposes two +peer-sets that event producers can attach to: `Validation` and `Collation`. More information can be found on the +documentation of the [`NetworkBridgeMessage`][NBM]. ## Protocol Input: [`NetworkBridgeMessage`][NBM] - -Output: - - [`ApprovalDistributionMessage`][AppD]`::NetworkBridgeUpdate` - - [`BitfieldDistributionMessage`][BitD]`::NetworkBridgeUpdate` - - [`CollatorProtocolMessage`][CollP]`::NetworkBridgeUpdate` - - [`StatementDistributionMessage`][StmtD]`::NetworkBridgeUpdate` +Output: - [`ApprovalDistributionMessage`][AppD]`::NetworkBridgeUpdate` - + [`BitfieldDistributionMessage`][BitD]`::NetworkBridgeUpdate` - + [`CollatorProtocolMessage`][CollP]`::NetworkBridgeUpdate` - + [`StatementDistributionMessage`][StmtD]`::NetworkBridgeUpdate` ## Functionality @@ -37,7 +50,8 @@ enum WireMessage { } ``` -and instantiates this type twice, once using the [`ValidationProtocolV1`][VP1] message type, and once with the [`CollationProtocolV1`][CP1] message type. +and instantiates this type twice, once using the [`ValidationProtocolV1`][VP1] message type, and once with the +[`CollationProtocolV1`][CP1] message type. ```rust type ValidationV1Message = WireMessage; @@ -46,17 +60,21 @@ type CollationV1Message = WireMessage; ### Startup -On startup, we register two protocols with the underlying network utility. One for validation and one for collation. We register only version 1 of each of these protocols. +On startup, we register two protocols with the underlying network utility. One for validation and one for collation. We +register only version 1 of each of these protocols. ### Main Loop -The bulk of the work done by this subsystem is in responding to network events, signals from the overseer, and messages from other subsystems. +The bulk of the work done by this subsystem is in responding to network events, signals from the overseer, and messages +from other subsystems. Each network event is associated with a particular peer-set. ### Overseer Signal: `ActiveLeavesUpdate` -The `activated` and `deactivated` lists determine the evolution of our local view over time. A `ProtocolMessage::ViewUpdate` is issued to each connected peer on each peer-set, and a `NetworkBridgeEvent::OurViewChange` is issued to each event handler for each protocol. +The `activated` and `deactivated` lists determine the evolution of our local view over time. A +`ProtocolMessage::ViewUpdate` is issued to each connected peer on each peer-set, and a +`NetworkBridgeEvent::OurViewChange` is issued to each event handler for each protocol. We only send view updates if the node has indicated that it has finished major blockchain synchronization. @@ -64,24 +82,31 @@ If we are connected to the same peer on both peer-sets, we will send the peer tw ### Overseer Signal: `BlockFinalized` -We update our view's `finalized_number` to the provided one and delay `ProtocolMessage::ViewUpdate` and `NetworkBridgeEvent::OurViewChange` till the next `ActiveLeavesUpdate`. +We update our view's `finalized_number` to the provided one and delay `ProtocolMessage::ViewUpdate` and +`NetworkBridgeEvent::OurViewChange` till the next `ActiveLeavesUpdate`. ### Network Event: `PeerConnected` -Issue a `NetworkBridgeEvent::PeerConnected` for each [Event Handler](#event-handlers) of the peer-set and negotiated protocol version of the peer. Also issue a `NetworkBridgeEvent::PeerViewChange` and send the peer our current view, but only if the node has indicated that it has finished major blockchain synchronization. Otherwise, we only send the peer an empty view. +Issue a `NetworkBridgeEvent::PeerConnected` for each [Event Handler](#event-handlers) of the peer-set and negotiated +protocol version of the peer. Also issue a `NetworkBridgeEvent::PeerViewChange` and send the peer our current view, but +only if the node has indicated that it has finished major blockchain synchronization. Otherwise, we only send the peer +an empty view. ### Network Event: `PeerDisconnected` -Issue a `NetworkBridgeEvent::PeerDisconnected` for each [Event Handler](#event-handlers) of the peer-set and negotiated protocol version of the peer. +Issue a `NetworkBridgeEvent::PeerDisconnected` for each [Event Handler](#event-handlers) of the peer-set and negotiated +protocol version of the peer. ### Network Event: `ProtocolMessage` -Map the message onto the corresponding [Event Handler](#event-handlers) based on the peer-set this message was received on and dispatch via overseer. +Map the message onto the corresponding [Event Handler](#event-handlers) based on the peer-set this message was received +on and dispatch via overseer. ### Network Event: `ViewUpdate` - Check that the new view is valid and note it as the most recent view update of the peer on this peer-set. -- Map a `NetworkBridgeEvent::PeerViewChange` onto the corresponding [Event Handler](#event-handlers) based on the peer-set this message was received on and dispatch via overseer. +- Map a `NetworkBridgeEvent::PeerViewChange` onto the corresponding [Event Handler](#event-handlers) based on the + peer-set this message was received on and dispatch via overseer. ### `ReportPeer` @@ -108,22 +133,23 @@ Map the message onto the corresponding [Event Handler](#event-handlers) based on ### `NewGossipTopology` -- Map all `AuthorityDiscoveryId`s to `PeerId`s and issue a corresponding `NetworkBridgeUpdate` - to all validation subsystems. +- Map all `AuthorityDiscoveryId`s to `PeerId`s and issue a corresponding `NetworkBridgeUpdate` to all validation + subsystems. ## Event Handlers -Network bridge event handlers are the intended recipients of particular network protocol messages. These are each a variant of a message to be sent via the overseer. +Network bridge event handlers are the intended recipients of particular network protocol messages. These are each a +variant of a message to be sent via the overseer. ### Validation V1 -* `ApprovalDistributionV1Message -> ApprovalDistributionMessage::NetworkBridgeUpdate` -* `BitfieldDistributionV1Message -> BitfieldDistributionMessage::NetworkBridgeUpdate` -* `StatementDistributionV1Message -> StatementDistributionMessage::NetworkBridgeUpdate` +- `ApprovalDistributionV1Message -> ApprovalDistributionMessage::NetworkBridgeUpdate` +- `BitfieldDistributionV1Message -> BitfieldDistributionMessage::NetworkBridgeUpdate` +- `StatementDistributionV1Message -> StatementDistributionMessage::NetworkBridgeUpdate` ### Collation V1 -* `CollatorProtocolV1Message -> CollatorProtocolMessage::NetworkBridgeUpdate` +- `CollatorProtocolV1Message -> CollatorProtocolMessage::NetworkBridgeUpdate` [NBM]: ../../types/overseer-protocol.md#network-bridge-message [AppD]: ../../types/overseer-protocol.md#approval-distribution-message diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/provisioner.md b/polkadot/roadmap/implementers-guide/src/node/utility/provisioner.md index a3998cabba6ce67f7eb43ff12591f6213530fa2a..0b4fe6a458732de10c2dcd23466ba30f9a46924c 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/provisioner.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/provisioner.md @@ -1,28 +1,43 @@ # Provisioner -Relay chain block authorship authority is governed by BABE and is beyond the scope of the Overseer and the rest of the subsystems. That said, ultimately the block author needs to select a set of backable parachain candidates and other consensus data, and assemble a block from them. This subsystem is responsible for providing the necessary data to all potential block authors. +Relay chain block authorship authority is governed by BABE and is beyond the scope of the Overseer and the rest of the +subsystems. That said, ultimately the block author needs to select a set of backable parachain candidates and other +consensus data, and assemble a block from them. This subsystem is responsible for providing the necessary data to all +potential block authors. ## Provisionable Data -There are several distinct types of provisionable data, but they share this property in common: all should eventually be included in a relay chain block. +There are several distinct types of provisionable data, but they share this property in common: all should eventually be +included in a relay chain block. ### Backed Candidates -The block author can choose 0 or 1 backed parachain candidates per parachain; the only constraint is that each backable candidate has the appropriate relay parent. However, the choice of a backed candidate must be the block author's. The provisioner subsystem is how those block authors make this choice in practice. +The block author can choose 0 or 1 backed parachain candidates per parachain; the only constraint is that each backable +candidate has the appropriate relay parent. However, the choice of a backed candidate must be the block author's. The +provisioner subsystem is how those block authors make this choice in practice. ### Signed Bitfields -[Signed bitfields](../../types/availability.md#signed-availability-bitfield) are attestations from a particular validator about which candidates it believes are available. Those will only be provided on fresh leaves. +[Signed bitfields](../../types/availability.md#signed-availability-bitfield) are attestations from a particular +validator about which candidates it believes are available. Those will only be provided on fresh leaves. ### Misbehavior Reports -Misbehavior reports are self-contained proofs of misbehavior by a validator or group of validators. For example, it is very easy to verify a double-voting misbehavior report: the report contains two votes signed by the same key, advocating different outcomes. Concretely, misbehavior reports become inherents which cause dots to be slashed. +Misbehavior reports are self-contained proofs of misbehavior by a validator or group of validators. For example, it is +very easy to verify a double-voting misbehavior report: the report contains two votes signed by the same key, advocating +different outcomes. Concretely, misbehavior reports become inherents which cause dots to be slashed. -Note that there is no mechanism in place which forces a block author to include a misbehavior report which it doesn't like, for example if it would be slashed by such a report. The chain's defense against this is to have a relatively long slash period, such that it's likely to encounter an honest author before the slash period expires. +Note that there is no mechanism in place which forces a block author to include a misbehavior report which it doesn't +like, for example if it would be slashed by such a report. The chain's defense against this is to have a relatively long +slash period, such that it's likely to encounter an honest author before the slash period expires. ### Dispute Inherent -The dispute inherent is similar to a misbehavior report in that it is an attestation of misbehavior on the part of a validator or group of validators. Unlike a misbehavior report, it is not self-contained: resolution requires coordinated action by several validators. The canonical example of a dispute inherent involves an approval checker discovering that a set of validators has improperly approved an invalid parachain block: resolving this requires the entire validator set to re-validate the block, so that the minority can be slashed. +The dispute inherent is similar to a misbehavior report in that it is an attestation of misbehavior on the part of a +validator or group of validators. Unlike a misbehavior report, it is not self-contained: resolution requires coordinated +action by several validators. The canonical example of a dispute inherent involves an approval checker discovering that +a set of validators has improperly approved an invalid parachain block: resolving this requires the entire validator set +to re-validate the block, so that the minority can be slashed. Dispute resolution is complex and is explained in substantially more detail [here](../../runtime/disputes.md). @@ -34,58 +49,85 @@ The subsystem should maintain a set of handles to Block Authorship Provisioning - `ActiveLeavesUpdate`: - For each `activated` head: - - spawn a Block Authorship Provisioning iteration with the given relay parent, storing a bidirectional channel with that iteration. + - spawn a Block Authorship Provisioning iteration with the given relay parent, storing a bidirectional channel with + that iteration. - For each `deactivated` head: - terminate the Block Authorship Provisioning iteration for the given relay parent, if any. -- `Conclude`: Forward `Conclude` to all iterations, waiting a small amount of time for them to join, and then hard-exiting. +- `Conclude`: Forward `Conclude` to all iterations, waiting a small amount of time for them to join, and then + hard-exiting. ### On `ProvisionerMessage` -Forward the message to the appropriate Block Authorship Provisioning iteration, or discard if no appropriate iteration is currently active. +Forward the message to the appropriate Block Authorship Provisioning iteration, or discard if no appropriate iteration +is currently active. ### Per Provisioning Iteration -Input: [`ProvisionerMessage`](../../types/overseer-protocol.md#provisioner-message). Backed candidates come from the [Candidate Backing subsystem](../backing/candidate-backing.md), signed bitfields come from the [Bitfield Distribution subsystem](../availability/bitfield-distribution.md), and disputes come from the [Disputes Subsystem](../disputes/dispute-coordinator.md). Misbehavior reports are currently sent from the [Candidate Backing subsystem](../backing/candidate-backing.md) and contain the following misbehaviors: +Input: [`ProvisionerMessage`](../../types/overseer-protocol.md#provisioner-message). Backed candidates come from the +[Candidate Backing subsystem](../backing/candidate-backing.md), signed bitfields come from the [Bitfield Distribution +subsystem](../availability/bitfield-distribution.md), and disputes come from the [Disputes +Subsystem](../disputes/dispute-coordinator.md). Misbehavior reports are currently sent from the [Candidate Backing +subsystem](../backing/candidate-backing.md) and contain the following misbehaviors: 1. `Misbehavior::ValidityDoubleVote` 2. `Misbehavior::MultipleCandidates` 3. `Misbehavior::UnauthorizedStatement` 4. `Misbehavior::DoubleSign` -But we choose not to punish these forms of misbehavior for the time being. Risks from misbehavior are sufficiently mitigated at the protocol level via reputation changes. Punitive actions here may become desirable enough to dedicate time to in the future. +But we choose not to punish these forms of misbehavior for the time being. Risks from misbehavior are sufficiently +mitigated at the protocol level via reputation changes. Punitive actions here may become desirable enough to dedicate +time to in the future. At initialization, this subsystem has no outputs. -Block authors request the inherent data they should use for constructing the inherent in the block which contains parachain execution information. +Block authors request the inherent data they should use for constructing the inherent in the block which contains +parachain execution information. ## Block Production -When a validator is selected by BABE to author a block, it becomes a block producer. The provisioner is the subsystem best suited to choosing which specific backed candidates and availability bitfields should be assembled into the block. To engage this functionality, a `ProvisionerMessage::RequestInherentData` is sent; the response is a [`ParaInherentData`](../../types/runtime.md#parainherentdata). Each relay chain block backs at most one backable parachain block candidate per parachain. Additionally no further block candidate can be backed until the previous one either gets declared available or expired. If bitfields indicate that candidate A, predecessor of B, should be declared available, then B can be backed in the same relay block. Appropriate bitfields, as outlined in the section on [bitfield selection](#bitfield-selection), and any dispute statements should be attached as well. +When a validator is selected by BABE to author a block, it becomes a block producer. The provisioner is the subsystem +best suited to choosing which specific backed candidates and availability bitfields should be assembled into the block. +To engage this functionality, a `ProvisionerMessage::RequestInherentData` is sent; the response is a +[`ParaInherentData`](../../types/runtime.md#parainherentdata). Each relay chain block backs at most one backable +parachain block candidate per parachain. Additionally no further block candidate can be backed until the previous one +either gets declared available or expired. If bitfields indicate that candidate A, predecessor of B, should be declared +available, then B can be backed in the same relay block. Appropriate bitfields, as outlined in the section on [bitfield +selection](#bitfield-selection), and any dispute statements should be attached as well. ### Bitfield Selection -Our goal with respect to bitfields is simple: maximize availability. However, it's not quite as simple as always including all bitfields; there are constraints which still need to be met: +Our goal with respect to bitfields is simple: maximize availability. However, it's not quite as simple as always +including all bitfields; there are constraints which still need to be met: - not more than one bitfield per validator - each 1 bit must correspond to an occupied core -Beyond that, a semi-arbitrary selection policy is fine. In order to meet the goal of maximizing availability, a heuristic of picking the bitfield with the greatest number of 1 bits set in the event of conflict is useful. +Beyond that, a semi-arbitrary selection policy is fine. In order to meet the goal of maximizing availability, a +heuristic of picking the bitfield with the greatest number of 1 bits set in the event of conflict is useful. ### Dispute Statement Selection -This is the point at which the block author provides further votes to active disputes or initiates new disputes in the runtime state. +This is the point at which the block author provides further votes to active disputes or initiates new disputes in the +runtime state. -The block-authoring logic of the runtime has an extra step between handling the inherent-data and producing the actual inherent call, which we assume performs the work of filtering out disputes which are not relevant to the on-chain state. Backing votes are always kept in the dispute statement set. This ensures we punish the maximum number of misbehaving backers. +The block-authoring logic of the runtime has an extra step between handling the inherent-data and producing the actual +inherent call, which we assume performs the work of filtering out disputes which are not relevant to the on-chain state. +Backing votes are always kept in the dispute statement set. This ensures we punish the maximum number of misbehaving +backers. To select disputes: -- Issue a `DisputeCoordinatorMessage::RecentDisputes` message and wait for the response. This is a set of all disputes in recent sessions which we are aware of. +- Issue a `DisputeCoordinatorMessage::RecentDisputes` message and wait for the response. This is a set of all disputes + in recent sessions which we are aware of. ### Determining Bitfield Availability -An occupied core has a `CoreAvailability` bitfield. We also have a list of `SignedAvailabilityBitfield`s. We need to determine from these whether or not a core at a particular index has become available. +An occupied core has a `CoreAvailability` bitfield. We also have a list of `SignedAvailabilityBitfield`s. We need to +determine from these whether or not a core at a particular index has become available. -The key insight required is that `CoreAvailability` is transverse to the `SignedAvailabilityBitfield`s: if we conceptualize the list of bitfields as many rows, each bit of which is its own column, then `CoreAvailability` for a given core index is the vertical slice of bits in the set at that index. +The key insight required is that `CoreAvailability` is transverse to the `SignedAvailabilityBitfield`s: if we +conceptualize the list of bitfields as many rows, each bit of which is its own column, then `CoreAvailability` for a +given core index is the vertical slice of bits in the set at that index. To compute bitfield availability, then: @@ -97,16 +139,22 @@ To compute bitfield availability, then: ### Candidate Selection: Prospective Parachains Mode -The state of the provisioner `PerRelayParent` tracks an important setting, `ProspectiveParachainsMode`. This setting determines which backable candidate selection method the provisioner uses. +The state of the provisioner `PerRelayParent` tracks an important setting, `ProspectiveParachainsMode`. This setting +determines which backable candidate selection method the provisioner uses. -`ProspectiveParachainsMode::Disabled` - The provisioner uses its own internal legacy candidate selection. -`ProspectiveParachainsMode::Enabled` - The provisioner requests that [prospective parachains](../backing/prospective-parachains.md) provide selected candidates. +`ProspectiveParachainsMode::Disabled` - The provisioner uses its own internal legacy candidate selection. +`ProspectiveParachainsMode::Enabled` - The provisioner requests that [prospective +parachains](../backing/prospective-parachains.md) provide selected candidates. -Candidates selected with `ProspectiveParachainsMode::Enabled` are able to benefit from the increased block production time asynchronous backing allows. For this reason all Polkadot protocol networks will eventually use prospective parachains candidate selection. Then legacy candidate selection will be removed as obsolete. +Candidates selected with `ProspectiveParachainsMode::Enabled` are able to benefit from the increased block production +time asynchronous backing allows. For this reason all Polkadot protocol networks will eventually use prospective +parachains candidate selection. Then legacy candidate selection will be removed as obsolete. ### Prospective Parachains Candidate Selection -The goal of candidate selection is to determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core. In prospective parachains candidate selection the provisioner handles the former process while [prospective parachains](../backing/prospective-parachains.md) handles the latter. +The goal of candidate selection is to determine which cores are free, and then to the degree possible, pick a candidate +appropriate to each free core. In prospective parachains candidate selection the provisioner handles the former process +while [prospective parachains](../backing/prospective-parachains.md) handles the latter. To select backable candidates: @@ -116,32 +164,50 @@ To select backable candidates: - The core is unscheduled and doesn’t need to be provisioned with a candidate - On `CoreState::Scheduled` - The core is unoccupied and scheduled to accept a backed block for a particular `para_id`. - - The provisioner requests a backable candidate from [prospective parachains](../backing/prospective-parachains.md) with the desired relay parent, the core’s scheduled `para_id`, and an empty required path. + - The provisioner requests a backable candidate from [prospective parachains](../backing/prospective-parachains.md) + with the desired relay parent, the core’s scheduled `para_id`, and an empty required path. - On `CoreState::Occupied` - - The availability core is occupied by a parachain block candidate pending availability. A further candidate need not be provided by the provisioner unless the core will be vacated this block. This is the case when either bitfields indicate the current core occupant has been made available or a timeout is reached. + - The availability core is occupied by a parachain block candidate pending availability. A further candidate need + not be provided by the provisioner unless the core will be vacated this block. This is the case when either + bitfields indicate the current core occupant has been made available or a timeout is reached. - If `bitfields_indicate_availability` - - If `Some(scheduled_core) = occupied_core.next_up_on_available`, the core will be vacated and in need of a provisioned candidate. The provisioner requests a backable candidate from [prospective parachains](../backing/prospective-parachains.md) with the core’s scheduled `para_id` and a required path with one entry. This entry corresponds to the parablock candidate previously occupying this core, which was made available and can be built upon even though it hasn’t been seen as included in a relay chain block yet. See the Required Path section below for more detail. - - If `occupied_core.next_up_on_available` is `None`, then the core being vacated is unscheduled and doesn’t need to be provisioned with a candidate. + - If `Some(scheduled_core) = occupied_core.next_up_on_available`, the core will be vacated and in need of a + provisioned candidate. The provisioner requests a backable candidate from [prospective + parachains](../backing/prospective-parachains.md) with the core’s scheduled `para_id` and a required path with + one entry. This entry corresponds to the parablock candidate previously occupying this core, which was made + available and can be built upon even though it hasn’t been seen as included in a relay chain block yet. See the + Required Path section below for more detail. + - If `occupied_core.next_up_on_available` is `None`, then the core being vacated is unscheduled and doesn’t need + to be provisioned with a candidate. - Else-if `occupied_core.time_out_at == block_number` - - If `Some(scheduled_core) = occupied_core.next_up_on_timeout`, the core will be vacated and in need of a provisioned candidate. A candidate is requested in exactly the same way as with `CoreState::Scheduled`. - - Else the core being vacated is unscheduled and doesn’t need to be provisioned with a candidate -The end result of this process is a vector of `CandidateHash`s, sorted in order of their core index. + - If `Some(scheduled_core) = occupied_core.next_up_on_timeout`, the core will be vacated and in need of a + provisioned candidate. A candidate is requested in exactly the same way as with `CoreState::Scheduled`. + - Else the core being vacated is unscheduled and doesn’t need to be provisioned with a candidate The end result of +this process is a vector of `CandidateHash`s, sorted in order of their core index. #### Required Path -Required path is a parameter for `ProspectiveParachainsMessage::GetBackableCandidate`, which the provisioner sends in candidate selection. +Required path is a parameter for `ProspectiveParachainsMessage::GetBackableCandidate`, which the provisioner sends in +candidate selection. -An empty required path indicates that the requested candidate should be a direct child of the most recently included parablock for the given `para_id` as of the given relay parent. +An empty required path indicates that the requested candidate should be a direct child of the most recently included +parablock for the given `para_id` as of the given relay parent. -In contrast, a required path with one or more entries prompts [prospective parachains](../backing/prospective-parachains.md) to step forward through its fragment tree for the given `para_id` and relay parent until the desired parablock is reached. We then select a direct child of that parablock to pass to the provisioner. +In contrast, a required path with one or more entries prompts [prospective +parachains](../backing/prospective-parachains.md) to step forward through its fragment tree for the given `para_id` and +relay parent until the desired parablock is reached. We then select a direct child of that parablock to pass to the +provisioner. -The parablocks making up a required path do not need to have been previously seen as included in relay chain blocks. Thus the ability to provision backable candidates based on a required path effectively decouples backing from inclusion. +The parablocks making up a required path do not need to have been previously seen as included in relay chain blocks. +Thus the ability to provision backable candidates based on a required path effectively decouples backing from inclusion. -### Legacy Candidate Selection +### Legacy Candidate Selection -Legacy candidate selection takes place in the provisioner. Thus the provisioner needs to keep an up to date record of all [backed_candidates](../../types/backing.md#backed-candidate) `PerRelayParent` to pick from. +Legacy candidate selection takes place in the provisioner. Thus the provisioner needs to keep an up to date record of +all [backed_candidates](../../types/backing.md#backed-candidate) `PerRelayParent` to pick from. -The goal of candidate selection is to determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core. +The goal of candidate selection is to determine which cores are free, and then to the degree possible, pick a candidate +appropriate to each free core. To determine availability: @@ -149,38 +215,54 @@ To determine availability: - For each core state: - On `CoreState::Scheduled`, then we can make an `OccupiedCoreAssumption::Free`. - On `CoreState::Occupied`, then we may be able to make an assumption: - - If the bitfields indicate availability and there is a scheduled `next_up_on_available`, then we can make an `OccupiedCoreAssumption::Included`. - - If the bitfields do not indicate availability, and there is a scheduled `next_up_on_time_out`, and `occupied_core.time_out_at == block_number_under_production`, then we can make an `OccupiedCoreAssumption::TimedOut`. + - If the bitfields indicate availability and there is a scheduled `next_up_on_available`, then we can make an + `OccupiedCoreAssumption::Included`. + - If the bitfields do not indicate availability, and there is a scheduled `next_up_on_time_out`, and + `occupied_core.time_out_at == block_number_under_production`, then we can make an + `OccupiedCoreAssumption::TimedOut`. - If we did not make an `OccupiedCoreAssumption`, then continue on to the next core. - - Now compute the core's `validation_data_hash`: get the `PersistedValidationData` from the runtime, given the known `ParaId` and `OccupiedCoreAssumption`; + - Now compute the core's `validation_data_hash`: get the `PersistedValidationData` from the runtime, given the known + `ParaId` and `OccupiedCoreAssumption`; - Find an appropriate candidate for the core. - - There are two constraints: `backed_candidate.candidate.descriptor.para_id == scheduled_core.para_id && candidate.candidate.descriptor.validation_data_hash == computed_validation_data_hash`. - - In the event that more than one candidate meets the constraints, selection between the candidates is arbitrary. However, not more than one candidate can be selected per core. + - There are two constraints: `backed_candidate.candidate.descriptor.para_id == scheduled_core.para_id && + candidate.candidate.descriptor.validation_data_hash == computed_validation_data_hash`. + - In the event that more than one candidate meets the constraints, selection between the candidates is arbitrary. + However, not more than one candidate can be selected per core. The end result of this process is a vector of `CandidateHash`s, sorted in order of their core index. ### Retrieving Full `BackedCandidate`s for Selected Hashes -Legacy candidate selection and prospective parachains candidate selection both leave us with a vector of `CandidateHash`s. These are passed to the backing subsystem with `CandidateBackingMessage::GetBackedCandidates`. +Legacy candidate selection and prospective parachains candidate selection both leave us with a vector of +`CandidateHash`s. These are passed to the backing subsystem with `CandidateBackingMessage::GetBackedCandidates`. -The response is a vector of `BackedCandidate`s, sorted in order of their core index and ready to be provisioned to block authoring. The candidate selection and retrieval process should select at maximum one candidate which upgrades the runtime validation code. +The response is a vector of `BackedCandidate`s, sorted in order of their core index and ready to be provisioned to block +authoring. The candidate selection and retrieval process should select at maximum one candidate which upgrades the +runtime validation code. ## Glossary -- **Relay-parent:** - - A particular relay-chain block which serves as an anchor and reference point for processes and data which depend on relay-chain state. -- **Active Leaf:** - - A relay chain block which is the head of an active fork of the relay chain. +- **Relay-parent:** + - A particular relay-chain block which serves as an anchor and reference point for processes and data which depend on + relay-chain state. +- **Active Leaf:** + - A relay chain block which is the head of an active fork of the relay chain. - Block authorship provisioning jobs are spawned per active leaf and concluded for any leaves which become inactive. -- **Candidate Selection:** +- **Candidate Selection:** - The process by which the provisioner selects backable parachain block candidates to pass to block authoring. - - Two versions, prospective parachains candidate selection and legacy candidate selection. See their respective protocol sections for details. -- **Availability Core:** - - Often referred to simply as "cores", availability cores are an abstraction used for resource management. For the provisioner, availability cores are most relevant in that core states determine which `para_id`s to provision backable candidates for. - - For more on availability cores see [Scheduler Module: Availability Cores](../../runtime/scheduler.md#availability-cores) + - Two versions, prospective parachains candidate selection and legacy candidate selection. See their respective + protocol sections for details. +- **Availability Core:** + - Often referred to simply as "cores", availability cores are an abstraction used for resource management. For the + provisioner, availability cores are most relevant in that core states determine which `para_id`s to provision + backable candidates for. + - For more on availability cores see [Scheduler Module: Availability + Cores](../../runtime/scheduler.md#availability-cores) - **Availability Bitfield:** - - Often referred to simply as a "bitfield", an availability bitfield represents the view of parablock candidate availability from a particular validator's perspective. Each bit in the bitfield corresponds to a single [availability core](../../runtime-api/availability-cores.md). + - Often referred to simply as a "bitfield", an availability bitfield represents the view of parablock candidate + availability from a particular validator's perspective. Each bit in the bitfield corresponds to a single + [availability core](../../runtime-api/availability-cores.md). - For more on availability bitfields see [availability](../../types/availability.md) - **Backable vs. Backed:** - Note that we sometimes use "backed" to refer to candidates that are "backable", but not yet backed on chain. - - Backable means that a quorum of the candidate's assigned backing group have provided signed affirming statements. \ No newline at end of file + - Backable means that a quorum of the candidate's assigned backing group have provided signed affirming statements. diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-prechecker.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-prechecker.md index 3cae12e65f33e7fff728124ffc1b75d1f785b7ef..b722bdc6539a6cf34c5870459180dedd590d7ec5 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-prechecker.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-prechecker.md @@ -1,45 +1,70 @@ # PVF Pre-checker -The PVF pre-checker is a subsystem that is responsible for watching the relay chain for new PVFs that require pre-checking. Head over to [overview] for the PVF pre-checking process overview. +The PVF pre-checker is a subsystem that is responsible for watching the relay chain for new PVFs that require +pre-checking. Head over to [overview] for the PVF pre-checking process overview. ## Protocol -There is no dedicated input mechanism for PVF pre-checker. Instead, PVF pre-checker looks on the `ActiveLeavesUpdate` event stream for work. +There is no dedicated input mechanism for PVF pre-checker. Instead, PVF pre-checker looks on the `ActiveLeavesUpdate` +event stream for work. -This subsytem does not produce any output messages either. The subsystem will, however, send messages to the [Runtime API] subsystem to query for the pending PVFs and to submit votes. In addition to that, it will also communicate with [Candidate Validation] Subsystem to request PVF pre-check. +This subsytem does not produce any output messages either. The subsystem will, however, send messages to the [Runtime +API] subsystem to query for the pending PVFs and to submit votes. In addition to that, it will also communicate with +[Candidate Validation] Subsystem to request PVF pre-check. ## Functionality -If the node is running in a collator mode, this subsystem will be disabled. The PVF pre-checker subsystem keeps track of the PVFs that are relevant for the subsystem. +If the node is running in a collator mode, this subsystem will be disabled. The PVF pre-checker subsystem keeps track of +the PVFs that are relevant for the subsystem. -To be relevant for the subsystem, a PVF must be returned by the [`pvfs_require_precheck` runtime API][PVF pre-checking runtime API] in any of the active leaves. If the PVF is not present in any of the active leaves, it ceases to be relevant. +To be relevant for the subsystem, a PVF must be returned by the [`pvfs_require_precheck` runtime API][PVF pre-checking +runtime API] in any of the active leaves. If the PVF is not present in any of the active leaves, it ceases to be +relevant. -When a PVF just becomes relevant, the subsystem will send a message to the [Candidate Validation] subsystem asking for the pre-check. +When a PVF just becomes relevant, the subsystem will send a message to the [Candidate Validation] subsystem asking for +the pre-check. -Upon receving a message from the candidate-validation subsystem, the pre-checker will note down that the PVF has its judgement and will also sign and submit a [`PvfCheckStatement`][PvfCheckStatement] via the [`submit_pvf_check_statement` runtime API][PVF pre-checking runtime API]. In case, a judgement was received for a PVF that is no longer in view it is ignored. +Upon receving a message from the candidate-validation subsystem, the pre-checker will note down that the PVF has its +judgement and will also sign and submit a [`PvfCheckStatement`][PvfCheckStatement] via the [`submit_pvf_check_statement` +runtime API][PVF pre-checking runtime API]. In case, a judgement was received for a PVF that is no longer in view it is +ignored. -Since a vote only is valid during [one session][overview], the subsystem will have to resign and submit the statements for the new session. The new session is assumed to be started if at least one of the leaves has a greater session index that was previously observed in any of the leaves. +Since a vote only is valid during [one session][overview], the subsystem will have to resign and submit the statements +for the new session. The new session is assumed to be started if at least one of the leaves has a greater session index +that was previously observed in any of the leaves. -The subsystem tracks all the statements that it submitted within a session. If for some reason a PVF became irrelevant and then becomes relevant again, the subsystem will not submit a new statement for that PVF within the same session. +The subsystem tracks all the statements that it submitted within a session. If for some reason a PVF became irrelevant +and then becomes relevant again, the subsystem will not submit a new statement for that PVF within the same session. -If the node is not in the active validator set, it will still perform all the checks. However, it will only submit the check statements when the node is in the active validator set. +If the node is not in the active validator set, it will still perform all the checks. However, it will only submit the +check statements when the node is in the active validator set. ### Rejecting failed PVFs -It is possible that the candidate validation was not able to check the PVF, e.g. if it timed out. In that case, the PVF pre-checker will vote against it. This is considered safe, as there is no slashing for being on the wrong side of a pre-check vote. +It is possible that the candidate validation was not able to check the PVF, e.g. if it timed out. In that case, the PVF +pre-checker will vote against it. This is considered safe, as there is no slashing for being on the wrong side of a +pre-check vote. Rejecting instead of abstaining is better in several ways: 1. Conclusion is reached faster - we have actual votes, instead of relying on a timeout. -1. Being strict in pre-checking makes it safer to be more lenient in preparation errors afterwards. Hence we have more leeway in avoiding raising dubious disputes, without making things less secure. +1. Being strict in pre-checking makes it safer to be more lenient in preparation errors afterwards. Hence we have more + leeway in avoiding raising dubious disputes, without making things less secure. -Also, if we only abstain, an attacker can specially craft a PVF wasm blob so that it will fail on e.g. 50% of the validators. In that case a supermajority will never be reached and the vote will repeat multiple times, most likely with the same result (since all votes are cleared on a session change). This is avoided by rejecting failed PVFs, and by only requiring 1/3 of validators to reject a PVF to reach a decision. +Also, if we only abstain, an attacker can specially craft a PVF wasm blob so that it will fail on e.g. 50% of the +validators. In that case a supermajority will never be reached and the vote will repeat multiple times, most likely with +the same result (since all votes are cleared on a session change). This is avoided by rejecting failed PVFs, and by only +requiring 1/3 of validators to reject a PVF to reach a decision. ### Note on Disputes -Having a pre-checking phase allows us to make certain assumptions later when preparing the PVF for execution. If a runtime passed pre-checking, then we know that the runtime should be valid, and therefore any issue during preparation for execution can be assumed to be a local problem on the current node. +Having a pre-checking phase allows us to make certain assumptions later when preparing the PVF for execution. If a +runtime passed pre-checking, then we know that the runtime should be valid, and therefore any issue during preparation +for execution can be assumed to be a local problem on the current node. -For this reason, even deterministic preparation errors should not trigger disputes. And since we do not dispute as a result of the pre-checking phase, as stated above, it should be impossible for preparation in general to result in disputes. +For this reason, even deterministic preparation errors should not trigger disputes. And since we do not dispute as a +result of the pre-checking phase, as stated above, it should be impossible for preparation in general to result in +disputes. [overview]: ../../pvf-prechecking.md [Runtime API]: runtime-api.md diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/runtime-api.md b/polkadot/roadmap/implementers-guide/src/node/utility/runtime-api.md index 6271429c2666fff7c3728fb6997042ac1fd5bacb..b687bd5684c5ae8eed412f7cec25846d7b8befd4 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/runtime-api.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/runtime-api.md @@ -1,6 +1,7 @@ # Runtime API -The Runtime API subsystem is responsible for providing a single point of access to runtime state data via a set of pre-determined queries. This prevents shared ownership of a blockchain client resource by providing +The Runtime API subsystem is responsible for providing a single point of access to runtime state data via a set of +pre-determined queries. This prevents shared ownership of a blockchain client resource by providing ## Protocol @@ -10,8 +11,11 @@ Output: None ## Functionality -On receipt of `RuntimeApiMessage::Request(relay_parent, request)`, answer the request using the post-state of the `relay_parent` provided and provide the response to the side-channel embedded within the request. +On receipt of `RuntimeApiMessage::Request(relay_parent, request)`, answer the request using the post-state of the +`relay_parent` provided and provide the response to the side-channel embedded within the request. ## Jobs -> TODO Don't limit requests based on parent hash, but limit caching. No caching should be done for any requests on `relay_parent`s that are not active based on `ActiveLeavesUpdate` messages. Maybe with some leeway for things that have just been stopped. +> TODO Don't limit requests based on parent hash, but limit caching. No caching should be done for any requests on +> `relay_parent`s that are not active based on `ActiveLeavesUpdate` messages. Maybe with some leeway for things that +> have just been stopped. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-approval.md b/polkadot/roadmap/implementers-guide/src/protocol-approval.md index 693822ce07973a2016e6ac2bed61a7670e559875..9ba7f6da17340ab786df423c248905db7ca76b94 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-approval.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-approval.md @@ -1,19 +1,39 @@ # Approval Process -The Approval Process is the mechanism by which the relay-chain ensures that only valid parablocks are finalized and that backing validators are held accountable for managing to get bad blocks included into the relay chain. +The Approval Process is the mechanism by which the relay-chain ensures that only valid parablocks are finalized and that +backing validators are held accountable for managing to get bad blocks included into the relay chain. -Having a parachain include a bad block into a fork of the relay-chain is not catastrophic as long as the block isn't finalized by the relay-chain's finality gadget, GRANDPA. If the block isn't finalized, that means that the fork of the relay-chain can be reverted in favor of another by means of a dynamic fork-choice rule which leads honest validators to ignore any forks containing that parablock. +Having a parachain include a bad block into a fork of the relay-chain is not catastrophic as long as the block isn't +finalized by the relay-chain's finality gadget, GRANDPA. If the block isn't finalized, that means that the fork of the +relay-chain can be reverted in favor of another by means of a dynamic fork-choice rule which leads honest validators to +ignore any forks containing that parablock. Dealing with a bad parablock proceeds in these stages: 1. Detection 2. Escalation 3. Consequences -First, the bad block must be detected by an honest party. Second, the honest party must escalate the bad block to be checked by all validators. And last, the correct consequences of a bad block must occur. The first consequence, as mentioned above, is to revert the chain so what full nodes perceive to be best no longer contains the bad parablock. The second consequence is to slash all malicious validators. Note that, if the chain containing the bad block is reverted, that the result of the dispute needs to be transplanted or at least transplantable to all other forks of the chain so that malicious validators are slashed in all possible histories. Phrased alternatively, there needs to be no possible relay-chain in which malicious validators get away cost-free. - -Accepting a parablock is the end result of having passed through the detection stage without dispute, or having passed through the escalation/dispute stage with a positive outcome. For this to work, we need the detection procedure to have the properties that enough honest validators are always selected to check the parablock and that they cannot be interfered with by an adversary. This needs to be balanced with the scaling concern of parachains in general: the easiest way to get the first property is to have everyone check everything, but that is clearly too heavy. So we also have a desired constraint on the other property that we have as few validators as possible check any particular parablock. Our assignment function is the method by which we select validators to do approval checks on parablocks. - -It often makes more sense to think of relay-chain blocks as having been approved or not as opposed to thinking about whether parablocks have been approved. A relay-chain block containing a single bad parablock needs to be reverted, and a relay-chain block that contains only approved parablocks can be called approved, as long as its parent relay-chain block is also approved. It is important that the validity of any particular relay-chain block depend on the validity of its ancestry, so we do not finalize a block which has a bad block in its ancestry. +First, the bad block must be detected by an honest party. Second, the honest party must escalate the bad block to be +checked by all validators. And last, the correct consequences of a bad block must occur. The first consequence, as +mentioned above, is to revert the chain so what full nodes perceive to be best no longer contains the bad parablock. The +second consequence is to slash all malicious validators. Note that, if the chain containing the bad block is reverted, +that the result of the dispute needs to be transplanted or at least transplantable to all other forks of the chain so +that malicious validators are slashed in all possible histories. Phrased alternatively, there needs to be no possible +relay-chain in which malicious validators get away cost-free. + +Accepting a parablock is the end result of having passed through the detection stage without dispute, or having passed +through the escalation/dispute stage with a positive outcome. For this to work, we need the detection procedure to have +the properties that enough honest validators are always selected to check the parablock and that they cannot be +interfered with by an adversary. This needs to be balanced with the scaling concern of parachains in general: the +easiest way to get the first property is to have everyone check everything, but that is clearly too heavy. So we also +have a desired constraint on the other property that we have as few validators as possible check any particular +parablock. Our assignment function is the method by which we select validators to do approval checks on parablocks. + +It often makes more sense to think of relay-chain blocks as having been approved or not as opposed to thinking about +whether parablocks have been approved. A relay-chain block containing a single bad parablock needs to be reverted, and a +relay-chain block that contains only approved parablocks can be called approved, as long as its parent relay-chain block +is also approved. It is important that the validity of any particular relay-chain block depend on the validity of its +ancestry, so we do not finalize a block which has a bad block in its ancestry. ```dot process Approval Process digraph { @@ -24,29 +44,56 @@ digraph { Approval has roughly two parts: -- **Assignments** determines which validators performs approval checks on which candidates. It ensures that each candidate receives enough random checkers, while reducing adversaries' odds for obtaining enough checkers, and limiting adversaries' foreknowledge. It tracks approval votes to identify when "no show" approval check takes suspiciously long, perhaps indicating the node being under attack, and assigns more checks in this case. It tracks relay chain equivocations to determine when adversaries possibly gained foreknowledge about assignments, and adds additional checks in this case. +- **Assignments** determines which validators performs approval checks on which candidates. It ensures that each + candidate receives enough random checkers, while reducing adversaries' odds for obtaining enough checkers, and + limiting adversaries' foreknowledge. It tracks approval votes to identify when "no show" approval check takes + suspiciously long, perhaps indicating the node being under attack, and assigns more checks in this case. It tracks + relay chain equivocations to determine when adversaries possibly gained foreknowledge about assignments, and adds + additional checks in this case. -- **Approval checks** listens to the assignments subsystem for outgoing assignment notices that we shall check specific candidates. It then performs these checks by first invoking the reconstruction subsystem to obtain the candidate, second invoking the candidate validity utility subsystem upon the candidate, and finally sending out an approval vote, or perhaps initiating a dispute. +- **Approval checks** listens to the assignments subsystem for outgoing assignment notices that we shall check specific + candidates. It then performs these checks by first invoking the reconstruction subsystem to obtain the candidate, + second invoking the candidate validity utility subsystem upon the candidate, and finally sending out an approval vote, + or perhaps initiating a dispute. -These both run first as off-chain consensus protocols using messages gossiped among all validators, and second as an on-chain record of this off-chain protocols' progress after the fact. We need the on-chain protocol to provide rewards for the off-chain protocol. +These both run first as off-chain consensus protocols using messages gossiped among all validators, and second as an +on-chain record of this off-chain protocols' progress after the fact. We need the on-chain protocol to provide rewards +for the off-chain protocol. -Approval requires two gossiped message types, assignment notices created by its assignments subsystem, and approval votes sent by our approval checks subsystem when authorized by the candidate validity utility subsystem. +Approval requires two gossiped message types, assignment notices created by its assignments subsystem, and approval +votes sent by our approval checks subsystem when authorized by the candidate validity utility subsystem. -### Approval keys +## Approval keys We need two separate keys for the approval subsystem: -- **Approval assignment keys** are sr25519/schnorrkel keys used only for the assignment criteria VRFs. We implicitly sign assignment notices with approval assignment keys by including their relay chain context and additional data in the VRF's extra message, but exclude these from its VRF input. +- **Approval assignment keys** are sr25519/schnorrkel keys used only for the assignment criteria VRFs. We implicitly + sign assignment notices with approval assignment keys by including their relay chain context and additional data in + the VRF's extra message, but exclude these from its VRF input. -- **Approval vote keys** would only sign off on candidate parablock validity and has no natural key type restrictions. There's no need for this to actually embody a new session key type. We just want to make a distinction between assignments and approvals, although distant future node configurations might favor separate roles. We re-use the same keys as are used for parachain backing in practice. +- **Approval vote keys** would only sign off on candidate parablock validity and has no natural key type restrictions. + There's no need for this to actually embody a new session key type. We just want to make a distinction between + assignments and approvals, although distant future node configurations might favor separate roles. We re-use the same + keys as are used for parachain backing in practice. -Approval vote keys could relatively easily be handled by some hardened signer tooling, perhaps even HSMs assuming we select ed25519 for approval vote keys. Approval assignment keys might or might not support hardened signer tooling, but doing so sounds far more complex. In fact, assignment keys determine only VRF outputs that determine approval checker assignments, for which they can only act or not act, so they cannot equivocate, lie, etc. and represent little if any slashing risk for validator operators. +Approval vote keys could relatively easily be handled by some hardened signer tooling, perhaps even HSMs assuming we +select ed25519 for approval vote keys. Approval assignment keys might or might not support hardened signer tooling, but +doing so sounds far more complex. In fact, assignment keys determine only VRF outputs that determine approval checker +assignments, for which they can only act or not act, so they cannot equivocate, lie, etc. and represent little if any +slashing risk for validator operators. -In future, we shall determine which among the several hardening techniques best benefits the network as a whole. We could provide a multi-process multi-machine architecture for validators, perhaps even reminiscent of GNUNet, or perhaps more resembling smart HSM tooling. We might instead design a system that more resembled full systems, like like Cosmos' sentry nodes. In either case, approval assignments might be handled by a slightly hardened machine, but not necessarily nearly as hardened as approval votes, but approval votes machines must similarly run foreign WASM code, which increases their risk, so assignments being separate sounds helpful. +In future, we shall determine which among the several hardening techniques best benefits the network as a whole. We +could provide a multi-process multi-machine architecture for validators, perhaps even reminiscent of GNUNet, or perhaps +more resembling smart HSM tooling. We might instead design a system that more resembled full systems, like like Cosmos' +sentry nodes. In either case, approval assignments might be handled by a slightly hardened machine, but not necessarily +nearly as hardened as approval votes, but approval votes machines must similarly run foreign WASM code, which increases +their risk, so assignments being separate sounds helpful. ## Assignments -Approval assignment determines on which candidate parachain blocks each validator performs approval checks. An approval session considers only one relay chain block and assigns only those candidates that relay chain block declares available. +Approval assignment determines on which candidate parachain blocks each validator performs approval checks. An approval +session considers only one relay chain block and assigns only those candidates that relay chain block declares +available. Assignment balances several concerns: @@ -54,149 +101,286 @@ Assignment balances several concerns: - ensures enough checkers, and - distributes assignments relatively equitably. -Assignees determine their own assignments to check specific candidates using two or three assignment criteria. Assignees never reveal their assignments until relevant, and gossip delays assignments sent early, which limits others' foreknowledge. Assignees learn their assignment only with the relay chain block. +Assignees determine their own assignments to check specific candidates using two or three assignment criteria. +Assignees never reveal their assignments until relevant, and gossip delays assignments sent early, which limits others' +foreknowledge. Assignees learn their assignment only with the relay chain block. -All criteria require the validator evaluate a verifiable random function (VRF) using their VRF secret key. All criteria input specific data called "stories" about the session's relay chain block, and output candidates to check and a precedence called a `DelayTranche`. +All criteria require the validator evaluate a verifiable random function (VRF) using their VRF secret key. All criteria +input specific data called "stories" about the session's relay chain block, and output candidates to check and a +precedence called a `DelayTranche`. -We liberate availability cores when their candidate becomes available of course, but one approval assignment criteria continues associating each candidate with the core number it occupied when it became available. +We liberate availability cores when their candidate becomes available of course, but one approval assignment criteria +continues associating each candidate with the core number it occupied when it became available. -Assignment operates in loosely timed rounds determined by this `DelayTranche`s, which proceed roughly 12 times faster than six second block production assuming half second gossip times. If a candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`. We continue until all candidates have enough approval checkers assigned. We take entire tranches together if we do not yet have enough, so we expect strictly more than enough checkers. We also take later tranches if some checkers return their approval votes too slow (see no shows below). +Assignment operates in loosely timed rounds determined by this `DelayTranche`s, which proceed roughly 12 times faster +than six second block production assuming half second gossip times. If a candidate `C` needs more approval checkers by +the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send +assignment notice for `C`. We continue until all candidates have enough approval checkers assigned. We take entire +tranches together if we do not yet have enough, so we expect strictly more than enough checkers. We also take later +tranches if some checkers return their approval votes too slow (see no shows below). -Assignment ensures validators check those relay chain blocks for which they have delay tranche zero aka the highest precedence, so that adversaries always face honest checkers equal to the expected number of assignments with delay tranche zero. +Assignment ensures validators check those relay chain blocks for which they have delay tranche zero aka the highest +precedence, so that adversaries always face honest checkers equal to the expected number of assignments with delay +tranche zero. -Among these criteria, the BABE VRF output provides the story for two, which reduces how frequently adversaries could position their own checkers. We have one criterion whose story consists of the candidate's block hash plus external knowledge that a relay chain equivocation exists with a conflicting candidate. It provides unforeseeable assignments when adversaries gain foreknowledge about the other two by committing an equivocation in relay chain block production. +Among these criteria, the BABE VRF output provides the story for two, which reduces how frequently adversaries could +position their own checkers. We have one criterion whose story consists of the candidate's block hash plus external +knowledge that a relay chain equivocation exists with a conflicting candidate. It provides unforeseeable assignments +when adversaries gain foreknowledge about the other two by committing an equivocation in relay chain block production. ## Announcements / Notices -We gossip assignment notices among nodes so that all validators know which validators should check each candidate, and if any candidate requires more checkers. +We gossip assignment notices among nodes so that all validators know which validators should check each candidate, and +if any candidate requires more checkers. -Assignment notices consist of a relay chain context given by a block hash, an assignment criteria, consisting of the criteria identifier and optionally a criteria specific field, an assignee identifier, and a VRF signature by the assignee, which itself consists of a VRF pre-output and a DLEQ proof. Its VRF input consists of the criteria, usually including a criteria specific field, and a "story" about its relay chain context block. +Assignment notices consist of a relay chain context given by a block hash, an assignment criteria, consisting of the +criteria identifier and optionally a criteria specific field, an assignee identifier, and a VRF signature by the +assignee, which itself consists of a VRF pre-output and a DLEQ proof. Its VRF input consists of the criteria, usually +including a criteria specific field, and a "story" about its relay chain context block. -We never include stories inside the gossip messages containing assignment notices, but require each validator reconstruct them. We never care about assignments in the disputes process, so this does not complicate remote disputes. +We never include stories inside the gossip messages containing assignment notices, but require each validator +reconstruct them. We never care about assignments in the disputes process, so this does not complicate remote disputes. -In a Schnorr VRF, there is an extra signed message distinct from this input, which we set to the relay chain block hash. As a result, assignment notices are self signing and can be "politely" gossiped without additional signatures, meaning between nodes who can compute the story from the relay chain context. In other words, if we cannot compute the story required by an assignment notice's VRF part then our self signing property fails and we cannot verify its origin. We could fix this with either another signature layer (64 bytes) or by including the VRF input point computed from the story (32 bytes), but doing so appears unhelpful. +In a Schnorr VRF, there is an extra signed message distinct from this input, which we set to the relay chain block hash. +As a result, assignment notices are self signing and can be "politely" gossiped without additional signatures, meaning +between nodes who can compute the story from the relay chain context. In other words, if we cannot compute the story +required by an assignment notice's VRF part then our self signing property fails and we cannot verify its origin. We +could fix this with either another signature layer (64 bytes) or by including the VRF input point computed from the +story (32 bytes), but doing so appears unhelpful. -Any validator could send their assignment notices and/or approval votes too early. We gossip the approval votes early because they represent a major commitment by the validator. We delay gossiping the assignment notices until they agree with our local clock however. We also impose a politeness condition that the recipient knows the relay chain context used by the assignment notice. +Any validator could send their assignment notices and/or approval votes too early. We gossip the approval votes early +because they represent a major commitment by the validator. We delay gossiping the assignment notices until they agree +with our local clock however. We also impose a politeness condition that the recipient knows the relay chain context +used by the assignment notice. ## Stories -We based assignment criteria upon two possible "stories" about the relay chain block `R` that included the candidate aka declared the candidate available. All stories have an output that attempts to minimize adversarial influence, which then acts as the VRF input for an assignment criteria. +We based assignment criteria upon two possible "stories" about the relay chain block `R` that included the candidate aka +declared the candidate available. All stories have an output that attempts to minimize adversarial influence, which +then acts as the VRF input for an assignment criteria. -We first have a `RelayVRFStory` that outputs the randomness from another VRF output produced by the relay chain block producer when creating `R`. Among honest nodes, only this one relay chain block producer who creates `R` knew the story in advance, and even they knew nothing two epochs previously. +We first have a `RelayVRFStory` that outputs the randomness from another VRF output produced by the relay chain block +producer when creating `R`. Among honest nodes, only this one relay chain block producer who creates `R` knew the story +in advance, and even they knew nothing two epochs previously. -In BABE, we create this value calling `schnorrkel::vrf::VRFInOut::make_bytes` with a context "A&V RC-VRF", with the `VRFInOut` coming from either the VRF that authorized block production for primary blocks, or else from the secondary block VRF for the secondary block type. +In BABE, we create this value calling `schnorrkel::vrf::VRFInOut::make_bytes` with a context "A&V RC-VRF", with the +`VRFInOut` coming from either the VRF that authorized block production for primary blocks, or else from the secondary +block VRF for the secondary block type. -In Sassafras, we shall always use the non-anonymized recycling VRF output, never the anonymized ring VRF that authorizes block production. We do not currently know if Sassafras shall have a separate schnorrkel key, but if it reuses its ring VRF key there is an equivalent `ring_vrf::VRFInOut::make_bytes`. +In Sassafras, we shall always use the non-anonymized recycling VRF output, never the anonymized ring VRF that authorizes +block production. We do not currently know if Sassafras shall have a separate schnorrkel key, but if it reuses its ring +VRF key there is an equivalent `ring_vrf::VRFInOut::make_bytes`. -We like that `RelayVRFStory` admits relatively few choices, but an adversary who equivocates in relay chain block production could learn assignments that depend upon the `RelayVRFStory` too early because the same relay chain VRF appears in multiple blocks. +We like that `RelayVRFStory` admits relatively few choices, but an adversary who equivocates in relay chain block +production could learn assignments that depend upon the `RelayVRFStory` too early because the same relay chain VRF +appears in multiple blocks. -We therefore provide a secondary `RelayEquivocationStory` that outputs the candidate's block hash, but only for candidate equivocations. We say a candidate `C` in `R` is an equivocation when there exists another relay chain block `R1` that equivocates for `R` in the sense that `R` and `R1` have the same `RelayVRFStory`, but `R` contains `C` and `R1` does not contain `C`. +We therefore provide a secondary `RelayEquivocationStory` that outputs the candidate's block hash, but only for +candidate equivocations. We say a candidate `C` in `R` is an equivocation when there exists another relay chain block +`R1` that equivocates for `R` in the sense that `R` and `R1` have the same `RelayVRFStory`, but `R` contains `C` and +`R1` does not contain `C`. -We want checkers for candidate equivocations that lie outside our preferred relay chain as well, which represents a slightly different usage for the assignments module, and might require more information in the gossip messages. +We want checkers for candidate equivocations that lie outside our preferred relay chain as well, which represents a +slightly different usage for the assignments module, and might require more information in the gossip messages. ## Assignment criteria -Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence `DelayTranche` for when the assignment becomes valid. +Assignment criteria compute actual assignments using stories and the validators' secret approval assignment key. +Assignment criteria output a `Position` consisting of both a `ParaId` to be checked, as well as a precedence +`DelayTranche` for when the assignment becomes valid. -Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both `RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` runs a VRF whose input is the output of a `RelayEquivocationStory`. +Assignment criteria come in three flavors, `RelayVRFModulo`, `RelayVRFDelay` and `RelayEquivocation`. Among these, both +`RelayVRFModulo` and `RelayVRFDelay` run a VRF whose input is the output of a `RelayVRFStory`, while `RelayEquivocation` +runs a VRF whose input is the output of a `RelayEquivocationStory`. Among these, we have two distinct VRF output computations: -`RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the sampled availability core in this relay chain block. We choose three samples initially, but we could make polkadot more secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful `RelayVRFModulo` samples are assigned delay tranche zero. +`RelayVRFModulo` runs several distinct samples whose VRF input is the `RelayVRFStory` and the sample number. It +computes the VRF output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Core", reduces this number +modulo the number of availability cores, and outputs the candidate just declared available by, and included by aka +leaving, that availability core. We drop any samples that return no candidate because no candidate was leaving the +sampled availability core in this relay chain block. We choose three samples initially, but we could make Polkadot more +secure and efficient by increasing this to four or five, and reducing the backing checks accordingly. All successful +`RelayVRFModulo` samples are assigned delay tranche zero. -There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. +There is no sampling process for `RelayVRFDelay` and `RelayEquivocation`. We instead run them on specific candidates +and they compute a delay from their VRF output. `RelayVRFDelay` runs for all candidates included under, aka declared +available by, a relay chain block, and inputs the associated VRF output via `RelayVRFStory`. `RelayEquivocation` runs +only on candidate block equivocations, and inputs their block hashes via the `RelayEquivocation` story. -`RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has `zeroth_delay_tranche_width+1` times as many assignments as any other tranche. +`RelayVRFDelay` and `RelayEquivocation` both compute their output with `schnorrkel::vrf::VRFInOut::make_bytes` using the +context "A&V Tranche" and reduce the result modulo `num_delay_tranches + zeroth_delay_tranche_width`, and consolidate +results 0 through `zeroth_delay_tranche_width` to be 0. In this way, they ensure the zeroth delay tranche has +`zeroth_delay_tranche_width+1` times as many assignments as any other tranche. -As future work (or TODO?), we should merge assignment notices with the same delay and story using `vrf_merge`. We cannot merge those with the same delay and different stories because `RelayEquivocationStory`s could change but `RelayVRFStory` never changes. +As future work (or TODO?), we should merge assignment notices with the same delay and story using `vrf_merge`. We +cannot merge those with the same delay and different stories because `RelayEquivocationStory`s could change but +`RelayVRFStory` never changes. ## Announcer and Watcher/Tracker -We track all validators' announced approval assignments for each candidate associated to each relay chain block, which tells us which validators were assigned to which candidates. +We track all validators' announced approval assignments for each candidate associated to each relay chain block, which +tells us which validators were assigned to which candidates. -We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any applicable criteria. +We permit at most one assignment per candidate per story per validator, so one validator could be assigned under both +the `RelayVRFDelay` and `RelayEquivocation` criteria, but not under both `RelayVRFModulo` and `RelayVRFDelay` criteria, +since those both use the same story. We permit only one approval vote per candidate per validator, which counts for any +applicable criteria. -We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get announced, which includes all `RelayVRFModulo` assignments. +We announce, and start checking for, our own assignments when the delay of their tranche is reached, but only if the +tracker says the assignee candidate requires more approval checkers. We never announce an assignment we believe +unnecessary because early announcements gives an adversary information. All delay tranche zero assignments always get +announced, which includes all `RelayVRFModulo` assignments. -In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing their assignments. +In other words, if some candidate `C` needs more approval checkers by the time we reach round `t` then any validators +with an assignment to `C` in delay tranche `t` gossip their send assignment notice for `C`, and begin reconstruction and +validation for 'C. If however `C` reached enough assignments, then validators with later assignments skip announcing +their assignments. -We continue until all candidates have enough approval checkers assigned. We never prioritize assignments within tranches and count all or no assignments for a given tranche together, so we often overshoot the target number of assigned approval checkers. +We continue until all candidates have enough approval checkers assigned. We never prioritize assignments within +tranches and count all or no assignments for a given tranche together, so we often overshoot the target number of +assigned approval checkers. ### No shows -We have a "no show" timeout longer than one relay chain slot, so at least 6 seconds, during which we expect approval checks should succeed in reconstructing the candidate block, in redoing its erasure coding to check the candidate receipt, and finally in rechecking the candidate block itself. +We have a "no show" timeout longer than one relay chain slot, so at least 6 seconds, during which we expect approval +checks should succeed in reconstructing the candidate block, in redoing its erasure coding to check the candidate +receipt, and finally in rechecking the candidate block itself. -We consider a validator a "no show" if they do not approve or dispute within this "no show" timeout from our receiving their assignment notice. We time this from our receipt of their assignment notice instead of our imagined real time for their tranche because otherwise receiving late assignment notices creates immediate "no shows" and unnecessary work. +We consider a validator a "no show" if they do not approve or dispute within this "no show" timeout from our receiving +their assignment notice. We time this from our receipt of their assignment notice instead of our imagined real time for +their tranche because otherwise receiving late assignment notices creates immediate "no shows" and unnecessary work. -We worry "no shows" represent a validator under denial of service attack, presumably to prevent it from reconstructing the candidate, but perhaps delaying it form gossiping a dispute too. We therefore always replace "no shows" by adding one entire extra delay tranche worth of validators, so such attacks always result in additional checkers. +We worry "no shows" represent a validator under denial of service attack, presumably to prevent it from reconstructing +the candidate, but perhaps delaying it form gossiping a dispute too. We therefore always replace "no shows" by adding +one entire extra delay tranche worth of validators, so such attacks always result in additional checkers. -As an example, imagine we need 20 checkers, but tranche zero produces only 14, and tranche one only 4, then we take all 5 from tranche two, and thus require 23 checkers for that candidate. If one checker Charlie from tranche one or two does not respond within say 8 seconds, then we add all 7 checkers from tranche three. If again one checker Cindy from tranche three does not respond within 8 seconds then we take all 3 checkers from tranche four. We now have 33 checkers working on the candidate, so this escalated quickly. +As an example, imagine we need 20 checkers, but tranche zero produces only 14, and tranche one only 4, then we take all +5 from tranche two, and thus require 23 checkers for that candidate. If one checker Charlie from tranche one or two +does not respond within say 8 seconds, then we add all 7 checkers from tranche three. If again one checker Cindy from +tranche three does not respond within 8 seconds then we take all 3 checkers from tranche four. We now have 33 checkers +working on the candidate, so this escalated quickly. -We escalated so quickly because we worried that Charlie and Cindy might be the only honest checkers assigned to that candidate. If therefore either Charlie or Cindy finally return an approval, then we can conclude approval, and abandon the checkers from tranche four. +We escalated so quickly because we worried that Charlie and Cindy might be the only honest checkers assigned to that +candidate. If therefore either Charlie or Cindy finally return an approval, then we can conclude approval, and abandon +the checkers from tranche four. -We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" on-chain. We discuss below how this helps reward validators who replace "no shows". +We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" +on-chain. We discuss below how this helps reward validators who replace "no shows". -We avoid slashing for "no shows" by itself, although being "no show" could enter into some computation that punishes repeated poor performance, presumably replaces `ImOnline`, and we could reduce their rewards and further rewards those who filled in. +We avoid slashing for "no shows" by itself, although being "no show" could enter into some computation that punishes +repeated poor performance, presumably replaces `ImOnline`, and we could reduce their rewards and further rewards those +who filled in. -As future work, we foresee expanding the "no show" scheme to anonymize the additional checkers, like by using assignment noticed with a new criteria that employs a ring VRF and then all validators providing cover by requesting a couple erasure coded pieces, but such anonymity scheme sound extremely complex and lie far beyond our initial functionality. +As future work, we foresee expanding the "no show" scheme to anonymize the additional checkers, like by using assignment +noticed with a new criteria that employs a ring VRF and then all validators providing cover by requesting a couple +erasure coded pieces, but such anonymity scheme sound extremely complex and lie far beyond our initial functionality. ## Assignment postponement -We expect validators could occasionally overloaded when they randomly acquire too many assignments. All these fluctuations amortize over multiple blocks fairly well, but this slows down finality. +We expect validators could occasionally overloaded when they randomly acquire too many assignments. All these +fluctuations amortize over multiple blocks fairly well, but this slows down finality. -We therefore permit validators to delay sending their assignment noticed intentionally. If nobody knows about their assignment then they avoid creating "no shows" and the workload progresses normally. +We therefore permit validators to delay sending their assignment noticed intentionally. If nobody knows about their +assignment then they avoid creating "no shows" and the workload progresses normally. -We strongly prefer if postponements come from tranches higher aka less important than zero because tranche zero checks provide somewhat more security. +We strongly prefer if postponements come from tranches higher aka less important than zero because tranche zero checks +provide somewhat more security. TODO: When? Is this optimal for the network? etc. ## On-chain verification -We should verify approval on-chain to reward approval checkers. We therefore require the "no show" timeout to be longer than a relay chain slot so that we can witness "no shows" on-chain, which helps with this goal. The major challenge with an on-chain record of the off-chain process is adversarial block producers who may either censor votes or publish votes to the chain which cause other votes to be ignored and unrewarded (reward stealing). +We should verify approval on-chain to reward approval checkers. We therefore require the "no show" timeout to be longer +than a relay chain slot so that we can witness "no shows" on-chain, which helps with this goal. The major challenge with +an on-chain record of the off-chain process is adversarial block producers who may either censor votes or publish votes +to the chain which cause other votes to be ignored and unrewarded (reward stealing). -In principle, all validators have some "tranche" at which they're assigned to the parachain candidate, which ensures we reach enough validators eventually. As noted above, we often retract "no shows" when the slow validator eventually shows up, so witnessing their initially being a "no show" helps manage rewards. +In principle, all validators have some "tranche" at which they're assigned to the parachain candidate, which ensures we +reach enough validators eventually. As noted above, we often retract "no shows" when the slow validator eventually +shows up, so witnessing their initially being a "no show" helps manage rewards. -We expect on-chain verification should work in two phases: We first record assignments notices and approval votes on-chain in relay chain block, doing the VRF or regular signature verification again in block verification, and inserting chain authenticated unsigned notes into the relay chain state that contain the checker, tranche, paraid, and relay block height for each assignment notice. We then later have another relay chain block that runs some "approved" intrinsic, which extract all these notes from the state and feeds them into our approval code. +We expect on-chain verification should work in two phases: We first record assignments notices and approval votes +on-chain in relay chain block, doing the VRF or regular signature verification again in block verification, and +inserting chain authenticated unsigned notes into the relay chain state that contain the checker, tranche, paraid, and +relay block height for each assignment notice. We then later have another relay chain block that runs some "approved" +intrinsic, which extract all these notes from the state and feeds them into our approval code. -We now encounter one niche concern in the interaction between postponement and on-chain verification: Any validator with a tranche zero (or other low) assignment could delay sending an assignment notice, like because they postponed their assigned tranche (which is allowed). If they later send this assignment notices right around finality time, then they race with this approved. intrinsic: If their announcement gets on-chain (also allowed), then yes it delays finality. If it does not get on-chain, then yes we've one announcement that the off-chain consensus system says is valid, but the chain ignores for being too slow. +We now encounter one niche concern in the interaction between postponement and on-chain verification: Any validator +with a tranche zero (or other low) assignment could delay sending an assignment notice, like because they postponed +their assigned tranche (which is allowed). If they later send this assignment notices right around finality time, then +they race with this approved. intrinsic: If their announcement gets on-chain (also allowed), then yes it delays +finality. If it does not get on-chain, then yes we've one announcement that the off-chain consensus system says is +valid, but the chain ignores for being too slow. -We need the chain to win in this case, but doing this requires imposing an annoyingly long overarching delay upon finality. We might explore limits on postponement too, but this sounds much harder. +We need the chain to win in this case, but doing this requires imposing an annoyingly long overarching delay upon +finality. We might explore limits on postponement too, but this sounds much harder. ## Parameters -We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because `RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been properly analyzed. +We prefer doing approval checkers assignments under `RelayVRFModulo` as opposed to `RelayVRFDelay` because +`RelayVRFModulo` avoids giving individual checkers too many assignments and tranche zero assignments benefit security +the most. We suggest assigning at least 16 checkers under `RelayVRFModulo` although assignment levels have never been +properly analyzed. -Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per tranche and the zeroth delay tranche width. +Our delay criteria `RelayVRFDelay` and `RelayEquivocation` both have two primary paramaters, expected checkers per +tranche and the zeroth delay tranche width. -We require expected checkers per tranche to be less than three because otherwise an adversary with 1/3 stake could force all nodes into checking all blocks. We strongly recommend expected checkers per tranche to be less than two, which helps avoid both accidental and intentional explosions. We also suggest expected checkers per tranche be larger than one, which helps prevent adversaries from predicting than advancing one tranche adds only their own validators. +We require expected checkers per tranche to be less than three because otherwise an adversary with 1/3 stake could force +all nodes into checking all blocks. We strongly recommend expected checkers per tranche to be less than two, which +helps avoid both accidental and intentional explosions. We also suggest expected checkers per tranche be larger than +one, which helps prevent adversaries from predicting than advancing one tranche adds only their own validators. -We improve security more with tranche zero assignments, so `RelayEquivocation` should consolidates its first several tranches into tranche zero. We describe this as the zeroth delay tranche width, which initially we set to 12 for `RelayEquivocation` and `1` for `RelayVRFDelay`. +We improve security more with tranche zero assignments, so `RelayEquivocation` should consolidates its first several +tranches into tranche zero. We describe this as the zeroth delay tranche width, which initially we set to 12 for +`RelayEquivocation` and `1` for `RelayVRFDelay`. ## Why VRFs? We do assignments with VRFs to give "enough" checkers some meaning beyond merely "expected" checkers: -We could specify a protocol that used only system randomness, which works because our strongest defense is the expected number of honest checkers who assign themselves. In this, adversaries could trivially flood their own blocks with their own checkers, so this strong defense becomes our only defense, and delay tranches become useless, so some blocks actually have zero approval checkers and possibly only one checker overall. +We could specify a protocol that used only system randomness, which works because our strongest defense is the expected +number of honest checkers who assign themselves. In this, adversaries could trivially flood their own blocks with their +own checkers, so this strong defense becomes our only defense, and delay tranches become useless, so some blocks +actually have zero approval checkers and possibly only one checker overall. -VRFs though require adversaries wait far longer between such attacks, which also helps against adversaries with little at stake because they compromised validators. VRFs raise user confidence that no such "drive by" attacks occurred because the delay tranche system ensure at least some minimum number of approval checkers. In this vein, VRFs permit reducing backing checks and increasing approval checks, which makes polkadot more efficient. +VRFs though require adversaries wait far longer between such attacks, which also helps against adversaries with little +at stake because they compromised validators. VRFs raise user confidence that no such "drive by" attacks occurred +because the delay tranche system ensure at least some minimum number of approval checkers. In this vein, VRFs permit +reducing backing checks and increasing approval checks, which makes Polkadot more efficient. ## Gossip -Any validator could send their assignment notices and/or approval votes too early. We gossip the approval votes because they represent a major commitment by the validator. We retain but delay gossiping the assignment notices until they agree with our local clock. +Any validator could send their assignment notices and/or approval votes too early. We gossip the approval votes because +they represent a major commitment by the validator. We retain but delay gossiping the assignment notices until they +agree with our local clock. -Assignment notices being gossiped too early might create a denial of service vector. If so, we might exploit the relative time scheme that synchronizes our clocks, which conceivably permits just dropping excessively early assignments. +Assignment notices being gossiped too early might create a denial of service vector. If so, we might exploit the +relative time scheme that synchronizes our clocks, which conceivably permits just dropping excessively early +assignments. ## Finality GRANDPA Voting Rule -The relay-chain requires validators to participate in GRANDPA. In GRANDPA, validators submit off-chain votes on what they believe to be the best block of the chain, and GRANDPA determines the common block contained by a supermajority of sub-chains. There are also additional constraints on what can be submitted based on results of previous rounds of voting. +The relay-chain requires validators to participate in GRANDPA. In GRANDPA, validators submit off-chain votes on what +they believe to be the best block of the chain, and GRANDPA determines the common block contained by a supermajority of +sub-chains. There are also additional constraints on what can be submitted based on results of previous rounds of +voting. -In order to avoid finalizing anything which has not received enough approval votes or is disputed, we will pair the approval protocol with an alteration to the GRANDPA voting strategy for honest nodes which causes them to vote only on chains where every parachain candidate within has been approved. Furthermore, the voting rule prevents voting for chains where there is any live dispute or any dispute has resolved to a candidate being invalid. +In order to avoid finalizing anything which has not received enough approval votes or is disputed, we will pair the +approval protocol with an alteration to the GRANDPA voting strategy for honest nodes which causes them to vote only on +chains where every parachain candidate within has been approved. Furthermore, the voting rule prevents voting for +chains where there is any live dispute or any dispute has resolved to a candidate being invalid. -Thus, the finalized relay-chain should contain only relay-chain blocks where a majority believe that every block within has been sufficiently approved. +Thus, the finalized relay-chain should contain only relay-chain blocks where a majority believe that every block within +has been sufficiently approved. ### Future work -We could consider additional gossip messages with which nodes claims "slow availability" and/or "slow candidate" to fine tune the assignments "no show" system, but long enough "no show" delays suffice probably. +We could consider additional gossip messages with which nodes claims "slow availability" and/or "slow candidate" to fine +tune the assignments "no show" system, but long enough "no show" delays suffice probably. -We shall develop more practical experience with UDP once the availability system works using direct UDP connections. In this, we should discover if reconstruction performs adequately with a complete graphs or -benefits from topology restrictions. At this point, an assignment notices could implicitly request pieces from a random 1/3rd, perhaps topology restricted, which saves one gossip round. If this preliminary fast reconstruction fails, then nodes' request alternative pieces directly. There is an interesting design space in how this overlaps with "slow availability" claims. +We shall develop more practical experience with UDP once the availability system works using direct UDP connections. In +this, we should discover if reconstruction performs adequately with a complete graphs or benefits from topology +restrictions. At this point, an assignment notices could implicitly request pieces from a random 1/3rd, perhaps +topology restricted, which saves one gossip round. If this preliminary fast reconstruction fails, then nodes' request +alternative pieces directly. There is an interesting design space in how this overlaps with "slow availability" claims. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-chain-selection.md b/polkadot/roadmap/implementers-guide/src/protocol-chain-selection.md index dd066df43cdde8e0f9c0e4ecd9364e2b7fb68fbd..9f0262243bd2267e25613e0bf28b0d9dc5893253 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-chain-selection.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-chain-selection.md @@ -1,12 +1,21 @@ # Chain Selection -Chain selection processes in blockchains are used for the purpose of selecting blocks to build on and finalize. It is important for these processes to be consistent among nodes and resilient to a maximum proportion of malicious nodes which do not obey the chain selection process. +Chain selection processes in blockchains are used for the purpose of selecting blocks to build on and finalize. It is +important for these processes to be consistent among nodes and resilient to a maximum proportion of malicious nodes +which do not obey the chain selection process. -The parachain host uses both a block authoring system and a finality gadget. The chain selection strategy of the parachain host involves two key components: a _leaf-selection_ rule and a set of _finality constraints_. When it's a validator's turn to author on a block, they are expected to select the best block via the leaf-selection rule to build on top of. When a validator is participating in finality, there is a minimum block which can be voted on, which is usually the finalized block. The validator should select the best chain according to the leaf-selection rule and subsequently apply the finality constraints to arrive at the actual vote cast by that validator. +The parachain host uses both a block authoring system and a finality gadget. The chain selection strategy of the +parachain host involves two key components: a _leaf-selection_ rule and a set of _finality constraints_. When it's a +validator's turn to author on a block, they are expected to select the best block via the leaf-selection rule to build +on top of. When a validator is participating in finality, there is a minimum block which can be voted on, which is +usually the finalized block. The validator should select the best chain according to the leaf-selection rule and +subsequently apply the finality constraints to arrive at the actual vote cast by that validator. -Before diving into the particularities of the leaf-selection rule and the finality constraints, it's important to discuss the goals that these components are meant to achieve. For this it is useful to create the definitions of _viable_ and _finalizable_ blocks. +Before diving into the particularities of the leaf-selection rule and the finality constraints, it's important to +discuss the goals that these components are meant to achieve. For this it is useful to create the definitions of +_viable_ and _finalizable_ blocks. -### Property Definitions +## Property Definitions A block is considered **viable** when all of the following hold: 1. It is or descends from the finalized block @@ -32,17 +41,27 @@ A block is considered **finalizable** when all of the following hold: 4. It is either finalized or includes no candidates which have unresolved disputes or have lost a dispute. -### The leaf-selection rule +## The leaf-selection rule -We assume that every block has an implicit weight or score which can be used to compare blocks. In BABE, this is determined by the number of primary slots included in the chain. In PoW, this is the chain with either the most work or GHOST weight. +We assume that every block has an implicit weight or score which can be used to compare blocks. In BABE, this is +determined by the number of primary slots included in the chain. In PoW, this is the chain with either the most work or +GHOST weight. -The leaf-selection rule based on our definitions above is simple: we take the maximum-scoring viable leaf we are aware of. In the case of a tie we select the one with a lower lexicographical block hash. +The leaf-selection rule based on our definitions above is simple: we take the maximum-scoring viable leaf we are aware +of. In the case of a tie we select the one with a lower lexicographical block hash. -### The best-chain-containing rule +## The best-chain-containing rule -Finality gadgets, as mentioned above, will often impose an additional requirement to vote on a chain containing a specific block, known as the **required** block. Although this is typically the most recently finalized block, it is possible that it may be a block that is unfinalized. When receiving such a request: +Finality gadgets, as mentioned above, will often impose an additional requirement to vote on a chain containing a +specific block, known as the **required** block. Although this is typically the most recently finalized block, it is +possible that it may be a block that is unfinalized. When receiving such a request: 1. If the required block is the best finalized block, then select the best viable leaf. -2. If the required block is unfinalized and non-viable, then select the required block and go no further. This is likely an indication that something bad will be finalized in the network, which will never happen when approvals & disputes are functioning correctly. Nevertheless we account for the case here. -3. If the required block is unfinalized and viable, then iterate over the viable leaves in descending order by score and select the first one which contains the required block in its chain. Backwards iteration is a simple way to check this, but if unfinalized chains grow long then Merkle Mountain-Ranges will most likely be more efficient. - -Once selecting a leaf, the chain should be constrained to the maximum of the required block or the highest **finalizable** ancestor. +2. If the required block is unfinalized and non-viable, then select the required block and go no further. This is likely + an indication that something bad will be finalized in the network, which will never happen when approvals & disputes + are functioning correctly. Nevertheless we account for the case here. +3. If the required block is unfinalized and viable, then iterate over the viable leaves in descending order by score and + select the first one which contains the required block in its chain. Backwards iteration is a simple way to check + this, but if unfinalized chains grow long then Merkle Mountain-Ranges will most likely be more efficient. + +Once selecting a leaf, the chain should be constrained to the maximum of the required block or the highest +**finalizable** ancestor. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-disputes.md b/polkadot/roadmap/implementers-guide/src/protocol-disputes.md index ebbc534f199237d5fd56348b4f6b0f180d16866f..2a4082cc07f92aa99425e9e708ec2e90117c02c5 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-disputes.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-disputes.md @@ -4,14 +4,27 @@ Fast forward to [more detailed disputes requirements](./disputes-flow.md). ## Motivation and Background -All parachain blocks that end up in the finalized relay chain should be valid. This does not apply to blocks that are only backed, but not included. +All parachain blocks that end up in the finalized relay chain should be valid. This does not apply to blocks that are +only backed, but not included. We have two primary components for ensuring that nothing invalid ends up in the finalized relay chain: - * Approval Checking, as described [here](./protocol-approval.md) and implemented according to the [Approval Voting](node/approval/approval-voting.md) subsystem. This protocol can be shown to prevent invalid parachain blocks from making their way into the finalized relay chain as long as the amount of attempts are limited. - * Disputes, this protocol, which ensures that each attempt to include something bad is caught, and the offending validators are punished. -Disputes differ from backing and approval process (and can not be part of those) in that a dispute is independent of a particular fork, while both backing and approval operate on particular forks. This distinction is important! Approval voting stops, if an alternative fork which might not contain the currently approved candidate gets finalized. This is totally fine from the perspective of approval voting as its sole purpose is to make sure invalid blocks won't get finalized. For disputes on the other hand we have different requirements: Even though the "danger" is past and the adversaries were not able to get their invalid block approved, we still want them to get slashed for the attempt. Otherwise they just have been able to get a free try, but this is something we need to avoid in our security model, as it is based on the assumption that the probability of getting an invalid block finalized is very low and an attacker would get bankrupt before it could have tried often enough. - -Every dispute stems from a disagreement among two or more validators. If a bad actor creates a bad block, but the bad actor never distributes it to honest validators, then nobody will dispute it. Of course, such a situation is not even an attack on the network, so we don't need to worry about defending against it. + * Approval Checking, as described [here](./protocol-approval.md) and implemented according to the [Approval + Voting](node/approval/approval-voting.md) subsystem. This protocol can be shown to prevent invalid parachain blocks + from making their way into the finalized relay chain as long as the amount of attempts are limited. + * Disputes, this protocol, which ensures that each attempt to include something bad is caught, and the offending +validators are punished. Disputes differ from backing and approval process (and can not be part of those) in that a +dispute is independent of a particular fork, while both backing and approval operate on particular forks. This +distinction is important! Approval voting stops, if an alternative fork which might not contain the currently approved +candidate gets finalized. This is totally fine from the perspective of approval voting as its sole purpose is to make +sure invalid blocks won't get finalized. For disputes on the other hand we have different requirements: Even though the +"danger" is past and the adversaries were not able to get their invalid block approved, we still want them to get +slashed for the attempt. Otherwise they just have been able to get a free try, but this is something we need to avoid in +our security model, as it is based on the assumption that the probability of getting an invalid block finalized is very +low and an attacker would get bankrupt before it could have tried often enough. + +Every dispute stems from a disagreement among two or more validators. If a bad actor creates a bad block, but the bad +actor never distributes it to honest validators, then nobody will dispute it. Of course, such a situation is not even an +attack on the network, so we don't need to worry about defending against it. We are interested in identifying and deterring the following attack scenario: * A parablock included on a branch of the relay chain is bad @@ -20,48 +33,101 @@ We are also interested in identifying these additional scenarios: * A parablock backed on a branch of the relay chain is bad * A parablock seconded, but not backed on any branch of the relay chain, is bad. -Punishing misbehavior in the latter two scenarios doesn't effect our security guarantees and introduces substantial technical challenges as described in the `No Disputes for Non Included Candidates` section of [Dispute Coordinator](./node/disputes/dispute-coordinator.md). We therefore choose to punt on disputes in these cases, instead favoring the protocol simplicity resulting from only punishing in the first scenario. - -As covered in the [protocol overview](./protocol-overview.md), checking a parachain block requires 3 pieces of data: the parachain validation code, the [`AvailableData`](types/availability.md), and the [`CandidateReceipt`](types/candidate.md). The validation code is available on-chain, and published ahead of time, so that no two branches of the relay chain have diverging views of the validation code for a given parachain. Note that only for the first scenario, where the parablock has been included on a branch of the relay chain, is the data necessarily available. Thus, dispute processes should begin with an availability process to ensure availability of the `AvailableData`. This availability process will conclude quickly if the data is already available. If the data is not already available, then the initiator of the dispute must make it available. - -Disputes have both an on-chain and an off-chain component. Slashing and punishment is handled on-chain, so votes by validators on either side of the dispute must be placed on-chain. Furthermore, a dispute on one branch of the relay chain should be transposed to all other active branches of the relay chain. The fact that slashing occurs _in all histories_ is crucial for deterring attempts to attack the network. The attacker should not be able to escape with their funds because the network has moved on to another branch of the relay chain where no attack was attempted. - -In fact, this is why we introduce a distinction between _local_ and _remote_ disputes. We categorize disputes as either local or remote relative to any particular branch of the relay chain. Local disputes are about dealing with our first scenario, where a parablock has been included on the specific branch we are looking at. In these cases, the chain is corrupted all the way back to the point where the parablock was backed and must be discarded. However, as mentioned before, the dispute must propagate to all other branches of the relay chain. All other disputes are considered _remote_. For the on-chain component, when handling a dispute for a block which was not included in the current fork of the relay chain, it is impossible to discern between our attack scenarios. It is possible that the parablock was included somewhere, or backed somewhere, or wasn't backed anywhere. The on-chain component for handling these cases will be the same. +Punishing misbehavior in the latter two scenarios doesn't effect our security guarantees and introduces substantial +technical challenges as described in the `No Disputes for Non Included Candidates` section of [Dispute +Coordinator](./node/disputes/dispute-coordinator.md). We therefore choose to punt on disputes in these cases, instead +favoring the protocol simplicity resulting from only punishing in the first scenario. + +As covered in the [protocol overview](./protocol-overview.md), checking a parachain block requires 3 pieces of data: the +parachain validation code, the [`AvailableData`](types/availability.md), and the +[`CandidateReceipt`](types/candidate.md). The validation code is available on-chain, and published ahead of time, so +that no two branches of the relay chain have diverging views of the validation code for a given parachain. Note that +only for the first scenario, where the parablock has been included on a branch of the relay chain, is the data +necessarily available. Thus, dispute processes should begin with an availability process to ensure availability of the +`AvailableData`. This availability process will conclude quickly if the data is already available. If the data is not +already available, then the initiator of the dispute must make it available. + +Disputes have both an on-chain and an off-chain component. Slashing and punishment is handled on-chain, so votes by +validators on either side of the dispute must be placed on-chain. Furthermore, a dispute on one branch of the relay +chain should be transposed to all other active branches of the relay chain. The fact that slashing occurs _in all +histories_ is crucial for deterring attempts to attack the network. The attacker should not be able to escape with their +funds because the network has moved on to another branch of the relay chain where no attack was attempted. + +In fact, this is why we introduce a distinction between _local_ and _remote_ disputes. We categorize disputes as either +local or remote relative to any particular branch of the relay chain. Local disputes are about dealing with our first +scenario, where a parablock has been included on the specific branch we are looking at. In these cases, the chain is +corrupted all the way back to the point where the parablock was backed and must be discarded. However, as mentioned +before, the dispute must propagate to all other branches of the relay chain. All other disputes are considered _remote_. +For the on-chain component, when handling a dispute for a block which was not included in the current fork of the relay +chain, it is impossible to discern between our attack scenarios. It is possible that the parablock was included +somewhere, or backed somewhere, or wasn't backed anywhere. The on-chain component for handling these cases will be the +same. ## Initiation -Disputes are initiated by any validator who finds their opinion on the validity of a parablock in opposition to another issued statement. As all statements currently gathered by the relay chain imply validity, disputes will be initiated only by nodes which perceive that the parablock is bad. - -The initiation of a dispute begins off-chain. A validator signs a message indicating that it disputes the validity of the parablock and notifies all other validators, off-chain, of all of the statements it is aware of for the disputed parablock. These may be backing statements or approval-checking statements. It is worth noting that there is no special message type for initiating a dispute. It is the same message as is used to participate in a dispute and vote negatively. As such, there is no consensus required on who initiated a dispute, only on the fact that there is a dispute in-progress. - -In practice, the initiator of a dispute will be either one of the backers or one of the approval checkers for the parablock. If the result of execution is found to be invalid, the validator will initiate the dispute as described above. Furthermore, if the dispute occurs during the backing phase, the initiator must make the data available to other validators. If the dispute occurs during approval checking, the data is already available. - -Lastly, it is possible that for backing disputes, i.e. where the data is not already available among all validators, that an adversary may DoS the few parties who are checking the block to prevent them from distributing the data to other validators participating in the dispute process. Note that this can only occur pre-inclusion for any given parablock, so the downside of this attack is small and it is not security-critical to address these cases. However, we assume that the adversary can only prevent the validator from issuing messages for a limited amount of time. We also assume that there is a side-channel where the relay chain's governance mechanisms can trigger disputes by providing the full PoV and candidate receipt on-chain manually. +Disputes are initiated by any validator who finds their opinion on the validity of a parablock in opposition to another +issued statement. As all statements currently gathered by the relay chain imply validity, disputes will be initiated +only by nodes which perceive that the parablock is bad. + +The initiation of a dispute begins off-chain. A validator signs a message indicating that it disputes the validity of +the parablock and notifies all other validators, off-chain, of all of the statements it is aware of for the disputed +parablock. These may be backing statements or approval-checking statements. It is worth noting that there is no special +message type for initiating a dispute. It is the same message as is used to participate in a dispute and vote +negatively. As such, there is no consensus required on who initiated a dispute, only on the fact that there is a dispute +in-progress. + +In practice, the initiator of a dispute will be either one of the backers or one of the approval checkers for the +parablock. If the result of execution is found to be invalid, the validator will initiate the dispute as described +above. Furthermore, if the dispute occurs during the backing phase, the initiator must make the data available to other +validators. If the dispute occurs during approval checking, the data is already available. + +Lastly, it is possible that for backing disputes, i.e. where the data is not already available among all validators, +that an adversary may DoS the few parties who are checking the block to prevent them from distributing the data to other +validators participating in the dispute process. Note that this can only occur pre-inclusion for any given parablock, so +the downside of this attack is small and it is not security-critical to address these cases. However, we assume that the +adversary can only prevent the validator from issuing messages for a limited amount of time. We also assume that there +is a side-channel where the relay chain's governance mechanisms can trigger disputes by providing the full PoV and +candidate receipt on-chain manually. ## Dispute Participation -Once becoming aware of a dispute, it is the responsibility of all validators to participate in the dispute. Concretely, this means: - * Circulate all statements about the candidate that we are aware of - backing statements, approval checking statements, and dispute statements. +Once becoming aware of a dispute, it is the responsibility of all validators to participate in the dispute. Concretely, +this means: + * Circulate all statements about the candidate that we are aware of - backing statements, approval checking + statements, and dispute statements. * If we have already issued any type of statement about the candidate, go no further. - * Download the [`AvailableData`](types/availability.md). If possible, this should first be attempted from other dispute participants or backing validators, and then [(via erasure-coding)](node/availability/availability-recovery.md) from all validators. - * Extract the Validation Code from any recent relay chain block. Code is guaranteed to be kept available on-chain, so we don't need to download any particular fork of the chain. - * Execute the block under the validation code, using the `AvailableData`, and check that all outputs are correct, including the `erasure-root` of the [`CandidateReceipt`](types/candidate.md). + * Download the [`AvailableData`](types/availability.md). If possible, this should first be attempted from other + dispute participants or backing validators, and then [(via + erasure-coding)](node/availability/availability-recovery.md) from all validators. + * Extract the Validation Code from any recent relay chain block. Code is guaranteed to be kept available on-chain, so + we don't need to download any particular fork of the chain. + * Execute the block under the validation code, using the `AvailableData`, and check that all outputs are correct, + including the `erasure-root` of the [`CandidateReceipt`](types/candidate.md). * Issue a dispute participation statement to the effect of the validity of the candidate block. Disputes _conclude_ after ⅔ supermajority is reached in either direction. -The on-chain component of disputes can be initiated by providing any two conflicting votes and it also waits for a ⅔ supermajority on either side. The on-chain component also tracks which parablocks have already been disputed so the same parablock may only be disputed once on any particular branch of the relay chain. Lastly, it also tracks which blocks have been included on the current branch of the relay chain. When a dispute is initiated for a para, inclusion is halted for the para until the dispute concludes. +The on-chain component of disputes can be initiated by providing any two conflicting votes and it also waits for a ⅔ +supermajority on either side. The on-chain component also tracks which parablocks have already been disputed so the same +parablock may only be disputed once on any particular branch of the relay chain. Lastly, it also tracks which blocks +have been included on the current branch of the relay chain. When a dispute is initiated for a para, inclusion is halted +for the para until the dispute concludes. -The author of a relay chain block should initiate the on-chain component of disputes for all disputes which the chain is not aware of, and provide all statements to the on-chain component as well. This should all be done via _inherents_. +The author of a relay chain block should initiate the on-chain component of disputes for all disputes which the chain is +not aware of, and provide all statements to the on-chain component as well. This should all be done via _inherents_. Validators can learn about dispute statements in two ways: * Receiving them from other validators over gossip - * Scraping them from imported blocks of the relay chain. This is also used for validators to track other types of statements, such as backing statements. + * Scraping them from imported blocks of the relay chain. This is also used for validators to track other types of + statements, such as backing statements. -Validators are rewarded for providing statements to the chain as well as for participating in the dispute, on either side. However, the losing side of the dispute is slashed. +Validators are rewarded for providing statements to the chain as well as for participating in the dispute, on either +side. However, the losing side of the dispute is slashed. ## Dispute Conclusion -Disputes, roughly, are over when one side reaches a ⅔ supermajority. They may also never conclude without either side witnessing supermajority, which will only happen if the majority of validators are unable to vote for some reason. Furthermore, disputes on-chain will stay open for some fixed amount of time even after concluding, to accept new votes. +Disputes, roughly, are over when one side reaches a ⅔ supermajority. They may also never conclude without either side +witnessing supermajority, which will only happen if the majority of validators are unable to vote for some reason. +Furthermore, disputes on-chain will stay open for some fixed amount of time even after concluding, to accept new votes. Late votes, after the dispute already reached a ⅔ supermajority, must be rewarded (albeit a smaller amount) as well. diff --git a/polkadot/roadmap/implementers-guide/src/protocol-overview.md b/polkadot/roadmap/implementers-guide/src/protocol-overview.md index fa5a866e6121b4e1607332f7c9bb5ffb795ed658..96827e8c06bbed580f91f1fb2adb45f9bb8e4fb1 100644 --- a/polkadot/roadmap/implementers-guide/src/protocol-overview.md +++ b/polkadot/roadmap/implementers-guide/src/protocol-overview.md @@ -1,28 +1,61 @@ # Protocol Overview -This section aims to describe, at a high level, the actors and protocols involved in running parachains in Polkadot. Specifically, we describe how different actors communicate with each other, what data structures they keep both individually and collectively, and the high-level purpose on why they do these things. +This section aims to describe, at a high level, the actors and protocols involved in running parachains in Polkadot. +Specifically, we describe how different actors communicate with each other, what data structures they keep both +individually and collectively, and the high-level purpose on why they do these things. -Our top-level goal is to carry a parachain block from authoring to secure inclusion, and define a process which can be carried out repeatedly and in parallel for many different parachains to extend them over time. Understanding of the high-level approach taken here is important to provide context for the proposed architecture further on. The key parts of Polkadot relevant to this are the main Polkadot blockchain, known as the relay-chain, and the actors which provide security and inputs to this blockchain. +Our top-level goal is to carry a parachain block from authoring to secure inclusion, and define a process which can be +carried out repeatedly and in parallel for many different parachains to extend them over time. Understanding of the +high-level approach taken here is important to provide context for the proposed architecture further on. The key parts +of Polkadot relevant to this are the main Polkadot blockchain, known as the relay-chain, and the actors which provide +security and inputs to this blockchain. First, it's important to go over the main actors we have involved in this protocol. -1. Validators. These nodes are responsible for validating proposed parachain blocks. They do so by checking a Proof-of-Validity (PoV) of the block and ensuring that the PoV remains available. They put financial capital down as "skin in the game" which can be slashed (destroyed) if they are proven to have misvalidated. -1. Collators. These nodes are responsible for creating the Proofs-of-Validity that validators know how to check. Creating a PoV typically requires familiarity with the transaction format and block authoring rules of the parachain, as well as having access to the full state of the parachain. - -This implies a simple pipeline where collators send validators parachain blocks and their requisite PoV to check. Then, validators validate the block using the PoV, signing statements which describe either the positive or negative outcome, and with enough positive statements, the block can be noted on the relay-chain. Negative statements are not a veto but will lead to a dispute, with those on the wrong side being slashed. If another validator later detects that a validator or group of validators incorrectly signed a statement claiming a block was valid, then those validators will be _slashed_, with the checker receiving a bounty. - -However, there is a problem with this formulation. In order for another validator to check the previous group of validators' work after the fact, the PoV must remain _available_ so the other validator can fetch it in order to check the work. The PoVs are expected to be too large to include in the blockchain directly, so we require an alternate _data availability_ scheme which requires validators to prove that the inputs to their work will remain available, and so their work can be checked. Empirical tests tell us that many PoVs may be between 1 and 10MB during periods of heavy load. - -Here is a description of the Inclusion Pipeline: the path a parachain block (or parablock, for short) takes from creation to inclusion: +1. Validators. These nodes are responsible for validating proposed parachain blocks. They do so by checking a + Proof-of-Validity (PoV) of the block and ensuring that the PoV remains available. They put financial capital down as + "skin in the game" which can be slashed (destroyed) if they are proven to have misvalidated. +1. Collators. These nodes are responsible for creating the Proofs-of-Validity that validators know how to check. + Creating a PoV typically requires familiarity with the transaction format and block authoring rules of the parachain, + as well as having access to the full state of the parachain. + +This implies a simple pipeline where collators send validators parachain blocks and their requisite PoV to check. Then, +validators validate the block using the PoV, signing statements which describe either the positive or negative outcome, +and with enough positive statements, the block can be noted on the relay-chain. Negative statements are not a veto but +will lead to a dispute, with those on the wrong side being slashed. If another validator later detects that a validator +or group of validators incorrectly signed a statement claiming a block was valid, then those validators will be +_slashed_, with the checker receiving a bounty. + +However, there is a problem with this formulation. In order for another validator to check the previous group of +validators' work after the fact, the PoV must remain _available_ so the other validator can fetch it in order to check +the work. The PoVs are expected to be too large to include in the blockchain directly, so we require an alternate _data +availability_ scheme which requires validators to prove that the inputs to their work will remain available, and so +their work can be checked. Empirical tests tell us that many PoVs may be between 1 and 10MB during periods of heavy +load. + +Here is a description of the Inclusion Pipeline: the path a parachain block (or parablock, for short) takes from +creation to inclusion: 1. Validators are selected and assigned to parachains by the Validator Assignment routine. -1. A collator produces the parachain block, which is known as a parachain candidate or candidate, along with a PoV for the candidate. -1. The collator forwards the candidate and PoV to validators assigned to the same parachain via the [Collator Protocol](node/collators/collator-protocol.md). -1. The validators assigned to a parachain at a given point in time participate in the [Candidate Backing subsystem](node/backing/candidate-backing.md) to validate candidates that were put forward for validation. Candidates which gather enough signed validity statements from validators are considered "backable". Their backing is the set of signed validity statements. -1. A relay-chain block author, selected by BABE, can note up to one (1) backable candidate for each parachain to include in the relay-chain block alongside its backing. A backable candidate once included in the relay-chain is considered backed in that fork of the relay-chain. -1. Once backed in the relay-chain, the parachain candidate is considered to be "pending availability". It is not considered to be included as part of the parachain until it is proven available. -1. In the following relay-chain blocks, validators will participate in the [Availability Distribution subsystem](node/availability/availability-distribution.md) to ensure availability of the candidate. Information regarding the availability of the candidate will be noted in the subsequent relay-chain blocks. -1. Once the relay-chain state machine has enough information to consider the candidate's PoV as being available, the candidate is considered to be part of the parachain and is graduated to being a full parachain block, or parablock for short. +1. A collator produces the parachain block, which is known as a parachain candidate or candidate, along with a PoV for + the candidate. +1. The collator forwards the candidate and PoV to validators assigned to the same parachain via the [Collator + Protocol](node/collators/collator-protocol.md). +1. The validators assigned to a parachain at a given point in time participate in the [Candidate Backing + subsystem](node/backing/candidate-backing.md) to validate candidates that were put forward for validation. Candidates + which gather enough signed validity statements from validators are considered "backable". Their backing is the set of + signed validity statements. +1. A relay-chain block author, selected by BABE, can note up to one (1) backable candidate for each parachain to include + in the relay-chain block alongside its backing. A backable candidate once included in the relay-chain is considered + backed in that fork of the relay-chain. +1. Once backed in the relay-chain, the parachain candidate is considered to be "pending availability". It is not + considered to be included as part of the parachain until it is proven available. +1. In the following relay-chain blocks, validators will participate in the [Availability Distribution + subsystem](node/availability/availability-distribution.md) to ensure availability of the candidate. Information + regarding the availability of the candidate will be noted in the subsequent relay-chain blocks. +1. Once the relay-chain state machine has enough information to consider the candidate's PoV as being available, the + candidate is considered to be part of the parachain and is graduated to being a full parachain block, or parablock + for short. Note that the candidate can fail to be included in any of the following ways: @@ -31,21 +64,47 @@ Note that the candidate can fail to be included in any of the following ways: - The candidate is not selected by a relay-chain block author to be included in the relay chain - The candidate's PoV is not considered as available within a timeout and is discarded from the relay chain. -This process can be divided further down. Steps 2 & 3 relate to the work of the collator in collating and distributing the candidate to validators via the Collation Distribution Subsystem. Steps 3 & 4 relate to the work of the validators in the Candidate Backing Subsystem and the block author (itself a validator) to include the block into the relay chain. Steps 6, 7, and 8 correspond to the logic of the relay-chain state-machine (otherwise known as the Runtime) used to fully incorporate the block into the chain. Step 7 requires further work on the validators' parts to participate in the Availability Distribution Subsystem and include that information into the relay chain for step 8 to be fully realized. - -This brings us to the second part of the process. Once a parablock is considered available and part of the parachain, it is still "pending approval". At this stage in the pipeline, the parablock has been backed by a majority of validators in the group assigned to that parachain, and its data has been guaranteed available by the set of validators as a whole. Once it's considered available, the host will even begin to accept children of that block. At this point, we can consider the parablock as having been tentatively included in the parachain, although more confirmations are desired. However, the validators in the parachain-group (known as the "Parachain Validators" for that parachain) are sampled from a validator set which contains some proportion of byzantine, or arbitrarily malicious members. This implies that the Parachain Validators for some parachain may be majority-dishonest, which means that (secondary) approval checks must be done on the block before it can be considered approved. This is necessary only because the Parachain Validators for a given parachain are sampled from an overall validator set which is assumed to be up to <1/3 dishonest - meaning that there is a chance to randomly sample Parachain Validators for a parachain that are majority or fully dishonest and can back a candidate wrongly. The Approval Process allows us to detect such misbehavior after-the-fact without allocating more Parachain Validators and reducing the throughput of the system. A parablock's failure to pass the approval process will invalidate the block as well as all of its descendants. However, only the validators who backed the block in question will be slashed, not the validators who backed the descendants. +This process can be divided further down. Steps 2 & 3 relate to the work of the collator in collating and distributing +the candidate to validators via the Collation Distribution Subsystem. Steps 3 & 4 relate to the work of the validators +in the Candidate Backing Subsystem and the block author (itself a validator) to include the block into the relay chain. +Steps 6, 7, and 8 correspond to the logic of the relay-chain state-machine (otherwise known as the Runtime) used to +fully incorporate the block into the chain. Step 7 requires further work on the validators' parts to participate in the +Availability Distribution Subsystem and include that information into the relay chain for step 8 to be fully realized. + +This brings us to the second part of the process. Once a parablock is considered available and part of the parachain, it +is still "pending approval". At this stage in the pipeline, the parablock has been backed by a majority of validators in +the group assigned to that parachain, and its data has been guaranteed available by the set of validators as a whole. +Once it's considered available, the host will even begin to accept children of that block. At this point, we can +consider the parablock as having been tentatively included in the parachain, although more confirmations are desired. +However, the validators in the parachain-group (known as the "Parachain Validators" for that parachain) are sampled from +a validator set which contains some proportion of byzantine, or arbitrarily malicious members. This implies that the +Parachain Validators for some parachain may be majority-dishonest, which means that (secondary) approval checks must be +done on the block before it can be considered approved. This is necessary only because the Parachain Validators for a +given parachain are sampled from an overall validator set which is assumed to be up to <1/3 dishonest - meaning that +there is a chance to randomly sample Parachain Validators for a parachain that are majority or fully dishonest and can +back a candidate wrongly. The Approval Process allows us to detect such misbehavior after-the-fact without allocating +more Parachain Validators and reducing the throughput of the system. A parablock's failure to pass the approval process +will invalidate the block as well as all of its descendants. However, only the validators who backed the block in +question will be slashed, not the validators who backed the descendants. The Approval Process, at a glance, looks like this: -1. Parablocks that have been included by the Inclusion Pipeline are pending approval for a time-window known as the secondary checking window. +1. Parablocks that have been included by the Inclusion Pipeline are pending approval for a time-window known as the + secondary checking window. 1. During the secondary-checking window, validators randomly self-select to perform secondary checks on the parablock. -1. These validators, known in this context as secondary checkers, acquire the parablock and its PoV, and re-run the validation function. -1. The secondary checkers gossip the result of their checks. Contradictory results lead to escalation, where all validators are required to check the block. The validators on the losing side of the dispute are slashed. -1. At the end of the Approval Process, the parablock is either Approved or it is rejected. More on the rejection process later. +1. These validators, known in this context as secondary checkers, acquire the parablock and its PoV, and re-run the + validation function. +1. The secondary checkers gossip the result of their checks. Contradictory results lead to escalation, where all + validators are required to check the block. The validators on the losing side of the dispute are slashed. +1. At the end of the Approval Process, the parablock is either Approved or it is rejected. More on the rejection process + later. -More information on the Approval Process can be found in the dedicated section on [Approval](protocol-approval.md). More information on Disputes can be found in the dedicated section on [Disputes](protocol-disputes.md). +More information on the Approval Process can be found in the dedicated section on [Approval](protocol-approval.md). More +information on Disputes can be found in the dedicated section on [Disputes](protocol-disputes.md). -These two pipelines sum up the sequence of events necessary to extend and acquire full security on a Parablock. Note that the Inclusion Pipeline must conclude for a specific parachain before a new block can be accepted on that parachain. After inclusion, the Approval Process kicks off, and can be running for many parachain blocks at once. +These two pipelines sum up the sequence of events necessary to extend and acquire full security on a Parablock. Note +that the Inclusion Pipeline must conclude for a specific parachain before a new block can be accepted on that parachain. +After inclusion, the Approval Process kicks off, and can be running for many parachain blocks at once. Reiterating the lifecycle of a candidate: @@ -129,8 +188,11 @@ digraph { The diagram above shows the happy path of a block from (1) Candidate to the (7) Approved state. -It is also important to take note of the fact that the relay-chain is extended by BABE, which is a forkful algorithm. That means that different block authors can be chosen at the same time, and may not be building on the same block parent. Furthermore, the set of validators is not fixed, nor is the set of parachains. And even with the same set of validators and parachains, the validators' assignments to parachains is flexible. This means that the architecture proposed in the next chapters must deal with the variability and multiplicity of the network state. - +It is also important to take note of the fact that the relay-chain is extended by BABE, which is a forkful algorithm. +That means that different block authors can be chosen at the same time, and may not be building on the same block +parent. Furthermore, the set of validators is not fixed, nor is the set of parachains. And even with the same set of +validators and parachains, the validators' assignments to parachains is flexible. This means that the architecture +proposed in the next chapters must deal with the variability and multiplicity of the network state. ```dot process digraph { @@ -169,7 +231,9 @@ digraph { } ``` -In this example, group 1 has received block C while the others have not due to network asynchrony. Now, a validator from group 2 may be able to build another block on top of B, called `C'`. Assume that afterwards, some validators become aware of both C and `C'`, while others remain only aware of one. +In this example, group 1 has received block C while the others have not due to network asynchrony. Now, a validator from +group 2 may be able to build another block on top of B, called `C'`. Assume that afterwards, some validators become +aware of both C and `C'`, while others remain only aware of one. ```dot process digraph { @@ -216,4 +280,7 @@ digraph { } ``` -Those validators that are aware of many competing heads must be aware of the work happening on each one. They may contribute to some or a full extent on both. It is possible that due to network asynchrony two forks may grow in parallel for some time, although in the absence of an adversarial network this is unlikely in the case where there are validators who are aware of both chain heads. +Those validators that are aware of many competing heads must be aware of the work happening on each one. They may +contribute to some or a full extent on both. It is possible that due to network asynchrony two forks may grow in +parallel for some time, although in the absence of an adversarial network this is unlikely in the case where there are +validators who are aware of both chain heads. diff --git a/polkadot/roadmap/implementers-guide/src/pvf-prechecking.md b/polkadot/roadmap/implementers-guide/src/pvf-prechecking.md index 91cc8f9b6a20d72912704fe943420037b22144cf..44ddb0abbe08bff1972c7a7433f88a330e98f103 100644 --- a/polkadot/roadmap/implementers-guide/src/pvf-prechecking.md +++ b/polkadot/roadmap/implementers-guide/src/pvf-prechecking.md @@ -2,20 +2,28 @@ ## Motivation -Parachains' validation function is described by a wasm module that we refer to as a PVF. Since a PVF is a wasm module the typical way of executing it is to compile it to machine code. +Parachains' validation function is described by a wasm module that we refer to as a PVF. Since a PVF is a wasm module +the typical way of executing it is to compile it to machine code. -Typically an optimizing compiler consists of algorithms that are able to optimize the resulting machine code heavily. However, while those algorithms perform quite well for a typical wasm code produced by standard toolchains (e.g. rustc/LLVM), those algorithms can be abused to consume a lot of resources. Moreover, since those algorithms are rather complex there is a lot of room for a bug that can crash the compiler. +Typically an optimizing compiler consists of algorithms that are able to optimize the resulting machine code heavily. +However, while those algorithms perform quite well for a typical wasm code produced by standard toolchains (e.g. +rustc/LLVM), those algorithms can be abused to consume a lot of resources. Moreover, since those algorithms are rather +complex there is a lot of room for a bug that can crash the compiler. -If compilation of a Parachain Validation Function (PVF) takes too long or uses too much memory, this can leave a node in limbo as to whether a candidate of that parachain is valid or not. +If compilation of a Parachain Validation Function (PVF) takes too long or uses too much memory, this can leave a node in +limbo as to whether a candidate of that parachain is valid or not. -The amount of time that a PVF takes to compile is a subjective resource limit and as such PVFs may be maliciously crafted so that there is e.g. a 50/50 split of validators which can and cannot compile and execute the PVF. +The amount of time that a PVF takes to compile is a subjective resource limit and as such PVFs may be maliciously +crafted so that there is e.g. a 50/50 split of validators which can and cannot compile and execute the PVF. This has the following implications: - In backing, inclusion may be slow due to backing groups being unable to execute the block - In approval checking, there may be many no-shows, leading to slow finality -- In disputes, neither side may reach supermajority. Nobody will get slashed and the chain will not be reverted or finalized. +- In disputes, neither side may reach supermajority. Nobody will get slashed and the chain will not be reverted or + finalized. -As a result of this issue we need a fairly hard guarantee that the PVFs of registered parachains/threads can be compiled within a reasonable amount of time. +As a result of this issue we need a fairly hard guarantee that the PVFs of registered parachains/threads can be compiled +within a reasonable amount of time. ## Solution @@ -23,9 +31,12 @@ The problem is solved by having a pre-checking process. ### Pre-checking -Pre-checking mostly consists of attempting to prepare (compile) the PVF WASM blob. We use more strict limits (e.g. timeouts) here compared to regular preparation for execution. This way errors during preparation later are likely unrelated to the PVF itself, as it already passed pre-checking. We can treat such errors as local node issues. +Pre-checking mostly consists of attempting to prepare (compile) the PVF WASM blob. We use more strict limits (e.g. +timeouts) here compared to regular preparation for execution. This way errors during preparation later are likely +unrelated to the PVF itself, as it already passed pre-checking. We can treat such errors as local node issues. -We also have an additional step where we attempt to instantiate the WASM runtime without running it. This is unrelated to preparation so we don't time it, but it does help us catch more issues. +We also have an additional step where we attempt to instantiate the WASM runtime without running it. This is unrelated +to preparation so we don't time it, but it does help us catch more issues. ### Protocol @@ -34,27 +45,41 @@ Pre-checking is run when a new validation code is included in the chain. A new P - A new parachain is registered. - An existing parachain signalled an upgrade of its validation code. -Before any of those operations finish, the PVF pre-checking vote is initiated. The PVF pre-checking vote is identified by the PVF code hash that is being voted on. If there is already PVF pre-checking process running, then no -new PVF pre-checking vote will be started. Instead, the operation just subscribes to the existing vote. +Before any of those operations finish, the PVF pre-checking vote is initiated. The PVF pre-checking vote is identified +by the PVF code hash that is being voted on. If there is already PVF pre-checking process running, then no new PVF +pre-checking vote will be started. Instead, the operation just subscribes to the existing vote. -The pre-checking vote can be concluded either by obtaining a threshold of votes for a decision, or if it expires. The threshold to accept is a supermajority of 2/3 of validators. We reject once a supermajority is no longer possible. +The pre-checking vote can be concluded either by obtaining a threshold of votes for a decision, or if it expires. The +threshold to accept is a supermajority of 2/3 of validators. We reject once a supermajority is no longer possible. -Each validator checks the list of PVFs available for voting. The vote is binary, i.e. accept or reject a given PVF. As soon as the threshold of votes are collected for one of the sides of the vote, the voting is concluded in that direction and the effects of the voting are enacted. +Each validator checks the list of PVFs available for voting. The vote is binary, i.e. accept or reject a given PVF. As +soon as the threshold of votes are collected for one of the sides of the vote, the voting is concluded in that direction +and the effects of the voting are enacted. -Only validators from the active set can participate in the vote. The set of active validators can change each session. That's why we reset the votes each session. A voting that observed a certain number of sessions will be rejected. +Only validators from the active set can participate in the vote. The set of active validators can change each session. +That's why we reset the votes each session. A voting that observed a certain number of sessions will be rejected. The effects of the PVF accepting depend on the operations requested it: -1. All onboardings subscribed to the approved PVF pre-checking process will get scheduled and after passing 2 session boundaries they will be onboarded. -1. All upgrades subscribed to the approved PVF pre-checking process will get scheduled very similarly to the existing process. Upgrades with pre-checking are really the same process that is just delayed by the time required for pre-checking voting. In case of instant approval the mechanism is exactly the same. +1. All onboardings subscribed to the approved PVF pre-checking process will get scheduled and after passing 2 session + boundaries they will be onboarded. +1. All upgrades subscribed to the approved PVF pre-checking process will get scheduled very similarly to the existing + process. Upgrades with pre-checking are really the same process that is just delayed by the time required for + pre-checking voting. In case of instant approval the mechanism is exactly the same. -In case PVF pre-checking process was concluded with rejection, then all the operations that are subscribed to the rejected PVF pre-checking process will be processed as follows. That is, onboarding or upgrading will be cancelled. +In case PVF pre-checking process was concluded with rejection, then all the operations that are subscribed to the +rejected PVF pre-checking process will be processed as follows. That is, onboarding or upgrading will be cancelled. The logic described above is implemented by the [paras] module. ### Subsystem -On the node-side, there is a PVF pre-checking [subsystem][pvf-prechecker-subsystem] that scans the chain for new PVFs via using [runtime APIs][pvf-runtime-api]. Upon finding a new PVF, the subsystem will initiate a PVF pre-checking request and wait for the result. Whenever the result is obtained, the subsystem will use the [runtime API][pvf-runtime-api] to submit a vote for the PVF. The vote is an unsigned transaction. The vote will be distributed via the gossip similarly to a normal transaction. Eventually a block producer will include the vote into the block where it will be handled by the [runtime][paras]. +On the node-side, there is a PVF pre-checking [subsystem][pvf-prechecker-subsystem] that scans the chain for new PVFs +via using [runtime APIs][pvf-runtime-api]. Upon finding a new PVF, the subsystem will initiate a PVF pre-checking +request and wait for the result. Whenever the result is obtained, the subsystem will use the [runtime +API][pvf-runtime-api] to submit a vote for the PVF. The vote is an unsigned transaction. The vote will be distributed +via the gossip similarly to a normal transaction. Eventually a block producer will include the vote into the block where +it will be handled by the [runtime][paras]. ## Summary @@ -62,11 +87,15 @@ Parachains' validation function is described by a wasm module that we refer to a In order to make the PVF usable for candidate validation it has to be registered on-chain. -As part of the registration process, it has to go through pre-checking. Pre-checking is a game of attempting preparation and additional checks, and reporting the results back on-chain. +As part of the registration process, it has to go through pre-checking. Pre-checking is a game of attempting preparation +and additional checks, and reporting the results back on-chain. -We define preparation as a process that: validates the consistency of the wasm binary (aka prevalidation) and the compilation of the wasm module into machine code (referred to as an artifact). +We define preparation as a process that: validates the consistency of the wasm binary (aka prevalidation) and the +compilation of the wasm module into machine code (referred to as an artifact). -Besides pre-checking, preparation can also be triggered by execution, since a compiled artifact is needed for the execution. If an artifact already exists, execution will skip preparation. If it does do preparation, execution uses a more lenient timeout than preparation, to avoid the situation where honest validators fail on valid, pre-checked PVFs. +Besides pre-checking, preparation can also be triggered by execution, since a compiled artifact is needed for the +execution. If an artifact already exists, execution will skip preparation. If it does do preparation, execution uses a +more lenient timeout than preparation, to avoid the situation where honest validators fail on valid, pre-checked PVFs. [paras]: runtime/paras.md [pvf-runtime-api]: runtime-api/pvf-prechecking.md diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/README.md b/polkadot/roadmap/implementers-guide/src/runtime-api/README.md index 740ffd38ccee831a58e7fda2e7d1d64171b46856..d74100f2e262c91e27cd317de65d6408db8c20eb 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/README.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/README.md @@ -2,9 +2,16 @@ Runtime APIs are the means by which the node-side code extracts information from the state of the runtime. -Every block in the relay-chain contains a *state root* which is the root hash of a state trie encapsulating all storage of runtime modules after execution of the block. This is a cryptographic commitment to a unique state. We use the terminology of accessing the *state at* a block to refer accessing the state referred to by the state root of that block. +Every block in the relay-chain contains a *state root* which is the root hash of a state trie encapsulating all storage +of runtime modules after execution of the block. This is a cryptographic commitment to a unique state. We use the +terminology of accessing the *state at* a block to refer accessing the state referred to by the state root of that +block. -Although Runtime APIs are often used for simple storage access, they are actually empowered to do arbitrary computation. The implementation of the Runtime APIs lives within the Runtime as Wasm code and exposes `extern` functions that can be invoked with arguments and have a return value. Runtime APIs have access to a variety of host functions, which are contextual functions provided by the Wasm execution context, that allow it to carry out many different types of behaviors. +Although Runtime APIs are often used for simple storage access, they are actually empowered to do arbitrary computation. +The implementation of the Runtime APIs lives within the Runtime as Wasm code and exposes `extern` functions that can be +invoked with arguments and have a return value. Runtime APIs have access to a variety of host functions, which are +contextual functions provided by the Wasm execution context, that allow it to carry out many different types of +behaviors. Abilities provided by host functions includes: @@ -14,16 +21,25 @@ Abilities provided by host functions includes: * Optimized versions of cryptographic functions * More -So it is clear that Runtime APIs are a versatile and powerful tool to leverage the state of the chain. In general, we will use Runtime APIs for these purposes: +So it is clear that Runtime APIs are a versatile and powerful tool to leverage the state of the chain. In general, we +will use Runtime APIs for these purposes: * Access of a storage item * Access of a bundle of related storage items * Deriving a value from storage based on arguments * Submitting misbehavior reports -More broadly, we have the goal of using Runtime APIs to write Node-side code that fulfills the requirements set by the Runtime. In particular, the constraints set forth by the [Scheduler](../runtime/scheduler.md) and [Inclusion](../runtime/inclusion.md) modules. These modules are responsible for advancing paras with a two-phase protocol where validators are first chosen to validate and back a candidate and then required to ensure availability of referenced data. In the second phase, validators are meant to attest to those para-candidates that they have their availability chunk for. As the Node-side code needs to generate the inputs into these two phases, the runtime API needs to transmit information from the runtime that is aware of the Availability Cores model instantiated by the Scheduler and Inclusion modules. +More broadly, we have the goal of using Runtime APIs to write Node-side code that fulfills the requirements set by the +Runtime. In particular, the constraints set forth by the [Scheduler](../runtime/scheduler.md) and +[Inclusion](../runtime/inclusion.md) modules. These modules are responsible for advancing paras with a two-phase +protocol where validators are first chosen to validate and back a candidate and then required to ensure availability of +referenced data. In the second phase, validators are meant to attest to those para-candidates that they have their +availability chunk for. As the Node-side code needs to generate the inputs into these two phases, the runtime API needs +to transmit information from the runtime that is aware of the Availability Cores model instantiated by the Scheduler and +Inclusion modules. -Node-side code is also responsible for detecting and reporting misbehavior performed by other validators, and the set of Runtime APIs needs to provide methods for observing live disputes and submitting reports as transactions. +Node-side code is also responsible for detecting and reporting misbehavior performed by other validators, and the set of +Runtime APIs needs to provide methods for observing live disputes and submitting reports as transactions. The next sections will contain information on specific runtime APIs. The format is this: @@ -38,9 +54,16 @@ The next sections will contain information on specific runtime APIs. The format fn some_runtime_api(at: Block, arg1: Type1, arg2: Type2, ...) -> ReturnValue; ``` -Certain runtime APIs concerning the state of a para require the caller to provide an `OccupiedCoreAssumption`. This indicates how the result of the runtime API should be computed if there is a candidate from the para occupying an availability core in the [Inclusion Module](../runtime/inclusion.md). +Certain runtime APIs concerning the state of a para require the caller to provide an `OccupiedCoreAssumption`. This +indicates how the result of the runtime API should be computed if there is a candidate from the para occupying an +availability core in the [Inclusion Module](../runtime/inclusion.md). -The choices of assumption are whether the candidate occupying that core should be assumed to have been made available and included or timed out and discarded, along with a third option to assert that the core was not occupied. This choice affects everything from the parent head-data, the validation code, and the state of message-queues. Typically, users will take the assumption that either the core was free or that the occupying candidate was included, as timeouts are expected only in adversarial circumstances and even so, only in a small minority of blocks directly following validator set rotations. +The choices of assumption are whether the candidate occupying that core should be assumed to have been made available +and included or timed out and discarded, along with a third option to assert that the core was not occupied. This choice +affects everything from the parent head-data, the validation code, and the state of message-queues. Typically, users +will take the assumption that either the core was free or that the occupying candidate was included, as timeouts are +expected only in adversarial circumstances and even so, only in a small minority of blocks directly following validator +set rotations. ```rust /// An assumption being made about the state of an occupied core. @@ -52,4 +75,4 @@ enum OccupiedCoreAssumption { /// The core was not occupied to begin with. Free, } -``` \ No newline at end of file +``` diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/availability-cores.md b/polkadot/roadmap/implementers-guide/src/runtime-api/availability-cores.md index 9402924f0013a67e2623fdc48710f7ef60bf8b76..43288116089617fbd85da980d345df876d4c6e41 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/availability-cores.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/availability-cores.md @@ -1,14 +1,26 @@ # Availability Cores -Yields information on all availability cores. Cores are either free or occupied. Free cores can have paras assigned to them. Occupied cores don't, but they can become available part-way through a block due to bitfields and then have something scheduled on them. To allow optimistic validation of candidates, the occupied cores are accompanied by information on what is upcoming. This information can be leveraged when validators perceive that there is a high likelihood of a core becoming available based on bitfields seen, and then optimistically validate something that would become scheduled based on that, although there is no guarantee on what the block producer will actually include in the block. +Yields information on all availability cores. Cores are either free or occupied. Free cores can have paras assigned to +them. Occupied cores don't, but they can become available part-way through a block due to bitfields and then have +something scheduled on them. To allow optimistic validation of candidates, the occupied cores are accompanied by +information on what is upcoming. This information can be leveraged when validators perceive that there is a high +likelihood of a core becoming available based on bitfields seen, and then optimistically validate something that would +become scheduled based on that, although there is no guarantee on what the block producer will actually include in the +block. -See also the [Scheduler Module](../runtime/scheduler.md) for a high-level description of what an availability core is and why it exists. +See also the [Scheduler Module](../runtime/scheduler.md) for a high-level description of what an availability core is +and why it exists. ```rust fn availability_cores(at: Block) -> Vec; ``` -This is all the information that a validator needs about scheduling for the current block. It includes all information on [Scheduler](../runtime/scheduler.md) core-assignments and [Inclusion](../runtime/inclusion.md) state of blocks occupying availability cores. It includes data necessary to determine not only which paras are assigned now, but which cores are likely to become freed after processing bitfields, and exactly which bitfields would be necessary to make them so. The implementation of this runtime API should invoke `Scheduler::clear` and `Scheduler::schedule(Vec::new(), current_block_number + 1)` to ensure that scheduling is accurate. +This is all the information that a validator needs about scheduling for the current block. It includes all information +on [Scheduler](../runtime/scheduler.md) core-assignments and [Inclusion](../runtime/inclusion.md) state of blocks +occupying availability cores. It includes data necessary to determine not only which paras are assigned now, but which +cores are likely to become freed after processing bitfields, and exactly which bitfields would be necessary to make them +so. The implementation of this runtime API should invoke `Scheduler::clear` and `Scheduler::schedule(Vec::new(), +current_block_number + 1)` to ensure that scheduling is accurate. ```rust struct OccupiedCore { diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/candidate-pending-availability.md b/polkadot/roadmap/implementers-guide/src/runtime-api/candidate-pending-availability.md index 9c8969f6a958b2cce360ad83c1dc1e57d91e207e..e118757d83cede12468be2d8d444958ca847eadf 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/candidate-pending-availability.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/candidate-pending-availability.md @@ -1,6 +1,7 @@ # Candidate Pending Availability -Get the receipt of a candidate pending availability. This returns `Some` for any paras assigned to occupied cores in `availability_cores` and `None` otherwise. +Get the receipt of a candidate pending availability. This returns `Some` for any paras assigned to occupied cores in +`availability_cores` and `None` otherwise. ```rust fn candidate_pending_availability(at: Block, ParaId) -> Option; diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/disputes-info.md b/polkadot/roadmap/implementers-guide/src/runtime-api/disputes-info.md index 3548d5fb5793a2e664f4ec3d681ec5e609471919..24f64a81538256c32ce3057eb4972a34af05f47c 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/disputes-info.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/disputes-info.md @@ -1,6 +1,9 @@ # Disputes Info -Get information about all disputes known by the chain as well as information about which validators the disputes subsystem will accept disputes from. These disputes may be either live or concluded. The [`DisputeState`](../types/disputes.md#disputestate) can be used to determine whether the dispute still accepts votes, as well as which validators' votes may be included. +Get information about all disputes known by the chain as well as information about which validators the disputes +subsystem will accept disputes from. These disputes may be either live or concluded. The +[`DisputeState`](../types/disputes.md#disputestate) can be used to determine whether the dispute still accepts votes, as +well as which validators' votes may be included. ```rust struct Dispute { diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/persisted-validation-data.md b/polkadot/roadmap/implementers-guide/src/runtime-api/persisted-validation-data.md index 2fd3e55c8712c5c85193ee60b9e6c736dc0ec2b7..e741008a7370eab76ca0039b5f4b278c22bac352 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/persisted-validation-data.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/persisted-validation-data.md @@ -1,6 +1,8 @@ # Persisted Validation Data -Yields the [`PersistedValidationData`](../types/candidate.md#persistedvalidationdata) for the given [`ParaId`](../types/candidate.md#paraid) along with an assumption that should be used if the para currently occupies a core: +Yields the [`PersistedValidationData`](../types/candidate.md#persistedvalidationdata) for the given +[`ParaId`](../types/candidate.md#paraid) along with an assumption that should be used if the para currently occupies a +core: ```rust /// Returns the persisted validation data for the given para and occupied core assumption. @@ -8,4 +10,4 @@ Yields the [`PersistedValidationData`](../types/candidate.md#persistedvalidation /// Returns `None` if either the para is not registered or the assumption is `Freed` /// and the para already occupies a core. fn persisted_validation_data(at: Block, ParaId, OccupiedCoreAssumption) -> Option; -``` \ No newline at end of file +``` diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/pvf-prechecking.md b/polkadot/roadmap/implementers-guide/src/runtime-api/pvf-prechecking.md index c74232367bff5de1084f1fe41312c6f5767b3cfa..ef9b801a81c60889b3fd3861d2e4f974df995928 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/pvf-prechecking.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/pvf-prechecking.md @@ -4,18 +4,18 @@ There are two main runtime APIs to work with PVF pre-checking. -The first runtime API is designed to fetch all PVFs that require pre-checking voting. The PVFs are -identified by their code hashes. As soon as the PVF gains required support, the runtime API will -not return the PVF anymore. +The first runtime API is designed to fetch all PVFs that require pre-checking voting. The PVFs are identified by their +code hashes. As soon as the PVF gains required support, the runtime API will not return the PVF anymore. ```rust fn pvfs_require_precheck() -> Vec; ``` -The second runtime API is needed to submit the judgement for a PVF, whether it is approved or not. -The voting process uses unsigned transactions. The [`PvfCheckStatement`](../types/pvf-prechecking.md) is circulated through the network via gossip similar to a normal transaction. At some point the validator -will include the statement in the block, where it will be processed by the runtime. If that was the -last vote before gaining the super-majority, this PVF will not be returned by `pvfs_require_precheck` anymore. +The second runtime API is needed to submit the judgement for a PVF, whether it is approved or not. The voting process +uses unsigned transactions. The [`PvfCheckStatement`](../types/pvf-prechecking.md) is circulated through the network via +gossip similar to a normal transaction. At some point the validator will include the statement in the block, where it +will be processed by the runtime. If that was the last vote before gaining the super-majority, this PVF will not be +returned by `pvfs_require_precheck` anymore. ```rust fn submit_pvf_check_statement(stmt: PvfCheckStatement, signature: ValidatorSignature); diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/session-index.md b/polkadot/roadmap/implementers-guide/src/runtime-api/session-index.md index 1baf6a167dbb2543a1b6080dbe22dac266827fd5..d1441a3a739704f9a704f202d864fa1d38930d9b 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/session-index.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/session-index.md @@ -2,7 +2,8 @@ Get the session index that is expected at the child of a block. -In the [`Initializer`](../runtime/initializer.md) module, session changes are buffered by one block. The session index of the child of any relay block is always predictable by that block's state. +In the [`Initializer`](../runtime/initializer.md) module, session changes are buffered by one block. The session index +of the child of any relay block is always predictable by that block's state. This session index can be used to derive a [`SigningContext`](../types/candidate.md#signing-context). diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/validator-groups.md b/polkadot/roadmap/implementers-guide/src/runtime-api/validator-groups.md index 8815a0217411773789f5d844991976ab84e70b67..e5ed7b43aa949978b88a6a2cfe678791a1e41ef4 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/validator-groups.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/validator-groups.md @@ -1,6 +1,7 @@ # Validator Groups -Yields the validator groups used during the current session. The validators in the groups are referred to by their index into the validator-set and this is assumed to be as-of the child of the block whose state is being queried. +Yields the validator groups used during the current session. The validators in the groups are referred to by their index +into the validator-set and this is assumed to be as-of the child of the block whose state is being queried. ```rust /// A helper data-type for tracking validator-group rotations. diff --git a/polkadot/roadmap/implementers-guide/src/runtime-api/validators.md b/polkadot/roadmap/implementers-guide/src/runtime-api/validators.md index b7f1d964754755c2224dc89a3503835398a581a2..e9d94d736b20d269971b244e7e277cc096e671f2 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime-api/validators.md +++ b/polkadot/roadmap/implementers-guide/src/runtime-api/validators.md @@ -1,6 +1,7 @@ # Validators -Yields the validator-set at the state of a given block. This validator set is always the one responsible for backing parachains in the child of the provided block. +Yields the validator-set at the state of a given block. This validator set is always the one responsible for backing +parachains in the child of the provided block. ```rust fn validators(at: Block) -> Vec; diff --git a/polkadot/roadmap/implementers-guide/src/runtime/README.md b/polkadot/roadmap/implementers-guide/src/runtime/README.md index 995b684b1f06d07bb56b4b51bad6861f0082bd68..459f0e6b69d98bddb19a9ccea85830ed8dc01150 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/README.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/README.md @@ -1,42 +1,83 @@ # Runtime Architecture -It's clear that we want to separate different aspects of the runtime logic into different modules. Modules define their own storage, routines, and entry-points. They also define initialization and finalization logic. +It's clear that we want to separate different aspects of the runtime logic into different modules. Modules define their +own storage, routines, and entry-points. They also define initialization and finalization logic. -Due to the (lack of) guarantees provided by a particular blockchain-runtime framework, there is no defined or dependable order in which modules' initialization or finalization logic will run. Supporting this blockchain-runtime framework is important enough to include that same uncertainty in our model of runtime modules in this guide. Furthermore, initialization logic of modules can trigger the entry-points or routines of other modules. This is one architectural pressure against dividing the runtime logic into multiple modules. However, in this case the benefits of splitting things up outweigh the costs, provided that we take certain precautions against initialization and entry-point races. +Due to the (lack of) guarantees provided by a particular blockchain-runtime framework, there is no defined or dependable +order in which modules' initialization or finalization logic will run. Supporting this blockchain-runtime framework is +important enough to include that same uncertainty in our model of runtime modules in this guide. Furthermore, +initialization logic of modules can trigger the entry-points or routines of other modules. This is one architectural +pressure against dividing the runtime logic into multiple modules. However, in this case the benefits of splitting +things up outweigh the costs, provided that we take certain precautions against initialization and entry-point races. -We also expect, although it's beyond the scope of this guide, that these runtime modules will exist alongside various other modules. This has two facets to consider. First, even if the modules that we describe here don't invoke each others' entry points or routines during initialization, we still have to protect against those other modules doing that. Second, some of those modules are expected to provide governance capabilities for the chain. Configuration exposed by parachain-host modules is mostly for the benefit of these governance modules, to allow the operators or community of the chain to tweak parameters. +We also expect, although it's beyond the scope of this guide, that these runtime modules will exist alongside various +other modules. This has two facets to consider. First, even if the modules that we describe here don't invoke each +others' entry points or routines during initialization, we still have to protect against those other modules doing that. +Second, some of those modules are expected to provide governance capabilities for the chain. Configuration exposed by +parachain-host modules is mostly for the benefit of these governance modules, to allow the operators or community of the +chain to tweak parameters. -The runtime's primary role is to manage scheduling and updating of parachains, as well as handling misbehavior reports and slashing. This guide doesn't focus on how parachains are registered, only that they are. Also, this runtime description assumes that validator sets are selected somehow, but doesn't assume any other details than a periodic _session change_ event. Session changes give information about the incoming validator set and the validator set of the following session. +The runtime's primary role is to manage scheduling and updating of parachains, as well as handling misbehavior reports +and slashing. This guide doesn't focus on how parachains are registered, only that they are. Also, this runtime +description assumes that validator sets are selected somehow, but doesn't assume any other details than a periodic +_session change_ event. Session changes give information about the incoming validator set and the validator set of the +following session. -The runtime also serves another role, which is to make data available to the Node-side logic via Runtime APIs. These Runtime APIs should be sufficient for the Node-side code to author blocks correctly. +The runtime also serves another role, which is to make data available to the Node-side logic via Runtime APIs. These +Runtime APIs should be sufficient for the Node-side code to author blocks correctly. -There is some functionality of the relay chain relating to parachains that we also consider beyond the scope of this document. In particular, all modules related to how parachains are registered aren't part of this guide, although we do provide routines that should be called by the registration process. +There is some functionality of the relay chain relating to parachains that we also consider beyond the scope of this +document. In particular, all modules related to how parachains are registered aren't part of this guide, although we do +provide routines that should be called by the registration process. We will split the logic of the runtime up into these modules: -* Initializer: manages initialization order of the other modules. -* Shared: manages shared storage and configurations for other modules. -* Configuration: manages configuration and configuration updates in a non-racy manner. -* Paras: manages chain-head and validation code for parachains. -* Scheduler: manages parachain scheduling as well as validator assignments. -* Inclusion: handles the inclusion and availability of scheduled parachains. -* SessionInfo: manages various session keys of validators and other params stored per session. -* Disputes: handles dispute resolution for included, available parablocks. -* Slashing: handles slashing logic for concluded disputes. -* HRMP: handles horizontal messages between paras. -* UMP: handles upward messages from a para to the relay chain. -* DMP: handles downward messages from the relay chain to the para. +- Initializer: manages initialization order of the other modules. +- Shared: manages shared storage and configurations for other modules. +- Configuration: manages configuration and configuration updates in a non-racy manner. +- Paras: manages chain-head and validation code for parachains. +- Scheduler: manages parachain scheduling as well as validator assignments. +- Inclusion: handles the inclusion and availability of scheduled parachains. +- SessionInfo: manages various session keys of validators and other params stored per session. +- Disputes: handles dispute resolution for included, available parablocks. +- Slashing: handles slashing logic for concluded disputes. +- HRMP: handles horizontal messages between paras. +- UMP: handles upward messages from a para to the relay chain. +- DMP: handles downward messages from the relay chain to the para. -The [Initializer module](initializer.md) is special - it's responsible for handling the initialization logic of the other modules to ensure that the correct initialization order and related invariants are maintained. The other modules won't specify a on-initialize logic, but will instead expose a special semi-private routine that the initialization module will call. The other modules are relatively straightforward and perform the roles described above. +The [Initializer module](initializer.md) is special - it's responsible for handling the initialization logic of the +other modules to ensure that the correct initialization order and related invariants are maintained. The other modules +won't specify a on-initialize logic, but will instead expose a special semi-private routine that the initialization +module will call. The other modules are relatively straightforward and perform the roles described above. -The Parachain Host operates under a changing set of validators. Time is split up into periodic sessions, where each session brings a potentially new set of validators. Sessions are buffered by one, meaning that the validators of the upcoming session `n+1` are determined at the end of session `n-1`, right before session `n` starts. Parachain Host runtime modules need to react to changes in the validator set, as it will affect the runtime logic for processing candidate backing, availability bitfields, and misbehavior reports. The Parachain Host modules can't determine ahead-of-time exactly when session change notifications are going to happen within the block (note: this depends on module initialization order again - better to put session before parachains modules). +The Parachain Host operates under a changing set of validators. Time is split up into periodic sessions, where each +session brings a potentially new set of validators. Sessions are buffered by one, meaning that the validators of the +upcoming session `n+1` are determined at the end of session `n-1`, right before session `n` starts. Parachain Host +runtime modules need to react to changes in the validator set, as it will affect the runtime logic for processing +candidate backing, availability bitfields, and misbehavior reports. The Parachain Host modules can't determine +ahead-of-time exactly when session change notifications are going to happen within the block (note: this depends on +module initialization order again - better to put session before parachains modules). -The relay chain is intended to use BABE or SASSAFRAS, which both have the property that a session changing at a block is determined not by the number of the block but instead by the time the block is authored. In some sense, sessions change in-between blocks, not at blocks. This has the side effect that the session of a child block cannot be determined solely by the parent block's identifier. Being able to unilaterally determine the validator-set at a specific block based on its parent hash would make a lot of Node-side logic much simpler. +The relay chain is intended to use BABE or SASSAFRAS, which both have the property that a session changing at a block is +determined not by the number of the block but instead by the time the block is authored. In some sense, sessions change +in-between blocks, not at blocks. This has the side effect that the session of a child block cannot be determined solely +by the parent block's identifier. Being able to unilaterally determine the validator-set at a specific block based on +its parent hash would make a lot of Node-side logic much simpler. -In order to regain the property that the validator set of a block is predictable by its parent block, we delay session changes' application to Parachains by 1 block. This means that if there is a session change at block X, that session change will be stored and applied during initialization of direct descendants of X. This principal side effect of this change is that the Parachains runtime can disagree with session or consensus modules about which session it currently is. Misbehavior reporting routines in particular will be affected by this, although not severely. The parachains runtime might believe it is the last block of the session while the system is really in the first block of the next session. In such cases, a historical validator-set membership proof will need to accompany any misbehavior report, although they typically do not need to during current-session misbehavior reports. +In order to regain the property that the validator set of a block is predictable by its parent block, we delay session +changes' application to Parachains by 1 block. This means that if there is a session change at block X, that session +change will be stored and applied during initialization of direct descendants of X. This principal side effect of this +change is that the Parachains runtime can disagree with session or consensus modules about which session it currently +is. Misbehavior reporting routines in particular will be affected by this, although not severely. The parachains runtime +might believe it is the last block of the session while the system is really in the first block of the next session. In +such cases, a historical validator-set membership proof will need to accompany any misbehavior report, although they +typically do not need to during current-session misbehavior reports. -So the other role of the initializer module is to forward session change notifications to modules in the initialization order. Session change is also the point at which the [Configuration Module](configuration.md) updates the configuration. Most of the other modules will handle changes in the configuration during their session change operation, so the initializer should provide both the old and new configuration to all the other -modules alongside the session change notification. This means that a session change notification should consist of the following data: +So the other role of the initializer module is to forward session change notifications to modules in the initialization +order. Session change is also the point at which the [Configuration Module](configuration.md) updates the configuration. +Most of the other modules will handle changes in the configuration during their session change operation, so the +initializer should provide both the old and new configuration to all the other modules alongside the session change +notification. This means that a session change notification should consist of the following data: ```rust struct SessionChangeNotification { diff --git a/polkadot/roadmap/implementers-guide/src/runtime/configuration.md b/polkadot/roadmap/implementers-guide/src/runtime/configuration.md index be62ab2d4d5e0eda4b14358673a152328afe98b1..37885bd481598f192db6862f987a0849b80d0586 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/configuration.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/configuration.md @@ -1,6 +1,11 @@ # Configuration Pallet -This module is responsible for managing all configuration of the parachain host in-flight. It provides a central point for configuration updates to prevent races between configuration changes and parachain-processing logic. Configuration can only change during the session change routine, and as this module handles the session change notification first it provides an invariant that the configuration does not change throughout the entire session. Both the [scheduler](scheduler.md) and [inclusion](inclusion.md) modules rely on this invariant to ensure proper behavior of the scheduler. +This module is responsible for managing all configuration of the parachain host in-flight. It provides a central point +for configuration updates to prevent races between configuration changes and parachain-processing logic. Configuration +can only change during the session change routine, and as this module handles the session change notification first it +provides an invariant that the configuration does not change throughout the entire session. Both the +[scheduler](scheduler.md) and [inclusion](inclusion.md) modules rely on this invariant to ensure proper behavior of the +scheduler. The configuration that we will be tracking is the [`HostConfiguration`](../types/runtime.md#host-configuration) struct. @@ -23,7 +28,8 @@ The session change routine works as follows: - If there is no pending configurations, then return early. - Take all pending configurations that are less than or equal to the current session index. - - Get the pending configuration with the highest session index and apply it to the current configuration. Discard the earlier ones if any. + - Get the pending configuration with the highest session index and apply it to the current configuration. Discard the + earlier ones if any. ## Routines @@ -41,17 +47,17 @@ pub fn configuration() -> HostConfiguration { Configuration::get() } -/// Schedules updating the host configuration. The update is given by the `updater` closure. The -/// closure takes the current version of the configuration and returns the new version. -/// Returns an `Err` if the closure returns a broken configuration. However, there are a couple of -/// exceptions: +/// Schedules updating the host configuration. The update is given by the `updater` closure. The +/// closure takes the current version of the configuration and returns the new version. +/// Returns an `Err` if the closure returns a broken configuration. However, there are a couple of +/// exceptions: /// -/// - if the configuration that was passed in the closure is already broken, then it will pass the +/// - if the configuration that was passed in the closure is already broken, then it will pass the /// update: you cannot break something that is already broken. /// - If the `BypassConsistencyCheck` flag is set, then the checks will be skipped. /// /// The changes made by this function will always be scheduled at session X, where X is the current session index + 2. -/// If there is already a pending update for X, then the closure will receive the already pending configuration for +/// If there is already a pending update for X, then the closure will receive the already pending configuration for /// session X. /// /// If there is already a pending update for the current session index + 1, then it won't be touched. Otherwise, @@ -61,4 +67,6 @@ fn schedule_config_update(updater: impl FnOnce(&mut HostConfiguration Option, Frozen: Option, ``` -> `byzantine_threshold` refers to the maximum number `f` of validators which may be byzantine. The total number of validators is `n = 3f + e` where `e in { 1, 2, 3 }`. +> `byzantine_threshold` refers to the maximum number `f` of validators which may be byzantine. The total number of +> validators is `n = 3f + e` where `e in { 1, 2, 3 }`. ## Session Change 1. If the current session is not greater than `config.dispute_period + 1`, nothing to do here. -1. Set `pruning_target = current_session - config.dispute_period - 1`. We add the extra `1` because we want to keep things for `config.dispute_period` _full_ sessions. - The stuff at the end of the most recent session has been around for a little over 0 sessions, not a little over 1. +1. Set `pruning_target = current_session - config.dispute_period - 1`. We add the extra `1` because we want to keep + things for `config.dispute_period` _full_ sessions. The stuff at the end of the most recent session has been around + for a little over 0 sessions, not a little over 1. 1. If `LastPrunedSession` is `None`, then set `LastPrunedSession` to `Some(pruning_target)` and return. -2. Otherwise, clear out all disputes and included candidates entries in the range `last_pruned..=pruning_target` and set `LastPrunedSession` to `Some(pruning_target)`. +1. Otherwise, clear out all disputes and included candidates entries in the range `last_pruned..=pruning_target` and set + `LastPrunedSession` to `Some(pruning_target)`. ## Block Initialization @@ -61,11 +82,10 @@ This is currently a `no op`. ## Routines * `filter_multi_dispute_data(MultiDisputeStatementSet) -> MultiDisputeStatementSet`: - 1. Takes a `MultiDisputeStatementSet` and filters it down to a `MultiDisputeStatementSet` - that satisfies all the criteria of `provide_multi_dispute_data`. That is, eliminating - ancient votes, duplicates and unconfirmed disputes. - This can be used by block authors to create the final submission in a block which is - guaranteed to pass the `provide_multi_dispute_data` checks. + 1. Takes a `MultiDisputeStatementSet` and filters it down to a `MultiDisputeStatementSet` that satisfies all the + criteria of `provide_multi_dispute_data`. That is, eliminating ancient votes, duplicates and unconfirmed disputes. + This can be used by block authors to create the final submission in a block which is guaranteed to pass the + `provide_multi_dispute_data` checks. * `provide_multi_dispute_data(MultiDisputeStatementSet) -> Vec<(SessionIndex, Hash)>`: 1. Pass on each dispute statement set to `provide_dispute_data`, propagating failure. @@ -76,46 +96,55 @@ This is currently a `no op`. 1. `SessionInfo` is used to check statement signatures and this function should fail if any signatures are invalid. 1. If there is no dispute under `Disputes`, create a new `DisputeState` with blank bitfields. 1. If `concluded_at` is `Some`, and is `concluded_at + config.post_conclusion_acceptance_period < now`, return false. - 2. Import all statements into the dispute. This should fail if any statements are duplicate or if the corresponding bit for the corresponding validator is set in the dispute already. - 3. If `concluded_at` is `None`, reward all statements. - 4. If `concluded_at` is `Some`, reward all statements slightly less. - 5. If either side now has supermajority and did not previously, slash the other side. This may be both sides, and we support this possibility in code, but note that this requires validators to participate on both sides which has negative expected value. Set `concluded_at` to `Some(now)` if it was `None`. - 6. If just concluded against the candidate and the `Included` map contains `(session, candidate)`: invoke `revert_and_freeze` with the stored block number. - 7. Return true if just initiated, false otherwise. - -* `disputes() -> Vec<(SessionIndex, CandidateHash, DisputeState)>`: Get a list of all disputes and info about dispute state. + 1. Import all statements into the dispute. This should fail if any statements are duplicate or if the corresponding + bit for the corresponding validator is set in the dispute already. + 1. If `concluded_at` is `None`, reward all statements. + 1. If `concluded_at` is `Some`, reward all statements slightly less. + 1. If either side now has supermajority and did not previously, slash the other side. This may be both sides, and we + support this possibility in code, but note that this requires validators to participate on both sides which has + negative expected value. Set `concluded_at` to `Some(now)` if it was `None`. + 1. If just concluded against the candidate and the `Included` map contains `(session, candidate)`: invoke + `revert_and_freeze` with the stored block number. + 1. Return true if just initiated, false otherwise. + +* `disputes() -> Vec<(SessionIndex, CandidateHash, DisputeState)>`: Get a list of all disputes and info about dispute + state. 1. Iterate over all disputes in `Disputes` and collect into a vector. * `note_included(SessionIndex, CandidateHash, included_in: BlockNumber)`: 1. Add `(SessionIndex, CandidateHash)` to the `Included` map with `included_in - 1` as the value. - 1. If there is a dispute under `(SessionIndex, CandidateHash)` that has concluded against the candidate, invoke `revert_and_freeze` with the stored block number. + 1. If there is a dispute under `(SessionIndex, CandidateHash)` that has concluded against the candidate, invoke + `revert_and_freeze` with the stored block number. -* `concluded_invalid(SessionIndex, CandidateHash) -> bool`: Returns whether a candidate has already concluded a dispute in the negative. +* `concluded_invalid(SessionIndex, CandidateHash) -> bool`: Returns whether a candidate has already concluded a dispute + in the negative. * `is_frozen()`: Load the value of `Frozen` from storage. Return true if `Some` and false if `None`. -* `last_valid_block()`: Load the value of `Frozen` from storage and return. None indicates that all blocks in the chain are potentially valid. +* `last_valid_block()`: Load the value of `Frozen` from storage and return. None indicates that all blocks in the chain + are potentially valid. * `revert_and_freeze(BlockNumber)`: 1. If `is_frozen()` return. 1. Set `Frozen` to `Some(BlockNumber)` to indicate a rollback to the block number. - 1. Issue a `Revert(BlockNumber + 1)` log to indicate a rollback of the block's child in the header chain, which is the same as a rollback to the block number. + 1. Issue a `Revert(BlockNumber + 1)` log to indicate a rollback of the block's child in the header chain, which is the + same as a rollback to the block number. # Disputes filtering All disputes delivered to the runtime by the client are filtered before the actual import. In this context actual import means persisted in the runtime storage. The filtering has got two purposes: -- Limit the amount of data saved onchain. -- Prevent persisting malicious dispute data onchain. +* Limit the amount of data saved onchain. +* Prevent persisting malicious dispute data onchain. *Implementation note*: Filtering is performed in function `filter_dispute_data` from `Disputes` pallet. The filtering is performed on the whole statement set which is about to be imported onchain. The following filters are applied: 1. Remove ancient disputes - if a dispute is concluded before the block number indicated in `OLDEST_ACCEPTED` parameter - it is removed from the set. `OLDEST_ACCEPTED` is a runtime configuration option. - *Implementation note*: `dispute_post_conclusion_acceptance_period` from - `HostConfiguration` is used in the current Polkadot/Kusama implementation. + it is removed from the set. `OLDEST_ACCEPTED` is a runtime configuration option. *Implementation note*: + `dispute_post_conclusion_acceptance_period` from `HostConfiguration` is used in the current Polkadot/Kusama + implementation. 2. Remove votes from unknown validators. If there is a vote from a validator which wasn't an authority in the session where the dispute was raised - they are removed. Please note that this step removes only single votes instead of removing the whole dispute. @@ -138,4 +167,4 @@ inconclusive disputes are not slashed. Thanks to the applied filtering (describe confident that there are no spam disputes in the runtime. So if a validator is not voting it is due to another reason (e.g. being under DoS attack). There is no reason to punish such validators with a slash. -*Implementation note*: Slashing is performed in `process_checked_dispute_data` from `Disputes` pallet. \ No newline at end of file +*Implementation note*: Slashing is performed in `process_checked_dispute_data` from `Disputes` pallet. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/dmp.md b/polkadot/roadmap/implementers-guide/src/runtime/dmp.md index f56df31934efefd9d5025835cd103ced22913669..5aeb3bd68ef38d9a2c9565ea10e7ccd1978b6629 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/dmp.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/dmp.md @@ -28,7 +28,8 @@ No initialization routine runs for this module. Candidate Acceptance Function: * `check_processed_downward_messages(P: ParaId, relay_parent_number: BlockNumber, processed_downward_messages: u32)`: - 1. Checks that `processed_downward_messages` is at least 1 if `DownwardMessageQueues` for `P` is not empty at the given `relay_parent_number`. + 1. Checks that `processed_downward_messages` is at least 1 if `DownwardMessageQueues` for `P` is not empty at the + given `relay_parent_number`. 1. Checks that `DownwardMessageQueues` for `P` is at least `processed_downward_messages` long. Candidate Enactment: @@ -38,11 +39,11 @@ Candidate Enactment: Utility routines. -`queue_downward_message(P: ParaId, M: DownwardMessage)`: - 1. Check if the size of `M` exceeds the `config.max_downward_message_size`. If so, return an error. - 1. Wrap `M` into `InboundDownwardMessage` using the current block number for `sent_at`. - 1. Obtain a new MQC link for the resulting `InboundDownwardMessage` and replace `DownwardMessageQueueHeads` for `P` with the resulting hash. - 1. Add the resulting `InboundDownwardMessage` into `DownwardMessageQueues` for `P`. +`queue_downward_message(P: ParaId, M: DownwardMessage)`: 1. Check if the size of `M` exceeds the + `config.max_downward_message_size`. If so, return an error. 1. Wrap `M` into `InboundDownwardMessage` using the + current block number for `sent_at`. 1. Obtain a new MQC link for the resulting `InboundDownwardMessage` and replace + `DownwardMessageQueueHeads` for `P` with the resulting hash. 1. Add the resulting `InboundDownwardMessage` into + `DownwardMessageQueues` for `P`. ## Session Change diff --git a/polkadot/roadmap/implementers-guide/src/runtime/hrmp.md b/polkadot/roadmap/implementers-guide/src/runtime/hrmp.md index 927c14cd596968bc7d2e82030140a02263093e20..aa31491d72f6c6e142059e7bcfbe881516772d25 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/hrmp.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/hrmp.md @@ -1,6 +1,7 @@ # HRMP Pallet -A module responsible for Horizontally Relay-routed Message Passing (HRMP). See [Messaging Overview](../messaging.md) for more details. +A module responsible for Horizontally Relay-routed Message Passing (HRMP). See [Messaging Overview](../messaging.md) for +more details. ## Storage @@ -132,7 +133,8 @@ Candidate Acceptance Function: 1. or in `HrmpChannelDigests` for `P` an entry with the block number should exist * `check_outbound_hrmp(sender: ParaId, Vec)`: 1. Checks that there are at most `config.hrmp_max_message_num_per_candidate` messages. - 1. Checks that horizontal messages are sorted by ascending recipient ParaId and there is no two horizontal messages have the same recipient. + 1. Checks that horizontal messages are sorted by ascending recipient ParaId and there is no two horizontal messages + have the same recipient. 1. For each horizontal message `M` with the channel `C` identified by `(sender, M.recipient)` check: 1. exists 1. `M`'s payload size doesn't exceed a preconfigured limit `C.max_message_size` @@ -143,42 +145,48 @@ Candidate Enactment: * `queue_outbound_hrmp(sender: ParaId, Vec)`: 1. For each horizontal message `HM` with the channel `C` identified by `(sender, HM.recipient)`: - 1. Append `HM` into `HrmpChannelContents` that corresponds to `C` with `sent_at` equals to the current block number. - 1. Locate or create an entry in `HrmpChannelDigests` for `HM.recipient` and append `sender` into the entry's list. + 1. Append `HM` into `HrmpChannelContents` that corresponds to `C` with `sent_at` equals to the current block + number. + 1. Locate or create an entry in `HrmpChannelDigests` for `HM.recipient` and append `sender` into the entry's + list. 1. Increment `C.msg_count` 1. Increment `C.total_size` by `HM`'s payload size - 1. Append a new link to the MQC and save the new head in `C.mqc_head`. Note that the current block number as of enactment is used for the link. + 1. Append a new link to the MQC and save the new head in `C.mqc_head`. Note that the current block number as of + enactment is used for the link. * `prune_hrmp(recipient, new_hrmp_watermark)`: - 1. From `HrmpChannelDigests` for `recipient` remove all entries up to an entry with block number equal to `new_hrmp_watermark`. - 1. From the removed digests construct a set of paras that sent new messages within the interval between the old and new watermarks. - 1. For each channel `C` identified by `(sender, recipient)` for each `sender` coming from the set, prune messages up to the `new_hrmp_watermark`. + 1. From `HrmpChannelDigests` for `recipient` remove all entries up to an entry with block number equal to + `new_hrmp_watermark`. + 1. From the removed digests construct a set of paras that sent new messages within the interval between the old and + new watermarks. + 1. For each channel `C` identified by `(sender, recipient)` for each `sender` coming from the set, prune messages up + to the `new_hrmp_watermark`. 1. For each pruned message `M` from channel `C`: 1. Decrement `C.msg_count` 1. Decrement `C.total_size` by `M`'s payload size. 1. Set `HrmpWatermarks` for `P` to be equal to `new_hrmp_watermark` > NOTE: That collecting digests can be inefficient and the time it takes grows very fast. Thanks to the aggressive - > parameterization this shouldn't be a big of a deal. - > If that becomes a problem consider introducing an extra dictionary which says at what block the given sender - > sent a message to the recipient. + > parameterization this shouldn't be a big of a deal. If that becomes a problem consider introducing an extra + > dictionary which says at what block the given sender sent a message to the recipient. ## Entry-points The following entry-points are meant to be used for HRMP channel management. -Those entry-points are meant to be called from a parachain. `origin` is defined as the `ParaId` of -the parachain executed the message. +Those entry-points are meant to be called from a parachain. `origin` is defined as the `ParaId` of the parachain +executed the message. * `hrmp_init_open_channel(recipient, proposed_max_capacity, proposed_max_message_size)`: 1. Check that the `origin` is not `recipient`. 1. Check that `proposed_max_capacity` is less or equal to `config.hrmp_channel_max_capacity` and greater than zero. - 1. Check that `proposed_max_message_size` is less or equal to `config.hrmp_channel_max_message_size` and greater than zero. + 1. Check that `proposed_max_message_size` is less or equal to `config.hrmp_channel_max_message_size` and greater + than zero. 1. Check that `recipient` is a valid para. 1. Check that there is no existing channel for `(origin, recipient)` in `HrmpChannels`. 1. Check that there is no existing open channel request (`origin`, `recipient`) in `HrmpOpenChannelRequests`. - 1. Check that the sum of the number of already opened HRMP channels by the `origin` (the size - of the set found `HrmpEgressChannelsIndex` for `origin`) and the number of open requests by the - `origin` (the value from `HrmpOpenChannelRequestCount` for `origin`) doesn't exceed the limit of - channels (`config.hrmp_max_parachain_outbound_channels` or `config.hrmp_max_parathread_outbound_channels`) minus 1. + 1. Check that the sum of the number of already opened HRMP channels by the `origin` (the size of the set found + `HrmpEgressChannelsIndex` for `origin`) and the number of open requests by the `origin` (the value from + `HrmpOpenChannelRequestCount` for `origin`) doesn't exceed the limit of channels + (`config.hrmp_max_parachain_outbound_channels` or `config.hrmp_max_parathread_outbound_channels`) minus 1. 1. Check that `origin`'s balance is more or equal to `config.hrmp_sender_deposit` 1. Reserve the deposit for the `origin` according to `config.hrmp_sender_deposit` 1. Increase `HrmpOpenChannelRequestCount` by 1 for `origin`. @@ -189,27 +197,26 @@ the parachain executed the message. 1. Set `max_message_size` to `proposed_max_message_size` 1. Set `max_total_size` to `config.hrmp_channel_max_total_size` 1. Send a downward message to `recipient` notifying about an inbound HRMP channel request. - - The DM is sent using `queue_downward_message`. - - The DM is represented by the `HrmpNewChannelOpenRequest` XCM message. - - `sender` is set to `origin`, - - `max_message_size` is set to `proposed_max_message_size`, - - `max_capacity` is set to `proposed_max_capacity`. + * The DM is sent using `queue_downward_message`. + * The DM is represented by the `HrmpNewChannelOpenRequest` XCM message. + * `sender` is set to `origin`, + * `max_message_size` is set to `proposed_max_message_size`, + * `max_capacity` is set to `proposed_max_capacity`. * `hrmp_accept_open_channel(sender)`: 1. Check that there is an existing request between (`sender`, `origin`) in `HrmpOpenChannelRequests` 1. Check that it is not confirmed. - 1. Check that the sum of the number of inbound HRMP channels opened to `origin` (the size of the set - found in `HrmpIngressChannelsIndex` for `origin`) and the number of accepted open requests by the `origin` - (the value from `HrmpAcceptedChannelRequestCount` for `origin`) doesn't exceed the limit of channels - (`config.hrmp_max_parachain_inbound_channels` or `config.hrmp_max_parathread_inbound_channels`) - minus 1. + 1. Check that the sum of the number of inbound HRMP channels opened to `origin` (the size of the set found in + `HrmpIngressChannelsIndex` for `origin`) and the number of accepted open requests by the `origin` (the value from + `HrmpAcceptedChannelRequestCount` for `origin`) doesn't exceed the limit of channels + (`config.hrmp_max_parachain_inbound_channels` or `config.hrmp_max_parathread_inbound_channels`) minus 1. 1. Check that `origin`'s balance is more or equal to `config.hrmp_recipient_deposit`. 1. Reserve the deposit for the `origin` according to `config.hrmp_recipient_deposit` 1. For the request in `HrmpOpenChannelRequests` identified by `(sender, P)`, set `confirmed` flag to `true`. 1. Increase `HrmpAcceptedChannelRequestCount` by 1 for `origin`. 1. Send a downward message to `sender` notifying that the channel request was accepted. - - The DM is sent using `queue_downward_message`. - - The DM is represented by the `HrmpChannelAccepted` XCM message. - - `recipient` is set to `origin`. + * The DM is sent using `queue_downward_message`. + * The DM is represented by the `HrmpChannelAccepted` XCM message. + * `recipient` is set to `origin`. * `hrmp_cancel_open_request(ch)`: 1. Check that `origin` is either `ch.sender` or `ch.recipient` 1. Check that the open channel request `ch` exists. @@ -221,15 +228,15 @@ the parachain executed the message. 1. Check that `origin` is either `ch.sender` or `ch.recipient` 1. Check that `HrmpChannels` for `ch` exists. 1. Check that `ch` is not in the `HrmpCloseChannelRequests` set. - 1. If not already there, insert a new entry `Some(())` to `HrmpCloseChannelRequests` for `ch` - and append `ch` to `HrmpCloseChannelRequestsList`. + 1. If not already there, insert a new entry `Some(())` to `HrmpCloseChannelRequests` for `ch` and append `ch` to + `HrmpCloseChannelRequestsList`. 1. Send a downward message to the opposite party notifying about the channel closing. - - The DM is sent using `queue_downward_message`. - - The DM is represented by the `HrmpChannelClosing` XCM message with: - - `initator` is set to `origin`, - - `sender` is set to `ch.sender`, - - `recipient` is set to `ch.recipient`. - - The opposite party is `ch.sender` if `origin` is `ch.recipient` and `ch.recipient` if `origin` is `ch.sender`. + * The DM is sent using `queue_downward_message`. + * The DM is represented by the `HrmpChannelClosing` XCM message with: + * `initator` is set to `origin`, + * `sender` is set to `ch.sender`, + * `recipient` is set to `ch.recipient`. + * The opposite party is `ch.sender` if `origin` is `ch.recipient` and `ch.recipient` if `origin` is `ch.sender`. ## Session Change @@ -241,13 +248,15 @@ the parachain executed the message. 1. Remove `HrmpOpenChannelRequests` and `HrmpOpenChannelRequestsList` for `(P, _)` and `(_, P)`. 1. For each removed channel request `C`: 1. Unreserve the sender's deposit if the sender is not present in `outgoing_paras` - 1. Unreserve the recipient's deposit if `C` is confirmed and the recipient is not present in `outgoing_paras` -1. For each channel designator `D` in `HrmpOpenChannelRequestsList` we query the request `R` from `HrmpOpenChannelRequests`: + 1. Unreserve the recipient's deposit if `C` is confirmed and the recipient is not present in + `outgoing_paras` +1. For each channel designator `D` in `HrmpOpenChannelRequestsList` we query the request `R` from + `HrmpOpenChannelRequests`: 1. if `R.confirmed = true`, 1. if both `D.sender` and `D.recipient` are not offboarded. 1. create a new channel `C` between `(D.sender, D.recipient)`. - 1. Initialize the `C.sender_deposit` with `R.sender_deposit` and `C.recipient_deposit` - with the value found in the configuration `config.hrmp_recipient_deposit`. + 1. Initialize the `C.sender_deposit` with `R.sender_deposit` and `C.recipient_deposit` with the value + found in the configuration `config.hrmp_recipient_deposit`. 1. Insert `sender` into the set `HrmpIngressChannelsIndex` for the `recipient`. 1. Insert `recipient` into the set `HrmpEgressChannelsIndex` for the `sender`. 1. decrement `HrmpOpenChannelRequestCount` for `D.sender` by 1. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md b/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md index 3fe7711ae2d0ee8a1ae5dc7a0d6bb0a1cf27789d..f73df209981af32acf3e99d8023da0d9bed9115d 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/inclusion.md @@ -1,6 +1,7 @@ # Inclusion Pallet -The inclusion module is responsible for inclusion and availability of scheduled parachains. It also manages the UMP dispatch queue of each parachain. +The inclusion module is responsible for inclusion and availability of scheduled parachains. It also manages the UMP +dispatch queue of each parachain. ## Storage @@ -37,11 +38,9 @@ PendingAvailabilityCommitments: map ParaId => CandidateCommitments; ## Config Dependencies -* `MessageQueue`: - The message queue provides general queueing and processing functionality. Currently it - replaces the old `UMP` dispatch queue. Other use-cases can be implemented as well by - adding new variants to `AggregateMessageOrigin`. Normally it should be set to an instance - of the `MessageQueue` pallet. +* `MessageQueue`: The message queue provides general queueing and processing functionality. Currently it replaces the + old `UMP` dispatch queue. Other use-cases can be implemented as well by adding new variants to + `AggregateMessageOrigin`. Normally it should be set to an instance of the `MessageQueue` pallet. ## Session Change @@ -49,11 +48,13 @@ PendingAvailabilityCommitments: map ParaId => CandidateCommitments; 1. Clear out all validator bitfields. Optional: -1. The UMP queue of all outgoing paras can be "swept". This would prevent the dispatch queue from automatically being serviced. It is a consideration for the chain and specific behaviour is not defined. +1. The UMP queue of all outgoing paras can be "swept". This would prevent the dispatch queue from automatically being + serviced. It is a consideration for the chain and specific behaviour is not defined. ## Initialization -No initialization routine runs for this module. However, the initialization of the `MessageQueue` pallet will attempt to process any pending UMP messages. +No initialization routine runs for this module. However, the initialization of the `MessageQueue` pallet will attempt to +process any pending UMP messages. ## Routines @@ -63,20 +64,19 @@ All failed checks should lead to an unrecoverable error making the block invalid * `process_bitfields(expected_bits, Bitfields, core_lookup: Fn(CoreIndex) -> Option)`: 1. Call `sanitize_bitfields` and use the sanitized `signed_bitfields` from now on. 1. Call `sanitize_backed_candidates` and use the sanitized `backed_candidates` from now on. - 1. Apply each bit of bitfield to the corresponding pending candidate, looking up on-demand parachain cores using the `core_lookup`. Disregard bitfields that have a `1` bit for any free cores. - 1. For each applied bit of each availability-bitfield, set the bit for the validator in the `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set in their `availability_votes`. These candidates are now available and can be enacted. + 1. Apply each bit of bitfield to the corresponding pending candidate, looking up on-demand parachain cores using the + `core_lookup`. Disregard bitfields that have a `1` bit for any free cores. + 1. For each applied bit of each availability-bitfield, set the bit for the validator in the + `CandidatePendingAvailability`'s `availability_votes` bitfield. Track all candidates that now have >2/3 of bits set + in their `availability_votes`. These candidates are now available and can be enacted. 1. For all now-available candidates, invoke the `enact_candidate` routine with the candidate and relay-parent number. - 1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become available. -* `sanitize_bitfields( - unchecked_bitfields: UncheckedSignedAvailabilityBitfields, - disputed_bitfield: DisputedBitfield, - expected_bits: usize, - parent_hash: T::Hash, - session_index: SessionIndex, - validators: &[ValidatorId], - full_check: FullCheck, - )`: - 1. check that `disputed_bitfield` has the same number of bits as the `expected_bits`, iff not return early with an empty vec. + 1. Return a list of `(CoreIndex, CandidateHash)` from freed cores consisting of the cores where candidates have become + available. +* `sanitize_bitfields( unchecked_bitfields: UncheckedSignedAvailabilityBitfields, + disputed_bitfield: DisputedBitfield, expected_bits: usize, parent_hash: T::Hash, session_index: SessionIndex, + validators: &[ValidatorId], full_check: FullCheck, )`: + 1. check that `disputed_bitfield` has the same number of bits as the `expected_bits`, iff not return early with an + empty vec. 1. each of the below checks is for each bitfield. If a check does not pass the bitfield will be skipped. 1. check that there are no bits set that reference a disputed candidate. 1. check that the number of bits is equal to `expected_bits`. @@ -84,41 +84,58 @@ All failed checks should lead to an unrecoverable error making the block invalid 1. check that the validator bit index is not out of bounds. 1. check the validators signature, iff `full_check=FullCheck::Yes`. -* `sanitize_backed_candidates) -> bool>( - mut backed_candidates: Vec>, - candidate_has_concluded_invalid_dispute: F, - scheduled: &[CoreAssignment], - ) ` +* `sanitize_backed_candidates) -> bool>( mut + backed_candidates: Vec>, candidate_has_concluded_invalid_dispute: F, scheduled: + &[CoreAssignment], )` 1. filter out any backed candidates that have concluded invalid. 1. filters backed candidates whom's paraid was scheduled by means of the provided `scheduled` parameter. 1. sorts remaining candidates with respect to the core index assigned to them. -* `process_candidates(allowed_relay_parents, BackedCandidates, scheduled: Vec, group_validators: Fn(GroupIndex) -> Option>)`: +* `process_candidates(allowed_relay_parents, BackedCandidates, scheduled: Vec, group_validators: + Fn(GroupIndex) -> Option>)`: > For details on `AllowedRelayParentsTracker` see documentation for [Shared](./shared.md) module. - 1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores appear in assignments in `scheduled`. + 1. check that each candidate corresponds to a scheduled core and that they are ordered in the same order the cores + appear in assignments in `scheduled`. 1. check that `scheduled` is sorted ascending by `CoreIndex`, without duplicates. 1. check that the relay-parent from each candidate receipt is one of the allowed relay-parents. 1. check that there is no candidate pending availability for any scheduled `ParaId`. - 1. check that each candidate's `validation_data_hash` corresponds to a `PersistedValidationData` computed from the state of the context block. + 1. check that each candidate's `validation_data_hash` corresponds to a `PersistedValidationData` computed from the + state of the context block. 1. If the core assignment includes a specific collator, ensure the backed candidate is issued by that collator. - 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_cooldown` of `Paras::last_code_upgrade(para_id, true)`, if any, comparing against the value of `Paras::FutureCodeUpgrades` for the given para ID. + 1. Ensure that any code upgrade scheduled by the candidate does not happen within `config.validation_upgrade_cooldown` + of `Paras::last_code_upgrade(para_id, true)`, if any, comparing against the value of `Paras::FutureCodeUpgrades` + for the given para ID. 1. Check the collator's signature on the candidate data. - 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators assigned to the groups, fetched with the `group_validators` lookup, while group indices are computed by `Scheduler` according to group rotation info. - 1. call `check_upward_messages(config, para, commitments.upward_messages)` to check that the upward messages are valid. - 1. call `Dmp::check_processed_downward_messages(para, commitments.processed_downward_messages)` to check that the DMQ is properly drained. - 1. call `Hrmp::check_hrmp_watermark(para, commitments.hrmp_watermark)` for each candidate to check rules of processing the HRMP watermark. - 1. using `Hrmp::check_outbound_hrmp(sender, commitments.horizontal_messages)` ensure that the each candidate sent a valid set of horizontal messages - 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` bitfield. + 1. check the backing of the candidate using the signatures and the bitfields, comparing against the validators + assigned to the groups, fetched with the `group_validators` lookup, while group indices are computed by `Scheduler` + according to group rotation info. + 1. call `check_upward_messages(config, para, commitments.upward_messages)` to check that the upward messages are + valid. + 1. call `Dmp::check_processed_downward_messages(para, commitments.processed_downward_messages)` to check that the DMQ + is properly drained. + 1. call `Hrmp::check_hrmp_watermark(para, commitments.hrmp_watermark)` for each candidate to check rules of processing + the HRMP watermark. + 1. using `Hrmp::check_outbound_hrmp(sender, commitments.horizontal_messages)` ensure that the each candidate sent a + valid set of horizontal messages + 1. create an entry in the `PendingAvailability` map for each backed candidate with a blank `availability_votes` + bitfield. 1. create a corresponding entry in the `PendingAvailabilityCommitments` with the commitments. - 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was successfully backed for, sorted ascending by CoreIndex. + 1. Return a `Vec` of all scheduled cores of the list of passed assignments that a candidate was + successfully backed for, sorted ascending by CoreIndex. * `enact_candidate(relay_parent_number: BlockNumber, CommittedCandidateReceipt)`: - 1. If the receipt contains a code upgrade, Call `Paras::schedule_code_upgrade(para_id, code, relay_parent_number, config)`. - > TODO: Note that this is safe as long as we never enact candidates where the relay parent is across a session boundary. In that case, which we should be careful to avoid with contextual execution, the configuration might have changed and the para may de-sync from the host's understanding of it. + 1. If the receipt contains a code upgrade, Call `Paras::schedule_code_upgrade(para_id, code, relay_parent_number, + config)`. + > TODO: Note that this is safe as long as we never enact candidates where the relay parent is across a session + > boundary. In that case, which we should be careful to avoid with contextual execution, the configuration might + > have changed and the para may de-sync from the host's understanding of it. 1. Reward all backing validators of each candidate, contained within the `backers` field. - 1. call `receive_upward_messages` for each backed candidate, using the [`UpwardMessage`s](../types/messages.md#upward-message) from the [`CandidateCommitments`](../types/candidate.md#candidate-commitments). + 1. call `receive_upward_messages` for each backed candidate, using the + [`UpwardMessage`s](../types/messages.md#upward-message) from the + [`CandidateCommitments`](../types/candidate.md#candidate-commitments). 1. call `Dmp::prune_dmq` with the para id of the candidate and the candidate's `processed_downward_messages`. 1. call `Hrmp::prune_hrmp` with the para id of the candiate and the candidate's `hrmp_watermark`. - 1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from the commitment, + 1. call `Hrmp::queue_outbound_hrmp` with the para id of the candidate and the list of horizontal messages taken from + the commitment, 1. Call `Paras::note_new_head` using the `HeadData` from the receipt and `relay_parent_number`. * `collect_pending`: @@ -130,21 +147,28 @@ All failed checks should lead to an unrecoverable error making the block invalid // return a vector of cleaned-up core IDs. } ``` -* `force_enact(ParaId)`: Forcibly enact the candidate with the given ID as though it had been deemed available by bitfields. Is a no-op if there is no candidate pending availability for this para-id. This should generally not be used but it is useful during execution of Runtime APIs, where the changes to the state are expected to be discarded directly after. -* `candidate_pending_availability(ParaId) -> Option`: returns the `CommittedCandidateReceipt` pending availability for the para provided, if any. -* `pending_availability(ParaId) -> Option`: returns the metadata around the candidate pending availability for the para, if any. -* `collect_disputed(disputed: Vec) -> Vec`: Sweeps through all paras pending availability. If the candidate hash is one of the disputed candidates, then clean up the corresponding storage for that candidate and the commitments. Return a vector of cleaned-up core IDs. +* `force_enact(ParaId)`: Forcibly enact the candidate with the given ID as though it had been deemed available by + bitfields. Is a no-op if there is no candidate pending availability for this para-id. This should generally not be + used but it is useful during execution of Runtime APIs, where the changes to the state are expected to be discarded + directly after. +* `candidate_pending_availability(ParaId) -> Option`: returns the `CommittedCandidateReceipt` + pending availability for the para provided, if any. +* `pending_availability(ParaId) -> Option`: returns the metadata around the candidate + pending availability for the para, if any. +* `collect_disputed(disputed: Vec) -> Vec`: Sweeps through all paras pending availability. If + the candidate hash is one of the disputed candidates, then clean up the corresponding storage for that candidate and + the commitments. Return a vector of cleaned-up core IDs. These functions were formerly part of the UMP pallet: * `check_upward_messages(P: ParaId, Vec)`: - 1. Checks that the parachain is not currently offboarding and error otherwise. + 1. Checks that the parachain is not currently offboarding and error otherwise. 1. Checks that there are at most `config.max_upward_message_num_per_candidate` messages to be enqueued. 1. Checks that no message exceeds `config.max_upward_message_size`. 1. Checks that the total resulting queue size would not exceed `co`. - 1. Verify that queuing up the messages could not result in exceeding the queue's footprint - according to the config items `config.max_upward_queue_count` and `config.max_upward_queue_size`. The queue's current footprint is provided in `well_known_keys` - in order to facilitate oraclisation on to the para. + 1. Verify that queuing up the messages could not result in exceeding the queue's footprint according to the config + items `config.max_upward_queue_count` and `config.max_upward_queue_size`. The queue's current footprint is provided + in `well_known_keys` in order to facilitate oraclisation on to the para. Candidate Enactment: diff --git a/polkadot/roadmap/implementers-guide/src/runtime/initializer.md b/polkadot/roadmap/implementers-guide/src/runtime/initializer.md index 19dfcbde50a9266305c3cb952655cb8870bfc5e2..a8522b369692de0f5a2dcec1c59b5584039b6141 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/initializer.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/initializer.md @@ -1,6 +1,7 @@ # Initializer Pallet -This module is responsible for initializing the other modules in a deterministic order. It also has one other purpose as described in the overview of the runtime: accepting and forwarding session change notifications. +This module is responsible for initializing the other modules in a deterministic order. It also has one other purpose as +described in the overview of the runtime: accepting and forwarding session change notifications. ## Storage @@ -15,7 +16,9 @@ BufferedSessionChanges: Vec<(BlockNumber, ValidatorSet, ValidatorSet)>; ## Initialization -Before initializing modules, remove all changes from the `BufferedSessionChanges` with number less than or equal to the current block number, and apply the last one. The session change is applied to all modules in the same order as initialization. +Before initializing modules, remove all changes from the `BufferedSessionChanges` with number less than or equal to the +current block number, and apply the last one. The session change is applied to all modules in the same order as +initialization. The other parachains modules are initialized in this order: @@ -30,16 +33,24 @@ The other parachains modules are initialized in this order: 1. UMP 1. HRMP -The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared](shared.md) module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module. +The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same +configuration as each other. Then the [Shared](shared.md) module is invoked, which determines the set of active +validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was +updated before the Inclusion module. Set `HasInitialized` to true. ## Session Change -Store the session change information in `BufferedSessionChange` along with the block number at which it was submitted, plus one. Although the expected operational parameters of the block authorship system should prevent more than one change from being buffered at any time, it may occur. Regardless, we always need to track the block number at which the session change can be applied so as to remain flexible over session change notifications being issued before or after initialization of the current block. +Store the session change information in `BufferedSessionChange` along with the block number at which it was submitted, +plus one. Although the expected operational parameters of the block authorship system should prevent more than one +change from being buffered at any time, it may occur. Regardless, we always need to track the block number at which the +session change can be applied so as to remain flexible over session change notifications being issued before or after +initialization of the current block. ## Finalization -Finalization order is less important in this case than initialization order, so we finalize the modules in the reverse order from initialization. +Finalization order is less important in this case than initialization order, so we finalize the modules in the reverse +order from initialization. Set `HasInitialized` to false. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/parainherent.md b/polkadot/roadmap/implementers-guide/src/runtime/parainherent.md index 405468c609a701b1e12147ba1a22442562a0b921..4a771f1df6441696c6bff95d74bb531047e17ace 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/parainherent.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/parainherent.md @@ -1,15 +1,27 @@ # `ParaInherent` -This module is responsible for providing all data given to the runtime by the block author to the various parachains modules. The entry-point is mandatory, in that it must be invoked exactly once within every block, and it is also "inherent", in that it is provided with no origin by the block author. The data within it carries its own authentication; i.e. the data takes the form of signed statements by validators. Invalid data will be filtered and not applied. +This module is responsible for providing all data given to the runtime by the block author to the various parachains +modules. The entry-point is mandatory, in that it must be invoked exactly once within every block, and it is also +"inherent", in that it is provided with no origin by the block author. The data within it carries its own +authentication; i.e. the data takes the form of signed statements by validators. Invalid data will be filtered and not +applied. -This module does not have the same initialization/finalization concerns as the others, as it only requires that entry points be triggered after all modules have initialized and that finalization happens after entry points are triggered. Both of these are assumptions we have already made about the runtime's order of operations, so this module doesn't need to be initialized or finalized by the `Initializer`. +This module does not have the same initialization/finalization concerns as the others, as it only requires that entry +points be triggered after all modules have initialized and that finalization happens after entry points are triggered. +Both of these are assumptions we have already made about the runtime's order of operations, so this module doesn't need +to be initialized or finalized by the `Initializer`. There are a couple of important notes to the operations in this inherent as they relate to disputes. -1. We don't accept bitfields or backed candidates if in "governance-only" mode from having a local dispute conclude on this fork. -1. When disputes are initiated, we remove the block from pending availability. This allows us to roll back chains to the block before blocks are included as opposed to backing. It's important to do this before processing bitfields. -1. `Inclusion::collect_disputed` is kind of expensive so it's important to gate this on whether there are actually any new disputes. Which should be never. -1. And we don't accept parablocks that have open disputes or disputes that have concluded against the candidate. It's important to import dispute statements before backing, but this is already the case as disputes are imported before processing bitfields. +1. We don't accept bitfields or backed candidates if in "governance-only" mode from having a local dispute conclude on + this fork. +1. When disputes are initiated, we remove the block from pending availability. This allows us to roll back chains to the + block before blocks are included as opposed to backing. It's important to do this before processing bitfields. +1. `Inclusion::collect_disputed` is kind of expensive so it's important to gate this on whether there are actually any + new disputes. Which should be never. +1. And we don't accept parablocks that have open disputes or disputes that have concluded against the candidate. It's + important to import dispute statements before backing, but this is already the case as disputes are imported before + processing bitfields. ## Storage @@ -32,26 +44,19 @@ OnChainVotes: Option, * `enter`: This entry-point accepts one parameter: [`ParaInherentData`](../types/runtime.md#ParaInherentData). * `create_inherent`: This entry-point accepts one parameter: `InherentData`. -Both entry points share mostly the same code. `create_inherent` will -meaningfully limit inherent data to adhere to the weight limit, in addition to -sanitizing any inputs and filtering out invalid data. Conceptually it is part of -the block production. The `enter` call on the other hand is part of block import -and consumes/imports the data previously produced by `create_inherent`. - -In practice both calls process inherent data and apply it to the state. Block -production and block import should arrive at the same new state. Hence we re-use -the same logic to ensure this is the case. - -The only real difference between the two is, that on `create_inherent` we -actually need the processed and filtered inherent data to build the block, while -on `enter` the processed data should for one be identical to the incoming -inherent data (assuming honest block producers) and second it is irrelevant, as -we are not building a block but just processing it, so the processed inherent -data is simply dropped. - -This also means that the `enter` function keeps data around for no good reason. -This seems acceptable though as the size of a block is rather limited. -Nevertheless if we ever wanted to optimize this we can easily implement an -inherent collector that has two implementations, where one clones and stores the -data and the other just passes it on. +Both entry points share mostly the same code. `create_inherent` will meaningfully limit inherent data to adhere to the +weight limit, in addition to sanitizing any inputs and filtering out invalid data. Conceptually it is part of the block +production. The `enter` call on the other hand is part of block import and consumes/imports the data previously produced +by `create_inherent`. +In practice both calls process inherent data and apply it to the state. Block production and block import should arrive +at the same new state. Hence we re-use the same logic to ensure this is the case. + +The only real difference between the two is, that on `create_inherent` we actually need the processed and filtered +inherent data to build the block, while on `enter` the processed data should for one be identical to the incoming +inherent data (assuming honest block producers) and second it is irrelevant, as we are not building a block but just +processing it, so the processed inherent data is simply dropped. + +This also means that the `enter` function keeps data around for no good reason. This seems acceptable though as the size +of a block is rather limited. Nevertheless if we ever wanted to optimize this we can easily implement an inherent +collector that has two implementations, where one clones and stores the data and the other just passes it on. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/paras.md b/polkadot/roadmap/implementers-guide/src/runtime/paras.md index b3015bd5729052db4b99e6285465f46efc07cb03..ac9b1520c3dfb588e06142d6e40b5fe286e0b6ac 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/paras.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/paras.md @@ -1,14 +1,12 @@ # Paras Pallet -The Paras module is responsible for storing information on parachains. Registered -parachains cannot change except at session boundaries and after at least a full -session has passed. This is primarily to ensure that the number and meaning of bits required for the -availability bitfields does not change except at session boundaries. +The Paras module is responsible for storing information on parachains. Registered parachains cannot change except at +session boundaries and after at least a full session has passed. This is primarily to ensure that the number and meaning +of bits required for the availability bitfields does not change except at session boundaries. It's also responsible for: -- managing parachain validation code upgrades as well as maintaining availability of old parachain -code and its pruning. +- managing parachain validation code upgrades as well as maintaining availability of old parachain code and its pruning. - vetting PVFs by means of the PVF pre-checking mechanism. ## Storage @@ -102,8 +100,8 @@ struct PvfCheckActiveVoteState { #### Para Lifecycle -Because the state changes of parachains are delayed, we track the specific state of -the para using the `ParaLifecycle` enum. +Because the state changes of parachains are delayed, we track the specific state of the para using the `ParaLifecycle` +enum. ``` None Parathread (on-demand parachain) Parachain @@ -132,8 +130,8 @@ None Parathread (on-demand parachain) Parachain + + + ``` -Note that if PVF pre-checking is enabled, onboarding of a para may potentially be delayed. This can -happen due to PVF pre-checking voting concluding late. +Note that if PVF pre-checking is enabled, onboarding of a para may potentially be delayed. This can happen due to PVF +pre-checking voting concluding late. During the transition period, the para object is still considered in its existing state. @@ -210,7 +208,7 @@ UpcomingUpgrades: Vec<(ParaId, BlockNumberFor)>; ActionsQueue: map SessionIndex => Vec; /// Upcoming paras instantiation arguments. /// -/// NOTE that after PVF pre-checking is enabled the para genesis arg will have it's code set +/// NOTE that after PVF pre-checking is enabled the para genesis arg will have it's code set /// to empty. Instead, the code will be saved into the storage right away via `CodeByHash`. UpcomingParasGenesis: map ParaId => Option; /// The number of references on the validation code in `CodeByHash` storage. @@ -223,12 +221,13 @@ CodeByHash: map ValidationCodeHash => Option 1. Execute all queued actions for paralifecycle changes: 1. Clean up outgoing paras. - 1. This means removing the entries under `Heads`, `CurrentCode`, `FutureCodeUpgrades`, - `FutureCode` and `MostRecentContext`. An according entry should be added to `PastCode`, `PastCodeMeta`, and `PastCodePruning` using the outgoing `ParaId` and removed `CurrentCode` value. This is because any outdated validation code must remain available on-chain for a determined amount - of blocks, and validation code outdated by de-registering the para is still subject to that - invariant. - 1. Apply all incoming paras by initializing the `Heads` and `CurrentCode` using the genesis - parameters as well as `MostRecentContext` to `0`. + 1. This means removing the entries under `Heads`, `CurrentCode`, `FutureCodeUpgrades`, `FutureCode` and + `MostRecentContext`. An according entry should be added to `PastCode`, `PastCodeMeta`, and `PastCodePruning` + using the outgoing `ParaId` and removed `CurrentCode` value. This is because any outdated validation code must + remain available on-chain for a determined amount of blocks, and validation code outdated by de-registering the + para is still subject to that invariant. + 1. Apply all incoming paras by initializing the `Heads` and `CurrentCode` using the genesis parameters as well as + `MostRecentContext` to `0`. 1. Amend the `Parachains` list and `ParaLifecycle` to reflect changes in registered parachains. 1. Amend the `ParaLifecycle` set to reflect changes in registered on-demand parachains. 1. Upgrade all on-demand parachains that should become lease holding parachains, updating the `Parachains` list and @@ -239,40 +238,50 @@ CodeByHash: map ValidationCodeHash => Option 1. Go over all active PVF pre-checking votes: 1. Increment `age` of the vote. 1. If `age` reached `cfg.pvf_voting_ttl`, then enact PVF rejection and remove the vote from the active list. - 1. Otherwise, reinitialize the ballots. - 1. Resize the `votes_accept`/`votes_reject` to have the same length as the incoming validator set. - 1. Zero all the votes. + 1. Otherwise, reinitialize the ballots. 1. Resize the `votes_accept`/`votes_reject` to have the same length as the + incoming validator set. 1. Zero all the votes. ## Initialization -1. Do pruning based on all entries in `PastCodePruning` with `BlockNumber <= now`. Update the - corresponding `PastCodeMeta` and `PastCode` accordingly. +1. Do pruning based on all entries in `PastCodePruning` with `BlockNumber <= now`. Update the corresponding + `PastCodeMeta` and `PastCode` accordingly. 1. Toggle the upgrade related signals - 1. Collect all `(para_id, expected_at)` from `UpcomingUpgrades` where `expected_at <= now` and prune them. For each para pruned set `UpgradeGoAheadSignal` to `GoAhead`. Reserve weight for the state modification to upgrade each para pruned. - 1. Collect all `(para_id, next_possible_upgrade_at)` from `UpgradeCooldowns` where `next_possible_upgrade_at <= now`. For each para obtained this way reserve weight to remove its `UpgradeRestrictionSignal` on finalization. + 1. Collect all `(para_id, expected_at)` from `UpcomingUpgrades` where `expected_at <= now` and prune them. For each + para pruned set `UpgradeGoAheadSignal` to `GoAhead`. Reserve weight for the state modification to upgrade each para + pruned. + 1. Collect all `(para_id, next_possible_upgrade_at)` from `UpgradeCooldowns` where `next_possible_upgrade_at <= now`. + For each para obtained this way reserve weight to remove its `UpgradeRestrictionSignal` on finalization. ## Routines -* `schedule_para_initialize(ParaId, ParaGenesisArgs)`: Schedule a para to be initialized at the next - session. Noop if para is already registered in the system with some `ParaLifecycle`. -* `schedule_para_cleanup(ParaId)`: Schedule a para to be cleaned up after the next full session. -* `schedule_parathread_upgrade(ParaId)`: Schedule a parathread (on-demand parachain) to be upgraded to a parachain. -* `schedule_parachain_downgrade(ParaId)`: Schedule a parachain to be downgraded from lease holding to on-demand. -* `schedule_code_upgrade(ParaId, new_code, relay_parent: BlockNumber, HostConfiguration)`: Schedule a future code - upgrade of the given parachain. In case the PVF pre-checking is disabled, or the new code is already present in the storage, the upgrade will be applied after inclusion of a block of the same parachain - executed in the context of a relay-chain block with number >= `relay_parent + config.validation_upgrade_delay`. If the upgrade is scheduled `UpgradeRestrictionSignal` is set and it will remain set until `relay_parent + config.validation_upgrade_cooldown`. -In case the PVF pre-checking is enabled, or the new code is not already present in the storage, then the PVF pre-checking run will be scheduled for that validation code. If the pre-checking concludes with rejection, then the upgrade is canceled. Otherwise, after pre-checking is concluded the upgrade will be scheduled and be enacted as described above. -* `note_new_head(ParaId, HeadData, BlockNumber)`: note that a para has progressed to a new head, - where the new head was executed in the context of a relay-chain block with given number, the latter value is inserted into the `MostRecentContext` mapping. This will apply pending code upgrades based on the block number provided. If an upgrade took place it will clear the `UpgradeGoAheadSignal`. -* `lifecycle(ParaId) -> Option`: Return the `ParaLifecycle` of a para. -* `is_parachain(ParaId) -> bool`: Returns true if the para ID references any live lease holding parachain, - including those which may be transitioning to an on-demand parachain in the future. -* `is_parathread(ParaId) -> bool`: Returns true if the para ID references any live parathread (on-demand parachain), +- `schedule_para_initialize(ParaId, ParaGenesisArgs)`: Schedule a para to be initialized at the next session. Noop if + para is already registered in the system with some `ParaLifecycle`. +- `schedule_para_cleanup(ParaId)`: Schedule a para to be cleaned up after the next full session. +- `schedule_parathread_upgrade(ParaId)`: Schedule a parathread (on-demand parachain) to be upgraded to a parachain. +- `schedule_parachain_downgrade(ParaId)`: Schedule a parachain to be downgraded from lease holding to on-demand. +- `schedule_code_upgrade(ParaId, new_code, relay_parent: BlockNumber, HostConfiguration)`: Schedule a future code + upgrade of the given parachain. In case the PVF pre-checking is disabled, or the new code is already present in the + storage, the upgrade will be applied after inclusion of a block of the same parachain executed in the context of a +relay-chain block with number >= `relay_parent + config.validation_upgrade_delay`. If the upgrade is scheduled +`UpgradeRestrictionSignal` is set and it will remain set until `relay_parent + config.validation_upgrade_cooldown`. In +case the PVF pre-checking is enabled, or the new code is not already present in the storage, then the PVF pre-checking +run will be scheduled for that validation code. If the pre-checking concludes with rejection, then the upgrade is +canceled. Otherwise, after pre-checking is concluded the upgrade will be scheduled and be enacted as described above. +- `note_new_head(ParaId, HeadData, BlockNumber)`: note that a para has progressed to a new head, where the new head was + executed in the context of a relay-chain block with given number, the latter value is inserted into the + `MostRecentContext` mapping. This will apply pending code upgrades based on the block number provided. If an upgrade + took place it will clear the `UpgradeGoAheadSignal`. +- `lifecycle(ParaId) -> Option`: Return the `ParaLifecycle` of a para. +- `is_parachain(ParaId) -> bool`: Returns true if the para ID references any live lease holding parachain, including + those which may be transitioning to an on-demand parachain in the future. +- `is_parathread(ParaId) -> bool`: Returns true if the para ID references any live parathread (on-demand parachain), including those which may be transitioning to a lease holding parachain in the future. -* `is_valid_para(ParaId) -> bool`: Returns true if the para ID references either a live on-demand parachain - or live lease holding parachain. -* `can_upgrade_validation_code(ParaId) -> bool`: Returns true if the given para can signal code upgrade right now. -* `pvfs_require_prechecking() -> Vec`: Returns the list of PVF validation code hashes that require PVF pre-checking votes. +- `is_valid_para(ParaId) -> bool`: Returns true if the para ID references either a live on-demand parachain or live + lease holding parachain. +- `can_upgrade_validation_code(ParaId) -> bool`: Returns true if the given para can signal code upgrade right now. +- `pvfs_require_prechecking() -> Vec`: Returns the list of PVF validation code hashes that require + PVF pre-checking votes. ## Finalization -Collect all `(para_id, next_possible_upgrade_at)` from `UpgradeCooldowns` where `next_possible_upgrade_at <= now` and prune them. For each para pruned remove its `UpgradeRestrictionSignal`. +Collect all `(para_id, next_possible_upgrade_at)` from `UpgradeCooldowns` where `next_possible_upgrade_at <= now` and +prune them. For each para pruned remove its `UpgradeRestrictionSignal`. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/scheduler.md b/polkadot/roadmap/implementers-guide/src/runtime/scheduler.md index 312ecedcb50f7c0108248f4dc9803465be2a0839..2b8832946fc990e5cf2d1e38211e6bec9e78fd89 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/scheduler.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/scheduler.md @@ -1,6 +1,7 @@ # Scheduler Pallet -> TODO: this section is still heavily under construction. key questions about availability cores and validator assignment are still open and the flow of the the section may be contradictory or inconsistent +> TODO: this section is still heavily under construction. key questions about availability cores and validator +> assignment are still open and the flow of the the section may be contradictory or inconsistent The Scheduler module is responsible for two main tasks: @@ -9,16 +10,29 @@ The Scheduler module is responsible for two main tasks: It aims to achieve these tasks with these goals in mind: -- It should be possible to know at least a block ahead-of-time, ideally more, which validators are going to be assigned to which parachains. +- It should be possible to know at least a block ahead-of-time, ideally more, which validators are going to be assigned + to which parachains. - Parachains that have a candidate pending availability in this fork of the chain should not be assigned. -- Validator assignments should not be gameable. Malicious cartels should not be able to manipulate the scheduler to assign themselves as desired. +- Validator assignments should not be gameable. Malicious cartels should not be able to manipulate the scheduler to + assign themselves as desired. - High or close to optimal throughput of parachains. Work among validator groups should be balanced. ## Availability Cores -The Scheduler manages resource allocation using the concept of "Availability Cores". There will be one availability core for each lease holding parachain, and a fixed number of cores used for multiplexing on-demand parachains. Validators will be partitioned into groups, with the same number of groups as availability cores. Validator groups will be assigned to different availability cores over time. - -An availability core can exist in either one of two states at the beginning or end of a block: free or occupied. A free availability core can have a lease holding or on-demand parachain assigned to it for the potential to have a backed candidate included. After backing, the core enters the occupied state as the backed candidate is pending availability. There is an important distinction: a core is not considered occupied until it is in charge of a block pending availability, although the implementation may treat scheduled cores the same as occupied ones for brevity. A core exits the occupied state when the candidate is no longer pending availability - either on timeout or on availability. A core starting in the occupied state can move to the free state and back to occupied all within a single block, as availability bitfields are processed before backed candidates. At the end of the block, there is a possible timeout on availability which can move the core back to the free state if occupied. +The Scheduler manages resource allocation using the concept of "Availability Cores". There will be one availability core +for each lease holding parachain, and a fixed number of cores used for multiplexing on-demand parachains. Validators +will be partitioned into groups, with the same number of groups as availability cores. Validator groups will be assigned +to different availability cores over time. + +An availability core can exist in either one of two states at the beginning or end of a block: free or occupied. A free +availability core can have a lease holding or on-demand parachain assigned to it for the potential to have a backed +candidate included. After backing, the core enters the occupied state as the backed candidate is pending availability. +There is an important distinction: a core is not considered occupied until it is in charge of a block pending +availability, although the implementation may treat scheduled cores the same as occupied ones for brevity. A core exits +the occupied state when the candidate is no longer pending availability - either on timeout or on availability. A core +starting in the occupied state can move to the free state and back to occupied all within a single block, as +availability bitfields are processed before backed candidates. At the end of the block, there is a possible timeout on +availability which can move the core back to the free state if occupied. Cores are treated as an ordered list and are typically referred to by their index in that list. @@ -82,19 +96,47 @@ digraph { ## Validator Groups -Validator group assignments do not need to change very quickly. The security benefits of fast rotation are redundant with the challenge mechanism in the [Approval process](../protocol-approval.md). Because of this, we only divide validators into groups at the beginning of the session and do not shuffle membership during the session. However, we do take steps to ensure that no particular validator group has dominance over a single lease holding parachain or on-demand parachain-multiplexer for an entire session to provide better guarantees of live-ness. - -Validator groups rotate across availability cores in a round-robin fashion, with rotation occurring at fixed intervals. The i'th group will be assigned to the `(i+k)%n`'th core at any point in time, where `k` is the number of rotations that have occurred in the session, and `n` is the number of cores. This makes upcoming rotations within the same session predictable. - -When a rotation occurs, validator groups are still responsible for distributing availability chunks for any previous cores that are still occupied and pending availability. In practice, rotation and availability-timeout frequencies should be set so this will only be the core they have just been rotated from. It is possible that a validator group is rotated onto a core which is currently occupied. In this case, the validator group will have nothing to do until the previously-assigned group finishes their availability work and frees the core or the availability process times out. Depending on if the core is for a lease holding parachain or on-demand parachain, a different timeout `t` from the [`HostConfiguration`](../types/runtime.md#host-configuration) will apply. Availability timeouts should only be triggered in the first `t-1` blocks after the beginning of a rotation. +Validator group assignments do not need to change very quickly. The security benefits of fast rotation are redundant +with the challenge mechanism in the [Approval process](../protocol-approval.md). Because of this, we only divide +validators into groups at the beginning of the session and do not shuffle membership during the session. However, we do +take steps to ensure that no particular validator group has dominance over a single lease holding parachain or on-demand +parachain-multiplexer for an entire session to provide better guarantees of live-ness. + +Validator groups rotate across availability cores in a round-robin fashion, with rotation occurring at fixed intervals. +The i'th group will be assigned to the `(i+k)%n`'th core at any point in time, where `k` is the number of rotations that +have occurred in the session, and `n` is the number of cores. This makes upcoming rotations within the same session +predictable. + +When a rotation occurs, validator groups are still responsible for distributing availability chunks for any previous +cores that are still occupied and pending availability. In practice, rotation and availability-timeout frequencies +should be set so this will only be the core they have just been rotated from. It is possible that a validator group is +rotated onto a core which is currently occupied. In this case, the validator group will have nothing to do until the +previously-assigned group finishes their availability work and frees the core or the availability process times out. +Depending on if the core is for a lease holding parachain or on-demand parachain, a different timeout `t` from the +[`HostConfiguration`](../types/runtime.md#host-configuration) will apply. Availability timeouts should only be triggered +in the first `t-1` blocks after the beginning of a rotation. ## Claims -On-demand parachains operate on a system of claims. Collators purchase claims on authoring the next block of an on-demand parachain, although the purchase mechanism is beyond the scope of the scheduler. The scheduler guarantees that they'll be given at least a certain number of attempts to author a candidate that is backed. Attempts that fail during the availability phase are not counted, since ensuring availability at that stage is the responsibility of the backing validators, not of the collator. When a claim is accepted, it is placed into a queue of claims, and each claim is assigned to a particular on-demand parachain-multiplexing core in advance. Given that the current assignments of validator groups to cores are known, and the upcoming assignments are predictable, it is possible for on-demand parachain collators to know who they should be talking to now and how they should begin establishing connections with as a fallback. - -With this information, the Node-side can be aware of which on-demand parachains have a good chance of being includable within the relay-chain block and can focus any additional resources on backing candidates from those on-demand parachains. Furthermore, Node-side code is aware of which validator group will be responsible for that thread. If the necessary conditions are reached for core reassignment, those candidates can be backed within the same block as the core being freed. - -On-demand claims, when scheduled onto a free core, may not result in a block pending availability. This may be due to collator error, networking timeout, or censorship by the validator group. In this case, the claims should be retried a certain number of times to give the collator a fair shot. +On-demand parachains operate on a system of claims. Collators purchase claims on authoring the next block of an +on-demand parachain, although the purchase mechanism is beyond the scope of the scheduler. The scheduler guarantees that +they'll be given at least a certain number of attempts to author a candidate that is backed. Attempts that fail during +the availability phase are not counted, since ensuring availability at that stage is the responsibility of the backing +validators, not of the collator. When a claim is accepted, it is placed into a queue of claims, and each claim is +assigned to a particular on-demand parachain-multiplexing core in advance. Given that the current assignments of +validator groups to cores are known, and the upcoming assignments are predictable, it is possible for on-demand +parachain collators to know who they should be talking to now and how they should begin establishing connections with as +a fallback. + +With this information, the Node-side can be aware of which on-demand parachains have a good chance of being includable +within the relay-chain block and can focus any additional resources on backing candidates from those on-demand +parachains. Furthermore, Node-side code is aware of which validator group will be responsible for that thread. If the +necessary conditions are reached for core reassignment, those candidates can be backed within the same block as the core +being freed. + +On-demand claims, when scheduled onto a free core, may not result in a block pending availability. This may be due to +collator error, networking timeout, or censorship by the validator group. In this case, the claims should be retried a +certain number of times to give the collator a fair shot. ## Storage @@ -104,7 +146,7 @@ Utility structs: // A claim on authoring the next block for a given parathread (on-demand parachain). struct ParathreadClaim(ParaId, CollatorId); -// An entry tracking a parathread (on-demand parachain) claim to ensure it does not +// An entry tracking a parathread (on-demand parachain) claim to ensure it does not // pass the maximum number of retries. struct ParathreadEntry { claim: ParathreadClaim, @@ -165,7 +207,7 @@ ParathreadClaimIndex: Vec; /// The block number where the session start occurred. Used to track how many group rotations have occurred. SessionStartBlock: BlockNumber; /// Currently scheduled cores - free but up to be occupied. -/// The value contained here will not be valid after the end of a block. +/// The value contained here will not be valid after the end of a block. /// Runtime APIs should be used to determine scheduled cores /// for the upcoming block. Scheduled: Vec, // sorted ascending by CoreIndex. @@ -173,13 +215,17 @@ Scheduled: Vec, // sorted ascending by CoreIndex. ## Session Change -Session changes are the only time that configuration can change, and the [Configuration module](configuration.md)'s session-change logic is handled before this module's. We also lean on the behavior of the [Inclusion module](inclusion.md) which clears all its occupied cores on session change. Thus we don't have to worry about cores being occupied across session boundaries and it is safe to re-size the `AvailabilityCores` bitfield. +Session changes are the only time that configuration can change, and the [Configuration module](configuration.md)'s +session-change logic is handled before this module's. We also lean on the behavior of the [Inclusion +module](inclusion.md) which clears all its occupied cores on session change. Thus we don't have to worry about cores +being occupied across session boundaries and it is safe to re-size the `AvailabilityCores` bitfield. Actions: 1. Set `SessionStartBlock` to current block number + 1, as session changes are applied at the end of the block. 1. Clear all `Some` members of `AvailabilityCores`. Return all parathread claims to queue with retries un-incremented. -1. Set `configuration = Configuration::configuration()` (see [`HostConfiguration`](../types/runtime.md#host-configuration)) +1. Set `configuration = Configuration::configuration()` (see + [`HostConfiguration`](../types/runtime.md#host-configuration)) 1. Fetch `Shared::ActiveValidators` as AV. 1. Determine the number of cores & validator groups as `n_cores`. This is the maximum of 1. `Paras::parachains().len() + configuration.parathread_cores` @@ -187,13 +233,18 @@ Actions: 1. Resize `AvailabilityCores` to have length `n_cores` with all `None` entries. 1. Compute new validator groups by shuffling using a secure randomness beacon - Note that the total number of validators `V` in AV may not be evenly divided by `n_cores`. - - The groups are selected by partitioning AV. The first `V % N` groups will have `(V / n_cores) + 1` members, while the remaining groups will have `(V / N)` members each. - - Instead of using the indices within AV, which point to the broader set, indices _into_ AV should be used. This implies that groups should have simply ascending validator indices. + - The groups are selected by partitioning AV. The first `V % N` groups will have `(V / n_cores) + 1` members, while + the remaining groups will have `(V / N)` members each. + - Instead of using the indices within AV, which point to the broader set, indices _into_ AV should be used. This + implies that groups should have simply ascending validator indices. 1. Prune the parathread (on-demand parachain) queue to remove all retries beyond `configuration.parathread_retries`. - Also prune all on-demand claims corresponding to de-registered parachains. - all pruned claims should have their entry removed from the parathread (on-demand parachain) index. - - assign all non-pruned claims to new cores if the number of on-demand parachain cores has changed between the `new_config` and `old_config` of the `SessionChangeNotification`. - - Assign claims in equal balance across all cores if rebalancing, and set the `next_core` of the `ParathreadQueue` (on-demand queue) by incrementing the relative index of the last assigned core and taking it modulo the number of on-demand cores. + - assign all non-pruned claims to new cores if the number of on-demand parachain cores has changed between the + `new_config` and `old_config` of the `SessionChangeNotification`. + - Assign claims in equal balance across all cores if rebalancing, and set the `next_core` of the `ParathreadQueue` + (on-demand queue) by incrementing the relative index of the last assigned core and taking it modulo the number of + on-demand cores. ## Initialization @@ -208,28 +259,52 @@ No finalization routine runs for this module. - `add_parathread_claim(ParathreadClaim)`: Add a parathread (on-demand parachain) claim to the queue. - Fails if any on-demand claim on the same parachain is currently indexed. - Fails if the queue length is >= `config.scheduling_lookahead * config.parathread_cores`. - - The core used for the on-demand claim is the `next_core` field of the `ParathreadQueue` (on-demand queue) and adding `Paras::parachains().len()` to it. + - The core used for the on-demand claim is the `next_core` field of the `ParathreadQueue` (on-demand queue) and adding + `Paras::parachains().len()` to it. - `next_core` is then updated by adding 1 and taking it modulo `config.parathread_cores`. - The claim is then added to the claim index. -- `free_cores(Vec<(CoreIndex, FreedReason)>)`: indicate previosuly-occupied cores which are to be considered returned and why they are being returned. +- `free_cores(Vec<(CoreIndex, FreedReason)>)`: indicate previosuly-occupied cores which are to be considered returned + and why they are being returned. - All freed lease holding parachain cores should be assigned to their respective parachain - - All freed on-demand parachain cores whose reason for freeing was `FreedReason::Concluded` should have the claim removed from the claim index. - - All freed on-demand cores whose reason for freeing was `FreedReason::TimedOut` should have the claim added to the parathread queue (on-demand queue) again without retries incremented + - All freed on-demand parachain cores whose reason for freeing was `FreedReason::Concluded` should have the claim + removed from the claim index. + - All freed on-demand cores whose reason for freeing was `FreedReason::TimedOut` should have the claim added to the + parathread queue (on-demand queue) again without retries incremented - All freed on-demand cores should take the next on-demand parachain entry from the queue. -- `schedule(Vec<(CoreIndex, FreedReason)>, now: BlockNumber)`: schedule new core assignments, with a parameter indicating previously-occupied cores which are to be considered returned and why they are being returned. +- `schedule(Vec<(CoreIndex, FreedReason)>, now: BlockNumber)`: schedule new core assignments, with a parameter + indicating previously-occupied cores which are to be considered returned and why they are being returned. - Invoke `free_cores(freed_cores)` - - The i'th validator group will be assigned to the `(i+k)%n`'th core at any point in time, where `k` is the number of rotations that have occurred in the session, and `n` is the total number of cores. This makes upcoming rotations within the same session predictable. Rotations are based off of `now`. + - The i'th validator group will be assigned to the `(i+k)%n`'th core at any point in time, where `k` is the number of + rotations that have occurred in the session, and `n` is the total number of cores. This makes upcoming rotations + within the same session predictable. Rotations are based off of `now`. - `scheduled() -> Vec`: Get currently scheduled core assignments. - `occupied(Vec)`. Note that the given cores have become occupied. - Behavior undefined if any given cores were not scheduled. - Behavior undefined if the given cores are not sorted ascending by core index - This clears them from `Scheduled` and marks each corresponding `core` in the `AvailabilityCores` as occupied. - - Since both the availability cores and the newly-occupied cores lists are sorted ascending, this method can be implemented efficiently. + - Since both the availability cores and the newly-occupied cores lists are sorted ascending, this method can be + implemented efficiently. - `core_para(CoreIndex) -> ParaId`: return the currently-scheduled or occupied ParaId for the given core. -- `group_validators(GroupIndex) -> Option>`: return all validators in a given group, if the group index is valid for this session. -- `availability_timeout_predicate() -> Option bool>`: returns an optional predicate that should be used for timing out occupied cores. if `None`, no timing-out should be done. The predicate accepts the index of the core, and the block number since which it has been occupied. The predicate should be implemented based on the time since the last validator group rotation, and the respective parachain timeouts, i.e. only within `max(config.chain_availability_period, config.thread_availability_period)` of the last rotation would this return `Some`. +- `group_validators(GroupIndex) -> Option>`: return all validators in a given group, if the group + index is valid for this session. +- `availability_timeout_predicate() -> Option bool>`: returns an optional predicate + that should be used for timing out occupied cores. if `None`, no timing-out should be done. The predicate accepts the + index of the core, and the block number since which it has been occupied. The predicate should be implemented based on + the time since the last validator group rotation, and the respective parachain timeouts, i.e. only within + `max(config.chain_availability_period, config.thread_availability_period)` of the last rotation would this return + `Some`. - `group_rotation_info(now: BlockNumber) -> GroupRotationInfo`: Returns a helper for determining group rotation. -- `next_up_on_available(CoreIndex) -> Option`: Return the next thing that will be scheduled on this core assuming it is currently occupied and the candidate occupying it became available. Returns in `ScheduledCore` format (todo: link to Runtime APIs page; linkcheck doesn't allow this right now). For lease holding parachains, this is always the ID of the parachain and no specified collator. For on-demand parachains, this is based on the next item in the `ParathreadQueue` (on-demand queue) assigned to that core, and is `None` if there isn't one. -- `next_up_on_time_out(CoreIndex) -> Option`: Return the next thing that will be scheduled on this core assuming it is currently occupied and the candidate occupying it timed out. Returns in `ScheduledCore` format (todo: link to Runtime APIs page; linkcheck doesn't allow this right now). For parachains, this is always the ID of the parachain and no specified collator. For on-demand parachains, this is based on the next item in the `ParathreadQueue` (on-demand queue) assigned to that core, or if there isn't one, the claim that is currently occupying the core. Otherwise `None`. +- `next_up_on_available(CoreIndex) -> Option`: Return the next thing that will be scheduled on this core + assuming it is currently occupied and the candidate occupying it became available. Returns in `ScheduledCore` format + (todo: link to Runtime APIs page; linkcheck doesn't allow this right now). For lease holding parachains, this is + always the ID of the parachain and no specified collator. For on-demand parachains, this is based on the next item in + the `ParathreadQueue` (on-demand queue) assigned to that core, and is `None` if there isn't one. +- `next_up_on_time_out(CoreIndex) -> Option`: Return the next thing that will be scheduled on this core + assuming it is currently occupied and the candidate occupying it timed out. Returns in `ScheduledCore` format (todo: + link to Runtime APIs page; linkcheck doesn't allow this right now). For parachains, this is always the ID of the + parachain and no specified collator. For on-demand parachains, this is based on the next item in the `ParathreadQueue` + (on-demand queue) assigned to that core, or if there isn't one, the claim that is currently occupying the core. + Otherwise `None`. - `clear()`: - - Free all scheduled cores and return on-demand claims to queue, with retries incremented. Skip on-demand parachains which no longer exist under paras. + - Free all scheduled cores and return on-demand claims to queue, with retries incremented. Skip on-demand parachains + which no longer exist under paras. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/session_info.md b/polkadot/roadmap/implementers-guide/src/runtime/session_info.md index 5ee63ab5a903072935b75abc909d164af5ae5d6b..0442ee57505d0823b98b54bed79596e09a16f641 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/session_info.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/session_info.md @@ -1,6 +1,8 @@ # Session Info -For disputes and approvals, we need access to information about validator sets from prior sessions. We also often want easy access to the same information about the current session's validator set. This module aggregates and stores this information in a rolling window while providing easy APIs for access. +For disputes and approvals, we need access to information about validator sets from prior sessions. We also often want +easy access to the same information about the current session's validator set. This module aggregates and stores this +information in a rolling window while providing easy APIs for access. ## Storage @@ -66,10 +68,14 @@ Sessions: map SessionIndex => Option, ## Session Change -1. Update `EarliestStoredSession` based on `config.dispute_period` and remove all entries from `Sessions` from the previous value up to the new value. -1. Create a new entry in `Sessions` with information about the current session. Use `shared::ActiveValidators` to determine the indices into the broader validator sets (validation, assignment, discovery) which are actually used for parachain validation. Only these validators should appear in the `SessionInfo`. +1. Update `EarliestStoredSession` based on `config.dispute_period` and remove all entries from `Sessions` from the + previous value up to the new value. +1. Create a new entry in `Sessions` with information about the current session. Use `shared::ActiveValidators` to + determine the indices into the broader validator sets (validation, assignment, discovery) which are actually used for + parachain validation. Only these validators should appear in the `SessionInfo`. ## Routines * `earliest_stored_session() -> SessionIndex`: Yields the earliest session for which we have information stored. -* `session_info(session: SessionIndex) -> Option`: Yields the session info for the given session, if stored. +* `session_info(session: SessionIndex) -> Option`: Yields the session info for the given session, if + stored. diff --git a/polkadot/roadmap/implementers-guide/src/runtime/shared.md b/polkadot/roadmap/implementers-guide/src/runtime/shared.md index 0f173134e2a2c8b205f6429ff3267d3706842f6f..ebed240838ab93e2cabaa9cde01cf8828337e708 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/shared.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/shared.md @@ -2,11 +2,11 @@ This module is responsible for managing shared storage and configuration for other modules. -It is important that other pallets are able to use the Shared Module, so it should not have a -dependency on any other modules in the Parachains Runtime. +It is important that other pallets are able to use the Shared Module, so it should not have a dependency on any other +modules in the Parachains Runtime. -For the moment, it is used exclusively to track the current session index across the Parachains -Runtime system, and when it should be allowed to schedule future changes to Paras or Configurations. +For the moment, it is used exclusively to track the current session index across the Parachains Runtime system, and when +it should be allowed to schedule future changes to Paras or Configurations. ## Constants @@ -57,24 +57,26 @@ AllowedRelayParents: AllowedRelayParentsTracker, The Shared Module currently has no initialization routines. -The Shared Module is initialized directly after the Configuration module, but before all other -modules. It is important to update the Shared Module before any other module since its state may be -used within the logic of other modules, and it is important that the state is consistent across -them. +The Shared Module is initialized directly after the Configuration module, but before all other modules. It is important +to update the Shared Module before any other module since its state may be used within the logic of other modules, and +it is important that the state is consistent across them. ## Session Change -During a session change, the Shared Module receives and stores the current Session Index directly from the initializer module, along with the broader validator set, and it returns the new list of validators. +During a session change, the Shared Module receives and stores the current Session Index directly from the initializer +module, along with the broader validator set, and it returns the new list of validators. -The list of validators should be first shuffled according to the chain's random seed and then truncated. The indices of these validators should be set to `ActiveValidatorIndices` and then returned back to the initializer. `ActiveValidatorKeys` should be set accordingly. +The list of validators should be first shuffled according to the chain's random seed and then truncated. The indices of +these validators should be set to `ActiveValidatorIndices` and then returned back to the initializer. +`ActiveValidatorKeys` should be set accordingly. This information is used in the: -* Configuration Module: For delaying updates to configurations until at lease one full session has - passed. +* Configuration Module: For delaying updates to configurations until at lease one full session has passed. * Paras Module: For delaying updates to paras until at least one full session has passed. -Allowed relay parents buffer, which is maintained by [ParaInherent](./parainherent.md) module, is cleared on every session change. +Allowed relay parents buffer, which is maintained by [ParaInherent](./parainherent.md) module, is cleared on every +session change. ## Finalization @@ -82,6 +84,6 @@ The Shared Module currently has no finalization routines. ## Functions -* `scheduled_sessions() -> SessionIndex`: Return the next session index where updates to the - Parachains Runtime system would be safe to apply. +* `scheduled_sessions() -> SessionIndex`: Return the next session index where updates to the Parachains Runtime system + would be safe to apply. * `set_session_index(SessionIndex)`: For tests. Set the current session index in the Shared Module. diff --git a/polkadot/roadmap/implementers-guide/src/types/approval.md b/polkadot/roadmap/implementers-guide/src/types/approval.md index b58e0a8187e1d3af2eae4a7a0ec8d1ee64368b56..bc33f024426fefcf0809feeeddef2f83882c1914 100644 --- a/polkadot/roadmap/implementers-guide/src/types/approval.md +++ b/polkadot/roadmap/implementers-guide/src/types/approval.md @@ -6,9 +6,11 @@ The public key of a keypair used by a validator for determining assignments to a ## `AssignmentCert` -An `AssignmentCert`, short for Assignment Certificate, is a piece of data provided by a validator to prove that they have been selected to perform approval checks on an included candidate. +An `AssignmentCert`, short for Assignment Certificate, is a piece of data provided by a validator to prove that they +have been selected to perform approval checks on an included candidate. -These certificates can be checked in the context of a specific block, candidate, and validator assignment VRF key. The block state will also provide further context about the availability core states at that block. +These certificates can be checked in the context of a specific block, candidate, and validator assignment VRF key. The +block state will also provide further context about the availability core states at that block. ```rust enum AssignmentCertKind { @@ -53,7 +55,8 @@ struct ApprovalVote(Hash); ## `SignedApprovalVote` -An approval vote signed with a validator's key. This should be verifiable under the `ValidatorId` corresponding to the `ValidatorIndex` of the session, which should be implicit from context. +An approval vote signed with a validator's key. This should be verifiable under the `ValidatorId` corresponding to the +`ValidatorIndex` of the session, which should be implicit from context. ```rust struct SignedApprovalVote { @@ -65,9 +68,11 @@ struct SignedApprovalVote { ## `IndirectSignedApprovalVote` -A signed approval vote which references the candidate indirectly via the block. If there exists a look-up to the candidate hash from the block hash and candidate index, then this can be transformed into a `SignedApprovalVote`. +A signed approval vote which references the candidate indirectly via the block. If there exists a look-up to the +candidate hash from the block hash and candidate index, then this can be transformed into a `SignedApprovalVote`. -Although this vote references the candidate by a specific block hash and candidate index, the signature is computed on the actual `SignedApprovalVote` payload. +Although this vote references the candidate by a specific block hash and candidate index, the signature is computed on +the actual `SignedApprovalVote` payload. ```rust struct IndirectSignedApprovalVote { @@ -82,7 +87,9 @@ struct IndirectSignedApprovalVote { ## `CheckedAssignmentCert` -An assignment cert which has checked both the VRF and the validity of the implied assignment according to the selection criteria rules of the protocol. This type should be declared in such a way as to be instantiatable only when the checks have actually been done. Fields should be accessible via getters, not direct struct access. +An assignment cert which has checked both the VRF and the validity of the implied assignment according to the selection +criteria rules of the protocol. This type should be declared in such a way as to be instantiatable only when the checks +have actually been done. Fields should be accessible via getters, not direct struct access. ```rust struct CheckedAssignmentCert { diff --git a/polkadot/roadmap/implementers-guide/src/types/availability.md b/polkadot/roadmap/implementers-guide/src/types/availability.md index e2b90e86f43fe6ce4d8e679866d7f6e7ce002865..a04b896c12816795b27850810d536a44dc3e16eb 100644 --- a/polkadot/roadmap/implementers-guide/src/types/availability.md +++ b/polkadot/roadmap/implementers-guide/src/types/availability.md @@ -1,13 +1,12 @@ # Availability -One of the key roles of validators is to ensure availability of all data necessary to validate -candidates for the duration of a challenge period. This is done via an erasure-coding of the data to keep available. +One of the key roles of validators is to ensure availability of all data necessary to validate candidates for the +duration of a challenge period. This is done via an erasure-coding of the data to keep available. ## Signed Availability Bitfield A bitfield [signed](backing.md#signed-wrapper) by a particular validator about the availability of pending candidates. - ```rust type SignedAvailabilityBitfield = Signed; @@ -16,26 +15,30 @@ struct Bitfields(Vec<(SignedAvailabilityBitfield)>), // bitfields sorted by vali ### Semantics -A `SignedAvailabilityBitfield` represents the view from a particular validator's perspective. Each bit in the bitfield corresponds to a single [availability core](../runtime-api/availability-cores.md). A `1` bit indicates that the validator believes the following statements to be true for a core: +A `SignedAvailabilityBitfield` represents the view from a particular validator's perspective. Each bit in the bitfield +corresponds to a single [availability core](../runtime-api/availability-cores.md). A `1` bit indicates that the +validator believes the following statements to be true for a core: - the availability core is occupied -- there exists a [`CommittedCandidateReceipt`](candidate.html#committed-candidate-receipt) corresponding to that core. In other words, that para has a block in progress. +- there exists a [`CommittedCandidateReceipt`](candidate.html#committed-candidate-receipt) corresponding to that core. + In other words, that para has a block in progress. - the validator's [Availability Store](../node/utility/availability-store.md) contains a chunk of that parablock's PoV. In other words, it is the transpose of [`OccupiedCore::availability`](../runtime-api/availability-cores.md). ## Proof-of-Validity -Often referred to as PoV, this is a type-safe wrapper around bytes (`Vec`) when referring to data that acts as a stateless-client proof of validity of a candidate, when used as input to the validation function of the para. +Often referred to as PoV, this is a type-safe wrapper around bytes (`Vec`) when referring to data that acts as a +stateless-client proof of validity of a candidate, when used as input to the validation function of the para. ```rust struct PoV(Vec); ``` - ## Available Data -This is the data we want to keep available for each [candidate](candidate.md) included in the relay chain. This is the PoV of the block, as well as the [`PersistedValidationData`](candidate.md#persistedvalidationdata) +This is the data we want to keep available for each [candidate](candidate.md) included in the relay chain. This is the +PoV of the block, as well as the [`PersistedValidationData`](candidate.md#persistedvalidationdata) ```rust struct AvailableData { @@ -50,7 +53,10 @@ struct AvailableData { ## Erasure Chunk -The [`AvailableData`](#availabledata) is split up into an erasure-coding as part of the availability process. Each validator gets a chunk. This describes one of those chunks, along with its proof against a merkle root hash, which should be apparent from context, and is the `erasure_root` field of a [`CandidateDescriptor`](candidate.md#candidatedescriptor). +The [`AvailableData`](#availabledata) is split up into an erasure-coding as part of the availability process. Each +validator gets a chunk. This describes one of those chunks, along with its proof against a merkle root hash, which +should be apparent from context, and is the `erasure_root` field of a +[`CandidateDescriptor`](candidate.md#candidatedescriptor). ```rust diff --git a/polkadot/roadmap/implementers-guide/src/types/backing.md b/polkadot/roadmap/implementers-guide/src/types/backing.md index 5fcb3ae161b621b9fd1cc7e91bcc89561980618d..7e43325ec5cdadcbd7d233d00f9a59dd56dd22df 100644 --- a/polkadot/roadmap/implementers-guide/src/types/backing.md +++ b/polkadot/roadmap/implementers-guide/src/types/backing.md @@ -1,12 +1,15 @@ # Backing Types -[Candidates](candidate.md) go through many phases before being considered included in a fork of the relay chain and eventually accepted. +[Candidates](candidate.md) go through many phases before being considered included in a fork of the relay chain and +eventually accepted. -These types describe the data used in the backing phase. Some are sent over the wire within subsystems, and some are simply included in the relay-chain block. +These types describe the data used in the backing phase. Some are sent over the wire within subsystems, and some are +simply included in the relay-chain block. ## Validity Attestation -An attestation of validity for a candidate, used as part of a backing. Both the `Seconded` and `Valid` statements are considered attestations of validity. This structure is only useful where the candidate referenced is apparent. +An attestation of validity for a candidate, used as part of a backing. Both the `Seconded` and `Valid` statements are +considered attestations of validity. This structure is only useful where the candidate referenced is apparent. ```rust enum ValidityAttestation { @@ -21,7 +24,8 @@ enum ValidityAttestation { ## Signed Wrapper -There are a few distinct types which we desire to sign, and validate the signatures of. Instead of duplicating this work, we extract a signed wrapper. +There are a few distinct types which we desire to sign, and validate the signatures of. Instead of duplicating this +work, we extract a signed wrapper. ```rust,ignore /// A signed type which encapsulates the common desire to sign some data and validate a signature. @@ -44,13 +48,18 @@ impl, RealPayload: Encode> Signed`. Therefore, for the generic case where `RealPayload = Payload`, it changes nothing. However, we `impl EncodeAs for Statement`, which helps efficiency. +`EncodeAs` is a helper trait with a blanket impl which ensures that any `T` can `EncodeAs`. Therefore, for the +generic case where `RealPayload = Payload`, it changes nothing. However, we `impl EncodeAs for +Statement`, which helps efficiency. ## Statement Type -The [Candidate Backing subsystem](../node/backing/candidate-backing.md) issues and signs these after candidate validation. +The [Candidate Backing subsystem](../node/backing/candidate-backing.md) issues and signs these after candidate +validation. ```rust /// A statement about the validity of a parachain candidate. @@ -94,11 +103,13 @@ pub type SignedFullStatement = Signed; pub type SignedStatement = Signed; ``` -Munging the signed `Statement` into a `CompactStatement` before signing allows the candidate receipt itself to be omitted when checking a signature on a `Seconded` statement. +Munging the signed `Statement` into a `CompactStatement` before signing allows the candidate receipt itself to be +omitted when checking a signature on a `Seconded` statement. ## Backed Candidate -An [`CommittedCandidateReceipt`](candidate.md#committed-candidate-receipt) along with all data necessary to prove its backing. This is submitted to the relay-chain to process and move along the candidate to the pending-availability stage. +An [`CommittedCandidateReceipt`](candidate.md#committed-candidate-receipt) along with all data necessary to prove its +backing. This is submitted to the relay-chain to process and move along the candidate to the pending-availability stage. ```rust struct BackedCandidate { diff --git a/polkadot/roadmap/implementers-guide/src/types/candidate.md b/polkadot/roadmap/implementers-guide/src/types/candidate.md index a37f98054c5e5a8430c5d2f3509b2efa3eb5988e..00176229e5a4356a9f88ae91c16eb46617b47216 100644 --- a/polkadot/roadmap/implementers-guide/src/types/candidate.md +++ b/polkadot/roadmap/implementers-guide/src/types/candidate.md @@ -1,15 +1,18 @@ # Candidate Types -Para candidates are some of the most common types, both within the runtime and on the Node-side. -Candidates are the fundamental datatype for advancing parachains, encapsulating the collator's signature, the context of the parablock, the commitments to the output, and a commitment to the data which proves it valid. +Para candidates are some of the most common types, both within the runtime and on the Node-side. Candidates are the +fundamental datatype for advancing parachains, encapsulating the collator's signature, the context of the parablock, the +commitments to the output, and a commitment to the data which proves it valid. -In a way, this entire guide is about these candidates: how they are scheduled, constructed, backed, included, and challenged. +In a way, this entire guide is about these candidates: how they are scheduled, constructed, backed, included, and +challenged. This section will describe the base candidate type, its components, and variants that contain extra data. ## Para Id -A unique 32-bit identifier referring to a specific para (chain or thread). The relay-chain runtime guarantees that `ParaId`s are unique for the duration of any session, but recycling and reuse over a longer period of time is permitted. +A unique 32-bit identifier referring to a specific para (chain or thread). The relay-chain runtime guarantees that +`ParaId`s are unique for the duration of any session, but recycling and reuse over a longer period of time is permitted. ```rust struct ParaId(u32); @@ -17,9 +20,12 @@ struct ParaId(u32); ## Candidate Receipt -Much info in a [`FullCandidateReceipt`](#full-candidate-receipt) is duplicated from the relay-chain state. When the corresponding relay-chain state is considered widely available, the Candidate Receipt should be favored over the `FullCandidateReceipt`. +Much info in a [`FullCandidateReceipt`](#full-candidate-receipt) is duplicated from the relay-chain state. When the +corresponding relay-chain state is considered widely available, the Candidate Receipt should be favored over the +`FullCandidateReceipt`. -Examples of situations where the state is readily available includes within the scope of work done by subsystems working on a given relay-parent, or within the logic of the runtime importing a backed candidate. +Examples of situations where the state is readily available includes within the scope of work done by subsystems working +on a given relay-parent, or within the logic of the runtime importing a backed candidate. ```rust /// A candidate-receipt. @@ -33,9 +39,13 @@ struct CandidateReceipt { ## Full Candidate Receipt -This is the full receipt type. The `PersistedValidationData` are technically redundant with the `inner.relay_parent`, which uniquely describes the block in the blockchain from whose state these values are derived. The [`CandidateReceipt`](#candidate-receipt) variant is often used instead for this reason. +This is the full receipt type. The `PersistedValidationData` are technically redundant with the `inner.relay_parent`, +which uniquely describes the block in the blockchain from whose state these values are derived. The +[`CandidateReceipt`](#candidate-receipt) variant is often used instead for this reason. -However, the Full Candidate Receipt type is useful as a means of avoiding the implicit dependency on availability of old blockchain state. In situations such as availability and approval, having the full description of the candidate within a self-contained struct is convenient. +However, the Full Candidate Receipt type is useful as a means of avoiding the implicit dependency on availability of old +blockchain state. In situations such as availability and approval, having the full description of the candidate within a +self-contained struct is convenient. ```rust /// All data pertaining to the execution of a para candidate. @@ -47,9 +57,13 @@ struct FullCandidateReceipt { ## Committed Candidate Receipt -This is a variant of the candidate receipt which includes the commitments of the candidate receipt alongside the descriptor. This should be favored over the [`Candidate Receipt`](#candidate-receipt) in situations where the candidate is not going to be executed but the actual data committed to is important. This is often the case in the backing phase. +This is a variant of the candidate receipt which includes the commitments of the candidate receipt alongside the +descriptor. This should be favored over the [`Candidate Receipt`](#candidate-receipt) in situations where the candidate +is not going to be executed but the actual data committed to is important. This is often the case in the backing phase. -The hash of the committed candidate receipt will be the same as the corresponding [`Candidate Receipt`](#candidate-receipt), because it is computed by first hashing the encoding of the commitments to form a plain [`Candidate Receipt`](#candidate-receipt). +The hash of the committed candidate receipt will be the same as the corresponding [`Candidate +Receipt`](#candidate-receipt), because it is computed by first hashing the encoding of the commitments to form a plain +[`Candidate Receipt`](#candidate-receipt). ```rust /// A candidate-receipt with commitments directly included. @@ -110,15 +124,24 @@ pub struct ValidationParams { ## `PersistedValidationData` -The validation data provides information about how to create the inputs for validation of a candidate. This information is derived from the chain state and will vary from para to para, although some of the fields may be the same for every para. +The validation data provides information about how to create the inputs for validation of a candidate. This information +is derived from the chain state and will vary from para to para, although some of the fields may be the same for every +para. -Since this data is used to form inputs to the validation function, it needs to be persisted by the availability system to avoid dependence on availability of the relay-chain state. +Since this data is used to form inputs to the validation function, it needs to be persisted by the availability system +to avoid dependence on availability of the relay-chain state. -Furthermore, the validation data acts as a way to authorize the additional data the collator needs to pass to the validation function. For example, the validation function can check whether the incoming messages (e.g. downward messages) were actually sent by using the data provided in the validation data using so called MQC heads. +Furthermore, the validation data acts as a way to authorize the additional data the collator needs to pass to the +validation function. For example, the validation function can check whether the incoming messages (e.g. downward +messages) were actually sent by using the data provided in the validation data using so called MQC heads. -Since the commitments of the validation function are checked by the relay-chain, approval checkers can rely on the invariant that the relay-chain only includes para-blocks for which these checks have already been done. As such, there is no need for the validation data used to inform validators and collators about the checks the relay-chain will perform to be persisted by the availability system. +Since the commitments of the validation function are checked by the relay-chain, approval checkers can rely on the +invariant that the relay-chain only includes para-blocks for which these checks have already been done. As such, there +is no need for the validation data used to inform validators and collators about the checks the relay-chain will perform +to be persisted by the availability system. -The `PersistedValidationData` should be relatively lightweight primarily because it is constructed during inclusion for each candidate and therefore lies on the critical path of inclusion. +The `PersistedValidationData` should be relatively lightweight primarily because it is constructed during inclusion for +each candidate and therefore lies on the critical path of inclusion. ```rust struct PersistedValidationData { @@ -150,7 +173,8 @@ struct HeadData(Vec); ## Candidate Commitments -The execution and validation of parachain candidates produces a number of values which either must be committed to blocks on the relay chain or committed to the state of the relay chain. +The execution and validation of parachain candidates produces a number of values which either must be committed to +blocks on the relay chain or committed to the state of the relay chain. ```rust /// Commitments made in a `CandidateReceipt`. Many of these are outputs of validation. @@ -174,7 +198,9 @@ struct CandidateCommitments { ## Signing Context -This struct provides context to signatures by combining with various payloads to localize the signature to a particular session index and relay-chain hash. Having these fields included in the signature makes misbehavior attribution much simpler. +This struct provides context to signatures by combining with various payloads to localize the signature to a particular +session index and relay-chain hash. Having these fields included in the signature makes misbehavior attribution much +simpler. ```rust struct SigningContext { diff --git a/polkadot/roadmap/implementers-guide/src/types/disputes.md b/polkadot/roadmap/implementers-guide/src/types/disputes.md index 24f152b1308f7a1613958cb210aeeef12901fefe..c49e0fea262510ef2e1585f9aa8433dc02883b50 100644 --- a/polkadot/roadmap/implementers-guide/src/types/disputes.md +++ b/polkadot/roadmap/implementers-guide/src/types/disputes.md @@ -28,7 +28,8 @@ enum DisputeStatement { ## Dispute Statement Kinds -Kinds of dispute statements. Each of these can be combined with a candidate hash, session index, validator public key, and validator signature to reproduce and check the original statement. +Kinds of dispute statements. Each of these can be combined with a candidate hash, session index, validator public key, +and validator signature to reproduce and check the original statement. ```rust enum ValidDisputeStatementKind { diff --git a/polkadot/roadmap/implementers-guide/src/types/network.md b/polkadot/roadmap/implementers-guide/src/types/network.md index b698ca2075bfe6e40a2098b2beaaa8c29dd60d27..60f5bda28fa9c40ad0e323279979a62dab74fd5e 100644 --- a/polkadot/roadmap/implementers-guide/src/types/network.md +++ b/polkadot/roadmap/implementers-guide/src/types/network.md @@ -139,7 +139,8 @@ enum CollationProtocolV1 { ## Network Bridge Event -These updates are posted from the [Network Bridge Subsystem](../node/utility/network-bridge.md) to other subsystems based on registered listeners. +These updates are posted from the [Network Bridge Subsystem](../node/utility/network-bridge.md) to other subsystems +based on registered listeners. ```rust struct NewGossipTopology { diff --git a/polkadot/roadmap/implementers-guide/src/types/overseer-protocol.md b/polkadot/roadmap/implementers-guide/src/types/overseer-protocol.md index 3d9037699da6127de0af46a3e67d8b8e531f1f43..fababbff354007c5fcf8f47d6b8e52e69d0879bb 100644 --- a/polkadot/roadmap/implementers-guide/src/types/overseer-protocol.md +++ b/polkadot/roadmap/implementers-guide/src/types/overseer-protocol.md @@ -1,6 +1,7 @@ # Overseer Protocol -This chapter contains message types sent to and from the overseer, and the underlying subsystem message types that are transmitted using these. +This chapter contains message types sent to and from the overseer, and the underlying subsystem message types that are +transmitted using these. ## Overseer Signal @@ -17,7 +18,8 @@ enum OverseerSignal { } ``` -All subsystems have their own message types; all of them need to be able to listen for overseer signals as well. There are currently two proposals for how to handle that with unified communication channels: +All subsystems have their own message types; all of them need to be able to listen for overseer signals as well. There +are currently two proposals for how to handle that with unified communication channels: 1. Retaining the `OverseerSignal` definition above, add `enum FromOrchestra {Signal(OverseerSignal), Message(T)}`. 1. Add a generic varint to `OverseerSignal`: `Message(T)`. @@ -26,7 +28,8 @@ Either way, there will be some top-level type encapsulating messages from the ov ## Active Leaves Update -Indicates a change in active leaves. Activated leaves should have jobs, whereas deactivated leaves should lead to winding-down of work based on those leaves. +Indicates a change in active leaves. Activated leaves should have jobs, whereas deactivated leaves should lead to +winding-down of work based on those leaves. ```rust enum LeafStatus { @@ -201,7 +204,8 @@ enum ApprovalDistributionMessage { Messages received by the availability distribution subsystem. -This is a network protocol that receives messages of type [`AvailabilityDistributionV1Message`][AvailabilityDistributionV1NetworkMessage]. +This is a network protocol that receives messages of type +[`AvailabilityDistributionV1Message`][AvailabilityDistributionV1NetworkMessage]. ```rust enum AvailabilityDistributionMessage { @@ -293,7 +297,7 @@ pub enum AvailabilityStoreMessage { tx: oneshot::Sender>, }, - /// Computes and checks the erasure root of `AvailableData` before storing all of its chunks in + /// Computes and checks the erasure root of `AvailableData` before storing all of its chunks in /// the AV store. /// /// Return `Ok(())` if the store operation succeeded, `Err(StoreAvailableData)` if it failed. @@ -319,8 +323,8 @@ pub enum StoreAvailableDataError { ## Bitfield Distribution Message -Messages received by the bitfield distribution subsystem. -This is a network protocol that receives messages of type [`BitfieldDistributionV1Message`][BitfieldDistributionV1NetworkMessage]. +Messages received by the bitfield distribution subsystem. This is a network protocol that receives messages of type +[`BitfieldDistributionV1Message`][BitfieldDistributionV1NetworkMessage]. ```rust enum BitfieldDistributionMessage { @@ -354,7 +358,7 @@ enum CandidateBackingMessage { /// The PoV is expected to match the `pov_hash` in the descriptor. Second(Hash, CandidateReceipt, PoV), /// Note a peer validator's statement about a particular candidate. Disagreements about validity must be escalated - /// to a broader check by the Disputes Subsystem, though that escalation is deferred until the approval voting + /// to a broader check by the Disputes Subsystem, though that escalation is deferred until the approval voting /// stage to guarantee availability. Agreements are simply tallied until a quorum is reached. Statement(Statement), } @@ -420,7 +424,8 @@ enum ChainSelectionMessage { Messages received by the [Collator Protocol subsystem](../node/collators/collator-protocol.md) -This is a network protocol that receives messages of type [`CollatorProtocolV1Message`][CollatorProtocolV1NetworkMessage]. +This is a network protocol that receives messages of type +[`CollatorProtocolV1Message`][CollatorProtocolV1NetworkMessage]. ```rust enum CollatorProtocolMessage { @@ -452,7 +457,8 @@ enum CollatorProtocolMessage { Messages received by the [Collation Generation subsystem](../node/collators/collation-generation.md) -This is the core interface by which collators built on top of a Polkadot node submit collations to validators. As such, these messages are not sent by any subsystem but are instead sent from outside of the overseer. +This is the core interface by which collators built on top of a Polkadot node submit collations to validators. As such, +these messages are not sent by any subsystem but are instead sent from outside of the overseer. ```rust /// A function provided to the subsystem which it uses to pull new collations. @@ -503,7 +509,8 @@ enum CollationGenerationMessage { Messages received by the [Dispute Coordinator subsystem](../node/disputes/dispute-coordinator.md) -This subsystem coordinates participation in disputes, tracks live disputes, and observed statements of validators from subsystems. +This subsystem coordinates participation in disputes, tracks live disputes, and observed statements of validators from +subsystems. ```rust enum DisputeCoordinatorMessage { @@ -572,11 +579,9 @@ pub enum ImportStatementsResult { } ``` - ## Dispute Distribution Message -Messages received by the [Dispute Distribution -subsystem](../node/disputes/dispute-distribution.md). This subsystem is +Messages received by the [Dispute Distribution subsystem](../node/disputes/dispute-distribution.md). This subsystem is responsible of distributing explicit dispute statements. ```rust @@ -602,8 +607,8 @@ enum DisputeDistributionMessage { ## Network Bridge Message -Messages received by the network bridge. This subsystem is invoked by others to manipulate access -to the low-level networking code. +Messages received by the network bridge. This subsystem is invoked by others to manipulate access to the low-level +networking code. ```rust /// Peer-sets handled by the network bridge. @@ -778,7 +783,8 @@ enum ProvisionerMessage { The Runtime API subsystem is responsible for providing an interface to the state of the chain's runtime. -This is fueled by an auxiliary type encapsulating all request types defined in the [Runtime API section](../runtime-api) of the guide. +This is fueled by an auxiliary type encapsulating all request types defined in the [Runtime API section](../runtime-api) +of the guide. ```rust enum RuntimeApiRequest { @@ -835,10 +841,12 @@ enum RuntimeApiMessage { ## Statement Distribution Message -The Statement Distribution subsystem distributes signed statements and candidates from validators to other validators. It does this by distributing full statements, which embed the candidate receipt, as opposed to compact statements which don't. -It receives updates from the network bridge and signed statements to share with other validators. +The Statement Distribution subsystem distributes signed statements and candidates from validators to other validators. +It does this by distributing full statements, which embed the candidate receipt, as opposed to compact statements which +don't. It receives updates from the network bridge and signed statements to share with other validators. -This is a network protocol that receives messages of type [`StatementDistributionV1Message`][StatementDistributionV1NetworkMessage]. +This is a network protocol that receives messages of type +[`StatementDistributionV1Message`][StatementDistributionV1NetworkMessage]. ```rust enum StatementDistributionMessage { @@ -855,7 +863,8 @@ enum StatementDistributionMessage { ## Validation Request Type -Various modules request that the [Candidate Validation subsystem](../node/utility/candidate-validation.md) validate a block with this message. It returns [`ValidationOutputs`](candidate.md#validationoutputs) for successful validation. +Various modules request that the [Candidate Validation subsystem](../node/utility/candidate-validation.md) validate a +block with this message. It returns [`ValidationOutputs`](candidate.md#validationoutputs) for successful validation. ```rust diff --git a/polkadot/roadmap/implementers-guide/src/whence-parachains.md b/polkadot/roadmap/implementers-guide/src/whence-parachains.md index 41842e93943b41699a6ddb06ee5589a9c1e713c5..faec46be2c6258eeb105ae5e06cbe8b30017d703 100644 --- a/polkadot/roadmap/implementers-guide/src/whence-parachains.md +++ b/polkadot/roadmap/implementers-guide/src/whence-parachains.md @@ -1,29 +1,50 @@ # Whence Parachains -Parachains are the solution to a problem. As with any solution, it cannot be understood without first understanding the problem. So let's start by going over the issues faced by blockchain technology that led to us beginning to explore the design space for something like parachains. +Parachains are the solution to a problem. As with any solution, it cannot be understood without first understanding the +problem. So let's start by going over the issues faced by blockchain technology that led to us beginning to explore the +design space for something like parachains. ## Issue 1: Scalability -It became clear a few years ago that the transaction throughput of simple Proof-of-Work (PoW) blockchains such as Bitcoin, Ethereum, and myriad others was simply too low. +It became clear a few years ago that the transaction throughput of simple Proof-of-Work (PoW) blockchains such as +Bitcoin, Ethereum, and myriad others was simply too low. > TODO: what if there were more blockchains, etc. -Proof-of-Stake (PoS) systems can accomplish higher throughput than PoW blockchains. PoS systems are secured by bonded capital as opposed to spent effort - liquidity opportunity cost vs. burning electricity. The way they work is by selecting a set of validators with known economic identity who lock up tokens in exchange for earning the right to "validate" or participate in the consensus process. If they are found to carry out that process wrongly, they will be slashed, meaning some or all of the locked tokens will be burned. This provides a strong disincentive in the direction of misbehavior. +Proof-of-Stake (PoS) systems can accomplish higher throughput than PoW blockchains. PoS systems are secured by bonded +capital as opposed to spent effort - liquidity opportunity cost vs. burning electricity. The way they work is by +selecting a set of validators with known economic identity who lock up tokens in exchange for earning the right to +"validate" or participate in the consensus process. If they are found to carry out that process wrongly, they will be +slashed, meaning some or all of the locked tokens will be burned. This provides a strong disincentive in the direction +of misbehavior. -Since the consensus protocol doesn't revolve around wasting effort, block times and agreement can occur much faster. Solutions to PoW challenges don't have to be found before a block can be authored, so the overhead of authoring a block is reduced to only the costs of creating and distributing the block. +Since the consensus protocol doesn't revolve around wasting effort, block times and agreement can occur much faster. +Solutions to PoW challenges don't have to be found before a block can be authored, so the overhead of authoring a block +is reduced to only the costs of creating and distributing the block. -However, consensus on a PoS chain requires full agreement of 2/3+ of the validator set for everything that occurs at Layer 1: all logic which is carried out as part of the blockchain's state machine. This means that everybody still needs to check everything. Furthermore, validators may have different views of the system based on the information that they receive over an asynchronous network, making agreement on the latest state more difficult. +However, consensus on a PoS chain requires full agreement of 2/3+ of the validator set for everything that occurs at +Layer 1: all logic which is carried out as part of the blockchain's state machine. This means that everybody still needs +to check everything. Furthermore, validators may have different views of the system based on the information that they +receive over an asynchronous network, making agreement on the latest state more difficult. -Parachains are an example of a **sharded** protocol. Sharding is a concept borrowed from traditional database architecture. Rather than requiring every participant to check every transaction, we require each participant to check some subset of transactions, with enough redundancy baked in that byzantine (arbitrarily malicious) participants can't sneak in invalid transactions - at least not without being detected and getting slashed, with those transactions reverted. +Parachains are an example of a **sharded** protocol. Sharding is a concept borrowed from traditional database +architecture. Rather than requiring every participant to check every transaction, we require each participant to check +some subset of transactions, with enough redundancy baked in that byzantine (arbitrarily malicious) participants can't +sneak in invalid transactions - at least not without being detected and getting slashed, with those transactions +reverted. -Sharding and Proof-of-Stake in coordination with each other allow a parachain host to provide full security on many parachains, even without all participants checking all state transitions. +Sharding and Proof-of-Stake in coordination with each other allow a parachain host to provide full security on many +parachains, even without all participants checking all state transitions. > TODO: note about network effects & bridging ## Issue 2: Flexibility / Specialization -"dumb" VMs don't give you the flexibility. Any engineer knows that being able to specialize on a problem gives them and their users more _leverage_. +"dumb" VMs don't give you the flexibility. Any engineer knows that being able to specialize on a problem gives them and +their users more _leverage_. > TODO: expand on leverage -Having recognized these issues, we set out to find a solution to these problems, which could allow developers to create and deploy purpose-built blockchains unified under a common source of security, with the capability of message-passing between them; a _heterogeneous sharding solution_, which we have come to know as **Parachains**. +Having recognized these issues, we set out to find a solution to these problems, which could allow developers to create +and deploy purpose-built blockchains unified under a common source of security, with the capability of message-passing +between them; a _heterogeneous sharding solution_, which we have come to know as **Parachains**. diff --git a/polkadot/roadmap/parachains.md b/polkadot/roadmap/parachains.md index 9d6c014a1c7c48e88c274998f1e7fae0aff904d3..77e9743367673265a86be795715992291369743a 100644 --- a/polkadot/roadmap/parachains.md +++ b/polkadot/roadmap/parachains.md @@ -1,17 +1,23 @@ # Parachains Roadmap -This is a roadmap for the core technology underlying Parachains - what protocols, APIs, and code paths need to be in place to fully instantiate a self-sufficient and secure parachain. We don't attempt to cover anything on what APIs a parachain toolkit might expose in order to make use of parachain features - only how those features are implemented and the low-level APIs that they expose to the validation function, if any. + +This is a roadmap for the core technology underlying Parachains - what protocols, APIs, and code paths need to be in +place to fully instantiate a self-sufficient and secure parachain. We don't attempt to cover anything on what APIs a +parachain toolkit might expose in order to make use of parachain features - only how those features are implemented and +the low-level APIs that they expose to the validation function, if any. ## Categories We will use these categories to delineate features: -*Runtime*: Runtime code for the Relay chain specifying consensus-critical state and updates that all full nodes must maintain or perform. +*Runtime*: Runtime code for the Relay chain specifying consensus-critical state and updates that all full nodes must +maintain or perform. *Networking*: Protocols for nodes to speak to each other and transmit information across the network. -*Node*: State or updates that must be maintained or performed by some or all nodes off-chain. Often interfaces with networking components, and references runtime state. +*Node*: State or updates that must be maintained or performed by some or all nodes off-chain. Often interfaces with +networking components, and references runtime state. --- -## Sub-projects and features: +## Sub-projects and features This section contains various sub-projects and the features that make them up. ### Infrastructure/API @@ -20,7 +26,8 @@ This section contains various sub-projects and the features that make them up. Category: Networking -Validators assigned to a parachain need a way to discover and connect to collators in order to get fresh parachain blocks to validate. +Validators assigned to a parachain need a way to discover and connect to collators in order to get fresh parachain +blocks to validate. Collators need to discover and connect to validators in order to submit parachain blocks. @@ -30,7 +37,9 @@ Some connections are long-lived, some are just for a single request. #### Custom libp2p sub-protocols -Polkadot parachains involve many distinct networking protocols. Ideally, we'd be able to spawn each of these as a separate futures task which communicates via channel with other protocols or node code as necessary. This requires changes in Substrate and libp2p. +Polkadot parachains involve many distinct networking protocols. Ideally, we'd be able to spawn each of these as a +separate futures task which communicates via channel with other protocols or node code as necessary. This requires +changes in Substrate and libp2p. --- ### Assignment @@ -39,21 +48,26 @@ Polkadot parachains involve many distinct networking protocols. Ideally, we'd be Category: Runtime -Auctioning and registration of parachains. This is already implemented and follows the [Parachain Allocation — Research at W3F](https://research.web3.foundation/en/latest/polkadot/Parachain-Allocation.html) document. +Auctioning and registration of parachains. This is already implemented and follows the [Parachain Allocation — Research +at W3F](https://research.web3.foundation/en/latest/polkadot/Parachain-Allocation.html) document. #### *On-demand Blockspace Purchase* Category: Runtime -The blockspace purchasing system for on-demand parachains consists of an on-chain mechanism for resolving block space purchases by collators and ensuring that they author a block. +The blockspace purchasing system for on-demand parachains consists of an on-chain mechanism for resolving block space +purchases by collators and ensuring that they author a block. -The node-side portion of on-demand parachains is for collators to actually purchase blockspace and to configure the conditions in which purchases are made. +The node-side portion of on-demand parachains is for collators to actually purchase blockspace and to configure the +conditions in which purchases are made. #### *Validator Assignment* Category: Runtime -Assignment of validators to parachains. Validators are only assigned to parachains for a short period of time. Tweakable parameters include length of time assigned to each parachain and length of time in advance that the network is aware of validators' assignments. +Assignment of validators to parachains. Validators are only assigned to parachains for a short period of time. Tweakable +parameters include length of time assigned to each parachain and length of time in advance that the network is aware of +validators' assignments. --- ### Agreement @@ -62,19 +76,26 @@ Assignment of validators to parachains. Validators are only assigned to parachai Category: Networking -A black-box networking component for circulating attestation messages (`Candidate`, `Valid`, `Invalid`) between validators of any given parachain to create a quorum on which blocks can be included. +A black-box networking component for circulating attestation messages (`Candidate`, `Valid`, `Invalid`) between +validators of any given parachain to create a quorum on which blocks can be included. #### *Availability Erasure-coding* Category: Node, Networking -For each potential, considered parachain block, perform an erasure-coding of the PoV and outgoing messages of the block. Call the number of validators on the relay chain for the Relay-chain block this parachain block is being considered for inclusion in `n`. Erasure-code into `n` pieces, where any `f + 1` can recover (`f` being the maximum number of tolerated faulty nodes = ~ `n / 3`). The `i'th` validator stores the `i'th` piece of the coding and provides it to any who ask. +For each potential, considered parachain block, perform an erasure-coding of the PoV and outgoing messages of the block. +Call the number of validators on the relay chain for the Relay-chain block this parachain block is being considered for +inclusion in `n`. Erasure-code into `n` pieces, where any `f + 1` can recover (`f` being the maximum number of tolerated +faulty nodes = ~ `n / 3`). The `i'th` validator stores the `i'th` piece of the coding and provides it to any who ask. #### *PoV block fetching* Category: Networking -A black-box networking component for validators or fishermen on a parachain to obtain the PoV block referenced by hash in an attestation, for the purpose of validating. When fetching "current" PoV blocks (close to the head of the chain, or relating to the block currently being built), this should be fast. When fetching "old" PoV blocks, it should be possible and fall back on recovering from the availability erasure-coding. +A black-box networking component for validators or fishermen on a parachain to obtain the PoV block referenced by hash +in an attestation, for the purpose of validating. When fetching "current" PoV blocks (close to the head of the chain, or +relating to the block currently being built), this should be fast. When fetching "old" PoV blocks, it should be possible +and fall back on recovering from the availability erasure-coding. #### *On-demand Blockspace Purchase* @@ -95,58 +116,80 @@ The main event loop of a collator node: --- ### Cross-chain Messaging -https://hackmd.io/ILoQltEISP697oMYe4HbrA?view -https://github.com/paritytech/polkadot/issues/597 +https://hackmd.io/ILoQltEISP697oMYe4HbrA?view https://github.com/paritytech/polkadot/issues/597 -The biggest sub-project of the parachains roadmap - how messages are sent between parachains. This involves the state-machine ordering of incoming messages, protocols for fetching those messages, and node logic for persisting the messages. +The biggest sub-project of the parachains roadmap - how messages are sent between parachains. This involves the +state-machine ordering of incoming messages, protocols for fetching those messages, and node logic for persisting the +messages. -This is designed around a concept of unidirectional _channels_ between paras, which consist of a sender and receiver. At each relay chain block, each para has an opportunity to send a message on each channel for which it controls the sending half. It will also attempt to process messages on each receiving half of the channel which it controls _in order_: messages sent at block height `b` must be processed before those sent at block height `b+1`. For messages on different channels sent at the same block height, there will be some well-defined order in which they should be processed. +This is designed around a concept of unidirectional _channels_ between paras, which consist of a sender and receiver. At +each relay chain block, each para has an opportunity to send a message on each channel for which it controls the sending +half. It will also attempt to process messages on each receiving half of the channel which it controls _in order_: +messages sent at block height `b` must be processed before those sent at block height `b+1`. For messages on different +channels sent at the same block height, there will be some well-defined order in which they should be processed. -This means that a receiving para will have a maximum height differential of `1` in terms of the most recently processed message's send-height across all of the channels it is receiving on. The minimum processed send-height of a receiving para is known as its _watermark_. All messages on all channels sending to this para before or at the watermark have been processed. +This means that a receiving para will have a maximum height differential of `1` in terms of the most recently processed +message's send-height across all of the channels it is receiving on. The minimum processed send-height of a receiving +para is known as its _watermark_. All messages on all channels sending to this para before or at the watermark have been +processed. #### *Finalize CandidateReceipt format* Category: Runtime / Node -The `CandidateReceipt` is the wrapper around a parablock header which is submitted to the runtime. It contains cryptographic commitments to data which is important for validation or interpretation of the parablock, including the hash of the witness data and outgoing message data. +The `CandidateReceipt` is the wrapper around a parablock header which is submitted to the runtime. It contains +cryptographic commitments to data which is important for validation or interpretation of the parablock, including the +hash of the witness data and outgoing message data. -The `CandidateReceipt` format should be finalized in accordance to the XCMP writeups linked above - most importantly, to be altered to hold `bitfield` and `message_root` fields which cryptographically commit to the state of each open channel. +The `CandidateReceipt` format should be finalized in accordance to the XCMP writeups linked above - most importantly, to +be altered to hold `bitfield` and `message_root` fields which cryptographically commit to the state of each open +channel. #### *Finalize PovBlock format* Category: Runtime / Node -The `PovBlock` or `Proof-of-Validity` block contains all the data you need to validate a parablock. It will need to contain incoming message queues and potentially outgoing ones as well. +The `PovBlock` or `Proof-of-Validity` block contains all the data you need to validate a parablock. It will need to +contain incoming message queues and potentially outgoing ones as well. #### *CST Update Procedure* Category: Runtime -Storage definitions and update logic of the Channel State Table (CST) based on the supplied `CandidateReceipt`s in a relay chain block. +Storage definitions and update logic of the Channel State Table (CST) based on the supplied `CandidateReceipt`s in a +relay chain block. #### *CST Entry Proof Generation and Checking* Category: Node -Means for full nodes of the relay chain to generate proofs of items in the CST and for light clients or pruned nodes to check those proofs. +Means for full nodes of the relay chain to generate proofs of items in the CST and for light clients or pruned nodes to +check those proofs. #### *MQC Storage and Distribution Protocol* Category: Node -Every channel's state is described by a Message Queue Chain (MQC) which is a hash-chain, where the links are defined by `(M, b, H)`: the message most recently sent, the block height at which the prior message was sent, and the hash of the prior link. +Every channel's state is described by a Message Queue Chain (MQC) which is a hash-chain, where the links are defined by +`(M, b, H)`: the message most recently sent, the block height at which the prior message was sent, and the hash of the +prior link. -It is the responsibility of the full nodes of the _sending_ para to maintain all links of the MQC up to and including the link where `b` is less than the watermark of the _receiving_ para. +It is the responsibility of the full nodes of the _sending_ para to maintain all links of the MQC up to and including +the link where `b` is less than the watermark of the _receiving_ para. -Full nodes of the para will be aware of the head of all MQCs for its channels because they are produced by execution of the block. This will take collaboration with the Cumulus team (https://github.com/paritytech/cumulus) on APIs. +Full nodes of the para will be aware of the head of all MQCs for its channels because they are produced by execution of +the block. This will take collaboration with the Cumulus team (https://github.com/paritytech/cumulus) on APIs. -We will need a network where collators of paras can discover and fetch the relevant portion of the MQC incoming from all channels. +We will need a network where collators of paras can discover and fetch the relevant portion of the MQC incoming from all +channels. #### *Channel Registrar and Economics* Category: Runtime -Runtime logic for paras to open and close channels by putting down a deposit. The amount of channels an on-demand parachain can open will be limited. Channels that are pending close should remain open until the watermark of the recipient has reached the block height of the close request. +Runtime logic for paras to open and close channels by putting down a deposit. The amount of channels an on-demand +parachain can open will be limited. Channels that are pending close should remain open until the watermark of the +recipient has reached the block height of the close request. --- ### Fishing/Slashing @@ -155,17 +198,29 @@ Runtime logic for paras to open and close channels by putting down a deposit. Th Category: Runtime -In Polkadot, a bad parachain group can force inclusion of an invalid or unavailable parachain block. It is the job of fishermen to detect those blocks and report them to the runtime. This item is about the report handler +In Polkadot, a bad parachain group can force inclusion of an invalid or unavailable parachain block. It is the job of +fishermen to detect those blocks and report them to the runtime. This item is about the report handler -The W3F-research writeup on availability/validity provides a high-level view of the dispute resolution process: [Availability and Validity — Research at W3F](https://research.web3.foundation/en/latest/polkadot/Availability_and_Validity.html) +The W3F-research writeup on availability/validity provides a high-level view of the dispute resolution process: +[Availability and Validity — Research at +W3F](https://research.web3.foundation/en/latest/polkadot/Availability_and_Validity.html) -One of the main behaviors that is unimplemented and needs to be is the _rollback_ that occurs when the dispute resolution process concludes that an error has been made. When we mark a parachain block as having been invalid or unavailable, we need to roll back all parachains to a point from just before this state. We would also need to roll back relay chain state, because there may have been messages from a parachain to a relay chain that now need to be rolled back. The easiest thing to do would be to side-step that by putting a delay on upwards messages, but this would impact the UX of parachain participation in slot auctions, council votes, etc. considerably. Assuming we can't side-step this, we will have to find a way to roll back selected state of the relay chain. +One of the main behaviors that is unimplemented and needs to be is the _rollback_ that occurs when the dispute +resolution process concludes that an error has been made. When we mark a parachain block as having been invalid or +unavailable, we need to roll back all parachains to a point from just before this state. We would also need to roll +back relay chain state, because there may have been messages from a parachain to a relay chain that now need to be +rolled back. The easiest thing to do would be to side-step that by putting a delay on upwards messages, but this would +impact the UX of parachain participation in slot auctions, council votes, etc. considerably. Assuming we can't side-step +this, we will have to find a way to roll back selected state of the relay chain. #### *Double-vote Slash Handler* Category: Runtime -In the attestation process, validators may submit only one `Candidate` message for a given relay chain block. If issuing a `Candidate` message on a parachain block, neither a `Valid` or `Invalid` vote cannot be issued on that parachain block, as the `Candidate` message is an implicit validity vote. Otherwise, it is illegal to cast both a `Valid` and `Invalid` vote on a given parachain block. +In the attestation process, validators may submit only one `Candidate` message for a given relay chain block. If issuing +a `Candidate` message on a parachain block, neither a `Valid` or `Invalid` vote cannot be issued on that parachain +block, as the `Candidate` message is an implicit validity vote. Otherwise, it is illegal to cast both a `Valid` and +`Invalid` vote on a given parachain block. Runtime handlers that take two conflicting votes as arguments and slash the offender are needed. @@ -173,9 +228,11 @@ Runtime handlers that take two conflicting votes as arguments and slash the offe Category: Node -This code-path is also taken by validators who self-select based on VRF [Availability and Validity — Research at W3F](https://research.web3.foundation/en/latest/polkadot/Availability_and_Validity.html). Validators and fishermen will select parachain blocks to re-validate. In these steps: -* Attempt to recover the PoV block, falling back on the erasure-coding. If not available, issue report. -* Attempt to validate the PoV block. If invalid, issue report. +This code-path is also taken by validators who self-select based on VRF [Availability and Validity — Research at +W3F](https://research.web3.foundation/en/latest/polkadot/Availability_and_Validity.html). Validators and fishermen will +select parachain blocks to re-validate. In these steps: +- Attempt to recover the PoV block, falling back on the erasure-coding. If not available, issue report. +- Attempt to validate the PoV block. If invalid, issue report. #### *Double-vote Fishing* @@ -186,45 +243,50 @@ Nodes that observe a double-vote in the attestation process should submit a repo --- # Phases -This roadmap is divided up into phases, where each represents another set of deliverables or iteration on a black-box component with respect to the prior phase. +This roadmap is divided up into phases, where each represents another set of deliverables or iteration on a black-box +component with respect to the prior phase. ## Phase 0: MVP -The very first phase - this is parachains without slashing (full security) or cross-chain messaging. It is primarily a PoC that registration and validation are working correctly. +The very first phase - this is parachains without slashing (full security) or cross-chain messaging. It is primarily a +PoC that registration and validation are working correctly. -### Infrastructure/API: - - Custom libp2p sub-protocols - - Peer Set Management +### Infrastructure/API +- Custom libp2p sub-protocols +- Peer Set Management -### Assignment: - - Auctions - - On-demand Blockspace purchase - - Validator Assignment +### Assignment +- Auctions +- On-demand Blockspace purchase +- Validator Assignment -### Agreement: - - Attestation Circulation (black box: gossip) - - Availability Erasure-coding (black box: gossip) - - PoV block fetching (black box: gossip) - - Collation Loop +### Agreement +- Attestation Circulation (black box: gossip) +- Availability Erasure-coding (black box: gossip) +- PoV block fetching (black box: gossip) +- Collation Loop -### Cross-chain Messaging: - - Finalize `CandidateReceipt` format +### Cross-chain Messaging +- Finalize `CandidateReceipt` format ## Phase 1: Fishing and Slashing -This phase marks advancement in the security of parachains. Once completed, parachains are a full-fledged cryptoeconomically secure rollup primitive. This phase also includes implementation work on XCMP, but does not enable it fully. +This phase marks advancement in the security of parachains. Once completed, parachains are a full-fledged +cryptoeconomically secure rollup primitive. This phase also includes implementation work on XCMP, but does not enable it +fully. ### Agreement - - Availability Erasure-coding (black box: targeted distribution) - - PoV block fetching (black box: targeted distribution and fetching) +- Availability Erasure-coding (black box: targeted distribution) +- PoV block fetching (black box: targeted distribution and fetching) ### Fishing/Slashing - - Validity/Availability Report Handler - - Double-vote Slash Handler - - Validity/Availability Fishing - - Double-vote Fishing -### Cross-chain Messaging: - - Finalize `PoVBlock` format. +- Validity/Availability Report Handler +- Double-vote Slash Handler +- Validity/Availability Fishing +- Double-vote Fishing + +### Cross-chain Messaging +- Finalize `PoVBlock` format. ## Phase 2: Messaging diff --git a/polkadot/runtime/common/Cargo.toml b/polkadot/runtime/common/Cargo.toml index 83aa70cd308a0576a628486dc9bd8f6a839912bc..17617bf4ada3fd6e567c9453656f717266615c5d 100644 --- a/polkadot/runtime/common/Cargo.toml +++ b/polkadot/runtime/common/Cargo.toml @@ -58,13 +58,12 @@ pallet-babe = { path = "../../../substrate/frame/babe" } pallet-treasury = { path = "../../../substrate/frame/treasury" } sp-keystore = { path = "../../../substrate/primitives/keystore" } sp-keyring = { path = "../../../substrate/primitives/keyring" } -serde_json = "1.0.96" +serde_json = "1.0.107" libsecp256k1 = "0.7.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } [features] default = [ "std" ] -experimental = [ "frame-support/experimental" ] no_std = [] std = [ "bitvec/std", diff --git a/polkadot/runtime/common/slot_range_helper/Cargo.toml b/polkadot/runtime/common/slot_range_helper/Cargo.toml index 30d5dc84e9dc2dd7c819ad86bf9707943619dde2..f65717519d5e1e715f478e37381e943e20bbdca4 100644 --- a/polkadot/runtime/common/slot_range_helper/Cargo.toml +++ b/polkadot/runtime/common/slot_range_helper/Cargo.toml @@ -7,7 +7,7 @@ license.workspace = true [dependencies] paste = "1.0" -enumn = "0.1.8" +enumn = "0.1.12" parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } sp-std = { package = "sp-std", path = "../../../../substrate/primitives/std", default-features = false } sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } diff --git a/polkadot/runtime/common/src/assigned_slots/migration.rs b/polkadot/runtime/common/src/assigned_slots/migration.rs index 656e0836bba3fa34572e45812f9e2a24b165e568..0e88b27a1ff8ae7bc1c7fba7fbeb4b057fdaa5fe 100644 --- a/polkadot/runtime/common/src/assigned_slots/migration.rs +++ b/polkadot/runtime/common/src/assigned_slots/migration.rs @@ -63,7 +63,6 @@ pub mod v1 { /// [`MigrateToV1`] wrapped in a /// [`VersionedMigration`](frame_support::migrations::VersionedMigration), ensuring the /// migration is only performed when on-chain version is 0. - #[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV1 = frame_support::migrations::VersionedMigration< 0, 1, diff --git a/polkadot/runtime/common/src/assigned_slots/mod.rs b/polkadot/runtime/common/src/assigned_slots/mod.rs index 3683cfc210fa0dcfc3d7995753639879bf3386f8..cc8ec339c11847b60aaf74e909ab798a6cb45b51 100644 --- a/polkadot/runtime/common/src/assigned_slots/mod.rs +++ b/polkadot/runtime/common/src/assigned_slots/mod.rs @@ -739,6 +739,7 @@ mod tests { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = (); type NextSessionRotation = crate::mock::TestNextSessionRotation; + type OnNewHead = (); } impl parachains_shared::Config for Test {} diff --git a/polkadot/runtime/common/src/crowdloan/mod.rs b/polkadot/runtime/common/src/crowdloan/mod.rs index 0303808e074775a5dd9309291d129d526bcb2f58..5a2939145925c453f9c86b9c6e5f7c9f71de9757 100644 --- a/polkadot/runtime/common/src/crowdloan/mod.rs +++ b/polkadot/runtime/common/src/crowdloan/mod.rs @@ -441,8 +441,6 @@ pub mod pallet { ); NextFundIndex::::put(new_fund_index); - // Add a lock to the para so that the configuration cannot be changed. - T::Registrar::apply_lock(index); Self::deposit_event(Event::::Created { para_id: index }); Ok(()) diff --git a/polkadot/runtime/common/src/integration_tests.rs b/polkadot/runtime/common/src/integration_tests.rs index f78347dedd8caadc209001bfa8894d8fa2f900f6..f14db68267d5b550553bd6f48819890adf011c48 100644 --- a/polkadot/runtime/common/src/integration_tests.rs +++ b/polkadot/runtime/common/src/integration_tests.rs @@ -204,6 +204,7 @@ impl paras::Config for Test { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = (); type NextSessionRotation = crate::mock::TestNextSessionRotation; + type OnNewHead = (); } parameter_types! { diff --git a/polkadot/runtime/common/src/paras_registrar/migration.rs b/polkadot/runtime/common/src/paras_registrar/migration.rs new file mode 100644 index 0000000000000000000000000000000000000000..b767985489d3622e6da01e689609c8913929e78a --- /dev/null +++ b/polkadot/runtime/common/src/paras_registrar/migration.rs @@ -0,0 +1,70 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use frame_support::traits::{Contains, OnRuntimeUpgrade}; + +#[derive(Encode, Decode)] +pub struct ParaInfoV1 { + manager: Account, + deposit: Balance, + locked: bool, +} + +pub struct VersionUncheckedMigrateToV1( + sp_std::marker::PhantomData<(T, UnlockParaIds)>, +); +impl> OnRuntimeUpgrade + for VersionUncheckedMigrateToV1 +{ + fn on_runtime_upgrade() -> Weight { + let mut count = 0u64; + Paras::::translate::>, _>(|key, v1| { + count.saturating_inc(); + Some(ParaInfo { + manager: v1.manager, + deposit: v1.deposit, + locked: if UnlockParaIds::contains(&key) { None } else { Some(v1.locked) }, + }) + }); + + log::info!(target: "runtime::registrar", "Upgraded {} storages to version 1", count); + T::DbWeight::get().reads_writes(count, count) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result, sp_runtime::TryRuntimeError> { + Ok((Paras::::iter_keys().count() as u32).encode()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade(state: Vec) -> Result<(), sp_runtime::TryRuntimeError> { + let old_count = u32::decode(&mut &state[..]).expect("Known good"); + let new_count = Paras::::iter_values().count() as u32; + + ensure!(old_count == new_count, "Paras count should not change"); + Ok(()) + } +} + +pub type VersionCheckedMigrateToV1 = + frame_support::migrations::VersionedMigration< + 0, + 1, + VersionUncheckedMigrateToV1, + super::Pallet, + ::DbWeight, + >; diff --git a/polkadot/runtime/common/src/paras_registrar.rs b/polkadot/runtime/common/src/paras_registrar/mod.rs similarity index 97% rename from polkadot/runtime/common/src/paras_registrar.rs rename to polkadot/runtime/common/src/paras_registrar/mod.rs index 3f5a8e1a5f93caca0fe5c243107825eeab068ca2..f2751803a413395b11ccc7126c81ee693ab981ce 100644 --- a/polkadot/runtime/common/src/paras_registrar.rs +++ b/polkadot/runtime/common/src/paras_registrar/mod.rs @@ -17,6 +17,8 @@ //! Pallet to handle parachain registration and related fund management. //! In essence this is a simple wrapper around `paras`. +pub mod migration; + use frame_support::{ dispatch::DispatchResult, ensure, @@ -35,7 +37,7 @@ use sp_std::{prelude::*, result}; use crate::traits::{OnSwap, Registrar}; pub use pallet::*; use parity_scale_codec::{Decode, Encode}; -use runtime_parachains::paras::ParaKind; +use runtime_parachains::paras::{OnNewHead, ParaKind}; use scale_info::TypeInfo; use sp_runtime::{ traits::{CheckedSub, Saturating}, @@ -49,7 +51,15 @@ pub struct ParaInfo { /// The amount reserved by the `manager` account for the registration. deposit: Balance, /// Whether the para registration should be locked from being controlled by the manager. - locked: bool, + /// None means the lock had not been explicitly set, and should be treated as false. + locked: Option, +} + +impl ParaInfo { + /// Returns if the para is locked. + pub fn is_locked(&self) -> bool { + self.locked.unwrap_or(false) + } } type BalanceOf = @@ -96,8 +106,12 @@ pub mod pallet { use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; + /// The current storage version. + const STORAGE_VERSION: StorageVersion = StorageVersion::new(1); + #[pallet::pallet] #[pallet::without_storage_info] + #[pallet::storage_version(STORAGE_VERSION)] pub struct Pallet(_); #[pallet::config] @@ -446,12 +460,12 @@ impl Registrar for Pallet { // Apply a lock to the parachain. fn apply_lock(id: ParaId) { - Paras::::mutate(id, |x| x.as_mut().map(|info| info.locked = true)); + Paras::::mutate(id, |x| x.as_mut().map(|info| info.locked = Some(true))); } // Remove a lock from the parachain. fn remove_lock(id: ParaId) { - Paras::::mutate(id, |x| x.as_mut().map(|info| info.locked = false)); + Paras::::mutate(id, |x| x.as_mut().map(|info| info.locked = Some(false))); } // Register a Para ID under control of `manager`. @@ -481,9 +495,7 @@ impl Registrar for Pallet { ); runtime_parachains::schedule_parathread_upgrade::(id) .map_err(|_| Error::::CannotUpgrade)?; - // Once a para has upgraded to a parachain, it can no longer be managed by the owner. - // Intentionally, the flag stays with the para even after downgrade. - Self::apply_lock(id); + Ok(()) } @@ -533,7 +545,7 @@ impl Pallet { .map_err(|e| e.into()) .and_then(|who| -> DispatchResult { let para_info = Paras::::get(id).ok_or(Error::::NotRegistered)?; - ensure!(!para_info.locked, Error::::ParaLocked); + ensure!(!para_info.is_locked(), Error::::ParaLocked); ensure!(para_info.manager == who, Error::::NotOwner); Ok(()) }) @@ -566,7 +578,7 @@ impl Pallet { let deposit = deposit_override.unwrap_or_else(T::ParaDeposit::get); ::Currency::reserve(&who, deposit)?; - let info = ParaInfo { manager: who.clone(), deposit, locked: false }; + let info = ParaInfo { manager: who.clone(), deposit, locked: None }; Paras::::insert(id, info); Self::deposit_event(Event::::Reserved { para_id: id, who }); @@ -585,7 +597,7 @@ impl Pallet { ) -> DispatchResult { let deposited = if let Some(para_data) = Paras::::get(id) { ensure!(para_data.manager == who, Error::::NotOwner); - ensure!(!para_data.locked, Error::::ParaLocked); + ensure!(!para_data.is_locked(), Error::::ParaLocked); para_data.deposit } else { ensure!(!ensure_reserved, Error::::NotReserved); @@ -601,7 +613,7 @@ impl Pallet { } else if let Some(rebate) = deposited.checked_sub(&deposit) { ::Currency::unreserve(&who, rebate); }; - let info = ParaInfo { manager: who.clone(), deposit, locked: false }; + let info = ParaInfo { manager: who.clone(), deposit, locked: None }; Paras::::insert(id, info); // We check above that para has no lifecycle, so this should not fail. @@ -665,6 +677,21 @@ impl Pallet { } } +impl OnNewHead for Pallet { + fn on_new_head(id: ParaId, _head: &HeadData) -> Weight { + // mark the parachain locked if the locked value is not already set + let mut writes = 0; + if let Some(mut info) = Paras::::get(id) { + if info.locked.is_none() { + info.locked = Some(true); + Paras::::insert(id, info); + writes += 1; + } + } + T::DbWeight::get().reads_writes(1, writes) + } +} + #[cfg(test)] mod tests { use super::*; @@ -784,6 +811,7 @@ mod tests { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = (); type NextSessionRotation = crate::mock::TestNextSessionRotation; + type OnNewHead = (); } impl configuration::Config for Test { @@ -1270,8 +1298,10 @@ mod tests { )); assert_noop!(Registrar::add_lock(RuntimeOrigin::signed(2), para_id), BadOrigin); - // Once they begin onboarding, we lock them in. - assert_ok!(Registrar::add_lock(RuntimeOrigin::signed(1), para_id)); + + // Once they produces new block, we lock them in. + Registrar::on_new_head(para_id, &Default::default()); + // Owner cannot pass origin check when checking lock assert_noop!( Registrar::ensure_root_para_or_owner(RuntimeOrigin::signed(1), para_id), @@ -1283,6 +1313,11 @@ mod tests { assert_ok!(Registrar::remove_lock(para_origin(para_id), para_id)); // Owner can pass origin check again assert_ok!(Registrar::ensure_root_para_or_owner(RuntimeOrigin::signed(1), para_id)); + + // Won't lock again after it is unlocked + Registrar::on_new_head(para_id, &Default::default()); + + assert_ok!(Registrar::ensure_root_para_or_owner(RuntimeOrigin::signed(1), para_id)); }); } diff --git a/polkadot/runtime/kusama/Cargo.toml b/polkadot/runtime/kusama/Cargo.toml index 8b0f59516c6df385762b56349f340f0a004d5bca..565b34637e29875169f2de6701e4862f2e7ea984 100644 --- a/polkadot/runtime/kusama/Cargo.toml +++ b/polkadot/runtime/kusama/Cargo.toml @@ -77,7 +77,7 @@ pallet-recovery = { path = "../../../substrate/frame/recovery", default-features pallet-referenda = { path = "../../../substrate/frame/referenda", default-features = false } pallet-scheduler = { path = "../../../substrate/frame/scheduler", default-features = false } pallet-session = { path = "../../../substrate/frame/session", default-features = false } -pallet-society = { path = "../../../substrate/frame/society", default-features = false, features = ["experimental"] } +pallet-society = { path = "../../../substrate/frame/society", default-features = false } frame-support = { path = "../../../substrate/frame/support", default-features = false } pallet-staking = { path = "../../../substrate/frame/staking", default-features = false } pallet-state-trie-migration = { path = "../../../substrate/frame/state-trie-migration", default-features = false } @@ -116,7 +116,7 @@ tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } separator = "0.4.1" -serde_json = "1.0.96" +serde_json = "1.0.107" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } diff --git a/polkadot/runtime/kusama/src/lib.rs b/polkadot/runtime/kusama/src/lib.rs index 4b5f03b38c6c61956582c091ec41b18d5bec2d11..8d8bd4baacf8e612263c7453a38b0e53ce14974a 100644 --- a/polkadot/runtime/kusama/src/lib.rs +++ b/polkadot/runtime/kusama/src/lib.rs @@ -63,8 +63,9 @@ use frame_election_provider_support::{ use frame_support::{ construct_runtime, parameter_types, traits::{ - ConstU32, Contains, EitherOf, EitherOfDiverse, InstanceFilter, KeyOwnerProofSystem, - PrivilegeCmp, ProcessMessage, ProcessMessageError, StorageMapShim, WithdrawReasons, + fungible::HoldConsideration, ConstU32, Contains, EitherOf, EitherOfDiverse, EverythingBut, + InstanceFilter, KeyOwnerProofSystem, LinearStoragePrice, PrivilegeCmp, ProcessMessage, + ProcessMessageError, StorageMapShim, WithdrawReasons, }, weights::{ConstantMultiplier, WeightMeter}, PalletId, @@ -92,7 +93,7 @@ use xcm::latest::Junction; pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; -pub use pallet_election_provider_multi_phase::Call as EPMCall; +pub use pallet_election_provider_multi_phase::{Call as EPMCall, GeometricDepositBase}; #[cfg(feature = "std")] pub use pallet_staking::StakerStatus; use pallet_staking::UseValidatorsMap; @@ -157,11 +158,14 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } -/// We currently allow all calls. -pub struct BaseFilter; -impl Contains for BaseFilter { - fn contains(_c: &RuntimeCall) -> bool { - true +/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, +/// locking the state of the pallet and preventing further updates to identities and sub-identities. +/// The locked state will be the genesis state of a new system chain and then removed from the Relay +/// Chain. +pub struct IdentityCalls; +impl Contains for IdentityCalls { + fn contains(c: &RuntimeCall) -> bool { + matches!(c, RuntimeCall::Identity(_)) } } @@ -171,7 +175,7 @@ parameter_types! { } impl frame_system::Config for Runtime { - type BaseCallFilter = frame_support::traits::Everything; + type BaseCallFilter = EverythingBut; type BlockWeights = BlockWeights; type BlockLength = BlockLength; type RuntimeOrigin = RuntimeOrigin; @@ -228,7 +232,8 @@ impl pallet_scheduler::Config for Runtime { type MaximumWeight = MaximumSchedulerWeight; // The goal of having ScheduleOrigin include AuctionAdmin is to allow the auctions track of // OpenGov to schedule periodic auctions. - type ScheduleOrigin = EitherOf, AuctionAdmin>; + // Also allow Treasurer to schedule recurring payments. + type ScheduleOrigin = EitherOf, AuctionAdmin>, Treasurer>; type MaxScheduledPerBlock = MaxScheduledPerBlock; type WeightInfo = weights::pallet_scheduler::WeightInfo; type OriginPrivilegeCmp = OriginPrivilegeCmp; @@ -238,6 +243,7 @@ impl pallet_scheduler::Config for Runtime { parameter_types! { pub const PreimageBaseDeposit: Balance = deposit(2, 64); pub const PreimageByteDeposit: Balance = deposit(0, 1); + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -245,8 +251,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } parameter_types! { @@ -504,7 +514,8 @@ parameter_types! { // signed config pub const SignedMaxSubmissions: u32 = 16; pub const SignedMaxRefunds: u32 = 16 / 4; - pub const SignedDepositBase: Balance = deposit(2, 0); + pub const SignedFixedDeposit: Balance = deposit(2, 0); + pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); pub const SignedDepositByte: Balance = deposit(0, 10) / 1024; // Each good submission will get 1/10 KSM as reward pub SignedRewardBase: Balance = UNITS / 10; @@ -577,7 +588,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SignedMaxSubmissions = SignedMaxSubmissions; type SignedMaxRefunds = SignedMaxRefunds; type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; + type SignedDepositBase = + GeometricDepositBase; type SignedDepositByte = SignedDepositByte; type SignedDepositWeight = (); type SignedMaxWeight = @@ -636,7 +648,7 @@ impl pallet_staking::EraPayout for EraPayout { // all para-ids that are currently active. let auctioned_slots = Paras::parachains() .into_iter() - // all active para-ids that do not belong to a system or common good chain is the number + // all active para-ids that do not belong to a system chain is the number // of parachains that we should take into account for inflation. .filter(|i| *i >= LOWEST_PUBLIC_ID) .count() as u64; @@ -1214,6 +1226,7 @@ impl parachains_paras::Config for Runtime { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = Babe; + type OnNewHead = Registrar; } parameter_types! { @@ -1558,7 +1571,7 @@ construct_runtime! { Multisig: pallet_multisig::{Pallet, Call, Storage, Event} = 31, // Preimage registrar. - Preimage: pallet_preimage::{Pallet, Call, Storage, Event} = 32, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 32, // Bounties modules. Bounties: pallet_bounties::{Pallet, Call, Storage, Event} = 35, @@ -1709,6 +1722,19 @@ pub mod migrations { } } + pub struct ParachainsToUnlock; + impl Contains for ParachainsToUnlock { + fn contains(id: &ParaId) -> bool { + let id: u32 = (*id).into(); + // ksuama parachains/parathreads that are locked and never produced block + match id { + 2003 | 2008 | 2018 | 2077 | 2089 | 2111 | 2112 | 2120 | 2126 | 2127 | 2130 | + 2226 | 2227 | 2231 | 2233 | 2237 | 2256 | 2257 | 2261 | 2268 | 2275 => true, + _ => false, + } + } + } + /// Unreleased migrations. Add new ones here: pub type Unreleased = ( init_state_migration::InitMigrate, @@ -1740,6 +1766,8 @@ pub mod migrations { UpgradeSessionKeys, parachains_configuration::migration::v9::MigrateToV9, + // Migrate parachain info format + paras_registrar::migration::VersionCheckedMigrateToV1, ); } @@ -2461,7 +2489,7 @@ mod fees_tests { fn signed_deposit_is_sensible() { // ensure this number does not change, or that it is checked after each change. // a 1 MB solution should need around 0.16 KSM deposit - let deposit = SignedDepositBase::get() + (SignedDepositByte::get() * 1024 * 1024); + let deposit = SignedFixedDeposit::get() + (SignedDepositByte::get() * 1024 * 1024); assert_eq_error_rate!(deposit, UNITS * 167 / 100, UNITS / 100); } } diff --git a/polkadot/runtime/kusama/src/weights/pallet_preimage.rs b/polkadot/runtime/kusama/src/weights/pallet_preimage.rs index 8c04eb2cd4ea424cc77dcdf6e882eff987408096..d0bf205666097d11b644fe7c31da4b86838f9e72 100644 --- a/polkadot/runtime/kusama/src/weights/pallet_preimage.rs +++ b/polkadot/runtime/kusama/src/weights/pallet_preimage.rs @@ -50,6 +50,21 @@ use core::marker::PhantomData; /// Weight functions for `pallet_preimage`. pub struct WeightInfo(PhantomData); impl pallet_preimage::WeightInfo for WeightInfo { + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_000_000 picoseconds. + Weight::from_parts(2_000_000, 3593) + // Standard Error: 13_720 + .saturating_add(Weight::from_parts(17_309_199, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) + } + /// Storage: Preimage StatusFor (r:1 w:1) /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) /// Storage: Preimage PreimageFor (r:0 w:1) diff --git a/polkadot/runtime/kusama/src/weights/runtime_parachains_hrmp.rs b/polkadot/runtime/kusama/src/weights/runtime_parachains_hrmp.rs index c13a8413e4102ac4fbedb64fab0c92fcb6dcf247..f2bc2aa2b085bf13594c144b335f9076465b5d56 100644 --- a/polkadot/runtime/kusama/src/weights/runtime_parachains_hrmp.rs +++ b/polkadot/runtime/kusama/src/weights/runtime_parachains_hrmp.rs @@ -282,4 +282,46 @@ impl runtime_parachains::hrmp::WeightInfo for WeightInf .saturating_add(T::DbWeight::get().reads(13)) .saturating_add(T::DbWeight::get().writes(8)) } + /// Storage: `Paras::ParaLifecycles` (r:1 w:0) + /// Proof: `Paras::ParaLifecycles` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequests` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequests` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpChannels` (r:1 w:0) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpEgressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpEgressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestsList` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestsList` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueues` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueues` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueueHeads` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueueHeads` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpIngressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpIngressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpAcceptedChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpAcceptedChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn establish_system_channel() -> Weight { + // Proof Size summary in bytes: + // Measured: `417` + // Estimated: `6357` + // Minimum execution time: 629_674_000 picoseconds. + Weight::from_parts(640_174_000, 0) + .saturating_add(Weight::from_parts(0, 6357)) + .saturating_add(T::DbWeight::get().reads(12)) + .saturating_add(T::DbWeight::get().writes(8)) + } + /// Storage: `Hrmp::HrmpChannels` (r:1 w:1) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn poke_channel_deposits() -> Weight { + // Proof Size summary in bytes: + // Measured: `263` + // Estimated: `3728` + // Minimum execution time: 173_371_000 picoseconds. + Weight::from_parts(175_860_000, 0) + .saturating_add(Weight::from_parts(0, 3728)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } } diff --git a/polkadot/runtime/kusama/src/xcm_config.rs b/polkadot/runtime/kusama/src/xcm_config.rs index c90e6c55a94b42d9e3c1273dbfbc8a7ae105bb89..40ecc83fe641b37929279974ae14a29ed482d28f 100644 --- a/polkadot/runtime/kusama/src/xcm_config.rs +++ b/polkadot/runtime/kusama/src/xcm_config.rs @@ -23,13 +23,12 @@ use super::{ }; use frame_support::{ match_types, parameter_types, - traits::{Contains, Everything, Nothing}, + traits::{Everything, Nothing}, weights::Weight, }; use frame_system::EnsureRoot; use kusama_runtime_constants::currency::CENTS; use runtime_common::{ - crowdloan, paras_registrar, xcm_sender::{ChildParachainRouter, ExponentialPrice}, ToAuthor, }; @@ -38,13 +37,12 @@ use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, ChildParachainAsNative, - ChildParachainConvertsVia, ChildSystemParachainAsSuperuser, - CurrencyAdapter as XcmCurrencyAdapter, IsChildSystemParachain, IsConcrete, MintLocation, + ChildParachainConvertsVia, CurrencyAdapter as XcmCurrencyAdapter, DescribeBodyTerminal, + DescribeFamily, HashedDescription, IsChildSystemParachain, IsConcrete, MintLocation, OriginToPluralityVoice, SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, WeightInfoBounds, WithComputedOrigin, WithUniqueTopic, }; -use xcm_executor::traits::WithOriginFilter; parameter_types! { /// The location of the KSM token, from the context of this chain. Since this token is native to this @@ -70,6 +68,8 @@ pub type SovereignAccountOf = ( ChildParachainConvertsVia, // We can directly alias an `AccountId32` into a local account. AccountId32Aliases, + // Allow governance body to be used as a sovereign account. + HashedDescription>, ); /// Our asset transactor. This is what allows us to interest with the runtime facilities from the @@ -97,8 +97,6 @@ type LocalOriginConverter = ( ChildParachainAsNative, // The AccountId32 location type can be expressed natively as a `Signed` origin. SignedAccountId32AsNative, - // A system child parachain, expressed as a Superuser, converts to the `Root` origin. - ChildSystemParachainAsSuperuser, ); parameter_types! { @@ -161,158 +159,6 @@ pub type Barrier = TrailingSetTopicAsId<( >, )>; -/// A call filter for the XCM Transact instruction. This is a temporary measure until we properly -/// account for proof size weights. -/// -/// Calls that are allowed through this filter must: -/// 1. Have a fixed weight; -/// 2. Cannot lead to another call being made; -/// 3. Have a defined proof size weight, e.g. no unbounded vecs in call parameters. -pub struct SafeCallFilter; -impl Contains for SafeCallFilter { - fn contains(call: &RuntimeCall) -> bool { - #[cfg(feature = "runtime-benchmarks")] - { - if matches!(call, RuntimeCall::System(frame_system::Call::remark_with_event { .. })) { - return true - } - } - - match call { - RuntimeCall::System( - frame_system::Call::kill_prefix { .. } | frame_system::Call::set_heap_pages { .. }, - ) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(..) | - RuntimeCall::Balances(..) | - RuntimeCall::Crowdloan( - crowdloan::Call::create { .. } | - crowdloan::Call::contribute { .. } | - crowdloan::Call::withdraw { .. } | - crowdloan::Call::refund { .. } | - crowdloan::Call::dissolve { .. } | - crowdloan::Call::edit { .. } | - crowdloan::Call::poke { .. } | - crowdloan::Call::contribute_all { .. }, - ) | - RuntimeCall::Staking( - pallet_staking::Call::bond { .. } | - pallet_staking::Call::bond_extra { .. } | - pallet_staking::Call::unbond { .. } | - pallet_staking::Call::withdraw_unbonded { .. } | - pallet_staking::Call::validate { .. } | - pallet_staking::Call::nominate { .. } | - pallet_staking::Call::chill { .. } | - pallet_staking::Call::set_payee { .. } | - pallet_staking::Call::set_controller { .. } | - pallet_staking::Call::set_validator_count { .. } | - pallet_staking::Call::increase_validator_count { .. } | - pallet_staking::Call::scale_validator_count { .. } | - pallet_staking::Call::force_no_eras { .. } | - pallet_staking::Call::force_new_era { .. } | - pallet_staking::Call::set_invulnerables { .. } | - pallet_staking::Call::force_unstake { .. } | - pallet_staking::Call::force_new_era_always { .. } | - pallet_staking::Call::payout_stakers { .. } | - pallet_staking::Call::rebond { .. } | - pallet_staking::Call::reap_stash { .. } | - pallet_staking::Call::set_staking_configs { .. } | - pallet_staking::Call::chill_other { .. } | - pallet_staking::Call::force_apply_min_commission { .. }, - ) | - RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Treasury(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda( - pallet_referenda::Call::place_decision_deposit { .. } | - pallet_referenda::Call::refund_decision_deposit { .. } | - pallet_referenda::Call::cancel { .. } | - pallet_referenda::Call::kill { .. } | - pallet_referenda::Call::nudge_referendum { .. } | - pallet_referenda::Call::one_fewer_deciding { .. }, - ) | - RuntimeCall::FellowshipCollective(..) | - RuntimeCall::FellowshipReferenda( - pallet_referenda::Call::place_decision_deposit { .. } | - pallet_referenda::Call::refund_decision_deposit { .. } | - pallet_referenda::Call::cancel { .. } | - pallet_referenda::Call::kill { .. } | - pallet_referenda::Call::nudge_referendum { .. } | - pallet_referenda::Call::one_fewer_deciding { .. }, - ) | - RuntimeCall::Claims( - super::claims::Call::claim { .. } | - super::claims::Call::mint_claim { .. } | - super::claims::Call::move_claim { .. }, - ) | - RuntimeCall::Utility(pallet_utility::Call::as_derivative { .. }) | - RuntimeCall::Identity( - pallet_identity::Call::add_registrar { .. } | - pallet_identity::Call::set_identity { .. } | - pallet_identity::Call::clear_identity { .. } | - pallet_identity::Call::request_judgement { .. } | - pallet_identity::Call::cancel_request { .. } | - pallet_identity::Call::set_fee { .. } | - pallet_identity::Call::set_account_id { .. } | - pallet_identity::Call::set_fields { .. } | - pallet_identity::Call::provide_judgement { .. } | - pallet_identity::Call::kill_identity { .. } | - pallet_identity::Call::add_sub { .. } | - pallet_identity::Call::rename_sub { .. } | - pallet_identity::Call::remove_sub { .. } | - pallet_identity::Call::quit_sub { .. }, - ) | - RuntimeCall::Society(..) | - RuntimeCall::Recovery(..) | - RuntimeCall::Vesting(..) | - RuntimeCall::Bounties( - pallet_bounties::Call::propose_bounty { .. } | - pallet_bounties::Call::approve_bounty { .. } | - pallet_bounties::Call::propose_curator { .. } | - pallet_bounties::Call::unassign_curator { .. } | - pallet_bounties::Call::accept_curator { .. } | - pallet_bounties::Call::award_bounty { .. } | - pallet_bounties::Call::claim_bounty { .. } | - pallet_bounties::Call::close_bounty { .. }, - ) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ElectionProviderMultiPhase(..) | - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools( - pallet_nomination_pools::Call::join { .. } | - pallet_nomination_pools::Call::bond_extra { .. } | - pallet_nomination_pools::Call::claim_payout { .. } | - pallet_nomination_pools::Call::unbond { .. } | - pallet_nomination_pools::Call::pool_withdraw_unbonded { .. } | - pallet_nomination_pools::Call::withdraw_unbonded { .. } | - pallet_nomination_pools::Call::create { .. } | - pallet_nomination_pools::Call::create_with_pool_id { .. } | - pallet_nomination_pools::Call::set_state { .. } | - pallet_nomination_pools::Call::set_configs { .. } | - pallet_nomination_pools::Call::update_roles { .. } | - pallet_nomination_pools::Call::chill { .. }, - ) | - RuntimeCall::Hrmp(..) | - RuntimeCall::Registrar( - paras_registrar::Call::deregister { .. } | - paras_registrar::Call::swap { .. } | - paras_registrar::Call::remove_lock { .. } | - paras_registrar::Call::reserve { .. } | - paras_registrar::Call::add_lock { .. }, - ) | - RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { - .. - }) | - RuntimeCall::Whitelist(pallet_whitelist::Call::whitelist_call { .. }) | - RuntimeCall::Proxy(..) => true, - _ => false, - } - } -} - pub struct XcmConfig; impl xcm_executor::Config for XcmConfig { type RuntimeCall = RuntimeCall; @@ -343,8 +189,8 @@ impl xcm_executor::Config for XcmConfig { // No bridges yet... type MessageExporter = (); type UniversalAliases = Nothing; - type CallDispatcher = WithOriginFilter; - type SafeCallFilter = SafeCallFilter; + type CallDispatcher = RuntimeCall; + type SafeCallFilter = Everything; type Aliasers = Nothing; } diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index 9a8bd5017e074f6827799824beb38d2c0b8a5c59..9f1ec57257f821901c2febde5a02c0881aadc1be 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -6,6 +6,7 @@ edition.workspace = true license.workspace = true [dependencies] +impl-trait-for-tuples = "0.2.2" bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] } parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive", "max-encoded-len"] } log = { version = "0.4.17", default-features = false } @@ -49,6 +50,7 @@ rand_chacha = { version = "0.3.1", default-features = false } static_assertions = { version = "1.1.0", optional = true } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } polkadot-runtime-metrics = { path = "../metrics", default-features = false} +polkadot-core-primitives = { path = "../../core-primitives", default-features = false } [dev-dependencies] futures = "0.3.21" @@ -60,7 +62,7 @@ test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../pri sp-tracing = { path = "../../../substrate/primitives/tracing" } thousands = "0.2.0" assert_matches = "1" -serde_json = "1.0.96" +serde_json = "1.0.107" [features] default = [ "std" ] @@ -82,6 +84,7 @@ std = [ "pallet-timestamp/std", "pallet-vesting/std", "parity-scale-codec/std", + "polkadot-core-primitives/std", "polkadot-parachain-primitives/std", "polkadot-runtime-metrics/std", "primitives/std", diff --git a/polkadot/runtime/parachains/src/assigner.rs b/polkadot/runtime/parachains/src/assigner.rs index 55434da11f307cd2a7af86f6d81d29460df6678f..b21e857a471379a57af4974d2280e9da69448ecd 100644 --- a/polkadot/runtime/parachains/src/assigner.rs +++ b/polkadot/runtime/parachains/src/assigner.rs @@ -17,11 +17,11 @@ //! The Polkadot multiplexing assignment provider. //! Provides blockspace assignments for both bulk and on demand parachains. use frame_system::pallet_prelude::BlockNumberFor; -use primitives::{v5::Assignment, CoreIndex, Id as ParaId}; +use primitives::{CoreIndex, Id as ParaId}; use crate::{ configuration, paras, - scheduler::common::{AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, }; pub use pallet::*; diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs index 0c9813d144f34611d28536c2ed93f1b6a98ad088..75c29bd6fbe4f79532823468a4c0a596942491a6 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/mod.rs @@ -34,7 +34,7 @@ mod tests; use crate::{ configuration, paras, - scheduler::common::{AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, }; use frame_support::{ @@ -46,7 +46,7 @@ use frame_support::{ }, }; use frame_system::pallet_prelude::*; -use primitives::{v5::Assignment, CoreIndex, Id as ParaId}; +use primitives::{CoreIndex, Id as ParaId}; use sp_runtime::{ traits::{One, SaturatedConversion}, FixedPointNumber, FixedPointOperand, FixedU128, Perbill, Saturating, @@ -606,7 +606,6 @@ impl AssignmentProvider> for Pallet { fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { let config = >::config(); AssignmentProviderConfig { - availability_period: config.paras_availability_period, max_availability_timeouts: config.on_demand_retries, ttl: config.on_demand_ttl, } diff --git a/polkadot/runtime/parachains/src/assigner_on_demand/tests.rs b/polkadot/runtime/parachains/src/assigner_on_demand/tests.rs index 8041179cd90c51c6a1d6bb820ba0572539f1e271..fe9a4e52bd076e2698c9e153a770c0261f6349fe 100644 --- a/polkadot/runtime/parachains/src/assigner_on_demand/tests.rs +++ b/polkadot/runtime/parachains/src/assigner_on_demand/tests.rs @@ -24,13 +24,11 @@ use crate::{ System, Test, }, paras::{ParaGenesisArgs, ParaKind}, + scheduler::common::Assignment, }; use frame_support::{assert_noop, assert_ok, error::BadOrigin}; use pallet_balances::Error as BalancesError; -use primitives::{ - v5::{Assignment, ValidationCode}, - BlockNumber, SessionIndex, -}; +use primitives::{v5::ValidationCode, BlockNumber, SessionIndex}; use sp_std::collections::btree_map::BTreeMap; fn schedule_blank_para(id: ParaId, parakind: ParaKind) { diff --git a/polkadot/runtime/parachains/src/assigner_parachains.rs b/polkadot/runtime/parachains/src/assigner_parachains.rs index 9a6b970597d52e58d595afad642ffdde7681e857..d605d86605151071b3c5dda9c825585f723ecaf1 100644 --- a/polkadot/runtime/parachains/src/assigner_parachains.rs +++ b/polkadot/runtime/parachains/src/assigner_parachains.rs @@ -19,11 +19,11 @@ use crate::{ configuration, paras, - scheduler::common::{AssignmentProvider, AssignmentProviderConfig}, + scheduler::common::{Assignment, AssignmentProvider, AssignmentProviderConfig}, }; use frame_system::pallet_prelude::BlockNumberFor; pub use pallet::*; -use primitives::{v5::Assignment, CoreIndex, Id as ParaId}; +use primitives::{CoreIndex, Id as ParaId}; #[frame_support::pallet] pub mod pallet { @@ -57,9 +57,7 @@ impl AssignmentProvider> for Pallet { fn push_assignment_for_core(_: CoreIndex, _: Assignment) {} fn get_provider_config(_core_idx: CoreIndex) -> AssignmentProviderConfig> { - let config = >::config(); AssignmentProviderConfig { - availability_period: config.paras_availability_period, // The next assignment already goes to the same [`ParaId`], no timeout tracking needed. max_availability_timeouts: 0, // The next assignment already goes to the same [`ParaId`], this can be any number diff --git a/polkadot/runtime/parachains/src/builder.rs b/polkadot/runtime/parachains/src/builder.rs index 4921af5bedda7c0354c30eeef85a153556eb7fee..dced24df0aec83d1f3aba619426f607f0dd88d1b 100644 --- a/polkadot/runtime/parachains/src/builder.rs +++ b/polkadot/runtime/parachains/src/builder.rs @@ -18,18 +18,20 @@ use crate::{ configuration, inclusion, initializer, paras, paras::ParaKind, paras_inherent, - scheduler::{self, common::AssignmentProviderConfig}, + scheduler::{ + self, + common::{Assignment, AssignmentProviderConfig}, + CoreOccupied, ParasEntry, + }, session_info, shared, }; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; use primitives::{ - collator_signature_payload, - v5::{Assignment, ParasEntry}, - AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor, - CandidateHash, CollatorId, CollatorSignature, CommittedCandidateReceipt, CompactStatement, - CoreIndex, CoreOccupied, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, + collator_signature_payload, AvailabilityBitfield, BackedCandidate, CandidateCommitments, + CandidateDescriptor, CandidateHash, CollatorId, CollatorSignature, CommittedCandidateReceipt, + CompactStatement, CoreIndex, DisputeStatement, DisputeStatementSet, GroupIndex, HeadData, Id as ParaId, IndexedVec, InherentData as ParachainsInherentData, InvalidDisputeStatementKind, PersistedValidationData, SessionIndex, SigningContext, UncheckedSigned, ValidDisputeStatementKind, ValidationCode, ValidatorId, ValidatorIndex, ValidityAttestation, diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index 0c66bb5bdf961c21898a3d2e9bb220f7a56ae6c4..33039cd08ca4b90e58b2a1e415242402d81dad92 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -190,11 +190,20 @@ pub struct HostConfiguration { /// /// Must be non-zero. pub group_rotation_frequency: BlockNumber, - /// The availability period, in blocks. This is the amount of blocks - /// after inclusion that validators have to make the block available and signal its - /// availability to the chain. + /// The minimum availability period, in blocks. /// - /// Must be at least 1. + /// This is the minimum amount of blocks after a core became occupied that validators have time + /// to make the block available. + /// + /// This value only has effect on group rotations. If backers backed something at the end of + /// their rotation, the occupied core affects the backing group that comes afterwards. We limit + /// the effect one backing group can have on the next to `paras_availability_period` blocks. + /// + /// Within a group rotation there is no timeout as backers are only affecting themselves. + /// + /// Must be at least 1. With a value of 1, the previous group will not be able to negatively + /// affect the following group at the expense of a tight availability timeline at group + /// rotation boundaries. pub paras_availability_period: BlockNumber, /// The amount of blocks ahead to schedule paras. pub scheduling_lookahead: u32, diff --git a/polkadot/runtime/parachains/src/hrmp.rs b/polkadot/runtime/parachains/src/hrmp.rs index cdb38ba6a8eaf855a6977d01c0c2fa9183cb6396..3f0a5e0830cb0ecfacfeeff9b6e3317d7376dd1f 100644 --- a/polkadot/runtime/parachains/src/hrmp.rs +++ b/polkadot/runtime/parachains/src/hrmp.rs @@ -21,13 +21,16 @@ use crate::{ use frame_support::{pallet_prelude::*, traits::ReservableCurrency, DefaultNoBound}; use frame_system::pallet_prelude::*; use parity_scale_codec::{Decode, Encode}; -use polkadot_parachain_primitives::primitives::HorizontalMessages; +use polkadot_parachain_primitives::primitives::{HorizontalMessages, IsSystem}; use primitives::{ Balance, Hash, HrmpChannelId, Id as ParaId, InboundHrmpMessage, OutboundHrmpMessage, SessionIndex, }; use scale_info::TypeInfo; -use sp_runtime::traits::{AccountIdConversion, BlakeTwo256, Hash as HashT, UniqueSaturatedInto}; +use sp_runtime::{ + traits::{AccountIdConversion, BlakeTwo256, Hash as HashT, UniqueSaturatedInto, Zero}, + ArithmeticError, +}; use sp_std::{ collections::{btree_map::BTreeMap, btree_set::BTreeSet}, fmt, mem, @@ -60,6 +63,8 @@ pub trait WeightInfo { fn hrmp_cancel_open_request(c: u32) -> Weight; fn clean_open_channel_requests(c: u32) -> Weight; fn force_open_hrmp_channel(c: u32) -> Weight; + fn establish_system_channel() -> Weight; + fn poke_channel_deposits() -> Weight; } /// A weight info that is only suitable for testing. @@ -93,6 +98,12 @@ impl WeightInfo for TestWeightInfo { fn force_open_hrmp_channel(_: u32) -> Weight { Weight::MAX } + fn establish_system_channel() -> Weight { + Weight::MAX + } + fn poke_channel_deposits() -> Weight { + Weight::MAX + } } /// A description of a request to open an HRMP channel. @@ -279,6 +290,12 @@ pub mod pallet { /// An HRMP channel was opened via Root origin. /// `[sender, recipient, proposed_max_capacity, proposed_max_message_size]` HrmpChannelForceOpened(ParaId, ParaId, u32, u32), + /// An HRMP channel was opened between two system chains. + /// `[sender, recipient, proposed_max_capacity, proposed_max_message_size]` + HrmpSystemChannelOpened(ParaId, ParaId, u32, u32), + /// An HRMP channel's deposits were updated. + /// `[sender, recipient]` + OpenChannelDepositsUpdated(ParaId, ParaId), } #[pallet::error] @@ -321,6 +338,8 @@ pub mod pallet { OpenHrmpChannelAlreadyConfirmed, /// The provided witness data is wrong. WrongWitness, + /// The channel between these two chains cannot be authorized. + ChannelCreationNotAuthorized, } /// The set of pending HRMP open channel requests. @@ -600,8 +619,8 @@ pub mod pallet { /// the `max_capacity` and `max_message_size` are still subject to the Relay Chain's /// configured limits. /// - /// Expected use is when one of the `ParaId`s involved in the channel is governed by the - /// Relay Chain, e.g. a system parachain. + /// Expected use is when one (and only one) of the `ParaId`s involved in the channel is + /// governed by the system, e.g. a system parachain. /// /// Origin must be the `ChannelManager`. #[pallet::call_index(7)] @@ -628,7 +647,8 @@ pub mod pallet { 0 }; - // Now we proceed with normal init/accept. + // Now we proceed with normal init/accept, except that we set `no_deposit` to true such + // that it will not require deposits from either member. Self::init_open_channel(sender, recipient, max_capacity, max_message_size)?; Self::accept_open_channel(recipient, sender)?; Self::deposit_event(Event::HrmpChannelForceOpened( @@ -640,6 +660,146 @@ pub mod pallet { Ok(Some(::WeightInfo::force_open_hrmp_channel(cancel_request)).into()) } + + /// Establish an HRMP channel between two system chains. If the channel does not already + /// exist, the transaction fees will be refunded to the caller. The system does not take + /// deposits for channels between system chains, and automatically sets the message number + /// and size limits to the maximum allowed by the network's configuration. + /// + /// Arguments: + /// + /// - `sender`: A system chain, `ParaId`. + /// - `recipient`: A system chain, `ParaId`. + /// + /// Any signed origin can call this function, but _both_ inputs MUST be system chains. If + /// the channel does not exist yet, there is no fee. + #[pallet::call_index(8)] + #[pallet::weight(::WeightInfo::establish_system_channel())] + pub fn establish_system_channel( + origin: OriginFor, + sender: ParaId, + recipient: ParaId, + ) -> DispatchResultWithPostInfo { + let _caller = ensure_signed(origin)?; + + // both chains must be system + ensure!( + sender.is_system() && recipient.is_system(), + Error::::ChannelCreationNotAuthorized + ); + + let config = >::config(); + let max_message_size = config.hrmp_channel_max_message_size; + let max_capacity = config.hrmp_channel_max_capacity; + + Self::init_open_channel(sender, recipient, max_capacity, max_message_size)?; + Self::accept_open_channel(recipient, sender)?; + + Self::deposit_event(Event::HrmpSystemChannelOpened( + sender, + recipient, + max_capacity, + max_message_size, + )); + + Ok(Pays::No.into()) + } + + /// Update the deposits held for an HRMP channel to the latest `Configuration`. Channels + /// with system chains do not require a deposit. + /// + /// Arguments: + /// + /// - `sender`: A chain, `ParaId`. + /// - `recipient`: A chain, `ParaId`. + /// + /// Any signed origin can call this function. + #[pallet::call_index(9)] + #[pallet::weight(::WeightInfo::poke_channel_deposits())] + pub fn poke_channel_deposits( + origin: OriginFor, + sender: ParaId, + recipient: ParaId, + ) -> DispatchResult { + let _caller = ensure_signed(origin)?; + let channel_id = HrmpChannelId { sender, recipient }; + let is_system = sender.is_system() || recipient.is_system(); + + let config = >::config(); + + // Channels with and amongst the system do not require a deposit. + let (new_sender_deposit, new_recipient_deposit) = if is_system { + (0, 0) + } else { + (config.hrmp_sender_deposit, config.hrmp_recipient_deposit) + }; + + let _ = HrmpChannels::::mutate(&channel_id, |channel| -> DispatchResult { + if let Some(ref mut channel) = channel { + let current_sender_deposit = channel.sender_deposit; + let current_recipient_deposit = channel.recipient_deposit; + + // nothing to update + if current_sender_deposit == new_sender_deposit && + current_recipient_deposit == new_recipient_deposit + { + return Ok(()) + } + + // sender + if current_sender_deposit > new_sender_deposit { + // Can never underflow, but be paranoid. + let amount = current_sender_deposit + .checked_sub(new_sender_deposit) + .ok_or(ArithmeticError::Underflow)?; + T::Currency::unreserve( + &channel_id.sender.into_account_truncating(), + // The difference should always be convertable into `Balance`, but be + // paranoid and do nothing in case. + amount.try_into().unwrap_or(Zero::zero()), + ); + } else if current_sender_deposit < new_sender_deposit { + let amount = new_sender_deposit + .checked_sub(current_sender_deposit) + .ok_or(ArithmeticError::Underflow)?; + T::Currency::reserve( + &channel_id.sender.into_account_truncating(), + amount.try_into().unwrap_or(Zero::zero()), + )?; + } + + // recipient + if current_recipient_deposit > new_recipient_deposit { + let amount = current_recipient_deposit + .checked_sub(new_recipient_deposit) + .ok_or(ArithmeticError::Underflow)?; + T::Currency::unreserve( + &channel_id.recipient.into_account_truncating(), + amount.try_into().unwrap_or(Zero::zero()), + ); + } else if current_recipient_deposit < new_recipient_deposit { + let amount = new_recipient_deposit + .checked_sub(current_recipient_deposit) + .ok_or(ArithmeticError::Underflow)?; + T::Currency::reserve( + &channel_id.recipient.into_account_truncating(), + amount.try_into().unwrap_or(Zero::zero()), + )?; + } + + // update storage + channel.sender_deposit = new_sender_deposit; + channel.recipient_deposit = new_recipient_deposit; + } else { + return Err(Error::::OpenHrmpChannelDoesntExist.into()) + } + Ok(()) + })?; + + Self::deposit_event(Event::OpenChannelDepositsUpdated(sender, recipient)); + + Ok(()) + } } } @@ -817,6 +977,10 @@ impl Pallet { "can't be `None` due to the invariant that the list contains the same items as the set; qed", ); + let system_channel = channel_id.sender.is_system() || channel_id.recipient.is_system(); + let sender_deposit = request.sender_deposit; + let recipient_deposit = if system_channel { 0 } else { config.hrmp_recipient_deposit }; + if request.confirmed { if >::is_valid_para(channel_id.sender) && >::is_valid_para(channel_id.recipient) @@ -824,8 +988,8 @@ impl Pallet { HrmpChannels::::insert( &channel_id, HrmpChannel { - sender_deposit: request.sender_deposit, - recipient_deposit: config.hrmp_recipient_deposit, + sender_deposit, + recipient_deposit, max_capacity: request.max_capacity, max_total_size: request.max_total_size, max_message_size: request.max_message_size, @@ -1173,7 +1337,9 @@ impl Pallet { } /// Initiate opening a channel from a parachain to a given recipient with given channel - /// parameters. + /// parameters. If neither chain is part of the system, then a deposit from the `Configuration` + /// will be required for `origin` (the sender) upon opening the request and the `recipient` upon + /// accepting it. /// /// Basically the same as [`hrmp_init_open_channel`](Pallet::hrmp_init_open_channel) but /// intended for calling directly from other pallets rather than dispatched. @@ -1219,10 +1385,15 @@ impl Pallet { Error::::OpenHrmpChannelLimitExceeded, ); - T::Currency::reserve( - &origin.into_account_truncating(), - config.hrmp_sender_deposit.unique_saturated_into(), - )?; + // Do not require deposits for channels with or amongst the system. + let is_system = origin.is_system() || recipient.is_system(); + let deposit = if is_system { 0 } else { config.hrmp_sender_deposit }; + if !deposit.is_zero() { + T::Currency::reserve( + &origin.into_account_truncating(), + deposit.unique_saturated_into(), + )?; + } // mutating storage directly now -- shall not bail henceforth. @@ -1232,7 +1403,7 @@ impl Pallet { HrmpOpenChannelRequest { confirmed: false, _age: 0, - sender_deposit: config.hrmp_sender_deposit, + sender_deposit: deposit, max_capacity: proposed_max_capacity, max_message_size: proposed_max_message_size, max_total_size: config.hrmp_channel_max_total_size, @@ -1254,7 +1425,7 @@ impl Pallet { if let Err(dmp::QueueDownwardMessageError::ExceedsMaxMessageSize) = >::queue_downward_message(&config, recipient, notification_bytes) { - // this should never happen unless the max downward message size is configured to an + // this should never happen unless the max downward message size is configured to a // jokingly small number. log::error!( target: "runtime::hrmp", @@ -1287,10 +1458,15 @@ impl Pallet { Error::::AcceptHrmpChannelLimitExceeded, ); - T::Currency::reserve( - &origin.into_account_truncating(), - config.hrmp_recipient_deposit.unique_saturated_into(), - )?; + // Do not require deposits for channels with or amongst the system. + let is_system = origin.is_system() || sender.is_system(); + let deposit = if is_system { 0 } else { config.hrmp_recipient_deposit }; + if !deposit.is_zero() { + T::Currency::reserve( + &origin.into_account_truncating(), + deposit.unique_saturated_into(), + )?; + } // persist the updated open channel request and then increment the number of accepted // channels. diff --git a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs index 3fe347a7bcbaf054b23afa67ecaa7bb20d9100b0..9ffa264a7dc86f4a1c5ef637a4b4ab85c7027841 100644 --- a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs +++ b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs @@ -304,14 +304,13 @@ frame_benchmarking::benchmarks! { let sender_origin: crate::Origin = 1u32.into(); let recipient_id: ParaId = 2u32.into(); - // make sure para is registered, and has enough balance. + // Make sure para is registered. The sender does actually need the normal deposit because it + // is first going the "init" route. let ed = T::Currency::minimum_balance(); let sender_deposit: BalanceOf = Configuration::::config().hrmp_sender_deposit.unique_saturated_into(); - let recipient_deposit: BalanceOf = - Configuration::::config().hrmp_recipient_deposit.unique_saturated_into(); register_parachain_with_balance::(sender_id, sender_deposit + ed); - register_parachain_with_balance::(recipient_id, recipient_deposit + ed); + register_parachain_with_balance::(recipient_id, Zero::zero()); let capacity = Configuration::::config().hrmp_channel_max_capacity; let message_size = Configuration::::config().hrmp_channel_max_message_size; @@ -351,6 +350,72 @@ frame_benchmarking::benchmarks! { Event::::HrmpChannelForceOpened(sender_id, recipient_id, capacity, message_size).into() ); } + + establish_system_channel { + let sender_id: ParaId = 1u32.into(); + let recipient_id: ParaId = 2u32.into(); + + let caller: T::AccountId = frame_benchmarking::whitelisted_caller(); + let config = Configuration::::config(); + + // make sure para is registered, and has zero balance. + register_parachain_with_balance::(sender_id, Zero::zero()); + register_parachain_with_balance::(recipient_id, Zero::zero()); + + let capacity = config.hrmp_channel_max_capacity; + let message_size = config.hrmp_channel_max_message_size; + }: _(frame_system::RawOrigin::Signed(caller), sender_id, recipient_id) + verify { + assert_last_event::( + Event::::HrmpSystemChannelOpened(sender_id, recipient_id, capacity, message_size).into() + ); + } + + poke_channel_deposits { + let sender_id: ParaId = 1u32.into(); + let recipient_id: ParaId = 2u32.into(); + let channel_id = HrmpChannelId {sender: sender_id, recipient: recipient_id }; + + let caller: T::AccountId = frame_benchmarking::whitelisted_caller(); + let config = Configuration::::config(); + + // make sure para is registered, and has balance to reserve. + let ed = T::Currency::minimum_balance(); + let sender_deposit: BalanceOf = config.hrmp_sender_deposit.unique_saturated_into(); + let recipient_deposit: BalanceOf = config.hrmp_recipient_deposit.unique_saturated_into(); + register_parachain_with_balance::(sender_id, ed + sender_deposit); + register_parachain_with_balance::(recipient_id, ed + recipient_deposit); + + // Our normal establishment won't actually reserve deposits, so just insert them directly. + HrmpChannels::::insert( + &channel_id, + HrmpChannel { + sender_deposit: config.hrmp_sender_deposit, + recipient_deposit: config.hrmp_recipient_deposit, + max_capacity: config.hrmp_channel_max_capacity, + max_total_size: config.hrmp_channel_max_total_size, + max_message_size: config.hrmp_channel_max_message_size, + msg_count: 0, + total_size: 0, + mqc_head: None, + }, + ); + // Actually reserve the deposits. + let _ = T::Currency::reserve(&sender_id.into_account_truncating(), sender_deposit); + let _ = T::Currency::reserve(&recipient_id.into_account_truncating(), recipient_deposit); + }: _(frame_system::RawOrigin::Signed(caller), sender_id, recipient_id) + verify { + assert_last_event::( + Event::::OpenChannelDepositsUpdated(sender_id, recipient_id).into() + ); + let channel = HrmpChannels::::get(&channel_id).unwrap(); + // Check that the deposit was updated in the channel state. + assert_eq!(channel.sender_deposit, 0); + assert_eq!(channel.recipient_deposit, 0); + // And that the funds were unreserved. + assert_eq!(T::Currency::reserved_balance(&sender_id.into_account_truncating()), 0u128.unique_saturated_into()); + assert_eq!(T::Currency::reserved_balance(&recipient_id.into_account_truncating()), 0u128.unique_saturated_into()); + } } frame_benchmarking::impl_benchmark_test_suite!( diff --git a/polkadot/runtime/parachains/src/hrmp/tests.rs b/polkadot/runtime/parachains/src/hrmp/tests.rs index 8cfaf48d10ef7f688ef37a4fb3a12182a7e06c13..236745b7cc359370f99c48129cbcc93318f43f28 100644 --- a/polkadot/runtime/parachains/src/hrmp/tests.rs +++ b/polkadot/runtime/parachains/src/hrmp/tests.rs @@ -14,13 +14,17 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +// NOTE: System chains, identified by ParaId < 2000, are treated as special in HRMP channel +// initialization. Namely, they do not require a deposit if even one ParaId is a system para. If +// both paras are system chains, then they are also configured to the system's max configuration. + use super::*; use crate::mock::{ deregister_parachain, new_test_ext, register_parachain, register_parachain_with_balance, Configuration, Hrmp, MockGenesisConfig, Paras, ParasShared, RuntimeEvent as MockEvent, RuntimeOrigin, System, Test, }; -use frame_support::assert_noop; +use frame_support::{assert_noop, assert_ok}; use primitives::BlockNumber; use std::collections::BTreeMap; @@ -133,10 +137,10 @@ fn empty_state_consistent_state() { #[test] fn open_channel_works() { - let para_a = 1.into(); - let para_a_origin: crate::Origin = 1.into(); - let para_b = 3.into(); - let para_b_origin: crate::Origin = 3.into(); + let para_a = 2001.into(); + let para_a_origin: crate::Origin = 2001.into(); + let para_b = 2003.into(); + let para_b_origin: crate::Origin = 2003.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { // We need both A & B to be registered and alive parachains. @@ -171,13 +175,16 @@ fn open_channel_works() { #[test] fn force_open_channel_works() { let para_a = 1.into(); - let para_b = 3.into(); + let para_b = 2003.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { // We need both A & B to be registered and live parachains. register_parachain(para_a); register_parachain(para_b); + let para_b_free_balance = + ::Currency::free_balance(¶_b.into_account_truncating()); + run_to_block(5, Some(vec![4, 5])); Hrmp::force_open_hrmp_channel(RuntimeOrigin::root(), para_a, para_b, 2, 8).unwrap(); Hrmp::assert_storage_consistency_exhaustive(); @@ -193,14 +200,19 @@ fn force_open_channel_works() { // Now let the session change happen and thus open the channel. run_to_block(8, Some(vec![8])); assert!(channel_exists(para_a, para_b)); + // Because para_a is a system chain, para_b's free balance should not have changed. + assert_eq!( + ::Currency::free_balance(¶_b.into_account_truncating()), + para_b_free_balance + ); }); } #[test] fn force_open_channel_works_with_existing_request() { - let para_a = 1.into(); - let para_a_origin: crate::Origin = 1.into(); - let para_b = 3.into(); + let para_a = 2001.into(); + let para_a_origin: crate::Origin = 2001.into(); + let para_b = 2003.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { // We need both A & B to be registered and live parachains. @@ -240,11 +252,127 @@ fn force_open_channel_works_with_existing_request() { }); } +#[test] +fn open_system_channel_works() { + let para_a = 1.into(); + let para_b = 3.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + // We need both A & B to be registered and live parachains. + register_parachain(para_a); + register_parachain(para_b); + + run_to_block(5, Some(vec![4, 5])); + Hrmp::establish_system_channel(RuntimeOrigin::signed(1), para_a, para_b).unwrap(); + Hrmp::assert_storage_consistency_exhaustive(); + assert!(System::events().iter().any(|record| record.event == + MockEvent::Hrmp(Event::HrmpSystemChannelOpened(para_a, para_b, 2, 8)))); + + // Advance to a block 6, but without session change. That means that the channel has + // not been created yet. + run_to_block(6, None); + assert!(!channel_exists(para_a, para_b)); + Hrmp::assert_storage_consistency_exhaustive(); + + // Now let the session change happen and thus open the channel. + run_to_block(8, Some(vec![8])); + assert!(channel_exists(para_a, para_b)); + }); +} + +#[test] +fn open_system_channel_does_not_work_for_non_system_chains() { + let para_a = 2001.into(); + let para_b = 2003.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + // We need both A & B to be registered and live parachains. + register_parachain(para_a); + register_parachain(para_b); + + run_to_block(5, Some(vec![4, 5])); + assert_noop!( + Hrmp::establish_system_channel(RuntimeOrigin::signed(1), para_a, para_b), + Error::::ChannelCreationNotAuthorized + ); + Hrmp::assert_storage_consistency_exhaustive(); + }); +} + +#[test] +fn open_system_channel_does_not_work_with_one_non_system_chain() { + let para_a = 1.into(); + let para_b = 2003.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + // We need both A & B to be registered and live parachains. + register_parachain(para_a); + register_parachain(para_b); + + run_to_block(5, Some(vec![4, 5])); + assert_noop!( + Hrmp::establish_system_channel(RuntimeOrigin::signed(1), para_a, para_b), + Error::::ChannelCreationNotAuthorized + ); + Hrmp::assert_storage_consistency_exhaustive(); + }); +} + +#[test] +fn poke_deposits_works() { + let para_a = 1.into(); + let para_b = 2001.into(); + + new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { + // We need both A & B to be registered and live parachains. + register_parachain_with_balance(para_a, 200); + register_parachain_with_balance(para_b, 200); + + let config = Configuration::config(); + let channel_id = HrmpChannelId { sender: para_a, recipient: para_b }; + + // Our normal establishment won't actually reserve deposits, so just insert them directly. + HrmpChannels::::insert( + &channel_id, + HrmpChannel { + sender_deposit: config.hrmp_sender_deposit, + recipient_deposit: config.hrmp_recipient_deposit, + max_capacity: config.hrmp_channel_max_capacity, + max_total_size: config.hrmp_channel_max_total_size, + max_message_size: config.hrmp_channel_max_message_size, + msg_count: 0, + total_size: 0, + mqc_head: None, + }, + ); + // reserve funds + assert_ok!(::Currency::reserve( + ¶_a.into_account_truncating(), + config.hrmp_sender_deposit + )); + assert_ok!(::Currency::reserve( + ¶_b.into_account_truncating(), + config.hrmp_recipient_deposit + )); + + assert_ok!(Hrmp::poke_channel_deposits(RuntimeOrigin::signed(1), para_a, para_b)); + + assert_eq!( + ::Currency::reserved_balance(¶_a.into_account_truncating()), + 0 + ); + assert_eq!( + ::Currency::reserved_balance(¶_b.into_account_truncating()), + 0 + ); + }); +} + #[test] fn close_channel_works() { - let para_a = 5.into(); - let para_b = 2.into(); - let para_b_origin: crate::Origin = 2.into(); + let para_a = 2005.into(); + let para_b = 2002.into(); + let para_b_origin: crate::Origin = 2002.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { register_parachain(para_a); @@ -275,8 +403,8 @@ fn close_channel_works() { #[test] fn send_recv_messages() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_channel_max_message_size = 20; @@ -358,8 +486,8 @@ fn hrmp_mqc_head_fixture() { #[test] fn accept_incoming_request_and_offboard() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { register_parachain(para_a); @@ -380,9 +508,9 @@ fn accept_incoming_request_and_offboard() { #[test] fn check_sent_messages() { - let para_a = 32.into(); - let para_b = 64.into(); - let para_c = 97.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); + let para_c = 2097.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { register_parachain(para_a); @@ -444,8 +572,8 @@ fn check_sent_messages() { fn verify_externally_accessible() { use primitives::{well_known_keys, AbridgedHrmpChannel}; - let para_a = 20.into(); - let para_b = 21.into(); + let para_a = 2020.into(); + let para_b = 2021.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { // Register two parachains, wait until a session change, then initiate channel open @@ -502,8 +630,8 @@ fn verify_externally_accessible() { #[test] fn charging_deposits() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); new_test_ext(GenesisConfigBuilder::default().build()).execute_with(|| { register_parachain_with_balance(para_a, 0); @@ -532,8 +660,8 @@ fn charging_deposits() { #[test] fn refund_deposit_on_normal_closure() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_sender_deposit = 20; @@ -565,8 +693,8 @@ fn refund_deposit_on_normal_closure() { #[test] fn refund_deposit_on_offboarding() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_sender_deposit = 20; @@ -605,8 +733,8 @@ fn refund_deposit_on_offboarding() { #[test] fn no_dangling_open_requests() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_sender_deposit = 20; @@ -643,8 +771,8 @@ fn no_dangling_open_requests() { #[test] fn cancel_pending_open_channel_request() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_sender_deposit = 20; @@ -675,8 +803,8 @@ fn cancel_pending_open_channel_request() { #[test] fn watermark_maxed_out_at_relay_parent() { - let para_a = 32.into(); - let para_b = 64.into(); + let para_a = 2032.into(); + let para_b = 2064.into(); let mut genesis = GenesisConfigBuilder::default(); genesis.hrmp_channel_max_message_size = 20; diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs index a9ee2d1b961281a7de4ab342c9cb150b120185db..bb16c804150dc7ca35aecaa2ee5b6c16cff98e28 100644 --- a/polkadot/runtime/parachains/src/inclusion/mod.rs +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -22,7 +22,7 @@ use crate::{ configuration::{self, HostConfiguration}, disputes, dmp, hrmp, paras, - scheduler::{self, common::CoreAssignment}, + scheduler::{self, AvailabilityTimeoutStatus}, shared::{self, AllowedRelayParentsTracker}, }; use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; @@ -46,7 +46,10 @@ use scale_info::TypeInfo; use sp_runtime::{traits::One, DispatchError, SaturatedConversion, Saturating}; #[cfg(feature = "std")] use sp_std::fmt; -use sp_std::{collections::btree_set::BTreeSet, prelude::*}; +use sp_std::{ + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + prelude::*, +}; pub use pallet::*; @@ -597,7 +600,7 @@ impl Pallet { pub(crate) fn process_candidates( allowed_relay_parents: &AllowedRelayParentsTracker>, candidates: Vec>, - scheduled: Vec>>, + scheduled: &BTreeMap, group_validators: GV, ) -> Result, DispatchError> where @@ -620,20 +623,18 @@ impl Pallet { // Do all checks before writing storage. let core_indices_and_backers = { - let mut skip = 0; let mut core_indices_and_backers = Vec::with_capacity(candidates.len()); let mut last_core = None; - let mut check_assignment_in_order = - |assignment: &CoreAssignment>| -> DispatchResult { - ensure!( - last_core.map_or(true, |core| assignment.core > core), - Error::::ScheduledOutOfOrder, - ); + let mut check_assignment_in_order = |core_idx| -> DispatchResult { + ensure!( + last_core.map_or(true, |core| core_idx > core), + Error::::ScheduledOutOfOrder, + ); - last_core = Some(assignment.core); - Ok(()) - }; + last_core = Some(core_idx); + Ok(()) + }; // We combine an outer loop over candidates with an inner loop over the scheduled, // where each iteration of the outer loop picks up at the position @@ -645,9 +646,7 @@ impl Pallet { // // In the meantime, we do certain sanity checks on the candidates and on the scheduled // list. - 'next_backed_candidate: for (candidate_idx, backed_candidate) in - candidates.iter().enumerate() - { + for (candidate_idx, backed_candidate) in candidates.iter().enumerate() { let relay_parent_hash = backed_candidate.descriptor().relay_parent; let para_id = backed_candidate.descriptor().para_id; @@ -681,108 +680,89 @@ impl Pallet { let para_id = backed_candidate.descriptor().para_id; let mut backers = bitvec::bitvec![u8, BitOrderLsb0; 0; validators.len()]; - for (i, core_assignment) in scheduled[skip..].iter().enumerate() { - check_assignment_in_order(core_assignment)?; + let core_idx = *scheduled.get(¶_id).ok_or(Error::::UnscheduledCandidate)?; + check_assignment_in_order(core_idx)?; + ensure!( + >::get(¶_id).is_none() && + >::get(¶_id).is_none(), + Error::::CandidateScheduledBeforeParaFree, + ); - if para_id == core_assignment.paras_entry.para_id() { - ensure!( - >::get(¶_id).is_none() && - >::get(¶_id).is_none(), - Error::::CandidateScheduledBeforeParaFree, - ); + // The candidate based upon relay parent `N` should be backed by a group + // assigned to core at block `N + 1`. Thus, `relay_parent_number + 1` + // will always land in the current session. + let group_idx = >::group_assigned_to_core( + core_idx, + relay_parent_number + One::one(), + ) + .ok_or_else(|| { + log::warn!( + target: LOG_TARGET, + "Failed to compute group index for candidate {}", + candidate_idx + ); + Error::::InvalidAssignment + })?; + let group_vals = + group_validators(group_idx).ok_or_else(|| Error::::InvalidGroupIndex)?; - // account for already skipped, and then skip this one. - skip = i + skip + 1; - - // The candidate based upon relay parent `N` should be backed by a group - // assigned to core at block `N + 1`. Thus, `relay_parent_number + 1` - // will always land in the current session. - let group_idx = >::group_assigned_to_core( - core_assignment.core, - relay_parent_number + One::one(), - ) - .ok_or_else(|| { - log::warn!( - target: LOG_TARGET, - "Failed to compute group index for candidate {}", - candidate_idx - ); - Error::::InvalidAssignment - })?; - let group_vals = group_validators(group_idx) - .ok_or_else(|| Error::::InvalidGroupIndex)?; - - // check the signatures in the backing and that it is a majority. - { - let maybe_amount_validated = primitives::check_candidate_backing( - &backed_candidate, - &signing_context, - group_vals.len(), - |intra_group_vi| { - group_vals - .get(intra_group_vi) - .and_then(|vi| validators.get(vi.0 as usize)) - .map(|v| v.clone()) - }, - ); - - match maybe_amount_validated { - Ok(amount_validated) => ensure!( - amount_validated >= - effective_minimum_backing_votes( - group_vals.len(), - minimum_backing_votes - ), - Error::::InsufficientBacking, + // check the signatures in the backing and that it is a majority. + { + let maybe_amount_validated = primitives::check_candidate_backing( + &backed_candidate, + &signing_context, + group_vals.len(), + |intra_group_vi| { + group_vals + .get(intra_group_vi) + .and_then(|vi| validators.get(vi.0 as usize)) + .map(|v| v.clone()) + }, + ); + + match maybe_amount_validated { + Ok(amount_validated) => ensure!( + amount_validated >= + effective_minimum_backing_votes( + group_vals.len(), + minimum_backing_votes ), - Err(()) => { - Err(Error::::InvalidBacking)?; - }, - } - - let mut backer_idx_and_attestation = - Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity( - backed_candidate.validator_indices.count_ones(), - ); - let candidate_receipt = backed_candidate.receipt(); - - for ((bit_idx, _), attestation) in backed_candidate - .validator_indices - .iter() - .enumerate() - .filter(|(_, signed)| **signed) - .zip(backed_candidate.validity_votes.iter().cloned()) - { - let val_idx = group_vals - .get(bit_idx) - .expect("this query succeeded above; qed"); - backer_idx_and_attestation.push((*val_idx, attestation)); - - backers.set(val_idx.0 as _, true); - } - candidate_receipt_with_backing_validator_indices - .push((candidate_receipt, backer_idx_and_attestation)); - } - - core_indices_and_backers.push(( - (core_assignment.core, core_assignment.paras_entry.para_id()), - backers, - group_idx, - relay_parent_number, - )); - continue 'next_backed_candidate + Error::::InsufficientBacking, + ), + Err(()) => { + Err(Error::::InvalidBacking)?; + }, } - } - // end of loop reached means that the candidate didn't appear in the non-traversed - // section of the `scheduled` slice. either it was not scheduled or didn't appear in - // `candidates` in the correct order. - ensure!(false, Error::::UnscheduledCandidate); - } + let mut backer_idx_and_attestation = + Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity( + backed_candidate.validator_indices.count_ones(), + ); + let candidate_receipt = backed_candidate.receipt(); + + for ((bit_idx, _), attestation) in backed_candidate + .validator_indices + .iter() + .enumerate() + .filter(|(_, signed)| **signed) + .zip(backed_candidate.validity_votes.iter().cloned()) + { + let val_idx = + group_vals.get(bit_idx).expect("this query succeeded above; qed"); + backer_idx_and_attestation.push((*val_idx, attestation)); + + backers.set(val_idx.0 as _, true); + } + candidate_receipt_with_backing_validator_indices + .push((candidate_receipt, backer_idx_and_attestation)); + } - // check remainder of scheduled cores, if any. - for assignment in scheduled[skip..].iter() { - check_assignment_in_order(assignment)?; + core_indices_and_backers.push(( + (core_idx, para_id), + backers, + group_idx, + relay_parent_number, + )); } core_indices_and_backers @@ -1043,13 +1023,13 @@ impl Pallet { /// /// Returns a vector of cleaned-up core IDs. pub(crate) fn collect_pending( - pred: impl Fn(CoreIndex, BlockNumberFor) -> bool, + pred: impl Fn(BlockNumberFor) -> AvailabilityTimeoutStatus>, ) -> Vec { let mut cleaned_up_ids = Vec::new(); let mut cleaned_up_cores = Vec::new(); for (para_id, pending_record) in >::iter() { - if pred(pending_record.core, pending_record.backed_in_number) { + if pred(pending_record.backed_in_number).timed_out { cleaned_up_ids.push(para_id); cleaned_up_cores.push(pending_record.core); } diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs index 70c5e959038ab3e083367f02023fb5990f301627..7677108d73decf816d62b8ea5491e058dcfac126 100644 --- a/polkadot/runtime/parachains/src/inclusion/tests.rs +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -36,7 +36,6 @@ use frame_support::assert_noop; use keyring::Sr25519Keyring; use parity_scale_codec::DecodeAll; use primitives::{ - v5::{Assignment, ParasEntry}, BlockNumber, CandidateCommitments, CandidateDescriptor, CollatorId, CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, ValidationCode, ValidatorId, ValidityAttestation, PARACHAIN_KEY_TYPE_ID, @@ -380,7 +379,9 @@ fn collect_pending_cleans_up_pending() { (chain_b, ParaKind::Parachain), (thread_a, ParaKind::Parathread), ]; - new_test_ext(genesis_config(paras)).execute_with(|| { + let mut config = genesis_config(paras); + config.configuration.config.group_rotation_frequency = 3; + new_test_ext(config).execute_with(|| { let default_candidate = TestCandidateBuilder::default().build(); >::insert( chain_a, @@ -408,7 +409,7 @@ fn collect_pending_cleans_up_pending() { descriptor: default_candidate.descriptor, availability_votes: default_availability_votes(), relay_parent_number: 0, - backed_in_number: 0, + backed_in_number: 5, backers: default_backing_bitfield(), backing_group: GroupIndex::from(1), }, @@ -422,7 +423,7 @@ fn collect_pending_cleans_up_pending() { assert!(>::get(&chain_a).is_some()); assert!(>::get(&chain_b).is_some()); - ParaInclusion::collect_pending(|core, _since| core == CoreIndex::from(0)); + ParaInclusion::collect_pending(Scheduler::availability_timeout_predicate()); assert!(>::get(&chain_a).is_none()); assert!(>::get(&chain_b).is_some()); @@ -910,23 +911,12 @@ fn candidate_checks() { ]; Scheduler::set_validator_groups(validator_groups); - let entry_ttl = 10_000; let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - paras_entry: ParasEntry::new(Assignment::new(chain_a), entry_ttl), - }; - - let chain_b_assignment = CoreAssignment { - core: CoreIndex::from(1), - paras_entry: ParasEntry::new(Assignment::new(chain_b), entry_ttl), - }; + let chain_a_assignment = (chain_a, CoreIndex::from(0)); - let thread_a_assignment = CoreAssignment { - core: CoreIndex::from(2), - paras_entry: ParasEntry::new(Assignment::new(thread_a), entry_ttl), - }; + let chain_b_assignment = (chain_b, CoreIndex::from(1)); + let thread_a_assignment = (thread_a, CoreIndex::from(2)); let allowed_relay_parents = default_allowed_relay_parent_tracker(); // unscheduled candidate. @@ -955,7 +945,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_b_assignment.clone()], + &[chain_b_assignment].into_iter().collect(), &group_validators, ), Error::::UnscheduledCandidate @@ -1010,10 +1000,10 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed_b, backed_a], - vec![chain_a_assignment.clone(), chain_b_assignment.clone()], + &[chain_a_assignment, chain_b_assignment].into_iter().collect(), &group_validators, ), - Error::::UnscheduledCandidate + Error::::ScheduledOutOfOrder ); } @@ -1043,7 +1033,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::InsufficientBacking @@ -1100,7 +1090,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed_b, backed_a], - vec![chain_a_assignment.clone(), chain_b_assignment.clone()], + &[chain_a_assignment, chain_b_assignment].into_iter().collect(), &group_validators, ), Error::::DisallowedRelayParent @@ -1138,7 +1128,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![thread_a_assignment.clone()], + &[thread_a_assignment].into_iter().collect(), &group_validators, ), Error::::NotCollatorSigned @@ -1188,7 +1178,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::CandidateScheduledBeforeParaFree @@ -1228,7 +1218,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::CandidateScheduledBeforeParaFree @@ -1272,7 +1262,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::PrematureCodeUpgrade @@ -1306,7 +1296,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Err(Error::::ValidationDataHashMismatch.into()), @@ -1341,7 +1331,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::InvalidValidationCodeHash @@ -1376,7 +1366,7 @@ fn candidate_checks() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ), Error::::ParaHeadMismatch @@ -1446,21 +1436,9 @@ fn backing_works() { let allowed_relay_parents = default_allowed_relay_parent_tracker(); - let entry_ttl = 10_000; - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - paras_entry: ParasEntry::new(Assignment::new(chain_a), entry_ttl), - }; - - let chain_b_assignment = CoreAssignment { - core: CoreIndex::from(1), - paras_entry: ParasEntry::new(Assignment::new(chain_b), entry_ttl), - }; - - let thread_a_assignment = CoreAssignment { - core: CoreIndex::from(2), - paras_entry: ParasEntry::new(Assignment::new(thread_a), entry_ttl), - }; + let chain_a_assignment = (chain_a, CoreIndex::from(0)); + let chain_b_assignment = (chain_b, CoreIndex::from(1)); + let thread_a_assignment = (thread_a, CoreIndex::from(2)); let mut candidate_a = TestCandidateBuilder { para_id: chain_a, @@ -1548,11 +1526,9 @@ fn backing_works() { } = ParaInclusion::process_candidates( &allowed_relay_parents, backed_candidates.clone(), - vec![ - chain_a_assignment.clone(), - chain_b_assignment.clone(), - thread_a_assignment.clone(), - ], + &[chain_a_assignment, chain_b_assignment, thread_a_assignment] + .into_iter() + .collect(), &group_validators, ) .expect("candidates scheduled, in order, and backed"); @@ -1738,12 +1714,7 @@ fn can_include_candidate_with_ok_code_upgrade() { Scheduler::set_validator_groups(validator_groups); let allowed_relay_parents = default_allowed_relay_parent_tracker(); - let entry_ttl = 10_000; - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - paras_entry: ParasEntry::new(Assignment::new(chain_a), entry_ttl), - }; - + let chain_a_assignment = (chain_a, CoreIndex::from(0)); let mut candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), @@ -1769,7 +1740,7 @@ fn can_include_candidate_with_ok_code_upgrade() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed_a], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ) .expect("candidates scheduled, in order, and backed"); @@ -1895,28 +1866,10 @@ fn check_allowed_relay_parents() { max_ancestry_len, ); - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_a }, - availability_timeouts: 0, - ttl: 5, - }, - }; + let chain_a_assignment = (chain_a, CoreIndex::from(0)); - let chain_b_assignment = CoreAssignment { - core: CoreIndex::from(1), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_b }, - availability_timeouts: 0, - ttl: 5, - }, - }; - - let thread_a_assignment = CoreAssignment { - core: CoreIndex::from(2), - paras_entry: ParasEntry::new(Assignment::new(thread_a), 5), - }; + let chain_b_assignment = (chain_b, CoreIndex::from(1)); + let thread_a_assignment = (thread_a, CoreIndex::from(2)); let mut candidate_a = TestCandidateBuilder { para_id: chain_a, @@ -1998,11 +1951,9 @@ fn check_allowed_relay_parents() { ParaInclusion::process_candidates( &allowed_relay_parents, backed_candidates.clone(), - vec![ - chain_a_assignment.clone(), - chain_b_assignment.clone(), - thread_a_assignment.clone(), - ], + &[chain_a_assignment, chain_b_assignment, thread_a_assignment] + .into_iter() + .collect(), &group_validators, ) .expect("candidates scheduled, in order, and backed"); @@ -2212,15 +2163,7 @@ fn para_upgrade_delay_scheduled_from_inclusion() { let allowed_relay_parents = default_allowed_relay_parent_tracker(); - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_a }, - availability_timeouts: 0, - ttl: 5, - }, - }; - + let chain_a_assignment = (chain_a, CoreIndex::from(0)); let mut candidate_a = TestCandidateBuilder { para_id: chain_a, relay_parent: System::parent_hash(), @@ -2246,7 +2189,7 @@ fn para_upgrade_delay_scheduled_from_inclusion() { ParaInclusion::process_candidates( &allowed_relay_parents, vec![backed_a], - vec![chain_a_assignment.clone()], + &[chain_a_assignment].into_iter().collect(), &group_validators, ) .expect("candidates scheduled, in order, and backed"); diff --git a/polkadot/runtime/parachains/src/mock.rs b/polkadot/runtime/parachains/src/mock.rs index f978b6c3360e49d9755267b56e34328102326caf..ded7de08e4fa95db7fd9ba7ae172514d38a86129 100644 --- a/polkadot/runtime/parachains/src/mock.rs +++ b/polkadot/runtime/parachains/src/mock.rs @@ -214,6 +214,7 @@ impl crate::paras::Config for Test { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = TestNextSessionRotation; + type OnNewHead = (); } impl crate::dmp::Config for Test {} diff --git a/polkadot/runtime/parachains/src/paras/mod.rs b/polkadot/runtime/parachains/src/paras/mod.rs index 95b89a1ca2c3a91ccde730b1254d213e6157a6b6..2f370b5bfe472a3994a4aabf6df9e8245d7b4f88 100644 --- a/polkadot/runtime/parachains/src/paras/mod.rs +++ b/polkadot/runtime/parachains/src/paras/mod.rs @@ -481,6 +481,22 @@ impl PvfCheckActiveVoteState { } } +/// Runtime hook for when a parachain head is updated. +pub trait OnNewHead { + /// Called when a parachain head is updated. + /// Returns the weight consumed by this function. + fn on_new_head(id: ParaId, head: &HeadData) -> Weight; +} + +#[impl_trait_for_tuples::impl_for_tuples(30)] +impl OnNewHead for Tuple { + fn on_new_head(id: ParaId, head: &HeadData) -> Weight { + let mut weight: Weight = Default::default(); + for_tuples!( #( weight.saturating_accrue(Tuple::on_new_head(id, head)); )* ); + weight + } +} + pub trait WeightInfo { fn force_set_current_code(c: u32) -> Weight; fn force_set_current_head(s: u32) -> Weight; @@ -575,6 +591,9 @@ pub mod pallet { /// be set to the `ParaInclusion` pallet. type QueueFootprinter: QueueFootprinter; + /// Runtime hook for when a parachain head is updated. + type OnNewHead: OnNewHead; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } @@ -1962,10 +1981,10 @@ impl Pallet { new_head: HeadData, execution_context: BlockNumberFor, ) -> Weight { - Heads::::insert(&id, new_head); + Heads::::insert(&id, &new_head); MostRecentContext::::insert(&id, execution_context); - if let Some(expected_at) = FutureCodeUpgrades::::get(&id) { + let weight = if let Some(expected_at) = FutureCodeUpgrades::::get(&id) { if expected_at <= execution_context { FutureCodeUpgrades::::remove(&id); UpgradeGoAheadSignal::::remove(&id); @@ -2005,7 +2024,9 @@ impl Pallet { // the `Abort` signal. UpgradeGoAheadSignal::::remove(&id); T::DbWeight::get().reads_writes(1, 2) - } + }; + + weight.saturating_add(T::OnNewHead::on_new_head(id, &new_head)) } /// Returns the list of PVFs (aka validation code) that require casting a vote by a validator in diff --git a/polkadot/runtime/parachains/src/paras/tests.rs b/polkadot/runtime/parachains/src/paras/tests.rs index a024525c817a81b176d160240ec4854b67990e37..c511c65d473304b230897b036c73d63daca52024 100644 --- a/polkadot/runtime/parachains/src/paras/tests.rs +++ b/polkadot/runtime/parachains/src/paras/tests.rs @@ -17,11 +17,11 @@ use super::*; use frame_support::{assert_err, assert_ok, assert_storage_noop}; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, ValidatorId, PARACHAIN_KEY_TYPE_ID}; +use primitives::{BlockNumber, PARACHAIN_KEY_TYPE_ID}; use sc_keystore::LocalKeystore; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; -use test_helpers::{dummy_head_data, dummy_validation_code}; +use test_helpers::{dummy_head_data, dummy_validation_code, validator_pubkeys}; use crate::{ configuration::HostConfiguration, @@ -39,10 +39,6 @@ static VALIDATORS: &[Sr25519Keyring] = &[ Sr25519Keyring::Ferdie, ]; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - fn sign_and_include_pvf_check_statement(stmt: PvfCheckStatement) { let validators = &[ Sr25519Keyring::Alice, diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs index 6244f44e434b03106393879b815fcbfb0a3a5e0d..8e918d35d5ff0d8af9ae77408b92fc53f9670853 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/mod.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -29,10 +29,7 @@ use crate::{ initializer, metrics::METRICS, paras, - scheduler::{ - self, - common::{CoreAssignment, FreedReason}, - }, + scheduler::{self, FreedReason}, shared, ParaId, }; use bitvec::prelude::BitVec; @@ -245,8 +242,8 @@ pub mod pallet { T: Config, { // Handle timeouts for any availability core work. - let availability_pred = >::availability_timeout_predicate(); - let freed_timeout = if let Some(pred) = availability_pred { + let freed_timeout = if >::availability_timeout_check_required() { + let pred = >::availability_timeout_predicate(); >::collect_pending(pred) } else { Vec::new() @@ -320,7 +317,7 @@ impl Pallet { /// /// When called from `create_inherent` the `context` must be set to /// `ProcessInherentDataContext::ProvideInherent` so it guarantees the invariant that inherent - /// is not overweight. + /// is not overweight. /// It is **mandatory** that calls from `enter` set `context` to /// `ProcessInherentDataContext::Enter` to ensure the weight invariant is checked. /// @@ -583,7 +580,10 @@ impl Pallet { let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); - let scheduled = >::update_claimqueue(freed, now); + >::update_claimqueue(freed, now); + let scheduled = >::scheduled_paras() + .map(|(core_idx, para_id)| (para_id, core_idx)) + .collect(); METRICS.on_candidates_processed_total(backed_candidates.len() as u64); @@ -608,7 +608,7 @@ impl Pallet { .verify_backed_candidate(&allowed_relay_parents, candidate_idx, backed_candidate) .is_err() }, - &scheduled[..], + &scheduled, ); METRICS.on_candidates_sanitized(backed_candidates.len() as u64); @@ -620,7 +620,7 @@ impl Pallet { } = >::process_candidates( &allowed_relay_parents, backed_candidates.clone(), - scheduled, + &scheduled, >::group_validators, )?; // Note which of the scheduled cores were actually occupied by a backed candidate. @@ -917,7 +917,7 @@ fn sanitize_backed_candidates< >( mut backed_candidates: Vec>, mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, - scheduled: &[CoreAssignment>], + scheduled: &BTreeMap, ) -> Vec> { // Remove any candidates that were concluded invalid. // This does not assume sorting. @@ -925,11 +925,6 @@ fn sanitize_backed_candidates< !candidate_has_concluded_invalid_dispute_or_is_invalid(candidate_idx, backed_candidate) }); - let scheduled_paras_to_core_idx = scheduled - .into_iter() - .map(|core_assignment| (core_assignment.paras_entry.para_id(), core_assignment.core)) - .collect::>(); - // Assure the backed candidate's `ParaId`'s core is free. // This holds under the assumption that `Scheduler::schedule` is called _before_. // We don't check the relay-parent because this is done in the closure when @@ -938,7 +933,7 @@ fn sanitize_backed_candidates< backed_candidates.retain(|backed_candidate| { let desc = backed_candidate.descriptor(); - scheduled_paras_to_core_idx.get(&desc.para_id).is_some() + scheduled.get(&desc.para_id).is_some() }); // Sort the `Vec` last, once there is a guarantee that these @@ -948,8 +943,7 @@ fn sanitize_backed_candidates< // but also allows this to be done in place. backed_candidates.sort_by(|x, y| { // Never panics, since we filtered all panic arguments out in the previous `fn retain`. - scheduled_paras_to_core_idx[&x.descriptor().para_id] - .cmp(&scheduled_paras_to_core_idx[&y.descriptor().para_id]) + scheduled[&x.descriptor().para_id].cmp(&scheduled[&y.descriptor().para_id]) }); backed_candidates diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs index ab515cb37565bbef2e22a436a5150d9559df26d0..7c70fcea1943415e9882bacb75e9dfc39b23b90d 100644 --- a/polkadot/runtime/parachains/src/paras_inherent/tests.rs +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -963,10 +963,7 @@ mod sanitizers { use crate::mock::Test; use keyring::Sr25519Keyring; - use primitives::{ - v5::{Assignment, ParasEntry}, - PARACHAIN_KEY_TYPE_ID, - }; + use primitives::PARACHAIN_KEY_TYPE_ID; use sc_keystore::LocalKeystore; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; @@ -1239,21 +1236,10 @@ mod sanitizers { let has_concluded_invalid = |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; - let entry_ttl = 10_000; let scheduled = (0_usize..2) .into_iter() - .map(|idx| { - let core_idx = CoreIndex::from(idx as u32); - let ca = CoreAssignment { - paras_entry: ParasEntry::new( - Assignment::new(ParaId::from(1_u32 + idx as u32)), - entry_ttl, - ), - core: core_idx, - }; - ca - }) - .collect::>(); + .map(|idx| (ParaId::from(1_u32 + idx as u32), CoreIndex::from(idx as u32))) + .collect::>(); let group_validators = |group_index: GroupIndex| { match group_index { @@ -1304,7 +1290,7 @@ mod sanitizers { // nothing is scheduled, so no paraids match, thus all backed candidates are skipped { - let scheduled = &Vec::new(); + let scheduled = &BTreeMap::new(); assert!(sanitize_backed_candidates::( backed_candidates.clone(), has_concluded_invalid, diff --git a/polkadot/runtime/parachains/src/runtime_api_impl/v5.rs b/polkadot/runtime/parachains/src/runtime_api_impl/v5.rs index bac1268f53bd34af67fa0ab98cd583114b4c64f9..46a609e0368dd65224fb24376245c4883c97e4c9 100644 --- a/polkadot/runtime/parachains/src/runtime_api_impl/v5.rs +++ b/polkadot/runtime/parachains/src/runtime_api_impl/v5.rs @@ -18,17 +18,17 @@ //! functions. use crate::{ - configuration, disputes, dmp, hrmp, inclusion, initializer, paras, paras_inherent, scheduler, + disputes, dmp, hrmp, inclusion, initializer, paras, paras_inherent, + scheduler::{self, CoreOccupied}, session_info, shared, }; use frame_system::pallet_prelude::*; use primitives::{ slashing, AuthorityDiscoveryId, CandidateEvent, CandidateHash, CommittedCandidateReceipt, - CoreIndex, CoreOccupied, CoreState, DisputeState, ExecutorParams, GroupIndex, - GroupRotationInfo, Hash, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, - OccupiedCore, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, - ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, - ValidatorId, ValidatorIndex, ValidatorSignature, + CoreIndex, CoreState, DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, + Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCore, OccupiedCoreAssumption, + PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, + ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; use sp_runtime::traits::One; use sp_std::{collections::btree_map::BTreeMap, prelude::*}; @@ -52,29 +52,15 @@ pub fn validator_groups( /// Implementation for the `availability_cores` function of the runtime API. pub fn availability_cores() -> Vec>> { let cores = >::availability_cores(); - let config = >::config(); let now = >::block_number() + One::one(); - let rotation_info = >::group_rotation_info(now); - let time_out_at = |backed_in_number, availability_period| { - let time_out_at = backed_in_number + availability_period; - - let current_window = rotation_info.last_rotation_at() + availability_period; - let next_rotation = rotation_info.next_rotation_at(); - - // If we are within `period` blocks of rotation, timeouts are being checked - // actively. We could even time out this block. - if time_out_at < current_window { - time_out_at - } else if time_out_at <= next_rotation { - // Otherwise, it will time out at the sooner of the next rotation - next_rotation - } else { - // or the scheduled time-out. This is by definition within `period` blocks - // of `next_rotation` and is thus a valid timeout block. - time_out_at - } - }; + // This explicit update is only strictly required for session boundaries: + // + // At the end of a session we clear the claim queues: Without this update call, nothing would be + // scheduled to the client. + >::update_claimqueue(Vec::new(), now); + + let time_out_for = >::availability_timeout_predicate(); let group_responsible_for = |backed_in_number, core_index| match >::group_assigned_to_core( @@ -93,7 +79,9 @@ pub fn availability_cores() -> Vec = cores + let scheduled: BTreeMap<_, _> = >::scheduled_paras().collect(); + + cores .into_iter() .enumerate() .map(|(i, core)| match core { @@ -108,7 +96,7 @@ pub fn availability_cores() -> Vec>::next_up_on_time_out(CoreIndex( i as u32, )), @@ -121,19 +109,15 @@ pub fn availability_cores() -> Vec CoreState::Free, + CoreOccupied::Free => { + if let Some(para_id) = scheduled.get(&CoreIndex(i as _)).cloned() { + CoreState::Scheduled(primitives::ScheduledCore { para_id, collator: None }) + } else { + CoreState::Free + } + }, }) - .collect(); - - // This will overwrite only `Free` cores if the scheduler module is working as intended. - for scheduled in >::scheduled_claimqueue() { - core_states[scheduled.core.0 as usize] = CoreState::Scheduled(primitives::ScheduledCore { - para_id: scheduled.paras_entry.para_id(), - collator: None, - }); - } - - core_states + .collect() } /// Returns current block number being processed and the corresponding root hash. diff --git a/polkadot/runtime/parachains/src/scheduler.rs b/polkadot/runtime/parachains/src/scheduler.rs index 577bcd153b5b032fbc447de74c93c0f4f1b156d3..60b2a9254600e56b40ca2027db06a98a7711f246 100644 --- a/polkadot/runtime/parachains/src/scheduler.rs +++ b/polkadot/runtime/parachains/src/scheduler.rs @@ -39,11 +39,11 @@ use crate::{configuration, initializer::SessionChangeNotification, paras}; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::BlockNumberFor; +pub use polkadot_core_primitives::v2::BlockNumber; use primitives::{ - v5::ParasEntry, CoreIndex, CoreOccupied, GroupIndex, GroupRotationInfo, Id as ParaId, - ScheduledCore, ValidatorIndex, + CoreIndex, GroupIndex, GroupRotationInfo, Id as ParaId, ScheduledCore, ValidatorIndex, }; -use sp_runtime::traits::{One, Saturating}; +use sp_runtime::traits::One; use sp_std::{ collections::{btree_map::BTreeMap, vec_deque::VecDeque}, prelude::*, @@ -51,7 +51,7 @@ use sp_std::{ pub mod common; -use common::{AssignmentProvider, AssignmentProviderConfig, CoreAssignment, FreedReason}; +use common::{Assignment, AssignmentProvider, AssignmentProviderConfig}; pub use pallet::*; @@ -101,6 +101,36 @@ pub mod pallet { pub(crate) type AvailabilityCores = StorageValue<_, Vec>>, ValueQuery>; + /// Representation of a core in `AvailabilityCores`. + /// + /// This is not to be confused with `CoreState` which is an enriched variant of this and exposed + /// to the node side. It also provides information about scheduled/upcoming assignments for + /// example and is computed on the fly in the `availability_cores` runtime call. + #[derive(Clone, Encode, Decode, TypeInfo, RuntimeDebug)] + #[cfg_attr(feature = "std", derive(PartialEq))] + pub enum CoreOccupied { + /// No candidate is waiting availability on this core right now (the core is not occupied). + Free, + /// A para is currently waiting for availability/inclusion on this core. + Paras(ParasEntry), + } + + impl CoreOccupied { + /// Is core free? + pub fn is_free(&self) -> bool { + matches!(self, Self::Free) + } + } + + /// Reasons a core might be freed. + #[derive(Clone, Copy)] + pub enum FreedReason { + /// The core's work concluded and the parablock assigned to it is considered available. + Concluded, + /// The core's work timed out. + TimedOut, + } + /// The block number where the session start occurred. Used to track how many group rotations /// have occurred. /// @@ -124,6 +154,68 @@ pub mod pallet { BTreeMap>>>>, ValueQuery, >; + + /// Assignments as tracked in the claim queue. + #[derive(Clone, Encode, Decode, TypeInfo, PartialEq, RuntimeDebug)] + pub struct ParasEntry { + /// The underlying `Assignment` + pub assignment: Assignment, + /// The number of times the entry has timed out in availability already. + pub availability_timeouts: u32, + /// The block height until this entry needs to be backed. + /// + /// If missed the entry will be removed from the claim queue without ever having occupied + /// the core. + pub ttl: N, + } + + impl ParasEntry { + /// Return `Id` from the underlying `Assignment`. + pub fn para_id(&self) -> ParaId { + self.assignment.para_id + } + + /// Create a new `ParasEntry`. + pub fn new(assignment: Assignment, now: N) -> Self { + ParasEntry { assignment, availability_timeouts: 0, ttl: now } + } + } + + /// How a core is mapped to a backing group and a `ParaId` + #[derive(Clone, Encode, Decode, PartialEq, TypeInfo)] + #[cfg_attr(feature = "std", derive(Debug))] + pub struct CoreAssignment { + /// The core that is assigned. + pub core: CoreIndex, + /// The para id and accompanying information needed to collate and back a parablock. + pub paras_entry: ParasEntry, + } + + impl CoreAssignment { + /// Returns the [`ParaId`] of the assignment. + pub fn para_id(&self) -> ParaId { + self.paras_entry.para_id() + } + + /// Returns the inner [`ParasEntry`] of the assignment. + pub fn to_paras_entry(self) -> ParasEntry { + self.paras_entry + } + } + + /// Availability timeout status of a core. + pub(crate) struct AvailabilityTimeoutStatus { + /// Is the core already timed out? + /// + /// If this is true the core will be freed at this block. + pub timed_out: bool, + + /// When does this core timeout. + /// + /// The block number the core times out. If `timed_out` is true, this will correspond to + /// now (current block number). + pub live_until: BlockNumber, + } } type PositionInClaimqueue = u32; @@ -368,50 +460,47 @@ impl Pallet { Some(GroupIndex(group_idx as u32)) } - /// Returns an optional predicate that should be used for timing out occupied cores. - /// - /// If `None`, no timing-out should be done. The predicate accepts the index of the core, and - /// the block number since which it has been occupied, and the respective parachain timeouts, - /// i.e. only within `config.paras_availability_period` of the last rotation would this return - /// `Some`, unless there are no rotations. + /// Returns a predicate that should be used for timing out occupied cores. /// - /// The timeout used to depend, but does not depend any more on group rotations. First of all - /// it only matters if a para got another chance (a retry). If there is a retry and it happens - /// still within the same group rotation a censoring backing group would need to censor again - /// and lose out again on backing rewards. This is bad for the censoring backing group, it does - /// not matter for the parachain as long as it is retried often enough (so it eventually gets a - /// try on another backing group) - the effect is similar to having a prolonged timeout. It - /// should also be noted that for both malicious and offline backing groups it is actually more - /// realistic that the candidate will not be backed to begin with, instead of getting backed - /// and then not made available. + /// This only ever times out cores that have been occupied across a group rotation boundary. pub(crate) fn availability_timeout_predicate( - ) -> Option) -> bool> { - let now = >::block_number(); + ) -> impl Fn(BlockNumberFor) -> AvailabilityTimeoutStatus> { let config = >::config(); - let session_start = >::get(); + let now = >::block_number(); + let rotation_info = Self::group_rotation_info(now); - let blocks_since_session_start = now.saturating_sub(session_start); - let blocks_since_last_rotation = - blocks_since_session_start % config.group_rotation_frequency.max(1u8.into()); + let next_rotation = rotation_info.next_rotation_at(); - if blocks_since_last_rotation >= config.paras_availability_period { - None - } else { - Some(|core_index: CoreIndex, pending_since| { - let availability_cores = AvailabilityCores::::get(); - let AssignmentProviderConfig { availability_period, .. } = - T::AssignmentProvider::get_provider_config(core_index); - let now = >::block_number(); - match availability_cores.get(core_index.0 as usize) { - None => true, // out-of-bounds, doesn't really matter what is returned. - Some(CoreOccupied::Free) => true, // core free, still doesn't matter. - Some(CoreOccupied::Paras(_)) => - now.saturating_sub(pending_since) >= availability_period, - } - }) + let times_out = Self::availability_timeout_check_required(); + + move |pending_since| { + let time_out_at = if times_out { + // We are at the beginning of the rotation, here availability period is relevant. + // Note: blocks backed in this rotation will never time out here as backed_in + + // config.paras_availability_period will always be > now for these blocks, as + // otherwise above condition would not be true. + pending_since + config.paras_availability_period + } else { + next_rotation + config.paras_availability_period + }; + + AvailabilityTimeoutStatus { timed_out: time_out_at <= now, live_until: time_out_at } } } + /// Is evaluation of `availability_timeout_predicate` necessary at the current block? + /// + /// This can be used to avoid calling `availability_timeout_predicate` for each core in case + /// this function returns false. + pub(crate) fn availability_timeout_check_required() -> bool { + let config = >::config(); + let now = >::block_number() + One::one(); + let rotation_info = Self::group_rotation_info(now); + + let current_window = rotation_info.last_rotation_at() + config.paras_availability_period; + now < current_window + } + /// Returns a helper for determining group rotation. pub(crate) fn group_rotation_info( now: BlockNumberFor, @@ -508,7 +597,7 @@ impl Pallet { pub(crate) fn update_claimqueue( just_freed_cores: impl IntoIterator, now: BlockNumberFor, - ) -> Vec>> { + ) { Self::move_claimqueue_forward(); Self::free_cores_and_fill_claimqueue(just_freed_cores, now) } @@ -534,61 +623,58 @@ impl Pallet { fn free_cores_and_fill_claimqueue( just_freed_cores: impl IntoIterator, now: BlockNumberFor, - ) -> Vec>> { + ) { let (mut concluded_paras, mut timedout_paras) = Self::free_cores(just_freed_cores); // This can only happen on new sessions at which we move all assignments back to the // provider. Hence, there's nothing we need to do here. if ValidatorGroups::::get().is_empty() { - vec![] - } else { - let n_lookahead = Self::claimqueue_lookahead(); - let n_session_cores = T::AssignmentProvider::session_core_count(); - let cq = ClaimQueue::::get(); - let ttl = >::config().on_demand_ttl; - - for core_idx in 0..n_session_cores { - let core_idx = CoreIndex::from(core_idx); - - // add previously timedout paras back into the queue - if let Some(mut entry) = timedout_paras.remove(&core_idx) { - let AssignmentProviderConfig { max_availability_timeouts, .. } = - T::AssignmentProvider::get_provider_config(core_idx); - if entry.availability_timeouts < max_availability_timeouts { - // Increment the timeout counter. - entry.availability_timeouts += 1; - // Reset the ttl so that a timed out assignment. - entry.ttl = now + ttl; - Self::add_to_claimqueue(core_idx, entry); - // The claim has been added back into the claimqueue. - // Do not pop another assignment for the core. - continue - } else { - // Consider timed out assignments for on demand parachains as concluded for - // the assignment provider - let ret = concluded_paras.insert(core_idx, entry.para_id()); - debug_assert!(ret.is_none()); - } + return + } + let n_lookahead = Self::claimqueue_lookahead(); + let n_session_cores = T::AssignmentProvider::session_core_count(); + let cq = ClaimQueue::::get(); + let ttl = >::config().on_demand_ttl; + + for core_idx in 0..n_session_cores { + let core_idx = CoreIndex::from(core_idx); + + // add previously timedout paras back into the queue + if let Some(mut entry) = timedout_paras.remove(&core_idx) { + let AssignmentProviderConfig { max_availability_timeouts, .. } = + T::AssignmentProvider::get_provider_config(core_idx); + if entry.availability_timeouts < max_availability_timeouts { + // Increment the timeout counter. + entry.availability_timeouts += 1; + // Reset the ttl so that a timed out assignment. + entry.ttl = now + ttl; + Self::add_to_claimqueue(core_idx, entry); + // The claim has been added back into the claimqueue. + // Do not pop another assignment for the core. + continue + } else { + // Consider timed out assignments for on demand parachains as concluded for + // the assignment provider + let ret = concluded_paras.insert(core_idx, entry.para_id()); + debug_assert!(ret.is_none()); } + } - // We consider occupied cores to be part of the claimqueue - let n_lookahead_used = cq.get(&core_idx).map_or(0, |v| v.len() as u32) + - if Self::is_core_occupied(core_idx) { 1 } else { 0 }; - for _ in n_lookahead_used..n_lookahead { - let concluded_para = concluded_paras.remove(&core_idx); - if let Some(assignment) = - T::AssignmentProvider::pop_assignment_for_core(core_idx, concluded_para) - { - Self::add_to_claimqueue(core_idx, ParasEntry::new(assignment, now + ttl)); - } + // We consider occupied cores to be part of the claimqueue + let n_lookahead_used = cq.get(&core_idx).map_or(0, |v| v.len() as u32) + + if Self::is_core_occupied(core_idx) { 1 } else { 0 }; + for _ in n_lookahead_used..n_lookahead { + let concluded_para = concluded_paras.remove(&core_idx); + if let Some(assignment) = + T::AssignmentProvider::pop_assignment_for_core(core_idx, concluded_para) + { + Self::add_to_claimqueue(core_idx, ParasEntry::new(assignment, now + ttl)); } } - - debug_assert!(timedout_paras.is_empty()); - debug_assert!(concluded_paras.is_empty()); - - Self::scheduled_claimqueue() } + + debug_assert!(timedout_paras.is_empty()); + debug_assert!(concluded_paras.is_empty()); } fn is_core_occupied(core_idx: CoreIndex) -> bool { @@ -623,29 +709,22 @@ impl Pallet { .ok_or("remove returned None")? .ok_or("Element in Claimqueue was None.")?; - // Since the core is now occupied, the next entry in the claimqueue in order to achieve - // 12 second block times needs to be None - if core_claims.front() != Some(&None) { - core_claims.push_front(None); - } Ok((pos as u32, pe)) }) } - // TODO: Temporary to imitate the old schedule() call. Will be adjusted when we make the - // scheduler AB ready - pub(crate) fn scheduled_claimqueue() -> Vec>> { + /// Paras scheduled next in the claim queue. + pub(crate) fn scheduled_paras() -> impl Iterator { + Self::scheduled_entries().map(|(core_idx, e)| (core_idx, e.assignment.para_id)) + } + + /// Internal access to entries at the top of the claim queue. + fn scheduled_entries() -> impl Iterator>)> { let claimqueue = ClaimQueue::::get(); claimqueue .into_iter() - .flat_map(|(core_idx, v)| { - v.front() - .cloned() - .flatten() - .map(|pe| CoreAssignment { core: core_idx, paras_entry: pe }) - }) - .collect() + .filter_map(|(core_idx, v)| v.front().cloned().flatten().map(|e| (core_idx, e))) } #[cfg(any(feature = "runtime-benchmarks", test))] diff --git a/polkadot/runtime/parachains/src/scheduler/common.rs b/polkadot/runtime/parachains/src/scheduler/common.rs index 0e8e8338b17b71c01e3c131e9ea5a7483f666571..316e8e3b760cc6a73c022f693e12ca537bf3443b 100644 --- a/polkadot/runtime/parachains/src/scheduler/common.rs +++ b/polkadot/runtime/parachains/src/scheduler/common.rs @@ -17,10 +17,7 @@ //! Common traits and types used by the scheduler and assignment providers. use frame_support::pallet_prelude::*; -use primitives::{ - v5::{Assignment, ParasEntry}, - CoreIndex, Id as ParaId, -}; +use primitives::{CoreIndex, Id as ParaId}; use scale_info::TypeInfo; use sp_std::prelude::*; @@ -28,21 +25,22 @@ use sp_std::prelude::*; #[allow(unused)] use crate::configuration::HostConfiguration; -/// Reasons a core might be freed -#[derive(Clone, Copy)] -pub enum FreedReason { - /// The core's work concluded and the parablock assigned to it is considered available. - Concluded, - /// The core's work timed out. - TimedOut, +/// An assignment for a parachain scheduled to be backed and included in a relay chain block. +#[derive(Clone, Encode, Decode, PartialEq, TypeInfo, RuntimeDebug)] +pub struct Assignment { + /// Assignment's ParaId + pub para_id: ParaId, +} + +impl Assignment { + /// Create a new `Assignment`. + pub fn new(para_id: ParaId) -> Self { + Self { para_id } + } } /// A set of variables required by the scheduler in order to operate. pub struct AssignmentProviderConfig { - /// The availability period specified by the implementation. - /// See [`HostConfiguration::paras_availability_period`] for more information. - pub availability_period: BlockNumber, - /// How many times a collation can time out on availability. /// Zero timeouts still means that a collation can be provided as per the slot auction /// assignment provider. @@ -72,25 +70,3 @@ pub trait AssignmentProvider { /// Returns a set of variables needed by the scheduler fn get_provider_config(core_idx: CoreIndex) -> AssignmentProviderConfig; } - -/// How a core is mapped to a backing group and a `ParaId` -#[derive(Clone, Encode, Decode, PartialEq, TypeInfo)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct CoreAssignment { - /// The core that is assigned. - pub core: CoreIndex, - /// The para id and accompanying information needed to collate and back a parablock. - pub paras_entry: ParasEntry, -} - -impl CoreAssignment { - /// Returns the [`ParaId`] of the assignment. - pub fn para_id(&self) -> ParaId { - self.paras_entry.para_id() - } - - /// Returns the inner [`ParasEntry`] of the assignment. - pub fn to_paras_entry(self) -> ParasEntry { - self.paras_entry - } -} diff --git a/polkadot/runtime/parachains/src/scheduler/migration.rs b/polkadot/runtime/parachains/src/scheduler/migration.rs index 32ac9deaf68f24149382bb9acaadc0368687d36b..accff7016ed1e37e585b8433b5cdfd37378e4fe3 100644 --- a/polkadot/runtime/parachains/src/scheduler/migration.rs +++ b/polkadot/runtime/parachains/src/scheduler/migration.rs @@ -20,7 +20,6 @@ use super::*; use frame_support::{ pallet_prelude::ValueQuery, storage_alias, traits::OnRuntimeUpgrade, weights::Weight, }; -use primitives::vstaging::Assignment; mod v0 { use super::*; diff --git a/polkadot/runtime/parachains/src/scheduler/tests.rs b/polkadot/runtime/parachains/src/scheduler/tests.rs index e203531ca49d2f985a32f32421db2ac49e2ac1cf..108f365d6b5c39567b57dc21243a30d0548a89bf 100644 --- a/polkadot/runtime/parachains/src/scheduler/tests.rs +++ b/polkadot/runtime/parachains/src/scheduler/tests.rs @@ -18,7 +18,7 @@ use super::*; use frame_support::assert_ok; use keyring::Sr25519Keyring; -use primitives::{v5::Assignment, BlockNumber, SessionIndex, ValidationCode, ValidatorId}; +use primitives::{BlockNumber, SessionIndex, ValidationCode, ValidatorId}; use sp_std::collections::{btree_map::BTreeMap, btree_set::BTreeSet}; use crate::{ @@ -427,33 +427,27 @@ fn fill_claimqueue_fills() { { assert_eq!(Scheduler::claimqueue_len(), 2 * lookahead); - let scheduled = Scheduler::scheduled_claimqueue(); + let scheduled: BTreeMap<_, _> = Scheduler::scheduled_entries().collect(); // Cannot assert on indices anymore as they depend on the assignment providers assert!(claimqueue_contains_para_ids::(vec![chain_a, chain_b])); assert_eq!( - scheduled[0], - CoreAssignment { - core: CoreIndex(0), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_a }, - availability_timeouts: 0, - ttl: 6 - }, - } + scheduled.get(&CoreIndex(0)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: chain_a }, + availability_timeouts: 0, + ttl: 6 + }, ); assert_eq!( - scheduled[1], - CoreAssignment { - core: CoreIndex(1), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_b }, - availability_timeouts: 0, - ttl: 6 - }, - } + scheduled.get(&CoreIndex(1)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: chain_b }, + availability_timeouts: 0, + ttl: 6 + }, ); } @@ -481,42 +475,33 @@ fn fill_claimqueue_fills() { { assert_eq!(Scheduler::claimqueue_len(), 5); - let scheduled = Scheduler::scheduled_claimqueue(); + let scheduled: BTreeMap<_, _> = Scheduler::scheduled_entries().collect(); assert_eq!( - scheduled[0], - CoreAssignment { - core: CoreIndex(0), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_a }, - availability_timeouts: 0, - ttl: 6 - }, - } + scheduled.get(&CoreIndex(0)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: chain_a }, + availability_timeouts: 0, + ttl: 6 + }, ); assert_eq!( - scheduled[1], - CoreAssignment { - core: CoreIndex(1), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_b }, - availability_timeouts: 0, - ttl: 6 - }, - } + scheduled.get(&CoreIndex(1)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: chain_b }, + availability_timeouts: 0, + ttl: 6 + }, ); // Was added a block later, note the TTL. assert_eq!( - scheduled[2], - CoreAssignment { - core: CoreIndex(2), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_a }, - availability_timeouts: 0, - ttl: 7 - }, - } + scheduled.get(&CoreIndex(2)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_a }, + availability_timeouts: 0, + ttl: 7 + }, ); // Sits on the same core as `thread_a` assert_eq!( @@ -528,15 +513,12 @@ fn fill_claimqueue_fills() { }) ); assert_eq!( - scheduled[3], - CoreAssignment { - core: CoreIndex(3), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_c }, - availability_timeouts: 0, - ttl: 7 - }, - } + scheduled.get(&CoreIndex(3)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_c }, + availability_timeouts: 0, + ttl: 7 + }, ); } }); @@ -608,7 +590,7 @@ fn schedule_schedules_including_just_freed() { let mut now = 2; run_to_block(now, |_| None); - assert_eq!(Scheduler::scheduled_claimqueue().len(), 4); + assert_eq!(Scheduler::scheduled_paras().collect::>().len(), 4); // cores 0, 1, 2, and 3 should be occupied. mark them as such. let mut occupied_map: BTreeMap = BTreeMap::new(); @@ -630,7 +612,7 @@ fn schedule_schedules_including_just_freed() { // core 4 is free assert!(cores[4] == CoreOccupied::Free); - assert!(Scheduler::scheduled_claimqueue().is_empty()); + assert!(Scheduler::scheduled_paras().collect::>().is_empty()); // All core index entries in the claimqueue should have `None` in them. Scheduler::claimqueue().iter().for_each(|(_core_idx, core_queue)| { @@ -657,21 +639,18 @@ fn schedule_schedules_including_just_freed() { run_to_block(now, |_| None); { - let scheduled = Scheduler::scheduled_claimqueue(); + let scheduled: BTreeMap<_, _> = Scheduler::scheduled_entries().collect(); // cores 0 and 1 are occupied by lease holding parachains. cores 2 and 3 are occupied by // on-demand parachain claims. core 4 was free. assert_eq!(scheduled.len(), 1); assert_eq!( - scheduled[0], - CoreAssignment { - core: CoreIndex(4), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_b }, - availability_timeouts: 0, - ttl: 8 - }, - } + scheduled.get(&CoreIndex(4)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_b }, + availability_timeouts: 0, + ttl: 8 + }, ); } @@ -686,54 +665,42 @@ fn schedule_schedules_including_just_freed() { Scheduler::update_claimqueue(just_updated, now); { - let scheduled = Scheduler::scheduled_claimqueue(); + let scheduled: BTreeMap<_, _> = Scheduler::scheduled_entries().collect(); // 1 thing scheduled before, + 3 cores freed. assert_eq!(scheduled.len(), 4); assert_eq!( - scheduled[0], - CoreAssignment { - core: CoreIndex(0), - paras_entry: ParasEntry { - assignment: Assignment { para_id: chain_a }, - availability_timeouts: 0, - ttl: 8 - }, - } + scheduled.get(&CoreIndex(0)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: chain_a }, + availability_timeouts: 0, + ttl: 8 + }, ); assert_eq!( - scheduled[1], - CoreAssignment { - core: CoreIndex(2), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_d }, - availability_timeouts: 0, - ttl: 8 - }, - } + scheduled.get(&CoreIndex(2)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_d }, + availability_timeouts: 0, + ttl: 8 + }, ); // Although C was descheduled, the core `4` was occupied so C goes back to the queue. assert_eq!( - scheduled[2], - CoreAssignment { - core: CoreIndex(3), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_c }, - availability_timeouts: 1, - ttl: 8 - }, - } + scheduled.get(&CoreIndex(3)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_c }, + availability_timeouts: 1, + ttl: 8 + }, ); assert_eq!( - scheduled[3], - CoreAssignment { - core: CoreIndex(4), - paras_entry: ParasEntry { - assignment: Assignment { para_id: thread_b }, - availability_timeouts: 0, - ttl: 8 - }, - } + scheduled.get(&CoreIndex(4)).unwrap(), + &ParasEntry { + assignment: Assignment { para_id: thread_b }, + availability_timeouts: 0, + ttl: 8 + }, ); // The only assignment yet to be popped on to the claim queue is `thread_e`. @@ -900,14 +867,14 @@ fn schedule_rotates_groups() { run_to_block(now, |_| None); let assert_groups_rotated = |rotations: u32, now: &BlockNumberFor| { - let scheduled = Scheduler::scheduled_claimqueue(); + let scheduled: BTreeMap<_, _> = Scheduler::scheduled_paras().collect(); assert_eq!(scheduled.len(), 2); assert_eq!( - Scheduler::group_assigned_to_core(scheduled[0].core, *now).unwrap(), + Scheduler::group_assigned_to_core(CoreIndex(0), *now).unwrap(), GroupIndex((0u32 + rotations) % on_demand_cores) ); assert_eq!( - Scheduler::group_assigned_to_core(scheduled[1].core, *now).unwrap(), + Scheduler::group_assigned_to_core(CoreIndex(1), *now).unwrap(), GroupIndex((1u32 + rotations) % on_demand_cores) ); }; @@ -999,7 +966,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { ] .into_iter() .collect(); - let core_assignments = Scheduler::update_claimqueue(just_updated, now); + Scheduler::update_claimqueue(just_updated, now); // ParaId a exists in the claim queue until max_retries is reached. if n < max_retries + now { @@ -1008,13 +975,9 @@ fn on_demand_claims_are_pruned_after_timing_out() { assert!(!claimqueue_contains_para_ids::(vec![thread_a])); } - // Occupy the cores based on the output of update_claimqueue. - Scheduler::occupied( - core_assignments - .iter() - .map(|core_assignment| (core_assignment.core, core_assignment.para_id())) - .collect(), - ); + let core_assignments = Scheduler::scheduled_paras().collect(); + // Occupy the cores based on the result of update_claimqueue. + Scheduler::occupied(core_assignments); } // ParaId a does not exist in the claimqueue/availability_cores after @@ -1054,7 +1017,7 @@ fn on_demand_claims_are_pruned_after_timing_out() { } } - let core_assignments = Scheduler::update_claimqueue(just_updated, now); + Scheduler::update_claimqueue(just_updated, now); // ParaId a exists in the claim queue until groups are rotated. if n < 31 { @@ -1063,13 +1026,9 @@ fn on_demand_claims_are_pruned_after_timing_out() { assert!(!claimqueue_contains_para_ids::(vec![thread_a])); } - // Occupy the cores based on the output of update_claimqueue. - Scheduler::occupied( - core_assignments - .iter() - .map(|core_assignment| (core_assignment.core, core_assignment.para_id())) - .collect(), - ); + let core_assignments = Scheduler::scheduled_paras().collect(); + // Occupy the cores based on the result of update_claimqueue. + Scheduler::occupied(core_assignments); } // ParaId a does not exist in the claimqueue/availability_cores after @@ -1124,33 +1083,25 @@ fn availability_predicate_works() { run_to_block(1 + paras_availability_period, |_| None); - assert!(Scheduler::availability_timeout_predicate().is_none()); + assert!(!Scheduler::availability_timeout_check_required()); run_to_block(1 + group_rotation_frequency, |_| None); { - let pred = Scheduler::availability_timeout_predicate() - .expect("predicate exists recently after rotation"); - let now = System::block_number(); - let would_be_timed_out = now - paras_availability_period; - for i in 0..AvailabilityCores::::get().len() { - // returns true for unoccupied cores. - // And can time out paras at this stage. - assert!(pred(CoreIndex(i as u32), would_be_timed_out)); - } + assert!(Scheduler::availability_timeout_check_required()); + let pred = Scheduler::availability_timeout_predicate(); + let last_rotation = Scheduler::group_rotation_info(now).last_rotation_at(); - assert!(!pred(CoreIndex(0), now)); - assert!(!pred(CoreIndex(1), now)); - assert!(pred(CoreIndex(2), now)); + let would_be_timed_out = now - paras_availability_period; + let should_not_be_timed_out = last_rotation; - // check the tight bound. - assert!(pred(CoreIndex(0), now - paras_availability_period)); - assert!(pred(CoreIndex(1), now - paras_availability_period)); + assert!(pred(would_be_timed_out).timed_out); + assert!(!pred(should_not_be_timed_out).timed_out); + assert!(!pred(now).timed_out); // check the threshold is exact. - assert!(!pred(CoreIndex(0), now - paras_availability_period + 1)); - assert!(!pred(CoreIndex(1), now - paras_availability_period + 1)); + assert!(!pred(would_be_timed_out + 1).timed_out); } run_to_block(1 + group_rotation_frequency + paras_availability_period, |_| None); diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index 91891ba8d75bb993474e06aafcc4b452a62b0fe7..ae12b4b3fc16f0d2c60c787aab8d8ce360391a8f 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -22,10 +22,7 @@ use crate::{ use assert_matches::assert_matches; use keyring::Sr25519Keyring; use primitives::Hash; - -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} +use test_helpers::validator_pubkeys; #[test] fn tracker_earliest_block_number() { diff --git a/polkadot/runtime/polkadot/Cargo.toml b/polkadot/runtime/polkadot/Cargo.toml index d185677ab8d27f142834cad31ef21d6f8fb73e9c..a659a4553220cd782d62b1b4db2b58030294077a 100644 --- a/polkadot/runtime/polkadot/Cargo.toml +++ b/polkadot/runtime/polkadot/Cargo.toml @@ -82,7 +82,7 @@ pallet-whitelist = { path = "../../../substrate/frame/whitelist", default-featur pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } frame-election-provider-support = { path = "../../../substrate/frame/election-provider-support", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -107,7 +107,7 @@ hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } -serde_json = "1.0.96" +serde_json = "1.0.107" separator = "0.4.1" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/runtime/polkadot/README.adoc b/polkadot/runtime/polkadot/README.adoc deleted file mode 100644 index 33373310819f975270260a27e94907dcca9498bb..0000000000000000000000000000000000000000 --- a/polkadot/runtime/polkadot/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot Runtime - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/runtime/polkadot/src/lib.rs b/polkadot/runtime/polkadot/src/lib.rs index 2efd329eea0ccc4d3c27b48df4246357c316f8e4..5956b0e155bb5b6f3a16cb3935b769625fbdb271 100644 --- a/polkadot/runtime/polkadot/src/lib.rs +++ b/polkadot/runtime/polkadot/src/lib.rs @@ -47,8 +47,9 @@ use frame_election_provider_support::{ use frame_support::{ construct_runtime, parameter_types, traits::{ - ConstU32, EitherOf, EitherOfDiverse, InstanceFilter, KeyOwnerProofSystem, PrivilegeCmp, - ProcessMessage, ProcessMessageError, WithdrawReasons, + fungible::HoldConsideration, ConstU32, Contains, EitherOf, EitherOfDiverse, EverythingBut, + InstanceFilter, KeyOwnerProofSystem, LinearStoragePrice, PrivilegeCmp, ProcessMessage, + ProcessMessageError, WithdrawReasons, }, weights::{ConstantMultiplier, WeightMeter}, PalletId, @@ -89,7 +90,7 @@ use xcm::latest::Junction; pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; -pub use pallet_election_provider_multi_phase::Call as EPMCall; +pub use pallet_election_provider_multi_phase::{Call as EPMCall, GeometricDepositBase}; #[cfg(feature = "std")] pub use pallet_staking::StakerStatus; use pallet_staking::UseValidatorsMap; @@ -148,13 +149,24 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } +/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, +/// locking the state of the pallet and preventing further updates to identities and sub-identities. +/// The locked state will be the genesis state of a new system chain and then removed from the Relay +/// Chain. +pub struct IdentityCalls; +impl Contains for IdentityCalls { + fn contains(c: &RuntimeCall) -> bool { + matches!(c, RuntimeCall::Identity(_)) + } +} + parameter_types! { pub const Version: RuntimeVersion = VERSION; pub const SS58Prefix: u8 = 0; } impl frame_system::Config for Runtime { - type BaseCallFilter = frame_support::traits::Everything; + type BaseCallFilter = EverythingBut; type BlockWeights = BlockWeights; type BlockLength = BlockLength; type RuntimeOrigin = RuntimeOrigin; @@ -212,7 +224,8 @@ impl pallet_scheduler::Config for Runtime { type MaximumWeight = MaximumSchedulerWeight; // The goal of having ScheduleOrigin include AuctionAdmin is to allow the auctions track of // OpenGov to schedule periodic auctions. - type ScheduleOrigin = EitherOf, AuctionAdmin>; + // Also allow Treasurer to schedule recurring payments. + type ScheduleOrigin = EitherOf, AuctionAdmin>, Treasurer>; type MaxScheduledPerBlock = MaxScheduledPerBlock; type WeightInfo = weights::pallet_scheduler::WeightInfo; type OriginPrivilegeCmp = OriginPrivilegeCmp; @@ -220,9 +233,9 @@ impl pallet_scheduler::Config for Runtime { } parameter_types! { - pub const PreimageMaxSize: u32 = 4096 * 1024; pub const PreimageBaseDeposit: Balance = deposit(2, 64); pub const PreimageByteDeposit: Balance = deposit(0, 1); + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -230,8 +243,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } parameter_types! { @@ -296,7 +313,7 @@ impl pallet_balances::Config for Runtime { type WeightInfo = weights::pallet_balances::WeightInfo; type RuntimeHoldReason = RuntimeHoldReason; type FreezeIdentifier = (); - type MaxHolds = ConstU32<0>; + type MaxHolds = ConstU32<1>; type MaxFreezes = ConstU32<0>; } @@ -376,6 +393,8 @@ parameter_types! { // signed config pub const SignedMaxSubmissions: u32 = 16; pub const SignedMaxRefunds: u32 = 16 / 4; + pub const SignedFixedDeposit: Balance = deposit(2, 0); + pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); // 40 DOTs fixed deposit.. pub const SignedDepositBase: Balance = deposit(2, 0); // 0.01 DOT per KB of solution data. @@ -450,7 +469,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SignedMaxSubmissions = SignedMaxSubmissions; type SignedMaxRefunds = SignedMaxRefunds; type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; + type SignedDepositBase = + GeometricDepositBase; type SignedDepositByte = SignedDepositByte; type SignedDepositWeight = (); type SignedMaxWeight = @@ -546,7 +566,7 @@ impl pallet_staking::EraPayout for EraPayout { // all para-ids that are not active. let auctioned_slots = Paras::parachains() .into_iter() - // all active para-ids that do not belong to a system or common good chain is the number + // all active para-ids that do not belong to a system chain is the number // of parachains that we should take into account for inflation. .filter(|i| *i >= LOWEST_PUBLIC_ID) .count() as u64; @@ -1090,6 +1110,7 @@ impl parachains_paras::Config for Runtime { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = Babe; + type OnNewHead = Registrar; } parameter_types! { @@ -1324,7 +1345,7 @@ construct_runtime! { // Basic stuff; balances is uncallable initially. System: frame_system::{Pallet, Call, Storage, Config, Event} = 0, Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event} = 1, - Preimage: pallet_preimage::{Pallet, Call, Storage, Event} = 10, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 10, // Babe must be before session. Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned} = 2, @@ -1502,6 +1523,19 @@ pub mod migrations { type PalletName = TipsPalletName; } + pub struct ParachainsToUnlock; + impl Contains for ParachainsToUnlock { + fn contains(id: &ParaId) -> bool { + let id: u32 = (*id).into(); + // polkadot parachains/parathreads that are locked and never produced block + match id { + 2003 | 2015 | 2017 | 2018 | 2025 | 2028 | 2036 | 2038 | 2053 | 2055 | 2090 | + 2097 | 2106 | 3336 | 3338 | 3342 => true, + _ => false, + } + } + } + /// Unreleased migrations. Add new ones here: pub type Unreleased = ( pallet_im_online::migration::v1::Migration, @@ -1524,6 +1558,8 @@ pub mod migrations { frame_support::migrations::RemovePallet::DbWeight>, parachains_configuration::migration::v9::MigrateToV9, + // Migrate parachain info format + paras_registrar::migration::VersionCheckedMigrateToV1, ); } @@ -2330,7 +2366,7 @@ mod test_fees { fn signed_deposit_is_sensible() { // ensure this number does not change, or that it is checked after each change. // a 1 MB solution should take (40 + 10) DOTs of deposit. - let deposit = SignedDepositBase::get() + (SignedDepositByte::get() * 1024 * 1024); + let deposit = SignedFixedDeposit::get() + (SignedDepositByte::get() * 1024 * 1024); assert_eq_error_rate!(deposit, 50 * DOLLARS, DOLLARS); } } diff --git a/polkadot/runtime/polkadot/src/weights/pallet_preimage.rs b/polkadot/runtime/polkadot/src/weights/pallet_preimage.rs index 91605e072f0930a9fc47acd0af3856b2128459fb..a283f09eae5f8224189bee22e461ee0d3eec6ab5 100644 --- a/polkadot/runtime/polkadot/src/weights/pallet_preimage.rs +++ b/polkadot/runtime/polkadot/src/weights/pallet_preimage.rs @@ -50,6 +50,21 @@ use core::marker::PhantomData; /// Weight functions for `pallet_preimage`. pub struct WeightInfo(PhantomData); impl pallet_preimage::WeightInfo for WeightInfo { + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_000_000 picoseconds. + Weight::from_parts(2_000_000, 3593) + // Standard Error: 13_720 + .saturating_add(Weight::from_parts(17_309_199, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) + } + /// Storage: Preimage StatusFor (r:1 w:1) /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) /// Storage: Preimage PreimageFor (r:0 w:1) diff --git a/polkadot/runtime/polkadot/src/weights/runtime_parachains_hrmp.rs b/polkadot/runtime/polkadot/src/weights/runtime_parachains_hrmp.rs index 82d8c30bacad485ba137614a3417547723f27bf3..73a08d33eed8e9f834a51d7c79461bd8d56d7111 100644 --- a/polkadot/runtime/polkadot/src/weights/runtime_parachains_hrmp.rs +++ b/polkadot/runtime/polkadot/src/weights/runtime_parachains_hrmp.rs @@ -292,4 +292,46 @@ impl runtime_parachains::hrmp::WeightInfo for WeightInf .saturating_add(T::DbWeight::get().reads(14)) .saturating_add(T::DbWeight::get().writes(8)) } + /// Storage: `Paras::ParaLifecycles` (r:1 w:0) + /// Proof: `Paras::ParaLifecycles` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequests` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequests` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpChannels` (r:1 w:0) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpEgressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpEgressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestsList` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestsList` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueues` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueues` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueueHeads` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueueHeads` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpIngressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpIngressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpAcceptedChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpAcceptedChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn establish_system_channel() -> Weight { + // Proof Size summary in bytes: + // Measured: `417` + // Estimated: `6357` + // Minimum execution time: 629_674_000 picoseconds. + Weight::from_parts(640_174_000, 0) + .saturating_add(Weight::from_parts(0, 6357)) + .saturating_add(T::DbWeight::get().reads(12)) + .saturating_add(T::DbWeight::get().writes(8)) + } + /// Storage: `Hrmp::HrmpChannels` (r:1 w:1) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn poke_channel_deposits() -> Weight { + // Proof Size summary in bytes: + // Measured: `263` + // Estimated: `3728` + // Minimum execution time: 173_371_000 picoseconds. + Weight::from_parts(175_860_000, 0) + .saturating_add(Weight::from_parts(0, 3728)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } } diff --git a/polkadot/runtime/polkadot/src/xcm_config.rs b/polkadot/runtime/polkadot/src/xcm_config.rs index 6c45f1cec402e83850d6b8d8ab3cccd31f6734f5..1110c0d8eb95bf38630fd427a168f818315a4ea8 100644 --- a/polkadot/runtime/polkadot/src/xcm_config.rs +++ b/polkadot/runtime/polkadot/src/xcm_config.rs @@ -23,7 +23,7 @@ use super::{ }; use frame_support::{ match_types, parameter_types, - traits::{Contains, Everything, Nothing}, + traits::{Everything, Nothing}, weights::Weight, }; use frame_system::EnsureRoot; @@ -32,7 +32,6 @@ use polkadot_runtime_constants::{ currency::CENTS, system_parachain::*, xcm::body::FELLOWSHIP_ADMIN_INDEX, }; use runtime_common::{ - crowdloan, paras_registrar, xcm_sender::{ChildParachainRouter, ExponentialPrice}, ToAuthor, }; @@ -41,12 +40,11 @@ use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, ChildParachainAsNative, - ChildParachainConvertsVia, CurrencyAdapter as XcmCurrencyAdapter, IsConcrete, MintLocation, - OriginToPluralityVoice, SignedAccountId32AsNative, SignedToAccountId32, - SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, - WeightInfoBounds, WithComputedOrigin, WithUniqueTopic, + ChildParachainConvertsVia, CurrencyAdapter as XcmCurrencyAdapter, DescribeBodyTerminal, + DescribeFamily, HashedDescription, IsConcrete, MintLocation, OriginToPluralityVoice, + SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, + TrailingSetTopicAsId, UsingComponents, WeightInfoBounds, WithComputedOrigin, WithUniqueTopic, }; -use xcm_executor::traits::WithOriginFilter; parameter_types! { /// The location of the DOT token, from the context of this chain. Since this token is native to this @@ -70,6 +68,8 @@ pub type SovereignAccountOf = ( ChildParachainConvertsVia, // We can directly alias an `AccountId32` into a local account. AccountId32Aliases, + // Allow governance body to be used as a sovereign account. + HashedDescription>, ); /// Our asset transactor. This is what allows us to interact with the runtime assets from the point @@ -172,147 +172,6 @@ pub type Barrier = TrailingSetTopicAsId<( >, )>; -/// A call filter for the XCM Transact instruction. This is a temporary measure until we -/// properly account for proof size weights. -/// -/// Calls that are allowed through this filter must: -/// 1. Have a fixed weight; -/// 2. Cannot lead to another call being made; -/// 3. Have a defined proof size weight, e.g. no unbounded vecs in call parameters. -pub struct SafeCallFilter; -impl Contains for SafeCallFilter { - fn contains(call: &RuntimeCall) -> bool { - #[cfg(feature = "runtime-benchmarks")] - { - if matches!(call, RuntimeCall::System(frame_system::Call::remark_with_event { .. })) { - return true - } - } - - match call { - RuntimeCall::System( - frame_system::Call::kill_prefix { .. } | frame_system::Call::set_heap_pages { .. }, - ) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(..) | - RuntimeCall::Balances(..) | - RuntimeCall::Crowdloan( - crowdloan::Call::create { .. } | - crowdloan::Call::contribute { .. } | - crowdloan::Call::withdraw { .. } | - crowdloan::Call::refund { .. } | - crowdloan::Call::dissolve { .. } | - crowdloan::Call::edit { .. } | - crowdloan::Call::poke { .. } | - crowdloan::Call::contribute_all { .. }, - ) | - RuntimeCall::Staking( - pallet_staking::Call::bond { .. } | - pallet_staking::Call::bond_extra { .. } | - pallet_staking::Call::unbond { .. } | - pallet_staking::Call::withdraw_unbonded { .. } | - pallet_staking::Call::validate { .. } | - pallet_staking::Call::nominate { .. } | - pallet_staking::Call::chill { .. } | - pallet_staking::Call::set_payee { .. } | - pallet_staking::Call::set_controller { .. } | - pallet_staking::Call::set_validator_count { .. } | - pallet_staking::Call::increase_validator_count { .. } | - pallet_staking::Call::scale_validator_count { .. } | - pallet_staking::Call::force_no_eras { .. } | - pallet_staking::Call::force_new_era { .. } | - pallet_staking::Call::set_invulnerables { .. } | - pallet_staking::Call::force_unstake { .. } | - pallet_staking::Call::force_new_era_always { .. } | - pallet_staking::Call::payout_stakers { .. } | - pallet_staking::Call::rebond { .. } | - pallet_staking::Call::reap_stash { .. } | - pallet_staking::Call::set_staking_configs { .. } | - pallet_staking::Call::chill_other { .. } | - pallet_staking::Call::force_apply_min_commission { .. }, - ) | - RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Treasury(..) | - RuntimeCall::ConvictionVoting(..) | - RuntimeCall::Referenda( - pallet_referenda::Call::place_decision_deposit { .. } | - pallet_referenda::Call::refund_decision_deposit { .. } | - pallet_referenda::Call::cancel { .. } | - pallet_referenda::Call::kill { .. } | - pallet_referenda::Call::nudge_referendum { .. } | - pallet_referenda::Call::one_fewer_deciding { .. }, - ) | - RuntimeCall::Claims( - super::claims::Call::claim { .. } | - super::claims::Call::mint_claim { .. } | - super::claims::Call::move_claim { .. }, - ) | - RuntimeCall::Utility(pallet_utility::Call::as_derivative { .. }) | - RuntimeCall::Identity( - pallet_identity::Call::add_registrar { .. } | - pallet_identity::Call::set_identity { .. } | - pallet_identity::Call::clear_identity { .. } | - pallet_identity::Call::request_judgement { .. } | - pallet_identity::Call::cancel_request { .. } | - pallet_identity::Call::set_fee { .. } | - pallet_identity::Call::set_account_id { .. } | - pallet_identity::Call::set_fields { .. } | - pallet_identity::Call::provide_judgement { .. } | - pallet_identity::Call::kill_identity { .. } | - pallet_identity::Call::add_sub { .. } | - pallet_identity::Call::rename_sub { .. } | - pallet_identity::Call::remove_sub { .. } | - pallet_identity::Call::quit_sub { .. }, - ) | - RuntimeCall::Vesting(..) | - RuntimeCall::Bounties( - pallet_bounties::Call::propose_bounty { .. } | - pallet_bounties::Call::approve_bounty { .. } | - pallet_bounties::Call::propose_curator { .. } | - pallet_bounties::Call::unassign_curator { .. } | - pallet_bounties::Call::accept_curator { .. } | - pallet_bounties::Call::award_bounty { .. } | - pallet_bounties::Call::claim_bounty { .. } | - pallet_bounties::Call::close_bounty { .. }, - ) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::ElectionProviderMultiPhase(..) | - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools( - pallet_nomination_pools::Call::join { .. } | - pallet_nomination_pools::Call::bond_extra { .. } | - pallet_nomination_pools::Call::claim_payout { .. } | - pallet_nomination_pools::Call::unbond { .. } | - pallet_nomination_pools::Call::pool_withdraw_unbonded { .. } | - pallet_nomination_pools::Call::withdraw_unbonded { .. } | - pallet_nomination_pools::Call::create { .. } | - pallet_nomination_pools::Call::create_with_pool_id { .. } | - pallet_nomination_pools::Call::set_state { .. } | - pallet_nomination_pools::Call::set_configs { .. } | - pallet_nomination_pools::Call::update_roles { .. } | - pallet_nomination_pools::Call::chill { .. }, - ) | - RuntimeCall::Hrmp(..) | - RuntimeCall::Registrar( - paras_registrar::Call::deregister { .. } | - paras_registrar::Call::swap { .. } | - paras_registrar::Call::remove_lock { .. } | - paras_registrar::Call::reserve { .. } | - paras_registrar::Call::add_lock { .. }, - ) | - RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { - .. - }) | - RuntimeCall::Whitelist(pallet_whitelist::Call::whitelist_call { .. }) | - RuntimeCall::Proxy(..) => true, - _ => false, - } - } -} - pub struct XcmConfig; impl xcm_executor::Config for XcmConfig { type RuntimeCall = RuntimeCall; @@ -344,8 +203,8 @@ impl xcm_executor::Config for XcmConfig { // No bridges yet... type MessageExporter = (); type UniversalAliases = Nothing; - type CallDispatcher = WithOriginFilter; - type SafeCallFilter = SafeCallFilter; + type CallDispatcher = RuntimeCall; + type SafeCallFilter = Everything; type Aliasers = Nothing; } diff --git a/polkadot/runtime/rococo/Cargo.toml b/polkadot/runtime/rococo/Cargo.toml index 6af9407a5879b3e7e091f0425b38b7ffc030663f..8f38659b84f5498d7744a70445205400fc1e420c 100644 --- a/polkadot/runtime/rococo/Cargo.toml +++ b/polkadot/runtime/rococo/Cargo.toml @@ -65,7 +65,7 @@ pallet-proxy = { path = "../../../substrate/frame/proxy", default-features = fal pallet-recovery = { path = "../../../substrate/frame/recovery", default-features = false } pallet-scheduler = { path = "../../../substrate/frame/scheduler", default-features = false } pallet-session = { path = "../../../substrate/frame/session", default-features = false } -pallet-society = { path = "../../../substrate/frame/society", default-features = false, features = ["experimental"] } +pallet-society = { path = "../../../substrate/frame/society", default-features = false } pallet-sudo = { path = "../../../substrate/frame/sudo", default-features = false } frame-support = { path = "../../../substrate/frame/support", default-features = false } pallet-staking = { path = "../../../substrate/frame/staking", default-features = false } @@ -76,7 +76,7 @@ pallet-tips = { path = "../../../substrate/frame/tips", default-features = false pallet-treasury = { path = "../../../substrate/frame/treasury", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -84,7 +84,7 @@ frame-try-runtime = { path = "../../../substrate/frame/try-runtime", default-fea frame-system-benchmarking = { path = "../../../substrate/frame/system/benchmarking", default-features = false, optional = true } hex-literal = { version = "0.4.1" } -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features=["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } @@ -99,7 +99,7 @@ keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyrin remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } sp-trie = { path = "../../../substrate/primitives/trie" } separator = "0.4.1" -serde_json = "1.0.96" +serde_json = "1.0.107" sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/runtime/rococo/README.md b/polkadot/runtime/rococo/README.md index 0f35665ca1726104a4d95b613894f99b05d476a5..465afd25549b69f8e3ef30f926bdefa4b81ed88f 100644 --- a/polkadot/runtime/rococo/README.md +++ b/polkadot/runtime/rococo/README.md @@ -4,8 +4,10 @@ Rococo is a testnet runtime with no stability guarantees. ## How to run `rococo-local` -The [Cumulus Tutorial](https://docs.substrate.io/tutorials/v3/cumulus/start-relay/) details building, starting, and testing `rococo-local` and parachains connecting to it. +The [Cumulus Tutorial](https://docs.substrate.io/tutorials/v3/cumulus/start-relay/) details building, starting, and +testing `rococo-local` and parachains connecting to it. ## How to register a parachain on the Rococo testnet -The [parachain registration process](https://docs.substrate.io/tutorials/v3/cumulus/rococo/) on the public Rococo testnet is also outlined. +The [parachain registration process](https://docs.substrate.io/tutorials/v3/cumulus/rococo/) on the public Rococo +testnet is also outlined. diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index a80f45c340d97b16472f5394c51fe279a2d9960b..7046c4640c040e4ddc13fb94b945b1ac3780de56 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -60,8 +60,9 @@ use beefy_primitives::{ use frame_support::{ construct_runtime, parameter_types, traits::{ - Contains, EitherOfDiverse, InstanceFilter, KeyOwnerProofSystem, LockIdentifier, - PrivilegeCmp, ProcessMessage, ProcessMessageError, StorageMapShim, WithdrawReasons, + fungible::HoldConsideration, Contains, EitherOfDiverse, EverythingBut, InstanceFilter, + KeyOwnerProofSystem, LinearStoragePrice, LockIdentifier, PrivilegeCmp, ProcessMessage, + ProcessMessageError, StorageMapShim, WithdrawReasons, }, weights::{ConstantMultiplier, WeightMeter}, PalletId, @@ -134,11 +135,14 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } -/// We currently allow all calls. -pub struct BaseFilter; -impl Contains for BaseFilter { - fn contains(_call: &RuntimeCall) -> bool { - true +/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, +/// locking the state of the pallet and preventing further updates to identities and sub-identities. +/// The locked state will be the genesis state of a new system chain and then removed from the Relay +/// Chain. +pub struct IdentityCalls; +impl Contains for IdentityCalls { + fn contains(c: &RuntimeCall) -> bool { + matches!(c, RuntimeCall::Identity(_)) } } @@ -148,7 +152,7 @@ parameter_types! { } impl frame_system::Config for Runtime { - type BaseCallFilter = BaseFilter; + type BaseCallFilter = EverythingBut; type BlockWeights = BlockWeights; type BlockLength = BlockLength; type DbWeight = RocksDbWeight; @@ -224,6 +228,7 @@ impl pallet_scheduler::Config for Runtime { parameter_types! { pub const PreimageBaseDeposit: Balance = deposit(2, 64); pub const PreimageByteDeposit: Balance = deposit(0, 1); + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -231,8 +236,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } parameter_types! { @@ -1031,6 +1040,7 @@ impl parachains_paras::Config for Runtime { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = Babe; + type OnNewHead = Registrar; } parameter_types! { @@ -1448,7 +1458,7 @@ construct_runtime! { Multisig: pallet_multisig::{Pallet, Call, Storage, Event} = 31, // Preimage registrar. - Preimage: pallet_preimage::{Pallet, Call, Storage, Event} = 32, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 32, // Bounties modules. Bounties: pallet_bounties::{Pallet, Call, Storage, Event} = 35, @@ -1550,6 +1560,7 @@ pub mod migrations { parachains_scheduler::migration::v1::MigrateToV1, parachains_configuration::migration::v8::MigrateToV8, parachains_configuration::migration::v9::MigrateToV9, + paras_registrar::migration::VersionCheckedMigrateToV1, ); } diff --git a/polkadot/runtime/rococo/src/weights/pallet_preimage.rs b/polkadot/runtime/rococo/src/weights/pallet_preimage.rs index b067e6a6d91e31e745f78c36eb96e6aced428452..e051ebd5bbab84cf98d34853666926a0f281e5c1 100644 --- a/polkadot/runtime/rococo/src/weights/pallet_preimage.rs +++ b/polkadot/runtime/rococo/src/weights/pallet_preimage.rs @@ -47,6 +47,21 @@ use core::marker::PhantomData; /// Weight functions for `pallet_preimage`. pub struct WeightInfo(PhantomData); impl pallet_preimage::WeightInfo for WeightInfo { + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_000_000 picoseconds. + Weight::from_parts(2_000_000, 3593) + // Standard Error: 13_720 + .saturating_add(Weight::from_parts(17_309_199, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) + } + /// Storage: Preimage StatusFor (r:1 w:1) /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) /// Storage: Preimage PreimageFor (r:0 w:1) diff --git a/polkadot/runtime/rococo/src/weights/runtime_parachains_hrmp.rs b/polkadot/runtime/rococo/src/weights/runtime_parachains_hrmp.rs index 9f1cf65efa649972f111be4029b00ca40c7554da..417820e6627f6e39c4107df722f4b83f9cca6a4b 100644 --- a/polkadot/runtime/rococo/src/weights/runtime_parachains_hrmp.rs +++ b/polkadot/runtime/rococo/src/weights/runtime_parachains_hrmp.rs @@ -289,4 +289,46 @@ impl runtime_parachains::hrmp::WeightInfo for WeightInf .saturating_add(T::DbWeight::get().reads(14)) .saturating_add(T::DbWeight::get().writes(8)) } + /// Storage: `Paras::ParaLifecycles` (r:1 w:0) + /// Proof: `Paras::ParaLifecycles` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequests` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequests` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpChannels` (r:1 w:0) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpEgressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpEgressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestsList` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestsList` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueues` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueues` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueueHeads` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueueHeads` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpIngressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpIngressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpAcceptedChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpAcceptedChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn establish_system_channel() -> Weight { + // Proof Size summary in bytes: + // Measured: `417` + // Estimated: `6357` + // Minimum execution time: 629_674_000 picoseconds. + Weight::from_parts(640_174_000, 0) + .saturating_add(Weight::from_parts(0, 6357)) + .saturating_add(T::DbWeight::get().reads(12)) + .saturating_add(T::DbWeight::get().writes(8)) + } + /// Storage: `Hrmp::HrmpChannels` (r:1 w:1) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn poke_channel_deposits() -> Weight { + // Proof Size summary in bytes: + // Measured: `263` + // Estimated: `3728` + // Minimum execution time: 173_371_000 picoseconds. + Weight::from_parts(175_860_000, 0) + .saturating_add(Weight::from_parts(0, 3728)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } } diff --git a/polkadot/runtime/rococo/src/xcm_config.rs b/polkadot/runtime/rococo/src/xcm_config.rs index 356cffaa0ba2b6582582c1bdc2e48f5dbe6bb951..445cb8014357d98584cac2486fbf3717ace74f8b 100644 --- a/polkadot/runtime/rococo/src/xcm_config.rs +++ b/polkadot/runtime/rococo/src/xcm_config.rs @@ -22,13 +22,12 @@ use super::{ }; use frame_support::{ match_types, parameter_types, - traits::{Contains, Everything, Nothing}, + traits::{Everything, Nothing}, weights::Weight, }; use frame_system::EnsureRoot; use rococo_runtime_constants::currency::CENTS; use runtime_common::{ - crowdloan, paras_registrar, xcm_sender::{ChildParachainRouter, ExponentialPrice}, ToAuthor, }; @@ -37,13 +36,13 @@ use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, ChildParachainAsNative, - ChildParachainConvertsVia, ChildSystemParachainAsSuperuser, - CurrencyAdapter as XcmCurrencyAdapter, FixedWeightBounds, IsChildSystemParachain, IsConcrete, + ChildParachainConvertsVia, CurrencyAdapter as XcmCurrencyAdapter, DescribeBodyTerminal, + DescribeFamily, FixedWeightBounds, HashedDescription, IsChildSystemParachain, IsConcrete, MintLocation, SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, WeightInfoBounds, WithComputedOrigin, WithUniqueTopic, }; -use xcm_executor::{traits::WithOriginFilter, XcmExecutor}; +use xcm_executor::XcmExecutor; parameter_types! { pub const TokenLocation: MultiLocation = Here.into_location(); @@ -53,8 +52,14 @@ parameter_types! { pub LocalCheckAccount: (AccountId, MintLocation) = (CheckAccount::get(), MintLocation::Local); } -pub type LocationConverter = - (ChildParachainConvertsVia, AccountId32Aliases); +pub type LocationConverter = ( + // We can convert a child parachain using the standard `AccountId` conversion. + ChildParachainConvertsVia, + // We can directly alias an `AccountId32` into a local account. + AccountId32Aliases, + // Allow governance body to be used as a sovereign account. + HashedDescription>, +); /// Our asset transactor. This is what allows us to interest with the runtime facilities from the /// point of view of XCM-only concepts like `MultiLocation` and `MultiAsset`. @@ -81,8 +86,6 @@ type LocalOriginConverter = ( ChildParachainAsNative, // The AccountId32 location type can be expressed natively as a `Signed` origin. SignedAccountId32AsNative, - // A system child parachain, expressed as a Superuser, converts to the `Root` origin. - ChildSystemParachainAsSuperuser, ); parameter_types! { @@ -157,138 +160,6 @@ pub type Barrier = TrailingSetTopicAsId<( >, )>; -/// A call filter for the XCM Transact instruction. This is a temporary measure until we -/// properly account for proof size weights. -/// -/// Calls that are allowed through this filter must: -/// 1. Have a fixed weight; -/// 2. Cannot lead to another call being made; -/// 3. Have a defined proof size weight, e.g. no unbounded vecs in call parameters. -pub struct SafeCallFilter; -impl Contains for SafeCallFilter { - fn contains(call: &RuntimeCall) -> bool { - #[cfg(feature = "runtime-benchmarks")] - { - if matches!(call, RuntimeCall::System(frame_system::Call::remark_with_event { .. })) { - return true - } - } - - match call { - RuntimeCall::System( - frame_system::Call::kill_prefix { .. } | frame_system::Call::set_heap_pages { .. }, - ) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(..) | - RuntimeCall::Balances(..) | - RuntimeCall::Crowdloan( - crowdloan::Call::create { .. } | - crowdloan::Call::contribute { .. } | - crowdloan::Call::withdraw { .. } | - crowdloan::Call::refund { .. } | - crowdloan::Call::dissolve { .. } | - crowdloan::Call::edit { .. } | - crowdloan::Call::poke { .. } | - crowdloan::Call::contribute_all { .. }, - ) | - RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Democracy( - pallet_democracy::Call::second { .. } | - pallet_democracy::Call::vote { .. } | - pallet_democracy::Call::emergency_cancel { .. } | - pallet_democracy::Call::fast_track { .. } | - pallet_democracy::Call::veto_external { .. } | - pallet_democracy::Call::cancel_referendum { .. } | - pallet_democracy::Call::delegate { .. } | - pallet_democracy::Call::undelegate { .. } | - pallet_democracy::Call::clear_public_proposals { .. } | - pallet_democracy::Call::unlock { .. } | - pallet_democracy::Call::remove_vote { .. } | - pallet_democracy::Call::remove_other_vote { .. } | - pallet_democracy::Call::blacklist { .. } | - pallet_democracy::Call::cancel_proposal { .. }, - ) | - RuntimeCall::Council( - pallet_collective::Call::vote { .. } | - pallet_collective::Call::disapprove_proposal { .. } | - pallet_collective::Call::close { .. }, - ) | - RuntimeCall::TechnicalCommittee( - pallet_collective::Call::vote { .. } | - pallet_collective::Call::disapprove_proposal { .. } | - pallet_collective::Call::close { .. }, - ) | - RuntimeCall::PhragmenElection( - pallet_elections_phragmen::Call::remove_voter { .. } | - pallet_elections_phragmen::Call::submit_candidacy { .. } | - pallet_elections_phragmen::Call::renounce_candidacy { .. } | - pallet_elections_phragmen::Call::remove_member { .. } | - pallet_elections_phragmen::Call::clean_defunct_voters { .. }, - ) | - RuntimeCall::TechnicalMembership( - pallet_membership::Call::add_member { .. } | - pallet_membership::Call::remove_member { .. } | - pallet_membership::Call::swap_member { .. } | - pallet_membership::Call::change_key { .. } | - pallet_membership::Call::set_prime { .. } | - pallet_membership::Call::clear_prime { .. }, - ) | - RuntimeCall::Treasury(..) | - RuntimeCall::Claims( - super::claims::Call::claim { .. } | - super::claims::Call::mint_claim { .. } | - super::claims::Call::move_claim { .. }, - ) | - RuntimeCall::Utility(pallet_utility::Call::as_derivative { .. }) | - RuntimeCall::Identity( - pallet_identity::Call::add_registrar { .. } | - pallet_identity::Call::set_identity { .. } | - pallet_identity::Call::clear_identity { .. } | - pallet_identity::Call::request_judgement { .. } | - pallet_identity::Call::cancel_request { .. } | - pallet_identity::Call::set_fee { .. } | - pallet_identity::Call::set_account_id { .. } | - pallet_identity::Call::set_fields { .. } | - pallet_identity::Call::provide_judgement { .. } | - pallet_identity::Call::kill_identity { .. } | - pallet_identity::Call::add_sub { .. } | - pallet_identity::Call::rename_sub { .. } | - pallet_identity::Call::remove_sub { .. } | - pallet_identity::Call::quit_sub { .. }, - ) | - RuntimeCall::Society(..) | - RuntimeCall::Recovery(..) | - RuntimeCall::Vesting(..) | - RuntimeCall::Bounties( - pallet_bounties::Call::propose_bounty { .. } | - pallet_bounties::Call::approve_bounty { .. } | - pallet_bounties::Call::propose_curator { .. } | - pallet_bounties::Call::unassign_curator { .. } | - pallet_bounties::Call::accept_curator { .. } | - pallet_bounties::Call::award_bounty { .. } | - pallet_bounties::Call::claim_bounty { .. } | - pallet_bounties::Call::close_bounty { .. }, - ) | - RuntimeCall::ChildBounties(..) | - RuntimeCall::Hrmp(..) | - RuntimeCall::Registrar( - paras_registrar::Call::deregister { .. } | - paras_registrar::Call::swap { .. } | - paras_registrar::Call::remove_lock { .. } | - paras_registrar::Call::reserve { .. } | - paras_registrar::Call::add_lock { .. }, - ) | - RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { - .. - }) => true, - _ => false, - } - } -} - pub struct XcmConfig; impl xcm_executor::Config for XcmConfig { type RuntimeCall = RuntimeCall; @@ -317,8 +188,8 @@ impl xcm_executor::Config for XcmConfig { type FeeManager = (); type MessageExporter = (); type UniversalAliases = Nothing; - type CallDispatcher = WithOriginFilter; - type SafeCallFilter = SafeCallFilter; + type CallDispatcher = RuntimeCall; + type SafeCallFilter = Everything; type Aliasers = Nothing; } diff --git a/polkadot/runtime/test-runtime/Cargo.toml b/polkadot/runtime/test-runtime/Cargo.toml index 1e411bc901ced86810745eef2e19357e1b5091b5..a8f26eb5b3d951a09c72a9b40cb78599a24299b6 100644 --- a/polkadot/runtime/test-runtime/Cargo.toml +++ b/polkadot/runtime/test-runtime/Cargo.toml @@ -70,7 +70,7 @@ hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } -serde_json = "1.0.96" +serde_json = "1.0.107" [build-dependencies] substrate-wasm-builder = { path = "../../../substrate/utils/wasm-builder" } diff --git a/polkadot/runtime/test-runtime/src/lib.rs b/polkadot/runtime/test-runtime/src/lib.rs index b2397299430de240ddc43b0812f69ecabcdf19aa..94852ad39f5aae110931c701793d104430ffc199 100644 --- a/polkadot/runtime/test-runtime/src/lib.rs +++ b/polkadot/runtime/test-runtime/src/lib.rs @@ -540,6 +540,7 @@ impl parachains_paras::Config for Runtime { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = Babe; + type OnNewHead = (); } impl parachains_dmp::Config for Runtime {} diff --git a/polkadot/runtime/westend/Cargo.toml b/polkadot/runtime/westend/Cargo.toml index 5a47288297befb78d58c53c6ca239440bc14223a..f4bb66f68cdfee559da0f7825a019e764dc2fde4 100644 --- a/polkadot/runtime/westend/Cargo.toml +++ b/polkadot/runtime/westend/Cargo.toml @@ -83,7 +83,7 @@ pallet-nomination-pools-runtime-api = { path = "../../../substrate/frame/nominat pallet-treasury = { path = "../../../substrate/frame/treasury", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -95,7 +95,7 @@ pallet-offences-benchmarking = { path = "../../../substrate/frame/offences/bench pallet-session-benchmarking = { path = "../../../substrate/frame/session/benchmarking", default-features = false, optional = true } hex-literal = { version = "0.4.1", optional = true } -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features=["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } @@ -108,7 +108,7 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../xcm/xcm-builder", hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } -serde_json = "1.0.96" +serde_json = "1.0.107" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 73aa4980151eaaf528067aad7d8e70e9a6b4c92b..9af18b5be2bbcb4300796eccc4a57afa8814110b 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -29,7 +29,8 @@ use frame_election_provider_support::{bounds::ElectionBoundsBuilder, onchain, Se use frame_support::{ construct_runtime, parameter_types, traits::{ - ConstU32, InstanceFilter, KeyOwnerProofSystem, ProcessMessage, ProcessMessageError, + fungible::HoldConsideration, ConstU32, Contains, EverythingBut, InstanceFilter, + KeyOwnerProofSystem, LinearStoragePrice, ProcessMessage, ProcessMessageError, WithdrawReasons, }, weights::{ConstantMultiplier, WeightMeter}, @@ -79,7 +80,7 @@ use sp_runtime::{ Keccak256, OpaqueKeys, SaturatedConversion, Verify, }, transaction_validity::{TransactionPriority, TransactionSource, TransactionValidity}, - ApplyExtrinsicResult, FixedU128, KeyTypeId, Perbill, + ApplyExtrinsicResult, FixedU128, KeyTypeId, Perbill, Percent, }; use sp_staking::SessionIndex; use sp_std::{collections::btree_map::BTreeMap, prelude::*}; @@ -90,7 +91,7 @@ use xcm::latest::Junction; pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; -pub use pallet_election_provider_multi_phase::Call as EPMCall; +pub use pallet_election_provider_multi_phase::{Call as EPMCall, GeometricDepositBase}; #[cfg(feature = "std")] pub use pallet_staking::StakerStatus; use pallet_staking::UseValidatorsMap; @@ -141,13 +142,24 @@ pub fn native_version() -> NativeVersion { NativeVersion { runtime_version: VERSION, can_author_with: Default::default() } } +/// A type to identify calls to the Identity pallet. These will be filtered to prevent invocation, +/// locking the state of the pallet and preventing further updates to identities and sub-identities. +/// The locked state will be the genesis state of a new system chain and then removed from the Relay +/// Chain. +pub struct IdentityCalls; +impl Contains for IdentityCalls { + fn contains(c: &RuntimeCall) -> bool { + matches!(c, RuntimeCall::Identity(_)) + } +} + parameter_types! { pub const Version: RuntimeVersion = VERSION; pub const SS58Prefix: u8 = 42; } impl frame_system::Config for Runtime { - type BaseCallFilter = frame_support::traits::Everything; + type BaseCallFilter = EverythingBut; type BlockWeights = BlockWeights; type BlockLength = BlockLength; type RuntimeOrigin = RuntimeOrigin; @@ -193,9 +205,9 @@ impl pallet_scheduler::Config for Runtime { } parameter_types! { - pub const PreimageMaxSize: u32 = 4096 * 1024; pub const PreimageBaseDeposit: Balance = deposit(2, 64); pub const PreimageByteDeposit: Balance = deposit(0, 1); + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -203,8 +215,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } parameter_types! { @@ -268,7 +284,7 @@ impl pallet_balances::Config for Runtime { type WeightInfo = weights::pallet_balances::WeightInfo; type RuntimeHoldReason = RuntimeHoldReason; type FreezeIdentifier = (); - type MaxHolds = ConstU32<0>; + type MaxHolds = ConstU32<1>; type MaxFreezes = ConstU32<0>; } @@ -477,7 +493,8 @@ parameter_types! { // signed config pub const SignedMaxSubmissions: u32 = 128; pub const SignedMaxRefunds: u32 = 128 / 4; - pub const SignedDepositBase: Balance = deposit(2, 0); + pub const SignedFixedDeposit: Balance = deposit(2, 0); + pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); pub const SignedDepositByte: Balance = deposit(0, 10) / 1024; // Each good submission will get 1 WND as reward pub SignedRewardBase: Balance = 1 * UNITS; @@ -549,7 +566,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type SignedMaxSubmissions = SignedMaxSubmissions; type SignedMaxRefunds = SignedMaxRefunds; type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; + type SignedDepositBase = + GeometricDepositBase; type SignedDepositByte = SignedDepositByte; type SignedDepositWeight = (); type SignedMaxWeight = @@ -1044,6 +1062,7 @@ impl parachains_paras::Config for Runtime { type UnsignedPriority = ParasUnsignedPriority; type QueueFootprinter = ParaInclusion; type NextSessionRotation = Babe; + type OnNewHead = (); } parameter_types! { @@ -1310,7 +1329,7 @@ construct_runtime! { Scheduler: pallet_scheduler::{Pallet, Call, Storage, Event} = 20, // Preimage registrar. - Preimage: pallet_preimage::{Pallet, Call, Storage, Event} = 28, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason} = 28, // Sudo. Sudo: pallet_sudo::{Pallet, Call, Storage, Event, Config} = 21, @@ -1425,6 +1444,7 @@ pub mod migrations { parachains_configuration::migration::v8::MigrateToV8, UpgradeSessionKeys, parachains_configuration::migration::v9::MigrateToV9, + paras_registrar::migration::VersionCheckedMigrateToV1, ); } diff --git a/polkadot/runtime/westend/src/weights/pallet_preimage.rs b/polkadot/runtime/westend/src/weights/pallet_preimage.rs index 39d3626b189f6e9078556cb68d59de9a3aa2d99d..0c4677a7d969423bb5bccbb938f6b8fd2c9ac402 100644 --- a/polkadot/runtime/westend/src/weights/pallet_preimage.rs +++ b/polkadot/runtime/westend/src/weights/pallet_preimage.rs @@ -50,6 +50,21 @@ use core::marker::PhantomData; /// Weight functions for `pallet_preimage`. pub struct WeightInfo(PhantomData); impl pallet_preimage::WeightInfo for WeightInfo { + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_000_000 picoseconds. + Weight::from_parts(2_000_000, 3593) + // Standard Error: 13_720 + .saturating_add(Weight::from_parts(17_309_199, 0).saturating_mul(n.into())) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) + .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) + } + /// Storage: Preimage StatusFor (r:1 w:1) /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) /// Storage: Preimage PreimageFor (r:0 w:1) diff --git a/polkadot/runtime/westend/src/weights/runtime_parachains_hrmp.rs b/polkadot/runtime/westend/src/weights/runtime_parachains_hrmp.rs index e6ff97fa08784d5f9fc75d9274295c2ee666843a..9beb15303d871cd4ae0b6b0ce3148291d745395b 100644 --- a/polkadot/runtime/westend/src/weights/runtime_parachains_hrmp.rs +++ b/polkadot/runtime/westend/src/weights/runtime_parachains_hrmp.rs @@ -282,4 +282,46 @@ impl runtime_parachains::hrmp::WeightInfo for WeightInf .saturating_add(T::DbWeight::get().reads(13)) .saturating_add(T::DbWeight::get().writes(8)) } + /// Storage: `Paras::ParaLifecycles` (r:1 w:0) + /// Proof: `Paras::ParaLifecycles` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequests` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequests` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpChannels` (r:1 w:0) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpEgressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpEgressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpOpenChannelRequestsList` (r:1 w:1) + /// Proof: `Hrmp::HrmpOpenChannelRequestsList` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueues` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueues` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Dmp::DownwardMessageQueueHeads` (r:2 w:2) + /// Proof: `Dmp::DownwardMessageQueueHeads` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpIngressChannelsIndex` (r:1 w:0) + /// Proof: `Hrmp::HrmpIngressChannelsIndex` (`max_values`: None, `max_size`: None, mode: `Measured`) + /// Storage: `Hrmp::HrmpAcceptedChannelRequestCount` (r:1 w:1) + /// Proof: `Hrmp::HrmpAcceptedChannelRequestCount` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn establish_system_channel() -> Weight { + // Proof Size summary in bytes: + // Measured: `417` + // Estimated: `6357` + // Minimum execution time: 629_674_000 picoseconds. + Weight::from_parts(640_174_000, 0) + .saturating_add(Weight::from_parts(0, 6357)) + .saturating_add(T::DbWeight::get().reads(12)) + .saturating_add(T::DbWeight::get().writes(8)) + } + /// Storage: `Hrmp::HrmpChannels` (r:1 w:1) + /// Proof: `Hrmp::HrmpChannels` (`max_values`: None, `max_size`: None, mode: `Measured`) + fn poke_channel_deposits() -> Weight { + // Proof Size summary in bytes: + // Measured: `263` + // Estimated: `3728` + // Minimum execution time: 173_371_000 picoseconds. + Weight::from_parts(175_860_000, 0) + .saturating_add(Weight::from_parts(0, 3728)) + .saturating_add(T::DbWeight::get().reads(1)) + .saturating_add(T::DbWeight::get().writes(1)) + } } diff --git a/polkadot/runtime/westend/src/xcm_config.rs b/polkadot/runtime/westend/src/xcm_config.rs index a83c38c9f66f655055c39ec12ceca5d890aa5b0f..92dcee150aab27a1939d2cff39e9002bdd7f8422 100644 --- a/polkadot/runtime/westend/src/xcm_config.rs +++ b/polkadot/runtime/westend/src/xcm_config.rs @@ -22,11 +22,10 @@ use super::{ }; use frame_support::{ parameter_types, - traits::{Contains, Everything, Nothing}, + traits::{Everything, Nothing}, }; use frame_system::EnsureRoot; use runtime_common::{ - crowdloan, paras_registrar, xcm_sender::{ChildParachainRouter, ExponentialPrice}, ToAuthor, }; @@ -36,12 +35,12 @@ use xcm::latest::prelude::*; use xcm_builder::{ AccountId32Aliases, AllowExplicitUnpaidExecutionFrom, AllowKnownQueryResponses, AllowSubscriptionsFrom, AllowTopLevelPaidExecutionFrom, ChildParachainAsNative, - ChildParachainConvertsVia, ChildSystemParachainAsSuperuser, - CurrencyAdapter as XcmCurrencyAdapter, IsChildSystemParachain, IsConcrete, MintLocation, + ChildParachainConvertsVia, CurrencyAdapter as XcmCurrencyAdapter, DescribeBodyTerminal, + DescribeFamily, HashedDescription, IsChildSystemParachain, IsConcrete, MintLocation, SignedAccountId32AsNative, SignedToAccountId32, SovereignSignedViaLocation, TakeWeightCredit, TrailingSetTopicAsId, UsingComponents, WeightInfoBounds, WithComputedOrigin, WithUniqueTopic, }; -use xcm_executor::{traits::WithOriginFilter, XcmExecutor}; +use xcm_executor::XcmExecutor; parameter_types! { pub const TokenLocation: MultiLocation = Here.into_location(); @@ -55,8 +54,14 @@ parameter_types! { pub const BaseDeliveryFee: u128 = CENTS.saturating_mul(3); } -pub type LocationConverter = - (ChildParachainConvertsVia, AccountId32Aliases); +pub type LocationConverter = ( + // We can convert a child parachain using the standard `AccountId` conversion. + ChildParachainConvertsVia, + // We can directly alias an `AccountId32` into a local account. + AccountId32Aliases, + // Allow governance body to be used as a sovereign account. + HashedDescription>, +); pub type LocalAssetTransactor = XcmCurrencyAdapter< // Use this currency: @@ -75,7 +80,6 @@ type LocalOriginConverter = ( SovereignSignedViaLocation, ChildParachainAsNative, SignedAccountId32AsNative, - ChildSystemParachainAsSuperuser, ); /// The XCM router. When we want to send an XCM message, we use this type. It amalgamates all of our @@ -127,120 +131,6 @@ pub type Barrier = TrailingSetTopicAsId<( >, )>; -/// A call filter for the XCM Transact instruction. This is a temporary measure until we -/// properly account for proof size weights. -/// -/// Calls that are allowed through this filter must: -/// 1. Have a fixed weight; -/// 2. Cannot lead to another call being made; -/// 3. Have a defined proof size weight, e.g. no unbounded vecs in call parameters. -pub struct SafeCallFilter; -impl Contains for SafeCallFilter { - fn contains(call: &RuntimeCall) -> bool { - #[cfg(feature = "runtime-benchmarks")] - { - if matches!(call, RuntimeCall::System(frame_system::Call::remark_with_event { .. })) { - return true - } - } - - match call { - RuntimeCall::System( - frame_system::Call::kill_prefix { .. } | frame_system::Call::set_heap_pages { .. }, - ) | - RuntimeCall::Babe(..) | - RuntimeCall::Timestamp(..) | - RuntimeCall::Indices(..) | - RuntimeCall::Balances(..) | - RuntimeCall::Crowdloan( - crowdloan::Call::create { .. } | - crowdloan::Call::contribute { .. } | - crowdloan::Call::withdraw { .. } | - crowdloan::Call::refund { .. } | - crowdloan::Call::dissolve { .. } | - crowdloan::Call::edit { .. } | - crowdloan::Call::poke { .. } | - crowdloan::Call::contribute_all { .. }, - ) | - RuntimeCall::Staking( - pallet_staking::Call::bond { .. } | - pallet_staking::Call::bond_extra { .. } | - pallet_staking::Call::unbond { .. } | - pallet_staking::Call::withdraw_unbonded { .. } | - pallet_staking::Call::validate { .. } | - pallet_staking::Call::nominate { .. } | - pallet_staking::Call::chill { .. } | - pallet_staking::Call::set_payee { .. } | - pallet_staking::Call::set_controller { .. } | - pallet_staking::Call::set_validator_count { .. } | - pallet_staking::Call::increase_validator_count { .. } | - pallet_staking::Call::scale_validator_count { .. } | - pallet_staking::Call::force_no_eras { .. } | - pallet_staking::Call::force_new_era { .. } | - pallet_staking::Call::set_invulnerables { .. } | - pallet_staking::Call::force_unstake { .. } | - pallet_staking::Call::force_new_era_always { .. } | - pallet_staking::Call::payout_stakers { .. } | - pallet_staking::Call::rebond { .. } | - pallet_staking::Call::reap_stash { .. } | - pallet_staking::Call::set_staking_configs { .. } | - pallet_staking::Call::chill_other { .. } | - pallet_staking::Call::force_apply_min_commission { .. }, - ) | - RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | - RuntimeCall::Grandpa(..) | - RuntimeCall::ImOnline(..) | - RuntimeCall::Utility(pallet_utility::Call::as_derivative { .. }) | - RuntimeCall::Identity( - pallet_identity::Call::add_registrar { .. } | - pallet_identity::Call::set_identity { .. } | - pallet_identity::Call::clear_identity { .. } | - pallet_identity::Call::request_judgement { .. } | - pallet_identity::Call::cancel_request { .. } | - pallet_identity::Call::set_fee { .. } | - pallet_identity::Call::set_account_id { .. } | - pallet_identity::Call::set_fields { .. } | - pallet_identity::Call::provide_judgement { .. } | - pallet_identity::Call::kill_identity { .. } | - pallet_identity::Call::add_sub { .. } | - pallet_identity::Call::rename_sub { .. } | - pallet_identity::Call::remove_sub { .. } | - pallet_identity::Call::quit_sub { .. }, - ) | - RuntimeCall::Recovery(..) | - RuntimeCall::Vesting(..) | - RuntimeCall::ElectionProviderMultiPhase(..) | - RuntimeCall::VoterList(..) | - RuntimeCall::NominationPools( - pallet_nomination_pools::Call::join { .. } | - pallet_nomination_pools::Call::bond_extra { .. } | - pallet_nomination_pools::Call::claim_payout { .. } | - pallet_nomination_pools::Call::unbond { .. } | - pallet_nomination_pools::Call::pool_withdraw_unbonded { .. } | - pallet_nomination_pools::Call::withdraw_unbonded { .. } | - pallet_nomination_pools::Call::create { .. } | - pallet_nomination_pools::Call::create_with_pool_id { .. } | - pallet_nomination_pools::Call::set_state { .. } | - pallet_nomination_pools::Call::set_configs { .. } | - pallet_nomination_pools::Call::update_roles { .. } | - pallet_nomination_pools::Call::chill { .. }, - ) | - RuntimeCall::Hrmp(..) | - RuntimeCall::Registrar( - paras_registrar::Call::deregister { .. } | - paras_registrar::Call::swap { .. } | - paras_registrar::Call::remove_lock { .. } | - paras_registrar::Call::reserve { .. } | - paras_registrar::Call::add_lock { .. }, - ) | - RuntimeCall::XcmPallet(pallet_xcm::Call::limited_reserve_transfer_assets { - .. - }) => true, - _ => false, - } - } -} - pub struct XcmConfig; impl xcm_executor::Config for XcmConfig { type RuntimeCall = RuntimeCall; @@ -266,8 +156,8 @@ impl xcm_executor::Config for XcmConfig { type FeeManager = (); type MessageExporter = (); type UniversalAliases = Nothing; - type CallDispatcher = WithOriginFilter; - type SafeCallFilter = SafeCallFilter; + type CallDispatcher = RuntimeCall; + type SafeCallFilter = Everything; type Aliasers = Nothing; } diff --git a/polkadot/src/README.adoc b/polkadot/src/README.adoc deleted file mode 100644 index 4ec8e18d8afe90105104bf0d5434f8f7fc8c94ff..0000000000000000000000000000000000000000 --- a/polkadot/src/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot Src - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/src/bin/execute-worker.rs b/polkadot/src/bin/execute-worker.rs index 72cab799d753e814beae0a337e30b7c21269f08d..1deb365809860502e8e298802814a185479c09c6 100644 --- a/polkadot/src/bin/execute-worker.rs +++ b/polkadot/src/bin/execute-worker.rs @@ -19,5 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "execute-worker", polkadot_node_core_pvf_execute_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") + polkadot_cli::NODE_VERSION, ); diff --git a/polkadot/src/bin/prepare-worker.rs b/polkadot/src/bin/prepare-worker.rs index 695f66cc7b7d3cae5c35a36af9570ce3911ed432..d731f8a30d065f0c9abdcc438df411be7d16417e 100644 --- a/polkadot/src/bin/prepare-worker.rs +++ b/polkadot/src/bin/prepare-worker.rs @@ -19,5 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "prepare-worker", polkadot_node_core_pvf_prepare_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") + polkadot_cli::NODE_VERSION, ); diff --git a/polkadot/statement-table/README.adoc b/polkadot/statement-table/README.adoc deleted file mode 100644 index a4da4dee80ff5fc9917d93fa705e66220fb34696..0000000000000000000000000000000000000000 --- a/polkadot/statement-table/README.adoc +++ /dev/null @@ -1,5 +0,0 @@ - -= Polkadot Statement table - -placeholder -//TODO Write content :) (https://github.com/paritytech/polkadot/issues/159) diff --git a/polkadot/tests/benchmark_block.rs b/polkadot/tests/benchmark_block.rs index bc91b31bc7b64430b757b9f87f39a0eb34b7238f..99f95ef611a48266105ba6b413641867fab21101 100644 --- a/polkadot/tests/benchmark_block.rs +++ b/polkadot/tests/benchmark_block.rs @@ -18,6 +18,7 @@ #![cfg(unix)] use assert_cmd::cargo::cargo_bin; +use common::run_with_timeout; use nix::{ sys::signal::{kill, Signal::SIGINT}, unistd::Pid, @@ -32,25 +33,28 @@ use tempfile::tempdir; pub mod common; -static RUNTIMES: [&str; 4] = ["polkadot", "kusama", "westend", "rococo"]; +static RUNTIMES: &[&str] = &["westend", "rococo"]; /// `benchmark block` works for all dev runtimes using the wasm executor. #[tokio::test] async fn benchmark_block_works() { for runtime in RUNTIMES { - let tmp_dir = tempdir().expect("could not create a temp dir"); - let base_path = tmp_dir.path(); - let runtime = format!("{}-dev", runtime); - - // Build a chain with a single block. - build_chain(&runtime, base_path).await.unwrap(); - // Benchmark the one block. - benchmark_block(&runtime, base_path, 1).unwrap(); + run_with_timeout(Duration::from_secs(10 * 60), async move { + let tmp_dir = tempdir().expect("could not create a temp dir"); + let base_path = tmp_dir.path(); + let runtime = format!("{}-dev", runtime); + + // Build a chain with a single block. + build_chain(&runtime, base_path).await; + // Benchmark the one block. + benchmark_block(&runtime, base_path, 1).unwrap(); + }) + .await } } /// Builds a chain with one block for the given runtime and base path. -async fn build_chain(runtime: &str, base_path: &Path) -> Result<(), String> { +async fn build_chain(runtime: &str, base_path: &Path) { let mut cmd = Command::new(cargo_bin("polkadot")) .stdout(process::Stdio::piped()) .stderr(process::Stdio::piped()) @@ -64,13 +68,10 @@ async fn build_chain(runtime: &str, base_path: &Path) -> Result<(), String> { let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); // Wait for the chain to produce one block. - let ok = common::wait_n_finalized_blocks(1, Duration::from_secs(60), &ws_url).await; + common::wait_n_finalized_blocks(1, &ws_url).await; // Send SIGINT to node. kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); - // Wait for the node to handle it and exit. - assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default()); - - ok.map_err(|e| format!("Node did not build the chain: {:?}", e)) + assert!(cmd.wait().unwrap().success()); } /// Benchmarks the given block with the wasm executor. diff --git a/polkadot/tests/benchmark_extrinsic.rs b/polkadot/tests/benchmark_extrinsic.rs index e701fd9c923c2eb736ed34493a865afcca6d6145..529eb29362d4dda4fba34592ce5ff9066ca61796 100644 --- a/polkadot/tests/benchmark_extrinsic.rs +++ b/polkadot/tests/benchmark_extrinsic.rs @@ -17,7 +17,7 @@ use assert_cmd::cargo::cargo_bin; use std::{process::Command, result::Result}; -static RUNTIMES: [&str; 4] = ["polkadot", "kusama", "westend", "rococo"]; +static RUNTIMES: &[&str] = &["westend", "rococo"]; static EXTRINSICS: [(&str, &str); 2] = [("system", "remark"), ("balances", "transfer_keep_alive")]; diff --git a/polkadot/tests/benchmark_overhead.rs b/polkadot/tests/benchmark_overhead.rs index 295479594eb9ba56a5d64e2df34a014bdd8ecd9f..b0912225347df5f4dd563f7dd0ff8075e76923c9 100644 --- a/polkadot/tests/benchmark_overhead.rs +++ b/polkadot/tests/benchmark_overhead.rs @@ -18,26 +18,26 @@ use assert_cmd::cargo::cargo_bin; use std::{process::Command, result::Result}; use tempfile::tempdir; -static RUNTIMES: [&str; 4] = ["polkadot", "kusama", "westend", "rococo"]; +static RUNTIMES: &[&str] = &["westend", "rococo"]; /// `benchmark overhead` works for all dev runtimes. #[test] fn benchmark_overhead_works() { for runtime in RUNTIMES { let runtime = format!("{}-dev", runtime); - assert!(benchmark_overhead(runtime).is_ok()); + assert!(benchmark_overhead(&runtime).is_ok()); } } /// `benchmark overhead` rejects all non-dev runtimes. #[test] fn benchmark_overhead_rejects_non_dev_runtimes() { - for runtime in RUNTIMES { - assert!(benchmark_overhead(runtime.into()).is_err()); + for runtime in RUNTIMES.into_iter() { + assert!(benchmark_overhead(runtime).is_err()); } } -fn benchmark_overhead(runtime: String) -> Result<(), String> { +fn benchmark_overhead(runtime: &str) -> Result<(), String> { let tmp_dir = tempdir().expect("could not create a temp dir"); let base_path = tmp_dir.path(); diff --git a/polkadot/tests/common.rs b/polkadot/tests/common.rs index 940a0c6f18d0fe47575df6177fcc4f84fa7f3071..10859ead5fe8b8e8ebe782bbbcd83d2c9bafd811 100644 --- a/polkadot/tests/common.rs +++ b/polkadot/tests/common.rs @@ -16,49 +16,25 @@ use polkadot_core_primitives::{Block, Hash, Header}; use std::{ + future::Future, io::{BufRead, BufReader, Read}, - process::{Child, ExitStatus}, - thread, time::Duration, }; use substrate_rpc_client::{ws_client, ChainApi}; -use tokio::time::timeout; -/// Wait for the given `child` the given amount of `secs`. -/// -/// Returns the `Some(exit status)` or `None` if the process did not finish in the given time. -pub fn wait_for(child: &mut Child, secs: usize) -> Option { - for _ in 0..secs { - match child.try_wait().unwrap() { - Some(status) => return Some(status), - None => thread::sleep(Duration::from_secs(1)), - } - } - eprintln!("Took to long to exit. Killing..."); - let _ = child.kill(); - child.wait().unwrap(); - - None -} - -/// Wait for at least `n` blocks to be finalized within the specified time. -pub async fn wait_n_finalized_blocks( - n: usize, - timeout_duration: Duration, - url: &str, -) -> Result<(), tokio::time::error::Elapsed> { - timeout(timeout_duration, wait_n_finalized_blocks_from(n, url)).await +/// Run the given `future` and panic if the `timeout` is hit. +pub async fn run_with_timeout(timeout: Duration, future: impl Future) { + tokio::time::timeout(timeout, future).await.expect("Hit timeout"); } /// Wait for at least `n` blocks to be finalized from a specified node. -async fn wait_n_finalized_blocks_from(n: usize, url: &str) { +pub async fn wait_n_finalized_blocks(n: usize, url: &str) { let mut built_blocks = std::collections::HashSet::new(); let mut interval = tokio::time::interval(Duration::from_secs(6)); loop { - let rpc = match ws_client(url).await { - Ok(rpc_service) => rpc_service, - Err(_) => continue, + let Ok(rpc) = ws_client(url).await else { + continue; }; if let Ok(block) = ChainApi::<(), Hash, Header, Block>::finalized_head(&rpc).await { @@ -67,6 +43,7 @@ async fn wait_n_finalized_blocks_from(n: usize, url: &str) { break } }; + interval.tick().await; } } diff --git a/polkadot/tests/purge_chain_works.rs b/polkadot/tests/purge_chain_works.rs index 3e9a378147810e5d2edf377fa7c5ed93ecf16952..831155fb4d7e1078d72d66c6b4999ffcaa4d99fc 100644 --- a/polkadot/tests/purge_chain_works.rs +++ b/polkadot/tests/purge_chain_works.rs @@ -14,7 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . +#![cfg(unix)] + use assert_cmd::cargo::cargo_bin; +use common::run_with_timeout; +use nix::{ + sys::signal::{kill, Signal::SIGINT}, + unistd::Pid, +}; use std::{ process::{self, Command}, time::Duration, @@ -24,105 +31,95 @@ use tempfile::tempdir; pub mod common; #[tokio::test] -#[cfg(unix)] async fn purge_chain_rocksdb_works() { - use nix::{ - sys::signal::{kill, Signal::SIGINT}, - unistd::Pid, - }; - - let tmpdir = tempdir().expect("could not create temp dir"); - - let mut cmd = Command::new(cargo_bin("polkadot")) - .stdout(process::Stdio::piped()) - .stderr(process::Stdio::piped()) - .args(["--dev", "-d"]) - .arg(tmpdir.path()) - .arg("--port") - .arg("33034") - .arg("--no-hardware-benchmarks") - .spawn() - .unwrap(); - - let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); - - // Let it produce 1 block. - common::wait_n_finalized_blocks(1, Duration::from_secs(60), &ws_url) - .await - .unwrap(); - - // Send SIGINT to node. - kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); - // Wait for the node to handle it and exit. - assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default()); - assert!(tmpdir.path().join("chains/polkadot_dev").exists()); - assert!(tmpdir.path().join("chains/polkadot_dev/db/full").exists()); - assert!(tmpdir.path().join("chains/polkadot_dev/db/full/parachains").exists()); - - // Purge chain - let status = Command::new(cargo_bin("polkadot")) - .args(["purge-chain", "--dev", "-d"]) - .arg(tmpdir.path()) - .arg("-y") - .status() - .unwrap(); - assert!(status.success()); - - // Make sure that the chain folder exists, but `db/full` is deleted. - assert!(tmpdir.path().join("chains/polkadot_dev").exists()); - assert!(!tmpdir.path().join("chains/polkadot_dev/db/full").exists()); + run_with_timeout(Duration::from_secs(10 * 60), async move { + let tmpdir = tempdir().expect("could not create temp dir"); + + let mut cmd = Command::new(cargo_bin("polkadot")) + .stdout(process::Stdio::piped()) + .stderr(process::Stdio::piped()) + .args(["--dev", "-d"]) + .arg(tmpdir.path()) + .arg("--port") + .arg("33034") + .arg("--no-hardware-benchmarks") + .spawn() + .unwrap(); + + let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); + + // Let it produce 1 block. + common::wait_n_finalized_blocks(1, &ws_url).await; + + // Send SIGINT to node. + kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); + // Wait for the node to handle it and exit. + assert!(cmd.wait().unwrap().success()); + assert!(tmpdir.path().join("chains/rococo_dev").exists()); + assert!(tmpdir.path().join("chains/rococo_dev/db/full").exists()); + assert!(tmpdir.path().join("chains/rococo_dev/db/full/parachains").exists()); + + // Purge chain + let status = Command::new(cargo_bin("polkadot")) + .args(["purge-chain", "--dev", "-d"]) + .arg(tmpdir.path()) + .arg("-y") + .status() + .unwrap(); + assert!(status.success()); + + // Make sure that the chain folder exists, but `db/full` is deleted. + assert!(tmpdir.path().join("chains/rococo_dev").exists()); + assert!(!tmpdir.path().join("chains/rococo_dev/db/full").exists()); + }) + .await; } #[tokio::test] -#[cfg(unix)] async fn purge_chain_paritydb_works() { - use nix::{ - sys::signal::{kill, Signal::SIGINT}, - unistd::Pid, - }; - - let tmpdir = tempdir().expect("could not create temp dir"); - - let mut cmd = Command::new(cargo_bin("polkadot")) - .stdout(process::Stdio::piped()) - .stderr(process::Stdio::piped()) - .args(["--dev", "-d"]) - .arg(tmpdir.path()) - .arg("--database") - .arg("paritydb-experimental") - .arg("--no-hardware-benchmarks") - .spawn() - .unwrap(); - - let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); - - // Let it produce 1 block. - common::wait_n_finalized_blocks(1, Duration::from_secs(60), &ws_url) - .await - .unwrap(); - - // Send SIGINT to node. - kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); - // Wait for the node to handle it and exit. - assert!(common::wait_for(&mut cmd, 30).map(|x| x.success()).unwrap_or_default()); - assert!(tmpdir.path().join("chains/polkadot_dev").exists()); - assert!(tmpdir.path().join("chains/polkadot_dev/paritydb/full").exists()); - assert!(tmpdir.path().join("chains/polkadot_dev/paritydb/parachains").exists()); - - // Purge chain - let status = Command::new(cargo_bin("polkadot")) - .args(["purge-chain", "--dev", "-d"]) - .arg(tmpdir.path()) - .arg("--database") - .arg("paritydb-experimental") - .arg("-y") - .status() - .unwrap(); - assert!(status.success()); - - // Make sure that the chain folder exists, but `db/full` is deleted. - assert!(tmpdir.path().join("chains/polkadot_dev").exists()); - assert!(!tmpdir.path().join("chains/polkadot_dev/paritydb/full").exists()); - // Parachains removal requires calling "purge-chain --parachains". - assert!(tmpdir.path().join("chains/polkadot_dev/paritydb/parachains").exists()); + run_with_timeout(Duration::from_secs(10 * 60), async move { + let tmpdir = tempdir().expect("could not create temp dir"); + + let mut cmd = Command::new(cargo_bin("polkadot")) + .stdout(process::Stdio::piped()) + .stderr(process::Stdio::piped()) + .args(["--dev", "-d"]) + .arg(tmpdir.path()) + .arg("--database") + .arg("paritydb-experimental") + .arg("--no-hardware-benchmarks") + .spawn() + .unwrap(); + + let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); + + // Let it produce 1 block. + common::wait_n_finalized_blocks(1, &ws_url).await; + + // Send SIGINT to node. + kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT).unwrap(); + // Wait for the node to handle it and exit. + assert!(cmd.wait().unwrap().success()); + assert!(tmpdir.path().join("chains/rococo_dev").exists()); + assert!(tmpdir.path().join("chains/rococo_dev/paritydb/full").exists()); + assert!(tmpdir.path().join("chains/rococo_dev/paritydb/parachains").exists()); + + // Purge chain + let status = Command::new(cargo_bin("polkadot")) + .args(["purge-chain", "--dev", "-d"]) + .arg(tmpdir.path()) + .arg("--database") + .arg("paritydb-experimental") + .arg("-y") + .status() + .unwrap(); + assert!(status.success()); + + // Make sure that the chain folder exists, but `db/full` is deleted. + assert!(tmpdir.path().join("chains/rococo_dev").exists()); + assert!(!tmpdir.path().join("chains/rococo_dev/paritydb/full").exists()); + // Parachains removal requires calling "purge-chain --parachains". + assert!(tmpdir.path().join("chains/rococo_dev/paritydb/parachains").exists()); + }) + .await; } diff --git a/polkadot/tests/running_the_node_and_interrupt.rs b/polkadot/tests/running_the_node_and_interrupt.rs index d3935cbb02ad176e56e1a248b9410f71b714ec95..079c34e0421e8a1936359b03bc46f3fea87f4b8b 100644 --- a/polkadot/tests/running_the_node_and_interrupt.rs +++ b/polkadot/tests/running_the_node_and_interrupt.rs @@ -15,10 +15,7 @@ // along with Substrate. If not, see . use assert_cmd::cargo::cargo_bin; -use std::{ - process::{self, Command}, - time::Duration, -}; +use std::process::{self, Command}; use tempfile::tempdir; pub mod common; @@ -49,17 +46,13 @@ async fn running_the_node_works_and_can_be_interrupted() { let (ws_url, _) = common::find_ws_url_from_output(cmd.stderr.take().unwrap()); // Let it produce three blocks. - common::wait_n_finalized_blocks(3, Duration::from_secs(60), &ws_url) - .await - .unwrap(); + common::wait_n_finalized_blocks(3, &ws_url).await; assert!(cmd.try_wait().unwrap().is_none(), "the process should still be running"); kill(Pid::from_raw(cmd.id().try_into().unwrap()), signal).unwrap(); - assert_eq!( - common::wait_for(&mut cmd, 30).map(|x| x.success()), - Some(true), - "the process must exit gracefully after signal {}", - signal, + assert!( + cmd.wait().unwrap().success(), + "the process must exit gracefully after signal {signal}", ); } diff --git a/polkadot/utils/generate-bags/Cargo.toml b/polkadot/utils/generate-bags/Cargo.toml index 98a0a9b6a88595ddb0de7c8910e35cdcaa57f98e..d270cabd3f03a52e0a149e2af5a1a6a1d266bd7d 100644 --- a/polkadot/utils/generate-bags/Cargo.toml +++ b/polkadot/utils/generate-bags/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true license.workspace = true [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } generate-bags = { path = "../../../substrate/utils/frame/generate-bags" } sp-io = { path = "../../../substrate/primitives/io" } diff --git a/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml b/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml index bbdddf7506449570dd16e1c7bced038dc12d4e26..cb9547ac7dc6e4348c60adf80149772a6031df00 100644 --- a/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml +++ b/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml @@ -19,6 +19,6 @@ sp-tracing = { path = "../../../../substrate/primitives/tracing" } frame-system = { path = "../../../../substrate/frame/system" } sp-core = { path = "../../../../substrate/primitives/core" } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } log = "0.4.17" tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/utils/staking-miner/.gitignore b/polkadot/utils/staking-miner/.gitignore deleted file mode 100644 index db7cff848330b52bc5135e811ba222a2fc303a0b..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.key -*.bin diff --git a/polkadot/utils/staking-miner/Cargo.toml b/polkadot/utils/staking-miner/Cargo.toml deleted file mode 100644 index 09e73bc10f2d1828d224fe565bee444aa3cbd924..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/Cargo.toml +++ /dev/null @@ -1,54 +0,0 @@ -[[bin]] -name = "staging-staking-miner" -path = "src/main.rs" - -[package] -name = "staging-staking-miner" -version = "1.0.0" -authors.workspace = true -edition.workspace = true -license.workspace = true -publish = false - -[dependencies] -codec = { package = "parity-scale-codec", version = "3.6.1" } -clap = { version = "4.4.2", features = ["derive", "env"] } -tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } -jsonrpsee = { version = "0.16.2", features = ["ws-client", "macros"] } -log = "0.4.17" -paste = "1.0.7" -serde = "1.0.188" -serde_json = "1.0" -thiserror = "1.0.31" -tokio = { version = "1.24.2", features = ["macros", "rt-multi-thread", "sync"] } -remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } -signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] } -sp-core = { path = "../../../substrate/primitives/core" } -sp-version = { path = "../../../substrate/primitives/version" } -sp-state-machine = { path = "../../../substrate/primitives/state-machine" } -sp-runtime = { path = "../../../substrate/primitives/runtime" } -sp-npos-elections = { path = "../../../substrate/primitives/npos-elections" } -sc-transaction-pool-api = { path = "../../../substrate/client/transaction-pool/api" } - -frame-system = { path = "../../../substrate/frame/system" } -frame-support = { path = "../../../substrate/frame/support" } -frame-election-provider-support = { path = "../../../substrate/frame/election-provider-support" } -pallet-election-provider-multi-phase = { path = "../../../substrate/frame/election-provider-multi-phase" } -pallet-staking = { path = "../../../substrate/frame/staking" } -pallet-balances = { path = "../../../substrate/frame/balances" } -pallet-transaction-payment = { path = "../../../substrate/frame/transaction-payment" } - -core-primitives = { package = "polkadot-core-primitives", path = "../../core-primitives" } - -runtime-common = { package = "polkadot-runtime-common", path = "../../runtime/common" } -polkadot-runtime = { path = "../../runtime/polkadot" } -kusama-runtime = { package = "staging-kusama-runtime", path = "../../runtime/kusama" } -westend-runtime = { path = "../../runtime/westend" } -exitcode = "1.1" - -sub-tokens = { git = "https://github.com/paritytech/substrate-debug-kit", branch = "master" } -signal-hook = "0.3" -futures-util = "0.3" - -[dev-dependencies] -assert_cmd = "2.0.4" diff --git a/polkadot/utils/staking-miner/README.md b/polkadot/utils/staking-miner/README.md deleted file mode 100644 index 7e7254dc7759f3cf811922eb670a4ff9aa542cff..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Staking Miner - -Substrate chains validators compute a basic solution for the NPoS election. The optimization of the solution is computing-intensive and can be delegated to the `staking-miner`. The `staking-miner` does not act as validator and focuses solely on the optimization of the solution. - -The staking miner connects to a specified chain and keeps listening to new Signed phase of the [pallet-election-provider-multi-phase](https://crates.parity.io/pallet_election_provider_multi_phase/index.html) in order to submit solutions to the NPoS election. When the correct time comes, it computes its solution and submit it to the chain. -The default miner algorithm is [sequential-phragmen](https://crates.parity.io/sp_npos_elections/phragmen/fn.seq_phragmen_core.html)] with a configurable number of balancing iterations that improve the score. - -Running the staking-miner requires passing the seed of a funded account in order to pay the fees for the transactions that will be sent. The same account's balance is used to reserve deposits as well. The best solution in each round is rewarded. All correct solutions will get their bond back. Any invalid solution will lose their bond. - -You can check the help with: -``` -staking-miner --help -``` - -## Building - -You can build from the root of the Polkadot repository using: -``` -cargo build --profile production --locked --package staking-miner --bin staking-miner -``` - -## Docker - -There are 2 options to build a staking-miner Docker image: -- injected binary: the binary is first built on a Linux host and then injected into a Docker base image. This method only works if you have a Linux host or access to a pre-built binary from a Linux host. -- multi-stage: the binary is entirely built within the multi-stage Docker image. There is no requirement on the host in terms of OS and the host does not even need to have any Rust toolchain installed. - -### Building the injected image - -First build the binary as documented [above](#building). -You may then inject the binary into a Docker base image: `parity/base-bin` (running the command from the root of the Polkadot repository): -``` -TODO: UPDATE THAT -docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile target/release -``` - -### Building the multi-stage image - -Unlike the injected image that requires a Linux pre-built binary, this option does not requires a Linux host, nor Rust to be installed. -The trade-off however is that it takes a little longer to build and this option is less ideal for CI tasks. -You may build the multi-stage image the root of the Polkadot repository with: -``` -TODO: UPDATE THAT -docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . -``` - -### Running - -A Docker container, especially one holding one of your `SEED` should be kept as secure as possible. -While it won't prevent a malicious actor to read your `SEED` if they gain access to your container, it is nonetheless recommended running this container in `read-only` mode: - -``` -# The following line starts with an extra space on purpose: - SEED=0x1234... - -docker run --rm -i \ - --name staking-miner \ - --read-only \ - -e RUST_LOG=info \ - -e SEED=$SEED \ - -e URI=wss://your-node:9944 \ - staking-miner dry-run -``` - -### Test locally - -Make sure you've built Polkadot, then: - -1. `cargo run -p polkadot --features fast-runtime -- --chain polkadot-dev --tmp --alice -lruntime=debug` -2. `cargo run -p staking-miner -- --uri ws://localhost:9944 monitor --seed-or-path //Alice phrag-mms` diff --git a/polkadot/utils/staking-miner/src/dry_run.rs b/polkadot/utils/staking-miner/src/dry_run.rs deleted file mode 100644 index 7e46f630a1f5e581143a1c91c87823d23b934bd0..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/dry_run.rs +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The dry-run command. - -use crate::{opts::DryRunConfig, prelude::*, rpc::*, signer::Signer, Error, SharedRpcClient}; -use codec::Encode; -use frame_support::traits::Currency; -use sp_core::Bytes; -use sp_npos_elections::ElectionScore; - -/// Forcefully create the snapshot. This can be used to compute the election at anytime. -fn force_create_snapshot(ext: &mut Ext) -> Result<(), Error> { - ext.execute_with(|| { - if >::exists() { - log::info!(target: LOG_TARGET, "snapshot already exists."); - Ok(()) - } else { - log::info!(target: LOG_TARGET, "creating a fake snapshot now."); - >::create_snapshot().map(|_| ()).map_err(Into::into) - } - }) -} - -/// Helper method to print the encoded size of the snapshot. -async fn print_info( - rpc: &SharedRpcClient, - ext: &mut Ext, - raw_solution: &EPM::RawSolution>, - extrinsic: &Bytes, -) where - ::Currency: Currency, -{ - ext.execute_with(|| { - log::info!( - target: LOG_TARGET, - "Snapshot Metadata: {:?}", - >::snapshot_metadata() - ); - log::info!( - target: LOG_TARGET, - "Snapshot Encoded Length: {:?}", - >::snapshot() - .expect("snapshot must exist before calling `measure_snapshot_size`") - .encode() - .len() - ); - - let snapshot_size = - >::snapshot_metadata().expect("snapshot must exist by now; qed."); - let deposit = EPM::Pallet::::deposit_for(raw_solution, snapshot_size); - - let score = { - let ElectionScore { minimal_stake, sum_stake, sum_stake_squared } = raw_solution.score; - [Token::from(minimal_stake), Token::from(sum_stake), Token::from(sum_stake_squared)] - }; - - log::info!( - target: LOG_TARGET, - "solution score {:?} / deposit {:?} / length {:?}", - score, - Token::from(deposit), - raw_solution.encode().len(), - ); - }); - - let info = rpc.payment_query_info(&extrinsic, None).await; - - log::info!( - target: LOG_TARGET, - "payment_queryInfo: (fee = {}) {:?}", - info.as_ref() - .map(|d| Token::from(d.partial_fee)) - .unwrap_or_else(|_| Token::from(0)), - info, - ); -} - -/// Find the stake threshold in order to have at most `count` voters. -#[allow(unused)] -fn find_threshold(ext: &mut Ext, count: usize) { - ext.execute_with(|| { - let mut voters = >::snapshot() - .expect("snapshot must exist before calling `measure_snapshot_size`") - .voters; - voters.sort_by_key(|(_voter, weight, _targets)| std::cmp::Reverse(*weight)); - match voters.get(count) { - Some(threshold_voter) => println!("smallest allowed voter is {:?}", threshold_voter), - None => { - println!("requested truncation to {} voters but had only {}", count, voters.len()); - println!("smallest current voter: {:?}", voters.last()); - }, - } - }) -} - -macro_rules! dry_run_cmd_for { ($runtime:ident) => { paste::paste! { - /// Execute the dry-run command. - pub(crate) async fn []( - rpc: SharedRpcClient, - config: DryRunConfig, - signer: Signer, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - let pallets = if config.force_snapshot { - vec!["Staking".to_string(), "BagsList".to_string()] - } else { - Default::default() - }; - let mut ext = crate::create_election_ext::(rpc.clone(), config.at, pallets).await?; - if config.force_snapshot { - force_create_snapshot::(&mut ext)?; - }; - - log::debug!(target: LOG_TARGET, "solving with {:?}", config.solver); - let raw_solution = crate::mine_with::(&config.solver, &mut ext, false)?; - - let nonce = crate::get_account_info::(&rpc, &signer.account, config.at) - .await? - .map(|i| i.nonce) - .expect("signer account is checked to exist upon startup; it can only die if it \ - transfers funds out of it, or get slashed. If it does not exist at this point, \ - it is likely due to a bug, or the signer got slashed. Terminating." - ); - let tip = 0 as Balance; - let era = sp_runtime::generic::Era::Immortal; - let extrinsic = ext.execute_with(|| create_uxt(raw_solution.clone(), signer.clone(), nonce, tip, era)); - - let bytes = sp_core::Bytes(extrinsic.encode().to_vec()); - print_info::(&rpc, &mut ext, &raw_solution, &bytes).await; - - let feasibility_result = ext.execute_with(|| { - EPM::Pallet::::feasibility_check(raw_solution.clone(), EPM::ElectionCompute::Signed) - }); - log::info!(target: LOG_TARGET, "feasibility result is {:?}", feasibility_result.map(|_| ())); - - let dispatch_result = ext.execute_with(|| { - // manually tweak the phase. - EPM::CurrentPhase::::put(EPM::Phase::Signed); - EPM::Pallet::::submit(frame_system::RawOrigin::Signed(signer.account).into(), Box::new(raw_solution)) - }); - log::info!(target: LOG_TARGET, "dispatch result is {:?}", dispatch_result); - - let dry_run_fut = rpc.dry_run(&bytes, None); - let outcome: sp_runtime::ApplyExtrinsicResult = await_request_and_decode(dry_run_fut).await.map_err::, _>(Into::into)?; - log::info!(target: LOG_TARGET, "dry-run outcome is {:?}", outcome); - Ok(()) - } -}}} - -dry_run_cmd_for!(polkadot); -dry_run_cmd_for!(kusama); -dry_run_cmd_for!(westend); diff --git a/polkadot/utils/staking-miner/src/emergency_solution.rs b/polkadot/utils/staking-miner/src/emergency_solution.rs deleted file mode 100644 index 9ea9f90756e22f458fc1c4e6872d77b7a246282b..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/emergency_solution.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The emergency-solution command. - -use crate::{prelude::*, EmergencySolutionConfig, Error, SharedRpcClient}; -use codec::Encode; -use std::io::Write; - -macro_rules! emergency_solution_cmd_for { ($runtime:ident) => { paste::paste! { - /// Execute the emergency-solution command. - pub(crate) async fn []( - client: SharedRpcClient, - config: EmergencySolutionConfig, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - - let mut ext = crate::create_election_ext::(client, config.at, vec![]).await?; - let raw_solution = crate::mine_with::(&config.solver, &mut ext, false)?; - - ext.execute_with(|| { - assert!(EPM::Pallet::::current_phase().is_emergency()); - - log::info!(target: LOG_TARGET, "mined solution with {:?}", &raw_solution.score); - - let ready_solution = EPM::Pallet::::feasibility_check(raw_solution, EPM::ElectionCompute::Signed)?; - let encoded_size = ready_solution.encoded_size(); - let score = ready_solution.score; - let mut supports = ready_solution.supports.into_inner(); - // maybe truncate. - if let Some(take) = config.take { - log::info!(target: LOG_TARGET, "truncating {} winners to {}", supports.len(), take); - supports.sort_unstable_by_key(|(_, s)| s.total); - supports.truncate(take); - } - - // write to file and stdout. - let encoded_support = supports.encode(); - let mut supports_file = std::fs::File::create("solution.supports.bin")?; - supports_file.write_all(&encoded_support)?; - - log::info!(target: LOG_TARGET, "ReadySolution: size {:?} / score = {:?}", encoded_size, score); - log::trace!(target: LOG_TARGET, "Supports: {}", sp_core::hexdisplay::HexDisplay::from(&encoded_support)); - - Ok(()) - }) - } -}}} - -emergency_solution_cmd_for!(polkadot); -emergency_solution_cmd_for!(kusama); -emergency_solution_cmd_for!(westend); diff --git a/polkadot/utils/staking-miner/src/main.rs b/polkadot/utils/staking-miner/src/main.rs deleted file mode 100644 index 90b2c7366a1bab6b1a742d0274f872341e3c3a9a..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/main.rs +++ /dev/null @@ -1,665 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! # Polkadot Staking Miner. -//! -//! Simple bot capable of monitoring a polkadot (and cousins) chain and submitting solutions to the -//! `pallet-election-provider-multi-phase`. See `--help` for more details. -//! -//! # Implementation Notes: -//! -//! - First draft: Be aware that this is the first draft and there might be bugs, or undefined -//! behaviors. Don't attach this bot to an account with lots of funds. -//! - Quick to crash: The bot is written so that it only continues to work if everything goes well. -//! In case of any failure (RPC, logic, IO), it will crash. This was a decision to simplify the -//! development. It is intended to run this bot with a `restart = true` way, so that it reports it -//! crash, but resumes work thereafter. - -// Silence erroneous warning about unsafe not being required whereas it is -// see https://github.com/rust-lang/rust/issues/49112 -#![allow(unused_unsafe)] - -mod dry_run; -mod emergency_solution; -mod monitor; -mod opts; -mod prelude; -mod rpc; -mod runtime_versions; -mod signer; - -pub(crate) use prelude::*; -pub(crate) use signer::get_account_info; - -use crate::opts::*; -use clap::Parser; -use frame_election_provider_support::NposSolver; -use frame_support::traits::Get; -use futures_util::StreamExt; -use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; -use remote_externalities::{Builder, Mode, OnlineConfig, Transport}; -use rpc::{RpcApiClient, SharedRpcClient}; -use runtime_versions::RuntimeVersions; -use signal_hook::consts::signal::*; -use signal_hook_tokio::Signals; -use sp_npos_elections::BalancingConfig; -use std::{ops::Deref, sync::Arc, time::Duration}; -use tracing_subscriber::{fmt, EnvFilter}; - -pub(crate) enum AnyRuntime { - Polkadot, - Kusama, - Westend, -} - -pub(crate) static mut RUNTIME: AnyRuntime = AnyRuntime::Polkadot; - -macro_rules! construct_runtime_prelude { - ($runtime:ident) => { paste::paste! { - pub(crate) mod [<$runtime _runtime_exports>] { - pub(crate) use crate::prelude::EPM; - pub(crate) use [<$runtime _runtime>]::*; - pub(crate) use crate::monitor::[] as monitor_cmd; - pub(crate) use crate::dry_run::[] as dry_run_cmd; - pub(crate) use crate::emergency_solution::[] as emergency_solution_cmd; - pub(crate) use private::{[] as create_uxt}; - - mod private { - use super::*; - pub(crate) fn []( - raw_solution: EPM::RawSolution>, - signer: crate::signer::Signer, - nonce: crate::prelude::Nonce, - tip: crate::prelude::Balance, - era: sp_runtime::generic::Era, - ) -> UncheckedExtrinsic { - use codec::Encode as _; - use sp_core::Pair as _; - use sp_runtime::traits::StaticLookup as _; - - let crate::signer::Signer { account, pair, .. } = signer; - - let local_call = EPMCall::::submit { raw_solution: Box::new(raw_solution) }; - let call: RuntimeCall = as std::convert::TryInto>::try_into(local_call) - .expect("election provider pallet must exist in the runtime, thus \ - inner call can be converted, qed." - ); - - let extra: SignedExtra = crate::[](nonce, tip, era); - let raw_payload = SignedPayload::new(call, extra).expect("creating signed payload infallible; qed."); - let signature = raw_payload.using_encoded(|payload| { - pair.sign(payload) - }); - let (call, extra, _) = raw_payload.deconstruct(); - let address = ::Lookup::unlookup(account); - let extrinsic = UncheckedExtrinsic::new_signed(call, address, signature.into(), extra); - log::debug!( - target: crate::LOG_TARGET, "constructed extrinsic {} with length {}", - sp_core::hexdisplay::HexDisplay::from(&extrinsic.encode()), - extrinsic.encode().len(), - ); - extrinsic - } - } - }} - }; -} - -// NOTE: we might be able to use some code from the bridges repo here. -fn signed_ext_builder_polkadot( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> polkadot_runtime_exports::SignedExtra { - use polkadot_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - runtime_common::claims::PrevalidateAttests::::new(), - ) -} - -fn signed_ext_builder_kusama( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> kusama_runtime_exports::SignedExtra { - use kusama_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - ) -} - -fn signed_ext_builder_westend( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> westend_runtime_exports::SignedExtra { - use westend_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - ) -} - -construct_runtime_prelude!(polkadot); -construct_runtime_prelude!(kusama); -construct_runtime_prelude!(westend); - -// NOTE: this is no longer used extensively, most of the per-runtime stuff us delegated to -// `construct_runtime_prelude` and macro's the import directly from it. A part of the code is also -// still generic over `T`. My hope is to still make everything generic over a `Runtime`, but sadly -// that is not currently possible as each runtime has its unique `Call`, and all Calls are not -// sharing any generic trait. In other words, to create the `UncheckedExtrinsic` of each chain, you -// need the concrete `Call` of that chain as well. -#[macro_export] -macro_rules! any_runtime { - ($($code:tt)*) => { - unsafe { - match $crate::RUNTIME { - $crate::AnyRuntime::Polkadot => { - #[allow(unused)] - use $crate::polkadot_runtime_exports::*; - $($code)* - }, - $crate::AnyRuntime::Kusama => { - #[allow(unused)] - use $crate::kusama_runtime_exports::*; - $($code)* - }, - $crate::AnyRuntime::Westend => { - #[allow(unused)] - use $crate::westend_runtime_exports::*; - $($code)* - } - } - } - } -} - -/// Same as [`any_runtime`], but instead of returning a `Result`, this simply returns `()`. Useful -/// for situations where the result is not useful and un-ergonomic to handle. -#[macro_export] -macro_rules! any_runtime_unit { - ($($code:tt)*) => { - unsafe { - match $crate::RUNTIME { - $crate::AnyRuntime::Polkadot => { - #[allow(unused)] - use $crate::polkadot_runtime_exports::*; - let _ = $($code)*; - }, - $crate::AnyRuntime::Kusama => { - #[allow(unused)] - use $crate::kusama_runtime_exports::*; - let _ = $($code)*; - }, - $crate::AnyRuntime::Westend => { - #[allow(unused)] - use $crate::westend_runtime_exports::*; - let _ = $($code)*; - } - } - } - } -} - -#[derive(frame_support::DebugNoBound, thiserror::Error)] -enum Error { - Io(#[from] std::io::Error), - JsonRpsee(#[from] jsonrpsee::core::Error), - RpcHelperError(#[from] rpc::RpcHelperError), - Codec(#[from] codec::Error), - Crypto(sp_core::crypto::SecretStringError), - RemoteExternalities(&'static str), - PalletMiner(EPM::unsigned::MinerError), - PalletElection(EPM::ElectionError), - PalletFeasibility(EPM::FeasibilityError), - AccountDoesNotExists, - IncorrectPhase, - AlreadySubmitted, - VersionMismatch, - StrategyNotSatisfied, - Other(String), -} - -impl From for Error { - fn from(e: sp_core::crypto::SecretStringError) -> Error { - Error::Crypto(e) - } -} - -impl From for Error { - fn from(e: EPM::unsigned::MinerError) -> Error { - Error::PalletMiner(e) - } -} - -impl From> for Error { - fn from(e: EPM::ElectionError) -> Error { - Error::PalletElection(e) - } -} - -impl From for Error { - fn from(e: EPM::FeasibilityError) -> Error { - Error::PalletFeasibility(e) - } -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - as std::fmt::Debug>::fmt(self, f) - } -} - -frame_support::parameter_types! { - /// Number of balancing iterations for a solution algorithm. Set based on the [`Solvers`] CLI - /// config. - pub static BalanceIterations: usize = 10; - pub static Balancing: Option = Some( BalancingConfig { iterations: BalanceIterations::get(), tolerance: 0 } ); -} - -/// Build the Ext at hash with all the data of `ElectionProviderMultiPhase` and any additional -/// pallets. -async fn create_election_ext( - client: SharedRpcClient, - at: Option, - additional: Vec, -) -> Result> -where - T: EPM::Config, -{ - use frame_support::{storage::generator::StorageMap, traits::PalletInfo}; - use sp_core::hashing::twox_128; - - let mut pallets = vec![::PalletInfo::name::>() - .expect("Pallet always has name; qed.") - .to_string()]; - pallets.extend(additional); - Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: Transport::Uri(client.uri().to_owned()), - at, - pallets, - hashed_prefixes: vec![>::prefix_hash()], - hashed_keys: vec![[twox_128(b"System"), twox_128(b"Number")].concat()], - ..Default::default() - })) - .build() - .await - .map_err(|why| Error::::RemoteExternalities(why)) - .map(|rx| rx.inner_ext) -} - -/// Compute the election. It expects to NOT be `Phase::Off`. In other words, the snapshot must -/// exists on the given externalities. -fn mine_solution( - ext: &mut Ext, - do_feasibility: bool, -) -> Result>, Error> -where - T: EPM::Config, - S: NposSolver< - Error = <::Solver as NposSolver>::Error, - AccountId = <::Solver as NposSolver>::AccountId, - >, -{ - ext.execute_with(|| { - let (solution, _) = >::mine_solution().map_err::, _>(Into::into)?; - if do_feasibility { - let _ = >::feasibility_check( - solution.clone(), - EPM::ElectionCompute::Signed, - )?; - } - Ok(solution) - }) -} - -/// Mine a solution with the given `solver`. -fn mine_with( - solver: &Solver, - ext: &mut Ext, - do_feasibility: bool, -) -> Result>, Error> -where - T: EPM::Config, - T::Solver: NposSolver, -{ - use frame_election_provider_support::{PhragMMS, SequentialPhragmen}; - - match solver { - Solver::SeqPhragmen { iterations } => { - BalanceIterations::set(*iterations); - mine_solution::< - T, - SequentialPhragmen< - ::AccountId, - sp_runtime::Perbill, - Balancing, - >, - >(ext, do_feasibility) - }, - Solver::PhragMMS { iterations } => { - BalanceIterations::set(*iterations); - mine_solution::< - T, - PhragMMS<::AccountId, sp_runtime::Perbill, Balancing>, - >(ext, do_feasibility) - }, - } -} - -#[allow(unused)] -fn mine_dpos(ext: &mut Ext) -> Result<(), Error> { - ext.execute_with(|| { - use std::collections::BTreeMap; - use EPM::RoundSnapshot; - let RoundSnapshot { voters, .. } = EPM::Snapshot::::get().unwrap(); - let desired_targets = EPM::DesiredTargets::::get().unwrap(); - let mut candidates_and_backing = BTreeMap::::new(); - voters.into_iter().for_each(|(who, stake, targets)| { - if targets.is_empty() { - println!("target = {:?}", (who, stake, targets)); - return - } - let share: u128 = (stake as u128) / (targets.len() as u128); - for target in targets { - *candidates_and_backing.entry(target.clone()).or_default() += share - } - }); - - let mut candidates_and_backing = - candidates_and_backing.into_iter().collect::>(); - candidates_and_backing.sort_by_key(|(_, total_stake)| *total_stake); - let winners = candidates_and_backing - .into_iter() - .rev() - .take(desired_targets as usize) - .collect::>(); - let score = { - let min_staker = *winners.last().map(|(_, stake)| stake).unwrap(); - let sum_stake = winners.iter().fold(0u128, |acc, (_, stake)| acc + stake); - let sum_squared = winners.iter().fold(0u128, |acc, (_, stake)| acc + stake); - [min_staker, sum_stake, sum_squared] - }; - println!("mined a dpos-like solution with score = {:?}", score); - Ok(()) - }) -} - -pub(crate) async fn check_versions( - rpc: &SharedRpcClient, - print: bool, -) -> Result<(), Error> { - let linked_version = T::Version::get(); - let on_chain_version = rpc - .runtime_version(None) - .await - .expect("runtime version RPC should always work; qed"); - - let do_print = || { - log::info!( - target: LOG_TARGET, - "linked version {:?}", - (&linked_version.spec_name, &linked_version.spec_version) - ); - log::info!( - target: LOG_TARGET, - "on-chain version {:?}", - (&on_chain_version.spec_name, &on_chain_version.spec_version) - ); - }; - - if print { - do_print(); - } - - // we relax the checking here a bit, which should not cause any issues in production (a chain - // that messes up its spec name is highly unlikely), but it allows us to do easier testing. - if linked_version.spec_name != on_chain_version.spec_name || - linked_version.spec_version != on_chain_version.spec_version - { - if !print { - do_print(); - } - log::error!( - target: LOG_TARGET, - "VERSION MISMATCH: any transaction will fail with bad-proof" - ); - Err(Error::VersionMismatch) - } else { - Ok(()) - } -} - -/// Control how we exit the application -fn controlled_exit(code: i32) { - log::info!(target: LOG_TARGET, "Exiting application"); - std::process::exit(code); -} - -/// Handles the various signal and exit the application -/// when appropriate. -async fn handle_signals(mut signals: Signals) { - let mut keyboard_sig_count: u8 = 0; - while let Some(signal) = signals.next().await { - match signal { - // Interrupts come from the keyboard - SIGQUIT | SIGINT => { - if keyboard_sig_count >= 1 { - log::info!( - target: LOG_TARGET, - "Received keyboard termination signal #{}/{}, quitting...", - keyboard_sig_count + 1, - 2 - ); - controlled_exit(exitcode::OK); - } - keyboard_sig_count += 1; - log::warn!( - target: LOG_TARGET, - "Received keyboard termination signal #{}, if you keep doing that I will really quit", - keyboard_sig_count - ); - }, - - SIGKILL | SIGTERM => { - log::info!(target: LOG_TARGET, "Received SIGKILL | SIGTERM, quitting..."); - controlled_exit(exitcode::OK); - }, - _ => unreachable!(), - } - } -} - -#[tokio::main] -async fn main() { - fmt().with_env_filter(EnvFilter::from_default_env()).init(); - - let Opt { uri, command, connection_timeout, request_timeout } = Opt::parse(); - log::debug!(target: LOG_TARGET, "attempting to connect to {:?}", uri); - - let signals = Signals::new(&[SIGTERM, SIGINT, SIGQUIT]).expect("Failed initializing Signals"); - let handle = signals.handle(); - let signals_task = tokio::spawn(handle_signals(signals)); - - let rpc = loop { - match SharedRpcClient::new( - &uri, - Duration::from_secs(connection_timeout as u64), - Duration::from_secs(request_timeout as u64), - ) - .await - { - Ok(client) => break client, - Err(why) => { - log::warn!( - target: LOG_TARGET, - "failed to connect to client due to {:?}, retrying soon..", - why - ); - tokio::time::sleep(std::time::Duration::from_millis(2500)).await; - }, - } - }; - - let chain: String = rpc.system_chain().await.expect("system_chain infallible; qed."); - match chain.to_lowercase().as_str() { - "polkadot" | "development" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::PolkadotAccount.into(), - ); - sub_tokens::dynamic::set_name("DOT"); - sub_tokens::dynamic::set_decimal_points(10_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Polkadot; - } - }, - "kusama" | "kusama-dev" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::KusamaAccount.into(), - ); - sub_tokens::dynamic::set_name("KSM"); - sub_tokens::dynamic::set_decimal_points(1_000_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Kusama; - } - }, - "westend" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::PolkadotAccount.into(), - ); - sub_tokens::dynamic::set_name("WND"); - sub_tokens::dynamic::set_decimal_points(1_000_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Westend; - } - }, - _ => { - eprintln!("unexpected chain: {:?}", chain); - return - }, - } - log::info!(target: LOG_TARGET, "connected to chain {:?}", chain); - - any_runtime_unit! { - check_versions::(&rpc, true).await - }; - - let outcome = any_runtime! { - match command { - Command::Monitor(monitor_config) => - { - let signer_account = any_runtime! { - signer::signer_uri_from_string::(&monitor_config.seed_or_path , &rpc) - .await - .expect("Provided account is invalid, terminating.") - }; - monitor_cmd(rpc, monitor_config, signer_account).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "Monitor error: {:?}", e); - })}, - Command::DryRun(dryrun_config) => { - let signer_account = any_runtime! { - signer::signer_uri_from_string::(&dryrun_config.seed_or_path , &rpc) - .await - .expect("Provided account is invalid, terminating.") - }; - dry_run_cmd(rpc, dryrun_config, signer_account).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "DryRun error: {:?}", e); - })}, - Command::EmergencySolution(emergency_solution_config) => - emergency_solution_cmd(rpc, emergency_solution_config).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "EmergencySolution error: {:?}", e); - }), - Command::Info(info_opts) => { - let remote_runtime_version = rpc.runtime_version(None).await.expect("runtime_version infallible; qed."); - - let builtin_version = any_runtime! { - Version::get() - }; - - let versions = RuntimeVersions::new(&remote_runtime_version, &builtin_version); - - if !info_opts.json { - println!("{}", versions); - } else { - let versions = serde_json::to_string_pretty(&versions).expect("Failed serializing version info"); - println!("{}", versions); - } - Ok(()) - } - } - }; - log::info!(target: LOG_TARGET, "round of execution finished. outcome = {:?}", outcome); - - handle.close(); - let _ = signals_task.await; -} - -#[cfg(test)] -mod tests { - use super::*; - - fn get_version() -> sp_version::RuntimeVersion { - T::Version::get() - } - - #[test] - fn any_runtime_works() { - unsafe { - RUNTIME = AnyRuntime::Polkadot; - } - let polkadot_version = any_runtime! { get_version::() }; - - unsafe { - RUNTIME = AnyRuntime::Kusama; - } - let kusama_version = any_runtime! { get_version::() }; - - assert_eq!(polkadot_version.spec_name, "polkadot".into()); - assert_eq!(kusama_version.spec_name, "kusama".into()); - } -} diff --git a/polkadot/utils/staking-miner/src/monitor.rs b/polkadot/utils/staking-miner/src/monitor.rs deleted file mode 100644 index 607ecb6baa42c9b24a2453044271a6446b16d285..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/monitor.rs +++ /dev/null @@ -1,478 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The monitor command. - -use crate::{ - prelude::*, rpc::*, signer::Signer, Error, MonitorConfig, SharedRpcClient, SubmissionStrategy, -}; -use codec::Encode; -use jsonrpsee::core::Error as RpcError; -use sc_transaction_pool_api::TransactionStatus; -use sp_core::storage::StorageKey; -use sp_runtime::Perbill; -use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; -use EPM::{signed::SubmissionIndicesOf, SignedSubmissionOf}; - -/// Ensure that now is the signed phase. -async fn ensure_signed_phase>( - rpc: &SharedRpcClient, - at: B::Hash, -) -> Result<(), Error> { - let key = StorageKey(EPM::CurrentPhase::::hashed_key().to_vec()); - let phase = rpc - .get_storage_and_decode::>(&key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - if phase.is_signed() { - Ok(()) - } else { - Err(Error::IncorrectPhase) - } -} - -/// Ensure that our current `us` have not submitted anything previously. -async fn ensure_no_previous_solution( - rpc: &SharedRpcClient, - at: Hash, - us: &AccountId, -) -> Result<(), Error> -where - T: EPM::Config + frame_system::Config, - B: BlockT, -{ - let indices_key = StorageKey(EPM::SignedSubmissionIndices::::hashed_key().to_vec()); - - let indices: SubmissionIndicesOf = rpc - .get_storage_and_decode(&indices_key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - for (_score, _bn, idx) in indices { - let key = StorageKey(EPM::SignedSubmissionsMap::::hashed_key_for(idx)); - - if let Some(submission) = rpc - .get_storage_and_decode::>(&key, Some(at)) - .await - .map_err::, _>(Into::into)? - { - if &submission.who == us { - return Err(Error::AlreadySubmitted) - } - } - } - - Ok(()) -} - -/// `true` if `our_score` should pass the onchain `best_score` with the given strategy. -pub(crate) fn score_passes_strategy( - our_score: sp_npos_elections::ElectionScore, - best_score: sp_npos_elections::ElectionScore, - strategy: SubmissionStrategy, -) -> bool { - match strategy { - SubmissionStrategy::Always => true, - SubmissionStrategy::IfLeading => - our_score == best_score || - our_score.strict_threshold_better(best_score, Perbill::zero()), - SubmissionStrategy::ClaimBetterThan(epsilon) => - our_score.strict_threshold_better(best_score, epsilon), - SubmissionStrategy::ClaimNoWorseThan(epsilon) => - !best_score.strict_threshold_better(our_score, epsilon), - } -} - -/// Reads all current solutions and checks the scores according to the `SubmissionStrategy`. -async fn ensure_strategy_met( - rpc: &SharedRpcClient, - at: Hash, - score: sp_npos_elections::ElectionScore, - strategy: SubmissionStrategy, - max_submissions: u32, -) -> Result<(), Error> { - // don't care about current scores. - if matches!(strategy, SubmissionStrategy::Always) { - return Ok(()) - } - - let indices_key = StorageKey(EPM::SignedSubmissionIndices::::hashed_key().to_vec()); - - let indices: SubmissionIndicesOf = rpc - .get_storage_and_decode(&indices_key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - if indices.len() >= max_submissions as usize { - log::debug!(target: LOG_TARGET, "The submissions queue is full"); - } - - // default score is all zeros, any score is better than it. - let best_score = indices.last().map(|(score, _, _)| *score).unwrap_or_default(); - log::debug!(target: LOG_TARGET, "best onchain score is {:?}", best_score); - - if score_passes_strategy(score, best_score, strategy) { - Ok(()) - } else { - Err(Error::StrategyNotSatisfied) - } -} - -async fn get_latest_head( - rpc: &SharedRpcClient, - mode: &str, -) -> Result> { - if mode == "head" { - match rpc.block_hash(None).await { - Ok(Some(hash)) => Ok(hash), - Ok(None) => Err(Error::Other("Best head not found".into())), - Err(e) => Err(e.into()), - } - } else { - rpc.finalized_head().await.map_err(Into::into) - } -} - -macro_rules! monitor_cmd_for { ($runtime:tt) => { paste::paste! { - - /// The monitor command. - pub(crate) async fn []( - rpc: SharedRpcClient, - config: MonitorConfig, - signer: Signer, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - type StakingMinerError = Error<$crate::[<$runtime _runtime_exports>]::Runtime>; - - let heads_subscription = || - if config.listen == "head" { - rpc.subscribe_new_heads() - } else { - rpc.subscribe_finalized_heads() - }; - - let mut subscription = heads_subscription().await?; - let (tx, mut rx) = mpsc::unbounded_channel::(); - let submit_lock = Arc::new(Mutex::new(())); - - loop { - let at = tokio::select! { - maybe_rp = subscription.next() => { - match maybe_rp { - Some(Ok(r)) => r, - Some(Err(e)) => { - log::error!(target: LOG_TARGET, "subscription failed to decode Header {:?}, this is bug please file an issue", e); - return Err(e.into()); - } - // The subscription was dropped, should only happen if: - // - the connection was closed. - // - the subscription could not keep up with the server. - None => { - log::warn!(target: LOG_TARGET, "subscription to `subscribeNewHeads/subscribeFinalizedHeads` terminated. Retrying.."); - subscription = heads_subscription().await?; - continue - } - } - }, - maybe_err = rx.recv() => { - match maybe_err { - Some(err) => return Err(err), - None => unreachable!("at least one sender kept in the main loop should always return Some; qed"), - } - } - }; - - // Spawn task and non-recoverable errors are sent back to the main task - // such as if the connection has been closed. - tokio::spawn( - send_and_watch_extrinsic(rpc.clone(), tx.clone(), at, signer.clone(), config.clone(), submit_lock.clone()) - ); - } - - /// Construct extrinsic at given block and watch it. - async fn send_and_watch_extrinsic( - rpc: SharedRpcClient, - tx: mpsc::UnboundedSender, - at: Header, - signer: Signer, - config: MonitorConfig, - submit_lock: Arc>, - ) { - - async fn flatten( - handle: tokio::task::JoinHandle> - ) -> Result { - match handle.await { - Ok(Ok(result)) => Ok(result), - Ok(Err(err)) => Err(err), - Err(err) => panic!("tokio spawn task failed; kill task: {:?}", err), - } - } - - let hash = at.hash(); - log::trace!(target: LOG_TARGET, "new event at #{:?} ({:?})", at.number, hash); - - // block on this because if this fails there is no way to recover from - // that error i.e, upgrade/downgrade required. - if let Err(err) = crate::check_versions::(&rpc, false).await { - let _ = tx.send(err.into()); - return; - } - - let rpc1 = rpc.clone(); - let rpc2 = rpc.clone(); - let account = signer.account.clone(); - - let signed_phase_fut = tokio::spawn(async move { - ensure_signed_phase::(&rpc1, hash).await - }); - - tokio::time::sleep(std::time::Duration::from_secs(config.delay as u64)).await; - - let no_prev_sol_fut = tokio::spawn(async move { - ensure_no_previous_solution::(&rpc2, hash, &account).await - }); - - // Run the calls in parallel and return once all has completed or any failed. - if let Err(err) = tokio::try_join!(flatten(signed_phase_fut), flatten(no_prev_sol_fut)) { - log::debug!(target: LOG_TARGET, "Skipping block {}; {}", at.number, err); - return; - } - - let _lock = submit_lock.lock().await; - - let mut ext = match crate::create_election_ext::(rpc.clone(), Some(hash), vec![]).await { - Ok(ext) => ext, - Err(err) => { - log::debug!(target: LOG_TARGET, "Skipping block {}; {}", at.number, err); - return; - } - }; - - // mine a solution, and run feasibility check on it as well. - let raw_solution = match crate::mine_with::(&config.solver, &mut ext, true) { - Ok(r) => r, - Err(err) => { - let _ = tx.send(err.into()); - return; - } - }; - - let score = raw_solution.score; - log::info!(target: LOG_TARGET, "mined solution with {:?}", score); - - let nonce = match crate::get_account_info::(&rpc, &signer.account, Some(hash)).await { - Ok(maybe_account) => { - let acc = maybe_account.expect(crate::signer::SIGNER_ACCOUNT_WILL_EXIST); - acc.nonce - } - Err(err) => { - let _ = tx.send(err); - return; - } - }; - - let tip = 0 as Balance; - let period = ::BlockHashCount::get() / 2; - let current_block = at.number.saturating_sub(1); - let era = sp_runtime::generic::Era::mortal(period.into(), current_block.into()); - - log::trace!( - target: LOG_TARGET, "transaction mortality: {:?} -> {:?}", - era.birth(current_block.into()), - era.death(current_block.into()), - ); - - let extrinsic = ext.execute_with(|| create_uxt(raw_solution, signer.clone(), nonce, tip, era)); - let bytes = sp_core::Bytes(extrinsic.encode()); - - let rpc1 = rpc.clone(); - let rpc2 = rpc.clone(); - let rpc3 = rpc.clone(); - - let latest_head = match get_latest_head::(&rpc, &config.listen).await { - Ok(hash) => hash, - Err(e) => { - log::debug!(target: LOG_TARGET, "Skipping to submit at block {}; {}", at.number, e); - return; - } - }; - - let ensure_strategy_met_fut = tokio::spawn(async move { - ensure_strategy_met::( - &rpc1, - latest_head, - score, - config.submission_strategy, - SignedMaxSubmissions::get() - ).await - }); - - let ensure_signed_phase_fut = tokio::spawn(async move { - ensure_signed_phase::(&rpc2, latest_head).await - }); - - let account = signer.account.clone(); - let no_prev_sol_fut = tokio::spawn(async move { - ensure_no_previous_solution::(&rpc3, latest_head, &account).await - }); - - // Run the calls in parallel and return once all has completed or any failed. - if let Err(err) = tokio::try_join!( - flatten(ensure_strategy_met_fut), - flatten(ensure_signed_phase_fut), - flatten(no_prev_sol_fut), - ) { - log::debug!(target: LOG_TARGET, "Skipping to submit at block {}; {}", at.number, err); - return; - } - - let mut tx_subscription = match rpc.watch_extrinsic(&bytes).await { - Ok(sub) => sub, - Err(RpcError::RestartNeeded(e)) => { - let _ = tx.send(RpcError::RestartNeeded(e).into()); - return - }, - Err(why) => { - // This usually happens when we've been busy with mining for a few blocks, and - // now we're receiving the subscriptions of blocks in which we were busy. In - // these blocks, we still don't have a solution, so we re-compute a new solution - // and submit it with an outdated `Nonce`, which yields most often `Stale` - // error. NOTE: to improve this overall, and to be able to introduce an array of - // other fancy features, we should make this multi-threaded and do the - // computation outside of this callback. - log::warn!( - target: LOG_TARGET, - "failing to submit a transaction {:?}. ignore block: {}", - why, at.number - ); - return; - }, - }; - - while let Some(rp) = tx_subscription.next().await { - let status_update = match rp { - Ok(r) => r, - Err(e) => { - log::error!(target: LOG_TARGET, "subscription failed to decode TransactionStatus {:?}, this is a bug please file an issue", e); - let _ = tx.send(e.into()); - return; - }, - }; - - log::trace!(target: LOG_TARGET, "status update {:?}", status_update); - match status_update { - TransactionStatus::Ready | - TransactionStatus::Broadcast(_) | - TransactionStatus::Future => continue, - TransactionStatus::InBlock((hash, _)) => { - log::info!(target: LOG_TARGET, "included at {:?}", hash); - let key = StorageKey( - frame_support::storage::storage_prefix(b"System", b"Events").to_vec(), - ); - - let events = match rpc.get_storage_and_decode::< - Vec::Hash>>, - >(&key, Some(hash)) - .await { - Ok(rp) => rp.unwrap_or_default(), - Err(RpcHelperError::JsonRpsee(RpcError::RestartNeeded(e))) => { - let _ = tx.send(RpcError::RestartNeeded(e).into()); - return; - } - // Decoding or other RPC error => just terminate the task. - Err(e) => { - log::warn!(target: LOG_TARGET, "get_storage [key: {:?}, hash: {:?}] failed: {:?}; skip block: {}", - key, hash, e, at.number - ); - return; - } - }; - - log::info!(target: LOG_TARGET, "events at inclusion {:?}", events); - }, - TransactionStatus::Retracted(hash) => { - log::info!(target: LOG_TARGET, "Retracted at {:?}", hash); - }, - TransactionStatus::Finalized((hash, _)) => { - log::info!(target: LOG_TARGET, "Finalized at {:?}", hash); - break - }, - _ => { - log::warn!( - target: LOG_TARGET, - "Stopping listen due to other status {:?}", - status_update - ); - break - }, - }; - } - } - } -}}} - -monitor_cmd_for!(polkadot); -monitor_cmd_for!(kusama); -monitor_cmd_for!(westend); - -#[cfg(test)] -pub mod tests { - use super::*; - - #[test] - fn score_passes_strategy_works() { - let s = |x| sp_npos_elections::ElectionScore { minimal_stake: x, ..Default::default() }; - let two = Perbill::from_percent(2); - - // anything passes Always - assert!(score_passes_strategy(s(0), s(0), SubmissionStrategy::Always)); - assert!(score_passes_strategy(s(5), s(0), SubmissionStrategy::Always)); - assert!(score_passes_strategy(s(5), s(10), SubmissionStrategy::Always)); - - // if leading - assert!(score_passes_strategy(s(0), s(0), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(1), s(0), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(2), s(0), SubmissionStrategy::IfLeading)); - assert!(!score_passes_strategy(s(5), s(10), SubmissionStrategy::IfLeading)); - assert!(!score_passes_strategy(s(9), s(10), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(10), s(10), SubmissionStrategy::IfLeading)); - - // if better by 2% - assert!(!score_passes_strategy(s(50), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(100), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(101), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(102), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(score_passes_strategy(s(103), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(score_passes_strategy(s(150), s(100), SubmissionStrategy::ClaimBetterThan(two))); - - // if no less than 2% worse - assert!(!score_passes_strategy(s(50), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(!score_passes_strategy(s(97), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(98), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(99), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(100), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(101), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(102), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(103), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(150), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - } -} diff --git a/polkadot/utils/staking-miner/src/opts.rs b/polkadot/utils/staking-miner/src/opts.rs deleted file mode 100644 index 4cf4d0a765199ecf790f45c579b6874b050a630e..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/opts.rs +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use crate::prelude::*; -use clap::Parser; -use sp_runtime::Perbill; -use std::str::FromStr; - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -#[command(author, version, about)] -pub(crate) struct Opt { - /// The `ws` node to connect to. - #[arg(long, short, default_value = DEFAULT_URI, env = "URI", global = true)] - pub uri: String, - - /// WS connection timeout in number of seconds. - #[arg(long, default_value_t = 60)] - pub connection_timeout: usize, - - /// WS request timeout in number of seconds. - #[arg(long, default_value_t = 60 * 10)] - pub request_timeout: usize, - - #[command(subcommand)] - pub command: Command, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) enum Command { - /// Monitor for the phase being signed, then compute. - Monitor(MonitorConfig), - - /// Just compute a solution now, and don't submit it. - DryRun(DryRunConfig), - - /// Provide a solution that can be submitted to the chain as an emergency response. - EmergencySolution(EmergencySolutionConfig), - - /// Return information about the current version - Info(InfoOpts), -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct MonitorConfig { - /// The path to a file containing the seed of the account. If the file is not found, the seed - /// is used as-is. - /// - /// Can also be provided via the `SEED` environment variable. - /// - /// WARNING: Don't use an account with a large stash for this. Based on how the bot is - /// configured, it might re-try and lose funds through transaction fees/deposits. - #[arg(long, short, env = "SEED")] - pub seed_or_path: String, - - /// They type of event to listen to. - /// - /// Typically, finalized is safer and there is no chance of anything going wrong, but it can be - /// slower. It is recommended to use finalized, if the duration of the signed phase is longer - /// than the the finality delay. - #[arg(long, default_value = "head", value_parser = ["head", "finalized"])] - pub listen: String, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// Submission strategy to use. - /// - /// Possible options: - /// - /// `--submission-strategy if-leading`: only submit if leading. - /// - /// `--submission-strategy always`: always submit. - /// - /// `--submission-strategy "percent-better percent"`: submit if the submission is `n` percent - /// better. - /// - /// `--submission-strategy "no-worse-than percent"`: submit if submission is no more than - /// `n` percent worse. - #[clap(long, default_value = "if-leading")] - pub submission_strategy: SubmissionStrategy, - - /// Delay in number seconds to wait until starting mining a solution. - /// - /// At every block when a solution is attempted - /// a delay can be enforced to avoid submitting at - /// "same time" and risk potential races with other miners. - /// - /// When this is enabled and there are competing solutions, your solution might not be - /// submitted if the scores are equal. - #[arg(long, default_value_t = 0)] - pub delay: usize, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct DryRunConfig { - /// The path to a file containing the seed of the account. If the file is not found, the seed - /// is used as-is. - /// - /// Can also be provided via the `SEED` environment variable. - /// - /// WARNING: Don't use an account with a large stash for this. Based on how the bot is - /// configured, it might re-try and lose funds through transaction fees/deposits. - #[arg(long, short, env = "SEED")] - pub seed_or_path: String, - - /// The block hash at which scraping happens. If none is provided, the latest head is used. - #[arg(long)] - pub at: Option, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// Force create a new snapshot, else expect one to exist onchain. - #[arg(long)] - pub force_snapshot: bool, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct EmergencySolutionConfig { - /// The block hash at which scraping happens. If none is provided, the latest head is used. - #[arg(long)] - pub at: Option, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// The number of top backed winners to take. All are taken, if not provided. - pub take: Option, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct InfoOpts { - /// Serialize the output as json - #[arg(long, short)] - pub json: bool, -} - -/// Submission strategy to use. -#[derive(Debug, Copy, Clone)] -#[cfg_attr(test, derive(PartialEq))] -pub enum SubmissionStrategy { - /// Always submit. - Always, - /// Only submit if at the time, we are the best (or equal to it). - IfLeading, - /// Submit if we are no worse than `Perbill` worse than the best. - ClaimNoWorseThan(Perbill), - /// Submit if we are leading, or if the solution that's leading is more that the given - /// `Perbill` better than us. This helps detect obviously fake solutions and still combat them. - ClaimBetterThan(Perbill), -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) enum Solver { - SeqPhragmen { - #[arg(long, default_value_t = 10)] - iterations: usize, - }, - PhragMMS { - #[arg(long, default_value_t = 10)] - iterations: usize, - }, -} - -/// Custom `impl` to parse `SubmissionStrategy` from CLI. -/// -/// Possible options: -/// * --submission-strategy if-leading: only submit if leading -/// * --submission-strategy always: always submit -/// * --submission-strategy "percent-better percent": submit if submission is `n` percent better. -/// * --submission-strategy "no-worse-than percent": submit if submission is no more than `n` -/// percent worse. -impl FromStr for SubmissionStrategy { - type Err = String; - - fn from_str(s: &str) -> Result { - let s = s.trim(); - - let res = if s == "if-leading" { - Self::IfLeading - } else if s == "always" { - Self::Always - } else if let Some(percent) = s.strip_prefix("no-worse-than ") { - let percent: u32 = percent.parse().map_err(|e| format!("{:?}", e))?; - Self::ClaimNoWorseThan(Perbill::from_percent(percent)) - } else if let Some(percent) = s.strip_prefix("percent-better ") { - let percent: u32 = percent.parse().map_err(|e| format!("{:?}", e))?; - Self::ClaimBetterThan(Perbill::from_percent(percent)) - } else { - return Err(s.into()) - }; - Ok(res) - } -} - -#[cfg(test)] -mod test_super { - use super::*; - - #[test] - fn cli_monitor_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "monitor", - "--seed-or-path", - "//Alice", - "--listen", - "head", - "--delay", - "12", - "seq-phragmen", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::Monitor(MonitorConfig { - seed_or_path: "//Alice".to_string(), - listen: "head".to_string(), - solver: Solver::SeqPhragmen { iterations: 10 }, - submission_strategy: SubmissionStrategy::IfLeading, - delay: 12, - }), - } - ); - } - - #[test] - fn cli_dry_run_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "dry-run", - "--seed-or-path", - "//Alice", - "phrag-mms", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::DryRun(DryRunConfig { - seed_or_path: "//Alice".to_string(), - at: None, - solver: Solver::PhragMMS { iterations: 10 }, - force_snapshot: false, - }), - } - ); - } - - #[test] - fn cli_emergency_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "emergency-solution", - "99", - "phrag-mms", - "--iterations", - "1337", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::EmergencySolution(EmergencySolutionConfig { - take: Some(99), - at: None, - solver: Solver::PhragMMS { iterations: 1337 } - }), - } - ); - } - - #[test] - fn cli_info_works() { - let opt = Opt::try_parse_from([env!("CARGO_PKG_NAME"), "--uri", "hi", "info"]).unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::Info(InfoOpts { json: false }) - } - ); - } - - #[test] - fn cli_request_conn_timeout_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "--request-timeout", - "10", - "--connection-timeout", - "9", - "info", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 9, - request_timeout: 10, - command: Command::Info(InfoOpts { json: false }) - } - ); - } - - #[test] - fn submission_strategy_from_str_works() { - use std::str::FromStr; - - assert_eq!(SubmissionStrategy::from_str("if-leading"), Ok(SubmissionStrategy::IfLeading)); - assert_eq!(SubmissionStrategy::from_str("always"), Ok(SubmissionStrategy::Always)); - assert_eq!( - SubmissionStrategy::from_str(" percent-better 99 "), - Ok(SubmissionStrategy::ClaimBetterThan(Perbill::from_percent(99))) - ); - } -} diff --git a/polkadot/utils/staking-miner/src/prelude.rs b/polkadot/utils/staking-miner/src/prelude.rs deleted file mode 100644 index fb701ece2384d8f1286e4573ebad0c00724dbb42..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/prelude.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Types that we don't fetch from a particular runtime and just assume that they are constant all -//! of the place. -//! -//! It is actually easy to convert the rest as well, but it'll be a lot of noise in our codebase, -//! needing to sprinkle `any_runtime` in a few extra places. - -/// The account id type. -pub type AccountId = core_primitives::AccountId; -/// The block number type. -pub type BlockNumber = core_primitives::BlockNumber; -/// The balance type. -pub type Balance = core_primitives::Balance; -/// Index of a transaction in the chain. -pub type Nonce = core_primitives::Nonce; -/// The hash type. We re-export it here, but we can easily get it from block as well. -pub type Hash = core_primitives::Hash; -/// The header type. We re-export it here, but we can easily get it from block as well. -pub type Header = core_primitives::Header; -/// The block type. -pub type Block = core_primitives::Block; - -pub use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -/// Default URI to connect to. -pub const DEFAULT_URI: &str = "wss://rpc.polkadot.io:443"; -/// The logging target. -pub const LOG_TARGET: &str = "staking-miner"; - -/// The election provider pallet. -pub use pallet_election_provider_multi_phase as EPM; - -/// The externalities type. -pub type Ext = sp_state_machine::TestExternalities>; - -/// The key pair type being used. We "strongly" assume sr25519 for simplicity. -pub type Pair = sp_core::sr25519::Pair; - -/// A dynamic token type used to represent account balances. -pub type Token = sub_tokens::dynamic::DynamicToken; diff --git a/polkadot/utils/staking-miner/src/rpc.rs b/polkadot/utils/staking-miner/src/rpc.rs deleted file mode 100644 index 2d25616e2a179e7eb993e2423dfe7496b022baca..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/rpc.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! JSON-RPC related types and helpers. - -use super::*; -use jsonrpsee::{ - core::{Error as RpcError, RpcResult}, - proc_macros::rpc, -}; -use pallet_transaction_payment::RuntimeDispatchInfo; -use sc_transaction_pool_api::TransactionStatus; -use sp_core::{storage::StorageKey, Bytes}; -use sp_version::RuntimeVersion; -use std::{future::Future, time::Duration}; - -#[derive(frame_support::DebugNoBound, thiserror::Error)] -pub(crate) enum RpcHelperError { - JsonRpsee(#[from] jsonrpsee::core::Error), - Codec(#[from] codec::Error), -} - -impl std::fmt::Display for RpcHelperError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - ::fmt(self, f) - } -} - -#[rpc(client)] -pub trait RpcApi { - /// Fetch system name. - #[method(name = "system_chain")] - async fn system_chain(&self) -> RpcResult; - - /// Fetch a storage key. - #[method(name = "state_getStorage")] - async fn storage(&self, key: &StorageKey, hash: Option) -> RpcResult>; - - /// Fetch the runtime version. - #[method(name = "state_getRuntimeVersion")] - async fn runtime_version(&self, at: Option) -> RpcResult; - - /// Fetch the payment query info. - #[method(name = "payment_queryInfo")] - async fn payment_query_info( - &self, - encoded_xt: &Bytes, - at: Option<&Hash>, - ) -> RpcResult>; - - /// Dry run an extrinsic at a given block. Return SCALE encoded - /// [`sp_runtime::ApplyExtrinsicResult`]. - #[method(name = "system_dryRun")] - async fn dry_run(&self, extrinsic: &Bytes, at: Option) -> RpcResult; - - /// Get hash of the n-th block in the canon chain. - /// - /// By default returns latest block hash. - #[method(name = "chain_getBlockHash", aliases = ["chain_getHead"], blocking)] - fn block_hash(&self, hash: Option) -> RpcResult>; - - /// Get hash of the last finalized block in the canon chain. - #[method(name = "chain_getFinalizedHead", aliases = ["chain_getFinalisedHead"], blocking)] - fn finalized_head(&self) -> RpcResult; - - /// Submit an extrinsic to watch. - /// - /// See [`TransactionStatus`](sc_transaction_pool_api::TransactionStatus) for details on - /// transaction life cycle. - #[subscription( - name = "author_submitAndWatchExtrinsic" => "author_extrinsicUpdate", - unsubscribe = "author_unwatchExtrinsic", - item = TransactionStatus - )] - fn watch_extrinsic(&self, bytes: &Bytes); - - /// New head subscription. - #[subscription( - name = "chain_subscribeNewHeads" => "newHead", - unsubscribe = "chain_unsubscribeNewHeads", - item = Header - )] - fn subscribe_new_heads(&self); - - /// Finalized head subscription. - #[subscription( - name = "chain_subscribeFinalizedHeads" => "chain_finalizedHead", - unsubscribe = "chain_unsubscribeFinalizedHeads", - item = Header - )] - fn subscribe_finalized_heads(&self); -} - -type Uri = String; - -/// Wraps a shared web-socket JSON-RPC client that can be cloned. -#[derive(Clone, Debug)] -pub(crate) struct SharedRpcClient(Arc, Uri); - -impl Deref for SharedRpcClient { - type Target = WsClient; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl SharedRpcClient { - /// Get the URI of the client. - pub fn uri(&self) -> &str { - &self.1 - } - - /// Create a new shared JSON-RPC web-socket client. - pub(crate) async fn new( - uri: &str, - connection_timeout: Duration, - request_timeout: Duration, - ) -> Result { - let client = WsClientBuilder::default() - .connection_timeout(connection_timeout) - .max_request_body_size(u32::MAX) - .request_timeout(request_timeout) - .max_concurrent_requests(u32::MAX as usize) - .build(uri) - .await?; - Ok(Self(Arc::new(client), uri.to_owned())) - } - - /// Get a storage item and decode it as `T`. - /// - /// # Return value: - /// - /// The function returns: - /// - /// * `Ok(Some(val))` if successful. - /// * `Ok(None)` if the storage item was not found. - /// * `Err(e)` if the JSON-RPC call failed. - pub(crate) async fn get_storage_and_decode<'a, T: codec::Decode>( - &self, - key: &StorageKey, - hash: Option, - ) -> Result, RpcHelperError> { - if let Some(bytes) = self.storage(key, hash).await? { - let decoded = ::decode(&mut &*bytes.0) - .map_err::(Into::into)?; - Ok(Some(decoded)) - } else { - Ok(None) - } - } -} - -/// Takes a future that returns `Bytes` and tries to decode those bytes into the type `Dec`. -/// Warning: don't use for storage, it will fail for non-existent storage items. -/// -/// # Return value: -/// -/// The function returns: -/// -/// * `Ok(val)` if successful. -/// * `Err(RpcHelperError::JsonRpsee)` if the JSON-RPC call failed. -/// * `Err(RpcHelperError::Codec)` if `Bytes` could not be decoded. -pub(crate) async fn await_request_and_decode<'a, Dec: codec::Decode>( - req: impl Future>, -) -> Result { - let bytes = req.await?; - Dec::decode(&mut &*bytes.0).map_err::(Into::into) -} diff --git a/polkadot/utils/staking-miner/src/runtime_versions.rs b/polkadot/utils/staking-miner/src/runtime_versions.rs deleted file mode 100644 index 38af05ead24143ae122e4e0ac2732c0de8f6c145..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/runtime_versions.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use sp_version::RuntimeVersion; -use std::fmt; - -#[derive(Debug, serde::Serialize)] -pub(crate) struct RuntimeWrapper<'a>(pub &'a RuntimeVersion); - -impl<'a> fmt::Display for RuntimeWrapper<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let width = 16; - - writeln!( - f, - r#" impl_name : {impl_name:>width$} - spec_name : {spec_name:>width$} - spec_version : {spec_version:>width$} - transaction_version : {transaction_version:>width$} - impl_version : {impl_version:>width$} - authoringVersion : {authoring_version:>width$} - state_version : {state_version:>width$}"#, - spec_name = self.0.spec_name.to_string(), - impl_name = self.0.impl_name.to_string(), - spec_version = self.0.spec_version, - impl_version = self.0.impl_version, - authoring_version = self.0.authoring_version, - transaction_version = self.0.transaction_version, - state_version = self.0.state_version, - ) - } -} - -impl<'a> From<&'a RuntimeVersion> for RuntimeWrapper<'a> { - fn from(r: &'a RuntimeVersion) -> Self { - RuntimeWrapper(r) - } -} - -#[derive(Debug, serde::Serialize)] -pub(crate) struct RuntimeVersions<'a> { - /// The `RuntimeVersion` linked in the staking-miner - pub linked: RuntimeWrapper<'a>, - - /// The `RuntimeVersion` reported by the node we connect to via RPC - pub remote: RuntimeWrapper<'a>, - - /// This `bool` reports whether both remote and linked `RuntimeVersion` are compatible - /// and if the staking-miner is expected to work properly against the remote runtime - compatible: bool, -} - -impl<'a> RuntimeVersions<'a> { - pub fn new( - remote_runtime_version: &'a RuntimeVersion, - linked_runtime_version: &'a RuntimeVersion, - ) -> Self { - Self { - remote: remote_runtime_version.into(), - linked: linked_runtime_version.into(), - compatible: are_runtimes_compatible(remote_runtime_version, linked_runtime_version), - } - } -} - -/// Check whether runtimes are compatible. Currently we only support equality. -fn are_runtimes_compatible(r1: &RuntimeVersion, r2: &RuntimeVersion) -> bool { - r1 == r2 -} - -impl<'a> fmt::Display for RuntimeVersions<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let _ = write!(f, "- linked:\n{}", self.linked); - let _ = write!(f, "- remote :\n{}", self.remote); - write!(f, "Compatible: {}", if self.compatible { "YES" } else { "NO" }) - } -} diff --git a/polkadot/utils/staking-miner/src/signer.rs b/polkadot/utils/staking-miner/src/signer.rs deleted file mode 100644 index e6677ccd3a66149e25affb5286140a5b44a6a3e4..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/src/signer.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Wrappers around creating a signer account. - -use crate::{prelude::*, rpc::SharedRpcClient, AccountId, Error, Nonce, Pair, LOG_TARGET}; -use frame_system::AccountInfo; -use sp_core::{crypto::Pair as _, storage::StorageKey}; - -pub(crate) const SIGNER_ACCOUNT_WILL_EXIST: &str = - "signer account is checked to exist upon startup; it can only die if it transfers funds out \ - of it, or get slashed. If it does not exist at this point, it is likely due to a bug, or the \ - signer got slashed. Terminating."; - -/// Some information about the signer. Redundant at this point, but makes life easier. -#[derive(Clone)] -pub(crate) struct Signer { - /// The account id. - pub(crate) account: AccountId, - - /// The full crypto key-pair. - pub(crate) pair: Pair, -} - -pub(crate) async fn get_account_info + EPM::Config>( - rpc: &SharedRpcClient, - who: &T::AccountId, - maybe_at: Option, -) -> Result>, Error> { - rpc.get_storage_and_decode::>( - &StorageKey(>::hashed_key_for(&who)), - maybe_at, - ) - .await - .map_err(Into::into) -} - -/// Read the signer account's URI -pub(crate) async fn signer_uri_from_string< - T: frame_system::Config< - AccountId = AccountId, - Nonce = Nonce, - AccountData = pallet_balances::AccountData, - Hash = Hash, - > + EPM::Config, ->( - mut seed_or_path: &str, - client: &SharedRpcClient, -) -> Result> { - seed_or_path = seed_or_path.trim(); - - let seed = match std::fs::read(seed_or_path) { - Ok(s) => String::from_utf8(s).map_err(|_| Error::::AccountDoesNotExists)?, - Err(_) => seed_or_path.to_string(), - }; - let seed = seed.trim(); - - let pair = Pair::from_string(seed, None)?; - let account = T::AccountId::from(pair.public()); - let _info = get_account_info::(client, &account, None) - .await? - .ok_or(Error::::AccountDoesNotExists)?; - log::info!( - target: LOG_TARGET, - "loaded account {:?}, free: {:?}, info: {:?}", - &account, - Token::from(_info.data.free), - _info - ); - Ok(Signer { account, pair }) -} diff --git a/polkadot/utils/staking-miner/tests/cli.rs b/polkadot/utils/staking-miner/tests/cli.rs deleted file mode 100644 index 1ced1239e5530c0b1a656ef3197fdaf9a4334dd1..0000000000000000000000000000000000000000 --- a/polkadot/utils/staking-miner/tests/cli.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use assert_cmd::{cargo::cargo_bin, Command}; -use serde_json::{Result, Value}; - -#[test] -fn cli_version_works() { - let crate_name = env!("CARGO_PKG_NAME"); - let output = Command::new(cargo_bin(crate_name)).arg("--version").output().unwrap(); - - assert!(output.status.success(), "command returned with non-success exit code"); - let version = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - - assert_eq!(version, format!("{} {}", crate_name, env!("CARGO_PKG_VERSION"))); -} - -#[test] -fn cli_info_works() { - let crate_name = env!("CARGO_PKG_NAME"); - let output = Command::new(cargo_bin(crate_name)) - .arg("info") - .arg("--json") - .env("RUST_LOG", "none") - .output() - .unwrap(); - - assert!(output.status.success(), "command returned with non-success exit code"); - let info = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - let v: Result = serde_json::from_str(&info); - let v = v.unwrap(); - assert!(!v["builtin"].to_string().is_empty()); - assert!(!v["builtin"]["spec_name"].to_string().is_empty()); - assert!(!v["builtin"]["spec_version"].to_string().is_empty()); - assert!(!v["remote"].to_string().is_empty()); -} diff --git a/polkadot/xcm/pallet-xcm/Cargo.toml b/polkadot/xcm/pallet-xcm/Cargo.toml index 17ee9840e8f47ad2436add35296b811dc38846e8..4946a8d11ce96aed88e5864a57e79875d62555f0 100644 --- a/polkadot/xcm/pallet-xcm/Cargo.toml +++ b/polkadot/xcm/pallet-xcm/Cargo.toml @@ -32,8 +32,6 @@ xcm-builder = { package = "staging-xcm-builder", path = "../xcm-builder" } [features] default = [ "std" ] -# Enable `VersionedMigration` for the migrations using the experimental feature. -experimental = [ "frame-support/experimental" ] std = [ "bounded-collections/std", "codec/std", diff --git a/polkadot/xcm/pallet-xcm/src/lib.rs b/polkadot/xcm/pallet-xcm/src/lib.rs index 9dfb63c2c9c6411f2d198cbdfc42c1d292b23c0a..5acd093d3bca259691e817921640d6d9907f8e03 100644 --- a/polkadot/xcm/pallet-xcm/src/lib.rs +++ b/polkadot/xcm/pallet-xcm/src/lib.rs @@ -787,6 +787,8 @@ pub mod pallet { /// Teleport some assets from the local chain to some destination chain. /// + /// **This function is deprecated: Use `limited_teleport_assets` instead.** + /// /// Fee payment on the destination side is made from the asset in the `assets` vector of /// index `fee_asset_item`. The weight limit for fees is not provided and thus is unlimited, /// with all fees taken as needed from the asset. @@ -830,12 +832,14 @@ pub mod pallet { assets: Box, fee_asset_item: u32, ) -> DispatchResult { - Self::do_teleport_assets(origin, dest, beneficiary, assets, fee_asset_item, None) + Self::do_teleport_assets(origin, dest, beneficiary, assets, fee_asset_item, Unlimited) } /// Transfer some assets from the local chain to the sovereign account of a destination /// chain and forward a notification XCM. /// + /// **This function is deprecated: Use `limited_reserve_transfer_assets` instead.** + /// /// Fee payment on the destination side is made from the asset in the `assets` vector of /// index `fee_asset_item`. The weight limit for fees is not provided and thus is unlimited, /// with all fees taken as needed from the asset. @@ -879,7 +883,7 @@ pub mod pallet { beneficiary, assets, fee_asset_item, - None, + Unlimited, ) } @@ -1055,7 +1059,7 @@ pub mod pallet { beneficiary, assets, fee_asset_item, - Some(weight_limit), + weight_limit, ) } @@ -1109,7 +1113,7 @@ pub mod pallet { beneficiary, assets, fee_asset_item, - Some(weight_limit), + weight_limit, ) } @@ -1197,7 +1201,7 @@ impl Pallet { beneficiary: Box, assets: Box, fee_asset_item: u32, - maybe_weight_limit: Option, + weight_limit: WeightLimit, ) -> DispatchResult { let origin_location = T::ExecuteXcmOrigin::ensure_origin(origin)?; let dest = (*dest).try_into().map_err(|()| Error::::BadVersion)?; @@ -1218,22 +1222,6 @@ impl Pallet { .map_err(|_| Error::::CannotReanchor)?; let max_assets = assets.len() as u32; let assets: MultiAssets = assets.into(); - let weight_limit = match maybe_weight_limit { - Some(weight_limit) => weight_limit, - None => { - let fees = fees.clone(); - let mut remote_message = Xcm(vec![ - ReserveAssetDeposited(assets.clone()), - ClearOrigin, - BuyExecution { fees, weight_limit: Limited(Weight::zero()) }, - DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, - ]); - // use local weight for remote message and hope for the best. - let remote_weight = T::Weigher::weight(&mut remote_message) - .map_err(|()| Error::::UnweighableMessage)?; - Limited(remote_weight) - }, - }; let xcm = Xcm(vec![ BuyExecution { fees, weight_limit }, DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, @@ -1257,7 +1245,7 @@ impl Pallet { beneficiary: Box, assets: Box, fee_asset_item: u32, - maybe_weight_limit: Option, + weight_limit: WeightLimit, ) -> DispatchResult { let origin_location = T::ExecuteXcmOrigin::ensure_origin(origin)?; let dest = (*dest).try_into().map_err(|()| Error::::BadVersion)?; @@ -1278,22 +1266,6 @@ impl Pallet { .map_err(|_| Error::::CannotReanchor)?; let max_assets = assets.len() as u32; let assets: MultiAssets = assets.into(); - let weight_limit = match maybe_weight_limit { - Some(weight_limit) => weight_limit, - None => { - let fees = fees.clone(); - let mut remote_message = Xcm(vec![ - ReceiveTeleportedAsset(assets.clone()), - ClearOrigin, - BuyExecution { fees, weight_limit: Limited(Weight::zero()) }, - DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, - ]); - // use local weight for remote message and hope for the best. - let remote_weight = T::Weigher::weight(&mut remote_message) - .map_err(|()| Error::::UnweighableMessage)?; - Limited(remote_weight) - }, - }; let xcm = Xcm(vec![ BuyExecution { fees, weight_limit }, DepositAsset { assets: Wild(AllCounted(max_assets)), beneficiary }, diff --git a/polkadot/xcm/pallet-xcm/src/migration.rs b/polkadot/xcm/pallet-xcm/src/migration.rs index e28ca56563c66909e196e6c4e7751229b1406dcd..ba3cdb5c51edc82712b3bf00feedc8b670365547 100644 --- a/polkadot/xcm/pallet-xcm/src/migration.rs +++ b/polkadot/xcm/pallet-xcm/src/migration.rs @@ -65,7 +65,6 @@ pub mod v1 { /// /// Wrapped in [`frame_support::migrations::VersionedMigration`] so the pre/post checks don't /// begin failing after the upgrade is enacted on-chain. - #[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV1 = frame_support::migrations::VersionedMigration< 0, 1, diff --git a/polkadot/xcm/pallet-xcm/src/tests.rs b/polkadot/xcm/pallet-xcm/src/tests.rs index 486887598c7829a5740ab951e8574f9feacb8c44..5d8aee8d665f4d2a98af0aeef3622219d3a87634 100644 --- a/polkadot/xcm/pallet-xcm/src/tests.rs +++ b/polkadot/xcm/pallet-xcm/src/tests.rs @@ -375,7 +375,7 @@ fn teleport_assets_works() { Xcm(vec![ ReceiveTeleportedAsset((Here, SEND_AMOUNT).into()), ClearOrigin, - buy_limited_execution((Here, SEND_AMOUNT), Weight::from_parts(4000, 4000)), + buy_execution((Here, SEND_AMOUNT)), DepositAsset { assets: AllCounted(1).into(), beneficiary: dest }, ]), )] @@ -508,7 +508,7 @@ fn reserve_transfer_assets_works() { Xcm(vec![ ReserveAssetDeposited((Parent, SEND_AMOUNT).into()), ClearOrigin, - buy_limited_execution((Parent, SEND_AMOUNT), Weight::from_parts(4000, 4000)), + buy_execution((Parent, SEND_AMOUNT)), DepositAsset { assets: AllCounted(1).into(), beneficiary: dest }, ]), )] diff --git a/polkadot/xcm/procedural/Cargo.toml b/polkadot/xcm/procedural/Cargo.toml index 6545fb68764d9081689ece4a694d029825a48f7c..1ff73c64780e6af913f6db37f8a325ef0edace92 100644 --- a/polkadot/xcm/procedural/Cargo.toml +++ b/polkadot/xcm/procedural/Cargo.toml @@ -11,5 +11,5 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.37" Inflector = "0.11.4" diff --git a/polkadot/xcm/src/v3/multiasset.rs b/polkadot/xcm/src/v3/multiasset.rs index 1668d1b870dcf12bb76ced97cd25aa3dfb929d83..188555318c8c0ba16e129407845ffdb3b056e81d 100644 --- a/polkadot/xcm/src/v3/multiasset.rs +++ b/polkadot/xcm/src/v3/multiasset.rs @@ -34,6 +34,7 @@ use crate::v2::{ WildMultiAsset as OldWildMultiAsset, }; use alloc::{vec, vec::Vec}; +use bounded_collections::{BoundedVec, ConstU32}; use core::{ cmp::Ordering, convert::{TryFrom, TryInto}, @@ -506,9 +507,8 @@ impl TryFrom for MultiAsset { #[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub struct MultiAssets(Vec); -/// Maximum number of items we expect in a single `MultiAssets` value. Note this is not (yet) -/// enforced, and just serves to provide a sensible `max_encoded_len` for `MultiAssets`. -const MAX_ITEMS_IN_MULTIASSETS: usize = 20; +/// Maximum number of items in a single `MultiAssets` value that can be decoded. +pub const MAX_ITEMS_IN_MULTIASSETS: usize = 20; impl MaxEncodedLen for MultiAssets { fn max_encoded_len() -> usize { @@ -517,8 +517,10 @@ impl MaxEncodedLen for MultiAssets { } impl Decode for MultiAssets { - fn decode(input: &mut I) -> Result { - Self::from_sorted_and_deduplicated(Vec::::decode(input)?) + fn decode(input: &mut I) -> Result { + let bounded_instructions = + BoundedVec::>::decode(input)?; + Self::from_sorted_and_deduplicated(bounded_instructions.into_inner()) .map_err(|()| "Out of order".into()) } } @@ -974,4 +976,31 @@ mod tests { let r = MultiAssets::from_sorted_and_deduplicated(mixed_bad); assert!(r.is_err()); } + + #[test] + fn decoding_respects_limit() { + use super::*; + + // Having lots of one asset will work since they are deduplicated + let lots_of_one_asset: MultiAssets = + vec![(GeneralIndex(1), 1u128).into(); MAX_ITEMS_IN_MULTIASSETS + 1].into(); + let encoded = lots_of_one_asset.encode(); + assert!(MultiAssets::decode(&mut &encoded[..]).is_ok()); + + // Fewer assets than the limit works + let mut few_assets: MultiAssets = Vec::new().into(); + for i in 0..MAX_ITEMS_IN_MULTIASSETS { + few_assets.push((GeneralIndex(i as u128), 1u128).into()); + } + let encoded = few_assets.encode(); + assert!(MultiAssets::decode(&mut &encoded[..]).is_ok()); + + // Having lots of different assets will not work + let mut too_many_different_assets: MultiAssets = Vec::new().into(); + for i in 0..MAX_ITEMS_IN_MULTIASSETS + 1 { + too_many_different_assets.push((GeneralIndex(i as u128), 1u128).into()); + } + let encoded = too_many_different_assets.encode(); + assert!(MultiAssets::decode(&mut &encoded[..]).is_err()); + } } diff --git a/polkadot/xcm/xcm-builder/src/lib.rs b/polkadot/xcm/xcm-builder/src/lib.rs index e3f91040963840d2b29ceaa30fc1a8e373de4817..26f2aadadf3de61d74eb6258d2a697477733dc52 100644 --- a/polkadot/xcm/xcm-builder/src/lib.rs +++ b/polkadot/xcm/xcm-builder/src/lib.rs @@ -32,8 +32,8 @@ pub use location_conversion::ForeignChainAliasAccount; pub use location_conversion::{ Account32Hash, AccountId32Aliases, AccountKey20Aliases, AliasesIntoAccountId32, ChildParachainConvertsVia, DescribeAccountId32Terminal, DescribeAccountIdTerminal, - DescribeAccountKey20Terminal, DescribeAllTerminal, DescribeFamily, DescribeLocation, - DescribePalletTerminal, DescribeTerminus, GlobalConsensusConvertsFor, + DescribeAccountKey20Terminal, DescribeAllTerminal, DescribeBodyTerminal, DescribeFamily, + DescribeLocation, DescribePalletTerminal, DescribeTerminus, GlobalConsensusConvertsFor, GlobalConsensusParachainConvertsFor, HashedDescription, ParentIsPreset, SiblingParachainConvertsVia, }; diff --git a/polkadot/xcm/xcm-builder/src/location_conversion.rs b/polkadot/xcm/xcm-builder/src/location_conversion.rs index 26b48fc88adcea2ca7cdb9d64402b6844a88942e..4a5fbbb123be167a60997e9dac19a33bd191b954 100644 --- a/polkadot/xcm/xcm-builder/src/location_conversion.rs +++ b/polkadot/xcm/xcm-builder/src/location_conversion.rs @@ -18,7 +18,7 @@ use crate::universal_exports::ensure_is_remote; use frame_support::traits::Get; use parity_scale_codec::{Compact, Decode, Encode}; use sp_io::hashing::blake2_256; -use sp_runtime::traits::{AccountIdConversion, Convert, TrailingZeroInput}; +use sp_runtime::traits::{AccountIdConversion, TrailingZeroInput, TryConvert}; use sp_std::{marker::PhantomData, prelude::*}; use xcm::latest::prelude::*; use xcm_executor::traits::ConvertLocation; @@ -86,11 +86,22 @@ impl DescribeLocation for DescribeAccountKey20Terminal { pub type DescribeAccountIdTerminal = (DescribeAccountId32Terminal, DescribeAccountKey20Terminal); +pub struct DescribeBodyTerminal; +impl DescribeLocation for DescribeBodyTerminal { + fn describe_location(l: &MultiLocation) -> Option> { + match (l.parents, &l.interior) { + (0, X1(Plurality { id, part })) => Some((b"Body", id, part).encode()), + _ => return None, + } + } +} + pub type DescribeAllTerminal = ( DescribeTerminus, DescribePalletTerminal, DescribeAccountId32Terminal, DescribeAccountKey20Terminal, + DescribeBodyTerminal, ); pub struct DescribeFamily(PhantomData); @@ -322,10 +333,10 @@ impl>, AccountId: From<[u8; 32]> + Into<[u8; 32]> /// network (provided by `Network`) and the `AccountId`'s `[u8; 32]` datum for the `id`. pub struct AliasesIntoAccountId32(PhantomData<(Network, AccountId)>); impl<'a, Network: Get>, AccountId: Clone + Into<[u8; 32]> + Clone> - Convert<&'a AccountId, MultiLocation> for AliasesIntoAccountId32 + TryConvert<&'a AccountId, MultiLocation> for AliasesIntoAccountId32 { - fn convert(who: &AccountId) -> MultiLocation { - AccountId32 { network: Network::get(), id: who.clone().into() }.into() + fn try_convert(who: &AccountId) -> Result { + Ok(AccountId32 { network: Network::get(), id: who.clone().into() }.into()) } } diff --git a/polkadot/xcm/xcm-builder/src/pay.rs b/polkadot/xcm/xcm-builder/src/pay.rs index 39e09e056772ed1318a05ecb948859fdebb62bc6..0f3a622f4ece1769a4314b507bcc4fd917b911f7 100644 --- a/polkadot/xcm/xcm-builder/src/pay.rs +++ b/polkadot/xcm/xcm-builder/src/pay.rs @@ -20,7 +20,7 @@ use frame_support::traits::{ tokens::{Pay, PaymentStatus}, Get, }; -use sp_runtime::traits::Convert; +use sp_runtime::traits::TryConvert; use sp_std::{marker::PhantomData, vec}; use xcm::{opaque::lts::Weight, prelude::*}; use xcm_executor::traits::{QueryHandler, QueryResponseStatus}; @@ -71,8 +71,8 @@ impl< Timeout: Get, Beneficiary: Clone, AssetKind, - AssetKindToLocatableAsset: Convert, - BeneficiaryRefToLocation: for<'a> Convert<&'a Beneficiary, MultiLocation>, + AssetKindToLocatableAsset: TryConvert, + BeneficiaryRefToLocation: for<'a> TryConvert<&'a Beneficiary, MultiLocation>, > Pay for PayOverXcm< Interior, @@ -96,12 +96,14 @@ impl< asset_kind: Self::AssetKind, amount: Self::Balance, ) -> Result { - let locatable = AssetKindToLocatableAsset::convert(asset_kind); + let locatable = AssetKindToLocatableAsset::try_convert(asset_kind) + .map_err(|_| xcm::latest::Error::InvalidLocation)?; let LocatableAssetId { asset_id, location: asset_location } = locatable; let destination = Querier::UniversalLocation::get() .invert_target(&asset_location) .map_err(|()| Self::Error::LocationNotInvertible)?; - let beneficiary = BeneficiaryRefToLocation::convert(&who); + let beneficiary = BeneficiaryRefToLocation::try_convert(&who) + .map_err(|_| xcm::latest::Error::InvalidLocation)?; let query_id = Querier::new_query(asset_location, Timeout::get(), Interior::get()); @@ -196,10 +198,10 @@ pub struct LocatableAssetId { /// Adapter `struct` which implements a conversion from any `AssetKind` into a [`LocatableAssetId`] /// value using a fixed `Location` for the `location` field. pub struct FixedLocation(sp_std::marker::PhantomData); -impl, AssetKind: Into> Convert +impl, AssetKind: Into> TryConvert for FixedLocation { - fn convert(value: AssetKind) -> LocatableAssetId { - LocatableAssetId { asset_id: value.into(), location: Location::get() } + fn try_convert(value: AssetKind) -> Result { + Ok(LocatableAssetId { asset_id: value.into(), location: Location::get() }) } } diff --git a/polkadot/xcm/xcm-builder/src/tests/pay/pay.rs b/polkadot/xcm/xcm-builder/src/tests/pay/pay.rs index 28b2feec0c231438c94d8e890a41e855498f6ded..491a2bcef7a0c8a3d026e13cf48f226161c0b9cf 100644 --- a/polkadot/xcm/xcm-builder/src/tests/pay/pay.rs +++ b/polkadot/xcm/xcm-builder/src/tests/pay/pay.rs @@ -29,9 +29,9 @@ pub struct AssetKind { } pub struct LocatableAssetKindConverter; -impl sp_runtime::traits::Convert for LocatableAssetKindConverter { - fn convert(value: AssetKind) -> LocatableAssetId { - LocatableAssetId { asset_id: value.asset_id, location: value.destination } +impl sp_runtime::traits::TryConvert for LocatableAssetKindConverter { + fn try_convert(value: AssetKind) -> Result { + Ok(LocatableAssetId { asset_id: value.asset_id, location: value.destination }) } } diff --git a/polkadot/xcm/xcm-builder/src/universal_exports.rs b/polkadot/xcm/xcm-builder/src/universal_exports.rs index 0ee627e0ee903d56b81a1b1f22ce80dd2b3a0fdc..a24631ffa9423c31c78b6f2860ab3df48e502725 100644 --- a/polkadot/xcm/xcm-builder/src/universal_exports.rs +++ b/polkadot/xcm/xcm-builder/src/universal_exports.rs @@ -163,15 +163,21 @@ impl, - xcm: &mut Option>, + msg: &mut Option>, ) -> SendResult { let d = dest.ok_or(MissingArgument)?; let devolved = ensure_is_remote(UniversalLocation::get(), d).map_err(|_| NotApplicable)?; let (remote_network, remote_location) = devolved; - let xcm = xcm.take().ok_or(MissingArgument)?; - - let (bridge, maybe_payment) = - Bridges::exporter_for(&remote_network, &remote_location, &xcm).ok_or(NotApplicable)?; + let xcm = msg.take().ok_or(MissingArgument)?; + + // find exporter + let Some((bridge, maybe_payment)) = + Bridges::exporter_for(&remote_network, &remote_location, &xcm) + else { + // We need to make sure that msg is not consumed in case of `NotApplicable`. + *msg = Some(xcm); + return Err(SendError::NotApplicable) + }; ensure!(maybe_payment.is_none(), Unroutable); // `xcm` should already end with `SetTopic` - if it does, then extract and derive into @@ -224,13 +230,21 @@ impl, - xcm: &mut Option>, + msg: &mut Option>, ) -> SendResult { let d = *dest.as_ref().ok_or(MissingArgument)?; let devolved = ensure_is_remote(UniversalLocation::get(), d).map_err(|_| NotApplicable)?; let (remote_network, remote_location) = devolved; - - let xcm = xcm.take().ok_or(MissingArgument)?; + let xcm = msg.take().ok_or(MissingArgument)?; + + // find exporter + let Some((bridge, maybe_payment)) = + Bridges::exporter_for(&remote_network, &remote_location, &xcm) + else { + // We need to make sure that msg is not consumed in case of `NotApplicable`. + *msg = Some(xcm); + return Err(SendError::NotApplicable) + }; // `xcm` should already end with `SetTopic` - if it does, then extract and derive into // an onward topic ID. @@ -239,9 +253,6 @@ impl None, }; - let (bridge, maybe_payment) = - Bridges::exporter_for(&remote_network, &remote_location, &xcm).ok_or(NotApplicable)?; - let local_from_bridge = UniversalLocation::get().invert_target(&bridge).map_err(|_| Unroutable)?; let export_instruction = @@ -452,4 +463,80 @@ mod tests { let x = ensure_is_remote((), (Parent, Polkadot, Parachain(1000))); assert_eq!(x, Err((Parent, Polkadot, Parachain(1000)).into())); } + + pub struct OkSender; + impl SendXcm for OkSender { + type Ticket = (); + + fn validate( + _destination: &mut Option, + _message: &mut Option>, + ) -> SendResult { + Ok(((), MultiAssets::new())) + } + + fn deliver(_ticket: Self::Ticket) -> Result { + Ok([0; 32]) + } + } + + /// Generic test case asserting that dest and msg is not consumed by `validate` implementation + /// of `SendXcm` in case of expected result. + fn ensure_validate_does_not_consume_dest_or_msg( + dest: MultiLocation, + assert_result: impl Fn(SendResult), + ) { + let mut dest_wrapper = Some(dest); + let msg = Xcm::<()>::new(); + let mut msg_wrapper = Some(msg.clone()); + + assert_result(S::validate(&mut dest_wrapper, &mut msg_wrapper)); + + // ensure dest and msg are untouched + assert_eq!(Some(dest), dest_wrapper); + assert_eq!(Some(msg), msg_wrapper); + } + + #[test] + fn remote_exporters_does_not_consume_dest_or_msg_on_not_applicable() { + frame_support::parameter_types! { + pub Local: NetworkId = ByGenesis([0; 32]); + pub UniversalLocation: InteriorMultiLocation = X2(GlobalConsensus(Local::get()), Parachain(1234)); + pub DifferentRemote: NetworkId = ByGenesis([22; 32]); + // no routers + pub BridgeTable: Vec<(NetworkId, MultiLocation, Option)> = vec![]; + } + + // check with local destination (should be remote) + let local_dest = (Parent, Parachain(5678)).into(); + assert!(ensure_is_remote(UniversalLocation::get(), local_dest).is_err()); + + ensure_validate_does_not_consume_dest_or_msg::< + UnpaidRemoteExporter, OkSender, UniversalLocation>, + >(local_dest, |result| assert_eq!(Err(NotApplicable), result)); + + ensure_validate_does_not_consume_dest_or_msg::< + SovereignPaidRemoteExporter< + NetworkExportTable, + OkSender, + UniversalLocation, + >, + >(local_dest, |result| assert_eq!(Err(NotApplicable), result)); + + // check with not applicable destination + let remote_dest = (Parent, Parent, DifferentRemote::get()).into(); + assert!(ensure_is_remote(UniversalLocation::get(), remote_dest).is_ok()); + + ensure_validate_does_not_consume_dest_or_msg::< + UnpaidRemoteExporter, OkSender, UniversalLocation>, + >(remote_dest, |result| assert_eq!(Err(NotApplicable), result)); + + ensure_validate_does_not_consume_dest_or_msg::< + SovereignPaidRemoteExporter< + NetworkExportTable, + OkSender, + UniversalLocation, + >, + >(remote_dest, |result| assert_eq!(Err(NotApplicable), result)); + } } diff --git a/polkadot/xcm/xcm-simulator/fuzzer/README.md b/polkadot/xcm/xcm-simulator/fuzzer/README.md index 69e8cd377b97739d0ad723fe380e668a979dcc82..0b3fdd8ec776d5e78d37aa5b57e6c0762af21320 100644 --- a/polkadot/xcm/xcm-simulator/fuzzer/README.md +++ b/polkadot/xcm/xcm-simulator/fuzzer/README.md @@ -1,6 +1,7 @@ # XCM Simulator Fuzzer -This project will fuzz-test the XCM simulator. It can catch reachable panics, timeouts as well as integer overflows and underflows. +This project will fuzz-test the XCM simulator. It can catch reachable panics, timeouts as well as integer overflows and +underflows. ## Install dependencies @@ -29,7 +30,8 @@ cargo hfuzz run-debug xcm-fuzzer hfuzz_workspace/xcm-fuzzer/fuzzer_input_file In this directory, run these four commands: ``` -RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build +RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" \ +CARGO_INCREMENTAL=0 SKIP_WASM_BUILD=1 CARGO_HOME=./cargo cargo build ../../../target/debug/xcm-fuzzer hfuzz_workspace/xcm-fuzzer/input/ zip -0 ccov.zip `find ../../../target/ \( -name "*.gc*" -o -name "test-*.gc*" \) -print` grcov ccov.zip -s ../../../ -t html --llvm --branch --ignore-not-existing -o ./coverage diff --git a/polkadot/zombienet_tests/README.md b/polkadot/zombienet_tests/README.md index 84334c3e1cfefadc053165c4646d384840268dc4..516ff880b2eb84ed7bc3ed7684b2f81def683e9b 100644 --- a/polkadot/zombienet_tests/README.md +++ b/polkadot/zombienet_tests/README.md @@ -1,34 +1,38 @@ # Zombienet tests -_The content of this directory is meant to be used by Parity's private CI/CD infrastructure with private tools. At the moment those tools are still early stage of development and we don't know if / when they will available for public use._ +_The content of this directory is meant to be used by Parity's private CI/CD infrastructure with private tools. At the +moment those tools are still early stage of development and we don't know if / when they will available for public use._ ## Contents of this directory -`parachains` - At the moment this directory only have one test related to parachains: `/parachains-smoke-test`, that check the parachain registration and the block height. +`parachains` At the moment this directory only have one test related to parachains: `/parachains-smoke-test`, that check + the parachain registration and the block height. ## Resources -* [zombienet repo](https://github.com/paritytech/zombienet) -* [zombienet book](https://paritytech.github.io/zombienet/) +- [zombienet repo](https://github.com/paritytech/zombienet) +- [zombienet book](https://paritytech.github.io/zombienet/) ## Running tests locally -To run any test locally use the native provider (`zombienet test -p native ...`) you need first build the binaries. They are: +To run any test locally use the native provider (`zombienet test -p native ...`) you need first build the binaries. They +are: -* adder-collator -> polkadot/target/testnet/adder-collator -* malus -> polkadot/target/testnet/malus -* polkadot -> polkadot/target/testnet/polkadot, polkadot/target/testnet/polkadot-prepare-worker, polkadot/target/testnet/polkadot-execute-worker -* polkadot-collator -> cumulus/target/release/polkadot-parachain -* undying-collator -> polkadot/target/testnet/undying-collator +- `adder-collator` -> `polkadot/target/testnet/adder-collator` +- `malus` -> `polkadot/target/testnet/malus` +- `polkadot` -> `polkadot/target/testnet/polkadot`, `polkadot/target/testnet/polkadot-prepare-worker`, + `polkadot/target/testnet/polkadot-execute-worker` +- `polkadot-collator` -> `cumulus/target/release/polkadot-parachain` +- `undying-collator` -> `polkadot/target/testnet/undying-collator` To build them use: -* adder-collator -> `cargo build --profile testnet -p test-parachain-adder-collator` -* undying-collator -> `cargo build --profile testnet -p test-parachain-undying-collator` -* malus -> `cargo build --profile testnet -p polkadot-test-malus` -* polkadot (in polkadot repo) and polkadot-collator (in cumulus repo) -> `cargo build --profile testnet` +- `adder-collator` -> `cargo build --profile testnet -p test-parachain-adder-collator` +- `undying-collator` -> `cargo build --profile testnet -p test-parachain-undying-collator` +- `malus` -> `cargo build --profile testnet -p polkadot-test-malus` +- `polkadot` (in the Polkadot repo) and `polkadot-collator` (in Cumulus repo) -> `cargo build --profile testnet` -One solution is to use the `.set_env` file (from this directory) and fill the `CUSTOM_PATHS` before *source* it to patch the PATH of your system to find the binaries you just built. +One solution is to use the `.set_env` file (from this directory) and fill the `CUSTOM_PATHS` before _source_ it to patch +the PATH of your system to find the binaries you just built. E.g.: ``` @@ -45,8 +49,9 @@ CUSTOM_PATHS=( source .set_env ``` -Then you have your `PATH` customized and ready to run `zombienet`. - **NOTE**: You should need to do this ones per terminal session, since we are patching the `PATH` and re-exporting. **Or** you can also `source` this file in your `.bashrc` file to get executed automatically in each new session. +Then you have your `PATH` customized and ready to run `zombienet`. **NOTE**: You should need to do this ones per + terminal session, since we are patching the `PATH` and re-exporting. **Or** you can also `source` this file in your + `.bashrc` file to get executed automatically in each new session. Example: @@ -57,4 +62,5 @@ zombienet test -p native 0001-parachains-pvf.zndsl ## Questions / permissions -Ping in element Javier (@javier:matrix.parity.io) to ask questions or grant permission to run the test from your local setup. +Ping in element Javier (`@javier:matrix.parity.io`) to ask questions or grant permission to run the test from your local +setup. diff --git a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml index 50c465950f727c566e95389af8c9b42ca942f242..25d922bb6827cbf8e129a07f64b537391bc72d7a 100644 --- a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml +++ b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.toml @@ -5,7 +5,7 @@ bootnode = true [relaychain.genesis.runtime.configuration.config] max_validators_per_core = 1 needed_approvals = 2 - group_rotation_frequency = 3 + group_rotation_frequency = 2 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}" @@ -39,8 +39,7 @@ id = 1000 cumulus_based = true [parachains.collator] - name = "collator" + name = "alice" command = "polkadot-parachain" - image = "docker.io/parity/polkadot-parachain:latest" - # image = "{{COL_IMAGE}}" + image = "{{CUMULUS_IMAGE}}" args = ["-lparachain=debug"] diff --git a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.zndsl b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.zndsl index bc3674f4f53ddef9ef87d1a2f301ed7a49d7161d..a3f1f0669ac9e85cea978ac2b62991cb3a092297 100644 --- a/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.zndsl +++ b/polkadot/zombienet_tests/functional/0005-parachains-disputes-past-session.zndsl @@ -21,18 +21,18 @@ malus-validator: resume malus-validator: log line matches "Suggesting malicious candidate" within 200 seconds # Pause first flaky node -# Availability and finality will continue with 3/4 nodes online (incl. malus) +# Availability will continue with 3/4 nodes online (incl. malus) honest-flaky-validator-0: pause # Wait for the dispute -honest-flaky-validator-1: reports parachain_candidate_disputes_total is at least 1 within 40 seconds +honest-flaky-validator-1: reports parachain_candidate_disputes_total is at least 1 within 60 seconds # Pause second flaky node so that we do not revert blocks due to f+1 invalid votes # Availability and finality will stop honest-flaky-validator-1: pause # Wait for 1 full session to pass after the last unconcluded dispute. -sleep 120 seconds +sleep 110 seconds # Now resume flaky validators honest-flaky-validator: resume diff --git a/prdoc/.gitkeep b/prdoc/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/prdoc/pr_1226.prdoc b/prdoc/pr_1226.prdoc new file mode 100644 index 0000000000000000000000000000000000000000..df7a425b538496d22040f2b5d9151623ea534585 --- /dev/null +++ b/prdoc/pr_1226.prdoc @@ -0,0 +1,17 @@ +title: Removed deprecated `Balances::transfer` and `Balances::set_balance_deprecated` functions. + +doc: + - audience: Builder + description: The Balances pallet's dispatchables `set_balance_deprecated` and `transfer` were deprecated in [paritytech/substrate#12951](https://github.com/paritytech/substrate/pull/12951) and have now been removed. + notes: + - Use `set_balance_deprecated` instead `force_set_balance` and `transfer_allow_death` instead of `transfer`. + +migrations: + db: [] + + runtime: [] + +crates: + - name: pallet-balances + +host_functions: [] diff --git a/prdoc/pr_1408_prodc-introduction.prdoc b/prdoc/pr_1408_prodc-introduction.prdoc new file mode 100644 index 0000000000000000000000000000000000000000..4b10e0fe2e8139e973ab31cabc7fb181c75f4fba --- /dev/null +++ b/prdoc/pr_1408_prodc-introduction.prdoc @@ -0,0 +1,19 @@ +# This PR does not need a prdoc but it is provided in order to test +title: PRdoc check + +doc: + - audience: Core Dev + description: | + This PRdoc is an **example**. + + This PR brings support and automated checks for documentation in the form of a + [`prdoc`](https://github.com/paritytech/prdoc/) file. + +migrations: + db: [] + + runtime: [] + +crates: [] + +host_functions: [] diff --git a/cumulus/scripts/bridges_update_subtree.sh b/scripts/bridges_update_subtree.sh similarity index 100% rename from cumulus/scripts/bridges_update_subtree.sh rename to scripts/bridges_update_subtree.sh diff --git a/substrate/.maintain/getgoing.sh b/substrate/.maintain/getgoing.sh deleted file mode 100644 index 98f360837d04aca82569eeb04dc2147803b69641..0000000000000000000000000000000000000000 --- a/substrate/.maintain/getgoing.sh +++ /dev/null @@ -1,6 +0,0 @@ -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -brew install openssl cmake -curl https://sh.rustup.rs -sSf | sh -source ~/.cargo/env -cargo install --git https://github.com/paritytech/substrate subkey -cargo install --git https://github.com/paritytech/substrate substrate diff --git a/substrate/.maintain/runtime-dep.py b/substrate/.maintain/runtime-dep.py deleted file mode 100755 index 3198bb3e2669e9bcc41e0c26e534028747d377e9..0000000000000000000000000000000000000000 --- a/substrate/.maintain/runtime-dep.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -# To run this script, you need to install the 'toml' python package and install the 'graphviz' package: -# pip install toml -# sudo apt-get install graphviz -# the first parameter is the runtime folder -# python ./.maintain/runtime-dep.py ./substrate/runtime | dot -Tpng -o output.png -import sys -import os -import toml - -if len(sys.argv) != 2: - print("needs the runtime folder.") - sys.exit(-1) - -runtime_dir = sys.argv[1] - -files = [os.path.join(runtime_dir, f, 'Cargo.toml') for f in os.listdir(runtime_dir) if os.path.isfile(os.path.join(runtime_dir, f, 'Cargo.toml')) and f != 'example'] - -print("digraph G {") - - -PREFIX = "substrate-runtime-" - -for f in files: - manifest = toml.load(f) - - package_name = manifest['package']['name'] - deps = [d for d in manifest['dependencies'].keys() if d.startswith(PREFIX)] - - for d in deps: - print(" \"{}\" -> \"{}\";".format(package_name, d)) - -print("}") diff --git a/substrate/.maintain/rustdocs-release.sh b/substrate/.maintain/rustdocs-release.sh index 2a1e141e63ad21c35e4b904764f64141ce838da1..091f9289e4e3438dda6976d5893a2ae85fe8f02f 100755 --- a/substrate/.maintain/rustdocs-release.sh +++ b/substrate/.maintain/rustdocs-release.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # set -x -# This script manages the deployment of Substrate rustdocs to https://paritytech.github.io/substrate/. +# This script used to manage the deployment of Substrate rustdocs to https://paritytech.github.io/substrate/. +# It is no longer used anywhere, and only here for historical/demonstration purposes. # - With `deploy` sub-command, it will checkout the passed-in branch/tag ref, build the rustdocs # locally (this takes some time), update the `index.html` index page, and push it to remote # `gh-pages` branch. So users running this command need to have write access to the remote diff --git a/substrate/.maintain/update-deps.sh b/substrate/.maintain/update-deps.sh deleted file mode 100755 index cd6b7c853825ed42a0b55d32018b81207a3d1642..0000000000000000000000000000000000000000 --- a/substrate/.maintain/update-deps.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -- -set -eu -case $0 in - (/*) dir=${0%/*}/;; - (*/*) dir=./${0%/*};; - (*) dir=.;; -esac - -find "$dir/.." -name Cargo.lock -execdir cargo update \; diff --git a/substrate/README.md b/substrate/README.md index 6ed64ac82eedda63aee7d615ab9a2a42aec98b6d..f7afa7a894d88f7bf57c8c7f6fb870a3e60c1b73 100644 --- a/substrate/README.md +++ b/substrate/README.md @@ -1,27 +1,35 @@ -# Substrate · [![GitHub license](https://img.shields.io/badge/license-GPL3%2FApache2-blue)](#LICENSE) [![GitLab Status](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/badges/master/pipeline.svg)](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/pipelines) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](docs/CONTRIBUTING.md) [![Stack Exchange](https://img.shields.io/badge/Substrate-Community%20&%20Support-24CC85?logo=stackexchange)](https://substrate.stackexchange.com/) +# Substrate + +[![GitHub license](https://img.shields.io/badge/license-GPL3%2FApache2-blue)](#LICENSE) +[![GitLab +Status](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/badges/master/pipeline.svg)](https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/pipelines) +[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](docs/CONTRIBUTING.md) +[![Stack +Exchange](https://img.shields.io/badge/Substrate-Community%20&%20Support-24CC85?logo=stackexchange)](https://substrate.stackexchange.com/)

- +

Substrate is a next-generation framework for blockchain innovation 🚀. ## Getting Started -Head to [docs.substrate.io](https://docs.substrate.io) and follow the -[installation](https://docs.substrate.io/install/) instructions. Then try out one of the -[tutorials](https://docs.substrate.io/tutorials/). Refer to the [Docker instructions](./docker/README.md) to quickly run Substrate, Substrate Node Template, Subkey, or to build a chain spec. +Head to [`docs.substrate.io`](https://docs.substrate.io) and follow the [installation](https://docs.substrate.io/install/) +instructions. Then try out one of the [tutorials](https://docs.substrate.io/tutorials/). Refer to the [Docker +instructions](./docker/README.md) to quickly run Substrate, Substrate Node Template, Subkey, or to build a chain spec. ## Community & Support -Join the highly active and supportive community on the -[Substrate Stack Exchange](https://substrate.stackexchange.com/) to ask questions about use and problems you run into using this software. -Please do report bugs and [issues here](https://github.com/paritytech/polkadot-sdk/issues) for anything you suspect requires action in the source. +Join the highly active and supportive community on the [Substrate Stack Exchange](https://substrate.stackexchange.com/) +to ask questions about use and problems you run into using this software. Please do report bugs and [issues +here](https://github.com/paritytech/polkadot-sdk/issues) for anything you suspect requires action in the source. ## Contributions & Code of Conduct Please follow the contributions guidelines as outlined in -[`docs/CONTRIBUTING.md`](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CONTRIBUTING.md). -In all communications and contributions, this project follows the [Contributor Covenant Code of Conduct](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CODE_OF_CONDUCT.md). +[`docs/CONTRIBUTING.md`](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CONTRIBUTING.md). In all +communications and contributions, this project follows the [Contributor Covenant Code of +Conduct](https://github.com/paritytech/polkadot-sdk/blob/master/docs/CODE_OF_CONDUCT.md). ## Security @@ -30,15 +38,13 @@ The security policy and procedures can be found in ## License -- Substrate Primitives (`sp-*`), Frame (`frame-*`) and the pallets (`pallets-*`), binaries (`/bin`) -and all other utilities are licensed under [Apache 2.0](LICENSE-APACHE2). - Substrate Client -(`/client/*` / `sc-*`) is licensed under [GPL v3.0 with a classpath linking -exception](LICENSE-GPL3). +- Substrate Primitives (`sp-*`), Frame (`frame-*`) and the pallets (`pallets-*`), binaries (`/bin`) and all other +utilities are licensed under [Apache 2.0](LICENSE-APACHE2). - Substrate Client (`/client/*` / `sc-*`) is licensed under +[GPL v3.0 with a classpath linking exception](LICENSE-GPL3). -The reason for the split-licensing is to ensure that for the vast majority of teams using Substrate -to create feature-chains, then all changes can be made entirely in Apache2-licensed code, allowing -teams full freedom over what and how they release and giving licensing clarity to commercial teams. +The reason for the split-licensing is to ensure that for the vast majority of teams using Substrate to create +feature-chains, then all changes can be made entirely in Apache2-licensed code, allowing teams full freedom over what +and how they release and giving licensing clarity to commercial teams. -In the interests of the community, we require any deeper improvements made to Substrate's core logic -(e.g. Substrate's internal consensus, crypto or database code) to be contributed back so everyone -can benefit. +In the interests of the community, we require any deeper improvements made to Substrate's core logic (e.g. Substrate's +internal consensus, crypto or database code) to be contributed back so everyone can benefit. diff --git a/substrate/bin/node-template/README.md b/substrate/bin/node-template/README.md index 337facaaf089864b1eec753be19836ed97224540..a07328df88c1bf86907bee784474907a17cf6bc8 100644 --- a/substrate/bin/node-template/README.md +++ b/substrate/bin/node-template/README.md @@ -2,17 +2,26 @@ A fresh [Substrate](https://substrate.io/) node, ready for hacking :rocket: -A standalone version of this template is available for each release of Polkadot in the [Substrate Developer Hub Parachain Template](https://github.com/substrate-developer-hub/substrate-parachain-template/) repository. -The parachain template is generated directly at each Polkadot release branch from the [Node Template in Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) upstream - -It is usually best to use the stand-alone version to start a new project. -All bugs, suggestions, and feature requests should be made upstream in the [Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) repository. +A standalone version of this template is available for each release of Polkadot +in the [Substrate Developer Hub Parachain +Template](https://github.com/substrate-developer-hub/substrate-parachain-template/) +repository. The parachain template is generated directly at each Polkadot +release branch from the [Node Template in +Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) +upstream + +It is usually best to use the stand-alone version to start a new project. All +bugs, suggestions, and feature requests should be made upstream in the +[Substrate](https://github.com/paritytech/substrate/tree/master/bin/node-template) +repository. ## Getting Started -Depending on your operating system and Rust version, there might be additional packages required to compile this template. -Check the [Install](https://docs.substrate.io/install/) instructions for your platform for the most common dependencies. -Alternatively, you can use one of the [alternative installation](#alternatives-installations) options. +Depending on your operating system and Rust version, there might be additional +packages required to compile this template. Check the +[Install](https://docs.substrate.io/install/) instructions for your platform for +the most common dependencies. Alternatively, you can use one of the [alternative +installation](#alternatives-installations) options. ### Build @@ -24,13 +33,16 @@ cargo build --release ### Embedded Docs -After you build the project, you can use the following command to explore its parameters and subcommands: +After you build the project, you can use the following command to explore its +parameters and subcommands: ```sh ./target/release/node-template -h ``` -You can generate and view the [Rust Docs](https://doc.rust-lang.org/cargo/commands/cargo-doc.html) for this template with this command: +You can generate and view the [Rust +Docs](https://doc.rust-lang.org/cargo/commands/cargo-doc.html) for this template +with this command: ```sh cargo +nightly doc --open @@ -38,7 +50,8 @@ cargo +nightly doc --open ### Single-Node Development Chain -The following command starts a single-node development chain that doesn't persist state: +The following command starts a single-node development chain that doesn't +persist state: ```sh ./target/release/node-template --dev @@ -61,10 +74,12 @@ Development chains: - Maintain state in a `tmp` folder while the node is running. - Use the **Alice** and **Bob** accounts as default validator authorities. - Use the **Alice** account as the default `sudo` account. -- Are preconfigured with a genesis state (`/node/src/chain_spec.rs`) that includes several prefunded development accounts. +- Are preconfigured with a genesis state (`/node/src/chain_spec.rs`) that + includes several prefunded development accounts. -To persist chain state between runs, specify a base path by running a command similar to the following: +To persist chain state between runs, specify a base path by running a command +similar to the following: ```sh // Create a folder to use as the db base path @@ -84,81 +99,127 @@ db keystore network ### Connect with Polkadot-JS Apps Front-End -After you start the node template locally, you can interact with it using the hosted version of the [Polkadot/Substrate Portal](https://polkadot.js.org/apps/#/explorer?rpc=ws://localhost:9944) front-end by connecting to the local node endpoint. -A hosted version is also available on [IPFS (redirect) here](https://dotapps.io/) or [IPNS (direct) here](ipns://dotapps.io/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer). -You can also find the source code and instructions for hosting your own instance on the [polkadot-js/apps](https://github.com/polkadot-js/apps) repository. +After you start the node template locally, you can interact with it using the +hosted version of the [Polkadot/Substrate +Portal](https://polkadot.js.org/apps/#/explorer?rpc=ws://localhost:9944) +front-end by connecting to the local node endpoint. A hosted version is also +available on [IPFS (redirect) here](https://dotapps.io/) or [IPNS (direct) +here](ipns://dotapps.io/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer). You can +also find the source code and instructions for hosting your own instance on the +[`polkadot-js/apps`](https://github.com/polkadot-js/apps) repository. ### Multi-Node Local Testnet -If you want to see the multi-node consensus algorithm in action, see [Simulate a network](https://docs.substrate.io/tutorials/build-a-blockchain/simulate-network/). +If you want to see the multi-node consensus algorithm in action, see [Simulate a +network](https://docs.substrate.io/tutorials/build-a-blockchain/simulate-network/). ## Template Structure -A Substrate project such as this consists of a number of components that are spread across a few directories. +A Substrate project such as this consists of a number of components that are +spread across a few directories. ### Node -A blockchain node is an application that allows users to participate in a blockchain network. -Substrate-based blockchain nodes expose a number of capabilities: - -- Networking: Substrate nodes use the [`libp2p`](https://libp2p.io/) networking stack to allow the - nodes in the network to communicate with one another. -- Consensus: Blockchains must have a way to come to [consensus](https://docs.substrate.io/fundamentals/consensus/) on the state of the network. - Substrate makes it possible to supply custom consensus engines and also ships with several consensus mechanisms that have been built on top of [Web3 Foundation research](https://research.web3.foundation/en/latest/polkadot/NPoS/index.html). -- RPC Server: A remote procedure call (RPC) server is used to interact with Substrate nodes. - -There are several files in the `node` directory. -Take special note of the following: - -- [`chain_spec.rs`](./node/src/chain_spec.rs): A [chain specification](https://docs.substrate.io/build/chain-spec/) is a source code file that defines a Substrate chain's initial (genesis) state. - Chain specifications are useful for development and testing, and critical when architecting the launch of a production chain. - Take note of the `development_config` and `testnet_genesis` functions,. - These functions are used to define the genesis state for the local development chain configuration. - These functions identify some [well-known accounts](https://docs.substrate.io/reference/command-line-tools/subkey/) and use them to configure the blockchain's initial state. -- [`service.rs`](./node/src/service.rs): This file defines the node implementation. - Take note of the libraries that this file imports and the names of the functions it invokes. - In particular, there are references to consensus-related topics, such as the [block finalization and forks](https://docs.substrate.io/fundamentals/consensus/#finalization-and-forks) and other [consensus mechanisms](https://docs.substrate.io/fundamentals/consensus/#default-consensus-models) such as Aura for block authoring and GRANDPA for finality. - +A blockchain node is an application that allows users to participate in a +blockchain network. Substrate-based blockchain nodes expose a number of +capabilities: + +- Networking: Substrate nodes use the [`libp2p`](https://libp2p.io/) networking + stack to allow the nodes in the network to communicate with one another. +- Consensus: Blockchains must have a way to come to + [consensus](https://docs.substrate.io/fundamentals/consensus/) on the state of + the network. Substrate makes it possible to supply custom consensus engines + and also ships with several consensus mechanisms that have been built on top + of [Web3 Foundation + research](https://research.web3.foundation/en/latest/polkadot/NPoS/index.html). +- RPC Server: A remote procedure call (RPC) server is used to interact with + Substrate nodes. + +There are several files in the `node` directory. Take special note of the +following: + +- [`chain_spec.rs`](./node/src/chain_spec.rs): A [chain + specification](https://docs.substrate.io/build/chain-spec/) is a source code + file that defines a Substrate chain's initial (genesis) state. Chain + specifications are useful for development and testing, and critical when + architecting the launch of a production chain. Take note of the + `development_config` and `testnet_genesis` functions,. These functions are + used to define the genesis state for the local development chain + configuration. These functions identify some [well-known + accounts](https://docs.substrate.io/reference/command-line-tools/subkey/) and + use them to configure the blockchain's initial state. +- [`service.rs`](./node/src/service.rs): This file defines the node + implementation. Take note of the libraries that this file imports and the + names of the functions it invokes. In particular, there are references to + consensus-related topics, such as the [block finalization and + forks](https://docs.substrate.io/fundamentals/consensus/#finalization-and-forks) + and other [consensus + mechanisms](https://docs.substrate.io/fundamentals/consensus/#default-consensus-models) + such as Aura for block authoring and GRANDPA for finality. ### Runtime In Substrate, the terms "runtime" and "state transition function" are analogous. -Both terms refer to the core logic of the blockchain that is responsible for validating blocks and executing the state changes they define. -The Substrate project in this repository uses [FRAME](https://docs.substrate.io/learn/runtime-development/#frame) to construct a blockchain runtime. -FRAME allows runtime developers to declare domain-specific logic in modules called "pallets". -At the heart of FRAME is a helpful [macro language](https://docs.substrate.io/reference/frame-macros/) that makes it easy to create pallets and flexibly compose them to create blockchains that can address [a variety of needs](https://substrate.io/ecosystem/projects/). - -Review the [FRAME runtime implementation](./runtime/src/lib.rs) included in this template and note the following: - -- This file configures several pallets to include in the runtime. - Each pallet configuration is defined by a code block that begins with `impl $PALLET_NAME::Config for Runtime`. -- The pallets are composed into a single runtime by way of the [`construct_runtime!`](https://paritytech.github.io/substrate/master/frame_support/macro.construct_runtime.html) macro, which is part of the [core FRAME pallet library](https://docs.substrate.io/reference/frame-pallets/#system-pallets). +Both terms refer to the core logic of the blockchain that is responsible for +validating blocks and executing the state changes they define. The Substrate +project in this repository uses +[FRAME](https://docs.substrate.io/learn/runtime-development/#frame) to construct +a blockchain runtime. FRAME allows runtime developers to declare domain-specific +logic in modules called "pallets". At the heart of FRAME is a helpful [macro +language](https://docs.substrate.io/reference/frame-macros/) that makes it easy +to create pallets and flexibly compose them to create blockchains that can +address [a variety of needs](https://substrate.io/ecosystem/projects/). + +Review the [FRAME runtime implementation](./runtime/src/lib.rs) included in this +template and note the following: + +- This file configures several pallets to include in the runtime. Each pallet + configuration is defined by a code block that begins with `impl + $PALLET_NAME::Config for Runtime`. +- The pallets are composed into a single runtime by way of the + [`construct_runtime!`](https://paritytech.github.io/substrate/master/frame_support/macro.construct_runtime.html) + macro, which is part of the [core FRAME pallet + library](https://docs.substrate.io/reference/frame-pallets/#system-pallets). ### Pallets -The runtime in this project is constructed using many FRAME pallets that ship with [the Substrate repository](https://github.com/paritytech/substrate/tree/master/frame) and a template pallet that is [defined in the `pallets`](./pallets/template/src/lib.rs) directory. +The runtime in this project is constructed using many FRAME pallets that ship +with [the Substrate +repository](https://github.com/paritytech/substrate/tree/master/frame) and a +template pallet that is [defined in the +`pallets`](./pallets/template/src/lib.rs) directory. A FRAME pallet is comprised of a number of blockchain primitives, including: -- Storage: FRAME defines a rich set of powerful [storage abstractions](https://docs.substrate.io/build/runtime-storage/) that makes it easy to use Substrate's efficient key-value database to manage the evolving state of a blockchain. -- Dispatchables: FRAME pallets define special types of functions that can be invoked (dispatched) from outside of the runtime in order to update its state. -- Events: Substrate uses [events](https://docs.substrate.io/build/events-and-errors/) to notify users of significant state changes. +- Storage: FRAME defines a rich set of powerful [storage + abstractions](https://docs.substrate.io/build/runtime-storage/) that makes it + easy to use Substrate's efficient key-value database to manage the evolving + state of a blockchain. +- Dispatchables: FRAME pallets define special types of functions that can be + invoked (dispatched) from outside of the runtime in order to update its state. +- Events: Substrate uses + [events](https://docs.substrate.io/build/events-and-errors/) to notify users + of significant state changes. - Errors: When a dispatchable fails, it returns an error. -Each pallet has its own `Config` trait which serves as a configuration interface to generically define the types and parameters it depends on. +Each pallet has its own `Config` trait which serves as a configuration interface +to generically define the types and parameters it depends on. ## Alternatives Installations -Instead of installing dependencies and building this source directly, consider the following alternatives. +Instead of installing dependencies and building this source directly, consider +the following alternatives. ### Nix Install [nix](https://nixos.org/) and -[nix-direnv](https://github.com/nix-community/nix-direnv) for a fully plug-and-play -experience for setting up the development environment. -To get all the correct dependencies, activate direnv `direnv allow`. +[nix-direnv](https://github.com/nix-community/nix-direnv) for a fully +plug-and-play experience for setting up the development environment. To get all +the correct dependencies, activate direnv `direnv allow`. ### Docker -Please follow the [Substrate Docker instructions here](https://github.com/paritytech/substrate/blob/master/docker/README.md) to build the Docker container with the Substrate Node Template binary. +Please follow the [Substrate Docker instructions +here](https://github.com/paritytech/substrate/blob/master/docker/README.md) to +build the Docker container with the Substrate Node Template binary. diff --git a/substrate/bin/node-template/docs/rust-setup.md b/substrate/bin/node-template/docs/rust-setup.md index 133db0277bb4b42be891dcc58bf04d3ada8b1bd4..38fddd5026bb0d81a38929ed1f1267b310cc0d73 100644 --- a/substrate/bin/node-template/docs/rust-setup.md +++ b/substrate/bin/node-template/docs/rust-setup.md @@ -1,22 +1,18 @@ ---- -title: Installation ---- +# Installation -This guide is for reference only, please check the latest information on getting started with Substrate -[here](https://docs.substrate.io/main-docs/install/). +This guide is for reference only, please check the latest information on getting started with Substrate [here](https://docs.substrate.io/main-docs/install/). -This page will guide you through the **2 steps** needed to prepare a computer for **Substrate** development. -Since Substrate is built with [the Rust programming language](https://www.rust-lang.org/), the first -thing you will need to do is prepare the computer for Rust development - these steps will vary based -on the computer's operating system. Once Rust is configured, you will use its toolchains to interact -with Rust projects; the commands for Rust's toolchains will be the same for all supported, -Unix-based operating systems. +This page will guide you through the **2 steps** needed to prepare a computer for **Substrate** development. Since +Substrate is built with [the Rust programming language](https://www.rust-lang.org/), the first thing you will need to do +is prepare the computer for Rust development - these steps will vary based on the computer's operating system. Once Rust +is configured, you will use its toolchains to interact with Rust projects; the commands for Rust's toolchains will be +the same for all supported, Unix-based operating systems. ## Build dependencies -Substrate development is easiest on Unix-based operating systems like macOS or Linux. The examples -in the [Substrate Docs](https://docs.substrate.io) use Unix-style terminals to demonstrate how to -interact with Substrate from the command line. +Substrate development is easiest on Unix-based operating systems like macOS or Linux. The examples in the [Substrate +Docs](https://docs.substrate.io) use Unix-style terminals to demonstrate how to interact with Substrate from the command +line. ### Ubuntu/Debian @@ -55,10 +51,9 @@ sudo zypper install clang curl git openssl-devel llvm-devel libudev-devel ### macOS -> **Apple M1 ARM** -> If you have an Apple M1 ARM system on a chip, make sure that you have Apple Rosetta 2 -> installed through `softwareupdate --install-rosetta`. This is only needed to run the -> `protoc` tool during the build. The build itself and the target binaries would remain native. +> **Apple M1 ARM** If you have an Apple M1 ARM system on a chip, make sure that you have Apple Rosetta 2 installed +> through `softwareupdate --install-rosetta`. This is only needed to run the `protoc` tool during the build. The build +> itself and the target binaries would remain native. Open the Terminal application and execute the following commands: @@ -81,8 +76,8 @@ Please refer to the separate ## Rust developer environment -This guide uses installer and the `rustup` tool to manage the Rust toolchain. -First install and configure `rustup`: +This guide uses installer and the `rustup` tool to manage the Rust toolchain. First install and +configure `rustup`: ```bash # Install @@ -102,13 +97,13 @@ rustup target add wasm32-unknown-unknown --toolchain nightly ## Test your set-up -Now the best way to ensure that you have successfully prepared a computer for Substrate -development is to follow the steps in [our first Substrate tutorial](https://docs.substrate.io/tutorials/v3/create-your-first-substrate-chain/). +Now the best way to ensure that you have successfully prepared a computer for Substrate development is to follow the +steps in [our first Substrate tutorial](https://docs.substrate.io/tutorials/v3/create-your-first-substrate-chain/). ## Troubleshooting Substrate builds -Sometimes you can't get the Substrate node template -to compile out of the box. Here are some tips to help you work through that. +Sometimes you can't get the Substrate node template to compile out of the box. Here are some tips to help you work +through that. ### Rust configuration check @@ -144,27 +139,27 @@ stable-x86_64-unknown-linux-gnu (default) rustc 1.50.0 (cb75ad5db 2021-02-10) ``` -As you can see above, the default toolchain is stable, and the -`nightly-x86_64-unknown-linux-gnu` toolchain as well as its `wasm32-unknown-unknown` target is installed. -You also see that `nightly-2020-10-06-x86_64-unknown-linux-gnu` is installed, but is not used unless explicitly defined as illustrated in the [specify your nightly version](#specifying-nightly-version) -section. +As you can see above, the default toolchain is stable, and the `nightly-x86_64-unknown-linux-gnu` toolchain as well as +its `wasm32-unknown-unknown` target is installed. You also see that `nightly-2020-10-06-x86_64-unknown-linux-gnu` is +installed, but is not used unless explicitly defined as illustrated in the [specify your nightly +version](#specifying-nightly-version) section. ### WebAssembly compilation -Substrate uses [WebAssembly](https://webassembly.org) (Wasm) to produce portable blockchain -runtimes. You will need to configure your Rust compiler to use -[`nightly` builds](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html) to allow you to -compile Substrate runtime code to the Wasm target. +Substrate uses [WebAssembly](https://webassembly.org) (Wasm) to produce portable blockchain runtimes. You will need to +configure your Rust compiler to use [`nightly` builds](https://doc.rust-lang.org/book/appendix-07-nightly-rust.html) to +allow you to compile Substrate runtime code to the Wasm target. > There are upstream issues in Rust that need to be resolved before all of Substrate can use the stable Rust toolchain. -> [This is our tracking issue](https://github.com/paritytech/substrate/issues/1252) if you're curious as to why and how this will be resolved. +> [This is our tracking issue](https://github.com/paritytech/substrate/issues/1252) if you're curious as to why and how +> this will be resolved. #### Latest nightly for Substrate `master` -Developers who are building Substrate _itself_ should always use the latest bug-free versions of -Rust stable and nightly. This is because the Substrate codebase follows the tip of Rust nightly, -which means that changes in Substrate often depend on upstream changes in the Rust nightly compiler. -To ensure your Rust compiler is always up to date, you should run: +Developers who are building Substrate _itself_ should always use the latest bug-free versions of Rust stable and +nightly. This is because the Substrate codebase follows the tip of Rust nightly, which means that changes in Substrate +often depend on upstream changes in the Rust nightly compiler. To ensure your Rust compiler is always up to date, you +should run: ```bash rustup update @@ -172,21 +167,19 @@ rustup update nightly rustup target add wasm32-unknown-unknown --toolchain nightly ``` -> NOTE: It may be necessary to occasionally rerun `rustup update` if a change in the upstream Substrate -> codebase depends on a new feature of the Rust compiler. When you do this, both your nightly -> and stable toolchains will be pulled to the most recent release, and for nightly, it is -> generally _not_ expected to compile WASM without error (although it very often does). -> Be sure to [specify your nightly version](#specifying-nightly-version) if you get WASM build errors -> from `rustup` and [downgrade nightly as needed](#downgrading-rust-nightly). +> NOTE: It may be necessary to occasionally rerun `rustup update` if a change in the upstream Substrate codebase depends +> on a new feature of the Rust compiler. When you do this, both your nightly and stable toolchains will be pulled to the +> most recent release, and for nightly, it is generally _not_ expected to compile WASM without error (although it very +> often does). Be sure to [specify your nightly version](#specifying-nightly-version) if you get WASM build errors from +> `rustup` and [downgrade nightly as needed](#downgrading-rust-nightly). #### Rust nightly toolchain -If you want to guarantee that your build works on your computer as you update Rust and other -dependencies, you should use a specific Rust nightly version that is known to be -compatible with the version of Substrate they are using; this version will vary from project to -project and different projects may use different mechanisms to communicate this version to -developers. For instance, the Polkadot client specifies this information in its -[release notes](https://github.com/paritytech/polkadot/releases). +If you want to guarantee that your build works on your computer as you update Rust and other dependencies, you should +use a specific Rust nightly version that is known to be compatible with the version of Substrate they are using; this +version will vary from project to project and different projects may use different mechanisms to communicate this +version to developers. For instance, the Polkadot client specifies this information in its [release +notes](https://github.com/paritytech/polkadot/releases). ```bash # Specify the specific nightly toolchain in the date below: @@ -203,20 +196,20 @@ rustup target add wasm32-unknown-unknown --toolchain nightly- ### Specifying nightly version -Use the `WASM_BUILD_TOOLCHAIN` environment variable to specify the Rust nightly version a Substrate -project should use for Wasm compilation: +Use the `WASM_BUILD_TOOLCHAIN` environment variable to specify the Rust nightly version a Substrate project should use +for Wasm compilation: ```bash WASM_BUILD_TOOLCHAIN=nightly- cargo build --release ``` -> Note that this only builds _the runtime_ with the specified nightly. The rest of project will be -> compiled with **your default toolchain**, i.e. the latest installed stable toolchain. +> Note that this only builds _the runtime_ with the specified nightly. The rest of project will be compiled with **your +> default toolchain**, i.e. the latest installed stable toolchain. ### Downgrading Rust nightly -If your computer is configured to use the latest Rust nightly and you would like to downgrade to a -specific nightly version, follow these steps: +If your computer is configured to use the latest Rust nightly and you would like to downgrade to a specific nightly +version, follow these steps: ```bash rustup uninstall nightly diff --git a/substrate/bin/node-template/node/Cargo.toml b/substrate/bin/node-template/node/Cargo.toml index 38eba5ee0e1f2e9119f9b93cfb64eb4795aa7656..35654e7d56499a535876f1d4e9434d4ac082cf9f 100644 --- a/substrate/bin/node-template/node/Cargo.toml +++ b/substrate/bin/node-template/node/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] name = "node-template" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } futures = { version = "0.3.21", features = ["thread-pool"]} sc-cli = { path = "../../../client/cli" } diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 7303f5cd6dd6d2eb23131fb102ef11ae955f4806..403202829241ee682a688a435ccdc687ffa3af62 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -183,6 +183,7 @@ pub fn new_full(config: Configuration) -> Result { import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; if config.offchain_worker.enabled { diff --git a/substrate/bin/node-template/pallets/template/README.md b/substrate/bin/node-template/pallets/template/README.md index d0d59537c12d7ef6490562eb4560d67506082a41..9e4dc55267d69c47fff971cb0427bcb2e0ff871c 100644 --- a/substrate/bin/node-template/pallets/template/README.md +++ b/substrate/bin/node-template/pallets/template/README.md @@ -1 +1 @@ -License: MIT-0 \ No newline at end of file +License: MIT-0 diff --git a/substrate/bin/node/bench/Cargo.toml b/substrate/bin/node/bench/Cargo.toml index a02d7393740aaa4bea4030eedb1e6bcc1028cd70..33560bca4923d38e6ce73bc999e5cce5859613ed 100644 --- a/substrate/bin/node/bench/Cargo.toml +++ b/substrate/bin/node/bench/Cargo.toml @@ -13,7 +13,7 @@ publish = false [dependencies] array-bytes = "6.1" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } log = "0.4.17" node-primitives = { path = "../primitives" } node-testing = { path = "../testing" } @@ -22,7 +22,7 @@ sc-client-api = { path = "../../../client/api" } sp-runtime = { path = "../../../primitives/runtime" } sp-state-machine = { path = "../../../primitives/state-machine" } serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" derive_more = { version = "0.99.17", default-features = false, features = ["display"] } kvdb = "0.13.0" kvdb-rocksdb = "0.19.0" diff --git a/substrate/bin/node/cli/Cargo.toml b/substrate/bin/node/cli/Cargo.toml index 28701db482d2be3aab69f78d7d4113282d250284..c47f8a5c3e52f4268cf5432d91e6f4cc9e956b52 100644 --- a/substrate/bin/node/cli/Cargo.toml +++ b/substrate/bin/node/cli/Cargo.toml @@ -38,7 +38,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies array-bytes = "6.1" -clap = { version = "4.4.2", features = ["derive"], optional = true } +clap = { version = "4.4.4", features = ["derive"], optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } serde = { version = "1.0.188", features = ["derive"] } jsonrpsee = { version = "0.16.2", features = ["server"] } @@ -106,7 +106,7 @@ sc-cli = { path = "../../../client/cli", optional = true} frame-benchmarking-cli = { path = "../../../utils/frame/benchmarking-cli", optional = true} node-inspect = { path = "../inspect", optional = true} try-runtime-cli = { path = "../../../utils/frame/try-runtime/cli", optional = true} -serde_json = "1.0.85" +serde_json = "1.0.107" [dev-dependencies] sc-keystore = { path = "../../../client/keystore" } @@ -135,7 +135,7 @@ pallet-timestamp = { path = "../../../frame/timestamp" } substrate-cli-test-utils = { path = "../../../test-utils/cli" } [build-dependencies] -clap = { version = "4.4.2", optional = true } +clap = { version = "4.4.4", optional = true } clap_complete = { version = "4.0.2", optional = true } node-inspect = { path = "../inspect", optional = true} frame-benchmarking-cli = { path = "../../../utils/frame/benchmarking-cli", optional = true} diff --git a/substrate/bin/node/cli/doc/shell-completion.adoc b/substrate/bin/node/cli/doc/shell-completion.md similarity index 75% rename from substrate/bin/node/cli/doc/shell-completion.adoc rename to substrate/bin/node/cli/doc/shell-completion.md index 168f00994fb2de175266447965576c6de6876591..3f009a04373d9decc3be3b3e12c204d7d18c33c0 100644 --- a/substrate/bin/node/cli/doc/shell-completion.adoc +++ b/substrate/bin/node/cli/doc/shell-completion.md @@ -1,12 +1,11 @@ +# Shell completion -== Shell completion - -The Substrate cli command supports shell auto-completion. For this to work, you will need to run the completion script matching your build and system. +The Substrate cli command supports shell auto-completion. For this to work, you will need to run the +completion script matching your build and system. Assuming you built a release version using `cargo build --release` and use `bash` run the following: -[source, shell] -source target/release/completion-scripts/substrate.bash +`source target/release/completion-scripts/substrate.bash` You can find completion scripts for: - bash @@ -17,25 +16,19 @@ You can find completion scripts for: To make this change persistent, you can proceed as follows: -.First install - -[source, shell] ----- +```shell COMPL_DIR=$HOME/.completion mkdir -p $COMPL_DIR cp -f target/release/completion-scripts/substrate.bash $COMPL_DIR/ echo "source $COMPL_DIR/substrate.bash" >> $HOME/.bash_profile source $HOME/.bash_profile ----- - -.Update +``` When you build a new version of Substrate, the following will ensure your auto-completion script matches the current binary: -[source, shell] ----- +```shell COMPL_DIR=$HOME/.completion mkdir -p $COMPL_DIR cp -f target/release/completion-scripts/substrate.bash $COMPL_DIR/ source $HOME/.bash_profile ----- +``` diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index ecca5c60db5156bbda32e012ed607db74e2d9fec..977c90e73e9ff0217ba8fb34cbd78be28a83092a 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -32,8 +32,7 @@ use sc_client_api::{Backend, BlockBackend}; use sc_consensus_babe::{self, SlotProportion}; use sc_executor::NativeElseWasmExecutor; use sc_network::{event::Event, NetworkEventStream, NetworkService}; -use sc_network_common::sync::warp::WarpSyncParams; -use sc_network_sync::SyncingService; +use sc_network_sync::{warp::WarpSyncParams, SyncingService}; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; use sc_statement_store::Store as StatementStore; use sc_telemetry::{Telemetry, TelemetryWorker}; @@ -389,6 +388,7 @@ pub fn new_full_base( import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; let role = config.role.clone(); diff --git a/substrate/bin/node/inspect/Cargo.toml b/substrate/bin/node/inspect/Cargo.toml index 2b6a5e1aea181df9d920e285522f5b5afe987875..06e9674117e687b61a5275036ca4319d521cdbf0 100644 --- a/substrate/bin/node/inspect/Cargo.toml +++ b/substrate/bin/node/inspect/Cargo.toml @@ -13,7 +13,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.6.1" } thiserror = "1.0" sc-cli = { path = "../../../client/cli" } diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 4f34e4ecd812221102c01e839df96472f5b8de99..df8fb06467d99dd74a5b749e66263170c34a3607 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -35,11 +35,12 @@ use frame_support::{ pallet_prelude::Get, parameter_types, traits::{ - fungible::{Balanced, Credit, ItemOf}, + fungible::{Balanced, Credit, HoldConsideration, ItemOf}, tokens::{nonfungibles_v2::Inspect, GetSalary, PayFromAccount}, AsEnsureOriginWithArg, ConstBool, ConstU128, ConstU16, ConstU32, Contains, Currency, EitherOfDiverse, EqualPrivilegeOnly, Imbalance, InsideBoth, InstanceFilter, - KeyOwnerProofSystem, LockIdentifier, Nothing, OnUnbalanced, WithdrawReasons, + KeyOwnerProofSystem, LinearStoragePrice, LockIdentifier, Nothing, OnUnbalanced, + WithdrawReasons, }, weights::{ constants::{ @@ -57,7 +58,7 @@ pub use node_primitives::{AccountId, Signature}; use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Moment, Nonce}; use pallet_asset_conversion::{NativeOrAssetId, NativeOrAssetIdConverter}; use pallet_broker::{CoreAssignment, CoreIndex, CoretimeInterface, PartsOf57600}; -use pallet_election_provider_multi_phase::SolutionAccuracyOf; +use pallet_election_provider_multi_phase::{GeometricDepositBase, SolutionAccuracyOf}; use pallet_im_online::sr25519::AuthorityId as ImOnlineId; use pallet_nfts::PalletFeatures; use pallet_nis::WithMaximumOf; @@ -447,10 +448,10 @@ impl pallet_glutton::Config for Runtime { } parameter_types! { - pub const PreimageMaxSize: u32 = 4096 * 1024; pub const PreimageBaseDeposit: Balance = 1 * DOLLARS; // One cent: $10,000 / MB pub const PreimageByteDeposit: Balance = 1 * CENTS; + pub const PreimageHoldReason: RuntimeHoldReason = RuntimeHoldReason::Preimage(pallet_preimage::HoldReason::Preimage); } impl pallet_preimage::Config for Runtime { @@ -458,8 +459,12 @@ impl pallet_preimage::Config for Runtime { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = PreimageBaseDeposit; - type ByteDeposit = PreimageByteDeposit; + type Consideration = HoldConsideration< + AccountId, + Balances, + PreimageHoldReason, + LinearStoragePrice, + >; } parameter_types! { @@ -689,7 +694,8 @@ parameter_types! { // signed config pub const SignedRewardBase: Balance = 1 * DOLLARS; - pub const SignedDepositBase: Balance = 1 * DOLLARS; + pub const SignedFixedDeposit: Balance = 1 * DOLLARS; + pub const SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); pub const SignedDepositByte: Balance = 1 * CENTS; pub BetterUnsignedThreshold: Perbill = Perbill::from_rational(1u32, 10_000); @@ -817,7 +823,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerConfig = Self; type SignedMaxSubmissions = ConstU32<10>; type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; + type SignedDepositBase = + GeometricDepositBase; type SignedDepositByte = SignedDepositByte; type SignedMaxRefunds = ConstU32<3>; type SignedDepositWeight = (); diff --git a/substrate/bin/utils/chain-spec-builder/Cargo.toml b/substrate/bin/utils/chain-spec-builder/Cargo.toml index fcc97e7809bbf1d96de8d2d270ad6891b2e08107..f564ff19af0f326de3ca989688391e1892a1c4a9 100644 --- a/substrate/bin/utils/chain-spec-builder/Cargo.toml +++ b/substrate/bin/utils/chain-spec-builder/Cargo.toml @@ -22,7 +22,7 @@ crate-type = ["rlib"] [dependencies] ansi_term = "0.12.1" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } rand = "0.8" node-cli = { path = "../../node/cli" } sc-chain-spec = { path = "../../../client/chain-spec" } diff --git a/substrate/bin/utils/subkey/Cargo.toml b/substrate/bin/utils/subkey/Cargo.toml index 98276863f8deb1c927f289f6fa62d0cb9d84f8f4..4e8cb606c94e5995ac03b3021a50ce78f0970e30 100644 --- a/substrate/bin/utils/subkey/Cargo.toml +++ b/substrate/bin/utils/subkey/Cargo.toml @@ -17,5 +17,5 @@ path = "src/main.rs" name = "subkey" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } sc-cli = { path = "../../../client/cli" } diff --git a/substrate/bin/utils/subkey/README.md b/substrate/bin/utils/subkey/README.md index d19ccefb59aae8efc581d848ce466bce23bf1825..60e5a9ca935008e81518263b2802b3eb1e90d636 100644 --- a/substrate/bin/utils/subkey/README.md +++ b/substrate/bin/utils/subkey/README.md @@ -1,26 +1,33 @@ # Subkey -Subkey is a commandline utility included with Substrate. It allows generating and restoring keys for Substrate based chains such as Polkadot, Kusama and a growing number of parachains and Substrate based projects. +Subkey is a commandline utility included with Substrate. It allows generating and restoring keys for Substrate based +chains such as Polkadot, Kusama and a growing number of parachains and Substrate based projects. `subkey` provides a few sub-commands to generate keys, check keys, sign messages, verify messages, etc... -You can see the full list of commands with `subkey --help`. Most commands have additional help available with for instance `subkey generate --help` for the `generate` command. +You can see the full list of commands with `subkey --help`. Most commands have additional help available with for +instance `subkey generate --help` for the `generate` command. ## Safety first -`subkey` does not need an internet connection to work. Indeed, for the best security, you should be using `subkey` on a machine that is **not connected** to the internet. +`subkey` does not need an internet connection to work. Indeed, for the best security, you should be using `subkey` on a +machine that is **not connected** to the internet. -`subkey` deals with **seeds** and **private keys**. Make sure to use `subkey` in a safe environment (ie. no one looking over your shoulder) and on a safe computer (ie. no one able to check your command history). +`subkey` deals with **seeds** and **private keys**. Make sure to use `subkey` in a safe environment (ie. no one looking +over your shoulder) and on a safe computer (ie. no one able to check your command history). -If you save any output of `subkey` into a file, make sure to apply proper permissions and/or delete the file as soon as possible. +If you save any output of `subkey` into a file, make sure to apply proper permissions and/or delete the file as soon as +possible. ## Usage -The following guide explains *some* of the `subkey` commands. For the full list and the most up to date documentation, make sure to check the integrated help with `subkey --help`. +The following guide explains *some* of the `subkey` commands. For the full list and the most up to date documentation, +make sure to check the integrated help with `subkey --help`. ### Install with Cargo -You will need to have the Substrate build dependencies to install Subkey. Use the following two commands to install the dependencies and Subkey, respectively: +You will need to have the Substrate build dependencies to install Subkey. Use the following two commands to install the +dependencies and Subkey, respectively: Command: @@ -58,19 +65,26 @@ Secret phrase `hotel forest jar hover kite book view eight stuff angle legend de --- ☠️ DO NT RE-USE ANY OF THE SEEDS AND SECRETS FROM THIS PAGE ☠️. -You can read more about security and risks in [SECURITY.md](./SECURITY.md) and in the [Polkadot Wiki](https://wiki.polkadot.network/docs/learn-account-generation). +You can read more about security and risks in [SECURITY.md](./SECURITY.md) and in the [Polkadot +Wiki](https://wiki.polkadot.network/docs/learn-account-generation). --- -The output above shows a **secret phrase** (also called **mnemonic phrase**) and the **secret seed** (also called **Private Key**). Those 2 secrets are the pieces of information you MUST keep safe and secret. All the other information below can be derived from those secrets. +The output above shows a **secret phrase** (also called **mnemonic phrase**) and the **secret seed** (also called +**Private Key**). Those 2 secrets are the pieces of information you MUST keep safe and secret. All the other information +below can be derived from those secrets. -The output above also show the **public key** and the **Account ID**. Those are the independant from the network where you will use the key. +The output above also show the **public key** and the **Account ID**. Those are the independant from the network where +you will use the key. -The **SS58 address** (or **Public Address**) of a new account is a reprensentation of the public keys of an account for a given network (for instance Kusama or Polkadot). +The **SS58 address** (or **Public Address**) of a new account is a reprensentation of the public keys of an account for +a given network (for instance Kusama or Polkadot). -You can read more about the [SS58 format in the Substrate Docs](https://docs.substrate.io/reference/address-formats/) and see the list of reserved prefixes in the [SS58 Registry](https://github.com/paritytech/ss58-registry). +You can read more about the [SS58 format in the Substrate Docs](https://docs.substrate.io/reference/address-formats/) +and see the list of reserved prefixes in the [SS58 Registry](https://github.com/paritytech/ss58-registry). -For instance, considering the previous seed `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` the SS58 addresses are: +For instance, considering the previous seed `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c92d` the +SS58 addresses are: - Polkadot: `16m4J167Mptt8UXL8aGSAi7U2FnPpPxZHPrCgMG9KJzVoFqM` - Kusama: `JLNozAv8QeLSbLFwe2UvWeKKE4yvmDbfGxTuiYkF2BUMx4M` @@ -129,13 +143,17 @@ Secret phrase `soup lyrics media market way crouch elevator put moon useful ques SS58 Address: 5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC ``` -Using the `inspect` command (see more details below), we see that knowning only the **secret seed** is no longer sufficient to recover the account: +Using the `inspect` command (see more details below), we see that knowning only the **secret seed** is no longer +sufficient to recover the account: ```bash subkey inspect "soup lyrics media market way crouch elevator put moon useful question wide" ``` -which recovers the account `5Fe4sqj2K4fRuzEGvToi4KATqZfiDU7TqynjXG6PZE2dxwyh` and not `5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC` as we expected. The additional user-defined **password** (`extra_secret` in our example) is now required to fully recover the account. Let's inspect the the previous mnemonic, this time passing also the required `password` as shown below: +which recovers the account `5Fe4sqj2K4fRuzEGvToi4KATqZfiDU7TqynjXG6PZE2dxwyh` and not +`5He5pZpc7AJ8evPuab37vJF6KkFDqq9uDq2WXh877Qw6iaVC` as we expected. The additional user-defined **password** +(`extra_secret` in our example) is now required to fully recover the account. Let's inspect the the previous mnemonic, +this time passing also the required `password` as shown below: ```bash subkey inspect --password extra_secret "soup lyrics media market way crouch elevator put moon useful question wide" @@ -161,7 +179,8 @@ subkey inspect --public < pubkey | address > **NOTE**: While you will be able to recover the secret seed from the mnemonic, the opposite is not possible. -**NOTE**: For obvious reasons, the **secrets** cannot be recovered from passing **public data** such as `pubkey` or `address` as input. +**NOTE**: For obvious reasons, the **secrets** cannot be recovered from passing **public data** such as `pubkey` or +`address` as input. command: @@ -181,7 +200,8 @@ Secret Key URI `0xa05c75731970cc7868a2fb7cb577353cd5b31f62dccced92c441acd8fee0c9 ### Signing -`subkey` allows using a **secret key** to sign a random message. The signature can then be verified by anyone using your **public key**: +`subkey` allows using a **secret key** to sign a random message. The signature can then be verified by anyone using your +**public key**: ```bash echo -n | subkey sign --suri @@ -201,11 +221,13 @@ output: 9201af3788ad4f986b800853c79da47155f2e08fde2070d866be4c27ab060466fea0623dc2b51f4392f4c61f25381a62848dd66c5d8217fae3858e469ebd668c ``` -**NOTE**: Each run of the `sign` command will yield a different output. While each signature is different, they are all valid. +**NOTE**: Each run of the `sign` command will yield a different output. While each signature is different, they are all +valid. ### Verifying a signature -Given a message, a signature and an address, `subkey` can verify whether the **message** has been digitally signed by the holder (or one of the holders) of the **private key** for the given **address**: +Given a message, a signature and an address, `subkey` can verify whether the **message** has been digitally signed by +the holder (or one of the holders) of the **private key** for the given **address**: ```bash echo -n | subkey verify
@@ -234,7 +256,8 @@ Error: SignatureInvalid ### Using the vanity generator -You can use the included vanity generator to find a seed that provides an address which includes the desired pattern. Be warned, depending on your hardware this may take a while. +You can use the included vanity generator to find a seed that provides an address which includes the desired pattern. Be +warned, depending on your hardware this may take a while. command: @@ -256,7 +279,9 @@ Secret Key URI `0x8c9a73097f235b84021a446bc2826a00c690ea0be3e0d81a84931cb4146d66 `Bob` now got a nice address starting with their name: 1**bob**YxBPjZWRPbVo35aSwci1u5Zmq8P6J2jpa4kkudBZMqE. -**Note**: While `Bob`, having a short name (3 chars), got a result rather quickly, it will take much longer for `Alice` who has a much longer name, thus the chances to generate a random address that contains the chain `alice` will be much smaller. +**Note**: While `Bob`, having a short name (3 chars), got a result rather quickly, it will take much longer for `Alice` +who has a much longer name, thus the chances to generate a random address that contains the chain `alice` will be much +smaller. ## License diff --git a/substrate/bin/utils/subkey/SECURITY.md b/substrate/bin/utils/subkey/SECURITY.md index 672d2965c7eae62b56a1ab6def39aea63bce4c00..698f119c29202120241b164abf06895b6c8ca5fb 100644 --- a/substrate/bin/utils/subkey/SECURITY.md +++ b/substrate/bin/utils/subkey/SECURITY.md @@ -1,8 +1,9 @@ # Keys and Security -The following information is not exhaustive but meant to prevent the most common mistakes. -You can read more about security and risks in the [Polkadot Wiki](https://wiki.polkadot.network/docs/learn-account-generation). -The Polkadot network has a few **test networks**, e.g. **Westend**. Test networks are a great way to experiment and learn safely as you can lose tokens on those networks without any financial consequences. +The following information is not exhaustive but meant to prevent the most common mistakes. You can read more about +security and risks in the [Polkadot Wiki](https://wiki.polkadot.network/docs/learn-account-generation). The Polkadot +network has a few **test networks**, e.g. **Westend**. Test networks are a great way to experiment and learn safely as +you can lose tokens on those networks without any financial consequences. `subkey` generates and provides 2 pieces of **secret** information: - **secret phrase**: a bunch of words, exactly 12 by default (can be 12, 15, 18, 21 or 24) @@ -10,16 +11,22 @@ The Polkadot network has a few **test networks**, e.g. **Westend**. Test network There are 2 risks related to private keys: - loss of keys: this can happen if you don't have a proper backup -- leak of the keys: this can unfortunately happen in many ways, including malware, phishing, key logger, backups on system that are online and not properly secured +- leak of the keys: this can unfortunately happen in many ways, including malware, phishing, key logger, backups on + system that are online and not properly secured You want to ensure that: - you **do not lose** those secrets - **no one but you can access** those secrets -☠️ **DO NOT SHARE** your mnemonic phrase or secret seed with ANYONE under **ANY** circumstances. Doing so would give them access to your funds and to send transactions on your behalf. +☠️ **DO NOT SHARE** your mnemonic phrase or secret seed with ANYONE under **ANY** circumstances. Doing so would give +them access to your funds and to send transactions on your behalf. -☠️ If someone is asking for your **secret** phrase or **secret** seed, you can be **SURE** they are attempting to steal your funds. +☠️ If someone is asking for your **secret** phrase or **secret** seed, you can be **SURE** they are attempting to steal +your funds. -✅ It is however fine to share your **SS58 Address** as this is meant to be public information and is needed by anyone you want to be able to make transfer to or otherwise interact with your account. They will only ever need your **Public Address**. +✅ It is however fine to share your **SS58 Address** as this is meant to be public information and is needed by anyone +you want to be able to make transfer to or otherwise interact with your account. They will only ever need your **Public +Address**. -⚠️ While using the same key on multiple networks is possible, it is usually **not** recommended unless you have good motivations for doing so and understand the associated risks and drawbacks. +⚠️ While using the same key on multiple networks is possible, it is usually **not** recommended unless you have good +motivations for doing so and understand the associated risks and drawbacks. diff --git a/substrate/client/allocator/Cargo.toml b/substrate/client/allocator/Cargo.toml index ffbfe14e86c2038d46b804d7c5baeb15cd5b20ea..31c714180ce57b0684ebfb7a8e27f7d2b42857ea 100644 --- a/substrate/client/allocator/Cargo.toml +++ b/substrate/client/allocator/Cargo.toml @@ -15,6 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] log = "0.4.17" -thiserror = "1.0.30" +thiserror = "1.0.48" sp-core = { path = "../../primitives/core" } sp-wasm-interface = { path = "../../primitives/wasm-interface" } diff --git a/substrate/client/allocator/README.md b/substrate/client/allocator/README.md index b89348b4c6950b4375c15f53ab4eef2031171150..e5a94e50f36d54e6a6c6671205e6f669da9826e7 100644 --- a/substrate/client/allocator/README.md +++ b/substrate/client/allocator/README.md @@ -3,4 +3,4 @@ Collection of allocator implementations. This crate provides the following allocator implementations: - A freeing-bump allocator: [`FreeingBumpHeapAllocator`](https://docs.rs/sc-allocator/latest/sc_allocator/struct.FreeingBumpHeapAllocator.html) -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/client/api/Cargo.toml b/substrate/client/api/Cargo.toml index 43545095c0ead237f37d1ec5d9eae7385ff17014..2b64c86038dda3ad5a37962c6dfc4a3230f7a4b2 100644 --- a/substrate/client/api/Cargo.toml +++ b/substrate/client/api/Cargo.toml @@ -35,8 +35,9 @@ sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-state-machine = { path = "../../primitives/state-machine" } sp-statement-store = { path = "../../primitives/statement-store" } sp-storage = { path = "../../primitives/storage" } +sp-trie = { path = "../../primitives/trie" } [dev-dependencies] -thiserror = "1.0.30" +thiserror = "1.0.48" sp-test-primitives = { path = "../../primitives/test-primitives" } substrate-test-runtime = { path = "../../test-utils/runtime" } diff --git a/substrate/client/api/README.md b/substrate/client/api/README.md index 142f5b32dd9a8de7c083aac5260ac6677042c3ad..7f94b1ca5e6912f7c14022f14d7ad6d65b4f6eca 100644 --- a/substrate/client/api/README.md +++ b/substrate/client/api/README.md @@ -1,3 +1,3 @@ Substrate client interfaces. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/api/src/backend.rs b/substrate/client/api/src/backend.rs index 2d8fdef77cdb9064076c3d38188d72bccedfdf9e..31b100433c7086719737275bfaa6b3931bffc88a 100644 --- a/substrate/client/api/src/backend.rs +++ b/substrate/client/api/src/backend.rs @@ -33,6 +33,7 @@ use sp_state_machine::{ OffchainChangesCollection, StorageCollection, StorageIterator, }; use sp_storage::{ChildInfo, StorageData, StorageKey}; +pub use sp_trie::MerkleValue; use crate::{blockchain::Backend as BlockchainBackend, UsageInfo}; @@ -470,6 +471,21 @@ pub trait StorageProvider> { child_info: &ChildInfo, key: &StorageKey, ) -> sp_blockchain::Result>; + + /// Given a block's `Hash` and a key, return the closest merkle value. + fn closest_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + ) -> sp_blockchain::Result>>; + + /// Given a block's `Hash`, a key and a child storage key, return the closest merkle value. + fn child_closest_merkle_value( + &self, + hash: Block::Hash, + child_info: &ChildInfo, + key: &StorageKey, + ) -> sp_blockchain::Result>>; } /// Client backend. diff --git a/substrate/client/api/src/lib.rs b/substrate/client/api/src/lib.rs index faadf3663a59d88871eb371c472826ddbec6f12f..f614a1e30b48e9c1daccb670a8f873f44b93372c 100644 --- a/substrate/client/api/src/lib.rs +++ b/substrate/client/api/src/lib.rs @@ -49,7 +49,6 @@ pub trait UsageProvider { pub mod utils { use sp_blockchain::{Error, HeaderBackend, HeaderMetadata}; use sp_runtime::traits::Block as BlockT; - use std::borrow::Borrow; /// Returns a function for checking block ancestry, the returned function will /// return `true` if the given hash (second parameter) is a descendent of the @@ -69,10 +68,8 @@ pub mod utils { return Ok(false) } - let current = current.as_ref().map(|(c, p)| (c.borrow(), p.borrow())); - let mut hash = hash; - if let Some((current_hash, current_parent_hash)) = current { + if let Some((current_hash, current_parent_hash)) = ¤t { if base == current_hash { return Ok(false) } diff --git a/substrate/client/block-builder/README.md b/substrate/client/block-builder/README.md index b105d4203362f2bf373a665f083d2169a2cb9801..f255b9a5480b14c5cceaa29e25c7cdb1e58308a9 100644 --- a/substrate/client/block-builder/README.md +++ b/substrate/client/block-builder/README.md @@ -6,4 +6,4 @@ This crate provides the [`BlockBuilder`] utility and the corresponding runtime a The block builder utility is used in the node as an abstraction over the runtime api to initialize a block, to push extrinsics and to finalize a block. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/chain-spec/Cargo.toml b/substrate/client/chain-spec/Cargo.toml index 6acd57ef51683d7d064dec5a652358979fc45481..e326d84ae50c6c6172965160a8ecd203319f3c0a 100644 --- a/substrate/client/chain-spec/Cargo.toml +++ b/substrate/client/chain-spec/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] memmap2 = "0.5.0" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" sc-client-api = { path = "../api" } sc-chain-spec-derive = { path = "derive" } sc-executor = { path = "../executor" } diff --git a/substrate/client/chain-spec/README.md b/substrate/client/chain-spec/README.md index 5525affbed81ccf343af1496ac3e384b110a32c1..dad1662d3230ec11b1bafa39fd78bf7d64d1ac5f 100644 --- a/substrate/client/chain-spec/README.md +++ b/substrate/client/chain-spec/README.md @@ -89,4 +89,4 @@ pub struct Extension { pub type MyChainSpec = GenericChainSpec; ``` -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/chain-spec/derive/Cargo.toml b/substrate/client/chain-spec/derive/Cargo.toml index b5ecc6107b943e18c2d5caf252bb083a94fa3959..202817438b7d80864a8a9db7e44bda23d244ffba 100644 --- a/substrate/client/chain-spec/derive/Cargo.toml +++ b/substrate/client/chain-spec/derive/Cargo.toml @@ -18,4 +18,4 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.37" diff --git a/substrate/client/cli/Cargo.toml b/substrate/client/cli/Cargo.toml index 917cdc04d1d9be07f99c15cfd243ca27e9e8897e..cfdcb39b1fa79d61c79cc613cd9a8b9db4c453bf 100644 --- a/substrate/client/cli/Cargo.toml +++ b/substrate/client/cli/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" chrono = "0.4.27" -clap = { version = "4.4.2", features = ["derive", "string"] } +clap = { version = "4.4.4", features = ["derive", "string"] } fdlimit = "0.2.1" futures = "0.3.21" libp2p-identity = { version = "0.1.3", features = ["peerid", "ed25519"]} @@ -26,8 +26,8 @@ rand = "0.8.5" regex = "1.6.0" rpassword = "7.0.0" serde = "1.0.188" -serde_json = "1.0.85" -thiserror = "1.0.30" +serde_json = "1.0.107" +thiserror = "1.0.48" tiny-bip39 = "1.0.0" tokio = { version = "1.22.0", features = ["signal", "rt-multi-thread", "parking_lot"] } sc-client-api = { path = "../api" } diff --git a/substrate/client/cli/README.adoc b/substrate/client/cli/README.adoc deleted file mode 100644 index fc58908fdf23d750409fc5ac8ce20fc40b3800e8..0000000000000000000000000000000000000000 --- a/substrate/client/cli/README.adoc +++ /dev/null @@ -1,6 +0,0 @@ - -= Substrate CLI - -Substrate CLI library - -include::doc/shell-completion.adoc[] diff --git a/substrate/client/cli/README.md b/substrate/client/cli/README.md index 2504dbb0c03b5f9525c580622445ac431fe61d6f..aeaee1e12194345b04b61b17b465d47198089b80 100644 --- a/substrate/client/cli/README.md +++ b/substrate/client/cli/README.md @@ -1,3 +1,3 @@ Substrate CLI library. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/cli/src/arg_enums.rs b/substrate/client/cli/src/arg_enums.rs index 40d86fd97988d23eaa209f6d067422af5ad89850..67acb82c2c3042fd8eb72f8241ec3e85e1a43688 100644 --- a/substrate/client/cli/src/arg_enums.rs +++ b/substrate/client/cli/src/arg_enums.rs @@ -38,12 +38,6 @@ pub enum WasmtimeInstantiationStrategy { /// Recreate the instance from scratch on every instantiation. Very slow. RecreateInstance, - - /// Legacy instance reuse mechanism. DEPRECATED. Will be removed in the future. - /// - /// Should only be used in case of encountering any issues with the new default - /// instantiation strategy. - LegacyInstanceReuse, } /// The default [`WasmtimeInstantiationStrategy`]. @@ -92,8 +86,6 @@ pub fn execution_method_from_cli( sc_service::config::WasmtimeInstantiationStrategy::Pooling, WasmtimeInstantiationStrategy::RecreateInstance => sc_service::config::WasmtimeInstantiationStrategy::RecreateInstance, - WasmtimeInstantiationStrategy::LegacyInstanceReuse => - sc_service::config::WasmtimeInstantiationStrategy::LegacyInstanceReuse, }, } } diff --git a/substrate/client/cli/src/params/keystore_params.rs b/substrate/client/cli/src/params/keystore_params.rs index a2fdd6b2218c4b2dcadb03a6ca179d00a59868e2..87210c3390cae3ee3860398ce74f041465f76caf 100644 --- a/substrate/client/cli/src/params/keystore_params.rs +++ b/substrate/client/cli/src/params/keystore_params.rs @@ -31,10 +31,6 @@ const DEFAULT_KEYSTORE_CONFIG_PATH: &str = "keystore"; /// Parameters of the keystore #[derive(Debug, Clone, Args)] pub struct KeystoreParams { - /// Specify custom URIs to connect to for keystore-services - #[arg(long)] - pub keystore_uri: Option, - /// Specify custom keystore path. #[arg(long, value_name = "PATH")] pub keystore_path: Option, diff --git a/substrate/client/consensus/aura/README.md b/substrate/client/consensus/aura/README.md index 85d82cd7dfd3b4e392c8210c40a32a6e57508413..cefa5f6c7d9d95e29cd3e62a7952dfda088d536f 100644 --- a/substrate/client/consensus/aura/README.md +++ b/substrate/client/consensus/aura/README.md @@ -1,4 +1,4 @@ -Aura (Authority-round) consensus in substrate. +Aura (Authority-round) consensus in Substrate. Aura works by having a list of authorities A who are expected to roughly agree on the current time. Time is divided up into discrete slots of t @@ -12,4 +12,4 @@ far in the future they are. NOTE: Aura itself is designed to be generic over the crypto used. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/babe/Cargo.toml b/substrate/client/consensus/babe/Cargo.toml index d95887945358d59bcc62d6b7df738691159c6ef3..c8cff0981b36f89e0b6f9120f4c6378f7c6793e7 100644 --- a/substrate/client/consensus/babe/Cargo.toml +++ b/substrate/client/consensus/babe/Cargo.toml @@ -15,7 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-trait = "0.1.57" -scale-info = { version = "2.5.0", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.6.1", features = ["derive"] } futures = "0.3.21" log = "0.4.17" @@ -45,10 +44,8 @@ sp-keystore = { path = "../../../primitives/keystore" } sp-runtime = { path = "../../../primitives/runtime" } [dev-dependencies] -rand_chacha = "0.2.2" sc-block-builder = { path = "../../block-builder" } sp-keyring = { path = "../../../primitives/keyring" } -sc-network = { path = "../../network" } sc-network-test = { path = "../../network/test" } sp-timestamp = { path = "../../../primitives/timestamp" } sp-tracing = { path = "../../../primitives/tracing" } diff --git a/substrate/client/consensus/babe/README.md b/substrate/client/consensus/babe/README.md index a404d2ea447064e083d05aac59752a8c41d97096..a3cf944b513b80eb96493342b821147bfc905592 100644 --- a/substrate/client/consensus/babe/README.md +++ b/substrate/client/consensus/babe/README.md @@ -45,4 +45,4 @@ blocks) and will go with the longest one in case of a tie. An in-depth description and analysis of the protocol can be found here: -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/babe/rpc/Cargo.toml b/substrate/client/consensus/babe/rpc/Cargo.toml index bfae5ad3fd0cef556d8180e55452a2cac572b314..f54edb296842ddf9b9f4390aa8e307c0bef69398 100644 --- a/substrate/client/consensus/babe/rpc/Cargo.toml +++ b/substrate/client/consensus/babe/rpc/Cargo.toml @@ -30,7 +30,7 @@ sp-keystore = { path = "../../../../primitives/keystore" } sp-runtime = { path = "../../../../primitives/runtime" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" tokio = "1.22.0" sc-consensus = { path = "../../common" } sc-keystore = { path = "../../../keystore" } diff --git a/substrate/client/consensus/babe/rpc/README.md b/substrate/client/consensus/babe/rpc/README.md index e76dd3dc67f81d3b789e718acbb8cb8b0e755cec..e1a366204cd0112d33a7b69a0fed4b81af9b7b57 100644 --- a/substrate/client/consensus/babe/rpc/README.md +++ b/substrate/client/consensus/babe/rpc/README.md @@ -1,3 +1,3 @@ RPC api for babe. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/babe/src/authorship.rs b/substrate/client/consensus/babe/src/authorship.rs index 758d5321a94c5b1c4cdaca6855a5739370ec012d..3580caba746141a8a9faf9e73216cb8903667acf 100644 --- a/substrate/client/consensus/babe/src/authorship.rs +++ b/substrate/client/consensus/babe/src/authorship.rs @@ -132,23 +132,22 @@ fn claim_secondary_slot( keystore: &KeystorePtr, author_secondary_vrf: bool, ) -> Option<(PreDigest, AuthorityId)> { - let Epoch { authorities, randomness, mut epoch_index, .. } = epoch; - - if authorities.is_empty() { + if epoch.authorities.is_empty() { return None } + let mut epoch_index = epoch.epoch_index; if epoch.end_slot() <= slot { // Slot doesn't strictly belong to the epoch, create a clone with fixed values. epoch_index = epoch.clone_for_slot(slot).epoch_index; } - let expected_author = secondary_slot_author(slot, authorities, *randomness)?; + let expected_author = secondary_slot_author(slot, &epoch.authorities, epoch.randomness)?; for (authority_id, authority_index) in keys { if authority_id == expected_author { let pre_digest = if author_secondary_vrf { - let data = make_vrf_sign_data(randomness, slot, epoch_index); + let data = make_vrf_sign_data(&epoch.randomness, slot, epoch_index); let result = keystore.sr25519_vrf_sign(AuthorityId::ID, authority_id.as_ref(), &data); if let Ok(Some(vrf_signature)) = result { @@ -232,19 +231,18 @@ fn claim_primary_slot( keystore: &KeystorePtr, keys: &[(AuthorityId, usize)], ) -> Option<(PreDigest, AuthorityId)> { - let Epoch { authorities, randomness, mut epoch_index, .. } = epoch; - + let mut epoch_index = epoch.epoch_index; if epoch.end_slot() <= slot { // Slot doesn't strictly belong to the epoch, create a clone with fixed values. epoch_index = epoch.clone_for_slot(slot).epoch_index; } - let data = make_vrf_sign_data(randomness, slot, epoch_index); + let data = make_vrf_sign_data(&epoch.randomness, slot, epoch_index); for (authority_id, authority_index) in keys { let result = keystore.sr25519_vrf_sign(AuthorityId::ID, authority_id.as_ref(), &data); if let Ok(Some(vrf_signature)) = result { - let threshold = calculate_primary_threshold(c, authorities, *authority_index); + let threshold = calculate_primary_threshold(c, &epoch.authorities, *authority_index); let can_claim = authority_id .as_inner_ref() @@ -274,7 +272,7 @@ fn claim_primary_slot( #[cfg(test)] mod tests { use super::*; - use sp_consensus_babe::{AllowedSlots, AuthorityId, BabeEpochConfiguration}; + use sp_consensus_babe::{AllowedSlots, AuthorityId, BabeEpochConfiguration, Epoch}; use sp_core::{crypto::Pair as _, sr25519::Pair}; use sp_keystore::testing::MemoryKeystore; @@ -300,7 +298,8 @@ mod tests { c: (3, 10), allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, }, - }; + } + .into(); assert!(claim_slot(10.into(), &epoch, &keystore).is_none()); diff --git a/substrate/client/consensus/babe/src/lib.rs b/substrate/client/consensus/babe/src/lib.rs index 90b7523ec18bba4d46ad17c55cd0ace64fee2dba..ccf72939631accae68c9e473cf30461c2fca65dc 100644 --- a/substrate/client/consensus/babe/src/lib.rs +++ b/substrate/client/consensus/babe/src/lib.rs @@ -69,6 +69,7 @@ use std::{ collections::HashSet, future::Future, + ops::{Deref, DerefMut}, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -156,20 +157,27 @@ const AUTHORING_SCORE_VRF_CONTEXT: &[u8] = b"substrate-babe-vrf"; const AUTHORING_SCORE_LENGTH: usize = 16; /// BABE epoch information -#[derive(Decode, Encode, PartialEq, Eq, Clone, Debug, scale_info::TypeInfo)] -pub struct Epoch { - /// The epoch index. - pub epoch_index: u64, - /// The starting slot of the epoch. - pub start_slot: Slot, - /// The duration of this epoch. - pub duration: u64, - /// The authorities and their weights. - pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>, - /// Randomness for this epoch. - pub randomness: Randomness, - /// Configuration of the epoch. - pub config: BabeEpochConfiguration, +#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)] +pub struct Epoch(sp_consensus_babe::Epoch); + +impl Deref for Epoch { + type Target = sp_consensus_babe::Epoch; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Epoch { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From for Epoch { + fn from(epoch: sp_consensus_babe::Epoch) -> Self { + Epoch(epoch) + } } impl EpochT for Epoch { @@ -180,7 +188,7 @@ impl EpochT for Epoch { &self, (descriptor, config): (NextEpochDescriptor, BabeEpochConfiguration), ) -> Epoch { - Epoch { + sp_consensus_babe::Epoch { epoch_index: self.epoch_index + 1, start_slot: self.start_slot + self.duration, duration: self.duration, @@ -188,6 +196,7 @@ impl EpochT for Epoch { randomness: descriptor.randomness, config, } + .into() } fn start_slot(&self) -> Slot { @@ -199,25 +208,12 @@ impl EpochT for Epoch { } } -impl From for Epoch { - fn from(epoch: sp_consensus_babe::Epoch) -> Self { - Epoch { - epoch_index: epoch.epoch_index, - start_slot: epoch.start_slot, - duration: epoch.duration, - authorities: epoch.authorities, - randomness: epoch.randomness, - config: epoch.config, - } - } -} - impl Epoch { /// Create the genesis epoch (epoch #0). /// /// This is defined to start at the slot of the first block, so that has to be provided. pub fn genesis(genesis_config: &BabeConfiguration, slot: Slot) -> Epoch { - Epoch { + sp_consensus_babe::Epoch { epoch_index: 0, start_slot: slot, duration: genesis_config.epoch_length, @@ -228,6 +224,7 @@ impl Epoch { allowed_slots: genesis_config.allowed_slots, }, } + .into() } /// Clone and tweak epoch information to refer to the specified slot. diff --git a/substrate/client/consensus/babe/src/migration.rs b/substrate/client/consensus/babe/src/migration.rs index 2b1396c41c268476c2dadc69c8051a04112b5d16..bec2d0a61f4b6d4f55236f69042bd764629b5f92 100644 --- a/substrate/client/consensus/babe/src/migration.rs +++ b/substrate/client/consensus/babe/src/migration.rs @@ -62,10 +62,11 @@ impl EpochT for EpochV0 { } } +// Implement From for Epoch impl EpochV0 { /// Migrate the sturct to current epoch version. pub fn migrate(self, config: &BabeConfiguration) -> Epoch { - Epoch { + sp_consensus_babe::Epoch { epoch_index: self.epoch_index, start_slot: self.start_slot, duration: self.duration, @@ -73,5 +74,6 @@ impl EpochV0 { randomness: self.randomness, config: BabeEpochConfiguration { c: config.c, allowed_slots: config.allowed_slots }, } + .into() } } diff --git a/substrate/client/consensus/babe/src/tests.rs b/substrate/client/consensus/babe/src/tests.rs index b3843f8acfa0a481ed456f13bdce5a7fb54173b4..02882d8baaeda2d8cf8ed2fdf2682c31989d1ab3 100644 --- a/substrate/client/consensus/babe/src/tests.rs +++ b/substrate/client/consensus/babe/src/tests.rs @@ -501,7 +501,7 @@ fn claim_epoch_slots() { let authority = Sr25519Keyring::Alice; let keystore = create_keystore(authority); - let mut epoch = Epoch { + let mut epoch: Epoch = sp_consensus_babe::Epoch { start_slot: 0.into(), authorities: vec![(authority.public().into(), 1)], randomness: [0; 32], @@ -511,7 +511,8 @@ fn claim_epoch_slots() { c: (3, 10), allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots, }, - }; + } + .into(); let claim_slot_wrap = |s, e| match claim_slot(Slot::from(s as u64), &e, &keystore) { None => 0, @@ -551,7 +552,7 @@ fn claim_vrf_check() { let public = authority.public(); - let epoch = Epoch { + let epoch: Epoch = sp_consensus_babe::Epoch { start_slot: 0.into(), authorities: vec![(public.into(), 1)], randomness: [0; 32], @@ -561,7 +562,8 @@ fn claim_vrf_check() { c: (3, 10), allowed_slots: AllowedSlots::PrimaryAndSecondaryVRFSlots, }, - }; + } + .into(); // We leverage the predictability of claim types given a constant randomness. diff --git a/substrate/client/consensus/beefy/rpc/Cargo.toml b/substrate/client/consensus/beefy/rpc/Cargo.toml index f9c3b4a99dfe8613f2aecbe7db8068f91a20f79e..74733ea9edd9d274a9578248f607b68fe1aacee4 100644 --- a/substrate/client/consensus/beefy/rpc/Cargo.toml +++ b/substrate/client/consensus/beefy/rpc/Cargo.toml @@ -23,7 +23,7 @@ sp-core = { path = "../../../../primitives/core" } sp-runtime = { path = "../../../../primitives/runtime" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" sc-rpc = { path = "../../../rpc", features = ["test-helpers"]} substrate-test-runtime-client = { path = "../../../../test-utils/runtime/client" } tokio = { version = "1.22.0", features = ["macros"] } diff --git a/substrate/client/consensus/common/Cargo.toml b/substrate/client/consensus/common/Cargo.toml index c9b3f221ecc0637a01e9d9f811e41ce87a7290fd..f269e3752d435488e9c843260454adbccd325b7b 100644 --- a/substrate/client/consensus/common/Cargo.toml +++ b/substrate/client/consensus/common/Cargo.toml @@ -21,7 +21,7 @@ log = "0.4.17" mockall = "0.11.3" parking_lot = "0.12.1" serde = { version = "1.0", features = ["derive"] } -thiserror = "1.0.30" +thiserror = "1.0.48" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus" } sc-client-api = { path = "../../api" } sc-utils = { path = "../../utils" } diff --git a/substrate/client/consensus/common/README.md b/substrate/client/consensus/common/README.md index a6717a1d7a6df682607e9ddf3c45d7bbf6115dfc..9b953fbf32740e62205f405f2e7a98552da71bcd 100644 --- a/substrate/client/consensus/common/README.md +++ b/substrate/client/consensus/common/README.md @@ -1,3 +1,3 @@ Collection of common consensus specific implementations -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/epochs/README.md b/substrate/client/consensus/epochs/README.md index 1e74e04172c2400f41d9d3c18037132412feb299..e4abc58c6c0d2dd7fa720ec835dc17d204aa0b30 100644 --- a/substrate/client/consensus/epochs/README.md +++ b/substrate/client/consensus/epochs/README.md @@ -1,3 +1,3 @@ Generic utilities for epoch-based consensus engines. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/grandpa/Cargo.toml b/substrate/client/consensus/grandpa/Cargo.toml index dfd9916850e54d85eb53dee3226b6b57ec304a4b..472bdd1c5b820e5590d6d38c1e15add9c94e6fd6 100644 --- a/substrate/client/consensus/grandpa/Cargo.toml +++ b/substrate/client/consensus/grandpa/Cargo.toml @@ -25,7 +25,7 @@ log = "0.4.17" parity-scale-codec = { version = "3.6.1", features = ["derive"] } parking_lot = "0.12.1" rand = "0.8.5" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" fork-tree = { path = "../../../utils/fork-tree" } prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus" } diff --git a/substrate/client/consensus/grandpa/README.md b/substrate/client/consensus/grandpa/README.md index 64a7e70bc6a52631016bba71343b38d3e0f9f005..f4045896599085d7436f1684f4612844c12184e9 100644 --- a/substrate/client/consensus/grandpa/README.md +++ b/substrate/client/consensus/grandpa/README.md @@ -1,4 +1,4 @@ -Integration of the GRANDPA finality gadget into substrate. +Integration of the GRANDPA finality gadget into Substrate. This crate is unstable and the API and usage may change. @@ -36,4 +36,4 @@ number (this is num(signal) + N). When finalizing a block, we either apply or prune any signaled changes based on whether the signaling block is included in the newly-finalized chain. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/grandpa/rpc/README.md b/substrate/client/consensus/grandpa/rpc/README.md index 0007f55dbd4dbfa5fe0b10b0cb9e0714def30b1a..ad73878a61bdde1b3fe0e059960b54051eefa7f2 100644 --- a/substrate/client/consensus/grandpa/rpc/README.md +++ b/substrate/client/consensus/grandpa/rpc/README.md @@ -1,3 +1,3 @@ RPC API for GRANDPA. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/grandpa/src/import.rs b/substrate/client/consensus/grandpa/src/import.rs index 8481b395847292173b4fa89206d9610d5aa767e0..ca5b7c400bfb2f83be8c1b22459ee0730072ada0 100644 --- a/substrate/client/consensus/grandpa/src/import.rs +++ b/substrate/client/consensus/grandpa/src/import.rs @@ -552,7 +552,6 @@ where .into(), )) } - assert!(block.justifications.is_some()); let mut authority_set = self.authority_set.inner_locked(); authority_set.authority_set_changes.insert(number); crate::aux_schema::update_authority_set::( diff --git a/substrate/client/consensus/manual-seal/README.md b/substrate/client/consensus/manual-seal/README.md index b355f8b73183cc9661b4c65d76a060d9f6cc8918..131b620cb2feab527818d6faafa175868938d045 100644 --- a/substrate/client/consensus/manual-seal/README.md +++ b/substrate/client/consensus/manual-seal/README.md @@ -1,4 +1,4 @@ A manual sealing engine: the engine listens for rpc calls to seal blocks and create forks. This is suitable for a testing environment. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/consensus/slots/README.md b/substrate/client/consensus/slots/README.md index 9ab3c3742f33076b46be61efbf3c3e5fd824b50c..aa896430f7dc2a56ce1b41a43de634e9e34f039e 100644 --- a/substrate/client/consensus/slots/README.md +++ b/substrate/client/consensus/slots/README.md @@ -4,4 +4,4 @@ Some consensus algorithms have a concept of *slots*, which are intervals in time during which certain events can and/or must occur. This crate provides generic functionality for slots. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/db/README.md b/substrate/client/db/README.md index e5fb3fce1d9763d92554b3d0c6262e4e3cfccd66..a11ae0d6f57effd7f7766dbc6f37fc42e60c8f6d 100644 --- a/substrate/client/db/README.md +++ b/substrate/client/db/README.md @@ -8,4 +8,4 @@ having discarded heavy state that will allow a chain reorganization. Finality implies canonicality but not vice-versa. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/db/src/bench.rs b/substrate/client/db/src/bench.rs index 38c37a42ede79659639268eb34e10316c8cbed15..03ad4817b53bcea2cb349d4d05c190bb239530fa 100644 --- a/substrate/client/db/src/bench.rs +++ b/substrate/client/db/src/bench.rs @@ -37,7 +37,7 @@ use sp_state_machine::{ }; use sp_trie::{ cache::{CacheSize, SharedTrieCache}, - prefixed_key, MemoryDB, + prefixed_key, MemoryDB, MerkleValue, }; use std::{ cell::{Cell, RefCell}, @@ -382,6 +382,27 @@ impl StateBackend> for BenchmarkingState { .child_storage_hash(child_info, key) } + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { + self.add_read_key(None, key); + self.state.borrow().as_ref().ok_or_else(state_err)?.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.add_read_key(None, key); + self.state + .borrow() + .as_ref() + .ok_or_else(state_err)? + .child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.add_read_key(None, key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 73fb4f8ce6db37f0857813d5b64821e77413b34b..194bec8a88eb46fd90537bf61a65c12dc02982c8 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -90,7 +90,7 @@ use sp_state_machine::{ OffchainChangesCollection, StateMachineStats, StorageCollection, StorageIterator, StorageKey, StorageValue, UsageInfo as StateUsageInfo, }; -use sp_trie::{cache::SharedTrieCache, prefixed_key, MemoryDB, PrefixedMemoryDB}; +use sp_trie::{cache::SharedTrieCache, prefixed_key, MemoryDB, MerkleValue, PrefixedMemoryDB}; // Re-export the Database trait so that one can pass an implementation of it. pub use sc_state_db::PruningMode; @@ -214,6 +214,21 @@ impl StateBackend> for RefTrackingState { self.state.child_storage_hash(child_info, key) } + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/substrate/client/db/src/record_stats_state.rs b/substrate/client/db/src/record_stats_state.rs index 29ece84f97e5744c7fb0902d5f5fc98ba0eb7ced..d9a35c075d79486683a632fabb7707891fa1ad8a 100644 --- a/substrate/client/db/src/record_stats_state.rs +++ b/substrate/client/db/src/record_stats_state.rs @@ -28,6 +28,7 @@ use sp_state_machine::{ backend::{AsTrieBackend, Backend as StateBackend}, BackendTransaction, IterArgs, StorageIterator, StorageKey, StorageValue, TrieBackend, }; +use sp_trie::MerkleValue; use std::sync::Arc; /// State abstraction for recording stats about state access. @@ -144,6 +145,21 @@ impl>, B: BlockT> StateBackend> self.state.child_storage_hash(child_info, key) } + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/substrate/client/executor/README.md b/substrate/client/executor/README.md index ab7b3d45206f9fc4ca4e0323dcc5b27bbf943226..6a35697962fa3a5dffb14abacd5e50341ff28dad 100644 --- a/substrate/client/executor/README.md +++ b/substrate/client/executor/README.md @@ -10,4 +10,4 @@ provided into the wasm runtime module. by the current value of `:code` in the provided externalities), i.e. interfacing with wasm engine used, instance cache. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/executor/benches/bench.rs b/substrate/client/executor/benches/bench.rs index 66a82a175221d6a9aa229c5a593f3531f80f656f..86c769f88811b9d004e373ac22a166ae84416df2 100644 --- a/substrate/client/executor/benches/bench.rs +++ b/substrate/client/executor/benches/bench.rs @@ -150,13 +150,6 @@ fn bench_call_instance(c: &mut Criterion) { let _ = env_logger::try_init(); let strategies = [ - ( - "legacy_instance_reuse", - Method::Compiled { - instantiation_strategy: InstantiationStrategy::LegacyInstanceReuse, - precompile: false, - }, - ), ( "recreate_instance_vanilla", Method::Compiled { diff --git a/substrate/client/executor/common/Cargo.toml b/substrate/client/executor/common/Cargo.toml index e84b9f9c85b86abef81b4068fa63dd3c801a16bc..5118279b43b44ad75df3b8882592640c5ec9bd11 100644 --- a/substrate/client/executor/common/Cargo.toml +++ b/substrate/client/executor/common/Cargo.toml @@ -14,7 +14,7 @@ readme = "README.md" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -thiserror = "1.0.30" +thiserror = "1.0.48" wasm-instrument = "0.3" sc-allocator = { path = "../../allocator" } sp-maybe-compressed-blob = { path = "../../../primitives/maybe-compressed-blob" } diff --git a/substrate/client/executor/common/README.md b/substrate/client/executor/common/README.md index 0c0d3bf08bcb20519013a428ee4daf9ce6c1a622..253d7340313765b1913798029a47807480004433 100644 --- a/substrate/client/executor/common/README.md +++ b/substrate/client/executor/common/README.md @@ -1,3 +1,3 @@ A set of common definitions that are needed for defining execution engines. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/executor/common/src/runtime_blob/data_segments_snapshot.rs b/substrate/client/executor/common/src/runtime_blob/data_segments_snapshot.rs deleted file mode 100644 index 3fd546ce4457b225819aed8ff785a43c81e93d4d..0000000000000000000000000000000000000000 --- a/substrate/client/executor/common/src/runtime_blob/data_segments_snapshot.rs +++ /dev/null @@ -1,87 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use super::RuntimeBlob; -use crate::error::{self, Error}; -use std::mem; -use wasm_instrument::parity_wasm::elements::Instruction; - -/// This is a snapshot of data segments specialzied for a particular instantiation. -/// -/// Note that this assumes that no mutable globals are used. -#[derive(Clone)] -pub struct DataSegmentsSnapshot { - /// The list of data segments represented by (offset, contents). - data_segments: Vec<(u32, Vec)>, -} - -impl DataSegmentsSnapshot { - /// Create a snapshot from the data segments from the module. - pub fn take(module: &RuntimeBlob) -> error::Result { - let data_segments = module - .data_segments() - .into_iter() - .map(|mut segment| { - // Just replace contents of the segment since the segments will be discarded later - // anyway. - let contents = mem::take(segment.value_mut()); - - let init_expr = match segment.offset() { - Some(offset) => offset.code(), - // Return if the segment is passive - None => return Err(Error::SharedMemUnsupported), - }; - - // [op, End] - if init_expr.len() != 2 { - return Err(Error::InitializerHasTooManyExpressions) - } - let offset = match &init_expr[0] { - Instruction::I32Const(v) => *v as u32, - Instruction::GetGlobal(_) => { - // In a valid wasm file, initializer expressions can only refer imported - // globals. - // - // At the moment of writing the Substrate Runtime Interface does not provide - // any globals. There is nothing that prevents us from supporting this - // if/when we gain those. - return Err(Error::ImportedGlobalsUnsupported) - }, - insn => return Err(Error::InvalidInitializerExpression(format!("{:?}", insn))), - }; - - Ok((offset, contents)) - }) - .collect::>>()?; - - Ok(Self { data_segments }) - } - - /// Apply the given snapshot to a linear memory. - /// - /// Linear memory interface is represented by a closure `memory_set`. - pub fn apply( - &self, - mut memory_set: impl FnMut(u32, &[u8]) -> Result<(), E>, - ) -> Result<(), E> { - for (offset, contents) in &self.data_segments { - memory_set(*offset, contents)?; - } - Ok(()) - } -} diff --git a/substrate/client/executor/common/src/runtime_blob/globals_snapshot.rs b/substrate/client/executor/common/src/runtime_blob/globals_snapshot.rs deleted file mode 100644 index 9ba6fc55e49c2a002d6e574a56e6b6865292e969..0000000000000000000000000000000000000000 --- a/substrate/client/executor/common/src/runtime_blob/globals_snapshot.rs +++ /dev/null @@ -1,112 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use super::RuntimeBlob; - -/// Saved value of particular exported global. -struct SavedValue { - /// The handle of this global which can be used to refer to this global. - handle: Global, - /// The global value that was observed during the snapshot creation. - value: sp_wasm_interface::Value, -} - -/// An adapter for a wasm module instance that is focused on getting and setting globals. -pub trait InstanceGlobals { - /// A handle to a global which can be used to get or set a global variable. This is supposed to - /// be a lightweight handle, like an index or an Rc-like smart-pointer, which is cheap to clone. - type Global: Clone; - /// Get a handle to a global by it's export name. - /// - /// The requested export is must exist in the exported list, and it should be a mutable global. - fn get_global(&mut self, export_name: &str) -> Self::Global; - /// Get the current value of the global. - fn get_global_value(&mut self, global: &Self::Global) -> sp_wasm_interface::Value; - /// Update the current value of the global. - /// - /// The global behind the handle is guaranteed to be mutable and the value to be the same type - /// as the global. - fn set_global_value(&mut self, global: &Self::Global, value: sp_wasm_interface::Value); -} - -/// A set of exposed mutable globals. -/// -/// This is set of globals required to create a [`GlobalsSnapshot`] and that are collected from -/// a runtime blob that was instrumented by -/// [`RuntimeBlob::expose_mutable_globals`](super::RuntimeBlob::expose_mutable_globals`). - -/// If the code wasn't instrumented then it would be empty and snapshot would do nothing. -pub struct ExposedMutableGlobalsSet(Vec); - -impl ExposedMutableGlobalsSet { - /// Collect the set from the given runtime blob. See the struct documentation for details. - pub fn collect(runtime_blob: &RuntimeBlob) -> Self { - let global_names = - runtime_blob.exported_internal_global_names().map(ToOwned::to_owned).collect(); - Self(global_names) - } -} - -/// A snapshot of a global variables values. This snapshot can be later used for restoring the -/// values to the preserved state. -/// -/// Technically, a snapshot stores only values of mutable global variables. This is because -/// immutable global variables always have the same values. -/// -/// We take it from an instance rather from a module because the start function could potentially -/// change any of the mutable global values. -pub struct GlobalsSnapshot(Vec>); - -impl GlobalsSnapshot { - /// Take a snapshot of global variables for a given instance. - /// - /// # Panics - /// - /// This function panics if the instance doesn't correspond to the module from which the - /// [`ExposedMutableGlobalsSet`] was collected. - pub fn take( - mutable_globals: &ExposedMutableGlobalsSet, - instance: &mut Instance, - ) -> Self - where - Instance: InstanceGlobals, - { - let global_names = &mutable_globals.0; - let mut saved_values = Vec::with_capacity(global_names.len()); - - for global_name in global_names { - let handle = instance.get_global(global_name); - let value = instance.get_global_value(&handle); - saved_values.push(SavedValue { handle, value }); - } - - Self(saved_values) - } - - /// Apply the snapshot to the given instance. - /// - /// This instance must be the same that was used for creation of this snapshot. - pub fn apply(&self, instance: &mut Instance) - where - Instance: InstanceGlobals, - { - for saved_value in &self.0 { - instance.set_global_value(&saved_value.handle, saved_value.value); - } - } -} diff --git a/substrate/client/executor/common/src/runtime_blob/mod.rs b/substrate/client/executor/common/src/runtime_blob/mod.rs index 07a0945cc2b668b1c9737b171bd3dc4707903d11..8261d07eda5efce09a0b73ebd33b8ec9720c5d5d 100644 --- a/substrate/client/executor/common/src/runtime_blob/mod.rs +++ b/substrate/client/executor/common/src/runtime_blob/mod.rs @@ -46,10 +46,6 @@ //! is free of any floating point operations, which is a useful step towards making instances //! produced from such a module deterministic. -mod data_segments_snapshot; -mod globals_snapshot; mod runtime_blob; -pub use data_segments_snapshot::DataSegmentsSnapshot; -pub use globals_snapshot::{ExposedMutableGlobalsSet, GlobalsSnapshot, InstanceGlobals}; pub use runtime_blob::RuntimeBlob; diff --git a/substrate/client/executor/common/src/runtime_blob/runtime_blob.rs b/substrate/client/executor/common/src/runtime_blob/runtime_blob.rs index 24dc7e393a4bccd1806f4ce5b6dccacaeeac8302..becf9e219b0b7a60f97bb08b4044897b143bd006 100644 --- a/substrate/client/executor/common/src/runtime_blob/runtime_blob.rs +++ b/substrate/client/executor/common/src/runtime_blob/runtime_blob.rs @@ -20,8 +20,8 @@ use crate::{error::WasmError, wasm_runtime::HeapAllocStrategy}; use wasm_instrument::{ export_mutable_globals, parity_wasm::elements::{ - deserialize_buffer, serialize, DataSegment, ExportEntry, External, Internal, MemorySection, - MemoryType, Module, Section, + deserialize_buffer, serialize, ExportEntry, External, Internal, MemorySection, MemoryType, + Module, Section, }, }; @@ -52,11 +52,6 @@ impl RuntimeBlob { Ok(Self { raw_module }) } - /// Extract the data segments from the given wasm code. - pub(super) fn data_segments(&self) -> Vec { - self.raw_module.data_section().map(|ds| ds.entries()).unwrap_or(&[]).to_vec() - } - /// The number of globals defined in locally in this module. pub fn declared_globals_count(&self) -> u32 { self.raw_module @@ -190,16 +185,6 @@ impl RuntimeBlob { Ok(()) } - /// Returns an iterator of all globals which were exported by [`expose_mutable_globals`]. - pub(super) fn exported_internal_global_names(&self) -> impl Iterator { - let exports = self.raw_module.export_section().map(|es| es.entries()).unwrap_or(&[]); - exports.iter().filter_map(|export| match export.internal() { - Internal::Global(_) if export.field().starts_with("exported_internal_global") => - Some(export.field()), - _ => None, - }) - } - /// Scans the wasm blob for the first section with the name that matches the given. Returns the /// contents of the custom section if found or `None` otherwise. pub fn custom_section_contents(&self, section_name: &str) -> Option<&[u8]> { diff --git a/substrate/client/executor/common/src/wasm_runtime.rs b/substrate/client/executor/common/src/wasm_runtime.rs index 5dac77e59fa7488e14fe35059f465ffce9b6407e..d8e142b9d55905e3e31d2247f5a48e8e2b0f987a 100644 --- a/substrate/client/executor/common/src/wasm_runtime.rs +++ b/substrate/client/executor/common/src/wasm_runtime.rs @@ -115,16 +115,6 @@ pub trait WasmInstance: Send { /// /// This method is only suitable for getting immutable globals. fn get_global_const(&mut self, name: &str) -> Result, Error>; - - /// **Testing Only**. This function returns the base address of the linear memory. - /// - /// This is meant to be the starting address of the memory mapped area for the linear memory. - /// - /// This function is intended only for a specific test that measures physical memory - /// consumption. - fn linear_memory_base_ptr(&self) -> Option<*const u8> { - None - } } /// Defines the heap pages allocation strategy the wasm runtime should use. diff --git a/substrate/client/executor/src/integration_tests/linux.rs b/substrate/client/executor/src/integration_tests/linux.rs deleted file mode 100644 index 68ac37e9011a18a811ad8acba7026ac974a2c888..0000000000000000000000000000000000000000 --- a/substrate/client/executor/src/integration_tests/linux.rs +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Tests that are only relevant for Linux. - -mod smaps; - -use super::mk_test_runtime; -use crate::WasmExecutionMethod; -use codec::Encode as _; -use sc_executor_common::wasm_runtime::DEFAULT_HEAP_ALLOC_STRATEGY; - -use self::smaps::Smaps; - -#[test] -fn memory_consumption_compiled() { - let _ = sp_tracing::try_init_simple(); - - if std::env::var("RUN_TEST").is_ok() { - memory_consumption(WasmExecutionMethod::Compiled { - instantiation_strategy: - sc_executor_wasmtime::InstantiationStrategy::LegacyInstanceReuse, - }); - } else { - // We need to run the test in isolation, to not getting interfered by the other tests. - let executable = std::env::current_exe().unwrap(); - let status = std::process::Command::new(executable) - .env("RUN_TEST", "1") - .args(&["--nocapture", "memory_consumption_compiled"]) - .status() - .unwrap(); - - assert!(status.success()); - } -} - -fn memory_consumption(wasm_method: WasmExecutionMethod) { - // This aims to see if linear memory stays backed by the physical memory after a runtime call. - // - // For that we make a series of runtime calls, probing the RSS for the VMA matching the linear - // memory. After the call we expect RSS to be equal to 0. - - let runtime = mk_test_runtime(wasm_method, DEFAULT_HEAP_ALLOC_STRATEGY); - - let mut instance = runtime.new_instance().unwrap(); - let heap_base = instance - .get_global_const("__heap_base") - .expect("`__heap_base` is valid") - .expect("`__heap_base` exists") - .as_i32() - .expect("`__heap_base` is an `i32`"); - - fn probe_rss(instance: &dyn sc_executor_common::wasm_runtime::WasmInstance) -> usize { - let base_addr = instance.linear_memory_base_ptr().unwrap() as usize; - Smaps::new().get_rss(base_addr).expect("failed to get rss") - } - - instance - .call_export("test_dirty_plenty_memory", &(heap_base as u32, 1u32).encode()) - .unwrap(); - let probe_1 = probe_rss(&*instance); - instance - .call_export("test_dirty_plenty_memory", &(heap_base as u32, 1024u32).encode()) - .unwrap(); - let probe_2 = probe_rss(&*instance); - - assert_eq!(probe_1, 0); - assert_eq!(probe_2, 0); -} diff --git a/substrate/client/executor/src/integration_tests/linux/smaps.rs b/substrate/client/executor/src/integration_tests/linux/smaps.rs deleted file mode 100644 index 1ac570dd8d5f237b61eb4006e39c8b25bb211a90..0000000000000000000000000000000000000000 --- a/substrate/client/executor/src/integration_tests/linux/smaps.rs +++ /dev/null @@ -1,82 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! A tool for extracting information about the memory consumption of the current process from -//! the procfs. - -use std::{collections::BTreeMap, ops::Range}; - -/// An interface to the /proc/self/smaps -/// -/// See docs about [procfs on kernel.org][procfs] -/// -/// [procfs]: https://www.kernel.org/doc/html/latest/filesystems/proc.html -pub struct Smaps(Vec<(Range, BTreeMap)>); - -impl Smaps { - pub fn new() -> Self { - let regex_start = regex::RegexBuilder::new("^([0-9a-f]+)-([0-9a-f]+)") - .multi_line(true) - .build() - .unwrap(); - let regex_kv = regex::RegexBuilder::new(r#"^([^:]+):\s*(\d+) kB"#) - .multi_line(true) - .build() - .unwrap(); - let smaps = std::fs::read_to_string("/proc/self/smaps").unwrap(); - let boundaries: Vec<_> = regex_start - .find_iter(&smaps) - .map(|matched| matched.start()) - .chain(std::iter::once(smaps.len())) - .collect(); - - let mut output = Vec::new(); - for window in boundaries.windows(2) { - let chunk = &smaps[window[0]..window[1]]; - let caps = regex_start.captures(chunk).unwrap(); - let start = usize::from_str_radix(caps.get(1).unwrap().as_str(), 16).unwrap(); - let end = usize::from_str_radix(caps.get(2).unwrap().as_str(), 16).unwrap(); - - let values = regex_kv - .captures_iter(chunk) - .map(|cap| { - let key = cap.get(1).unwrap().as_str().to_owned(); - let value = cap.get(2).unwrap().as_str().parse().unwrap(); - (key, value) - }) - .collect(); - - output.push((start..end, values)); - } - - Self(output) - } - - fn get_map(&self, addr: usize) -> &BTreeMap { - &self - .0 - .iter() - .find(|(range, _)| addr >= range.start && addr < range.end) - .unwrap() - .1 - } - - pub fn get_rss(&self, addr: usize) -> Option { - self.get_map(addr).get("Rss").cloned() - } -} diff --git a/substrate/client/executor/src/integration_tests/mod.rs b/substrate/client/executor/src/integration_tests/mod.rs index 37aed8eef96a1b81e703a26b9cd95e6a19c346a1..0bd080c243574c8af93e8f2bd66190bef35ebe24 100644 --- a/substrate/client/executor/src/integration_tests/mod.rs +++ b/substrate/client/executor/src/integration_tests/mod.rs @@ -16,9 +16,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -#[cfg(target_os = "linux")] -mod linux; - use assert_matches::assert_matches; use codec::{Decode, Encode}; use sc_executor_common::{ @@ -81,14 +78,6 @@ macro_rules! test_wasm_execution { instantiation_strategy: sc_executor_wasmtime::InstantiationStrategy::Pooling }); } - - #[test] - fn [<$method_name _compiled_legacy_instance_reuse>]() { - let _ = sp_tracing::try_init_simple(); - $method_name(WasmExecutionMethod::Compiled { - instantiation_strategy: sc_executor_wasmtime::InstantiationStrategy::LegacyInstanceReuse - }); - } } }; } diff --git a/substrate/client/executor/wasmtime/README.md b/substrate/client/executor/wasmtime/README.md index 3e9ac0bddbdc17539d12893553789983e38ab05f..64fc2fa0c28485566fc60329ad8cfc0057112bc7 100644 --- a/substrate/client/executor/wasmtime/README.md +++ b/substrate/client/executor/wasmtime/README.md @@ -1 +1 @@ -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/executor/wasmtime/src/host.rs b/substrate/client/executor/wasmtime/src/host.rs index 9bd3ca3dade5eaefc4f9457bd43663b0effaf03d..f8c78cbb660eefbe1a25a77a3a5301d99c9f3b9b 100644 --- a/substrate/client/executor/wasmtime/src/host.rs +++ b/substrate/client/executor/wasmtime/src/host.rs @@ -32,7 +32,7 @@ use crate::{instance_wrapper::MemoryWrapper, runtime::StoreData, util}; pub struct HostState { /// The allocator instance to keep track of allocated memory. /// - /// This is stored as an `Option` as we need to temporarly set this to `None` when we are + /// This is stored as an `Option` as we need to temporarily set this to `None` when we are /// allocating/deallocating memory. The problem being that we can only mutable access `caller` /// once. allocator: Option, diff --git a/substrate/client/executor/wasmtime/src/instance_wrapper.rs b/substrate/client/executor/wasmtime/src/instance_wrapper.rs index 6d319cce509e54c6c52c2f55359e6c4bedd64bae..acc799061c27ca9ab7b96bde403f7e092d127bd6 100644 --- a/substrate/client/executor/wasmtime/src/instance_wrapper.rs +++ b/substrate/client/executor/wasmtime/src/instance_wrapper.rs @@ -116,14 +116,14 @@ impl EntryPoint { pub(crate) struct MemoryWrapper<'a, C>(pub &'a wasmtime::Memory, pub &'a mut C); impl sc_allocator::Memory for MemoryWrapper<'_, C> { - fn with_access(&self, run: impl FnOnce(&[u8]) -> R) -> R { - run(self.0.data(&self.1)) - } - fn with_access_mut(&mut self, run: impl FnOnce(&mut [u8]) -> R) -> R { run(self.0.data_mut(&mut self.1)) } + fn with_access(&self, run: impl FnOnce(&[u8]) -> R) -> R { + run(self.0.data(&self.1)) + } + fn grow(&mut self, additional: u32) -> std::result::Result<(), ()> { self.0 .grow(&mut self.1, additional as u64) @@ -153,11 +153,6 @@ impl sc_allocator::Memory for MemoryWrapper<'_, C> { /// routines. pub struct InstanceWrapper { instance: Instance, - /// The memory instance of the `instance`. - /// - /// It is important to make sure that we don't make any copies of this to make it easier to - /// proof - memory: Memory, store: Store, } @@ -177,7 +172,7 @@ impl InstanceWrapper { store.data_mut().memory = Some(memory); store.data_mut().table = table; - Ok(InstanceWrapper { instance, memory, store }) + Ok(InstanceWrapper { instance, store }) } /// Resolves a substrate entrypoint by the given name. @@ -280,11 +275,6 @@ impl InstanceWrapper { _ => Err("Unknown value type".into()), } } - - /// Get a global with the given `name`. - pub fn get_global(&mut self, name: &str) -> Option { - self.instance.get_global(&mut self.store, name) - } } /// Extract linear memory instance from the given instance. @@ -311,76 +301,6 @@ fn get_table(instance: &Instance, ctx: &mut Store) -> Option
{ /// Functions related to memory. impl InstanceWrapper { - /// Returns the pointer to the first byte of the linear memory for this instance. - pub fn base_ptr(&self) -> *const u8 { - self.memory.data_ptr(&self.store) - } - - /// If possible removes physical backing from the allocated linear memory which - /// leads to returning the memory back to the system; this also zeroes the memory - /// as a side-effect. - pub fn decommit(&mut self) { - if self.memory.data_size(&self.store) == 0 { - return - } - - cfg_if::cfg_if! { - if #[cfg(target_os = "linux")] { - use std::sync::Once; - - unsafe { - let ptr = self.memory.data_ptr(&self.store); - let len = self.memory.data_size(&self.store); - - // Linux handles MADV_DONTNEED reliably. The result is that the given area - // is unmapped and will be zeroed on the next pagefault. - if libc::madvise(ptr as _, len, libc::MADV_DONTNEED) != 0 { - static LOGGED: Once = Once::new(); - LOGGED.call_once(|| { - log::warn!( - "madvise(MADV_DONTNEED) failed: {}", - std::io::Error::last_os_error(), - ); - }); - } else { - return; - } - } - } else if #[cfg(target_os = "macos")] { - use std::sync::Once; - - unsafe { - let ptr = self.memory.data_ptr(&self.store); - let len = self.memory.data_size(&self.store); - - // On MacOS we can simply overwrite memory mapping. - if libc::mmap( - ptr as _, - len, - libc::PROT_READ | libc::PROT_WRITE, - libc::MAP_FIXED | libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, - -1, - 0, - ) == libc::MAP_FAILED { - static LOGGED: Once = Once::new(); - LOGGED.call_once(|| { - log::warn!( - "Failed to decommit WASM instance memory through mmap: {}", - std::io::Error::last_os_error(), - ); - }); - } else { - return; - } - } - } - } - - // If we're on an unsupported OS or the memory couldn't have been - // decommited for some reason then just manually zero it out. - self.memory.data_mut(self.store.as_context_mut()).fill(0); - } - pub(crate) fn store(&self) -> &Store { &self.store } @@ -389,17 +309,3 @@ impl InstanceWrapper { &mut self.store } } - -#[test] -fn decommit_works() { - let engine = wasmtime::Engine::default(); - let code = wat::parse_str("(module (memory (export \"memory\") 1 4))").unwrap(); - let module = wasmtime::Module::new(&engine, code).unwrap(); - let linker = wasmtime::Linker::new(&engine); - let instance_pre = linker.instantiate_pre(&module).unwrap(); - let mut wrapper = InstanceWrapper::new(&engine, &instance_pre).unwrap(); - unsafe { *wrapper.memory.data_ptr(&wrapper.store) = 42 }; - assert_eq!(unsafe { *wrapper.memory.data_ptr(&wrapper.store) }, 42); - wrapper.decommit(); - assert_eq!(unsafe { *wrapper.memory.data_ptr(&wrapper.store) }, 0); -} diff --git a/substrate/client/executor/wasmtime/src/runtime.rs b/substrate/client/executor/wasmtime/src/runtime.rs index 23b069870aa36f8c7e2b2e0fa45f9126c86c24de..ae78137959be1a4799bce16d23d2654931be4c47 100644 --- a/substrate/client/executor/wasmtime/src/runtime.rs +++ b/substrate/client/executor/wasmtime/src/runtime.rs @@ -27,9 +27,7 @@ use crate::{ use sc_allocator::{AllocationStats, FreeingBumpHeapAllocator}; use sc_executor_common::{ error::{Error, Result, WasmError}, - runtime_blob::{ - self, DataSegmentsSnapshot, ExposedMutableGlobalsSet, GlobalsSnapshot, RuntimeBlob, - }, + runtime_blob::RuntimeBlob, util::checked_range, wasm_runtime::{HeapAllocStrategy, InvokeMethod, WasmInstance, WasmModule}, }; @@ -69,17 +67,11 @@ impl StoreData { pub(crate) type Store = wasmtime::Store; enum Strategy { - LegacyInstanceReuse { - instance_wrapper: InstanceWrapper, - globals_snapshot: GlobalsSnapshot, - data_segments_snapshot: Arc, - heap_base: u32, - }, RecreateInstance(InstanceCreator), } struct InstanceCreator { - engine: wasmtime::Engine, + engine: Engine, instance_pre: Arc>, } @@ -89,40 +81,10 @@ impl InstanceCreator { } } -struct InstanceGlobals<'a> { - instance: &'a mut InstanceWrapper, -} - -impl<'a> runtime_blob::InstanceGlobals for InstanceGlobals<'a> { - type Global = wasmtime::Global; - - fn get_global(&mut self, export_name: &str) -> Self::Global { - self.instance - .get_global(export_name) - .expect("get_global is guaranteed to be called with an export name of a global; qed") - } - - fn get_global_value(&mut self, global: &Self::Global) -> Value { - util::from_wasmtime_val(global.get(&mut self.instance.store_mut())) - } - - fn set_global_value(&mut self, global: &Self::Global, value: Value) { - global.set(&mut self.instance.store_mut(), util::into_wasmtime_val(value)).expect( - "the value is guaranteed to be of the same value; the global is guaranteed to be mutable; qed", - ); - } -} - -/// Data required for creating instances with the fast instance reuse strategy. -struct InstanceSnapshotData { - mutable_globals: ExposedMutableGlobalsSet, - data_segments_snapshot: Arc, -} - /// A `WasmModule` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - engine: wasmtime::Engine, + engine: Engine, instance_pre: Arc>, instantiation_strategy: InternalInstantiationStrategy, } @@ -130,26 +92,6 @@ pub struct WasmtimeRuntime { impl WasmModule for WasmtimeRuntime { fn new_instance(&self) -> Result> { let strategy = match self.instantiation_strategy { - InternalInstantiationStrategy::LegacyInstanceReuse(ref snapshot_data) => { - let mut instance_wrapper = InstanceWrapper::new(&self.engine, &self.instance_pre)?; - let heap_base = instance_wrapper.extract_heap_base()?; - - // This function panics if the instance was created from a runtime blob different - // from which the mutable globals were collected. Here, it is easy to see that there - // is only a single runtime blob and thus it's the same that was used for both - // creating the instance and collecting the mutable globals. - let globals_snapshot = GlobalsSnapshot::take( - &snapshot_data.mutable_globals, - &mut InstanceGlobals { instance: &mut instance_wrapper }, - ); - - Strategy::LegacyInstanceReuse { - instance_wrapper, - globals_snapshot, - data_segments_snapshot: snapshot_data.data_segments_snapshot.clone(), - heap_base, - } - }, InternalInstantiationStrategy::Builtin => Strategy::RecreateInstance(InstanceCreator { engine: self.engine.clone(), instance_pre: self.instance_pre.clone(), @@ -174,39 +116,12 @@ impl WasmtimeInstance { allocation_stats: &mut Option, ) -> Result> { match &mut self.strategy { - Strategy::LegacyInstanceReuse { - ref mut instance_wrapper, - globals_snapshot, - data_segments_snapshot, - heap_base, - } => { - let entrypoint = instance_wrapper.resolve_entrypoint(method)?; - - data_segments_snapshot.apply(|offset, contents| { - util::write_memory_from( - instance_wrapper.store_mut(), - Pointer::new(offset), - contents, - ) - })?; - globals_snapshot.apply(&mut InstanceGlobals { instance: instance_wrapper }); - let allocator = FreeingBumpHeapAllocator::new(*heap_base); - - let result = - perform_call(data, instance_wrapper, entrypoint, allocator, allocation_stats); - - // Signal to the OS that we are done with the linear memory and that it can be - // reclaimed. - instance_wrapper.decommit(); - - result - }, Strategy::RecreateInstance(ref mut instance_creator) => { let mut instance_wrapper = instance_creator.instantiate()?; let heap_base = instance_wrapper.extract_heap_base()?; let entrypoint = instance_wrapper.resolve_entrypoint(method)?; - let allocator = FreeingBumpHeapAllocator::new(heap_base); + perform_call(data, &mut instance_wrapper, entrypoint, allocator, allocation_stats) }, } @@ -226,24 +141,10 @@ impl WasmInstance for WasmtimeInstance { fn get_global_const(&mut self, name: &str) -> Result> { match &mut self.strategy { - Strategy::LegacyInstanceReuse { instance_wrapper, .. } => - instance_wrapper.get_global_val(name), Strategy::RecreateInstance(ref mut instance_creator) => instance_creator.instantiate()?.get_global_val(name), } } - - fn linear_memory_base_ptr(&self) -> Option<*const u8> { - match &self.strategy { - Strategy::RecreateInstance(_) => { - // We do not keep the wasm instance around, therefore there is no linear memory - // associated with it. - None - }, - Strategy::LegacyInstanceReuse { instance_wrapper, .. } => - Some(instance_wrapper.base_ptr()), - } - } } /// Prepare a directory structure and a config file to enable wasmtime caching. @@ -338,7 +239,6 @@ fn common_config(semantics: &Semantics) -> std::result::Result (true, false), InstantiationStrategy::RecreateInstanceCopyOnWrite => (false, true), InstantiationStrategy::RecreateInstance => (false, false), - InstantiationStrategy::LegacyInstanceReuse => (false, false), }; const WASM_PAGE_SIZE: u64 = 65536; @@ -409,7 +309,7 @@ fn common_config(semantics: &Semantics) -> std::result::Result { - let data_segments_snapshot = - DataSegmentsSnapshot::take(&blob).map_err(|e| { - WasmError::Other(format!("cannot take data segments snapshot: {}", e)) - })?; - let data_segments_snapshot = Arc::new(data_segments_snapshot); - let mutable_globals = ExposedMutableGlobalsSet::collect(&blob); - - ( - module, - InternalInstantiationStrategy::LegacyInstanceReuse(InstanceSnapshotData { - data_segments_snapshot, - mutable_globals, - }), - ) - }, InstantiationStrategy::Pooling | InstantiationStrategy::PoolingCopyOnWrite | InstantiationStrategy::RecreateInstance | @@ -679,12 +559,6 @@ where } }, CodeSupplyMode::Precompiled(compiled_artifact_path) => { - if let InstantiationStrategy::LegacyInstanceReuse = - config.semantics.instantiation_strategy - { - return Err(WasmError::Other("the legacy instance reuse instantiation strategy is incompatible with precompiled modules".into())); - } - // SAFETY: The unsafety of `deserialize_file` is covered by this function. The // responsibilities to maintain the invariants are passed to the caller. // @@ -695,12 +569,6 @@ where (module, InternalInstantiationStrategy::Builtin) }, CodeSupplyMode::PrecompiledBytes(compiled_artifact_bytes) => { - if let InstantiationStrategy::LegacyInstanceReuse = - config.semantics.instantiation_strategy - { - return Err(WasmError::Other("the legacy instance reuse instantiation strategy is incompatible with precompiled modules".into())); - } - // SAFETY: The unsafety of `deserialize` is covered by this function. The // responsibilities to maintain the invariants are passed to the caller. // @@ -730,13 +598,6 @@ fn prepare_blob_for_compilation( blob = blob.inject_stack_depth_metering(logical_max)?; } - if let InstantiationStrategy::LegacyInstanceReuse = semantics.instantiation_strategy { - // When this strategy is used this must be called after all other passes which may introduce - // new global variables, otherwise they will not be reset when we call into the runtime - // again. - blob.expose_mutable_globals(); - } - // We don't actually need the memory to be imported so we can just convert any memory // import into an export with impunity. This simplifies our code since `wasmtime` will // now automatically take care of creating the memory for us, and it is also necessary diff --git a/substrate/client/executor/wasmtime/src/tests.rs b/substrate/client/executor/wasmtime/src/tests.rs index 65093687822d49b650c7a2a639b8a9ac864a202a..e185754b07696fde80f61787c659c869ff112842 100644 --- a/substrate/client/executor/wasmtime/src/tests.rs +++ b/substrate/client/executor/wasmtime/src/tests.rs @@ -30,7 +30,7 @@ type HostFunctions = sp_io::SubstrateHostFunctions; #[macro_export] macro_rules! test_wasm_execution { - (@no_legacy_instance_reuse $method_name:ident) => { + ($method_name:ident) => { paste::item! { #[test] fn [<$method_name _recreate_instance_cow>]() { @@ -61,19 +61,6 @@ macro_rules! test_wasm_execution { } } }; - - ($method_name:ident) => { - test_wasm_execution!(@no_legacy_instance_reuse $method_name); - - paste::item! { - #[test] - fn [<$method_name _legacy_instance_reuse>]() { - $method_name( - InstantiationStrategy::LegacyInstanceReuse - ); - } - } - }; } struct RuntimeBuilder { @@ -330,14 +317,14 @@ fn test_max_memory_pages_exported_memory_without_precompilation( test_max_memory_pages(instantiation_strategy, false, false); } -test_wasm_execution!(@no_legacy_instance_reuse test_max_memory_pages_imported_memory_with_precompilation); +test_wasm_execution!(test_max_memory_pages_imported_memory_with_precompilation); fn test_max_memory_pages_imported_memory_with_precompilation( instantiation_strategy: InstantiationStrategy, ) { test_max_memory_pages(instantiation_strategy, true, true); } -test_wasm_execution!(@no_legacy_instance_reuse test_max_memory_pages_exported_memory_with_precompilation); +test_wasm_execution!(test_max_memory_pages_exported_memory_with_precompilation); fn test_max_memory_pages_exported_memory_with_precompilation( instantiation_strategy: InstantiationStrategy, ) { diff --git a/substrate/client/executor/wasmtime/src/util.rs b/substrate/client/executor/wasmtime/src/util.rs index c38d969ce9dcdc67575a6259265e10f8e9b0ed91..7af554c35e1b37274ff35a797b558a75f9032de4 100644 --- a/substrate/client/executor/wasmtime/src/util.rs +++ b/substrate/client/executor/wasmtime/src/util.rs @@ -21,33 +21,9 @@ use sc_executor_common::{ error::{Error, Result}, util::checked_range, }; -use sp_wasm_interface::{Pointer, Value}; +use sp_wasm_interface::Pointer; use wasmtime::{AsContext, AsContextMut}; -/// Converts a [`wasmtime::Val`] into a substrate runtime interface [`Value`]. -/// -/// Panics if the given value doesn't have a corresponding variant in `Value`. -pub fn from_wasmtime_val(val: wasmtime::Val) -> Value { - match val { - wasmtime::Val::I32(v) => Value::I32(v), - wasmtime::Val::I64(v) => Value::I64(v), - wasmtime::Val::F32(f_bits) => Value::F32(f_bits), - wasmtime::Val::F64(f_bits) => Value::F64(f_bits), - v => panic!("Given value type is unsupported by Substrate: {:?}", v), - } -} - -/// Converts a sp_wasm_interface's [`Value`] into the corresponding variant in wasmtime's -/// [`wasmtime::Val`]. -pub fn into_wasmtime_val(value: Value) -> wasmtime::Val { - match value { - Value::I32(v) => wasmtime::Val::I32(v), - Value::I64(v) => wasmtime::Val::I64(v), - Value::F32(f_bits) => wasmtime::Val::F32(f_bits), - Value::F64(f_bits) => wasmtime::Val::F64(f_bits), - } -} - /// Read data from the instance memory into a slice. /// /// Returns an error if the read would go out of the memory bounds. @@ -140,8 +116,8 @@ pub(crate) fn replace_strategy_if_broken(strategy: &mut InstantiationStrategy) { // These strategies require a working `madvise` to be sound. InstantiationStrategy::PoolingCopyOnWrite => InstantiationStrategy::Pooling, - InstantiationStrategy::RecreateInstanceCopyOnWrite | - InstantiationStrategy::LegacyInstanceReuse => InstantiationStrategy::RecreateInstance, + InstantiationStrategy::RecreateInstanceCopyOnWrite => + InstantiationStrategy::RecreateInstance, }; use std::sync::OnceLock; diff --git a/substrate/client/informant/README.md b/substrate/client/informant/README.md index b494042590a4218aaa9c63d5e4b3c8a09a804b5b..a5ad89a932d36e0dd0b7209676c98a8078d16723 100644 --- a/substrate/client/informant/README.md +++ b/substrate/client/informant/README.md @@ -1,3 +1,3 @@ Console informant. Prints sync progress and block events. Runs on the calling thread. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/keystore/Cargo.toml b/substrate/client/keystore/Cargo.toml index 9ead4265a84ad3bc6f64763abdb5dcae6f3ec86b..6303ff5c27e1d943a3dd1b3ff86a4d7dad962e9a 100644 --- a/substrate/client/keystore/Cargo.toml +++ b/substrate/client/keystore/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" parking_lot = "0.12.1" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" sp-application-crypto = { path = "../../primitives/application-crypto" } sp-core = { path = "../../primitives/core" } diff --git a/substrate/client/keystore/README.md b/substrate/client/keystore/README.md index 9946a61d6fde65e8c0988d03c6d3e34b9d3d7bfb..5d50f7ad2cfa06f932131f5c8c53d41e91bdaf53 100644 --- a/substrate/client/keystore/README.md +++ b/substrate/client/keystore/README.md @@ -1,3 +1,3 @@ Keystore (and session key management) for ed25519 based chains like Polkadot. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/merkle-mountain-range/rpc/Cargo.toml b/substrate/client/merkle-mountain-range/rpc/Cargo.toml index 6aa32333af9729d11193cd426dc0776ed3583ffd..05d8547d243ffa35f74783accddb7a0deac8e5db 100644 --- a/substrate/client/merkle-mountain-range/rpc/Cargo.toml +++ b/substrate/client/merkle-mountain-range/rpc/Cargo.toml @@ -23,4 +23,4 @@ sp-runtime = { path = "../../../primitives/runtime" } anyhow = "1" [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" diff --git a/substrate/client/network-gossip/README.md b/substrate/client/network-gossip/README.md index 9030fac056407aada04522af786058f8f75d8718..900f223251d7b7ea33e24b5df834b52f7503c1bd 100644 --- a/substrate/client/network-gossip/README.md +++ b/substrate/client/network-gossip/README.md @@ -38,4 +38,4 @@ opens the door for neighbor status packets to be baked into the gossip protocol. These status packets will typically contain light pieces of information used to inform peers of a current view of protocol state. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 72e38ce498d30421b53c860ab5621fa536b9080a..8b188176fc5e57f487ac74aaa7f9b3c179183846 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -34,7 +34,7 @@ partial_sort = "0.2.0" pin-project = "1.0.12" rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" smallvec = "1.11.0" thiserror = "1.0" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } diff --git a/substrate/client/network/README.md b/substrate/client/network/README.md index cad46d059092c366754df98181e978651b8e3fa6..f4031fbd308539c6235f3bc2697dfa13136f039d 100644 --- a/substrate/client/network/README.md +++ b/substrate/client/network/README.md @@ -80,8 +80,8 @@ is "dot". In the protocol names below, `` must be replaced with the protocol ID. > **Note**: It is possible for the same connection to be used for multiple chains. For example, -> one can use both the `/dot/sync/2` and `/sub/sync/2` protocols on the same -> connection, provided that the remote supports them. +> one can use both the `/dot/sync/2` and `/sub/sync/2` protocols on the same +> connection, provided that the remote supports them. Substrate uses the following standard libp2p protocols: @@ -138,7 +138,7 @@ substream is closed, the entire connection is closed as well. This is a bug that resolved by deprecating the protocol entirely. Within the unique Substrate substream, messages encoded using -[*parity-scale-codec*](https://github.com/paritytech/parity-scale-codec) are exchanged. +[`parity-scale-codec``](https://github.com/paritytech/parity-scale-codec) are exchanged. The detail of theses messages is not totally in place, but they can be found in the `message.rs` file. @@ -240,7 +240,7 @@ The state is then imported into the database and the keep-up sync starts in norm This is similar to fast sync, but instead of downloading and verifying full header chain, the algorithm only downloads finalized authority set changes. -### GRANDPA warp sync. +### GRANDPA warp sync GRANDPA keeps justifications for each finalized authority set change. Each change is signed by the authorities from the previous set. By downloading and verifying these signed hand-offs starting from genesis, @@ -254,7 +254,7 @@ the fast sync. The state is verified to match the header storage root. After the database it is queried for the information that allows GRANDPA and BABE to continue operating from that state. This includes BABE epoch information and GRANDPA authority set id. -### Background block download. +### Background block download After the latest state has been imported the node is fully operational, but is still missing historic block data. I.e. it is unable to serve bock bodies and headers other than the most recent one. To make sure all diff --git a/substrate/client/network/common/src/sync.rs b/substrate/client/network/common/src/sync.rs index b142925aeb10c90971bdadff33b6bc10ccde471e..5a6f90b290d2f20fc76737d1d32fa804ade51f2c 100644 --- a/substrate/client/network/common/src/sync.rs +++ b/substrate/client/network/common/src/sync.rs @@ -27,7 +27,7 @@ use futures::Stream; use libp2p_identity::PeerId; -use message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse}; +use message::{BlockAnnounce, BlockRequest, BlockResponse}; use sc_consensus::{import_queue::RuntimeOrigin, IncomingBlock}; use sp_consensus::BlockOrigin; use sp_runtime::{ @@ -157,38 +157,6 @@ pub enum ImportResult { JustificationImport(RuntimeOrigin, B::Hash, NumberFor, Justifications), } -/// Value polled from `ChainSync` -#[derive(Debug)] -pub enum PollResult { - Import(ImportResult), - Announce(PollBlockAnnounceValidation), -} - -/// Result of [`ChainSync::poll_block_announce_validation`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum PollBlockAnnounceValidation { - /// The announcement failed at validation. - /// - /// The peer reputation should be decreased. - Failure { - /// Who sent the processed block announcement? - who: PeerId, - /// Should the peer be disconnected? - disconnect: bool, - }, - /// The announcement does not require further handling. - Nothing { - /// Who sent the processed block announcement? - who: PeerId, - /// Was this their new best block? - is_best: bool, - /// The announcement. - announce: BlockAnnounce, - }, - /// The block announcement should be skipped. - Skip, -} - /// Sync operation mode. #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum SyncMode { @@ -258,28 +226,6 @@ impl fmt::Debug for OpaqueStateResponse { } } -/// Wrapper for implementation-specific block request. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueBlockRequest(pub Box); - -impl fmt::Debug for OpaqueBlockRequest { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueBlockRequest").finish() - } -} - -/// Wrapper for implementation-specific block response. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueBlockResponse(pub Box); - -impl fmt::Debug for OpaqueBlockResponse { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueBlockResponse").finish() - } -} - /// Provides high-level status of syncing. #[async_trait::async_trait] pub trait SyncStatusProvider: Send + Sync { @@ -408,29 +354,14 @@ pub trait ChainSync: Send { /// Notify about finalization of the given block. fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor); - /// Push a block announce validation. - /// - /// It is required that [`ChainSync::poll_block_announce_validation`] is called - /// to check for finished block announce validations. - fn push_block_announce_validation( + /// Notify about pre-validated block announcement. + fn on_validated_block_announce( &mut self, - who: PeerId, - hash: Block::Hash, - announce: BlockAnnounce, is_best: bool, + who: PeerId, + announce: &BlockAnnounce, ); - /// Poll block announce validation. - /// - /// Block announce validations can be pushed by using - /// [`ChainSync::push_block_announce_validation`]. - /// - /// This should be polled until it returns [`Poll::Pending`]. - fn poll_block_announce_validation( - &mut self, - cx: &mut std::task::Context<'_>, - ) -> Poll>; - /// Call when a peer has disconnected. /// Canceled obsolete block request may result in some blocks being ready for /// import, so this functions checks for such blocks and returns them. @@ -439,22 +370,8 @@ pub trait ChainSync: Send { /// Return some key metrics. fn metrics(&self) -> Metrics; - /// Access blocks from implementation-specific block response. - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String>; - /// Advance the state of `ChainSync` - /// - /// Internally calls [`ChainSync::poll_block_announce_validation()`] and - /// this function should be polled until it returns [`Poll::Pending`] to - /// consume all pending events. - fn poll( - &mut self, - cx: &mut std::task::Context, - ) -> Poll>; + fn poll(&mut self, cx: &mut std::task::Context) -> Poll<()>; /// Send block request to peer fn send_block_request(&mut self, who: PeerId, request: BlockRequest); diff --git a/substrate/client/network/common/src/sync/warp.rs b/substrate/client/network/common/src/sync/warp.rs index 91d6c4151a42d0ca82e565aeeddcf7afb6b024a2..f4e39f438512889742c13f3081fe66554f2688ae 100644 --- a/substrate/client/network/common/src/sync/warp.rs +++ b/substrate/client/network/common/src/sync/warp.rs @@ -15,10 +15,9 @@ // along with Substrate. If not, see . use codec::{Decode, Encode}; -use futures::channel::oneshot; pub use sp_consensus_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, NumberFor}; -use std::{fmt, sync::Arc}; +use std::fmt; /// Scale-encoded warp sync proof response. pub struct EncodedProof(pub Vec); @@ -30,16 +29,6 @@ pub struct WarpProofRequest { pub begin: B::Hash, } -/// The different types of warp syncing. -pub enum WarpSyncParams { - /// Standard warp sync for the chain. - WithProvider(Arc>), - /// Skip downloading proofs and wait for a header of the state that should be downloaded. - /// - /// It is expected that the header provider ensures that the header is trusted. - WaitForTarget(oneshot::Receiver<::Header>), -} - /// Proof verification result. pub enum VerificationResult { /// Proof is valid, but the target was not reached. diff --git a/substrate/client/network/sync/src/block_announce_validator.rs b/substrate/client/network/sync/src/block_announce_validator.rs new file mode 100644 index 0000000000000000000000000000000000000000..f083f9e29e44f46c038efd849c292aff9ad720c7 --- /dev/null +++ b/substrate/client/network/sync/src/block_announce_validator.rs @@ -0,0 +1,405 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! `BlockAnnounceValidator` is responsible for async validation of block announcements. + +use crate::futures_stream::FuturesStream; +use futures::{Future, FutureExt, Stream, StreamExt}; +use libp2p::PeerId; +use log::{debug, error, trace, warn}; +use sc_network_common::sync::message::BlockAnnounce; +use sp_consensus::block_validation::Validation; +use sp_runtime::traits::{Block as BlockT, Header, Zero}; +use std::{ + collections::{hash_map::Entry, HashMap}, + default::Default, + pin::Pin, + task::{Context, Poll}, +}; + +/// Log target for this file. +const LOG_TARGET: &str = "sync"; + +/// Maximum number of concurrent block announce validations. +/// +/// If the queue reaches the maximum, we drop any new block +/// announcements. +const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS: usize = 256; + +/// Maximum number of concurrent block announce validations per peer. +/// +/// See [`MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS`] for more information. +const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER: usize = 4; + +/// Item that yields [`Stream`] implementation of [`BlockAnnounceValidator`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum BlockAnnounceValidationResult { + /// The announcement failed at validation. + /// + /// The peer reputation should be decreased. + Failure { + /// The id of the peer that send us the announcement. + peer_id: PeerId, + /// Should the peer be disconnected? + disconnect: bool, + }, + /// The announcement was validated successfully and should be passed to [`crate::ChainSync`]. + Process { + /// The id of the peer that send us the announcement. + peer_id: PeerId, + /// Was this their new best block? + is_new_best: bool, + /// The announcement. + announce: BlockAnnounce, + }, + /// The block announcement should be skipped. + Skip { + /// The id of the peer that send us the announcement. + peer_id: PeerId, + }, +} + +impl BlockAnnounceValidationResult { + fn peer_id(&self) -> &PeerId { + match self { + BlockAnnounceValidationResult::Failure { peer_id, .. } | + BlockAnnounceValidationResult::Process { peer_id, .. } | + BlockAnnounceValidationResult::Skip { peer_id } => peer_id, + } + } +} + +/// Result of [`BlockAnnounceValidator::allocate_slot_for_block_announce_validation`]. +enum AllocateSlotForBlockAnnounceValidation { + /// Success, there is a slot for the block announce validation. + Allocated, + /// We reached the total maximum number of validation slots. + TotalMaximumSlotsReached, + /// We reached the maximum number of validation slots for the given peer. + MaximumPeerSlotsReached, +} + +pub(crate) struct BlockAnnounceValidator { + /// A type to check incoming block announcements. + validator: Box + Send>, + /// All block announcements that are currently being validated. + validations: FuturesStream< + Pin> + Send>>, + >, + /// Number of concurrent block announce validations per peer. + validations_per_peer: HashMap, +} + +impl BlockAnnounceValidator { + pub(crate) fn new( + validator: Box + Send>, + ) -> Self { + Self { + validator, + validations: Default::default(), + validations_per_peer: Default::default(), + } + } + + /// Push a block announce validation. + pub(crate) fn push_block_announce_validation( + &mut self, + peer_id: PeerId, + hash: B::Hash, + announce: BlockAnnounce, + is_best: bool, + ) { + let header = &announce.header; + let number = *header.number(); + debug!( + target: LOG_TARGET, + "Pre-validating received block announcement {:?} with number {:?} from {}", + hash, + number, + peer_id, + ); + + if number.is_zero() { + warn!( + target: LOG_TARGET, + "💔 Ignored genesis block (#0) announcement from {}: {}", + peer_id, + hash, + ); + return + } + + // Try to allocate a slot for this block announce validation. + match self.allocate_slot_for_block_announce_validation(&peer_id) { + AllocateSlotForBlockAnnounceValidation::Allocated => {}, + AllocateSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => { + warn!( + target: LOG_TARGET, + "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.", + number, + hash, + peer_id, + ); + return + }, + AllocateSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => { + warn!( + target: LOG_TARGET, + "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.", + number, + hash, + peer_id, + ); + return + }, + } + + // Let external validator check the block announcement. + let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); + let future = self.validator.validate(header, assoc_data); + + self.validations.push( + async move { + match future.await { + Ok(Validation::Success { is_new_best }) => { + let is_new_best = is_new_best || is_best; + + trace!( + target: LOG_TARGET, + "Block announcement validated successfully: from {}: {:?}. Local best: {}.", + peer_id, + announce.summary(), + is_new_best, + ); + + BlockAnnounceValidationResult::Process { is_new_best, announce, peer_id } + }, + Ok(Validation::Failure { disconnect }) => { + debug!( + target: LOG_TARGET, + "Block announcement validation failed: from {}, block {:?}. Disconnect: {}.", + peer_id, + hash, + disconnect, + ); + + BlockAnnounceValidationResult::Failure { peer_id, disconnect } + }, + Err(e) => { + debug!( + target: LOG_TARGET, + "💔 Ignoring block announcement validation from {} of block {:?} due to internal error: {}.", + peer_id, + hash, + e, + ); + + BlockAnnounceValidationResult::Skip { peer_id } + }, + } + } + .boxed(), + ); + } + + /// Checks if there is a slot for a block announce validation. + /// + /// The total number and the number per peer of concurrent block announce validations + /// is capped. + /// + /// Returns [`AllocateSlotForBlockAnnounceValidation`] to inform about the result. + /// + /// # Note + /// + /// It is *required* to call [`Self::deallocate_slot_for_block_announce_validation`] when the + /// validation is finished to clear the slot. + fn allocate_slot_for_block_announce_validation( + &mut self, + peer_id: &PeerId, + ) -> AllocateSlotForBlockAnnounceValidation { + if self.validations.len() >= MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS { + return AllocateSlotForBlockAnnounceValidation::TotalMaximumSlotsReached + } + + match self.validations_per_peer.entry(*peer_id) { + Entry::Vacant(entry) => { + entry.insert(1); + AllocateSlotForBlockAnnounceValidation::Allocated + }, + Entry::Occupied(mut entry) => { + if *entry.get() < MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER { + *entry.get_mut() += 1; + AllocateSlotForBlockAnnounceValidation::Allocated + } else { + AllocateSlotForBlockAnnounceValidation::MaximumPeerSlotsReached + } + }, + } + } + + /// Should be called when a block announce validation is finished, to update the slots + /// of the peer that send the block announce. + fn deallocate_slot_for_block_announce_validation(&mut self, peer_id: &PeerId) { + match self.validations_per_peer.entry(*peer_id) { + Entry::Vacant(_) => { + error!( + target: LOG_TARGET, + "💔 Block announcement validation from peer {} finished for a slot that was not allocated!", + peer_id, + ); + }, + Entry::Occupied(mut entry) => match entry.get().checked_sub(1) { + Some(value) => + if value == 0 { + entry.remove(); + } else { + *entry.get_mut() = value; + }, + None => { + entry.remove(); + + error!( + target: LOG_TARGET, + "Invalid (zero) block announce validation slot counter for peer {peer_id}.", + ); + debug_assert!( + false, + "Invalid (zero) block announce validation slot counter for peer {peer_id}.", + ); + }, + }, + } + } +} + +impl Stream for BlockAnnounceValidator { + type Item = BlockAnnounceValidationResult; + + /// Poll for finished block announce validations. The stream never terminates. + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let validation = futures::ready!(self.validations.poll_next_unpin(cx)) + .expect("`FuturesStream` never terminates; qed"); + self.deallocate_slot_for_block_announce_validation(validation.peer_id()); + + Poll::Ready(Some(validation)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::block_announce_validator::AllocateSlotForBlockAnnounceValidation; + use libp2p::PeerId; + use sp_consensus::block_validation::DefaultBlockAnnounceValidator; + use substrate_test_runtime_client::runtime::Block; + + #[test] + fn allocate_one_validation_slot() { + let mut validator = + BlockAnnounceValidator::::new(Box::new(DefaultBlockAnnounceValidator {})); + let peer_id = PeerId::random(); + + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + } + + #[test] + fn allocate_validation_slots_for_two_peers() { + let mut validator = + BlockAnnounceValidator::::new(Box::new(DefaultBlockAnnounceValidator {})); + let peer_id_1 = PeerId::random(); + let peer_id_2 = PeerId::random(); + + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id_1), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id_2), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + } + + #[test] + fn maximum_validation_slots_per_peer() { + let mut validator = + BlockAnnounceValidator::::new(Box::new(DefaultBlockAnnounceValidator {})); + let peer_id = PeerId::random(); + + for _ in 0..MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER { + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + } + + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::MaximumPeerSlotsReached, + )); + } + + #[test] + fn validation_slots_per_peer_deallocated() { + let mut validator = + BlockAnnounceValidator::::new(Box::new(DefaultBlockAnnounceValidator {})); + let peer_id = PeerId::random(); + + for _ in 0..MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER { + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + } + + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::MaximumPeerSlotsReached, + )); + + validator.deallocate_slot_for_block_announce_validation(&peer_id); + + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::Allocated, + )); + } + + #[test] + fn maximum_validation_slots_for_all_peers() { + let mut validator = + BlockAnnounceValidator::::new(Box::new(DefaultBlockAnnounceValidator {})); + + for _ in 0..MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS { + validator.validations.push( + futures::future::ready(BlockAnnounceValidationResult::Skip { + peer_id: PeerId::random(), + }) + .boxed(), + ); + } + + let peer_id = PeerId::random(); + assert!(matches!( + validator.allocate_slot_for_block_announce_validation(&peer_id), + AllocateSlotForBlockAnnounceValidation::TotalMaximumSlotsReached, + )); + } +} diff --git a/substrate/client/network/sync/src/block_relay_protocol.rs b/substrate/client/network/sync/src/block_relay_protocol.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a313458bf0344772c54dec6a5c5328b324e6383 --- /dev/null +++ b/substrate/client/network/sync/src/block_relay_protocol.rs @@ -0,0 +1,72 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block relay protocol related definitions. + +use futures::channel::oneshot; +use libp2p::PeerId; +use sc_network::request_responses::{ProtocolConfig, RequestFailure}; +use sc_network_common::sync::message::{BlockData, BlockRequest}; +use sp_runtime::traits::Block as BlockT; +use std::sync::Arc; + +/// The serving side of the block relay protocol. It runs a single instance +/// of the server task that processes the incoming protocol messages. +#[async_trait::async_trait] +pub trait BlockServer: Send { + /// Starts the protocol processing. + async fn run(&mut self); +} + +/// The client side stub to download blocks from peers. This is a handle +/// that can be used to initiate concurrent downloads. +#[async_trait::async_trait] +pub trait BlockDownloader: Send + Sync { + /// Performs the protocol specific sequence to fetch the blocks from the peer. + /// Output: if the download succeeds, the response is a `Vec` which is + /// in a format specific to the protocol implementation. The block data + /// can be extracted from this response using [`BlockDownloader::block_response_into_blocks`]. + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled>; + + /// Parses the protocol specific response to retrieve the block data. + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError>; +} + +/// Errors returned by [`BlockDownloader::block_response_into_blocks`]. +#[derive(Debug)] +pub enum BlockResponseError { + /// Failed to decode the response bytes. + DecodeFailed(String), + + /// Failed to extract the blocks from the decoded bytes. + ExtractionFailed(String), +} + +/// Block relay specific params for network creation, specified in +/// ['sc_service::BuildNetworkParams']. +pub struct BlockRelayParams { + pub server: Box>, + pub downloader: Arc>, + pub request_response_config: ProtocolConfig, +} diff --git a/substrate/client/network/sync/src/block_request_handler.rs b/substrate/client/network/sync/src/block_request_handler.rs index d90a00b37673beb1997cf240b7169c540e9a2ee4..c24083f6328783ef51f81a6806eca91772992820 100644 --- a/substrate/client/network/sync/src/block_request_handler.rs +++ b/substrate/client/network/sync/src/block_request_handler.rs @@ -18,29 +18,35 @@ //! `crate::request_responses::RequestResponsesBehaviour`. use crate::{ - schema::v1::{block_request::FromBlock, BlockResponse, Direction}, + block_relay_protocol::{BlockDownloader, BlockRelayParams, BlockResponseError, BlockServer}, + schema::v1::{ + block_request::FromBlock as FromBlockSchema, BlockRequest as BlockRequestSchema, + BlockResponse as BlockResponseSchema, BlockResponse, Direction, + }, + service::network::NetworkServiceHandle, MAX_BLOCKS_IN_RESPONSE, }; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeAll, Encode}; use futures::{channel::oneshot, stream::StreamExt}; use libp2p::PeerId; use log::debug; use prost::Message; -use schnellru::{ByLength, LruMap}; - use sc_client_api::BlockBackend; use sc_network::{ config::ProtocolId, - request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}, + request_responses::{ + IfDisconnected, IncomingRequest, OutgoingResponse, ProtocolConfig, RequestFailure, + }, + types::ProtocolName, }; -use sc_network_common::sync::message::BlockAttributes; +use sc_network_common::sync::message::{BlockAttributes, BlockData, BlockRequest, FromBlock}; +use schnellru::{ByLength, LruMap}; use sp_blockchain::HeaderBackend; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header, One, Zero}, }; - use std::{ cmp::min, hash::{Hash, Hasher}, @@ -129,7 +135,8 @@ enum SeenRequestsValue { Fulfilled(usize), } -/// Handler for incoming block requests from a remote peer. +/// The full block server implementation of [`BlockServer`]. It handles +/// the incoming block requests from a remote peer. pub struct BlockRequestHandler { client: Arc, request_receiver: async_channel::Receiver, @@ -146,11 +153,12 @@ where { /// Create a new [`BlockRequestHandler`]. pub fn new( + network: NetworkServiceHandle, protocol_id: &ProtocolId, fork_id: Option<&str>, client: Arc, num_peer_hint: usize, - ) -> (Self, ProtocolConfig) { + ) -> BlockRelayParams { // Reserve enough request slots for one request per peer when we are at the maximum // number of peers. let capacity = std::cmp::max(num_peer_hint, 1); @@ -170,11 +178,15 @@ where let capacity = ByLength::new(num_peer_hint.max(1) as u32 * 2); let seen_requests = LruMap::new(capacity); - (Self { client, request_receiver, seen_requests }, protocol_config) + BlockRelayParams { + server: Box::new(Self { client, request_receiver, seen_requests }), + downloader: Arc::new(FullBlockDownloader::new(protocol_config.name.clone(), network)), + request_response_config: protocol_config, + } } /// Run [`BlockRequestHandler`]. - pub async fn run(mut self) { + async fn process_requests(&mut self) { while let Some(request) = self.request_receiver.next().await { let IncomingRequest { peer, payload, pending_response } = request; @@ -197,11 +209,11 @@ where let request = crate::schema::v1::BlockRequest::decode(&payload[..])?; let from_block_id = match request.from_block.ok_or(HandleRequestError::MissingFromField)? { - FromBlock::Hash(ref h) => { + FromBlockSchema::Hash(ref h) => { let h = Decode::decode(&mut h.as_ref())?; BlockId::::Hash(h) }, - FromBlock::Number(ref n) => { + FromBlockSchema::Number(ref n) => { let n = Decode::decode(&mut n.as_ref())?; BlockId::::Number(n) }, @@ -448,6 +460,17 @@ where } } +#[async_trait::async_trait] +impl BlockServer for BlockRequestHandler +where + B: BlockT, + Client: HeaderBackend + BlockBackend + Send + Sync + 'static, +{ + async fn run(&mut self) { + self.process_requests().await; + } +} + #[derive(Debug, thiserror::Error)] enum HandleRequestError { #[error("Failed to decode request: {0}.")] @@ -465,3 +488,122 @@ enum HandleRequestError { #[error("Failed to send response.")] SendResponse, } + +/// The full block downloader implementation of [`BlockDownloader]. +pub struct FullBlockDownloader { + protocol_name: ProtocolName, + network: NetworkServiceHandle, +} + +impl FullBlockDownloader { + fn new(protocol_name: ProtocolName, network: NetworkServiceHandle) -> Self { + Self { protocol_name, network } + } + + /// Extracts the blocks from the response schema. + fn blocks_from_schema( + &self, + request: &BlockRequest, + response: BlockResponseSchema, + ) -> Result>, String> { + response + .blocks + .into_iter() + .map(|block_data| { + Ok(BlockData:: { + hash: Decode::decode(&mut block_data.hash.as_ref())?, + header: if !block_data.header.is_empty() { + Some(Decode::decode(&mut block_data.header.as_ref())?) + } else { + None + }, + body: if request.fields.contains(BlockAttributes::BODY) { + Some( + block_data + .body + .iter() + .map(|body| Decode::decode(&mut body.as_ref())) + .collect::, _>>()?, + ) + } else { + None + }, + indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { + Some(block_data.indexed_body) + } else { + None + }, + receipt: if !block_data.receipt.is_empty() { + Some(block_data.receipt) + } else { + None + }, + message_queue: if !block_data.message_queue.is_empty() { + Some(block_data.message_queue) + } else { + None + }, + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) + } else if block_data.is_empty_justification { + Some(Vec::new()) + } else { + None + }, + justifications: if !block_data.justifications.is_empty() { + Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) + } else { + None + }, + }) + }) + .collect::>() + .map_err(|error: codec::Error| error.to_string()) + } +} + +#[async_trait::async_trait] +impl BlockDownloader for FullBlockDownloader { + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled> { + // Build the request protobuf. + let bytes = BlockRequestSchema { + fields: request.fields.to_be_u32(), + from_block: match request.from { + FromBlock::Hash(h) => Some(FromBlockSchema::Hash(h.encode())), + FromBlock::Number(n) => Some(FromBlockSchema::Number(n.encode())), + }, + direction: request.direction as i32, + max_blocks: request.max.unwrap_or(0), + support_multiple_justifications: true, + } + .encode_to_vec(); + + let (tx, rx) = oneshot::channel(); + self.network.start_request( + who, + self.protocol_name.clone(), + bytes, + tx, + IfDisconnected::ImmediateError, + ); + rx.await + } + + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError> { + // Decode the response protobuf + let response_schema = BlockResponseSchema::decode(response.as_slice()) + .map_err(|error| BlockResponseError::DecodeFailed(error.to_string()))?; + + // Extract the block data from the protobuf + self.blocks_from_schema::(request, response_schema) + .map_err(|error| BlockResponseError::ExtractionFailed(error.to_string())) + } +} diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 65bd56a289584d231ac56e0155a556e221b3bea9..c93ba89c6220e7625cfd662bea984eebfa81d1b7 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -20,12 +20,21 @@ //! to tip and keep the blockchain up to date with network updates. use crate::{ + block_announce_validator::{ + BlockAnnounceValidationResult, BlockAnnounceValidator as BlockAnnounceValidatorStream, + }, + block_relay_protocol::BlockDownloader, service::{self, chain_sync::ToServiceCommand}, + warp::WarpSyncParams, ChainSync, ClientError, SyncingService, }; use codec::{Decode, Encode}; -use futures::{FutureExt, StreamExt}; +use futures::{ + channel::oneshot, + future::{BoxFuture, Fuse}, + FutureExt, StreamExt, +}; use futures_timer::Delay; use libp2p::PeerId; use prometheus_endpoint::{ @@ -44,8 +53,7 @@ use sc_network_common::{ role::Roles, sync::{ message::{BlockAnnounce, BlockAnnouncesHandshake, BlockState}, - warp::WarpSyncParams, - BadPeer, ChainSync as ChainSyncT, ExtendedPeerInfo, PollBlockAnnounceValidation, SyncEvent, + BadPeer, ChainSync as ChainSyncT, ExtendedPeerInfo, SyncEvent, }, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -64,6 +72,9 @@ use std::{ time::{Duration, Instant}, }; +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + /// Interval at which we perform time based maintenance const TICK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1100); @@ -239,12 +250,19 @@ pub struct SyncingEngine { /// Number of inbound peers accepted so far. num_in_peers: usize, + /// Async processor of block announce validations. + block_announce_validator: BlockAnnounceValidatorStream, + /// A cache for the data that was associated to a block announcement. block_announce_data_cache: LruMap>, /// The `PeerId`'s of all boot nodes. boot_node_ids: HashSet, + /// A channel to get target block header if we skip over proofs downloading during warp sync. + warp_sync_target_block_header_rx: + Fuse>>, + /// Protocol name used for block announcements block_announce_protocol_name: ProtocolName, @@ -283,7 +301,7 @@ where warp_sync_params: Option>, network_service: service::network::NetworkServiceHandle, import_queue: Box>, - block_request_protocol_name: ProtocolName, + block_downloader: Arc>, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, rx: sc_utils::mpsc::TracingUnboundedReceiver>, @@ -293,7 +311,11 @@ where let max_blocks_per_request = if net_config.network_config.max_blocks_per_request > crate::MAX_BLOCKS_IN_RESPONSE as u32 { - log::info!(target: "sync", "clamping maximum blocks per request to {}", crate::MAX_BLOCKS_IN_RESPONSE); + log::info!( + target: LOG_TARGET, + "clamping maximum blocks per request to {}", + crate::MAX_BLOCKS_IN_RESPONSE, + ); crate::MAX_BLOCKS_IN_RESPONSE as u32 } else { net_config.network_config.max_blocks_per_request @@ -346,20 +368,32 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; + // Split warp sync params into warp sync config and a channel to retreive target block + // header. + let (warp_sync_config, warp_sync_target_block_header_rx) = + warp_sync_params.map_or((None, None), |params| { + let (config, target_block_rx) = params.split(); + (Some(config), target_block_rx) + }); + + // Make sure polling of the target block channel is a no-op if there is no block to + // retrieve. + let warp_sync_target_block_header_rx = warp_sync_target_block_header_rx + .map_or(futures::future::pending().boxed().fuse(), |rx| rx.boxed().fuse()); + let (chain_sync, block_announce_config) = ChainSync::new( mode, client.clone(), protocol_id, fork_id, roles, - block_announce_validator, max_parallel_downloads, max_blocks_per_request, - warp_sync_params, + warp_sync_config, metrics_registry, network_service.clone(), import_queue, - block_request_protocol_name, + block_downloader, state_request_protocol_name, warp_sync_protocol_name, )?; @@ -389,6 +423,9 @@ where peers: HashMap::new(), block_announce_data_cache: LruMap::new(ByLength::new(cache_capacity)), block_announce_protocol_name, + block_announce_validator: BlockAnnounceValidatorStream::new( + block_announce_validator, + ), num_connected: num_connected.clone(), is_major_syncing: is_major_syncing.clone(), service_rx, @@ -396,6 +433,7 @@ where genesis_hash, important_peers, default_peers_set_no_slot_connected_peers: HashSet::new(), + warp_sync_target_block_header_rx, boot_node_ids, default_peers_set_no_slot_peers, default_peers_set_num_full, @@ -410,7 +448,7 @@ where match Metrics::register(r, is_major_syncing.clone()) { Ok(metrics) => Some(metrics), Err(err) => { - log::error!(target: "sync", "Failed to register metrics {err:?}"); + log::error!(target: LOG_TARGET, "Failed to register metrics {err:?}"); None }, } @@ -453,9 +491,9 @@ where } } - fn update_peer_info(&mut self, who: &PeerId) { - if let Some(info) = self.chain_sync.peer_info(who) { - if let Some(ref mut peer) = self.peers.get_mut(who) { + fn update_peer_info(&mut self, peer_id: &PeerId) { + if let Some(info) = self.chain_sync.peer_info(peer_id) { + if let Some(ref mut peer) = self.peers.get_mut(peer_id) { peer.info.best_hash = info.best_hash; peer.info.best_number = info.best_number; } @@ -463,14 +501,16 @@ where } /// Process the result of the block announce validation. - pub fn process_block_announce_validation_result( + fn process_block_announce_validation_result( &mut self, - validation_result: PollBlockAnnounceValidation, + validation_result: BlockAnnounceValidationResult, ) { match validation_result { - PollBlockAnnounceValidation::Skip => {}, - PollBlockAnnounceValidation::Nothing { is_best: _, who, announce } => { - self.update_peer_info(&who); + BlockAnnounceValidationResult::Skip { peer_id: _ } => {}, + BlockAnnounceValidationResult::Process { is_new_best, peer_id, announce } => { + self.chain_sync.on_validated_block_announce(is_new_best, peer_id, &announce); + + self.update_peer_info(&peer_id); if let Some(data) = announce.data { if !data.is_empty() { @@ -478,41 +518,32 @@ where } } }, - PollBlockAnnounceValidation::Failure { who, disconnect } => { + BlockAnnounceValidationResult::Failure { peer_id, disconnect } => { if disconnect { self.network_service - .disconnect_peer(who, self.block_announce_protocol_name.clone()); + .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); } - self.network_service.report_peer(who, rep::BAD_BLOCK_ANNOUNCEMENT); + self.network_service.report_peer(peer_id, rep::BAD_BLOCK_ANNOUNCEMENT); }, } } /// Push a block announce validation. - /// - /// It is required that [`ChainSync::poll_block_announce_validation`] is - /// called later to check for finished validations. The result of the validation - /// needs to be passed to [`SyncingEngine::process_block_announce_validation_result`] - /// to finish the processing. - /// - /// # Note - /// - /// This will internally create a future, but this future will not be registered - /// in the task before being polled once. So, it is required to call - /// [`ChainSync::poll_block_announce_validation`] to ensure that the future is - /// registered properly and will wake up the task when being ready. pub fn push_block_announce_validation( &mut self, - who: PeerId, + peer_id: PeerId, announce: BlockAnnounce, ) { let hash = announce.header.hash(); - let peer = match self.peers.get_mut(&who) { + let peer = match self.peers.get_mut(&peer_id) { Some(p) => p, None => { - log::error!(target: "sync", "Received block announce from disconnected peer {}", who); + log::error!( + target: LOG_TARGET, + "Received block announce from disconnected peer {peer_id}", + ); debug_assert!(false); return }, @@ -525,7 +556,8 @@ where BlockState::Normal => false, }; - self.chain_sync.push_block_announce_validation(who, hash, announce, is_best); + self.block_announce_validator + .push_block_announce_validation(peer_id, hash, announce, is_best); } } @@ -537,11 +569,11 @@ where let header = match self.client.header(hash) { Ok(Some(header)) => header, Ok(None) => { - log::warn!(target: "sync", "Trying to announce unknown block: {}", hash); + log::warn!(target: LOG_TARGET, "Trying to announce unknown block: {hash}"); return }, Err(e) => { - log::warn!(target: "sync", "Error reading block header {}: {}", hash, e); + log::warn!(target: LOG_TARGET, "Error reading block header {hash}: {e}"); return }, }; @@ -552,16 +584,16 @@ where } let is_best = self.client.info().best_hash == hash; - log::debug!(target: "sync", "Reannouncing block {:?} is_best: {}", hash, is_best); + log::debug!(target: LOG_TARGET, "Reannouncing block {hash:?} is_best: {is_best}"); let data = data .or_else(|| self.block_announce_data_cache.get(&hash).cloned()) .unwrap_or_default(); - for (who, ref mut peer) in self.peers.iter_mut() { + for (peer_id, ref mut peer) in self.peers.iter_mut() { let inserted = peer.known_blocks.insert(hash); if inserted { - log::trace!(target: "sync", "Announcing block {:?} to {}", hash, who); + log::trace!(target: LOG_TARGET, "Announcing block {hash:?} to {peer_id}"); let message = BlockAnnounce { header: header.clone(), state: if is_best { Some(BlockState::Best) } else { Some(BlockState::Normal) }, @@ -576,7 +608,7 @@ where /// Inform sync about new best imported block. pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { - log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); + log::debug!(target: LOG_TARGET, "New best block imported {hash:?}/#{number}"); self.chain_sync.update_chain_info(&hash, number); self.network_service.set_notification_handshake( @@ -620,7 +652,10 @@ where // consider it connected or are also all stalled. In order to unstall the node, // disconnect all peers and allow `ProtocolController` to establish new connections. if self.last_notification_io.elapsed() > INACTIVITY_EVICT_THRESHOLD { - log::debug!(target: "sync", "syncing has halted due to inactivity, evicting all peers"); + log::debug!( + target: LOG_TARGET, + "syncing has halted due to inactivity, evicting all peers", + ); for peer in self.peers.keys() { self.network_service.report_peer(*peer, rep::INACTIVE_SUBSTREAM); @@ -656,14 +691,17 @@ where } } }, - ToServiceCommand::JustificationImported(peer, hash, number, success) => { + ToServiceCommand::JustificationImported(peer_id, hash, number, success) => { self.chain_sync.on_justification_import(hash, number, success); if !success { - log::info!(target: "sync", "💔 Invalid justification provided by {} for #{}", peer, hash); + log::info!( + target: LOG_TARGET, + "💔 Invalid justification provided by {peer_id} for #{hash}", + ); self.network_service - .disconnect_peer(peer, self.block_announce_protocol_name.clone()); + .disconnect_peer(peer_id, self.block_announce_protocol_name.clone()); self.network_service.report_peer( - peer, + peer_id, ReputationChange::new_fatal("Invalid justification"), ); } @@ -698,8 +736,11 @@ where let _ = tx.send(self.chain_sync.num_sync_requests()); }, ToServiceCommand::PeersInfo(tx) => { - let peers_info = - self.peers.iter().map(|(id, peer)| (*id, peer.info.clone())).collect(); + let peers_info = self + .peers + .iter() + .map(|(peer_id, peer)| (*peer_id, peer.info.clone())) + .collect(); let _ = tx.send(peers_info); }, ToServiceCommand::OnBlockFinalized(hash, header) => @@ -721,7 +762,7 @@ where }, Err(()) => { log::debug!( - target: "sync", + target: LOG_TARGET, "Failed to register peer {remote:?}: {received_handshake:?}", ); let _ = tx.send(false); @@ -730,7 +771,7 @@ where sc_network::SyncEvent::NotificationStreamClosed { remote } => { if self.on_sync_peer_disconnected(remote).is_err() { log::trace!( - target: "sync", + target: LOG_TARGET, "Disconnected peer which had earlier been refused by on_sync_peer_connected {}", remote ); @@ -742,22 +783,13 @@ where if let Ok(announce) = BlockAnnounce::decode(&mut message.as_ref()) { self.last_notification_io = Instant::now(); self.push_block_announce_validation(remote, announce); - - // Make sure that the newly added block announce validation future - // was polled once to be registered in the task. - if let Poll::Ready(res) = - self.chain_sync.poll_block_announce_validation(cx) - { - self.process_block_announce_validation_result(res) - } } else { log::warn!(target: "sub-libp2p", "Failed to decode block announce"); } } else { log::trace!( - target: "sync", - "Received sync for peer earlier refused by sync layer: {}", - remote + target: LOG_TARGET, + "Received sync for peer earlier refused by sync layer: {remote}", ); } } @@ -770,10 +802,29 @@ where } } - // poll `ChainSync` last because of a block announcement was received through the - // event stream between `SyncingEngine` and `Protocol` and the validation finished - // right after it as queued, the resulting block request (if any) can be sent right away. - while let Poll::Ready(result) = self.chain_sync.poll(cx) { + // Retreive warp sync target block header just before polling `ChainSync` + // to make progress as soon as we receive it. + match self.warp_sync_target_block_header_rx.poll_unpin(cx) { + Poll::Ready(Ok(target)) => { + self.chain_sync.set_warp_sync_target_block(target); + }, + Poll::Ready(Err(err)) => { + log::error!( + target: LOG_TARGET, + "Failed to get target block for warp sync. Error: {err:?}", + ); + }, + Poll::Pending => {}, + } + + // Drive `ChainSync`. + while let Poll::Ready(()) = self.chain_sync.poll(cx) {} + + // Poll block announce validations last, because if a block announcement was received + // through the event stream between `SyncingEngine` and `Protocol` and the validation + // finished right after it is queued, the resulting block request (if any) can be sent + // right away. + while let Poll::Ready(Some(result)) = self.block_announce_validator.poll_next_unpin(cx) { self.process_block_announce_validation_result(result); } @@ -783,15 +834,15 @@ where /// Called by peer when it is disconnecting. /// /// Returns a result if the handshake of this peer was indeed accepted. - pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) -> Result<(), ()> { - if let Some(info) = self.peers.remove(&peer) { - if self.important_peers.contains(&peer) { - log::warn!(target: "sync", "Reserved peer {} disconnected", peer); + pub fn on_sync_peer_disconnected(&mut self, peer_id: PeerId) -> Result<(), ()> { + if let Some(info) = self.peers.remove(&peer_id) { + if self.important_peers.contains(&peer_id) { + log::warn!(target: LOG_TARGET, "Reserved peer {peer_id} disconnected"); } else { - log::debug!(target: "sync", "{} disconnected", peer); + log::debug!(target: LOG_TARGET, "{peer_id} disconnected"); } - if !self.default_peers_set_no_slot_connected_peers.remove(&peer) && + if !self.default_peers_set_no_slot_connected_peers.remove(&peer_id) && info.inbound && info.info.roles.is_full() { match self.num_in_peers.checked_sub(1) { @@ -800,7 +851,7 @@ where }, None => { log::error!( - target: "sync", + target: LOG_TARGET, "trying to disconnect an inbound node which is not counted as inbound" ); debug_assert!(false); @@ -808,9 +859,10 @@ where } } - self.chain_sync.peer_disconnected(&peer); - self.event_streams - .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); + self.chain_sync.peer_disconnected(&peer_id); + self.event_streams.retain(|stream| { + stream.unbounded_send(SyncEvent::PeerDisconnected(peer_id)).is_ok() + }); Ok(()) } else { Err(()) @@ -824,41 +876,44 @@ where /// from. pub fn on_sync_peer_connected( &mut self, - who: PeerId, + peer_id: PeerId, status: &BlockAnnouncesHandshake, sink: NotificationsSink, inbound: bool, ) -> Result<(), ()> { - log::trace!(target: "sync", "New peer {} {:?}", who, status); + log::trace!(target: LOG_TARGET, "New peer {peer_id} {status:?}"); - if self.peers.contains_key(&who) { - log::error!(target: "sync", "Called on_sync_peer_connected with already connected peer {}", who); + if self.peers.contains_key(&peer_id) { + log::error!( + target: LOG_TARGET, + "Called on_sync_peer_connected with already connected peer {peer_id}", + ); debug_assert!(false); return Err(()) } if status.genesis_hash != self.genesis_hash { - self.network_service.report_peer(who, rep::GENESIS_MISMATCH); + self.network_service.report_peer(peer_id, rep::GENESIS_MISMATCH); - if self.important_peers.contains(&who) { + if self.important_peers.contains(&peer_id) { log::error!( - target: "sync", + target: LOG_TARGET, "Reserved peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + peer_id, self.genesis_hash, status.genesis_hash, ); - } else if self.boot_node_ids.contains(&who) { + } else if self.boot_node_ids.contains(&peer_id) { log::error!( - target: "sync", + target: LOG_TARGET, "Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + peer_id, self.genesis_hash, status.genesis_hash, ); } else { log::debug!( - target: "sync", + target: LOG_TARGET, "Peer is on different chain (our genesis: {} theirs: {})", self.genesis_hash, status.genesis_hash ); @@ -867,7 +922,7 @@ where return Err(()) } - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&who); + let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer_id); let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; // make sure to accept no more than `--in-peers` many full nodes @@ -875,7 +930,10 @@ where status.roles.is_full() && inbound && self.num_in_peers == self.max_in_peers { - log::debug!(target: "sync", "All inbound slots have been consumed, rejecting {who}"); + log::debug!( + target: LOG_TARGET, + "All inbound slots have been consumed, rejecting {peer_id}", + ); return Err(()) } @@ -885,7 +943,7 @@ where self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot { - log::debug!(target: "sync", "Too many full nodes, rejecting {}", who); + log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer_id}"); return Err(()) } @@ -893,7 +951,7 @@ where (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { // Make sure that not all slots are occupied by light clients. - log::debug!(target: "sync", "Too many light nodes, rejecting {}", who); + log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); return Err(()) } @@ -911,7 +969,7 @@ where }; let req = if peer.info.roles.is_full() { - match self.chain_sync.new_peer(who, peer.info.best_hash, peer.info.best_number) { + match self.chain_sync.new_peer(peer_id, peer.info.best_hash, peer.info.best_number) { Ok(req) => req, Err(BadPeer(id, repu)) => { self.network_service.report_peer(id, repu); @@ -922,22 +980,22 @@ where None }; - log::debug!(target: "sync", "Connected {}", who); + log::debug!(target: LOG_TARGET, "Connected {peer_id}"); - self.peers.insert(who, peer); + self.peers.insert(peer_id, peer); if no_slot_peer { - self.default_peers_set_no_slot_connected_peers.insert(who); + self.default_peers_set_no_slot_connected_peers.insert(peer_id); } else if inbound && status.roles.is_full() { self.num_in_peers += 1; } if let Some(req) = req { - self.chain_sync.send_block_request(who, req); + self.chain_sync.send_block_request(peer_id, req); } self.event_streams - .retain(|stream| stream.unbounded_send(SyncEvent::PeerConnected(who)).is_ok()); + .retain(|stream| stream.unbounded_send(SyncEvent::PeerConnected(peer_id)).is_ok()); Ok(()) } diff --git a/substrate/client/network/sync/src/futures_stream.rs b/substrate/client/network/sync/src/futures_stream.rs new file mode 100644 index 0000000000000000000000000000000000000000..c33d582345b64ee077b4d1fed9b5c0e36103ed5d --- /dev/null +++ b/substrate/client/network/sync/src/futures_stream.rs @@ -0,0 +1,134 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! A wrapper for [`FuturesUnordered`] that wakes the task up once a new future is pushed +//! for it to be polled automatically. It's [`Stream`] never terminates. + +use futures::{stream::FuturesUnordered, Future, Stream, StreamExt}; +use std::{ + pin::Pin, + task::{Context, Poll, Waker}, +}; + +/// Wrapper around [`FuturesUnordered`] that wakes a task up automatically. +pub struct FuturesStream { + futures: FuturesUnordered, + waker: Option, +} + +/// Surprizingly, `#[derive(Default)]` doesn't work on [`FuturesStream`]. +impl Default for FuturesStream { + fn default() -> FuturesStream { + FuturesStream { futures: Default::default(), waker: None } + } +} + +impl FuturesStream { + /// Push a future for processing. + pub fn push(&mut self, future: F) { + self.futures.push(future); + + if let Some(waker) = self.waker.take() { + waker.wake(); + } + } + + /// The number of futures in the stream. + pub fn len(&self) -> usize { + self.futures.len() + } +} + +impl Stream for FuturesStream { + type Item = ::Output; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let Poll::Ready(Some(result)) = self.futures.poll_next_unpin(cx) else { + self.waker = Some(cx.waker().clone()); + + return Poll::Pending + }; + + Poll::Ready(Some(result)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::future::{BoxFuture, FutureExt}; + + /// [`Stream`] implementation for [`FuturesStream`] relies on the undocumented + /// feature that [`FuturesUnordered`] can be polled and repeatedly yield + /// `Poll::Ready(None)` before any futures are added into it. + #[tokio::test] + async fn empty_futures_unordered_can_be_polled() { + let mut unordered = FuturesUnordered::>::default(); + + futures::future::poll_fn(|cx| { + assert_eq!(unordered.poll_next_unpin(cx), Poll::Ready(None)); + assert_eq!(unordered.poll_next_unpin(cx), Poll::Ready(None)); + + Poll::Ready(()) + }) + .await; + } + + /// [`Stream`] implementation for [`FuturesStream`] relies on the undocumented + /// feature that [`FuturesUnordered`] can be polled and repeatedly yield + /// `Poll::Ready(None)` after all the futures in it have resolved. + #[tokio::test] + async fn deplenished_futures_unordered_can_be_polled() { + let mut unordered = FuturesUnordered::>::default(); + + unordered.push(futures::future::ready(()).boxed()); + assert_eq!(unordered.next().await, Some(())); + + futures::future::poll_fn(|cx| { + assert_eq!(unordered.poll_next_unpin(cx), Poll::Ready(None)); + assert_eq!(unordered.poll_next_unpin(cx), Poll::Ready(None)); + + Poll::Ready(()) + }) + .await; + } + + #[tokio::test] + async fn empty_futures_stream_yields_pending() { + let mut stream = FuturesStream::>::default(); + + futures::future::poll_fn(|cx| { + assert_eq!(stream.poll_next_unpin(cx), Poll::Pending); + Poll::Ready(()) + }) + .await; + } + + #[tokio::test] + async fn futures_stream_resolves_futures_and_yields_pending() { + let mut stream = FuturesStream::default(); + stream.push(futures::future::ready(17)); + + futures::future::poll_fn(|cx| { + assert_eq!(stream.poll_next_unpin(cx), Poll::Ready(Some(17))); + assert_eq!(stream.poll_next_unpin(cx), Poll::Pending); + Poll::Ready(()) + }) + .await; + } +} diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index 175c1c43f46f703d09404a79a1234eecf6149e30..20c0034966d5438b0b219c80e9721f6bde8df084 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -29,17 +29,16 @@ //! order to update it. use crate::{ + block_relay_protocol::{BlockDownloader, BlockResponseError}, blocks::BlockCollection, schema::v1::{StateRequest, StateResponse}, state::StateSync, - warp::{WarpProofImportResult, WarpSync}, + warp::{WarpProofImportResult, WarpSync, WarpSyncConfig}, }; -use codec::{Decode, DecodeAll, Encode}; +use codec::Encode; use extra_requests::ExtraRequests; -use futures::{ - channel::oneshot, stream::FuturesUnordered, task::Poll, Future, FutureExt, StreamExt, -}; +use futures::{channel::oneshot, task::Poll, Future, FutureExt}; use libp2p::{request_response::OutboundFailure, PeerId}; use log::{debug, error, info, trace, warn}; use prost::Message; @@ -63,19 +62,15 @@ use sc_network_common::{ BlockAnnounce, BlockAnnouncesHandshake, BlockAttributes, BlockData, BlockRequest, BlockResponse, Direction, FromBlock, }, - warp::{EncodedProof, WarpProofRequest, WarpSyncParams, WarpSyncPhase, WarpSyncProgress}, + warp::{EncodedProof, WarpProofRequest, WarpSyncPhase, WarpSyncProgress}, BadPeer, ChainSync as ChainSyncT, ImportResult, Metrics, OnBlockData, OnBlockJustification, - OnStateData, OpaqueBlockRequest, OpaqueBlockResponse, OpaqueStateRequest, - OpaqueStateResponse, PeerInfo, PeerRequest, PollBlockAnnounceValidation, SyncMode, + OnStateData, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, PeerRequest, SyncMode, SyncState, SyncStatus, }, }; use sp_arithmetic::traits::Saturating; use sp_blockchain::{Error as ClientError, HeaderBackend, HeaderMetadata}; -use sp_consensus::{ - block_validation::{BlockAnnounceValidator, Validation}, - BlockOrigin, BlockStatus, -}; +use sp_consensus::{BlockOrigin, BlockStatus}; use sp_runtime::{ traits::{ Block as BlockT, CheckedSub, Hash, HashingFor, Header as HeaderT, NumberFor, One, @@ -85,7 +80,7 @@ use sp_runtime::{ }; use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{HashMap, HashSet}, iter, ops::Range, pin::Pin, @@ -94,9 +89,12 @@ use std::{ pub use service::chain_sync::SyncingService; +mod block_announce_validator; mod extra_requests; +mod futures_stream; mod schema; +pub mod block_relay_protocol; pub mod block_request_handler; pub mod blocks; pub mod engine; @@ -107,6 +105,9 @@ pub mod state_request_handler; pub mod warp; pub mod warp_request_handler; +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + /// Maximum blocks to store in the import queue. const MAX_IMPORTING_BLOCKS: usize = 2048; @@ -117,17 +118,6 @@ const MAX_DOWNLOAD_AHEAD: u32 = 2048; /// common block of a node. const MAX_BLOCKS_TO_LOOK_BACKWARDS: u32 = MAX_DOWNLOAD_AHEAD / 2; -/// Maximum number of concurrent block announce validations. -/// -/// If the queue reaches the maximum, we drop any new block -/// announcements. -const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS: usize = 256; - -/// Maximum number of concurrent block announce validations per peer. -/// -/// See [`MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS`] for more information. -const MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER: usize = 4; - /// Pick the state to sync as the latest finalized number minus this. const STATE_SYNC_FINALITY_THRESHOLD: u32 = 8; @@ -307,27 +297,22 @@ pub struct ChainSync { fork_targets: HashMap>, /// A set of peers for which there might be potential block requests allowed_requests: AllowedRequests, - /// A type to check incoming block announcements. - block_announce_validator: Box + Send>, /// Maximum number of peers to ask the same blocks in parallel. max_parallel_downloads: u32, /// Maximum blocks per request. max_blocks_per_request: u32, /// Total number of downloaded blocks. downloaded_blocks: usize, - /// All block announcement that are currently being validated. - block_announce_validation: - FuturesUnordered> + Send>>>, - /// Stats per peer about the number of concurrent block announce validations. - block_announce_validation_per_peer_stats: HashMap, /// State sync in progress, if any. state_sync: Option>, /// Warp sync in progress, if any. warp_sync: Option>, - /// Warp sync params. + /// Warp sync configuration. /// /// Will be `None` after `self.warp_sync` is `Some(_)`. - warp_sync_params: Option>, + warp_sync_config: Option>, + /// A temporary storage for warp sync target block until warp sync is initialized. + warp_sync_target_block_header: Option, /// Enable importing existing blocks. This is used used after the state download to /// catch up to the latest state while re-importing blocks. import_existing: bool, @@ -337,8 +322,8 @@ pub struct ChainSync { network_service: service::network::NetworkServiceHandle, /// Protocol name used for block announcements block_announce_protocol_name: ProtocolName, - /// Protocol name used to send out block requests - block_request_protocol_name: ProtocolName, + /// Block downloader stub + block_downloader: Arc>, /// Protocol name used to send out state requests state_request_protocol_name: ProtocolName, /// Protocol name used to send out warp sync requests @@ -373,7 +358,7 @@ impl PeerSync { fn update_common_number(&mut self, new_common: NumberFor) { if self.common_number < new_common { trace!( - target: "sync", + target: LOG_TARGET, "Updating peer {} common number from={} => to={}.", self.peer_id, self.common_number, @@ -424,51 +409,6 @@ impl PeerSyncState { } } -/// Result of [`ChainSync::block_announce_validation`]. -#[derive(Debug, Clone, PartialEq, Eq)] -enum PreValidateBlockAnnounce { - /// The announcement failed at validation. - /// - /// The peer reputation should be decreased. - Failure { - /// Who sent the processed block announcement? - who: PeerId, - /// Should the peer be disconnected? - disconnect: bool, - }, - /// The pre-validation was sucessful and the announcement should be - /// further processed. - Process { - /// Is this the new best block of the peer? - is_new_best: bool, - /// The id of the peer that send us the announcement. - who: PeerId, - /// The announcement. - announce: BlockAnnounce, - }, - /// The announcement validation returned an error. - /// - /// An error means that *this* node failed to validate it because some internal error happened. - /// If the block announcement was invalid, [`Self::Failure`] is the correct variant to express - /// this. - Error { who: PeerId }, - /// The block announcement should be skipped. - /// - /// This should *only* be returned when there wasn't a slot registered - /// for this block announcement validation. - Skip, -} - -/// Result of [`ChainSync::has_slot_for_block_announce_validation`]. -enum HasSlotForBlockAnnounceValidation { - /// Yes, there is a slot for the block announce validation. - Yes, - /// We reached the total maximum number of validation slots. - TotalMaximumSlotsReached, - /// We reached the maximum number of validation slots for the given peer. - MaximumPeerSlotsReached, -} - impl ChainSyncT for ChainSync where B: BlockT, @@ -564,7 +504,7 @@ where // There is nothing sync can get from the node that has no blockchain data. match self.block_status(&best_hash) { Err(e) => { - debug!(target:"sync", "Error reading blockchain: {}", e); + debug!(target:LOG_TARGET, "Error reading blockchain: {e}"); Err(BadPeer(who, rep::BLOCKCHAIN_READ_ERROR)) }, Ok(BlockStatus::KnownBad) => { @@ -582,7 +522,7 @@ where // an ancestor search, which is what we do in the next match case below. if self.queue_blocks.len() > MAJOR_SYNC_BLOCKS.into() { debug!( - target:"sync", + target:LOG_TARGET, "New peer with unknown best hash {} ({}), assuming common block.", self.best_queued_hash, self.best_queued_number @@ -603,10 +543,8 @@ where // If we are at genesis, just start downloading. let (state, req) = if self.best_queued_number.is_zero() { debug!( - target:"sync", - "New peer with best hash {} ({}).", - best_hash, - best_number, + target:LOG_TARGET, + "New peer with best hash {best_hash} ({best_number}).", ); (PeerSyncState::Available, None) @@ -614,7 +552,7 @@ where let common_best = std::cmp::min(self.best_queued_number, best_number); debug!( - target:"sync", + target:LOG_TARGET, "New peer with unknown best hash {} ({}), searching for common ancestor.", best_hash, best_number @@ -645,9 +583,14 @@ where if let SyncMode::Warp = self.mode { if self.peers.len() >= MIN_PEERS_TO_START_WARP_SYNC && self.warp_sync.is_none() { - log::debug!(target: "sync", "Starting warp state sync."); - if let Some(params) = self.warp_sync_params.take() { - self.warp_sync = Some(WarpSync::new(self.client.clone(), params)); + log::debug!(target: LOG_TARGET, "Starting warp state sync."); + + if let Some(config) = self.warp_sync_config.take() { + let mut warp_sync = WarpSync::new(self.client.clone(), config); + if let Some(header) = self.warp_sync_target_block_header.take() { + warp_sync.set_target_block(header); + } + self.warp_sync = Some(warp_sync); } } } @@ -657,10 +600,8 @@ where Ok(BlockStatus::InChainWithState) | Ok(BlockStatus::InChainPruned) => { debug!( - target: "sync", - "New peer with known best hash {} ({}).", - best_hash, - best_number, + target: LOG_TARGET, + "New peer with known best hash {best_hash} ({best_number}).", ); self.peers.insert( who, @@ -692,7 +633,7 @@ where self.extra_justifications.reset(); } - // The implementation is similar to on_block_announce with unknown parent hash. + // The implementation is similar to `on_validated_block_announce` with unknown parent hash. fn set_sync_fork_request( &mut self, mut peers: Vec, @@ -709,21 +650,23 @@ where .collect(); debug!( - target: "sync", - "Explicit sync request for block {:?} with no peers specified. \ - Syncing from these peers {:?} instead.", - hash, peers, + target: LOG_TARGET, + "Explicit sync request for block {hash:?} with no peers specified. \ + Syncing from these peers {peers:?} instead.", ); } else { - debug!(target: "sync", "Explicit sync request for block {:?} with {:?}", hash, peers); + debug!( + target: LOG_TARGET, + "Explicit sync request for block {hash:?} with {peers:?}", + ); } if self.is_known(hash) { - debug!(target: "sync", "Refusing to sync known hash {:?}", hash); + debug!(target: LOG_TARGET, "Refusing to sync known hash {hash:?}"); return } - trace!(target: "sync", "Downloading requested old fork {:?}", hash); + trace!(target: LOG_TARGET, "Downloading requested old fork {hash:?}"); for peer_id in &peers { if let Some(peer) = self.peers.get_mut(peer_id) { if let PeerSyncState::AncestorSearch { .. } = peer.state { @@ -756,7 +699,7 @@ where let new_blocks: Vec> = if let Some(peer) = self.peers.get_mut(who) { let mut blocks = response.blocks; if request.as_ref().map_or(false, |r| r.direction == Direction::Descending) { - trace!(target: "sync", "Reversing incoming block list"); + trace!(target: LOG_TARGET, "Reversing incoming block list"); blocks.reverse() } self.allowed_requests.add(who); @@ -807,17 +750,22 @@ where } }) .collect(); - debug!(target: "sync", "Drained {} gap blocks from {}", blocks.len(), gap_sync.best_queued_number); + debug!( + target: LOG_TARGET, + "Drained {} gap blocks from {}", + blocks.len(), + gap_sync.best_queued_number, + ); blocks } else { - debug!(target: "sync", "Unexpected gap block response from {}", who); + debug!(target: LOG_TARGET, "Unexpected gap block response from {who}"); return Err(BadPeer(*who, rep::NO_BLOCK)) } }, PeerSyncState::DownloadingStale(_) => { peer.state = PeerSyncState::Available; if blocks.is_empty() { - debug!(target: "sync", "Empty block response from {}", who); + debug!(target: LOG_TARGET, "Empty block response from {who}"); return Err(BadPeer(*who, rep::NO_BLOCK)) } validate_blocks::(&blocks, who, Some(request))?; @@ -846,7 +794,7 @@ where let matching_hash = match (blocks.get(0), self.client.hash(*current)) { (Some(block), Ok(maybe_our_block_hash)) => { trace!( - target: "sync", + target: LOG_TARGET, "Got ancestry block #{} ({}) from peer {}", current, block.hash, @@ -856,17 +804,15 @@ where }, (None, _) => { debug!( - target: "sync", - "Invalid response when searching for ancestor from {}", - who, + target: LOG_TARGET, + "Invalid response when searching for ancestor from {who}", ); return Err(BadPeer(*who, rep::UNKNOWN_ANCESTOR)) }, (_, Err(e)) => { info!( - target: "sync", - "❌ Error answering legitimate blockchain query: {}", - e, + target: LOG_TARGET, + "❌ Error answering legitimate blockchain query: {e}", ); return Err(BadPeer(*who, rep::BLOCKCHAIN_READ_ERROR)) }, @@ -884,7 +830,10 @@ where } } if matching_hash.is_none() && current.is_zero() { - trace!(target:"sync", "Ancestry search: genesis mismatch for peer {}", who); + trace!( + target:LOG_TARGET, + "Ancestry search: genesis mismatch for peer {who}", + ); return Err(BadPeer(*who, rep::GENESIS_MISMATCH)) } if let Some((next_state, next_num)) = @@ -900,7 +849,7 @@ where // Ancestry search is complete. Check if peer is on a stale fork unknown // to us and add it to sync targets if necessary. trace!( - target: "sync", + target: LOG_TARGET, "Ancestry search complete. Ours={} ({}), Theirs={} ({}), Common={:?} ({})", self.best_queued_hash, self.best_queued_number, @@ -913,7 +862,7 @@ where peer.best_number < self.best_queued_number { trace!( - target: "sync", + target: LOG_TARGET, "Added fork target {} for {}", peer.best_hash, who, @@ -946,11 +895,11 @@ where return Err(BadPeer(*who, rep::VERIFICATION_FAIL)), } } else if blocks.is_empty() { - debug!(target: "sync", "Empty block response from {}", who); + debug!(target: LOG_TARGET, "Empty block response from {who}"); return Err(BadPeer(*who, rep::NO_BLOCK)) } else { debug!( - target: "sync", + target: LOG_TARGET, "Too many blocks ({}) in warp target block response from {}", blocks.len(), who, @@ -959,7 +908,7 @@ where } } else { debug!( - target: "sync", + target: LOG_TARGET, "Logic error: we think we are downloading warp target block from {}, but no warp sync is happening.", who, ); @@ -1011,7 +960,10 @@ where let peer = if let Some(peer) = self.peers.get_mut(&who) { peer } else { - error!(target: "sync", "💔 Called on_block_justification with a peer ID of an unknown peer"); + error!( + target: LOG_TARGET, + "💔 Called on_block_justification with a peer ID of an unknown peer", + ); return Ok(OnBlockJustification::Nothing) }; @@ -1023,7 +975,7 @@ where let justification = if let Some(block) = response.blocks.into_iter().next() { if hash != block.hash { warn!( - target: "sync", + target: LOG_TARGET, "💔 Invalid block justification provided by {}: requested: {:?} got: {:?}", who, hash, @@ -1039,10 +991,8 @@ where // we might have asked the peer for a justification on a block that we assumed it // had but didn't (regardless of whether it had a justification for it or not). trace!( - target: "sync", - "Peer {:?} provided empty response for justification request {:?}", - who, - hash, + target: LOG_TARGET, + "Peer {who:?} provided empty response for justification request {hash:?}", ); None @@ -1080,10 +1030,8 @@ where if number + STATE_SYNC_FINALITY_THRESHOLD.saturated_into() >= median { if let Ok(Some(header)) = self.client.header(*hash) { log::debug!( - target: "sync", - "Starting state sync for #{} ({})", - number, - hash, + target: LOG_TARGET, + "Starting state sync for #{number} ({hash})", ); self.state_sync = Some(StateSync::new( self.client.clone(), @@ -1100,126 +1048,94 @@ where if let Err(err) = r { warn!( - target: "sync", - "💔 Error cleaning up pending extra justification data requests: {}", - err, + target: LOG_TARGET, + "💔 Error cleaning up pending extra justification data requests: {err}", ); } } - fn push_block_announce_validation( + fn on_validated_block_announce( &mut self, - who: PeerId, - hash: B::Hash, - announce: BlockAnnounce, is_best: bool, + who: PeerId, + announce: &BlockAnnounce, ) { - let header = &announce.header; - let number = *header.number(); - debug!( - target: "sync", - "Pre-validating received block announcement {:?} with number {:?} from {}", - hash, - number, - who, - ); + let number = *announce.header.number(); + let hash = announce.header.hash(); + let parent_status = + self.block_status(announce.header.parent_hash()).unwrap_or(BlockStatus::Unknown); + let known_parent = parent_status != BlockStatus::Unknown; + let ancient_parent = parent_status == BlockStatus::InChainPruned; - if number.is_zero() { - self.block_announce_validation.push( - async move { - warn!( - target: "sync", - "💔 Ignored genesis block (#0) announcement from {}: {}", - who, - hash, - ); - PreValidateBlockAnnounce::Skip - } - .boxed(), - ); + let known = self.is_known(&hash); + let peer = if let Some(peer) = self.peers.get_mut(&who) { + peer + } else { + error!(target: LOG_TARGET, "💔 Called `on_validated_block_announce` with a bad peer ID"); + return + }; + + if let PeerSyncState::AncestorSearch { .. } = peer.state { + trace!(target: LOG_TARGET, "Peer {} is in the ancestor search state.", who); return } - // Check if there is a slot for this block announce validation. - match self.has_slot_for_block_announce_validation(&who) { - HasSlotForBlockAnnounceValidation::Yes => {}, - HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached => { - self.block_announce_validation.push( - async move { - warn!( - target: "sync", - "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots are occupied.", - number, - hash, - who, - ); - PreValidateBlockAnnounce::Skip - } - .boxed(), - ); - return - }, - HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached => { - self.block_announce_validation.push(async move { - warn!( - target: "sync", - "💔 Ignored block (#{} -- {}) announcement from {} because all validation slots for this peer are occupied.", - number, - hash, - who, - ); - PreValidateBlockAnnounce::Skip - }.boxed()); - return - }, + if is_best { + // update their best block + peer.best_number = number; + peer.best_hash = hash; } - // Let external validator check the block announcement. - let assoc_data = announce.data.as_ref().map_or(&[][..], |v| v.as_slice()); - let future = self.block_announce_validator.validate(header, assoc_data); + // If the announced block is the best they have and is not ahead of us, our common number + // is either one further ahead or it's the one they just announced, if we know about it. + if is_best { + if known && self.best_queued_number >= number { + self.update_peer_common_number(&who, number); + } else if announce.header.parent_hash() == &self.best_queued_hash || + known_parent && self.best_queued_number >= number + { + self.update_peer_common_number(&who, number.saturating_sub(One::one())); + } + } + self.allowed_requests.add(&who); - self.block_announce_validation.push( - async move { - match future.await { - Ok(Validation::Success { is_new_best }) => PreValidateBlockAnnounce::Process { - is_new_best: is_new_best || is_best, - announce, - who, - }, - Ok(Validation::Failure { disconnect }) => { - debug!( - target: "sync", - "Block announcement validation of block {:?} from {} failed", - hash, - who, - ); - PreValidateBlockAnnounce::Failure { who, disconnect } - }, - Err(e) => { - debug!( - target: "sync", - "💔 Block announcement validation of block {:?} errored: {}", - hash, - e, - ); - PreValidateBlockAnnounce::Error { who } - }, - } + // known block case + if known || self.is_already_downloading(&hash) { + trace!(target: "sync", "Known block announce from {}: {}", who, hash); + if let Some(target) = self.fork_targets.get_mut(&hash) { + target.peers.insert(who); } - .boxed(), - ); - } + return + } - fn poll_block_announce_validation( - &mut self, - cx: &mut std::task::Context, - ) -> Poll> { - match self.block_announce_validation.poll_next_unpin(cx) { - Poll::Ready(Some(res)) => { - self.peer_block_announce_validation_finished(&res); - Poll::Ready(self.finish_block_announce_validation(res)) - }, - _ => Poll::Pending, + if ancient_parent { + trace!( + target: "sync", + "Ignored ancient block announced from {}: {} {:?}", + who, + hash, + announce.header, + ); + return + } + + if self.status().state == SyncState::Idle { + trace!( + target: "sync", + "Added sync target for block announced from {}: {} {:?}", + who, + hash, + announce.summary(), + ); + self.fork_targets + .entry(hash) + .or_insert_with(|| ForkTarget { + number, + parent_hash: Some(*announce.header.parent_hash()), + peers: Default::default(), + }) + .peers + .insert(who); } } @@ -1253,82 +1169,7 @@ where } } - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String> { - let response: Box = response.0.downcast().map_err(|_error| { - "Failed to downcast opaque block response during encoding, this is an \ - implementation bug." - .to_string() - })?; - - response - .blocks - .into_iter() - .map(|block_data| { - Ok(BlockData:: { - hash: Decode::decode(&mut block_data.hash.as_ref())?, - header: if !block_data.header.is_empty() { - Some(Decode::decode(&mut block_data.header.as_ref())?) - } else { - None - }, - body: if request.fields.contains(BlockAttributes::BODY) { - Some( - block_data - .body - .iter() - .map(|body| Decode::decode(&mut body.as_ref())) - .collect::, _>>()?, - ) - } else { - None - }, - indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { - Some(block_data.indexed_body) - } else { - None - }, - receipt: if !block_data.receipt.is_empty() { - Some(block_data.receipt) - } else { - None - }, - message_queue: if !block_data.message_queue.is_empty() { - Some(block_data.message_queue) - } else { - None - }, - justification: if !block_data.justification.is_empty() { - Some(block_data.justification) - } else if block_data.is_empty_justification { - Some(Vec::new()) - } else { - None - }, - justifications: if !block_data.justifications.is_empty() { - Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) - } else { - None - }, - }) - }) - .collect::>() - .map_err(|error: codec::Error| error.to_string()) - } - - fn poll( - &mut self, - cx: &mut std::task::Context, - ) -> Poll> { - // Should be called before `process_outbound_requests` to ensure - // that a potential target block is directly leading to requests. - if let Some(warp_sync) = &mut self.warp_sync { - let _ = warp_sync.poll(cx); - } - + fn poll(&mut self, cx: &mut std::task::Context) -> Poll<()> { self.process_outbound_requests(); while let Poll::Ready(result) = self.poll_pending_responses(cx) { @@ -1339,39 +1180,22 @@ where } } - if let Poll::Ready(announce) = self.poll_block_announce_validation(cx) { - return Poll::Ready(announce) - } - Poll::Pending } fn send_block_request(&mut self, who: PeerId, request: BlockRequest) { - let (tx, rx) = oneshot::channel(); - let opaque_req = self.create_opaque_block_request(&request); - if self.peers.contains_key(&who) { - self.pending_responses - .insert(who, Box::pin(async move { (who, PeerRequest::Block(request), rx.await) })); - } - - match self.encode_block_request(&opaque_req) { - Ok(data) => { - self.network_service.start_request( - who, - self.block_request_protocol_name.clone(), - data, - tx, - IfDisconnected::ImmediateError, - ); - }, - Err(err) => { - log::warn!( - target: "sync", - "Failed to encode block request {:?}: {:?}", - opaque_req, err - ); - }, + let downloader = self.block_downloader.clone(); + self.pending_responses.insert( + who, + Box::pin(async move { + ( + who, + PeerRequest::Block(request.clone()), + downloader.download_blocks(who, request).await, + ) + }), + ); } } } @@ -1395,14 +1219,13 @@ where protocol_id: ProtocolId, fork_id: &Option, roles: Roles, - block_announce_validator: Box + Send>, max_parallel_downloads: u32, max_blocks_per_request: u32, - warp_sync_params: Option>, + warp_sync_config: Option>, metrics_registry: Option<&Registry>, network_service: service::network::NetworkServiceHandle, import_queue: Box>, - block_request_protocol_name: ProtocolName, + block_downloader: Arc>, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, ) -> Result<(Self, NonDefaultSetConfig), ClientError> { @@ -1430,20 +1253,18 @@ where queue_blocks: Default::default(), fork_targets: Default::default(), allowed_requests: Default::default(), - block_announce_validator, max_parallel_downloads, max_blocks_per_request, downloaded_blocks: 0, - block_announce_validation: Default::default(), - block_announce_validation_per_peer_stats: Default::default(), state_sync: None, warp_sync: None, import_existing: false, gap_sync: None, network_service, - block_request_protocol_name, + block_downloader, state_request_protocol_name, - warp_sync_params, + warp_sync_config, + warp_sync_target_block_header: None, warp_sync_protocol_name, block_announce_protocol_name: block_announce_config .notifications_protocol @@ -1455,7 +1276,10 @@ where match SyncingMetrics::register(r) { Ok(metrics) => Some(metrics), Err(err) => { - error!(target: "sync", "Failed to register metrics for ChainSync: {err:?}"); + error!( + target: LOG_TARGET, + "Failed to register metrics for ChainSync: {err:?}", + ); None }, } @@ -1512,7 +1336,7 @@ where new_blocks.retain(|b| !self.queue_blocks.contains(&b.hash)); if new_blocks.len() != orig_len { debug!( - target: "sync", + target: LOG_TARGET, "Ignoring {} blocks that are already queued", orig_len - new_blocks.len(), ); @@ -1529,7 +1353,7 @@ where .and_then(|b| b.header.as_ref().map(|h| (&b.hash, *h.number()))) { trace!( - target:"sync", + target:LOG_TARGET, "Accepted {} blocks ({:?}) with origin {:?}", new_blocks.len(), h, @@ -1553,7 +1377,7 @@ where /// through all peers to update our view of their state as well. fn on_block_queued(&mut self, hash: &B::Hash, number: NumberFor) { if self.fork_targets.remove(hash).is_some() { - trace!(target: "sync", "Completed fork sync {:?}", hash); + trace!(target: LOG_TARGET, "Completed fork sync {hash:?}"); } if let Some(gap_sync) = &mut self.gap_sync { if number > gap_sync.best_queued_number && number <= gap_sync.target { @@ -1572,7 +1396,7 @@ where let new_common_number = if peer.best_number >= number { number } else { peer.best_number }; trace!( - target: "sync", + target: LOG_TARGET, "Updating peer {} info, ours={}, common={}->{}, their best={}", n, number, @@ -1586,196 +1410,21 @@ where self.allowed_requests.set_all(); } - /// Checks if there is a slot for a block announce validation. - /// - /// The total number and the number per peer of concurrent block announce validations - /// is capped. - /// - /// Returns [`HasSlotForBlockAnnounceValidation`] to inform about the result. - /// - /// # Note - /// - /// It is *required* to call [`Self::peer_block_announce_validation_finished`] when the - /// validation is finished to clear the slot. - fn has_slot_for_block_announce_validation( - &mut self, - peer: &PeerId, - ) -> HasSlotForBlockAnnounceValidation { - if self.block_announce_validation.len() >= MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS { - return HasSlotForBlockAnnounceValidation::TotalMaximumSlotsReached - } - - match self.block_announce_validation_per_peer_stats.entry(*peer) { - Entry::Vacant(entry) => { - entry.insert(1); - HasSlotForBlockAnnounceValidation::Yes - }, - Entry::Occupied(mut entry) => { - if *entry.get() < MAX_CONCURRENT_BLOCK_ANNOUNCE_VALIDATIONS_PER_PEER { - *entry.get_mut() += 1; - HasSlotForBlockAnnounceValidation::Yes - } else { - HasSlotForBlockAnnounceValidation::MaximumPeerSlotsReached - } - }, - } - } - - /// Should be called when a block announce validation is finished, to update the slots - /// of the peer that send the block announce. - fn peer_block_announce_validation_finished( - &mut self, - res: &PreValidateBlockAnnounce, - ) { - let peer = match res { - PreValidateBlockAnnounce::Failure { who, .. } | - PreValidateBlockAnnounce::Process { who, .. } | - PreValidateBlockAnnounce::Error { who } => who, - PreValidateBlockAnnounce::Skip => return, - }; - - match self.block_announce_validation_per_peer_stats.entry(*peer) { - Entry::Vacant(_) => { - error!( - target: "sync", - "💔 Block announcement validation from peer {} finished for that no slot was allocated!", - peer, - ); - }, - Entry::Occupied(mut entry) => { - *entry.get_mut() = entry.get().saturating_sub(1); - if *entry.get() == 0 { - entry.remove(); - } - }, - } - } - - /// This will finish processing of the block announcement. - fn finish_block_announce_validation( - &mut self, - pre_validation_result: PreValidateBlockAnnounce, - ) -> PollBlockAnnounceValidation { - let (announce, is_best, who) = match pre_validation_result { - PreValidateBlockAnnounce::Failure { who, disconnect } => { - debug!( - target: "sync", - "Failed announce validation: {:?}, disconnect: {}", - who, - disconnect, - ); - return PollBlockAnnounceValidation::Failure { who, disconnect } - }, - PreValidateBlockAnnounce::Process { announce, is_new_best, who } => - (announce, is_new_best, who), - PreValidateBlockAnnounce::Error { .. } | PreValidateBlockAnnounce::Skip => { - debug!( - target: "sync", - "Ignored announce validation", - ); - return PollBlockAnnounceValidation::Skip - }, - }; - - trace!( - target: "sync", - "Finished block announce validation: from {:?}: {:?}. local_best={}", - who, - announce.summary(), - is_best, - ); - - let number = *announce.header.number(); - let hash = announce.header.hash(); - let parent_status = - self.block_status(announce.header.parent_hash()).unwrap_or(BlockStatus::Unknown); - let known_parent = parent_status != BlockStatus::Unknown; - let ancient_parent = parent_status == BlockStatus::InChainPruned; - - let known = self.is_known(&hash); - let peer = if let Some(peer) = self.peers.get_mut(&who) { - peer - } else { - error!(target: "sync", "💔 Called on_block_announce with a bad peer ID"); - return PollBlockAnnounceValidation::Nothing { is_best, who, announce } - }; - - if let PeerSyncState::AncestorSearch { .. } = peer.state { - trace!(target: "sync", "Peer state is ancestor search."); - return PollBlockAnnounceValidation::Nothing { is_best, who, announce } - } - - if is_best { - // update their best block - peer.best_number = number; - peer.best_hash = hash; - } - - // If the announced block is the best they have and is not ahead of us, our common number - // is either one further ahead or it's the one they just announced, if we know about it. - if is_best { - if known && self.best_queued_number >= number { - self.update_peer_common_number(&who, number); - } else if announce.header.parent_hash() == &self.best_queued_hash || - known_parent && self.best_queued_number >= number - { - self.update_peer_common_number(&who, number - One::one()); - } - } - self.allowed_requests.add(&who); - - // known block case - if known || self.is_already_downloading(&hash) { - trace!(target: "sync", "Known block announce from {}: {}", who, hash); - if let Some(target) = self.fork_targets.get_mut(&hash) { - target.peers.insert(who); - } - return PollBlockAnnounceValidation::Nothing { is_best, who, announce } - } - - if ancient_parent { - trace!( - target: "sync", - "Ignored ancient block announced from {}: {} {:?}", - who, - hash, - announce.header, - ); - return PollBlockAnnounceValidation::Nothing { is_best, who, announce } - } - - if self.status().state == SyncState::Idle { - trace!( - target: "sync", - "Added sync target for block announced from {}: {} {:?}", - who, - hash, - announce.summary(), - ); - self.fork_targets - .entry(hash) - .or_insert_with(|| ForkTarget { - number, - parent_hash: Some(*announce.header.parent_hash()), - peers: Default::default(), - }) - .peers - .insert(who); - } - - PollBlockAnnounceValidation::Nothing { is_best, who, announce } - } - /// Restart the sync process. This will reset all pending block requests and return an iterator /// of new block requests to make to peers. Peers that were downloading finality data (i.e. /// their state was `DownloadingJustification`) are unaffected and will stay in the same state. fn restart(&mut self) -> impl Iterator), BadPeer>> + '_ { self.blocks.clear(); if let Err(e) = self.reset_sync_start_point() { - warn!(target: "sync", "💔 Unable to restart sync: {}", e); + warn!(target: LOG_TARGET, "💔 Unable to restart sync: {e}"); } self.allowed_requests.set_all(); - debug!(target:"sync", "Restarted with {} ({})", self.best_queued_number, self.best_queued_hash); + debug!( + target: LOG_TARGET, + "Restarted with {} ({})", + self.best_queued_number, + self.best_queued_hash, + ); let old_peers = std::mem::take(&mut self.peers); old_peers.into_iter().filter_map(move |(id, mut p)| { @@ -1806,14 +1455,14 @@ where let info = self.client.info(); if matches!(self.mode, SyncMode::LightState { .. }) && info.finalized_state.is_some() { warn!( - target: "sync", + target: LOG_TARGET, "Can't use fast sync mode with a partially synced database. Reverting to full sync mode." ); self.mode = SyncMode::Full; } if matches!(self.mode, SyncMode::Warp) && info.finalized_state.is_some() { warn!( - target: "sync", + target: LOG_TARGET, "Can't use warp sync mode with a partially synced database. Reverting to full sync mode." ); self.mode = SyncMode::Full; @@ -1828,25 +1477,30 @@ where self.import_existing = true; // Latest state is missing, start with the last finalized state or genesis instead. if let Some((hash, number)) = info.finalized_state { - debug!(target: "sync", "Starting from finalized state #{}", number); + debug!(target: LOG_TARGET, "Starting from finalized state #{number}"); self.best_queued_hash = hash; self.best_queued_number = number; } else { - debug!(target: "sync", "Restarting from genesis"); + debug!(target: LOG_TARGET, "Restarting from genesis"); self.best_queued_hash = Default::default(); self.best_queued_number = Zero::zero(); } } if let Some((start, end)) = info.block_gap { - debug!(target: "sync", "Starting gap sync #{} - #{}", start, end); + debug!(target: LOG_TARGET, "Starting gap sync #{start} - #{end}"); self.gap_sync = Some(GapSync { best_queued_number: start - One::one(), target: end, blocks: BlockCollection::new(), }); } - trace!(target: "sync", "Restarted sync at #{} ({:?})", self.best_queued_number, self.best_queued_hash); + trace!( + target: LOG_TARGET, + "Restarted sync at #{} ({:?})", + self.best_queued_number, + self.best_queued_hash, + ); Ok(()) } @@ -1896,6 +1550,15 @@ where .collect() } + /// Set warp sync target block externally in case we skip warp proof downloading. + pub fn set_warp_sync_target_block(&mut self, header: B::Header) { + if let Some(ref mut warp_sync) = self.warp_sync { + warp_sync.set_target_block(header); + } else { + self.warp_sync_target_block_header = Some(header); + } + } + /// Generate block request for downloading of the target block body during warp sync. fn warp_target_block_request(&mut self) -> Option<(PeerId, BlockRequest)> { let sync = &self.warp_sync.as_ref()?; @@ -1914,7 +1577,7 @@ where // Find a random peer that has a block with the target number. for (id, peer) in self.peers.iter_mut() { if peer.state.is_available() && peer.best_number >= target_number { - trace!(target: "sync", "New warp target block request for {}", id); + trace!(target: LOG_TARGET, "New warp target block request for {id}"); peer.state = PeerSyncState::DownloadingWarpTargetBlock; self.allowed_requests.clear(); return Some((*id, request)) @@ -1971,13 +1634,6 @@ where } } - fn decode_block_response(response: &[u8]) -> Result { - let response = schema::v1::BlockResponse::decode(response) - .map_err(|error| format!("Failed to decode block response: {error}"))?; - - Ok(OpaqueBlockResponse(Box::new(response))) - } - fn decode_state_response(response: &[u8]) -> Result { let response = StateResponse::decode(response) .map_err(|error| format!("Failed to decode state response: {error}"))?; @@ -2005,9 +1661,8 @@ where }, Err(err) => { log::warn!( - target: "sync", - "Failed to encode state request {:?}: {:?}", - request, err + target: LOG_TARGET, + "Failed to encode state request {request:?}: {err:?}", ); }, } @@ -2031,9 +1686,8 @@ where ), None => { log::warn!( - target: "sync", - "Trying to send warp sync request when no protocol is configured {:?}", - request, + target: LOG_TARGET, + "Trying to send warp sync request when no protocol is configured {request:?}", ); }, } @@ -2043,17 +1697,8 @@ where &mut self, peer_id: PeerId, request: BlockRequest, - response: OpaqueBlockResponse, + blocks: Vec>, ) -> Option> { - let blocks = match self.block_response_into_blocks(&request, response) { - Ok(blocks) => blocks, - Err(err) => { - debug!(target: "sync", "Failed to decode block response from {}: {}", peer_id, err); - self.network_service.report_peer(peer_id, rep::BAD_MESSAGE); - return None - }, - }; - let block_response = BlockResponse:: { id: request.id, blocks }; let blocks_range = || match ( @@ -2068,7 +1713,7 @@ where _ => Default::default(), }; trace!( - target: "sync", + target: LOG_TARGET, "BlockResponse {} from {} with {} blocks {}", block_response.id, peer_id, @@ -2173,11 +1818,15 @@ where match response { Ok(Ok(resp)) => match request { PeerRequest::Block(req) => { - let response = match Self::decode_block_response(&resp[..]) { - Ok(proto) => proto, - Err(e) => { + match self.block_downloader.block_response_into_blocks(&req, resp) { + Ok(blocks) => { + if let Some(import) = self.on_block_response(id, req, blocks) { + return Poll::Ready(import) + } + }, + Err(BlockResponseError::DecodeFailed(e)) => { debug!( - target: "sync", + target: LOG_TARGET, "Failed to decode block response from peer {:?}: {:?}.", id, e @@ -2187,10 +1836,16 @@ where .disconnect_peer(id, self.block_announce_protocol_name.clone()); continue }, - }; - - if let Some(import) = self.on_block_response(id, req, response) { - return Poll::Ready(import) + Err(BlockResponseError::ExtractionFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to extract blocks from peer response {:?}: {:?}.", + id, + e + ); + self.network_service.report_peer(id, rep::BAD_MESSAGE); + continue + }, } }, PeerRequest::State => { @@ -2198,10 +1853,8 @@ where Ok(proto) => proto, Err(e) => { debug!( - target: "sync", - "Failed to decode state response from peer {:?}: {:?}.", - id, - e + target: LOG_TARGET, + "Failed to decode state response from peer {id:?}: {e:?}.", ); self.network_service.report_peer(id, rep::BAD_MESSAGE); self.network_service @@ -2219,7 +1872,7 @@ where }, }, Ok(Err(e)) => { - debug!(target: "sync", "Request to peer {:?} failed: {:?}.", id, e); + debug!(target: LOG_TARGET, "Request to peer {id:?} failed: {e:?}."); match e { RequestFailure::Network(OutboundFailure::Timeout) => { @@ -2260,9 +1913,8 @@ where }, Err(oneshot::Canceled) => { trace!( - target: "sync", - "Request to peer {:?} failed due to oneshot being canceled.", - id, + target: LOG_TARGET, + "Request to peer {id:?} failed due to oneshot being canceled.", ); self.network_service .disconnect_peer(id, self.block_announce_protocol_name.clone()); @@ -2273,31 +1925,6 @@ where Poll::Pending } - /// Create implementation-specific block request. - fn create_opaque_block_request(&self, request: &BlockRequest) -> OpaqueBlockRequest { - OpaqueBlockRequest(Box::new(schema::v1::BlockRequest { - fields: request.fields.to_be_u32(), - from_block: match request.from { - FromBlock::Hash(h) => Some(schema::v1::block_request::FromBlock::Hash(h.encode())), - FromBlock::Number(n) => - Some(schema::v1::block_request::FromBlock::Number(n.encode())), - }, - direction: request.direction as i32, - max_blocks: request.max.unwrap_or(0), - support_multiple_justifications: true, - })) - } - - fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result, String> { - let request: &schema::v1::BlockRequest = request.0.downcast_ref().ok_or_else(|| { - "Failed to downcast opaque block response during encoding, this is an \ - implementation bug." - .to_string() - })?; - - Ok(request.encode_to_vec()) - } - fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result, String> { let request: &StateRequest = request.0.downcast_ref().ok_or_else(|| { "Failed to downcast opaque state response during encoding, this is an \ @@ -2347,7 +1974,7 @@ where } if self.queue_blocks.len() > MAX_IMPORTING_BLOCKS { - trace!(target: "sync", "Too many blocks in the queue."); + trace!(target: LOG_TARGET, "Too many blocks in the queue."); return Vec::new() } let is_major_syncing = self.status().state.is_major_syncing(); @@ -2382,7 +2009,7 @@ where queue.len() <= MAJOR_SYNC_BLOCKS.into() { trace!( - target: "sync", + target: LOG_TARGET, "Peer {:?} common block {} too far behind of our best {}. Starting ancestry search.", id, peer.common_number, @@ -2407,7 +2034,7 @@ where ) { peer.state = PeerSyncState::DownloadingNew(range.start); trace!( - target: "sync", + target: LOG_TARGET, "New block request for {}, (best:{}, common:{}) {:?}", id, peer.best_number, @@ -2430,7 +2057,7 @@ where }, max_blocks_per_request, ) { - trace!(target: "sync", "Downloading fork {:?} from {}", hash, id); + trace!(target: LOG_TARGET, "Downloading fork {hash:?} from {id}"); peer.state = PeerSyncState::DownloadingStale(hash); Some((id, req)) } else if let Some((range, req)) = gap_sync.as_mut().and_then(|sync| { @@ -2446,7 +2073,7 @@ where }) { peer.state = PeerSyncState::DownloadingGap(range.start); trace!( - target: "sync", + target: LOG_TARGET, "New gap block request for {}, (best:{}, common:{}) {:?}", id, peer.best_number, @@ -2481,7 +2108,7 @@ where if peer.state.is_available() && peer.common_number >= sync.target_block_num() { peer.state = PeerSyncState::DownloadingState; let request = sync.next_request(); - trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); + trace!(target: LOG_TARGET, "New StateRequest for {}: {:?}", id, request); self.allowed_requests.clear(); return Some((*id, OpaqueStateRequest(Box::new(request)))) } @@ -2496,7 +2123,7 @@ where { for (id, peer) in self.peers.iter_mut() { if peer.state.is_available() && peer.best_number >= target { - trace!(target: "sync", "New StateRequest for {}: {:?}", id, request); + trace!(target: LOG_TARGET, "New StateRequest for {id}: {request:?}"); peer.state = PeerSyncState::DownloadingState; self.allowed_requests.clear(); return Some((*id, OpaqueStateRequest(Box::new(request)))) @@ -2526,7 +2153,7 @@ where // Find a random peer that is synced as much as peer majority. for (id, peer) in self.peers.iter_mut() { if peer.state.is_available() && peer.best_number >= median { - trace!(target: "sync", "New WarpProofRequest for {}", id); + trace!(target: LOG_TARGET, "New WarpProofRequest for {id}"); peer.state = PeerSyncState::DownloadingWarpProof; self.allowed_requests.clear(); return Some((*id, request)) @@ -2545,7 +2172,7 @@ where ) -> Result, BadPeer> { let response: Box = response.0.downcast().map_err(|_error| { error!( - target: "sync", + target: LOG_TARGET, "Failed to downcast opaque state response, this is an implementation bug." ); @@ -2560,7 +2187,7 @@ where } let import_result = if let Some(sync) = &mut self.state_sync { debug!( - target: "sync", + target: LOG_TARGET, "Importing state data from {} with {} keys, {} proof nodes.", who, response.entries.len(), @@ -2569,7 +2196,7 @@ where sync.import(*response) } else if let Some(sync) = &mut self.warp_sync { debug!( - target: "sync", + target: LOG_TARGET, "Importing state data from {} with {} keys, {} proof nodes.", who, response.entries.len(), @@ -2577,7 +2204,7 @@ where ); sync.import_state(*response) } else { - debug!(target: "sync", "Ignored obsolete state response from {}", who); + debug!(target: LOG_TARGET, "Ignored obsolete state response from {who}"); return Err(BadPeer(*who, rep::NOT_REQUESTED)) }; @@ -2596,12 +2223,12 @@ where skip_execution: self.skip_execution(), state: Some(state), }; - debug!(target: "sync", "State download is complete. Import is queued"); + debug!(target: LOG_TARGET, "State download is complete. Import is queued"); Ok(OnStateData::Import(origin, block)) }, state::ImportResult::Continue => Ok(OnStateData::Continue), state::ImportResult::BadResponse => { - debug!(target: "sync", "Bad state data received from {}", who); + debug!(target: LOG_TARGET, "Bad state data received from {who}"); Err(BadPeer(*who, rep::BAD_BLOCK)) }, } @@ -2616,21 +2243,21 @@ where } let import_result = if let Some(sync) = &mut self.warp_sync { debug!( - target: "sync", + target: LOG_TARGET, "Importing warp proof data from {}, {} bytes.", who, response.0.len(), ); sync.import_warp_proof(response) } else { - debug!(target: "sync", "Ignored obsolete warp sync response from {}", who); + debug!(target: LOG_TARGET, "Ignored obsolete warp sync response from {who}"); return Err(BadPeer(*who, rep::NOT_REQUESTED)) }; match import_result { WarpProofImportResult::Success => Ok(()), WarpProofImportResult::BadResponse => { - debug!(target: "sync", "Bad proof data received from {}", who); + debug!(target: LOG_TARGET, "Bad proof data received from {who}"); Err(BadPeer(*who, rep::BAD_BLOCK)) }, } @@ -2668,7 +2295,7 @@ where count: usize, results: Vec<(Result>, BlockImportError>, B::Hash)>, ) -> Box), BadPeer>>> { - trace!(target: "sync", "Imported {} of {}", imported, count); + trace!(target: LOG_TARGET, "Imported {imported} of {count}"); let mut output = Vec::new(); @@ -2695,7 +2322,7 @@ where Ok(BlockImportStatus::ImportedUnknown(number, aux, who)) => { if aux.clear_justification_requests { trace!( - target: "sync", + target: LOG_TARGET, "Block imported clears all pending justification requests {number}: {hash:?}", ); self.clear_justification_requests(); @@ -2703,7 +2330,7 @@ where if aux.needs_justification { trace!( - target: "sync", + target: LOG_TARGET, "Block imported but requires justification {number}: {hash:?}", ); self.request_justification(&hash, number); @@ -2723,7 +2350,7 @@ where self.state_sync.as_ref().map_or(false, |s| s.target() == hash); if state_sync_complete { info!( - target: "sync", + target: LOG_TARGET, "State sync is complete ({} MiB), restarting block sync.", self.state_sync.as_ref().map_or(0, |s| s.progress().size / (1024 * 1024)), ); @@ -2737,7 +2364,7 @@ where .map_or(false, |s| s.target_block_hash() == Some(hash)); if warp_sync_complete { info!( - target: "sync", + target: LOG_TARGET, "Warp sync is complete ({} MiB), restarting block sync.", self.warp_sync.as_ref().map_or(0, |s| s.progress().total_bytes / (1024 * 1024)), ); @@ -2749,7 +2376,7 @@ where self.gap_sync.as_ref().map_or(false, |s| s.target == number); if gap_sync_complete { info!( - target: "sync", + target: LOG_TARGET, "Block history download is complete." ); self.gap_sync = None; @@ -2758,7 +2385,7 @@ where Err(BlockImportError::IncompleteHeader(who)) => if let Some(peer) = who { warn!( - target: "sync", + target: LOG_TARGET, "💔 Peer sent block with incomplete header to import", ); output.push(Err(BadPeer(peer, rep::INCOMPLETE_HEADER))); @@ -2769,7 +2396,7 @@ where who.map_or_else(|| "".into(), |peer| format!(" received from ({peer})")); warn!( - target: "sync", + target: LOG_TARGET, "💔 Verification failed for block {hash:?}{extra_message}: {e:?}", ); @@ -2782,7 +2409,7 @@ where Err(BlockImportError::BadBlock(who)) => if let Some(peer) = who { warn!( - target: "sync", + target: LOG_TARGET, "💔 Block {hash:?} received from peer {peer} has been blacklisted", ); output.push(Err(BadPeer(peer, rep::BAD_BLOCK))); @@ -2791,10 +2418,10 @@ where // This may happen if the chain we were requesting upon has been discarded // in the meantime because other chain has been finalized. // Don't mark it as bad as it still may be synced if explicitly requested. - trace!(target: "sync", "Obsolete block {hash:?}"); + trace!(target: LOG_TARGET, "Obsolete block {hash:?}"); }, e @ Err(BlockImportError::UnknownParent) | e @ Err(BlockImportError::Other(_)) => { - warn!(target: "sync", "💔 Error importing block {hash:?}: {}", e.unwrap_err()); + warn!(target: LOG_TARGET, "💔 Error importing block {hash:?}: {}", e.unwrap_err()); self.state_sync = None; self.warp_sync = None; output.extend(self.restart()); @@ -2914,7 +2541,7 @@ fn peer_block_request( return None } else if peer.common_number < finalized { trace!( - target: "sync", + target: LOG_TARGET, "Requesting pre-finalized chain from {:?}, common={}, finalized={}, peer best={}, our best={}", id, peer.common_number, finalized, peer.best_number, best_num, ); @@ -2993,11 +2620,21 @@ fn fork_sync_request( ) -> Option<(B::Hash, BlockRequest)> { targets.retain(|hash, r| { if r.number <= finalized { - trace!(target: "sync", "Removed expired fork sync request {:?} (#{})", hash, r.number); + trace!( + target: LOG_TARGET, + "Removed expired fork sync request {:?} (#{})", + hash, + r.number, + ); return false } if check_block(hash) != BlockStatus::Unknown { - trace!(target: "sync", "Removed obsolete fork sync request {:?} (#{})", hash, r.number); + trace!( + target: LOG_TARGET, + "Removed obsolete fork sync request {:?} (#{})", + hash, + r.number, + ); return false } true @@ -3018,7 +2655,10 @@ fn fork_sync_request( // request only single block 1 }; - trace!(target: "sync", "Downloading requested fork {:?} from {}, {} blocks", hash, id, count); + trace!( + target: LOG_TARGET, + "Downloading requested fork {hash:?} from {id}, {count} blocks", + ); return Some(( *hash, BlockRequest:: { @@ -3030,7 +2670,7 @@ fn fork_sync_request( }, )) } else { - trace!(target: "sync", "Fork too far in the future: {:?} (#{})", hash, r.number); + trace!(target: LOG_TARGET, "Fork too far in the future: {:?} (#{})", hash, r.number); } } None @@ -3067,7 +2707,7 @@ fn validate_blocks( if let Some(request) = request { if Some(blocks.len() as _) > request.max { debug!( - target: "sync", + target: LOG_TARGET, "Received more blocks than requested from {}. Expected in maximum {:?}, got {}.", who, request.max, @@ -3088,7 +2728,7 @@ fn validate_blocks( if !expected_block { debug!( - target: "sync", + target: LOG_TARGET, "Received block that was not requested. Requested {:?}, got {:?}.", request.from, block_header, @@ -3101,9 +2741,8 @@ fn validate_blocks( blocks.iter().any(|b| b.header.is_none()) { trace!( - target: "sync", - "Missing requested header for a block in response from {}.", - who, + target: LOG_TARGET, + "Missing requested header for a block in response from {who}.", ); return Err(BadPeer(*who, rep::BAD_RESPONSE)) @@ -3112,9 +2751,8 @@ fn validate_blocks( if request.fields.contains(BlockAttributes::BODY) && blocks.iter().any(|b| b.body.is_none()) { trace!( - target: "sync", - "Missing requested body for a block in response from {}.", - who, + target: LOG_TARGET, + "Missing requested body for a block in response from {who}.", ); return Err(BadPeer(*who, rep::BAD_RESPONSE)) @@ -3126,7 +2764,7 @@ fn validate_blocks( let hash = header.hash(); if hash != b.hash { debug!( - target:"sync", + target:LOG_TARGET, "Bad header received from {}. Expected hash {:?}, got {:?}", who, b.hash, @@ -3143,7 +2781,7 @@ fn validate_blocks( ); if expected != got { debug!( - target:"sync", + target:LOG_TARGET, "Bad extrinsic root for a block {} received from {}. Expected {:?}, got {:?}", b.hash, who, @@ -3161,15 +2799,14 @@ fn validate_blocks( #[cfg(test)] mod test { use super::*; - use crate::service::network::NetworkServiceProvider; - use futures::{executor::block_on, future::poll_fn}; + use crate::{mock::MockBlockDownloader, service::network::NetworkServiceProvider}; + use futures::executor::block_on; use sc_block_builder::BlockBuilderProvider; use sc_network_common::{ role::Role, - sync::message::{BlockData, BlockState, FromBlock}, + sync::message::{BlockAnnounce, BlockData, BlockState, FromBlock}, }; use sp_blockchain::HeaderBackend; - use sp_consensus::block_validation::DefaultBlockAnnounceValidator; use substrate_test_runtime_client::{ runtime::{Block, Hash, Header}, BlockBuilderExt, ClientBlockImportExt, ClientExt, DefaultTestClientBuilderExt, TestClient, @@ -3183,7 +2820,6 @@ mod test { // internally we should process the response as the justification not being available. let client = Arc::new(TestClientBuilder::new().build()); - let block_announce_validator = Box::new(DefaultBlockAnnounceValidator); let peer_id = PeerId::random(); let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); @@ -3195,14 +2831,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - block_announce_validator, 1, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3262,14 +2897,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 1, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3344,23 +2978,16 @@ mod test { /// Send a block annoucnement for the given `header`. fn send_block_announce( header: Header, - peer_id: &PeerId, + peer_id: PeerId, sync: &mut ChainSync, ) { - let block_annnounce = BlockAnnounce { + let announce = BlockAnnounce { header: header.clone(), state: Some(BlockState::Best), data: Some(Vec::new()), }; - sync.push_block_announce_validation(*peer_id, header.hash(), block_annnounce, true); - - // Poll until we have procssed the block announcement - block_on(poll_fn(|cx| loop { - if sync.poll_block_announce_validation(cx).is_pending() { - break Poll::Ready(()) - } - })) + sync.on_validated_block_announce(true, peer_id, &announce); } /// Create a block response from the given `blocks`. @@ -3392,7 +3019,7 @@ mod test { ) -> BlockRequest { let requests = sync.block_requests(); - log::trace!(target: "sync", "Requests: {:?}", requests); + log::trace!(target: LOG_TARGET, "Requests: {requests:?}"); assert_eq!(1, requests.len()); assert_eq!(*peer, requests[0].0); @@ -3444,14 +3071,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 5, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3491,7 +3117,7 @@ mod test { assert!(sync.block_requests().is_empty()); // Let peer2 announce a fork of block 3 - send_block_announce(block3_fork.header().clone(), &peer_id2, &mut sync); + send_block_announce(block3_fork.header().clone(), peer_id2, &mut sync); // Import and tell sync that we now have the fork. block_on(client.import(BlockOrigin::Own, block3_fork.clone())).unwrap(); @@ -3500,13 +3126,13 @@ mod test { let block4 = build_block_at(block3_fork.hash(), false); // Let peer2 announce block 4 and check that sync wants to get the block. - send_block_announce(block4.header().clone(), &peer_id2, &mut sync); + send_block_announce(block4.header().clone(), peer_id2, &mut sync); let request = get_block_request(&mut sync, FromBlock::Hash(block4.hash()), 2, &peer_id2); // Peer1 announces the same block, but as the common block is still `1`, sync will request // block 2 again. - send_block_announce(block4.header().clone(), &peer_id1, &mut sync); + send_block_announce(block4.header().clone(), peer_id1, &mut sync); let request2 = get_block_request(&mut sync, FromBlock::Number(2), 1, &peer_id1); @@ -3571,14 +3197,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 5, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3647,7 +3272,7 @@ mod test { sync.queue_blocks.clear(); // Let peer2 announce that it finished syncing - send_block_announce(best_block.header().clone(), &peer_id2, &mut sync); + send_block_announce(best_block.header().clone(), peer_id2, &mut sync); let (peer1_req, peer2_req) = sync.block_requests().into_iter().fold((None, None), |res, req| { @@ -3729,14 +3354,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 5, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3754,7 +3378,7 @@ mod test { sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) .unwrap(); - send_block_announce(fork_blocks.last().unwrap().header().clone(), &peer_id1, &mut sync); + send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); let mut request = get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); @@ -3772,7 +3396,7 @@ mod test { break }; - log::trace!(target: "sync", "Request: {:?}", request); + log::trace!(target: LOG_TARGET, "Request: {request:?}"); } // Now request and import the fork. @@ -3872,14 +3496,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 5, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3897,7 +3520,7 @@ mod test { sync.new_peer(peer_id1, common_block.hash(), *common_block.header().number()) .unwrap(); - send_block_announce(fork_blocks.last().unwrap().header().clone(), &peer_id1, &mut sync); + send_block_announce(fork_blocks.last().unwrap().header().clone(), peer_id1, &mut sync); let mut request = get_block_request(&mut sync, FromBlock::Number(info.best_number), 1, &peer_id1); @@ -3915,7 +3538,7 @@ mod test { break }; - log::trace!(target: "sync", "Request: {:?}", request); + log::trace!(target: LOG_TARGET, "Request: {request:?}"); } // Now request and import the fork. @@ -4017,14 +3640,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 1, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -4039,7 +3661,7 @@ mod test { // Create a "new" header and announce it let mut header = blocks[0].header().clone(); header.number = 4; - send_block_announce(header, &peer_id1, &mut sync); + send_block_announce(header, peer_id1, &mut sync); assert!(sync.fork_targets.len() == 1); sync.peer_disconnected(&peer_id1); @@ -4063,14 +3685,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - Box::new(DefaultBlockAnnounceValidator), 1, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -4107,7 +3728,6 @@ mod test { #[test] fn sync_restart_removes_block_but_not_justification_requests() { let mut client = Arc::new(TestClientBuilder::new().build()); - let block_announce_validator = Box::new(DefaultBlockAnnounceValidator); let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); @@ -4117,14 +3737,13 @@ mod test { ProtocolId::from("test-protocol-name"), &Some(String::from("test-fork-id")), Roles::from(&Role::Full), - block_announce_validator, 1, 64, None, None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) diff --git a/substrate/client/network/sync/src/mock.rs b/substrate/client/network/sync/src/mock.rs index 838c6cf7667a2463a38dc093addce42577bfc89f..859f9fb9c54b620f67fc2729043e8a9a145d97af 100644 --- a/substrate/client/network/sync/src/mock.rs +++ b/substrate/client/network/sync/src/mock.rs @@ -16,15 +16,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains a mock implementation of `ChainSync` that can be used -//! for testing calls made to `ChainSync`. +//! Contains mock implementations of `ChainSync` and 'BlockDownloader'. -use futures::task::Poll; +use crate::block_relay_protocol::{BlockDownloader as BlockDownloaderT, BlockResponseError}; + +use futures::{channel::oneshot, task::Poll}; use libp2p::PeerId; +use sc_network::RequestFailure; use sc_network_common::sync::{ message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse}, - BadPeer, ChainSync as ChainSyncT, Metrics, OnBlockData, OnBlockJustification, - OpaqueBlockResponse, PeerInfo, PollBlockAnnounceValidation, SyncStatus, + BadPeer, ChainSync as ChainSyncT, Metrics, OnBlockData, OnBlockJustification, PeerInfo, + SyncStatus, }; use sp_runtime::traits::{Block as BlockT, NumberFor}; @@ -71,28 +73,18 @@ mockall::mock! { success: bool, ); fn on_block_finalized(&mut self, hash: &Block::Hash, number: NumberFor); - fn push_block_announce_validation( + fn on_validated_block_announce( &mut self, - who: PeerId, - hash: Block::Hash, - announce: BlockAnnounce, is_best: bool, + who: PeerId, + announce: &BlockAnnounce, ); - fn poll_block_announce_validation<'a>( - &mut self, - cx: &mut std::task::Context<'a>, - ) -> Poll>; fn peer_disconnected(&mut self, who: &PeerId); fn metrics(&self) -> Metrics; - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String>; fn poll<'a>( &mut self, cx: &mut std::task::Context<'a>, - ) -> Poll>; + ) -> Poll<()>; fn send_block_request( &mut self, who: PeerId, @@ -100,3 +92,21 @@ mockall::mock! { ); } } + +mockall::mock! { + pub BlockDownloader {} + + #[async_trait::async_trait] + impl BlockDownloaderT for BlockDownloader { + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled>; + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError>; + } +} diff --git a/substrate/client/network/sync/src/warp.rs b/substrate/client/network/sync/src/warp.rs index 912ad78dfdd0800db1233786cd2cd8ebed064394..74835a6e015e3e6d8506950a794a1659ae3fdae0 100644 --- a/substrate/client/network/sync/src/warp.rs +++ b/substrate/client/network/sync/src/warp.rs @@ -19,36 +19,75 @@ //! Warp sync support. use crate::{ - oneshot, schema::v1::{StateRequest, StateResponse}, state::{ImportResult, StateSync}, }; -use futures::FutureExt; +use futures::channel::oneshot; use log::error; use sc_client_api::ProofProvider; use sc_network_common::sync::{ message::{BlockAttributes, BlockData, BlockRequest, Direction, FromBlock}, warp::{ - EncodedProof, VerificationResult, WarpProofRequest, WarpSyncParams, WarpSyncPhase, - WarpSyncProgress, WarpSyncProvider, + EncodedProof, VerificationResult, WarpProofRequest, WarpSyncPhase, WarpSyncProgress, + WarpSyncProvider, }, }; use sp_blockchain::HeaderBackend; use sp_consensus_grandpa::{AuthorityList, SetId}; use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; -use std::{sync::Arc, task::Poll}; +use std::sync::Arc; +/// Log target for this file. +const LOG_TARGET: &'static str = "sync"; + +/// The different types of warp syncing, passed to `build_network`. +pub enum WarpSyncParams { + /// Standard warp sync for the chain. + WithProvider(Arc>), + /// Skip downloading proofs and wait for a header of the state that should be downloaded. + /// + /// It is expected that the header provider ensures that the header is trusted. + WaitForTarget(oneshot::Receiver<::Header>), +} + +/// Warp sync configuration as accepted by [`WarpSync`]. +pub enum WarpSyncConfig { + /// Standard warp sync for the chain. + WithProvider(Arc>), + /// Skip downloading proofs and wait for a header of the state that should be downloaded. + /// + /// It is expected that the header provider ensures that the header is trusted. + WaitForTarget, +} + +impl WarpSyncParams { + /// Split `WarpSyncParams` into `WarpSyncConfig` and warp sync target block header receiver. + pub fn split( + self, + ) -> (WarpSyncConfig, Option::Header>>) { + match self { + WarpSyncParams::WithProvider(provider) => + (WarpSyncConfig::WithProvider(provider), None), + WarpSyncParams::WaitForTarget(rx) => (WarpSyncConfig::WaitForTarget, Some(rx)), + } + } +} + +/// Warp sync phase. enum Phase { + /// Downloading warp proofs. WarpProof { set_id: SetId, authorities: AuthorityList, last_hash: B::Hash, warp_sync_provider: Arc>, }, - PendingTargetBlock { - target_block: Option>, - }, + /// Waiting for target block to be set externally if we skip warp proofs downloading, + /// and start straight from the target block (used by parachains warp sync). + PendingTargetBlock, + /// Downloading target block. TargetBlock(B::Header), + /// Downloading state. State(StateSync), } @@ -83,10 +122,10 @@ where /// Create a new instance. When passing a warp sync provider we will be checking for proof and /// authorities. Alternatively we can pass a target block when we want to skip downloading /// proofs, in this case we will continue polling until the target block is known. - pub fn new(client: Arc, warp_sync_params: WarpSyncParams) -> Self { + pub fn new(client: Arc, warp_sync_config: WarpSyncConfig) -> Self { let last_hash = client.hash(Zero::zero()).unwrap().expect("Genesis header always exists"); - match warp_sync_params { - WarpSyncParams::WithProvider(warp_sync_provider) => { + match warp_sync_config { + WarpSyncConfig::WithProvider(warp_sync_provider) => { let phase = Phase::WarpProof { set_id: 0, authorities: warp_sync_provider.current_authorities(), @@ -95,35 +134,23 @@ where }; Self { client, phase, total_proof_bytes: 0 } }, - WarpSyncParams::WaitForTarget(block) => Self { - client, - phase: Phase::PendingTargetBlock { target_block: Some(block) }, - total_proof_bytes: 0, - }, + WarpSyncConfig::WaitForTarget => + Self { client, phase: Phase::PendingTargetBlock, total_proof_bytes: 0 }, } } - /// Poll to make progress. - /// - /// This only makes progress when `phase = Phase::PendingTargetBlock` and the pending block was - /// sent. - pub fn poll(&mut self, cx: &mut std::task::Context) { - let new_phase = if let Phase::PendingTargetBlock { target_block: Some(target_block) } = - &mut self.phase - { - match target_block.poll_unpin(cx) { - Poll::Ready(Ok(target)) => Phase::TargetBlock(target), - Poll::Ready(Err(e)) => { - error!(target: "sync", "Failed to get target block. Error: {:?}",e); - Phase::PendingTargetBlock { target_block: None } - }, - _ => return, - } - } else { + /// Set target block externally in case we skip warp proof downloading. + pub fn set_target_block(&mut self, header: B::Header) { + let Phase::PendingTargetBlock = self.phase else { + error!( + target: LOG_TARGET, + "Attempt to set warp sync target block in invalid phase.", + ); + debug_assert!(false); return }; - self.phase = new_phase; + self.phase = Phase::TargetBlock(header); } /// Validate and import a state response. diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 2a20da5a556b7398397cd9845acdc049961f0ce2..11505903e35d70a04c5dd04d2a0935db8cfd50a3 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -62,15 +62,14 @@ use sc_network::{ }; use sc_network_common::{ role::Roles, - sync::warp::{ - AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncParams, WarpSyncProvider, - }, + sync::warp::{AuthorityList, EncodedProof, SetId, VerificationResult, WarpSyncProvider}, }; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ block_request_handler::BlockRequestHandler, service::{chain_sync::SyncingService, network::NetworkServiceProvider}, state_request_handler::StateRequestHandler, + warp::WarpSyncParams, warp_request_handler, }; use sc_service::client::Client; @@ -799,12 +798,18 @@ pub trait TestNetFactory: Default + Sized + Send { let fork_id = Some(String::from("test-fork-id")); - let block_request_protocol_config = { - let (handler, protocol_config) = - BlockRequestHandler::new(&protocol_id, None, client.clone(), 50); - self.spawn_task(handler.run().boxed()); - protocol_config - }; + let (chain_sync_network_provider, chain_sync_network_handle) = + NetworkServiceProvider::new(); + let mut block_relay_params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + None, + client.clone(), + 50, + ); + self.spawn_task(Box::pin(async move { + block_relay_params.server.run().await; + })); let state_request_protocol_config = { let (handler, protocol_config) = @@ -849,8 +854,6 @@ pub trait TestNetFactory: Default + Sized + Send { let block_announce_validator = config .block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); - let (chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = @@ -865,7 +868,7 @@ pub trait TestNetFactory: Default + Sized + Send { Some(warp_sync_params), chain_sync_network_handle, import_queue.service(), - block_request_protocol_config.name.clone(), + block_relay_params.downloader, state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), rx, @@ -878,7 +881,7 @@ pub trait TestNetFactory: Default + Sized + Send { full_net_config.add_request_response_protocol(config); } for config in [ - block_request_protocol_config, + block_relay_params.request_response_config, state_request_protocol_config, light_client_request_protocol_config, warp_protocol_config, diff --git a/substrate/client/network/test/src/service.rs b/substrate/client/network/test/src/service.rs index 68e780545bb173775fc49268a704e0e37b2fa8c3..62d7f9f9d1bb1839f3f1f12bc601ecefb270d6bf 100644 --- a/substrate/client/network/test/src/service.rs +++ b/substrate/client/network/test/src/service.rs @@ -156,12 +156,18 @@ impl TestNetworkBuilder { let fork_id = Some(String::from("test-fork-id")); let mut full_net_config = FullNetworkConfiguration::new(&network_config); - let block_request_protocol_config = { - let (handler, protocol_config) = - BlockRequestHandler::new(&protocol_id, None, client.clone(), 50); - tokio::spawn(handler.run().boxed()); - protocol_config - }; + let (chain_sync_network_provider, chain_sync_network_handle) = + self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); + let mut block_relay_params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + None, + client.clone(), + 50, + ); + tokio::spawn(Box::pin(async move { + block_relay_params.server.run().await; + })); let state_request_protocol_config = { let (handler, protocol_config) = @@ -177,8 +183,6 @@ impl TestNetworkBuilder { protocol_config }; - let (chain_sync_network_provider, chain_sync_network_handle) = - self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config::Role::Full), @@ -191,7 +195,7 @@ impl TestNetworkBuilder { None, chain_sync_network_handle, import_queue.service(), - block_request_protocol_config.name.clone(), + block_relay_params.downloader, state_request_protocol_config.name.clone(), None, rx, @@ -214,7 +218,7 @@ impl TestNetworkBuilder { } for config in [ - block_request_protocol_config, + block_relay_params.request_response_config, state_request_protocol_config, light_client_request_protocol_config, ] { diff --git a/substrate/client/offchain/README.md b/substrate/client/offchain/README.md index f7c097e8e0b2a9e3ac07bbf0a41359aadf4a758f..74d54c0c234dcf66a411447ea66ef153730e36d7 100644 --- a/substrate/client/offchain/README.md +++ b/substrate/client/offchain/README.md @@ -15,4 +15,4 @@ for instance via: 2. Majority voting for results 3. etc -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/proposer-metrics/README.md b/substrate/client/proposer-metrics/README.md index 9669c7d35191dc6dc6d4c6df5878c6b4c9bea30d..27a6b726814c38088c2322e205825fb990357baa 100644 --- a/substrate/client/proposer-metrics/README.md +++ b/substrate/client/proposer-metrics/README.md @@ -1,3 +1,3 @@ Prometheus basic proposer metrics. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/rpc-api/Cargo.toml b/substrate/client/rpc-api/Cargo.toml index fb0d0a8a1f8a156cf6b53a5d9fecf9ad88d29547..a2ee090b1c23994169dba9b1105c3972f0a83a07 100644 --- a/substrate/client/rpc-api/Cargo.toml +++ b/substrate/client/rpc-api/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1" } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" sc-chain-spec = { path = "../chain-spec" } sc-transaction-pool-api = { path = "../transaction-pool/api" } diff --git a/substrate/client/rpc-api/README.md b/substrate/client/rpc-api/README.md index e860e0c2334da50e57bee241b14e2807b25bc7f3..8ea6e686c1a6450f6daef9db01ccfa4ad27309df 100644 --- a/substrate/client/rpc-api/README.md +++ b/substrate/client/rpc-api/README.md @@ -1,5 +1,5 @@ Substrate RPC interfaces. -A collection of RPC methods and subscriptions supported by all substrate clients. +A collection of RPC methods and subscriptions supported by all Substrate clients. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/rpc-servers/Cargo.toml b/substrate/client/rpc-servers/Cargo.toml index 94a0d815f2e6b39664e05864fa88b503c2cc2177..674a8db39cb576f6a0461d43c2fe8c7e0acb2ff2 100644 --- a/substrate/client/rpc-servers/Cargo.toml +++ b/substrate/client/rpc-servers/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpsee = { version = "0.16.2", features = ["server"] } log = "0.4.17" -serde_json = "1.0.85" +serde_json = "1.0.107" tokio = { version = "1.22.0", features = ["parking_lot"] } prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } tower-http = { version = "0.4.0", features = ["cors"] } diff --git a/substrate/client/rpc-servers/README.md b/substrate/client/rpc-servers/README.md index cf00b3169a6276b048b5a132d02df7c77e21dbf0..16f1bb9f2a368fa7e19c082161be0ccf3c805a37 100644 --- a/substrate/client/rpc-servers/README.md +++ b/substrate/client/rpc-servers/README.md @@ -1,3 +1,3 @@ Substrate RPC servers. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/rpc-spec-v2/README.md b/substrate/client/rpc-spec-v2/README.md index e860e0c2334da50e57bee241b14e2807b25bc7f3..8ea6e686c1a6450f6daef9db01ccfa4ad27309df 100644 --- a/substrate/client/rpc-spec-v2/README.md +++ b/substrate/client/rpc-spec-v2/README.md @@ -1,5 +1,5 @@ Substrate RPC interfaces. -A collection of RPC methods and subscriptions supported by all substrate clients. +A collection of RPC methods and subscriptions supported by all Substrate clients. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/rpc-spec-v2/src/archive/api.rs b/substrate/client/rpc-spec-v2/src/archive/api.rs new file mode 100644 index 0000000000000000000000000000000000000000..ca94779c887cba2dcfae25eb46113ec343a51746 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/archive/api.rs @@ -0,0 +1,56 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! API trait of the archive methods. + +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; + +#[rpc(client, server)] +pub trait ArchiveApi { + /// Retrieves the body (list of transactions) of a given block hash. + /// + /// Returns an array of strings containing the hexadecimal-encoded SCALE-codec-encoded + /// transactions in that block. If no block with that hash is found, null. + /// + /// # Unstable + /// + /// This method is unstable and subject to change in the future. + #[method(name = "archive_unstable_body")] + fn archive_unstable_body(&self, hash: Hash) -> RpcResult>>; + + /// Get the chain's genesis hash. + /// + /// Returns a string containing the hexadecimal-encoded hash of the genesis block of the chain. + /// + /// # Unstable + /// + /// This method is unstable and subject to change in the future. + #[method(name = "archive_unstable_genesisHash")] + fn archive_unstable_genesis_hash(&self) -> RpcResult; + + /// Get the block's header. + /// + /// Returns a string containing the hexadecimal-encoded SCALE-codec encoding header of the + /// block. + /// + /// # Unstable + /// + /// This method is unstable and subject to change in the future. + #[method(name = "archive_unstable_header")] + fn archive_unstable_header(&self, hash: Hash) -> RpcResult>; +} diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs new file mode 100644 index 0000000000000000000000000000000000000000..4fb2e5671d30044f6e609f34d5453ffce8005b84 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -0,0 +1,86 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! API implementation for `archive`. + +use super::ArchiveApiServer; +use crate::chain_head::hex_string; +use codec::Encode; +use jsonrpsee::core::{async_trait, RpcResult}; +use sc_client_api::{Backend, BlockBackend, BlockchainEvents, ExecutorProvider, StorageProvider}; +use sp_api::CallApiAt; +use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata}; +use sp_runtime::traits::Block as BlockT; +use std::{marker::PhantomData, sync::Arc}; + +/// An API for archive RPC calls. +pub struct Archive, Block: BlockT, Client> { + /// Substrate client. + client: Arc, + /// The hexadecimal encoded hash of the genesis block. + genesis_hash: String, + /// Phantom member to pin the block type. + _phantom: PhantomData<(Block, BE)>, +} + +impl, Block: BlockT, Client> Archive { + /// Create a new [`Archive`]. + pub fn new>(client: Arc, genesis_hash: GenesisHash) -> Self { + let genesis_hash = hex_string(&genesis_hash.as_ref()); + Self { client, genesis_hash, _phantom: PhantomData } + } +} + +#[async_trait] +impl ArchiveApiServer for Archive +where + Block: BlockT + 'static, + Block::Header: Unpin, + BE: Backend + 'static, + Client: BlockBackend + + ExecutorProvider + + HeaderBackend + + HeaderMetadata + + BlockchainEvents + + CallApiAt + + StorageProvider + + 'static, +{ + fn archive_unstable_body(&self, hash: Block::Hash) -> RpcResult>> { + let Ok(Some(signed_block)) = self.client.block(hash) else { return Ok(None) }; + + let extrinsics = signed_block + .block + .extrinsics() + .iter() + .map(|extrinsic| hex_string(&extrinsic.encode())) + .collect(); + + Ok(Some(extrinsics)) + } + + fn archive_unstable_genesis_hash(&self) -> RpcResult { + Ok(self.genesis_hash.clone()) + } + + fn archive_unstable_header(&self, hash: Block::Hash) -> RpcResult> { + let Ok(Some(header)) = self.client.header(hash) else { return Ok(None) }; + + Ok(Some(hex_string(&header.encode()))) + } +} diff --git a/substrate/client/rpc-spec-v2/src/archive/mod.rs b/substrate/client/rpc-spec-v2/src/archive/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..767f658ecd7537074be16f9d72db97821f743cca --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/archive/mod.rs @@ -0,0 +1,31 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Substrate archive API. +//! +//! # Note +//! +//! Methods are prefixed by `archive`. + +#[cfg(test)] +mod tests; + +pub mod api; +pub mod archive; + +pub use api::ArchiveApiServer; diff --git a/substrate/client/rpc-spec-v2/src/archive/tests.rs b/substrate/client/rpc-spec-v2/src/archive/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..bc75fc749acc06f6e9638a0d7ff29696d1f64ee3 --- /dev/null +++ b/substrate/client/rpc-spec-v2/src/archive/tests.rs @@ -0,0 +1,113 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::chain_head::hex_string; + +use super::{archive::Archive, *}; + +use codec::{Decode, Encode}; +use jsonrpsee::{types::EmptyServerParams as EmptyParams, RpcModule}; +use sc_block_builder::BlockBuilderProvider; + +use sp_consensus::BlockOrigin; +use std::sync::Arc; +use substrate_test_runtime_client::{ + prelude::*, runtime, Backend, BlockBuilderExt, Client, ClientBlockImportExt, +}; + +const CHAIN_GENESIS: [u8; 32] = [0; 32]; +const INVALID_HASH: [u8; 32] = [1; 32]; + +type Header = substrate_test_runtime_client::runtime::Header; +type Block = substrate_test_runtime_client::runtime::Block; + +fn setup_api() -> (Arc>, RpcModule>>) { + let builder = TestClientBuilder::new(); + let client = Arc::new(builder.build()); + + let api = Archive::new(client.clone(), CHAIN_GENESIS).into_rpc(); + + (client, api) +} + +#[tokio::test] +async fn archive_genesis() { + let (_client, api) = setup_api(); + + let genesis: String = + api.call("archive_unstable_genesisHash", EmptyParams::new()).await.unwrap(); + assert_eq!(genesis, hex_string(&CHAIN_GENESIS)); +} + +#[tokio::test] +async fn archive_body() { + let (mut client, api) = setup_api(); + + // Invalid block hash. + let invalid_hash = hex_string(&INVALID_HASH); + let res: Option> = api.call("archive_unstable_body", [invalid_hash]).await.unwrap(); + assert!(res.is_none()); + + // Import a new block with an extrinsic. + let mut builder = client.new_block(Default::default()).unwrap(); + builder + .push_transfer(runtime::Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 42, + nonce: 0, + }) + .unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + let expected_tx = hex_string(&block.extrinsics[0].encode()); + + let body: Vec = api.call("archive_unstable_body", [block_hash]).await.unwrap(); + assert_eq!(vec![expected_tx], body); +} + +#[tokio::test] +async fn archive_header() { + let (mut client, api) = setup_api(); + + // Invalid block hash. + let invalid_hash = hex_string(&INVALID_HASH); + let res: Option = api.call("archive_unstable_header", [invalid_hash]).await.unwrap(); + assert!(res.is_none()); + + // Import a new block with an extrinsic. + let mut builder = client.new_block(Default::default()).unwrap(); + builder + .push_transfer(runtime::Transfer { + from: AccountKeyring::Alice.into(), + to: AccountKeyring::Ferdie.into(), + amount: 42, + nonce: 0, + }) + .unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + let header: String = api.call("archive_unstable_header", [block_hash]).await.unwrap(); + let bytes = array_bytes::hex2bytes(&header).unwrap(); + let header: Header = Decode::decode(&mut &bytes[..]).unwrap(); + assert_eq!(header, block.header); +} diff --git a/substrate/client/rpc-spec-v2/src/chain_head/api.rs b/substrate/client/rpc-spec-v2/src/chain_head/api.rs index 682cd690dd10cf45bebfe73f1abbb680b87417e5..d93c4018b60faee0928204f1a6ecdfb75dab845c 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/api.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/api.rs @@ -30,7 +30,7 @@ pub trait ChainHeadApi { /// /// This method is unstable and subject to change in the future. #[subscription( - name = "chainHead_unstable_follow", + name = "chainHead_unstable_follow" => "chainHead_unstable_followEvent", unsubscribe = "chainHead_unstable_unfollow", item = FollowEvent, )] diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs index 14364c331e6c45a32e30673f2dc88d5b3a22246c..a8c1c4f7e083d86b19906d8845a5ee84b3b5ca6e 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -27,7 +27,7 @@ use crate::{ api::ChainHeadApiServer, chain_head_follow::ChainHeadFollower, error::Error as ChainHeadRpcError, - event::{FollowEvent, MethodResponse, OperationError, StorageQuery, StorageQueryType}, + event::{FollowEvent, MethodResponse, OperationError, StorageQuery}, hex_string, subscription::{SubscriptionManagement, SubscriptionManagementError}, }, @@ -329,19 +329,10 @@ where let items = items .into_iter() .map(|query| { - if query.query_type == StorageQueryType::ClosestDescendantMerkleValue { - // Note: remove this once all types are implemented. - return Err(ChainHeadRpcError::InvalidParam( - "Storage query type not supported".into(), - )) - } - - Ok(StorageQuery { - key: StorageKey(parse_hex_param(query.key)?), - query_type: query.query_type, - }) + let key = StorageKey(parse_hex_param(query.key)?); + Ok(StorageQuery { key, query_type: query.query_type }) }) - .collect::, _>>()?; + .collect::, ChainHeadRpcError>>()?; let child_trie = child_trie .map(|child_trie| parse_hex_param(child_trie)) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 48a673f47e3b3aaa0389a21d2244e679589b24d9..7095548a2b16cd518e874a86acd44cc0fad26b73 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -145,6 +145,36 @@ where .unwrap_or_else(|error| QueryResult::Err(error.to_string())) } + /// Fetch the closest merkle value. + fn query_storage_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + child_key: Option<&ChildInfo>, + ) -> QueryResult { + let result = if let Some(child_key) = child_key { + self.client.child_closest_merkle_value(hash, child_key, key) + } else { + self.client.closest_merkle_value(hash, key) + }; + + result + .map(|opt| { + QueryResult::Ok(opt.map(|storage_data| { + let result = match &storage_data { + sc_client_api::MerkleValue::Node(data) => hex_string(&data.as_slice()), + sc_client_api::MerkleValue::Hash(hash) => hex_string(&hash.as_ref()), + }; + + StorageResult { + key: hex_string(&key.0), + result: StorageResultType::ClosestDescendantMerkleValue(result), + } + })) + }) + .unwrap_or_else(|error| QueryResult::Err(error.to_string())) + } + /// Iterate over at most `operation_max_storage_items` keys. /// /// Returns the storage result with a potential next key to resume iteration. @@ -286,13 +316,21 @@ where return }, }, + StorageQueryType::ClosestDescendantMerkleValue => + match self.query_storage_merkle_value(hash, &item.key, child_key.as_ref()) { + Ok(Some(value)) => storage_results.push(value), + Ok(None) => continue, + Err(error) => { + send_error::(&sender, operation.operation_id(), error); + return + }, + }, StorageQueryType::DescendantsValues => self .iter_operations .push_back(QueryIter { next_key: item.key, ty: IterQueryType::Value }), StorageQueryType::DescendantsHashes => self .iter_operations .push_back(QueryIter { next_key: item.key, ty: IterQueryType::Hash }), - _ => continue, }; } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs index 6e92e87608b44abab9a4133e456e4f4f7272819c..a901f3039ffeabb8b4e9999e1bba28d1aa10d687 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -20,8 +20,8 @@ use parking_lot::Mutex; use sc_client_api::{ execution_extensions::ExecutionExtensions, BlockBackend, BlockImportNotification, BlockchainEvents, CallExecutor, ChildInfo, ExecutorProvider, FinalityNotification, - FinalityNotifications, FinalizeSummary, ImportNotifications, KeysIter, PairsIter, StorageData, - StorageEventStream, StorageKey, StorageProvider, + FinalityNotifications, FinalizeSummary, ImportNotifications, KeysIter, MerkleValue, PairsIter, + StorageData, StorageEventStream, StorageKey, StorageProvider, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_api::{CallApiAt, CallApiAtParams, NumberFor, RuntimeVersion}; @@ -198,6 +198,23 @@ impl< ) -> sp_blockchain::Result> { self.client.child_storage_hash(hash, child_info, key) } + + fn closest_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + ) -> sp_blockchain::Result>> { + self.client.closest_merkle_value(hash, key) + } + + fn child_closest_merkle_value( + &self, + hash: Block::Hash, + child_info: &ChildInfo, + key: &StorageKey, + ) -> sp_blockchain::Result>> { + self.client.child_closest_merkle_value(hash, child_info, key) + } } impl> CallApiAt for ChainHeadMockClient { diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 1336cff84b6ff31d3edae83afe80dff6671648c9..3ab47991c4e551456241707d4ad7ef828b042ce7 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -43,7 +43,12 @@ use sp_core::{ Blake2Hasher, Hasher, }; use sp_version::RuntimeVersion; -use std::{collections::HashSet, fmt::Debug, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + sync::Arc, + time::Duration, +}; use substrate_test_runtime::Transfer; use substrate_test_runtime_client::{ prelude::*, runtime, runtime::RuntimeApi, Backend, BlockBuilderExt, Client, @@ -2583,3 +2588,186 @@ async fn stop_storage_operation() { ) .await; } + +#[tokio::test] +async fn storage_closest_merkle_value() { + let (mut client, api, mut sub, sub_id, _) = setup_api().await; + + /// The core of this test. + /// + /// Checks keys that are exact match, keys with descedant and keys that should not return + /// values. + /// + /// Returns (key, merkle value) pairs. + async fn expect_merkle_request( + api: &RpcModule>>, + mut sub: &mut RpcSubscription, + sub_id: String, + block_hash: String, + ) -> HashMap { + // Valid call with storage at the keys. + let response: MethodResponse = api + .call( + "chainHead_unstable_storage", + rpc_params![ + &sub_id, + &block_hash, + vec![ + StorageQuery { + key: hex_string(b":AAAA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAAB"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Key with descedent. + StorageQuery { + key: hex_string(b":A"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Keys below this comment do not produce a result. + // Key that exceed the keyspace of the trie. + StorageQuery { + key: hex_string(b":AAAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAABX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Key that are not part of the trie. + StorageQuery { + key: hex_string(b":AAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + ] + ], + ) + .await + .unwrap(); + let operation_id = match response { + MethodResponse::Started(started) => started.operation_id, + MethodResponse::LimitReached => panic!("Expected started response"), + }; + + let event = get_next_event::>(&mut sub).await; + let merkle_values: HashMap<_, _> = match event { + FollowEvent::OperationStorageItems(res) => { + assert_eq!(res.operation_id, operation_id); + + res.items + .into_iter() + .map(|res| { + let value = match res.result { + StorageResultType::ClosestDescendantMerkleValue(value) => value, + _ => panic!("Unexpected StorageResultType"), + }; + (res.key, value) + }) + .collect() + }, + _ => panic!("Expected OperationStorageItems event"), + }; + + // Finished. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::OperationStorageDone(done) if done.operation_id == operation_id + ); + + // Response for AAAA, AAAB, A and AA. + assert_eq!(merkle_values.len(), 4); + + // While checking for expected merkle values to align, + // the following will check that the returned keys are + // expected. + + // Values for AAAA and AAAB are different. + assert_ne!( + merkle_values.get(&hex_string(b":AAAA")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + // Values for A and AA should be on the same branch node. + assert_eq!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AA")).unwrap() + ); + // The branch node value must be different than the leaf of either + // AAAA and AAAB. + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAA")).unwrap() + ); + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + merkle_values + } + + // Import a new block with storage changes. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![2; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Ensure the imported block is propagated and pinned for this subscription. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::NewBlock(_) + ); + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::BestBlockChanged(_) + ); + + let merkle_values_lhs = expect_merkle_request(&api, &mut sub, sub_id.clone(), block_hash).await; + + // Import a new block with and change AAAB value. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![3; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Ensure the imported block is propagated and pinned for this subscription. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::NewBlock(_) + ); + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::BestBlockChanged(_) + ); + + let merkle_values_rhs = expect_merkle_request(&api, &mut sub, sub_id.clone(), block_hash).await; + + // Change propagated to the root. + assert_ne!( + merkle_values_lhs.get(&hex_string(b":A")).unwrap(), + merkle_values_rhs.get(&hex_string(b":A")).unwrap() + ); + assert_ne!( + merkle_values_lhs.get(&hex_string(b":AAAB")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAB")).unwrap() + ); + // However the AAAA branch leaf remains unchanged. + assert_eq!( + merkle_values_lhs.get(&hex_string(b":AAAA")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAA")).unwrap() + ); +} diff --git a/substrate/client/rpc-spec-v2/src/lib.rs b/substrate/client/rpc-spec-v2/src/lib.rs index 7c22ef5d523189f5a2d49b150c786976bf64c5a8..9a455c5984a5f758c7b0b274cdf0c89be2685163 100644 --- a/substrate/client/rpc-spec-v2/src/lib.rs +++ b/substrate/client/rpc-spec-v2/src/lib.rs @@ -23,6 +23,7 @@ #![warn(missing_docs)] #![deny(unused_crate_dependencies)] +pub mod archive; pub mod chain_head; pub mod chain_spec; pub mod transaction; diff --git a/substrate/client/rpc-spec-v2/src/transaction/api.rs b/substrate/client/rpc-spec-v2/src/transaction/api.rs index c226ab86787e9880438fbc780d0b604a816a1848..3eb6cb204f24db463cabe3ae4d5a9560bf624057 100644 --- a/substrate/client/rpc-spec-v2/src/transaction/api.rs +++ b/substrate/client/rpc-spec-v2/src/transaction/api.rs @@ -29,7 +29,7 @@ pub trait TransactionApi { /// See [`TransactionEvent`](crate::transaction::event::TransactionEvent) for details on /// transaction life cycle. #[subscription( - name = "transaction_unstable_submitAndWatch" => "transaction_unstable_submitExtrinsic", + name = "transaction_unstable_submitAndWatch" => "transaction_unstable_watchEvent", unsubscribe = "transaction_unstable_unwatch", item = TransactionEvent, )] diff --git a/substrate/client/rpc/Cargo.toml b/substrate/client/rpc/Cargo.toml index c7e9977cb2529e275acc9804385243817a38c3af..64aaa7c94aacb074bd577b94138c71b7f3fe3d7c 100644 --- a/substrate/client/rpc/Cargo.toml +++ b/substrate/client/rpc/Cargo.toml @@ -18,7 +18,7 @@ futures = "0.3.21" jsonrpsee = { version = "0.16.2", features = ["server"] } log = "0.4.17" parking_lot = "0.12.1" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-block-builder = { path = "../block-builder" } sc-chain-spec = { path = "../chain-spec" } sc-client-api = { path = "../api" } diff --git a/substrate/client/rpc/README.md b/substrate/client/rpc/README.md index 6066af4da71a36ed2748f1cce2c360592e2b51ab..7490d0dc2b0910f6998900112f5cef43a44bc866 100644 --- a/substrate/client/rpc/README.md +++ b/substrate/client/rpc/README.md @@ -2,4 +2,4 @@ Substrate RPC implementation. A core implementation of Substrate RPC interfaces. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/service/Cargo.toml b/substrate/client/service/Cargo.toml index 6f794d93fed30c33a455da3dcdbfd38066c87f3f..ccf23bc8994b344fb66ac658b0f47ac1d8ad5a50 100644 --- a/substrate/client/service/Cargo.toml +++ b/substrate/client/service/Cargo.toml @@ -26,7 +26,7 @@ runtime-benchmarks = [ [dependencies] jsonrpsee = { version = "0.16.2", features = ["server"] } -thiserror = "1.0.30" +thiserror = "1.0.48" futures = "0.3.21" rand = "0.8.5" parking_lot = "0.12.1" @@ -35,7 +35,7 @@ futures-timer = "3.0.1" exit-future = "0.2.0" pin-project = "1.0.12" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-keystore = { path = "../keystore" } sp-runtime = { path = "../../primitives/runtime" } sp-trie = { path = "../../primitives/trie" } diff --git a/substrate/client/service/README.md b/substrate/client/service/README.md index 26f940f16df02328ad03e728e2a033ce3013acc6..0a36138a366f4f081e2bae8bf6fa8e75f276bdfa 100644 --- a/substrate/client/service/README.md +++ b/substrate/client/service/README.md @@ -1,4 +1,4 @@ Substrate service. Starts a thread that spins up the network, client, and extrinsic pool. Manages communication between them. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index fe18d1d002d56c9a6ab7ae6a3175be488f660c90..3838accde02393b053943e944e86f1f02ba8944e 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -47,11 +47,12 @@ use sc_network::{ NetworkService, NetworkStateInfo, NetworkStatusProvider, }; use sc_network_bitswap::BitswapRequestHandler; -use sc_network_common::{role::Roles, sync::warp::WarpSyncParams}; +use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ - block_request_handler::BlockRequestHandler, engine::SyncingEngine, - service::network::NetworkServiceProvider, state_request_handler::StateRequestHandler, + block_relay_protocol::BlockRelayParams, block_request_handler::BlockRequestHandler, + engine::SyncingEngine, service::network::NetworkServiceProvider, + state_request_handler::StateRequestHandler, warp::WarpSyncParams, warp_request_handler::RequestHandler as WarpSyncRequestHandler, SyncingService, }; use sc_rpc::{ @@ -695,6 +696,9 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { Option) -> Box + Send> + Send>>, /// Optional warp sync params. pub warp_sync_params: Option>, + /// User specified block relay params. If not specified, the default + /// block request handler will be used. + pub block_relay: Option>, } /// Build the network service, the network status sinks and an RPC sender. @@ -733,6 +737,7 @@ where import_queue, block_announce_validator_builder, warp_sync_params, + block_relay, } = params; if warp_sync_params.is_none() && config.network.sync_mode.is_warp() { @@ -756,19 +761,26 @@ where Box::new(DefaultBlockAnnounceValidator) }; - let (block_request_protocol_config, block_request_protocol_name) = { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = BlockRequestHandler::new( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - net_config.network_config.default_peers_set.in_peers as usize + - net_config.network_config.default_peers_set.out_peers as usize, - ); - let config_name = protocol_config.name.clone(); - spawn_handle.spawn("block-request-handler", Some("networking"), handler.run()); - (protocol_config, config_name) + let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let (mut block_server, block_downloader, block_request_protocol_config) = match block_relay { + Some(params) => (params.server, params.downloader, params.request_response_config), + None => { + // Custom protocol was not specified, use the default block handler. + // Allow both outgoing and incoming requests. + let params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + config.network.default_peers_set.in_peers as usize + + config.network.default_peers_set.out_peers as usize, + ); + (params.server, params.downloader, params.request_response_config) + }, }; + spawn_handle.spawn("block-request-handler", Some("networking"), async move { + block_server.run().await; + }); let (state_request_protocol_config, state_request_protocol_name) = { let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize + @@ -859,7 +871,6 @@ where spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); - let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); let (engine, sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config.role), client.clone(), @@ -871,7 +882,7 @@ where warp_sync_params, chain_sync_network_handle, import_queue.service(), - block_request_protocol_name, + block_downloader, state_request_protocol_name, warp_request_protocol_name, rx, diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index a0983d823e5b19a5758e281e91499085f9d813da..26dcd0f9e21a6aa4299f3228321c1f2ba1ba1158 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -78,7 +78,7 @@ use sp_state_machine::{ ChildStorageCollection, KeyValueStates, KeyValueStorageLevel, StorageCollection, MAX_NESTED_TRIE_DEPTH, }; -use sp_trie::{CompactProof, StorageProof}; +use sp_trie::{CompactProof, MerkleValue, StorageProof}; use std::{ collections::{HashMap, HashSet}, marker::PhantomData, @@ -603,8 +603,6 @@ where .block_gap .map_or(false, |(start, _)| *import_headers.post().number() == start); - assert!(justifications.is_some() && finalized || justifications.is_none() || gap_block); - // the block is lower than our last finalized block so it must revert // finality, refusing import. if status == blockchain::BlockStatus::Unknown && @@ -1547,6 +1545,27 @@ where .child_storage_hash(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } + + fn closest_merkle_value( + &self, + hash: ::Hash, + key: &StorageKey, + ) -> blockchain::Result::Hash>>> { + self.state_at(hash)? + .closest_merkle_value(&key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) + } + + fn child_closest_merkle_value( + &self, + hash: ::Hash, + child_info: &ChildInfo, + key: &StorageKey, + ) -> blockchain::Result::Hash>>> { + self.state_at(hash)? + .child_closest_merkle_value(child_info, &key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) + } } impl HeaderMetadata for Client diff --git a/substrate/client/service/src/client/wasm_substitutes.rs b/substrate/client/service/src/client/wasm_substitutes.rs index a792ab87e771bd0b1652e5d7f4b5708f945b4346..70db0ef20f5a83907536f67d8ef36500a8065b48 100644 --- a/substrate/client/service/src/client/wasm_substitutes.rs +++ b/substrate/client/service/src/client/wasm_substitutes.rs @@ -126,7 +126,7 @@ where let runtime_code = RuntimeCode { code_fetcher: &WrappedRuntimeCode((&code).into()), heap_pages: None, - hash: Vec::new(), + hash: make_hash(&code), }; let version = Self::runtime_version(&executor, &runtime_code)?; let spec_version = version.spec_version; diff --git a/substrate/client/service/src/lib.rs b/substrate/client/service/src/lib.rs index 0961967f9ca20a039e12b560b3ba6cb6cea24659..cd720e1c1e096671a07fd849be40071cdfbea979 100644 --- a/substrate/client/service/src/lib.rs +++ b/substrate/client/service/src/lib.rs @@ -79,7 +79,7 @@ pub use sc_chain_spec::{ pub use sc_consensus::ImportQueue; pub use sc_executor::NativeExecutionDispatch; -pub use sc_network_common::sync::warp::WarpSyncParams; +pub use sc_network_sync::warp::WarpSyncParams; #[doc(hidden)] pub use sc_network_transactions::config::{TransactionImport, TransactionImportFuture}; pub use sc_rpc::{ diff --git a/substrate/client/state-db/README.md b/substrate/client/state-db/README.md index a02b3929088fc8289a22d4a372101ca703cb6876..c97cc23400fae7c82132bddab19bf6bf177e1cff 100644 --- a/substrate/client/state-db/README.md +++ b/substrate/client/state-db/README.md @@ -2,15 +2,15 @@ State database maintenance. Handles canonicalization and pruning in the database this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that were added or deleted during block execution. -# Canonicalization. +# Canonicalization Canonicalization window tracks a tree of blocks identified by header hash. The in-memory overlay allows to get any node that was inserted in any of the blocks within the window. The tree is journaled to the backing database and rebuilt on startup. Canonicalization function selects one root from the top of the tree and discards all other roots and their subtrees. -# Pruning. +# Pruning See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning constraints are satisfied. -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/storage-monitor/Cargo.toml b/substrate/client/storage-monitor/Cargo.toml index c5b71260f97e34aef6c9de0486e79dcd8dffb240..7538f5ba602cec3efe139814584ac7a23000a088 100644 --- a/substrate/client/storage-monitor/Cargo.toml +++ b/substrate/client/storage-monitor/Cargo.toml @@ -9,10 +9,10 @@ description = "Storage monitor service for substrate" homepage = "https://substrate.io" [dependencies] -clap = { version = "4.4.2", features = ["derive", "string"] } +clap = { version = "4.4.4", features = ["derive", "string"] } log = "0.4.17" fs4 = "0.6.3" sc-client-db = { path = "../db", default-features = false} sp-core = { path = "../../primitives/core" } tokio = "1.22.0" -thiserror = "1.0.30" +thiserror = "1.0.48" diff --git a/substrate/client/sync-state-rpc/Cargo.toml b/substrate/client/sync-state-rpc/Cargo.toml index 59cc6ba40481765a60d6f58ef6776b1d62b23883..e582d0b7e4ffc1159be31ffaf6ecf5c347db38c7 100644 --- a/substrate/client/sync-state-rpc/Cargo.toml +++ b/substrate/client/sync-state-rpc/Cargo.toml @@ -15,8 +15,8 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1" } jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" -thiserror = "1.0.30" +serde_json = "1.0.107" +thiserror = "1.0.48" sc-chain-spec = { path = "../chain-spec" } sc-client-api = { path = "../api" } sc-consensus-babe = { path = "../consensus/babe" } diff --git a/substrate/client/sysinfo/Cargo.toml b/substrate/client/sysinfo/Cargo.toml index 5834d8dec07778aeefe2d9e08d8ad6514b45a8e1..fdf987ed45f265f9bd54ec0060a23e2ffac1d585 100644 --- a/substrate/client/sysinfo/Cargo.toml +++ b/substrate/client/sysinfo/Cargo.toml @@ -21,7 +21,7 @@ rand = "0.8.5" rand_pcg = "0.3.1" regex = "1" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" sc-telemetry = { path = "../telemetry" } sp-core = { path = "../../primitives/core" } sp-io = { path = "../../primitives/io" } diff --git a/substrate/client/telemetry/Cargo.toml b/substrate/client/telemetry/Cargo.toml index 15362912909193178acaaa3dbb26605af338550d..6d59b9dfc1a6980fa08d1ade2cd12ec25713f358 100644 --- a/substrate/client/telemetry/Cargo.toml +++ b/substrate/client/telemetry/Cargo.toml @@ -23,6 +23,6 @@ pin-project = "1.0.12" sc-utils = { path = "../utils" } rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" -thiserror = "1.0.30" +serde_json = "1.0.107" +thiserror = "1.0.48" wasm-timer = "0.2.5" diff --git a/substrate/client/telemetry/README.md b/substrate/client/telemetry/README.md index 2e3e19bd2f628565dc22b6f18e24397fb4315f55..849fad8bec70c14fb7dd327446e6e8e25be85e0d 100644 --- a/substrate/client/telemetry/README.md +++ b/substrate/client/telemetry/README.md @@ -1,6 +1,6 @@ # sc-telemetry -Substrate's client telemetry is a part of substrate that allows ingesting telemetry data +Substrate's client telemetry is a part of Substrate that allows ingesting telemetry data with for example [Polkadot telemetry](https://github.com/paritytech/substrate-telemetry). It works using Tokio's [tracing](https://github.com/tokio-rs/tracing/) library. The telemetry @@ -9,8 +9,8 @@ tracing `Layer`. This layer will then send the data through an asynchronous chan background task called [`TelemetryWorker`] which will send the information to the configured remote telemetry servers. -If multiple substrate nodes are running in the same process, it uses a `tracing::Span` to -identify which substrate node is reporting the telemetry. Every task spawned using sc-service's +If multiple Substrate nodes are running in the same process, it uses a `tracing::Span` to +identify which Substrate node is reporting the telemetry. Every task spawned using sc-service's `TaskManager` automatically inherit this span. Substrate's nodes initialize/register with the [`TelemetryWorker`] using a [`TelemetryHandle`]. diff --git a/substrate/client/tracing/Cargo.toml b/substrate/client/tracing/Cargo.toml index c9cd6ca313cdb65251062c8f4054af4fccec5455..ffcbf07490836318cd17ccd983823bbde2f56f78 100644 --- a/substrate/client/tracing/Cargo.toml +++ b/substrate/client/tracing/Cargo.toml @@ -23,7 +23,7 @@ parking_lot = "0.12.1" regex = "1.6.0" rustc-hash = "1.1.0" serde = "1.0.188" -thiserror = "1.0.30" +thiserror = "1.0.48" tracing = "0.1.29" tracing-log = "0.1.3" tracing-subscriber = { version = "0.2.25", features = ["parking_lot"] } diff --git a/substrate/client/tracing/README.md b/substrate/client/tracing/README.md index b008436df9bbedd136f79be15c10ec5a7c7b9a07..f52e9d4dbc667112e356e1864c6696497dd79e61 100644 --- a/substrate/client/tracing/README.md +++ b/substrate/client/tracing/README.md @@ -1,4 +1,4 @@ -Instrumentation implementation for substrate. +Instrumentation implementation for Substrate. This crate is unstable and the API and usage may change. @@ -8,4 +8,4 @@ See `sp-tracing` for examples on how to use tracing. Currently we provide `Log` (default), `Telemetry` variants for `Receiver` -License: GPL-3.0-or-later WITH Classpath-exception-2.0 \ No newline at end of file +License: GPL-3.0-or-later WITH Classpath-exception-2.0 diff --git a/substrate/client/tracing/proc-macro/Cargo.toml b/substrate/client/tracing/proc-macro/Cargo.toml index 8c444ffb606b31638338a5b50d4c06b1c24e4b39..f18e0aacd3776906419f1ff14bfd277c65c89004 100644 --- a/substrate/client/tracing/proc-macro/Cargo.toml +++ b/substrate/client/tracing/proc-macro/Cargo.toml @@ -18,4 +18,4 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = { version = "1.0.28", features = ["proc-macro"] } -syn = { version = "2.0.31", features = ["proc-macro", "full", "extra-traits", "parsing"] } +syn = { version = "2.0.37", features = ["proc-macro", "full", "extra-traits", "parsing"] } diff --git a/substrate/client/transaction-pool/Cargo.toml b/substrate/client/transaction-pool/Cargo.toml index 0e502cb39fba02736c86f3dbe417f9484983c689..b893dc839edd4c8a5a6d179b616d857bcd4d2e3c 100644 --- a/substrate/client/transaction-pool/Cargo.toml +++ b/substrate/client/transaction-pool/Cargo.toml @@ -21,7 +21,7 @@ linked-hash-map = "0.5.4" log = "0.4.17" parking_lot = "0.12.1" serde = { version = "1.0.188", features = ["derive"] } -thiserror = "1.0.30" +thiserror = "1.0.48" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } sc-client-api = { path = "../api" } sc-transaction-pool-api = { path = "api" } diff --git a/substrate/client/transaction-pool/README.md b/substrate/client/transaction-pool/README.md index 4a2bbb8838f9c8e2b654c938365cedc65201f36d..b34b623b26e016c444458c66641e68adb795b45a 100644 --- a/substrate/client/transaction-pool/README.md +++ b/substrate/client/transaction-pool/README.md @@ -232,7 +232,7 @@ are interested in. Since the pool is expected to store more transactions than what can fit in a single block, validating the entire pool on every block might not be -feasible. This means that the actual implementation might need to take some +feasible. This means that the actual implementation might need to take some shortcuts. ## Suggestions & caveats @@ -255,7 +255,7 @@ shortcuts. lot of re-orgs. Make sure that transactions are never lost. 1. The UTXO model is quite challenging. A transaction becomes valid right after - it's included in a block, however it is waiting for exactly the same inputs + it's included in a block, however it is waiting for exactly the same inputs to be spent, so it will never really be included again. 1. Note that in a non-ideal implementation the state of the pool will most @@ -278,7 +278,7 @@ shortcuts. 1. We periodically validate all transactions in the pool in batches. -1. To minimize runtime calls, we introduce the batch-verify call. Note it should +1. To minimize runtime calls, we introduce the batch-verify call. Note it should reset the state (overlay) after every verification. 1. Consider leveraging finality. Maybe we could verify against latest finalised @@ -355,7 +355,7 @@ figure out what tags were satisfied by a transaction in that block. For each blo transaction we either call into the runtime to get it's `ValidTransaction` object, or we check the pool if that transaction is already known to spare the runtime call. From this we gather the full set of `provides` tags and perform pruning of -the `ready` pool based on that. Also, we promote all transactions from `future` +the `ready` pool based on that. Also, we promote all transactions from `future` that have their tags satisfied. In case we remove transactions that we are unsure if they were already included diff --git a/substrate/client/transaction-pool/api/Cargo.toml b/substrate/client/transaction-pool/api/Cargo.toml index edab1304e01c18b72c657aebc6ebfb8d5ca930ba..5ff5a4149ca983a36d7b940b15bf10bf8c802b0f 100644 --- a/substrate/client/transaction-pool/api/Cargo.toml +++ b/substrate/client/transaction-pool/api/Cargo.toml @@ -14,7 +14,7 @@ codec = { package = "parity-scale-codec", version = "3.6.1" } futures = "0.3.21" log = "0.4.17" serde = { version = "1.0.188", features = ["derive"] } -thiserror = "1.0.30" +thiserror = "1.0.48" sp-blockchain = { path = "../../../primitives/blockchain" } sp-core = { path = "../../../primitives/core", default-features = false} sp-runtime = { path = "../../../primitives/runtime", default-features = false} diff --git a/substrate/client/transaction-pool/api/src/lib.rs b/substrate/client/transaction-pool/api/src/lib.rs index a132cbc46e9b07eee1c448b458eec6232e13142e..73cc513708d2d8e55f85ac2b5c35e861c3383a65 100644 --- a/substrate/client/transaction-pool/api/src/lib.rs +++ b/substrate/client/transaction-pool/api/src/lib.rs @@ -22,6 +22,7 @@ pub mod error; use async_trait::async_trait; +use codec::Codec; use futures::{Future, Stream}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sp_core::offchain::TransactionPoolExt; @@ -187,7 +188,7 @@ pub trait TransactionPool: Send + Sync { /// Block type. type Block: BlockT; /// Transaction hash type. - type Hash: Hash + Eq + Member + Serialize + DeserializeOwned; + type Hash: Hash + Eq + Member + Serialize + DeserializeOwned + Codec; /// In-pool transaction type. type InPoolTransaction: InPoolTransaction< Transaction = TransactionFor, diff --git a/substrate/client/utils/src/metrics.rs b/substrate/client/utils/src/metrics.rs index 6bbdbe2e2e59954669c9bee51aafb6f1c25592ac..308e90cb25379d62fe02f1878a664ff44377c9df 100644 --- a/substrate/client/utils/src/metrics.rs +++ b/substrate/client/utils/src/metrics.rs @@ -24,7 +24,10 @@ use prometheus::{ Error as PrometheusError, Registry, }; -use prometheus::{core::GenericCounterVec, Opts}; +use prometheus::{ + core::{GenericCounterVec, GenericGaugeVec}, + Opts, +}; lazy_static! { pub static ref TOKIO_THREADS_TOTAL: GenericCounter = @@ -36,18 +39,32 @@ lazy_static! { } lazy_static! { - pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericCounterVec = GenericCounterVec::new( - Opts::new("substrate_unbounded_channel_len", "Items in each mpsc::unbounded instance"), - &["entity", "action"] // 'name of channel, send|received|dropped + pub static ref UNBOUNDED_CHANNELS_COUNTER: GenericCounterVec = GenericCounterVec::new( + Opts::new( + "substrate_unbounded_channel_len", + "Items sent/received/dropped on each mpsc::unbounded instance" + ), + &["entity", "action"], // name of channel, send|received|dropped + ).expect("Creating of statics doesn't fail. qed"); + pub static ref UNBOUNDED_CHANNELS_SIZE: GenericGaugeVec = GenericGaugeVec::new( + Opts::new( + "substrate_unbounded_channel_size", + "Size (number of messages to be processed) of each mpsc::unbounded instance", + ), + &["entity"], // name of channel ).expect("Creating of statics doesn't fail. qed"); - } +pub static SENT_LABEL: &'static str = "send"; +pub static RECEIVED_LABEL: &'static str = "received"; +pub static DROPPED_LABEL: &'static str = "dropped"; + /// Register the statics to report to registry pub fn register_globals(registry: &Registry) -> Result<(), PrometheusError> { registry.register(Box::new(TOKIO_THREADS_ALIVE.clone()))?; registry.register(Box::new(TOKIO_THREADS_TOTAL.clone()))?; registry.register(Box::new(UNBOUNDED_CHANNELS_COUNTER.clone()))?; + registry.register(Box::new(UNBOUNDED_CHANNELS_SIZE.clone()))?; Ok(()) } diff --git a/substrate/client/utils/src/mpsc.rs b/substrate/client/utils/src/mpsc.rs index 36e44be5e29507d1d3445f9e35f141a7df9a4adb..c24a5bd8904afc99109acf8f5c6fe5872b5d9874 100644 --- a/substrate/client/utils/src/mpsc.rs +++ b/substrate/client/utils/src/mpsc.rs @@ -20,7 +20,9 @@ pub use async_channel::{TryRecvError, TrySendError}; -use crate::metrics::UNBOUNDED_CHANNELS_COUNTER; +use crate::metrics::{ + DROPPED_LABEL, RECEIVED_LABEL, SENT_LABEL, UNBOUNDED_CHANNELS_COUNTER, UNBOUNDED_CHANNELS_SIZE, +}; use async_channel::{Receiver, Sender}; use futures::{ stream::{FusedStream, Stream}, @@ -102,7 +104,10 @@ impl TracingUnboundedSender { /// Proxy function to `async_channel::Sender::try_send`. pub fn unbounded_send(&self, msg: T) -> Result<(), TrySendError> { self.inner.try_send(msg).map(|s| { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, "send"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, SENT_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[self.name]) + .set(self.inner.len().saturated_into()); if self.inner.len() >= self.queue_size_warning && self.warning_fired @@ -123,6 +128,11 @@ impl TracingUnboundedSender { s }) } + + /// The number of elements in the channel (proxy function to [`async_channel::Sender`]). + pub fn len(&self) -> usize { + self.inner.len() + } } impl TracingUnboundedReceiver { @@ -135,24 +145,34 @@ impl TracingUnboundedReceiver { /// that discounts the messages taken out. pub fn try_recv(&mut self) -> Result { self.inner.try_recv().map(|s| { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, "received"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, RECEIVED_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[self.name]) + .set(self.inner.len().saturated_into()); s }) } + + /// The number of elements in the channel (proxy function to [`async_channel::Receiver`]). + pub fn len(&self) -> usize { + self.inner.len() + } } impl Drop for TracingUnboundedReceiver { fn drop(&mut self) { // Close the channel to prevent any further messages to be sent into the channel self.close(); - // the number of messages about to be dropped + // The number of messages about to be dropped let count = self.inner.len(); - // discount the messages + // Discount the messages if count > 0 { UNBOUNDED_CHANNELS_COUNTER - .with_label_values(&[self.name, "dropped"]) + .with_label_values(&[self.name, DROPPED_LABEL]) .inc_by(count.saturated_into()); } + // Reset the size metric to 0 + UNBOUNDED_CHANNELS_SIZE.with_label_values(&[self.name]).set(0); // Drain all the pending messages in the channel since they can never be accessed, // this can be removed once https://github.com/smol-rs/async-channel/issues/23 is // resolved @@ -170,7 +190,10 @@ impl Stream for TracingUnboundedReceiver { match Pin::new(&mut s.inner).poll_next(cx) { Poll::Ready(msg) => { if msg.is_some() { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[s.name, "received"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[s.name, RECEIVED_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[s.name]) + .set(s.inner.len().saturated_into()); } Poll::Ready(msg) }, diff --git a/substrate/docker/README.md b/substrate/docker/README.md index 71ddb2dffd1bbe575f67d71e51db728a920ab265..f9cc7ed14653035c22c8753b61c8dd88ca24cbeb 100644 --- a/substrate/docker/README.md +++ b/substrate/docker/README.md @@ -1,27 +1,32 @@ # Substrate Builder Docker Image -The Docker image in this folder is a `builder` image. It is self contained and allows users to build the binaries themselves. -There is no requirement on having Rust or any other toolchain installed but a working Docker environment. +The Docker image in this folder is a `builder` image. It is self contained and allows users to build the binaries +themselves. There is no requirement on having Rust or any other toolchain installed but a working Docker environment. -Unlike the `parity/polkadot` image which contains a single binary (`polkadot`!) used by default, the image in this folder builds and contains several binaries and you need to provide the name of the binary to be called. +Unlike the `parity/polkadot` image which contains a single binary (`polkadot`!) used by default, the image in this +folder builds and contains several binaries and you need to provide the name of the binary to be called. -You should refer to the [.Dockerfile](./substrate_builder.Dockerfile) for the actual list. At the time of editing, the list of included binaries is: +You should refer to the [.Dockerfile](./substrate_builder.Dockerfile) for the actual list. At the time of editing, the +list of included binaries is: -- substrate -- subkey -- node-template -- chain-spec-builder +- `substrate` +- `subkey` +- `node-template` +- `chain-spec-builder` First, install [Docker](https://docs.docker.com/get-docker/). -Then to generate the latest parity/substrate image. Please run: +Then to generate the latest `parity/substrate` image. Please run: ```sh ./build.sh ``` -> If you wish to create a debug build rather than a production build, then you may modify the [.Dockerfile](./substrate_builder.Dockerfile) replacing `cargo build --locked --release` with just `cargo build --locked` and replacing `target/release` with `target/debug`. +If you wish to create a debug build rather than a production build, then you may modify the +[.Dockerfile](./substrate_builder.Dockerfile) replacing `cargo build --locked --release` with just +`cargo build --locked` and replacing `target/release` with `target/debug`. -> If you get an error that a tcp port address is already in use then find an available port to use for the host port in the [.Dockerfile](./substrate_builder.Dockerfile). +If you get an error that a tcp port address is already in use then find an available port to use for the host port in +the [.Dockerfile](./substrate_builder.Dockerfile). The image can be used by passing the selected binary followed by the appropriate tags for this binary. @@ -32,7 +37,8 @@ Your best guess to get started is to pass the `--help flag`. Here are a few exam - `./run.sh node-template --version` - `./run.sh chain-spec-builder --help` -Then try running the following command to start a single node development chain using the Substrate Node Template binary `node-template`: +Then try running the following command to start a single node development chain using the Substrate Node Template binary +`node-template`: ```sh ./run.sh node-template --dev --ws-external diff --git a/substrate/docs/CHANGELOG.md b/substrate/docs/CHANGELOG.md index 25f8e582c78fc01f3b5ca46d8f57f70413f3c2b6..8a1b245a5b0cc1cbf81b11a03897b60bb701e283 100644 --- a/substrate/docs/CHANGELOG.md +++ b/substrate/docs/CHANGELOG.md @@ -8,24 +8,36 @@ The format is based on [Keep a Changelog]. ## 2.0.1-> 3.0.0 - Apollo 14 -Most notably, this is the first release of the new FRAME (2.0) with its new macro-syntax and some changes in types, and pallet versioning. This release also incorporates the faster and improve version 2.0 of the parity-scale-codec and upgraded dependencies all-around. While the `FinalityTracker` pallet has been dropped, this release marks the first public appearance of a few new pallets, too;Bounties, Lottery, Tips (extracted from the `Treasury`-pallet, see #7536) and Merkle-Mountain-Ranges (MMR). - -On the client side, the most notable changes are around the keystore, making it async and switching to a different signing model allowing for remote-signing to be implemented; and various changes to improve networking and light-client support, like adding the Grandpa warp sync request-response protocol (#7711). - -_Contracts_: Please note that the contracts pallet _is not part_ of this release. The pallet is not yet ready and will be released separately in the coming weeks. The currently released contracts pallet _is not compatible_ with the new FRAME, thus if you need the contracts pallet, we recommend you wait with the upgrade until it has been released, too. +Most notably, this is the first release of the new FRAME (2.0) with its new macro-syntax and some changes in types, and +pallet versioning. This release also incorporates the faster and improve version 2.0 of the `parity-scale-codec` and +upgraded dependencies all-around. While the `FinalityTracker` pallet has been dropped, this release marks the first +public appearance of a few new pallets, too;Bounties, Lottery, Tips (extracted from the `Treasury`-pallet, see #7536) +and Merkle-Mountain-Ranges (MMR). + +On the client side, the most notable changes are around the keystore, making it async and switching to a different +signing model allowing for remote-signing to be implemented; and various changes to improve networking and light-client +support, like adding the Grandpa warp sync request-response protocol (#7711). + +_Contracts_: Please note that the contracts pallet _is not part_ of this release. The pallet is not yet ready and will +be released separately in the coming weeks. The currently released contracts pallet _is not compatible_ with the new +FRAME, thus if you need the contracts pallet, we recommend you wait with the upgrade until it has been released, too. ### Upgrade instructions -Not too much has changed on the top and API level for developing Substrate between 2.0 and 3.0. The easiest and quickest path for upgrading is just to take the latest node-template and try applying your changes to it: +Not too much has changed on the top and API level for developing Substrate between 2.0 and 3.0. The easiest and quickest +path for upgrading is just to take the latest node-template and try applying your changes to it: 1. take a diff between 2.0 and your changes 2. store that diff 3. remove everything, copy over the 3.0 node-template 4. try re-applying your diff, manually, a hunk at a time. -If that doesn't work for you, we are working on an in-depth-guide for all major changes that took place and how you need to adapt your code for it. [You can find the upgrade guide under `docs/` in the repo](https://github.com/paritytech/substrate/blob/master/docs/Upgrading-2.0-to-3.0.md), if you have further questions or problem, please [feel free to ask in the github discussion board](https://github.com/paritytech/substrate/discussions). +If that doesn't work for you, we are working on an in-depth-guide for all major changes that took place and how you need +to adapt your code for it. [You can find the upgrade guide under `docs/` in the +repo](https://github.com/paritytech/substrate/blob/master/docs/Upgrading-2.0-to-3.0.md), if you have further questions +or problem, please [feel free to ask in the github discussion +board](https://github.com/paritytech/substrate/discussions). -Runtime -------- +#### Runtime * contracts: Charge rent for code storage (#7935) * contracts: Emit event on contract termination (#8014) @@ -63,8 +75,7 @@ Runtime * Move proxies migration (#7205) * Introduce `cancel_proposal` to rid us of those pesky proposals (#7111) -Client ------- +#### Client * Remove backwards-compatibility networking hack (#8068) * Extend SS58 network identifiers (#8039) @@ -97,8 +108,7 @@ Client * Refactor CurrencyToVote (#6896) * client/network: Stop sending noise legacy handshake (#7211) -API ---- +#### API * pallet macro: easier syntax for `#[pallet::pallet]` with `struct Pallet(_)` (#8091) * WasmExecutor takes a cache directory (#8057) @@ -106,7 +116,7 @@ API * Migrate assets pallet to new macros (#7984) * contracts: Make ChainExtension trait generic over the runtime (#8003) * Decouple the session validators from im-online (#7127) -* Update parity-scale-codec to 2.0 (#7994) +* Update `parity-scale-codec` to 2.0 (#7994) * Merkle Mountain Range pallet improvements (#7891) * Cleaner GRANDPA RPC API for proving finality (#7339) * Migrate frame-system to pallet attribute macro (#7898) @@ -136,8 +146,7 @@ API * SystemOrigin trait (#7226) * permit setting treasury pallet initial funding through genesis (#7214) -Runtime Migrations ------------------- +#### Runtime Migrations * Migrate assets pallet to new macros (#7984) * Fix elections-phragmen and proxy issue (#7040) @@ -149,8 +158,7 @@ Runtime Migrations ## 2.0.0-> 2.0.1 -Patch release with backports to fix broken nightly builds. -Namely contains backports of +Patch release with backports to fix broken nightly builds. Namely contains backports of * [#7381: Make Substrate compile with latest nightly](https://github.com/paritytech/substrate/pull/7381) * [#7238: Fix compilation with environmental on latest nightly](https://github.com/paritytech/substrate/pull/7238) @@ -161,8 +169,7 @@ Namely contains backports of ## 2.0.0-rc6 -> 2.0.0 – two dot 😮 -Runtime -------- +### Runtime * Rename `ModuleToIndex` to `PalletRuntimeSetup` (#7148) * Bounties (#5715) @@ -174,8 +181,7 @@ Runtime * Time-delay proxies (#6770) * Refcounts are now u32 (#7164) -Client ------- +### Client * Rename `inspect-key` to `inspect` (#7160) * Send import notification always for re-orgs (#7118) @@ -190,8 +196,7 @@ Client * Fix benchmark read/write key tracker for keys in child storages. (#6905) * *: Update to next libp2p version 0.24.0 (#6891) -API ---- +### API * grandpa-rpc: use FinalityProofProvider to check finality for rpc (#6215) * pow: replace the thread-base mining loop with a future-based mining worker (#7060) @@ -204,16 +209,14 @@ API * Add a `LightSyncState` field to the chain spec (#6894) * *: Update to next libp2p version 0.24.0 (#6891) -Runtime Migrations ------------------- +### Runtime Migrations * Time-delay proxies (#6770) ## 2.0.0-rc5 -> 2.0.0-rc6 – Rock Hyrax -Runtime -------- +### Runtime * Custom Codec Implementation for NPoS Election (#6720) * Successful `note_imminent_preimage` is free (#6793) @@ -224,8 +227,7 @@ Runtime * pallet-evm: add support for tuple-based precompile declarations (#6681) * grandpa: allow noting that the set has stalled (#6725) -Client ------- +#### Client * Merge Subkey into sc-cli (#4954) * RpcHandlers Refactorings (#6846) @@ -239,8 +241,7 @@ Client * Name all the tasks! (#6726) * Child nodes can be handled by adding a child `TaskManager` to the parent's `TaskManager` (#6771) -API ---- +### API * pow: add access to pre-digest for algorithm verifiers (#6900) * babe, aura, pow: only call check_inherents if authoring version is compatible (#6862) @@ -254,8 +255,7 @@ API ## 2.0.0-rc4 -> 2.0.0-rc5 – River Dolphin -Runtime -------- +### Runtime * Support using system storage directly for EVM balance and nonce (#6659) * Properly filter out duplicate voters in elections. (#6693) @@ -273,14 +273,13 @@ Runtime * pallet-evm: customizable chain id (#6537) * Refactor as_sub to make things clearer. (#6503) -Client ------- +### Client * Update wasmtime to (almost) latest master (#6662) * Update to latest sysinfo prevents leaking fd-handlers (#6708) * Tracing values (#6679) * Graceful shutdown for the task manager (#6654) -* Update substrate-networking Grafana dashboard (#6649) +* Update `substrate-networking` Grafana dashboard (#6649) * *: Update to libp2p v0.21.1 (#6559) * Send Status message on all newly-opened legacy substreams (#6593) * babe: report equivocations (#6362) @@ -288,8 +287,7 @@ Client * Remove the service, replacing it with a struct of individual chain components (#6352) * Fix tx-pool returning the same transaction multiple times (#6535) -API ---- +### API * Better handling of stable-only build (#6569) * Remove the service builder (#6557) @@ -302,8 +300,7 @@ API ## 2.0.0-rc3 -> 2.0.0-rc4 (Rhinoceros) -Runtime -------- +### Runtime * Staking Payout Creates Controller (#6496) * `pallet-scheduler`: Check that `when` is not in the past (#6480) @@ -321,8 +318,7 @@ Runtime * Add events for balance reserve and unreserve functions (#6330) * Introduce frozen indices. (#6307) -Client ------- +### Client * client/network/service: Add primary dimension to connection metrics (#6472) * Fix Babe secondary plain slots claiming (#6451) @@ -340,8 +336,7 @@ Client * new crate sc-light (#6235) * Allow adding a prefix to the informant (#6174) -API ---- +### API * seal: Remove ext_dispatch_call and ext_get_runtime_storage (#6464) * seal: Refactor ext_gas_price (#6478) @@ -356,16 +351,14 @@ API ## 2.0.0-rc2 -> 2.0.0-rc3 -Runtime -------- +### Runtime * Introduce stacked filtering (#6273) * Allow "pure" proxied accounts (#6236) * Allow over-weight collective proposals to be closed (#6163) * Fix Election when ForceNone V1 (#6166) -Client ------- +### Client * Make transaction pool prune transactions only of canonical blocks (#6123) * Rename all the election operations (#6245) @@ -381,14 +374,12 @@ Client ## 2.0.0-alpha.8 -> 2.0.0-rc1 -Runtime -------- +### Runtime * Allow operational recovery path if on_initialize use fullblock. (#6089) * Maximum extrinsic weight limit (#6067) -Client ------- +### Client * Add JSON format to import blocks and set it as default (#5816) * Upgrade to libp2p v0.19 - Changes the default PeerId representation (#6064) @@ -396,17 +387,17 @@ Client ## 2.0.0-alpha.7 -> 2.0.0-alpha.8 -**License Changed** -From this release forward, the code is released under a new – more relaxed – license scheme: Client (`sc-*`) is released under "GPL 3.0 or newer with the Classpath Exception", while primitives, FRAME, the pallets, utils and test-utils are released under "Apache 2.0". More details in the [Relax licensing scheme PR](https://github.com/paritytech/substrate/pull/5947). +**License Changed** From this release forward, the code is released under a new – more relaxed – license scheme: Client +(`sc-*`) is released under "GPL 3.0 or newer with the Classpath Exception", while primitives, FRAME, the pallets, utils +and test-utils are released under "Apache 2.0". More details in the [Relax licensing scheme +PR](https://github.com/paritytech/substrate/pull/5947). -Runtime -------- +### Runtime * Democracy weight (#5828) * Make `Digest` support `StorageAppend` (#5922) -Client ------- +### Client * Meter block import results via prometheus (#6025) * Added RuntimePublic for ecdsa public key. (#6029) @@ -418,8 +409,7 @@ Client ## 2.0.0-alpha.6 -> 2.0.0-alpha.7 -Runtime -------- +### Runtime * Use `storage::append` in the implementation of the storage types (#5889) * pallet-sudo: Store `DispatchResult` in `Sudid` event (#5804) @@ -431,8 +421,7 @@ Runtime * Transaction versioning in the RuntimeVersion (#5582) * emit TipClosed event on success tip payout (#5656) -Client ------- +### Client * Adds `export-state` subcommand (#5842) * Drop ClientProvider (#5823) @@ -454,8 +443,7 @@ Client * Use a Kademlia instance per `ProtocolId`. (#5045) * Report tasks metrics to Prometheus (#5619) -API ---- +### API * Child trie api changes BREAKING (#4857) * Pass max-total to RewardRemainder on end_era (#5697) @@ -463,8 +451,7 @@ API ## 2.0.0-alpha.5 -> 2.0.0-alpha.6 -Runtime -------- +### Runtime * Unsigned Validation best practices (#5563) * Generate Unit Tests for Benchmarks (#5527) @@ -473,8 +460,7 @@ Runtime * Pass transaction source to validate_transaction (#5366) * on_initialize return weight consumed and default cost to default DispatchInfo instead of zero (#5382) -Client ------- +### Client * Add new RPC method to get the chain type (#5576) * Reuse wasmtime instances, the PR (#5567) @@ -488,10 +474,10 @@ Client * Make transactions and block announces use notifications substre… (#5360) * Adds state_queryStorageAt (#5362) * Offchain Phragmén BREAKING. (#4517) -* `sc_rpc::system::SystemInfo.impl_version` now returns the full version (2.0.0-alpha.2-b950f731c-x86_64-linux-gnu) instead of the short version (1.0.0) (#5271) +* `sc_rpc::system::SystemInfo.impl_version` now returns the full version (2.0.0-alpha.2-b950f731c-x86_64-linux-gnu) + instead of the short version (1.0.0) (#5271) -API ---- +### API * Unsigned Validation best practices (#5563) * Split the Roles in three types (#5520) @@ -501,16 +487,14 @@ API ## 2.0.0-alpha.4 -> 2.0.0-alpha.5 -Runtime -------- +### Runtime * pallet-evm: configurable gasometer config (#5320) * Adds new event phase `Initialization` (#5302) ## 2.0.0-alpha.3 -> 2.0.0-alpha.4 -Runtime -------- +### Runtime * Move runtime upgrade to `frame-executive` (#5197) * Split fees and tips between author and treasury independently (#5207) @@ -520,22 +504,20 @@ Runtime * Adds `vested_transfer` to Vesting pallet (#5029) * Change extrinsic_count to extrinsic_index in pallet-utility (#5044) -Client ------- +### Client * client/finality-grandpa: Add Prometheus metrics to GossipValidator (#5237) * removes use of sc_client::Client from node-transaction-factory (#5158) * removes use of sc_client::Client from sc_network (#5147) * Use CLI to configure max instances cache (#5177) * client/service/src/builder.rs: Add build_info metric (#5192) -* Remove substrate-ui.parity.io from CORS whitelist (#5142) +* Remove `substrate-ui.parity.io` from CORS whitelist (#5142) * removes use of sc_client::Client from sc-rpc (#5063) * Use 128mb for db cache default (#5134) * Drop db-cache default from 1gig to 32mb (#5128) * Add more metrics to prometheus (#5034) -API ---- +### API * Produce block always on updated transaction pool state (#5227) * Add `ext_terminate` (#5234) diff --git a/substrate/docs/CODEOWNERS b/substrate/docs/CODEOWNERS deleted file mode 100644 index 63294d90e9d069949614bc1c703078f6828dd70d..0000000000000000000000000000000000000000 --- a/substrate/docs/CODEOWNERS +++ /dev/null @@ -1,74 +0,0 @@ -# Lists some code owners. -# -# A codeowner just oversees some part of the codebase. If an owned file is changed then the -# corresponding codeowner receives a review request. An approval of the codeowner is -# not required for merging a PR though. -# -# **This is pretty much an experiment at the moment**. Feel free to remove yourself at any time if -# you do not want to receive review requests any longer. -# -# For details about syntax, see: -# https://help.github.com/en/articles/about-code-owners -# But here are some important notes: -# -# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` which -# can be everywhere. -# - Multiple owners are supported. -# - Either handle (e.g, @pepyakin) or email can be used. Keep in mind, that handles might work better because they -# are more recognizable on GitHub, you can use them for mentioning unlike an email. -# - The latest matching rule, if multiple, takes precedence. - -# CI -/.github/ @paritytech/ci -/.gitlab-ci.yml @paritytech/ci -/scripts/ci/ @paritytech/ci - -# WASM executor, low-level client <-> WASM interface and other WASM-related code -/client/allocator/ @koute -/client/executor/ @koute -/primitives/panic-handler/ @koute -/primitives/runtime-interface/ @koute -/primitives/wasm-interface/ @koute -/utils/wasm-builder/ @koute - -# Systems-related bits and bobs on the client side -/client/sysinfo/ @koute -/client/tracing/ @koute - -# Documentation audit -/primitives/runtime @paritytech/docs-audit -/primitives/arithmetic @paritytech/docs-audit -# /primitives/core (to be added later) -# /primitives/io (to be added later) - -# FRAME -/frame/ @paritytech/frame-coders @paritytech/docs-audit -/frame/nfts/ @jsidorenko @paritytech/docs-audit -/frame/state-trie-migration/ @paritytech/frame-coders @cheme -/frame/uniques/ @jsidorenko @paritytech/docs-audit - -# GRANDPA, BABE, consensus stuff -/client/consensus/babe/ @andresilva -/client/consensus/grandpa/ @andresilva -/client/consensus/pow/ @sorpaas -/client/consensus/slots/ @andresilva -/frame/babe/ @andresilva -/frame/grandpa/ @andresilva -/primitives/consensus/pow/ @sorpaas - -# BEEFY, MMR -/frame/beefy/ @acatangiu -/frame/beefy-mmr/ @acatangiu -/frame/merkle-mountain-range/ @acatangiu -/primitives/merkle-mountain-range/ @acatangiu - -# Contracts -/frame/contracts/ @athei @paritytech/docs-audit - -# NPoS and election -/frame/election-provider-multi-phase/ @paritytech/staking-core @paritytech/docs-audit -/frame/election-provider-support/ @paritytech/staking-core @paritytech/docs-audit -/frame/elections-phragmen/ @paritytech/staking-core @paritytech/docs-audit -/frame/nomination-pools/ @paritytech/staking-core @paritytech/docs-audit -/frame/staking/ @paritytech/staking-core @paritytech/docs-audit -/primitives/npos-elections/ @paritytech/staking-core @paritytech/docs-audit diff --git a/substrate/docs/README.adoc b/substrate/docs/README.adoc deleted file mode 100644 index 3537e346a66e1697004f2e114491c03b51ffd356..0000000000000000000000000000000000000000 --- a/substrate/docs/README.adoc +++ /dev/null @@ -1,522 +0,0 @@ -= Substrate -:Author: Substrate developers -:Revision: 0.2.0 -:toc: -:sectnums: - -== Intro in one sentence - -Substrate is a next-generation framework for blockchain innovation. - -== Description - -At its heart, Substrate is a combination of three technologies: https://webassembly.org/[WebAssembly], https://libp2p.io/[Libp2p] and GRANDPA Consensus. About GRANDPA, see this https://hackmd.io/Jd0byWX0RiqFiXUVC78Bdw?view#GRANDPA[definition], https://medium.com/polkadot-network/grandpa-block-finality-in-polkadot-an-introduction-part-1-d08a24a021b5[introduction] and https://github.com/w3f/consensus/blob/master/pdf/grandpa.pdf[formal specification]. It is both a library for building new blockchains and a "skeleton key" of a blockchain client, able to synchronize to any Substrate-based chain. - -Substrate chains have three distinct features that make them "next-generation": a dynamic, self-defining state-transition function; light-client functionality from day one; and a progressive consensus algorithm with fast block production and adaptive, definite finality. The STF, encoded in WebAssembly, is known as the "runtime". This defines the `execute_block` function, and can specify everything from the staking algorithm, transaction semantics, logging mechanisms and procedures for replacing any aspect of itself or of the blockchain's state ("governance"). Because the runtime is entirely dynamic all of these can be switched out or upgraded at any time. A Substrate chain is very much a "living organism". - -See also https://www.parity.io/what-is-substrate/. - -== Usage - -Substrate is still an early stage project, and while it has already been used as the basis of major projects like Polkadot, using it is still a significant undertaking. In particular, you should have a good knowledge of blockchain concepts and basic cryptography. Terminology like header, block, client, hash, transaction and signature should be familiar. At present you will need a working knowledge of Rust to be able to do anything interesting (though eventually, we aim for this not to be the case). - -Substrate is designed for use in one of three ways: - -**1. Trivial**: By running the Substrate binary `substrate` and configuring it with a genesis block that includes the current demonstration runtime. In this case, you just build Substrate, configure a JSON file, and launch your own blockchain. This affords you the least amount of customizability, primarily allowing you to change the genesis parameters of the various included runtime modules such as balances, staking, block-period, fees, and governance. - -**2. Modular**: By hacking together pallets built with Substrate FRAME into a new runtime and possibly altering or reconfiguring the Substrate client's block authoring logic. This affords you a very large amount of freedom over your blockchain's logic, letting you change data types, add or remove modules, and crucially, add your own modules. Much can be changed without touching the block authoring logic (since it is generic). If this is the case, then the existing Substrate binary can be used for block authoring and syncing. If the block authoring logic needs to be tweaked, then a new, altered block authoring binary must be built as a separate project and used by validators. This is how the Polkadot relay chain is built and should suffice for almost all circumstances in the near to mid-term. - -**3. Generic**: The entire FRAME can be ignored and the entire runtime designed and implemented from scratch. If desired, this can be done in a language other than Rust, provided it can target WebAssembly. If the runtime can be made compatible with the existing client's block authoring logic, then you can simply construct a new genesis block from your Wasm blob and launch your chain with the existing Rust-based Substrate client. If not, then you'll need to alter the client's block authoring logic accordingly. This is probably a useless option for most projects right now, but provides complete flexibility allowing for a long-term, far-reaching upgrade path for the Substrate paradigm. - -=== The Basics of Substrate - -Substrate is a blockchain platform with a completely generic state transition function. That said, it does come with both standards and conventions (particularly regarding the Runtime Module Library) regarding underlying data structures. Roughly speaking, these core data types correspond to +trait+s in terms of the actual non-negotiable standard and generic +struct+s in terms of the convention. - -``` -Header := Parent + ExtrinsicsRoot + StorageRoot + Digest -Block := Header + Extrinsics + Justifications -``` - -=== Extrinsics - -Extrinsics in Substrate are pieces of information from "the outside world" that are contained in the blocks of the chain. You might think "ahh, that means *transactions*": in fact, no. Extrinsics fall into two broad categories of which only one is *transactions*. The other is known as *inherents*. The difference between these two is that transactions are signed and gossiped on the network and can be deemed useful *per se*. This fits the mold of what you would call transactions in Bitcoin or Ethereum. - -Inherents, meanwhile, are not passed on the network and are not signed. They represent data which describes the environment but which cannot call upon anything to prove it such as a signature. Rather they are assumed to be "true" simply because a sufficiently large number of validators have agreed on them being reasonable. - -To give an example, there is the timestamp inherent, which sets the current timestamp of the block. This is not a fixed part of Substrate, but does come as part of FRAME to be used as desired. No signature could fundamentally prove that a block were authored at a given time in quite the same way that a signature can "prove" the desire to spend some particular funds. Rather, it is the business of each validator to ensure that they believe the timestamp is set to something reasonable before they agree that the block candidate is valid. - -Other examples include the parachain-heads extrinsic in Polkadot and the "note-missed-proposal" extrinsic used in FRAME to determine and punish or deactivate offline validators. - - -=== Runtime and API - -Substrate chains all have a runtime. The runtime is a WebAssembly "blob" that includes a number of entry-points. Some entry-points are required as part of the underlying Substrate specification. Others are merely convention and required for the default implementation of the Substrate client to be able to author blocks. - -If you want to develop a chain with Substrate, you will need to implement the `Core` trait. This `Core` trait generates an API with the minimum necessary functionality to interact with your runtime. A special macro is provided called `impl_runtime_apis!` that help you implement runtime API traits. All runtime API trait implementations need to be done in one call of the `impl_runtime_apis!` macro. All parameters and return values need to implement https://crates.io/crates/parity-codec[`parity-codec`] to be encodable and decodable. - -Here's a snippet of the Polkadot API implementation as of PoC-3: - -```rust -impl_runtime_apis! { - impl client_api::Core for Runtime { - fn version() -> RuntimeVersion { - VERSION - } - - fn execute_block(block: Block) { - Executive::execute_block(block) - } - - fn initialize_block(header: ::Header) { - Executive::initialize_block(&header) - } - } - // ---snip--- -} -``` - - -=== Inherent Extrinsics - -Substrate FRAME includes functionality for timestamps and slashing. If used, these rely on "trusted" external information being passed in via inherent extrinsics. The Substrate reference block authoring client software will expect to be able to call into the runtime API with collated data (in the case of the reference Substrate authoring client, this is merely the current timestamp and which nodes were offline) in order to return the appropriate extrinsics ready for inclusion. If new inherent extrinsic types and data are to be used in a modified runtime, then it is this function (and its argument type) that would change. - -=== Block-authoring Logic - -In Substrate, there is a major distinction between blockchain *syncing* and block *authoring* ("authoring" is a more general term for what is called "mining" in Bitcoin). The first case might be referred to as a "full node" (or "light node" - Substrate supports both): authoring necessarily requires a synced node and, therefore, all authoring clients must necessarily be able to synchronize. However, the reverse is not true. The primary functionality that authoring nodes have which is not in "sync nodes" is threefold: transaction queue logic, inherent transaction knowledge and BFT consensus logic. BFT consensus logic is provided as a core element of Substrate and can be ignored since it is only exposed in the SDK under the `authorities()` API entry. - -Transaction queue logic in Substrate is designed to be as generic as possible, allowing a runtime to express which transactions are fit for inclusion in a block through the `initialize_block` and `apply_extrinsic` calls. However, more subtle aspects like prioritization and replacement policy must currently be expressed "hard coded" as part of the blockchain's authoring code. That said, Substrate's reference implementation for a transaction queue should be sufficient for an initial chain implementation. - -Inherent extrinsic knowledge is again somewhat generic, and the actual construction of the extrinsics is, by convention, delegated to the "soft code" in the runtime. If ever there needs to be additional extrinsic information in the chain, then both the block authoring logic will need to be altered to provide it into the runtime and the runtime's `inherent_extrinsics` call will need to use this extra information in order to construct any additional extrinsic transactions for inclusion in the block. - -== Roadmap - -=== So far - -- 0.1 "PoC-1": PBFT consensus, Wasm runtime engine, basic runtime modules. -- 0.2 "PoC-2": Libp2p - -=== In progress - -- AfG consensus -- Improved PoS -- Smart contract runtime module - -=== The future - -- Splitting out runtime modules into separate repo -- Introduce substrate executable (the skeleton-key runtime) -- Introduce basic but extensible transaction queue and block-builder and place them in the executable. -- DAO runtime module -- Audit - -== Trying out Substrate Node - -Substrate Node is Substrate's pre-baked blockchain client. You can run a development node locally or configure a new chain and launch your own global testnet. - -=== On Mac and Ubuntu - -To get going as fast as possible, there is a simple script that installs all required dependencies and installs Substrate into your path. Just open a terminal and run: - -[source, shell] ----- -curl https://getsubstrate.io -sSf | bash ----- - -You can start a local Substrate development chain with running `substrate --dev`. - -To create your own global network/cryptocurrency, you'll need to make a new Substrate Node chain specification file ("chainspec"). - -First let's get a template chainspec that you can edit. We'll use the "staging" chain, a sort of default chain that the node comes pre-configured with: - -[source, shell] ----- -substrate build-spec --chain=staging > ~/chainspec.json ----- - -Now, edit `~/chainspec.json` in your editor. There are a lot of individual fields for each module, and one very large one which contains the WebAssembly code blob for this chain. The easiest field to edit is the block `period`. Change it to 10 (seconds): - -[source, json] ----- - "timestamp": { - "minimumPeriod": 10 - }, ----- - -Now with this new chainspec file, you can build a "raw" chain definition for your new chain: - -[source, shell] ----- -substrate build-spec --chain ~/chainspec.json --raw > ~/mychain.json ----- - -This can be fed into Substrate: - -[source, shell] ----- -substrate --chain ~/mychain.json ----- - -It won't do much until you start producing blocks though, so to do that you'll need to use the `--validator` option together with passing the seed for the account(s) that is configured to be the initial authorities: - -[source, shell] ----- -substrate --chain ~/mychain.json --validator ----- - -You can distribute `mychain.json` so that everyone can synchronize and (depending on your authorities list) validate on your chain. - - -== Building - -=== Hacking on Substrate - -If you'd actually like to hack on Substrate, you can just grab the source code and -build it. Ensure you have Rust and the support software installed: - -==== Linux and Mac - -For Unix-based operating systems, you should run the following commands: - -[source, shell] ----- -curl https://sh.rustup.rs -sSf | sh - -rustup update nightly -rustup target add wasm32-unknown-unknown --toolchain nightly -rustup update stable ----- - -You will also need to install the following packages: - - - Linux: -[source, shell] -sudo apt install cmake pkg-config libssl-dev git clang libclang-dev llvm - -- Linux on ARM: -`rust-lld` is required for linking wasm, but is missing on non Tier 1 platforms. -So, use this https://github.com/Plume-org/Plume/blob/master/script/wasm-deps.sh[script] -to build `lld` and create the symlink `/usr/bin/rust-lld` to the build binary. - - - Mac: -[source, shell] -brew install cmake pkg-config openssl git llvm - -To finish installation of Substrate, jump down to <>. - -==== Windows - -If you are trying to set up Substrate on Windows, you should do the following: - -1. First, you will need to download and install "Build Tools for Visual Studio:" - - * You can get it at this link: https://aka.ms/buildtools - * Run the installation file: `vs_buildtools.exe` - * Please ensure the Windows 10 SDK component is included when installing the Visual C++ Build Tools. - * image:https://i.imgur.com/zayVLmu.png[image] - * Restart your computer. - -2. Next, you need to install Rust: - - * Detailed instructions are provided by the https://doc.rust-lang.org/book/ch01-01-installation.html#installing-rustup-on-windows[Rust Book]. - * Download from: https://www.rust-lang.org/tools/install - * Run the installation file: `rustup-init.exe` - > Note that it should not prompt you to install vs_buildtools since you did it in step 1. - * Choose "Default Installation." - * To get started, you need Cargo's bin directory (%USERPROFILE%\.cargo\bin) in your PATH environment variable. Future applications will automatically have the correct environment, but you may need to restart your current shell. - -3. Then, you will need to run some commands in CMD to set up your Wasm Build Environment: - - rustup update nightly - rustup update stable - rustup target add wasm32-unknown-unknown --toolchain nightly - -4. Then, you need to install LLVM: https://releases.llvm.org/download.html - -5. Next, you need to install OpenSSL, which we will do with `vcpkg`: - - mkdir \Tools - cd \Tools - git clone https://github.com/Microsoft/vcpkg.git - cd vcpkg - .\bootstrap-vcpkg.bat - .\vcpkg.exe install openssl:x64-windows-static - -6. After, you need to add OpenSSL to your System Variables. Note that in order for the following commands to work, you need to use Windows Powershell: - - $env:OPENSSL_DIR = 'C:\Tools\vcpkg\installed\x64-windows-static' - $env:OPENSSL_STATIC = 'Yes' - [System.Environment]::SetEnvironmentVariable('OPENSSL_DIR', $env:OPENSSL_DIR, [System.EnvironmentVariableTarget]::User) - [System.Environment]::SetEnvironmentVariable('OPENSSL_STATIC', $env:OPENSSL_STATIC, [System.EnvironmentVariableTarget]::User) - -7. Finally, you need to install `cmake`: https://cmake.org/download/ - -==== Docker - -You can use https://github.com/paritytech/scripts/tree/master/dockerfiles/ci-linux[Parity CI docker image] with all necessary dependencies to build Substrate: - -[source, shell] ----- -#run it in the folder with the Substrate source code -docker run --rm -it -w /shellhere/substrate \ - -v $(pwd):/shellhere/substrate \ - paritytech/ci-linux:production ----- - -You can find necessary cargo commands in <> - -==== Shared Steps - -Then, grab the Substrate source code: - -[source, shell] ----- -git clone https://github.com/paritytech/substrate.git -cd substrate ----- - -Then build the code: - -[source, shell] ----- -cargo build # Builds all native code ----- - -You can run all the tests if you like: - -[source, shell] -cargo test --all - -Or just run the tests of a specific package (i.e. `cargo test -p pallet-assets`) - -You can start a development chain with: - -[source, shell] -cargo run --release -- --dev - -Detailed logs may be shown by running the node with the following environment variables set: `RUST_LOG=debug RUST_BACKTRACE=1 cargo run --release \-- --dev`. - -If you want to see the multi-node consensus algorithm in action locally, then you can create a local testnet with two validator nodes for Alice and Bob, who are the initial authorities of the genesis chain specification that have been endowed with a testnet DOTs. We'll give each node a name and expose them so they are listed on link:https://telemetry.polkadot.io/#/Local%20Testnet[Telemetry]. You'll need two terminal windows open. - -We'll start Alice's Substrate node first on default TCP port 30333 with their chain database stored locally at `/tmp/alice`. The Bootnode ID of Alice's node is `QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR`, which is generated from the `--node-key` value that we specify below: - -[source, shell] -cargo run --release \-- \ - --base-path /tmp/alice \ - --chain=local \ - --alice \ - --node-key 0000000000000000000000000000000000000000000000000000000000000001 \ - --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ - --validator - -In the second terminal, we'll run the following to start Bob's Substrate node on a different TCP port of 30334, and with their chain database stored locally at `/tmp/bob`. We'll specify a value for the `--bootnodes` option that will connect Bob's node to Alice's Bootnode ID on TCP port 30333: - -[source, shell] -cargo run --release \-- \ - --base-path /tmp/bob \ - --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR \ - --chain=local \ - --bob \ - --port 30334 \ - --telemetry-url 'ws://telemetry.polkadot.io:1024 0' \ - --validator - -Additional Substrate CLI usage options are available and may be shown by running `cargo run \-- --help`. - -[[flaming-fir]] -=== Joining the Flaming Fir Testnet - -Flaming Fir is the new testnet for Substrate master (2.0) to test the latest development features. Please note that master is not compatible with the BBQ Birch, Charred Cherry, Dried Danta or Emberic Elm testnets. Ensure you have the dependencies listed above before compiling. - -Since Flaming Fir is targeting the master branch we make absolutely no guarantees of stability and/or persistence of the network. We might reset the chain at any time if it is necessary to deploy new changes. Currently, the validators are running with a client built from `d013bd900`, if you build from this commit you should be able to successfully sync, later commits may not work as new breaking changes may be introduced in master. - -Latest known working version: `a2a0eb5398d6223e531455b4c155ef053a4a3a2b` - -[source, shell] ----- -git clone https://github.com/paritytech/substrate.git -cd substrate -git checkout -b flaming-fir a2a0eb5398d6223e531455b4c155ef053a4a3a2b ----- - -You can run the tests if you like: - -[source, shell] -cargo test --all - -Start your node: - -[source, shell] -cargo run --release \-- - -To see a list of command line options, enter: - -[source, shell] -cargo run --release \-- --help - -For example, you can choose a custom node name: - -[source, shell] -cargo run --release \-- --name my_custom_name - -If you are successful, you will see your node syncing at https://telemetry.polkadot.io/#/Flaming%20Fir - -=== Joining the Emberic Elm Testnet - -Emberic Elm is the testnet for Substrate 1.0. Please note that 1.0 is not compatible with the BBQ Birch, Charred Cherry, Dried Danta or Flaming Fir testnets. -In order to join the Emberic Elm testnet you should build from the `v1.0` branch. Ensure you have the dependencies listed above before compiling. - -[source, shell] ----- -git clone https://github.com/paritytech/substrate.git -cd substrate -git checkout -b v1.0 origin/v1.0 ----- - -You can then follow the same steps for building and running as described above in <>. - -== Key management - -Keys in Substrate are stored in the keystore in the file system. To store keys into this keystore, -you need to use one of the two provided RPC calls. If your keys are encrypted or should be encrypted -by the keystore, you need to provide the key using one of the cli arguments `--password`, -`--password-interactive` or `--password-filename`. - -=== Recommended RPC call - -For most users who want to run a validator node, the `author_rotateKeys` RPC call is sufficient. -The RPC call will generate `N` Session keys for you and return their public keys. `N` is the number -of session keys configured in the runtime. The output of the RPC call can be used as input for the -`session::set_keys` transaction. - -``` -curl -H 'Content-Type: application/json' --data '{ "jsonrpc":"2.0", "method":"author_rotateKeys", "id":1 }' localhost:9933 -``` - -=== Advanced RPC call - -If the Session keys need to match a fixed seed, they can be set individually key by key. The RPC call -expects the key seed and the key type. The key types supported by default in Substrate are listed -https://github.com/paritytech/substrate/blob/master/core/primitives/src/crypto.rs#L767[here], but the -user can declare any key type. - -``` -curl -H 'Content-Type: application/json' --data '{ "jsonrpc":"2.0", "method":"author_insertKey", "params":["KEY_TYPE", "SEED", "PUBLIC"],"id":1 }' localhost:9933 -``` - -`KEY_TYPE` - needs to be replaced with the 4-character key type identifier. -`SEED` - is the seed of the key. -`PUBLIC` - public key for the given key. - -== Documentation - -=== Viewing documentation for Substrate packages - -You can generate documentation for a Substrate Rust package and have it automatically open in your web browser using https://doc.rust-lang.org/rustdoc/what-is-rustdoc.html#using-rustdoc-with-cargo[rustdoc with Cargo], -(of the The Rustdoc Book), by running the following command: - -``` -cargo doc --package --open -``` - -Replacing `` with one of the following (i.e. `cargo doc --package substrate --open`): - -* All Substrate Packages -[source, shell] -substrate -* Substrate Core -[source, shell] -substrate, substrate-cli, substrate-client, substrate-client-db, -substrate-consensus-common, substrate-consensus-rhd, -substrate-executor, substrate-finality-grandpa, substrate-keyring, substrate-keystore, substrate-network, -substrate-network-libp2p, substrate-primitives, substrate-rpc, substrate-rpc-servers, -substrate-serializer, substrate-service, substrate-service-test, substrate-state-db, -substrate-state-machine, substrate-telemetry, substrate-test-client, -substrate-test-runtime, substrate-transaction-graph, sp-transaction-pool, -substrate-trie -* Substrate Runtime -[source, shell] -sr-api, sr-io, sr-primitives, sr-sandbox, sr-std, sr-version -* FRAME Core -[source, shell] -frame-metadata, frame-support, frame-system -* FRAME Pallets -[source, shell] -pallet-assets, pallet-balances, pallet-consensus, pallet-contracts, pallet-council, pallet-democracy, pallet-example, -frame-executive, pallet-session, pallet-staking, pallet-timestamp, pallet-treasury -* Node -[source, shell] -node-cli, node-consensus, node-executor, node-network, node-primitives, kitchensink-runtime -* Subkey -[source, shell] -subkey - -=== Contributing to documentation for Substrate packages - -https://doc.rust-lang.org/1.9.0/book/documentation.html[Document source code] for Substrate packages by annotating the source code with documentation comments. - -Example (generic): -```markdown -/// Summary -/// -/// Description -/// -/// # Panics -/// -/// # Errors -/// -/// # Safety -/// -/// # Examples -/// -/// Summary of Example 1 -/// -/// ```rust -/// // insert example 1 code here -/// ``` -/// -``` - -* Important notes: -** Documentation comments must use annotations with a triple slash `///` -** Modules are documented using `//!` -``` -//! Summary (of module) -//! -//! Description (of module) -``` -* Special section header is indicated with a hash `#`. -** `Panics` section requires an explanation if the function triggers a panic -** `Errors` section is for describing conditions under which a function of method returns `Err(E)` if it returns a `Result` -** `Safety` section requires an explanation if the function is `unsafe` -** `Examples` section includes examples of using the function or method -* Code block annotations for examples are included between triple graves, as shown above. -Instead of including the programming language to use for syntax highlighting as the annotation -after the triple graves, alternative annotations include the `ignore`, `text`, `should_panic`, or `no_run`. -* Summary sentence is a short high level single sentence of its functionality -* Description paragraph is for details additional to the summary sentence -* Missing documentation annotations may be used to identify where to generate warnings with `#![warn(missing_docs)]` -or errors `#![deny(missing_docs)]` -* Hide documentation for items with `#[doc(hidden)]` - -=== Contributing to documentation (tests, extended examples, macros) for Substrate packages - -The code block annotations in the `# Example` section may be used as https://doc.rust-lang.org/1.9.0/book/documentation.html#documentation-as-tests[documentation as tests and for extended examples]. - -* Important notes: -** Rustdoc will automatically add a `main()` wrapper around the code block to test it -** https://doc.rust-lang.org/1.9.0/book/documentation.html#documenting-macros[Documenting macros]. -** Documentation as tests examples are included when running `cargo test` - -== Contributing - -=== Contributing Guidelines - -include::CONTRIBUTING.md[] - -=== Contributor Code of Conduct - -include::CODE_OF_CONDUCT.md[] - -== License - -https://github.com/paritytech/substrate/blob/master/LICENSE[LICENSE] diff --git a/substrate/docs/SECURITY.md b/substrate/docs/SECURITY.md index 19f5b145feb5eb30447c0f8755aa19cabb081570..0d2064863d85c885448f2caea9a113e663d61598 100644 --- a/substrate/docs/SECURITY.md +++ b/substrate/docs/SECURITY.md @@ -1,11 +1,14 @@ # Security Policy -Parity Technologies is committed to resolving security vulnerabilities in our software quickly and carefully. We take the necessary steps to minimize risk, provide timely information, and deliver vulnerability fixes and mitigations required to address security issues. +Parity Technologies is committed to resolving security vulnerabilities in our software quickly and carefully. We take +the necessary steps to minimize risk, provide timely information, and deliver vulnerability fixes and mitigations +required to address security issues. ## Reporting a Vulnerability -Security vulnerabilities in Parity software should be reported by email to security@parity.io. If you think your report might be eligible for the Parity Bug Bounty Program, your email should be send to bugbounty@parity.io. +Security vulnerabilities in Parity software should be reported by email to security@parity.io. If you think your report +might be eligible for the Parity Bug Bounty Program, your email should be send to bugbounty@parity.io. Your report should include the following: @@ -16,11 +19,16 @@ Your report should include the following: - reproduction - other details -Try to include as much information in your report as you can, including a description of the vulnerability, its potential impact, and steps for reproducing it. Be sure to use a descriptive subject line. +Try to include as much information in your report as you can, including a description of the vulnerability, its +potential impact, and steps for reproducing it. Be sure to use a descriptive subject line. -You'll receive a response to your email within two business days indicating the next steps in handling your report. We encourage finders to use encrypted communication channels to protect the confidentiality of vulnerability reports. You can encrypt your report using our public key. This key is [on MIT's key server](https://pgp.mit.edu/pks/lookup?op=get&search=0x5D0F03018D07DE73) server and reproduced below. +You'll receive a response to your email within two business days indicating the next steps in handling your report. We +encourage finders to use encrypted communication channels to protect the confidentiality of vulnerability reports. You +can encrypt your report using our public key. This key is [on MIT's key +server](https://pgp.mit.edu/pks/lookup?op=get&search=0x5D0F03018D07DE73) server and reproduced below. -After the initial reply to your report, our team will endeavor to keep you informed of the progress being made towards a fix. These updates will be sent at least every five business days. +After the initial reply to your report, our team will endeavor to keep you informed of the progress being made towards a +fix. These updates will be sent at least every five business days. Thank you for taking the time to responsibly disclose any vulnerabilities you find. @@ -29,19 +37,23 @@ Thank you for taking the time to responsibly disclose any vulnerabilities you fi Responsible investigation and reporting includes, but isn't limited to, the following: - Don't violate the privacy of other users, destroy data, etc. -- Don’t defraud or harm Parity Technologies Ltd or its users during your research; you should make a good faith effort to not interrupt or degrade our services. -- Don't target our physical security measures, or attempt to use social engineering, spam, distributed denial of service (DDOS) attacks, etc. +- Don’t defraud or harm Parity Technologies Ltd or its users during your research; you should make a good faith effort + to not interrupt or degrade our services. +- Don't target our physical security measures, or attempt to use social engineering, spam, distributed denial of service + (DDOS) attacks, etc. - Initially report the bug only to us and not to anyone else. -- Give us a reasonable amount of time to fix the bug before disclosing it to anyone else, and give us adequate written warning before disclosing it to anyone else. -- In general, please investigate and report bugs in a way that makes a reasonable, good faith effort not to be disruptive or harmful to us or our users. Otherwise your actions might be interpreted as an attack rather than an effort to be helpful. +- Give us a reasonable amount of time to fix the bug before disclosing it to anyone else, and give us adequate written + warning before disclosing it to anyone else. +- In general, please investigate and report bugs in a way that makes a reasonable, good faith effort not to be + disruptive or harmful to us or our users. Otherwise your actions might be interpreted as an attack rather than an + effort to be helpful. ## Bug Bounty Program -Our Bug Bounty Program allows us to recognize and reward members of the Parity community for helping us find and address significant bugs, in accordance with the terms of the Parity Bug Bounty Program. A detailed description on eligibility, rewards, legal information and terms & conditions for contributors can be found on [our website](https://paritytech.io/bug-bounty.html). - - - - +Our Bug Bounty Program allows us to recognize and reward members of the Parity community for helping us find and address +significant bugs, in accordance with the terms of the Parity Bug Bounty Program. A detailed description on eligibility, +rewards, legal information and terms & conditions for contributors can be found on [our +website](https://paritytech.io/bug-bounty.html). ## Plaintext PGP Key diff --git a/substrate/docs/STYLE_GUIDE.md b/substrate/docs/STYLE_GUIDE.md index a89dcf52ffc0b7778cd3b2257b8470b307ae9c6e..6ea0755d0807d4452b6f2b1347bb13999ca37fe5 100644 --- a/substrate/docs/STYLE_GUIDE.md +++ b/substrate/docs/STYLE_GUIDE.md @@ -2,19 +2,18 @@ title: Style Guide for Rust in Substrate --- -Where possible these styles are enforced by settings in `rustfmt.toml` so if you run `cargo fmt` -then you will adhere to most of these style guidelines automatically. +Where possible these styles are enforced by settings in `rustfmt.toml` so if you run `cargo fmt` then you will adhere to +most of these style guidelines automatically. # Code Formatting -- Indent using tabs. -- Lines should be longer than 100 characters long only in exceptional circumstances and certainly - no longer than 120. For this purpose, tabs are considered 4 characters wide. -- Indent levels should be greater than 5 only in exceptional circumstances and certainly no - greater than 8. If they are greater than 5, then consider using `let` or auxiliary functions in - order to strip out complex inline expressions. -- Never have spaces on a line prior to a non-whitespace character -- Follow-on lines are only ever a single indent from the original line. +- Indent using tabs. +- Lines should be longer than 100 characters long only in exceptional circumstances and certainly no longer than 120. + For this purpose, tabs are considered 4 characters wide. +- Indent levels should be greater than 5 only in exceptional circumstances and certainly no greater than 8. If they are + greater than 5, then consider using `let` or auxiliary functions in order to strip out complex inline expressions. +- Never have spaces on a line prior to a non-whitespace character +- Follow-on lines are only ever a single indent from the original line. ```rust fn calculation(some_long_variable_a: i8, some_long_variable_b: i8) -> bool { @@ -25,8 +24,8 @@ fn calculation(some_long_variable_a: i8, some_long_variable_b: i8) -> bool { } ``` -- Indent level should follow open parens/brackets, but should be collapsed to the smallest number - of levels actually used: +- Indent level should follow open parens/brackets, but should be collapsed to the smallest number of levels actually + used: ```rust fn calculate( @@ -45,10 +44,10 @@ fn calculate( } ``` -- `where` is indented, and its items are indented one further. -- Argument lists or function invocations that are too long to fit on one line are indented - similarly to code blocks, and once one param is indented in such a way, all others should be, - too. Run-on parameter lists are also acceptable for single-line run-ons of basic function calls. +- `where` is indented, and its items are indented one further. +- Argument lists or function invocations that are too long to fit on one line are indented similarly to code blocks, and + once one param is indented in such a way, all others should be, too. Run-on parameter lists are also acceptable for + single-line run-ons of basic function calls. ```rust // OK @@ -92,7 +91,7 @@ fn foo(really_long_parameter_name_1: SomeLongTypeName, really_long_parameter_nam } ``` -- Always end last item of a multi-line comma-delimited set with `,` when legal: +- Always end last item of a multi-line comma-delimited set with `,` when legal: ```rust struct Point { @@ -104,7 +103,7 @@ struct Point { enum Meal { Breakfast, Lunch, Dinner }; ``` -- Avoid trailing `;`s where unneeded. +- Avoid trailing `;`s where unneeded. ```rust if condition { @@ -112,8 +111,8 @@ if condition { } ``` -- `match` arms may be either blocks or have a trailing `,` but not both. -- Blocks should not be used unnecessarily. +- `match` arms may be either blocks or have a trailing `,` but not both. +- Blocks should not be used unnecessarily. ```rust match meal { @@ -126,9 +125,8 @@ match meal { # Style -- Panickers require explicit proofs they don't trigger. Calling `unwrap` is discouraged. The - exception to this rule is test code. Avoiding panickers by restructuring code is preferred if - feasible. +- Panickers require explicit proofs they don't trigger. Calling `unwrap` is discouraged. The exception to this rule is + test code. Avoiding panickers by restructuring code is preferred if feasible. ```rust let mut target_path = @@ -139,21 +137,22 @@ let mut target_path = ); ``` -- Unsafe code requires explicit proofs just as panickers do. When introducing unsafe code, - consider trade-offs between efficiency on one hand and reliability, maintenance costs, and - security on the other. Here is a list of questions that may help evaluating the trade-off while - preparing or reviewing a PR: - - how much more performant or compact the resulting code will be using unsafe code, - - how likely is it that invariants could be violated, - - are issues stemming from the use of unsafe code caught by existing tests/tooling, - - what are the consequences if the problems slip into production. +- Unsafe code requires explicit proofs just as panickers do. When introducing unsafe code, consider trade-offs between + efficiency on one hand and reliability, maintenance costs, and security on the other. Here is a list of questions + that may help evaluating the trade-off while preparing or reviewing a PR: + - how much more performant or compact the resulting code will be using unsafe code, + - how likely is it that invariants could be violated, + - are issues stemming from the use of unsafe code caught by existing tests/tooling, + - what are the consequences if the problems slip into production. # Manifest Formatting -> **TLDR** -> You can use the CLI tool [Zepter](https://crates.io/crates/zepter) to format the files: `zepter format features` +> **TLDR** You can use the CLI tool [Zepter](https://crates.io/crates/zepter) to format the files: `zepter format +> features` -Rust `Cargo.toml` files need to respect certain formatting rules. All entries need to be alphabetically sorted. This makes it easier to read them and insert new entries. The exhaustive list of rules is enforced by the CI. The general format looks like this: +Rust `Cargo.toml` files need to respect certain formatting rules. All entries need to be alphabetically sorted. This +makes it easier to read them and insert new entries. The exhaustive list of rules is enforced by the CI. The general +format looks like this: - The feature is written as a single line if it fits within 80 chars: ```toml @@ -161,7 +160,8 @@ Rust `Cargo.toml` files need to respect certain formatting rules. All entries ne default = [ "std" ] ``` -- Otherwise the feature is broken down into multiple lines with one entry per line. Each line is padded with one tab and no trailing spaces but a trailing comma. +- Otherwise the feature is broken down into multiple lines with one entry per line. Each line is padded with one tab and + no trailing spaces but a trailing comma. ```toml [features] default = [ diff --git a/substrate/docs/Structure.adoc b/substrate/docs/Structure.adoc deleted file mode 100644 index 6c810a83c51b91367203ba67b532935e36c85146..0000000000000000000000000000000000000000 --- a/substrate/docs/Structure.adoc +++ /dev/null @@ -1,121 +0,0 @@ -= Structure -:Author: Substrate developers -:Revision: 0.3.0 -:toc: -:sectnums: - - -== Overview - -Substrate is split into multiple levels with increasing opinion and decreasing flexibility: - -* primitives -* client -* FRAME (formerly `srml`) - -Putting all these components together we have: - -* Integration Tests -* Node -* Node template -* Subkey - -=== Runtime - -* _found in_: `/primitives` -* _crates prefix_: `sp-` -* _constraints_: -** must be `[no_std]` -** crates may not (dev-)depend on crates in other subfolders of this repo - -In the lowest level, Substrate defines primitives, interfaces and traits to implement any on-chain Substrate transition system and its interactions with the outside world. This is the lowest level of abstraction and opinion that everything else builds upon. - -=== Client - -* _found in_: `/client` -* _crates prefix_: `sc-` -* _constraints_: -** crates may not (dev-)depend on any `frame-`-crates - -In the client you can find a set of crates to construct the outer substrate-node, implementing outer runtime interfaces, thus it depends on `runtime`. It provides the outer building blocks like transaction queue, networking layer, database backend, full* and light-client support. - -=== FRAME (formerly `srml`) - -* _found in_: `/frame` -* _crates prefix_: `frame-` and `pallet-` -* _constraints_: -** all crates that go on chain must be `[no_std]` -** must not (dev-)depend on anything in `/client` - -FRAME is a set of modules that implement specific transition functions and features one might want to have in their runtime. - -_Pallets_ are individual modules within _FRAME._ These are containers that host domain-specific logic. They have the `pallet-` prefix. For example, `pallet-staking` contains logic for staking tokens. - -There are a few crates with the `frame-` prefix. These do not contain domain-specific logic. Rather, they are the main FRAME support infrastructure. These are: - -- Executive -- Metadata -- Support -- System -- Utility - -=== Integration Tests - -* _found in_: `/test` -* _crates prefix_: `substrate-test` -* _constraints_: -** only helpers may be published -** purely testing crates must be `publish = false` - -All tests that have to pull (dev)-dependencies out of their subtree and would thus break the dependency rules are considered integration tests and should be stored in here. Only helper-crates in here shall be published, everything else is expected to be non-publish. - -=== Binaries and template - -* _found in_: `/bin` - -We also provide some binaries pulling from the components creating full applications. - -==== Node - -* _found in_: `/bin/node` - -The default (testing) application pulling together our recommended setup of substrate-client with a wasm-contracts-supporting frame-runtime. The node pulls it all together, constructs the (upgradable) runtime, and wires up the client around it. You can find an example client, which includes a full wasm-contracts chain in `node`. This is also what is being built and run if you do `cargo run`. - -==== Node Template - -* _found in_: `/bin/node-template` - -We also provide a template to get you started building your own node. - -==== Utils - -* _found in_: `/bin/utils` - -- **subkey** - Subkey is a client library to generate keys and sign transactions to send to a substrate node. -- **chain-spec-builder** - The chain spec builder builds a chain specification that includes a Substrate runtime compiled as WASM. To ensure proper functioning of the included runtime compile (or run) the chain spec builder binary in `--release` mode. - -== Internal Dependency Tree - -[ditaa] -.... -+---------------+ +----------------+ -| | | | -| runtime +<------+ frame | -| | | | -+------+-----+--+ +-------------+--+ - ^ ^ ^ - | +----------------+ | - | | | -+------+--------+ | | -| | | | -| client | +--+-------+--------+ -| +<---------+ | -+---------------+ | | - | test /bin/* | - | | - | | - +-------------------+ - -.... diff --git a/substrate/docs/Upgrade.md b/substrate/docs/Upgrade.md index 4908d53f579e4028bff5615947a86b285f8f8039..08b5a7d37b647d43cb2b4f39210a4ff8d95bbb86 100644 --- a/substrate/docs/Upgrade.md +++ b/substrate/docs/Upgrade.md @@ -1,5 +1,7 @@ -# Upgrade path for you building on substrate +# Upgrade path for you building on Substrate ## master - - crate rename has been fixed `sp-application-crypto` (was `sc-application-crypto`); `.maintain/rename-crates-for-2.0.sh` has been updated accordingly, you can use it to upgrade to latest naming convention - - crates have been renamed, run `bash .maintain/rename-crates-for-2.0.sh` \ No newline at end of file + - crate rename has been fixed `sp-application-crypto` (was `sc-application-crypto`); + `.maintain/rename-crates-for-2.0.sh` has been updated accordingly, you can use it to upgrade to latest naming + convention + - crates have been renamed, run `bash .maintain/rename-crates-for-2.0.sh` diff --git a/substrate/docs/Upgrading-2.0-to-3.0.md b/substrate/docs/Upgrading-2.0-to-3.0.md index 906018db9a707b094b6bc051dedc3aa0be4d9ff3..58066ce074de63c83ea1a1def1273e67d79be1b4 100644 --- a/substrate/docs/Upgrading-2.0-to-3.0.md +++ b/substrate/docs/Upgrading-2.0-to-3.0.md @@ -4,7 +4,8 @@ An incomplete guide. ## Refreshing the node-template -Not much has changed on the top and API level for developing Substrate between 2.0 and 3.0. If you've made only small changes to the node-template, we recommend to do the following - it is easiest and quickest path forward: +Not much has changed on the top and API level for developing Substrate between 2.0 and 3.0. If you've made only small +changes to the node-template, we recommend to do the following - it is easiest and quickest path forward: 1. take a diff between 2.0 and your changes 2. store that diff 3. remove everything, copy over the 3.0 node-template @@ -12,19 +13,30 @@ Not much has changed on the top and API level for developing Substrate between 2 ## In-Depth guide on the changes -If you've made significant changes or diverted from the node-template a lot, starting out with that is probably not helping. For that case, we'll take a look at all changes between 2.0 and 3.0 to the fully-implemented node and explain them one by one, so you can follow up, what needs to be changing for your node. +If you've made significant changes or diverted from the node-template a lot, starting out with that is probably not +helping. For that case, we'll take a look at all changes between 2.0 and 3.0 to the fully-implemented node and explain +them one by one, so you can follow up, what needs to be changing for your node. _Note_: Of course, step 1 is to upgrade your `Cargo.toml`'s to use the latest version of Substrate and all dependencies. -We'll be taking the diff from 2.0.1 to 3.0.0 on `bin/node` as the baseline of what has changed between these two versions in terms of adapting ones code base. We will not be covering the changes made on the tests and bench-marking as they are mostly reactions to the other changes. +We'll be taking the diff from 2.0.1 to 3.0.0 on `bin/node` as the baseline of what has changed between these two +versions in terms of adapting ones code base. We will not be covering the changes made on the tests and bench-marking as +they are mostly reactions to the other changes. ### Versions upgrade -First and foremost you have to upgrade the version pf the dependencies of course, that's `0.8.x -> 0.9.0` and `2.0.x -> 3.0.0` for all `sc-`, `sp-`, `frame-`, and `pallet-` coming from Parity. Further more this release also upgraded its own dependencies, most notably, we are now using `parity-scale-codec 2.0`, `parking_lot 0.11` and `substrate-wasm-builder 3.0.0` (as build dependency). All other dependency upgrades should resolve automatically or are just internal. However you might see some error that another dependency/type you have as a dependency and one of our upgraded crates don't match up, if so please check the version of said dependency - we've probably upgraded it. +First and foremost you have to upgrade the version pf the dependencies of course, that's `0.8.x -> 0.9.0` and `2.0.x -> +3.0.0` for all `sc-`, `sp-`, `frame-`, and `pallet-` coming from Parity. Further more this release also upgraded its own +dependencies, most notably, we are now using `parity-scale-codec 2.0`, `parking_lot 0.11` and `substrate-wasm-builder +3.0.0` (as build dependency). All other dependency upgrades should resolve automatically or are just internal. However +you might see some error that another dependency/type you have as a dependency and one of our upgraded crates don't +match up, if so please check the version of said dependency - we've probably upgraded it. ### WASM-Builder -The new version of wasm-builder has gotten a bit smarter and a lot faster (you should definitely switch). Once you've upgraded the dependency, in most cases you just have to remove the now obsolete `with_wasm_builder_from_crates_or_path`-function and you are good to go: +The new version of wasm-builder has gotten a bit smarter and a lot faster (you should definitely switch). Once you've +upgraded the dependency, in most cases you just have to remove the now obsolete +`with_wasm_builder_from_crates_or_path`-function and you are good to go: ```diff: rust --- a/bin/node/runtime/build.rs @@ -49,11 +61,15 @@ The new version of wasm-builder has gotten a bit smarter and a lot faster (you s #### FRAME 2.0 -The new FRAME 2.0 macros are a lot nicer to use and easier to read. While we were on that change though, we also cleaned up some mainly internal names and traits. The old `macro`'s still work and also produce the new structure, however, when plugging all that together as a Runtime, there's some things we have to adapt now: +The new FRAME 2.0 macros are a lot nicer to use and easier to read. While we were on that change though, we also cleaned +up some mainly internal names and traits. The old `macro`'s still work and also produce the new structure, however, when +plugging all that together as a Runtime, there's some things we have to adapt now: ##### `::Trait for Runtime` becomes `::Config for Runtime` -The most visible and significant change is that the macros no longer generate the `$pallet::Trait` but now a much more aptly named `$pallet::Config`. Thus, we need to rename all `::Trait for Runtime` into`::Config for Runtime`, e.g. for the `sudo` pallet we must do: +The most visible and significant change is that the macros no longer generate the `$pallet::Trait` but now a much more +aptly named `$pallet::Config`. Thus, we need to rename all `::Trait for Runtime` into`::Config for Runtime`, e.g. for +the `sudo` pallet we must do: ```diff -impl pallet_sudo::Trait for Runtime { @@ -65,11 +81,15 @@ The same goes for all `` and alike, which simply be #### SS58 Prefix is now a runtime param -Since [#7810](https://github.com/paritytech/substrate/pull/7810) we don't define the ss58 prefix in the chainspec anymore but moved it into the runtime. Namely, `frame_system` now needs a new `SS58Prefix`, which in substrate node we have defined for ourselves as: `pub const SS58Prefix: u8 = 42;`. Use your own chain-specific value there. +Since [#7810](https://github.com/paritytech/substrate/pull/7810) we don't define the ss58 prefix in the chainspec +anymore but moved it into the runtime. Namely, `frame_system` now needs a new `SS58Prefix`, which in Substrate node we +have defined for ourselves as: `pub const SS58Prefix: u8 = 42;`. Use your own chain-specific value there. #### Weight Definition -`type WeightInfo` has changed and instead on `weights::pallet_$name::WeightInfo` is now bound to the Runtime as `pallet_$name::weights::SubstrateWeight`. As a result we have to the change the type definitions everywhere in our Runtime accordingly: +`type WeightInfo` has changed and instead on `weights::pallet_$name::WeightInfo` is now bound to the Runtime as +`pallet_$name::weights::SubstrateWeight`. As a result we have to the change the type definitions everywhere in +our Runtime accordingly: ```diff - type WeightInfo = weights::pallet_$name::WeightInfo; @@ -170,24 +190,29 @@ And update the overall definition for weights on frame and a few related types a + type SystemWeightInfo = frame_system::weights::SubstrateWeight; ``` -#### Pallets: +#### Pallets ##### Assets The assets pallet has seen a variety of changes: -- [Features needed for reserve-backed stablecoins #7152 ](https://github.com/paritytech/substrate/pull/7152) -- [Freeze Assets and Asset Metadata #7346 ](https://github.com/paritytech/substrate/pull/7346) -- [Introduces account existence providers reference counting #7363 ]((https://github.com/paritytech/substrate/pull/7363)) +- [Features needed for reserve-backed stablecoins #7152](https://github.com/paritytech/substrate/pull/7152) +- [Freeze Assets and Asset Metadata #7346](https://github.com/paritytech/substrate/pull/7346) +- [Introduces account existence providers reference counting #7363]((https://github.com/paritytech/substrate/pull/7363)) -have all altered the feature set and changed the concepts. However, it has some of the best documentation and explains the current state very well. If you are using the assets pallet and need to upgrade from an earlier version, we recommend you use the current docs to guide your way! +have all altered the feature set and changed the concepts. However, it has some of the best documentation and explains +the current state very well. If you are using the assets pallet and need to upgrade from an earlier version, we +recommend you use the current docs to guide your way! ##### Contracts -As noted in the changelog, the `contracts`-pallet is still undergoing massive changes and is not yet part of this release. We are expecting for it to be released a few weeks after. If your chain is dependent on this pallet, we recommend to wait until it has been released as the currently released version is not compatible with FRAME 2.0. +As noted in the changelog, the `contracts`-pallet is still undergoing massive changes and is not yet part of this +release. We are expecting for it to be released a few weeks after. If your chain is dependent on this pallet, we +recommend to wait until it has been released as the currently released version is not compatible with FRAME 2.0. #### (changes) Treasury -As mentioned above, Bounties, Tips and Lottery have been extracted out of treasury into their own pallets - removing these options here. Secondly we must now specify the `BurnDestination` and `SpendFunds`, which now go the `Bounties`. +As mentioned above, Bounties, Tips and Lottery have been extracted out of treasury into their own pallets - removing +these options here. Secondly we must now specify the `BurnDestination` and `SpendFunds`, which now go the `Bounties`. ```diff - type Tippers = Elections; @@ -206,9 +231,10 @@ As mentioned above, Bounties, Tips and Lottery have been extracted out of treasu + type SpendFunds = Bounties; ``` -Factoring out Bounties and Tips means most of these definitions have now moved there, while the parameter types can be left as they were: +Factoring out Bounties and Tips means most of these definitions have now moved there, while the parameter types can be +left as they were: -###### 🆕 Bounties +##### 🆕 Bounties ```rust= impl pallet_bounties::Config for Runtime { @@ -241,11 +267,15 @@ impl pallet_tips::Config for Runtime { #### `FinalityTracker` removed -Finality Tracker has been removed in favor of a different approach to handle the issue in GRANDPA, [see #7228 for details](https://github.com/paritytech/substrate/pull/7228). With latest GRANDPA this is not needed anymore and can be removed without worry. +Finality Tracker has been removed in favor of a different approach to handle the issue in GRANDPA, [see #7228 for +details](https://github.com/paritytech/substrate/pull/7228). With latest GRANDPA this is not needed anymore and can be +removed without worry. #### (changes) Elections Phragmen -The pallet has been moved to a new system in which the exact amount of deposit for each voter, candidate, member, or runner-up is now deposited on-chain. Moreover, the concept of a `defunct_voter` is removed, since votes now have adequate deposit associated with them. A number of configuration parameters has changed to reflect this, as shown below: +The pallet has been moved to a new system in which the exact amount of deposit for each voter, candidate, member, or +runner-up is now deposited on-chain. Moreover, the concept of a `defunct_voter` is removed, since votes now have +adequate deposit associated with them. A number of configuration parameters has changed to reflect this, as shown below: ```diff= parameter_types! { @@ -277,11 +307,16 @@ The pallet has been moved to a new system in which the exact amount of deposit f type TermDuration = TermDuration; ``` - **This upgrade requires storage [migration](https://github.com/paritytech/substrate/blob/master/frame/elections-phragmen/src/migrations_3_0_0.rs)**. Further details can be found in the [pallet-specific changelog](https://github.com/paritytech/substrate/blob/master/frame/elections-phragmen/CHANGELOG.md#security). + **This upgrade requires storage + [migration](https://github.com/paritytech/substrate/blob/master/frame/elections-phragmen/src/migrations_3_0_0.rs)**. + Further details can be found in the [pallet-specific + changelog](https://github.com/paritytech/substrate/blob/master/frame/elections-phragmen/CHANGELOG.md#security). #### (changes) Democracy -Democracy brings three new settings with this release, all to allow for better influx- and spam-control. Namely these allow to specify the maximum number of proposals at a time, who can blacklist and who can cancel proposals. This diff acts as a good starting point: +Democracy brings three new settings with this release, all to allow for better influx- and spam-control. Namely these +allow to specify the maximum number of proposals at a time, who can blacklist and who can cancel proposals. This diff +acts as a good starting point: ```diff= @@ -508,6 +537,14 @@ impl pallet_democracy::Trait for Runtime { @@ -311,7 +346,9 @@ Democracy brings three new settings with this release, all to allow for better i ### Primitives -The shared primitives define the API between Client and Runtime. Usually, you don't have to touch nor directly interact with them, unless you created your own client or frame-less runtime. Therefore we'd expect you to understand whether you are effected by changes and how to update your code yourself. +The shared primitives define the API between Client and Runtime. Usually, you don't have to touch nor directly interact +with them, unless you created your own client or frame-less runtime. Therefore we'd expect you to understand whether you +are effected by changes and how to update your code yourself. ---- @@ -321,10 +358,17 @@ The shared primitives define the API between Client and Runtime. Usually, you do A few minor things have changed in the `cli` (compared to 2.0.1): -1. we've [replaced the newly added `BuildSyncSpec` subcommand with an RPC API](https://github.com/paritytech/substrate/commit/65cc9af9b8df8d36928f6144ee7474cefbd70454#diff-c57da6fbeff8c46ce15f55ea42fedaa5a4684d79578006ce4af01ae04fd6b8f8) in an on-going effort to make light-client-support smoother, see below -2. we've [removed double accounts from our chainspec-builder](https://github.com/paritytech/substrate/commit/31499cd29ed30df932fb71b7459796f7160d0272) -3. we [don't fallback to `--chain flaming-fir` anymore](https://github.com/paritytech/substrate/commit/13cdf1c8cd2ee62d411f82b64dc7eba860c9c6c6), if no chain is given our substrate-node will error. -4. [the `subkey`-integration has seen a fix to the `insert`-command](https://github.com/paritytech/substrate/commit/54bde60cfd2c544c54e9e8623b6b8725b99557f8) that requires you to now add the `&cli` as a param. +1. we've [replaced the newly added `BuildSyncSpec` subcommand with an RPC + API](https://github.com/paritytech/substrate/commit/65cc9af9b8df8d36928f6144ee7474cefbd70454#diff-c57da6fbeff8c46ce15f55ea42fedaa5a4684d79578006ce4af01ae04fd6b8f8) + in an on-going effort to make light-client-support smoother, see below +2. we've [removed double accounts from our + chainspec-builder](https://github.com/paritytech/substrate/commit/31499cd29ed30df932fb71b7459796f7160d0272) +3. we [don't fallback to `--chain flaming-fir` + anymore](https://github.com/paritytech/substrate/commit/13cdf1c8cd2ee62d411f82b64dc7eba860c9c6c6), if no chain is + given our `substrate-node` will error. +4. [the `subkey`-integration has seen a fix to the + `insert`-command](https://github.com/paritytech/substrate/commit/54bde60cfd2c544c54e9e8623b6b8725b99557f8) that + requires you to now add the `&cli` as a param. ```diff= --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -344,7 +388,8 @@ A few minor things have changed in the `cli` (compared to 2.0.1): ##### Light client support -As said, we've added a new optional RPC service for improved light client support. For that to work, we need to pass the `chain_spec` and give access to the `AuxStore` to our `rpc`: +As said, we've added a new optional RPC service for improved light client support. For that to work, we need to pass the +`chain_spec` and give access to the `AuxStore` to our `rpc`: ```diff= @@ -438,17 +483,30 @@ and add the new service: ##### Telemetry -The telemetry subsystem has seen a few fixes and refactorings to allow for a more flexible handling, in particular in regards to parachains. Most notably `sc_service::spawn_tasks` now returns the `telemetry_connection_notifier` as the second member of the tuple, (`let (_rpc_handlers, telemetry_connection_notifier) = sc_service::spawn_tasks(`), which should be passed to `telemetry_on_connect` of `new_full_base` now: `telemetry_on_connect: telemetry_connection_notifier.map(|x| x.on_connect_stream()),` (see the service-section below for a full diff). +The telemetry subsystem has seen a few fixes and refactorings to allow for a more flexible handling, in particular in +regards to parachains. Most notably `sc_service::spawn_tasks` now returns the `telemetry_connection_notifier` as the +second member of the tuple, (`let (_rpc_handlers, telemetry_connection_notifier) = sc_service::spawn_tasks(`), which +should be passed to `telemetry_on_connect` of `new_full_base` now: `telemetry_on_connect: +telemetry_connection_notifier.map(|x| x.on_connect_stream()),` (see the service-section below for a full diff). ##### Async & Remote Keystore support -In order to allow for remote-keystores, the keystore-subsystem has been reworked to support async operations and generally refactored to not provide the keys itself but only sign on request. This allows for remote-keystore to never hand out keys and thus to operate any substrate-based node in a manner without ever having the private keys in the local system memory. +In order to allow for remote-keystores, the keystore-subsystem has been reworked to support async operations and +generally refactored to not provide the keys itself but only sign on request. This allows for remote-keystore to never +hand out keys and thus to operate any Substrate-based node in a manner without ever having the private keys in the local +system memory. -There are some operations, however, that the keystore must be local for performance reasons and for which a remote keystore won't work (in particular around parachains). As such, the keystore has both a slot for remote but also always a local instance, where some operations hard bind to the local variant, while most subsystems just ask the generic keystore which prefers a remote signer if given. To reflect this change, `sc_service::new_full_parts` now returns a `KeystoreContainer` rather than the keystore, and the other subsystems (e.g. `sc_service::PartialComponents`) expect to be given that. +There are some operations, however, that the keystore must be local for performance reasons and for which a remote +keystore won't work (in particular around parachains). As such, the keystore has both a slot for remote but also always +a local instance, where some operations hard bind to the local variant, while most subsystems just ask the generic +keystore which prefers a remote signer if given. To reflect this change, `sc_service::new_full_parts` now returns a +`KeystoreContainer` rather than the keystore, and the other subsystems (e.g. `sc_service::PartialComponents`) expect to +be given that. -###### on RPC: +###### on RPC -This has most visible changes for the rpc, where we are switching from the previous `KeyStorePtr` to the new `SyncCryptoStorePtr`: +This has most visible changes for the rpc, where we are switching from the previous `KeyStorePtr` to the new +`SyncCryptoStorePtr`: ```diff @@ -483,9 +541,15 @@ This has most visible changes for the rpc, where we are switching from the previ ##### GRANDPA -As already in the changelog, a few things significant things have changed in regards to GRANDPA: the finality tracker has been replaced, an RPC command has been added and WARP-sync-support for faster light client startup has been implemented. All this means we have to do a few changes to our GRANDPA setup procedures in the client. +As already in the changelog, a few things significant things have changed in regards to GRANDPA: the finality tracker +has been replaced, an RPC command has been added and WARP-sync-support for faster light client startup has been +implemented. All this means we have to do a few changes to our GRANDPA setup procedures in the client. -First and foremost, grandpa internalised a few aspects, and thus `new_partial` doesn't expect a tuple but only the `grandpa::SharedVoterState` as input now, and unpacking that again later is not needed anymore either. On the opposite side `grandpa::FinalityProofProvider::new_for_service` now requires the `Some(shared_authority_set)` to be passed as a new third parameter. This set also becomes relevant when adding warp-sync-support, which is added as an extra-protocol-layer to the networking as: +First and foremost, grandpa internalised a few aspects, and thus `new_partial` doesn't expect a tuple but only the +`grandpa::SharedVoterState` as input now, and unpacking that again later is not needed anymore either. On the opposite +side `grandpa::FinalityProofProvider::new_for_service` now requires the `Some(shared_authority_set)` to be passed as a +new third parameter. This set also becomes relevant when adding warp-sync-support, which is added as an +extra-protocol-layer to the networking as: ```diff= + config.network.extra_sets.push(grandpa::grandpa_peers_set_config()); @@ -496,11 +560,13 @@ First and foremost, grandpa internalised a few aspects, and thus `new_partial` d + )); ``` -As these changes pull through the entirety of `cli/src/service.rs`, we recommend looking at the final diff below for guidance. +As these changes pull through the entirety of `cli/src/service.rs`, we recommend looking at the final diff below for +guidance. ##### In a nutshell -Altogether this accumulates to the following diff for `node/cli/src/service.rs`. If you want these features and have modified your chain you should probably try to apply these patches: +Altogether this accumulates to the following diff for `node/cli/src/service.rs`. If you want these features and have +modified your chain you should probably try to apply these patches: ```diff= diff --git a/substrate/docs/node-template-release.md b/substrate/docs/node-template-release.md index 911e6a2bbe71aa370f34648d75f77c62ccc5a795..0acaf3bdc61100d1fc5311f95c15a7cf897dd4f2 100644 --- a/substrate/docs/node-template-release.md +++ b/substrate/docs/node-template-release.md @@ -1,71 +1,65 @@ # Substrate Node Template Release Process -1. This release process has to be run in a github checkout Substrate directory with your work -committed into `https://github.com/paritytech/substrate/`, because the build script will check -the existence of your current git commit ID in the remote repository. +## This release process has to be run in a github checkout Substrate directory with your work committed into +`https://github.com/paritytech/substrate/`, because the build script will check the existence of your current git commit +ID in the remote repository. - Assume you are in root directory of Substrate. Run: +Assume you are in root directory of Substrate. Run: - ```bash - cd scripts/ci/ - ./node-template-release.sh - ``` +```bash +cd scripts/ci/ ./node-template-release.sh +``` -2. Expand the output tar gzipped file and replace files in current Substrate Node Template -by running the following command. +## Expand the output tar gzipped file and replace files in current Substrate Node Template by running the following +command. - ```bash - # This is where the tar.gz file uncompressed - cd substrate-node-template - # rsync with force copying. Note the slash at the destination directory is important - rsync -avh * / - # For dry-running add `-n` argument - # rsync -avhn * / - ``` +```bash +# This is where the tar.gz file uncompressed cd substrate-node-template # rsync with force copying. Note the +slash at the destination directory is important rsync -avh * / # For dry-running +add `-n` argument # rsync -avhn * / +``` - The above command only copies existing files from the source to the destination, but does not - delete files/directories that are removed from the source. So you need to manually check and - remove them in the destination. +The above command only copies existing files from the source to the destination, but does not delete files/directories +that are removed from the source. So you need to manually check and remove them in the destination. -3. There is a `Cargo.toml` file in the root directory. Inside, dependencies are listed form and -linked to a certain git commit in Substrate remote repository, such as: +## There is a `Cargo.toml` file in the root directory. Inside, dependencies are listed form and linked to a certain git +commit in Substrate remote repository, such as: - ```toml - sp-core = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", rev = "de80d0107336a9c7a2efdc0199015e4d67fcbdb5", default-features = false } - ``` +```toml +toml sp-core = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", rev = +"de80d0107336a9c7a2efdc0199015e4d67fcbdb5", default-features = false } +``` - We will update each of them to link to the Rust [crate registry](https://crates.io/). -After confirming the versioned package is published in the crate, the above will become: +e will update each of them to link to the Rust [crate registry](https://crates.io/). After confirming the versioned +package is published in the crate, the above will become: - ```toml - [workspace.dependencies] - sp-core = { version = "7.0.0", default-features = false } - ``` +```toml +[workspace.dependencies] sp-core = { version = "7.0.0", default-features = false } +``` - P.S: This step can be automated if we update `node-template-release` package in - `scripts/ci/node-template-release`. +P.S: This step can be automated if we update `node-template-release` package in `scripts/ci/node-template-release`. -4. Once the `Cargo.toml` is updated, compile and confirm that the Node Template builds. Then commit -the changes to a new branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR. +## Once the `Cargo.toml` is updated, compile and confirm that the Node Template builds. Then commit the changes to a new +branch in [Substrate Node Template](https://github.com/substrate-developer-hub/substrate-node-template), and make a PR. - > Note that there is a chance the code in Substrate Node Template works with the linked Substrate git - commit but not with published packages due to the latest (as yet) unpublished features. In this case, - rollback that section of the Node Template to its previous version to ensure the Node Template builds. +> Note that there is a chance the code in Substrate Node Template works with the linked Substrate git commit but not +with published packages due to the latest (as yet) unpublished features. In this case, rollback that section of the +Node Template to its previous version to ensure the Node Template builds. -5. Once the PR is merged, tag the merged commit in master branch with the version number -`vX.Y.Z+A` (e.g. `v3.0.0+1`). The `X`(major), `Y`(minor), and `Z`(patch) version number should -follow Substrate release version. The last digit is any significant fixes made in the Substrate -Node Template apart from Substrate. When the Substrate version is updated, this digit is reset to 0. +## Once the PR is merged, tag the merged commit in master branch with the version number `vX.Y.Z+A` (e.g. `v3.0.0+1`) +The `X`(major), `Y`(minor), and `Z`(patch) version number should follow Substrate release version. The last digit is any +significant fixes made in the Substrate Node Template apart from Substrate. When the Substrate version is updated, this +digit is reset to 0. ## Troubleshooting -- Running the script `./node-template-release.sh `, after all tests passed - successfully, seeing the following error message: +- Running the script `./node-template-release.sh `, after all tests passed successfully, seeing the + following error message: - ``` - thread 'main' panicked at 'Creates output file: Os { code: 2, kind: NotFound, message: "No such file or directory" }', src/main.rs:250:10 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ``` +``` +thread 'main' panicked at 'Creates output file: Os { code: 2, kind: NotFound, message: "No such file or directory" +}', src/main.rs:250:10 note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +``` - This is likely due to that your output path is not a valid `tar.gz` filename or you don't have write - permission to the destination. Try with a simple output path such as `~/node-tpl.tar.gz`. +This is likely due to that your output path is not a valid `tar.gz` filename or you don't have write permission to the +destination. Try with a simple output path such as `~/node-tpl.tar.gz`. diff --git a/substrate/frame/README.md b/substrate/frame/README.md index 47a7892c2c8d0bb702453cbecf153f62dc1d5b9d..0a6c01fd035dffb53426d1718b9638258207837b 100644 --- a/substrate/frame/README.md +++ b/substrate/frame/README.md @@ -1,11 +1,12 @@ # FRAME -The FRAME development environment provides modules (called "pallets") and support libraries that you can use, modify, and extend to build the runtime logic to suit the needs of your blockchain. +The FRAME development environment provides modules (called "pallets") and support libraries that you can use, modify, +and extend to build the runtime logic to suit the needs of your blockchain. -### Documentation +## Documentation https://docs.substrate.io/reference/frame-pallets/ -### Issues +## Issues https://github.com/orgs/paritytech/projects/40 diff --git a/substrate/frame/asset-conversion/src/tests.rs b/substrate/frame/asset-conversion/src/tests.rs index 190e4fb621476924377f047205932abac587d68d..1c1267ab87b3fad874f6a06a3a917a81ac629d2d 100644 --- a/substrate/frame/asset-conversion/src/tests.rs +++ b/substrate/frame/asset-conversion/src/tests.rs @@ -569,6 +569,16 @@ fn can_quote_price() { ), Some(60) ); + // including fee so should get less out... + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 3000, + true, + ), + Some(46) + ); // Check it still gives same price: // (if the above accidentally exchanged then it would not give same quote as before) assert_eq!( @@ -580,6 +590,16 @@ fn can_quote_price() { ), Some(60) ); + // including fee so should get less out... + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 3000, + true, + ), + Some(46) + ); // Check inverse: assert_eq!( @@ -591,6 +611,247 @@ fn can_quote_price() { ), Some(3000) ); + // including fee so should get less out... + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + 60, + true, + ), + Some(2302) + ); + + // + // same tests as above but for quote_price_tokens_for_exact_tokens: + // + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 60, + false, + ), + Some(3000) + ); + // including fee so should need to put more in... + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 60, + true, + ), + Some(4299) + ); + // Check it still gives same price: + // (if the above accidentally exchanged then it would not give same quote as before) + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 60, + false, + ), + Some(3000) + ); + // including fee so should need to put more in... + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + 60, + true, + ), + Some(4299) + ); + + // Check inverse: + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + 3000, + false, + ), + Some(60) + ); + // including fee so should need to put more in... + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + 3000, + true, + ), + Some(86) + ); + + // + // roundtrip: Without fees one should get the original number + // + let amount_in = 100; + + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + amount_in, + false, + ) + .and_then(|amount| AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + amount, + false, + )), + Some(amount_in) + ); + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + amount_in, + false, + ) + .and_then(|amount| AssetConversion::quote_price_exact_tokens_for_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + amount, + false, + )), + Some(amount_in) + ); + + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + amount_in, + false, + ) + .and_then(|amount| AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + amount, + false, + )), + Some(amount_in) + ); + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Native, + NativeOrAssetId::Asset(2), + amount_in, + false, + ) + .and_then(|amount| AssetConversion::quote_price_tokens_for_exact_tokens( + NativeOrAssetId::Asset(2), + NativeOrAssetId::Native, + amount, + false, + )), + Some(amount_in) + ); + }); +} + +#[test] +fn quote_price_exact_tokens_for_tokens_matches_execution() { + new_test_ext().execute_with(|| { + let user = 1; + let user2 = 2; + let token_1 = NativeOrAssetId::Native; + let token_2 = NativeOrAssetId::Asset(2); + + create_tokens(user, vec![token_2]); + assert_ok!(AssetConversion::create_pool(RuntimeOrigin::signed(user), token_1, token_2)); + + assert_ok!(Balances::force_set_balance(RuntimeOrigin::root(), user, 100000)); + assert_ok!(Assets::mint(RuntimeOrigin::signed(user), 2, user, 1000)); + + assert_ok!(AssetConversion::add_liquidity( + RuntimeOrigin::signed(user), + token_1, + token_2, + 10000, + 200, + 1, + 1, + user, + )); + + let amount = 1; + let quoted_price = 49; + assert_eq!( + AssetConversion::quote_price_exact_tokens_for_tokens(token_2, token_1, amount, true,), + Some(quoted_price) + ); + + assert_ok!(Assets::mint(RuntimeOrigin::signed(user), 2, user2, amount)); + let prior_dot_balance = 20000; + assert_eq!(prior_dot_balance, balance(user2, token_1)); + assert_ok!(AssetConversion::swap_exact_tokens_for_tokens( + RuntimeOrigin::signed(user2), + bvec![token_2, token_1], + amount, + 1, + user2, + false, + )); + + assert_eq!(prior_dot_balance + quoted_price, balance(user2, token_1)); + }); +} + +#[test] +fn quote_price_tokens_for_exact_tokens_matches_execution() { + new_test_ext().execute_with(|| { + let user = 1; + let user2 = 2; + let token_1 = NativeOrAssetId::Native; + let token_2 = NativeOrAssetId::Asset(2); + + create_tokens(user, vec![token_2]); + assert_ok!(AssetConversion::create_pool(RuntimeOrigin::signed(user), token_1, token_2)); + + assert_ok!(Balances::force_set_balance(RuntimeOrigin::root(), user, 100000)); + assert_ok!(Assets::mint(RuntimeOrigin::signed(user), 2, user, 1000)); + + assert_ok!(AssetConversion::add_liquidity( + RuntimeOrigin::signed(user), + token_1, + token_2, + 10000, + 200, + 1, + 1, + user, + )); + + let amount = 49; + let quoted_price = 1; + assert_eq!( + AssetConversion::quote_price_tokens_for_exact_tokens(token_2, token_1, amount, true,), + Some(quoted_price) + ); + + assert_ok!(Assets::mint(RuntimeOrigin::signed(user), 2, user2, amount)); + let prior_dot_balance = 20000; + assert_eq!(prior_dot_balance, balance(user2, token_1)); + let prior_asset_balance = 49; + assert_eq!(prior_asset_balance, balance(user2, token_2)); + assert_ok!(AssetConversion::swap_tokens_for_exact_tokens( + RuntimeOrigin::signed(user2), + bvec![token_2, token_1], + amount, + 1, + user2, + false, + )); + + assert_eq!(prior_dot_balance + amount, balance(user2, token_1)); + assert_eq!(prior_asset_balance - quoted_price, balance(user2, token_2)); }); } @@ -1389,7 +1650,11 @@ fn cannot_block_pool_creation() { let pool_account = AssetConversion::get_pool_account(&AssetConversion::get_pool_id(token_2, token_1)); // And transfers the ED to that pool account - assert_ok!(Balances::transfer(RuntimeOrigin::signed(attacker), pool_account, ed)); + assert_ok!(Balances::transfer_allow_death( + RuntimeOrigin::signed(attacker), + pool_account, + ed + )); // Then, the attacker creates 14 tokens and sends one of each to the pool account for i in 10..25 { create_tokens(attacker, vec![NativeOrAssetId::Asset(i)]); diff --git a/substrate/frame/asset-rate/src/benchmarking.rs b/substrate/frame/asset-rate/src/benchmarking.rs index 12edcf5aee025c8d88601c67583bf02affac6bbd..21d53a89e3976987a54c3ce03e8fdc91260f2330 100644 --- a/substrate/frame/asset-rate/src/benchmarking.rs +++ b/substrate/frame/asset-rate/src/benchmarking.rs @@ -54,7 +54,7 @@ mod benchmarks { fn create() -> Result<(), BenchmarkError> { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone(), default_conversion_rate()); + _(RawOrigin::Root, Box::new(asset_kind.clone()), default_conversion_rate()); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(asset_kind), @@ -68,12 +68,12 @@ mod benchmarks { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); assert_ok!(AssetRate::::create( RawOrigin::Root.into(), - asset_kind.clone(), + Box::new(asset_kind.clone()), default_conversion_rate() )); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone(), FixedU128::from_u32(2)); + _(RawOrigin::Root, Box::new(asset_kind.clone()), FixedU128::from_u32(2)); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(asset_kind), @@ -87,12 +87,12 @@ mod benchmarks { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); assert_ok!(AssetRate::::create( RawOrigin::Root.into(), - asset_kind.clone(), + Box::new(asset_kind.clone()), default_conversion_rate() )); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone()); + _(RawOrigin::Root, Box::new(asset_kind.clone())); assert!(pallet_asset_rate::ConversionRateToNative::::get(asset_kind).is_none()); Ok(()) diff --git a/substrate/frame/asset-rate/src/lib.rs b/substrate/frame/asset-rate/src/lib.rs index 8b55f3d1d40296fb064aa8c2f450232d7eb02465..c3dc551f876d0a64663fe8c17eef854865d67d63 100644 --- a/substrate/frame/asset-rate/src/lib.rs +++ b/substrate/frame/asset-rate/src/lib.rs @@ -61,6 +61,7 @@ use frame_support::traits::{fungible::Inspect, tokens::ConversionFromAssetBalance}; use sp_runtime::{traits::Zero, FixedPointNumber, FixedU128}; +use sp_std::boxed::Box; pub use pallet::*; pub use weights::WeightInfo; @@ -155,18 +156,18 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::create())] pub fn create( origin: OriginFor, - asset_kind: T::AssetKind, + asset_kind: Box, rate: FixedU128, ) -> DispatchResult { T::CreateOrigin::ensure_origin(origin)?; ensure!( - !ConversionRateToNative::::contains_key(asset_kind.clone()), + !ConversionRateToNative::::contains_key(asset_kind.as_ref()), Error::::AlreadyExists ); - ConversionRateToNative::::set(asset_kind.clone(), Some(rate)); + ConversionRateToNative::::set(asset_kind.as_ref(), Some(rate)); - Self::deposit_event(Event::AssetRateCreated { asset_kind, rate }); + Self::deposit_event(Event::AssetRateCreated { asset_kind: *asset_kind, rate }); Ok(()) } @@ -178,13 +179,13 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::update())] pub fn update( origin: OriginFor, - asset_kind: T::AssetKind, + asset_kind: Box, rate: FixedU128, ) -> DispatchResult { T::UpdateOrigin::ensure_origin(origin)?; let mut old = FixedU128::zero(); - ConversionRateToNative::::mutate(asset_kind.clone(), |maybe_rate| { + ConversionRateToNative::::mutate(asset_kind.as_ref(), |maybe_rate| { if let Some(r) = maybe_rate { old = *r; *r = rate; @@ -195,7 +196,11 @@ pub mod pallet { } })?; - Self::deposit_event(Event::AssetRateUpdated { asset_kind, old, new: rate }); + Self::deposit_event(Event::AssetRateUpdated { + asset_kind: *asset_kind, + old, + new: rate, + }); Ok(()) } @@ -205,16 +210,16 @@ pub mod pallet { /// - O(1) #[pallet::call_index(2)] #[pallet::weight(T::WeightInfo::remove())] - pub fn remove(origin: OriginFor, asset_kind: T::AssetKind) -> DispatchResult { + pub fn remove(origin: OriginFor, asset_kind: Box) -> DispatchResult { T::RemoveOrigin::ensure_origin(origin)?; ensure!( - ConversionRateToNative::::contains_key(asset_kind.clone()), + ConversionRateToNative::::contains_key(asset_kind.as_ref()), Error::::UnknownAssetKind ); - ConversionRateToNative::::remove(asset_kind.clone()); + ConversionRateToNative::::remove(asset_kind.as_ref()); - Self::deposit_event(Event::AssetRateRemoved { asset_kind }); + Self::deposit_event(Event::AssetRateRemoved { asset_kind: *asset_kind }); Ok(()) } } diff --git a/substrate/frame/asset-rate/src/tests.rs b/substrate/frame/asset-rate/src/tests.rs index 8990ba9fc28d658bfab2b2bf1e3f73a6373232e1..452265476093b034256907e9d2a232c91041f772 100644 --- a/substrate/frame/asset-rate/src/tests.rs +++ b/substrate/frame/asset-rate/src/tests.rs @@ -29,7 +29,11 @@ const ASSET_ID: u32 = 42; fn create_works() { new_test_ext().execute_with(|| { assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID), @@ -42,10 +46,18 @@ fn create_works() { fn create_existing_throws() { new_test_ext().execute_with(|| { assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); assert_noop!( - AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1)), + AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + ), Error::::AlreadyExists ); }); @@ -54,9 +66,13 @@ fn create_existing_throws() { #[test] fn remove_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); - assert_ok!(AssetRate::remove(RuntimeOrigin::root(), ASSET_ID,)); + assert_ok!(AssetRate::remove(RuntimeOrigin::root(), Box::new(ASSET_ID),)); assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); }); } @@ -65,7 +81,7 @@ fn remove_works() { fn remove_unknown_throws() { new_test_ext().execute_with(|| { assert_noop!( - AssetRate::remove(RuntimeOrigin::root(), ASSET_ID,), + AssetRate::remove(RuntimeOrigin::root(), Box::new(ASSET_ID),), Error::::UnknownAssetKind ); }); @@ -74,8 +90,16 @@ fn remove_unknown_throws() { #[test] fn update_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); - assert_ok!(AssetRate::update(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.5))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); + assert_ok!(AssetRate::update( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.5) + )); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID), @@ -88,7 +112,11 @@ fn update_works() { fn update_unknown_throws() { new_test_ext().execute_with(|| { assert_noop!( - AssetRate::update(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.5)), + AssetRate::update( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.5) + ), Error::::UnknownAssetKind ); }); @@ -97,7 +125,11 @@ fn update_unknown_throws() { #[test] fn convert_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(2.51))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(2.51) + )); let conversion = , diff --git a/substrate/frame/assets/README.md b/substrate/frame/assets/README.md index aae5244953e50638510569af075ca45cbe4539f0..863bcccbbaf802cb938fd90aa61ee174164bc722 100644 --- a/substrate/frame/assets/README.md +++ b/substrate/frame/assets/README.md @@ -4,21 +4,21 @@ A simple, secure module for dealing with fungible assets. ## Overview -The Assets module provides functionality for asset management of fungible asset classes -with a fixed supply, including: +The Assets module provides functionality for asset management of fungible asset classes with a fixed supply, including: * Asset Issuance * Asset Transfer * Asset Destruction -To use it in your runtime, you need to implement the assets [`assets::Config`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/trait.Config.html). +To use it in your runtime, you need to implement the assets +[`assets::Config`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/trait.Config.html). -The supported dispatchable functions are documented in the [`assets::Call`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/enum.Call.html) enum. +The supported dispatchable functions are documented in the +[`assets::Call`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/enum.Call.html) enum. ### Terminology -* **Asset issuance:** The creation of a new asset, whose total supply will belong to the - account that issues the asset. +* **Asset issuance:** The creation of a new asset, whose total supply will belong to the account that issues the asset. * **Asset transfer:** The action of transferring assets from one account to another. * **Asset destruction:** The process of an account removing its entire holding of an asset. * **Fungible asset:** An asset whose units are interchangeable. @@ -30,20 +30,19 @@ The assets system in Substrate is designed to make the following possible: * Issue a unique asset to its creator's account. * Move assets between accounts. -* Remove an account's balance of an asset when requested by that account's owner and update - the asset's total supply. +* Remove an account's balance of an asset when requested by that account's owner and update the asset's total supply. ## Interface ### Dispatchable Functions * `issue` - Issues the total supply of a new fungible asset to the account of the caller of the function. -* `transfer` - Transfers an `amount` of units of fungible asset `id` from the balance of -the function caller's account (`origin`) to a `target` account. -* `destroy` - Destroys the entire holding of a fungible asset `id` associated with the account -that called the function. +* `transfer` - Transfers an `amount` of units of fungible asset `id` from the balance of the function caller's account +(`origin`) to a `target` account. +* `destroy` - Destroys the entire holding of a fungible asset `id` associated with the account that called the function. -Please refer to the [`Call`](https://docs.rs/pallet-assets/latest/pallet_assets/enum.Call.html) enum and its associated variants for documentation on each function. +Please refer to the [`Call`](https://docs.rs/pallet-assets/latest/pallet_assets/enum.Call.html) enum and its associated +variants for documentation on each function. ### Public Functions @@ -51,7 +50,8 @@ Please refer to the [`Call`](https://docs.rs/pallet-assets/latest/pallet_assets/ * `balance` - Get the asset `id` balance of `who`. * `total_supply` - Get the total supply of an asset `id`. -Please refer to the [`Pallet`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/struct.Pallet.html) struct for details on publicly available functions. +Please refer to the [`Pallet`](https://docs.rs/pallet-assets/latest/pallet_assets/pallet/struct.Pallet.html) struct for +details on publicly available functions. ## Usage @@ -111,11 +111,10 @@ pub mod pallet { ## Assumptions -Below are assumptions that must be held when using this module. If any of -them are violated, the behavior of this module is undefined. +Below are assumptions that must be held when using this module. If any of them are violated, the behavior of this +module is undefined. -* The total count of assets should be less than - `Config::AssetId::max_value()`. +* The total count of assets should be less than `Config::AssetId::max_value()`. ## Related Modules diff --git a/substrate/frame/atomic-swap/README.md b/substrate/frame/atomic-swap/README.md index 888a64ec7e0658e8cb6ffc6a95f59a300a8ae1aa..d5f924c64fc8dc90f7d946186448b23619ae83b4 100644 --- a/substrate/frame/atomic-swap/README.md +++ b/substrate/frame/atomic-swap/README.md @@ -16,8 +16,8 @@ claimed within a specified duration of time, the sender may cancel it. ### Dispatchable Functions -* `create_swap` - called by a sender to register a new atomic swap -* `claim_swap` - called by the target to approve a swap -* `cancel_swap` - may be called by a sender after a specified duration +- `create_swap` - called by a sender to register a new atomic swap +- `claim_swap` - called by the target to approve a swap +- `cancel_swap` - may be called by a sender after a specified duration License: Apache-2.0 diff --git a/substrate/frame/aura/README.md b/substrate/frame/aura/README.md index 263f158d790686aa34fb2a238aad1999e4934011..3ce9652a5385c9a1fd7d781eea99e04027ac7e57 100644 --- a/substrate/frame/aura/README.md +++ b/substrate/frame/aura/README.md @@ -23,6 +23,7 @@ consensus rounds (via `slots`). If you're interested in hacking on this module, it is useful to understand the interaction with `substrate/primitives/inherents/src/lib.rs` and, specifically, the required implementation of [`ProvideInherent`](https://docs.rs/sp-inherents/latest/sp_inherents/trait.ProvideInherent.html) and -[`ProvideInherentData`](https://docs.rs/sp-inherents/latest/sp_inherents/trait.ProvideInherentData.html) to create and check inherents. +[`ProvideInherentData`](https://docs.rs/sp-inherents/latest/sp_inherents/trait.ProvideInherentData.html) to create and +check inherents. License: Apache-2.0 diff --git a/substrate/frame/authority-discovery/README.md b/substrate/frame/authority-discovery/README.md index 9a534dcbeb6f8f98ec2260176ca4075f81179194..f4435a9f3161236951d4acd876df3f77598e5f51 100644 --- a/substrate/frame/authority-discovery/README.md +++ b/substrate/frame/authority-discovery/README.md @@ -1,6 +1,6 @@ -# Authority discovery module. +# Authority discovery module This module is used by the `client/authority-discovery` to retrieve the current set of authorities. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/authorship/README.md b/substrate/frame/authorship/README.md index d61747da3e101b4989b517a77c166cf98cbce6fe..db4b979319ab63f8f627f9b5707fe0560f2f2b08 100644 --- a/substrate/frame/authorship/README.md +++ b/substrate/frame/authorship/README.md @@ -2,4 +2,4 @@ Authorship tracking for FRAME runtimes. This tracks the current author of the block and recent uncles. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/babe/README.md b/substrate/frame/babe/README.md index 6f20be89efc0cd57bcf4ab1a082f1f5198e42fa7..3d1af534eeb63729924171421e0735d00154c077 100644 --- a/substrate/frame/babe/README.md +++ b/substrate/frame/babe/README.md @@ -1,4 +1,4 @@ Consensus extension module for BABE consensus. Collects on-chain randomness from VRF outputs and manages epoch transitions. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/bags-list/Cargo.toml b/substrate/frame/bags-list/Cargo.toml index 4d6f3d768aad8e533558175f312e20da0a3043d6..f4644890e2bafd4b9a25ec25dc81dddb020fae2e 100644 --- a/substrate/frame/bags-list/Cargo.toml +++ b/substrate/frame/bags-list/Cargo.toml @@ -27,7 +27,7 @@ frame-election-provider-support = { path = "../election-provider-support", defau # third party log = { version = "0.4.17", default-features = false } -docify = "0.2.1" +docify = "0.2.4" aquamarine = { version = "0.3.2" } # Optional imports for benchmarking diff --git a/substrate/frame/balances/README.md b/substrate/frame/balances/README.md index fa1ee622d48ce2858df0d3b9937d6645b290c7ac..1dc93a6bd8fddce5fd5912885a6aa2ed74bb9385 100644 --- a/substrate/frame/balances/README.md +++ b/substrate/frame/balances/README.md @@ -21,49 +21,48 @@ The Balances module provides functions for: ### Terminology -- **Existential Deposit:** The minimum balance required to create or keep an account open. This prevents -"dust accounts" from filling storage. When the free plus the reserved balance (i.e. the total balance) - fall below this, then the account is said to be dead; and it loses its functionality as well as any - prior history and all information on it is removed from the chain's state. - No account should ever have a total balance that is strictly between 0 and the existential - deposit (exclusive). If this ever happens, it indicates either a bug in this module or an - erroneous raw mutation of storage. +- **Existential Deposit:** The minimum balance required to create or keep an account open. This prevents "dust accounts" +from filling storage. When the free plus the reserved balance (i.e. the total balance) fall below this, then the account + is said to be dead; and it loses its functionality as well as any prior history and all information on it is removed + from the chain's state. No account should ever have a total balance that is strictly between 0 and the existential + deposit (exclusive). If this ever happens, it indicates either a bug in this module or an erroneous raw mutation of + storage. - **Total Issuance:** The total number of units in existence in a system. -- **Reaping an account:** The act of removing an account by resetting its nonce. Happens after its -total balance has become zero (or, strictly speaking, less than the Existential Deposit). +- **Reaping an account:** The act of removing an account by resetting its nonce. Happens after its total balance has +become zero (or, strictly speaking, less than the Existential Deposit). -- **Free Balance:** The portion of a balance that is not reserved. The free balance is the only - balance that matters for most operations. +- **Free Balance:** The portion of a balance that is not reserved. The free balance is the only balance that matters for + most operations. -- **Reserved Balance:** Reserved balance still belongs to the account holder, but is suspended. - Reserved balance can still be slashed, but only after all the free balance has been slashed. +- **Reserved Balance:** Reserved balance still belongs to the account holder, but is suspended. Reserved balance can + still be slashed, but only after all the free balance has been slashed. -- **Imbalance:** A condition when some funds were credited or debited without equal and opposite accounting -(i.e. a difference between total issuance and account balances). Functions that result in an imbalance will -return an object of the `Imbalance` trait that can be managed within your runtime logic. (If an imbalance is -simply dropped, it should automatically maintain any book-keeping such as total issuance.) +- **Imbalance:** A condition when some funds were credited or debited without equal and opposite accounting (i.e. a +difference between total issuance and account balances). Functions that result in an imbalance will return an object of +the `Imbalance` trait that can be managed within your runtime logic. (If an imbalance is simply dropped, it should +automatically maintain any book-keeping such as total issuance.) -- **Lock:** A freeze on a specified amount of an account's free balance until a specified block number. Multiple -locks always operate over the same funds, so they "overlay" rather than "stack". +- **Lock:** A freeze on a specified amount of an account's free balance until a specified block number. Multiple locks +always operate over the same funds, so they "overlay" rather than "stack". ### Implementations -The Balances module provides implementations for the following traits. If these traits provide the functionality -that you need, then you can avoid coupling with the Balances module. +The Balances module provides implementations for the following traits. If these traits provide the functionality that +you need, then you can avoid coupling with the Balances module. -- [`Currency`](https://docs.rs/frame-support/latest/frame_support/traits/trait.Currency.html): Functions for dealing with a -fungible assets system. +- [`Currency`](https://docs.rs/frame-support/latest/frame_support/traits/trait.Currency.html): Functions for dealing +with a fungible assets system. - [`ReservableCurrency`](https://docs.rs/frame-support/latest/frame_support/traits/trait.ReservableCurrency.html): Functions for dealing with assets that can be reserved from an account. -- [`LockableCurrency`](https://docs.rs/frame-support/latest/frame_support/traits/trait.LockableCurrency.html): Functions for -dealing with accounts that allow liquidity restrictions. +- [`LockableCurrency`](https://docs.rs/frame-support/latest/frame_support/traits/trait.LockableCurrency.html): Functions +for dealing with accounts that allow liquidity restrictions. - [`Imbalance`](https://docs.rs/frame-support/latest/frame_support/traits/trait.Imbalance.html): Functions for handling -imbalances between total issuance in the system and account balances. Must be used when a function -creates new funds (e.g. a reward) or destroys some funds (e.g. a system fee). -- [`IsDeadAccount`](https://docs.rs/frame-support/latest/frame_support/traits/trait.IsDeadAccount.html): Determiner to say whether a -given account is unused. +imbalances between total issuance in the system and account balances. Must be used when a function creates new funds +(e.g. a reward) or destroys some funds (e.g. a system fee). +- [`IsDeadAccount`](https://docs.rs/frame-support/latest/frame_support/traits/trait.IsDeadAccount.html): Determiner to +say whether a given account is unused. ## Interface @@ -113,10 +112,11 @@ fn update_ledger( ## Genesis config -The Balances module depends on the [`GenesisConfig`](https://docs.rs/pallet-balances/latest/pallet_balances/pallet/struct.GenesisConfig.html). +The Balances module depends on the +[`GenesisConfig`](https://docs.rs/pallet-balances/latest/pallet_balances/pallet/struct.GenesisConfig.html). ## Assumptions -* Total issued balanced of all accounts should be less than `Config::Balance::max_value()`. +- Total issued balanced of all accounts should be less than `Config::Balance::max_value()`. License: Apache-2.0 diff --git a/substrate/frame/balances/src/lib.rs b/substrate/frame/balances/src/lib.rs index f94b3230b917b79a046ce8f1e8fcece8ab0473d7..5da6600d8796780e19d85d383b58cdcf223caa9b 100644 --- a/substrate/frame/balances/src/lib.rs +++ b/substrate/frame/balances/src/lib.rs @@ -563,53 +563,6 @@ pub mod pallet { Ok(()) } - /// Set the regular balance of a given account; it also takes a reserved balance but this - /// must be the same as the account's current reserved balance. - /// - /// The dispatch origin for this call is `root`. - /// - /// WARNING: This call is DEPRECATED! Use `force_set_balance` instead. - #[pallet::call_index(1)] - #[pallet::weight( - T::WeightInfo::force_set_balance_creating() // Creates a new account. - .max(T::WeightInfo::force_set_balance_killing()) // Kills an existing account. - )] - pub fn set_balance_deprecated( - origin: OriginFor, - who: AccountIdLookupOf, - #[pallet::compact] new_free: T::Balance, - #[pallet::compact] old_reserved: T::Balance, - ) -> DispatchResult { - ensure_root(origin)?; - let who = T::Lookup::lookup(who)?; - let existential_deposit = Self::ed(); - - let wipeout = new_free < existential_deposit; - let new_free = if wipeout { Zero::zero() } else { new_free }; - - // First we try to modify the account's balance to the forced balance. - let old_free = Self::try_mutate_account_handling_dust( - &who, - |account, _is_new| -> Result { - let old_free = account.free; - ensure!(account.reserved == old_reserved, TokenError::Unsupported); - account.free = new_free; - Ok(old_free) - }, - )?; - - // This will adjust the total issuance, which was not done by the `mutate_account` - // above. - if new_free > old_free { - mem::drop(PositiveImbalance::::new(new_free - old_free)); - } else if new_free < old_free { - mem::drop(NegativeImbalance::::new(old_free - new_free)); - } - - Self::deposit_event(Event::BalanceSet { who, free: new_free }); - Ok(()) - } - /// Exactly as `transfer_allow_death`, except the origin must be root and the source account /// may be specified. #[pallet::call_index(2)] @@ -730,22 +683,6 @@ pub mod pallet { } } - /// Alias for `transfer_allow_death`, provided only for name-wise compatibility. - /// - /// WARNING: DEPRECATED! Will be released in approximately 3 months. - #[pallet::call_index(7)] - #[pallet::weight(T::WeightInfo::transfer_allow_death())] - pub fn transfer( - origin: OriginFor, - dest: AccountIdLookupOf, - #[pallet::compact] value: T::Balance, - ) -> DispatchResult { - let source = ensure_signed(origin)?; - let dest = T::Lookup::lookup(dest)?; - >::transfer(&source, &dest, value, Expendable)?; - Ok(()) - } - /// Set the regular balance of a given account. /// /// The dispatch origin for this call is `root`. diff --git a/substrate/frame/balances/src/tests/currency_tests.rs b/substrate/frame/balances/src/tests/currency_tests.rs index 969731b49a236c792c2ea6043c16696ac2e44b40..2449638788ddbfc8ea42a6d5b638a41f47ff1874 100644 --- a/substrate/frame/balances/src/tests/currency_tests.rs +++ b/substrate/frame/balances/src/tests/currency_tests.rs @@ -22,7 +22,7 @@ use crate::NegativeImbalance; use frame_support::traits::{ BalanceStatus::{Free, Reserved}, Currency, - ExistenceRequirement::{self, AllowDeath}, + ExistenceRequirement::{self, AllowDeath, KeepAlive}, Hooks, LockIdentifier, LockableCurrency, NamedReservableCurrency, ReservableCurrency, WithdrawReasons, }; @@ -33,6 +33,18 @@ const ID_2: LockIdentifier = *b"2 "; pub const CALL: &::RuntimeCall = &RuntimeCall::Balances(crate::Call::transfer_allow_death { dest: 0, value: 0 }); +#[test] +fn ed_should_work() { + ExtBuilder::default().existential_deposit(1).build_and_execute_with(|| { + assert_ok!(Balances::force_set_balance(RuntimeOrigin::root(), 1, 1000)); + assert_noop!( + >::transfer(&1, &10, 1000, KeepAlive), + TokenError::NotExpendable + ); + assert_ok!(>::transfer(&1, &10, 1000, AllowDeath)); + }); +} + #[test] fn set_lock_with_amount_zero_removes_lock() { ExtBuilder::default() diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs index b12eb95f650f7b65771ba5fa7482d802f848cddd..a0bf7cdcf86a28a139693e596938169e95df029b 100644 --- a/substrate/frame/beefy-mmr/src/lib.rs +++ b/substrate/frame/beefy-mmr/src/lib.rs @@ -79,7 +79,7 @@ impl Convert> for BeefyEc .to_eth_address() .map(|v| v.to_vec()) .map_err(|_| { - log::error!(target: "runtime::beefy", "Failed to convert BEEFY PublicKey to ETH address!"); + log::debug!(target: "runtime::beefy", "Failed to convert BEEFY PublicKey to ETH address!"); }) .unwrap_or_default() } @@ -199,7 +199,20 @@ impl Pallet { .cloned() .map(T::BeefyAuthorityToMerkleLeaf::convert) .collect::>(); + let default_eth_addr = [0u8; 20]; let len = beefy_addresses.len() as u32; + let uninitialized_addresses = beefy_addresses + .iter() + .filter(|&addr| addr.as_slice().eq(&default_eth_addr)) + .count(); + if uninitialized_addresses > 0 { + log::error!( + target: "runtime::beefy", + "Failed to convert {} out of {} BEEFY PublicKeys to ETH addresses!", + uninitialized_addresses, + len, + ); + } let keyset_commitment = binary_merkle_tree::merkle_root::< ::Hashing, _, diff --git a/substrate/frame/beefy/src/default_weights.rs b/substrate/frame/beefy/src/default_weights.rs index 091d58f47f97888b43259bcb62c44905fae568e3..8042f0c932eb6603c88e1ff5f4525743c347b9bb 100644 --- a/substrate/frame/beefy/src/default_weights.rs +++ b/substrate/frame/beefy/src/default_weights.rs @@ -49,4 +49,8 @@ impl crate::WeightInfo for () { // fetching set id -> session index mappings .saturating_add(DbWeight::get().reads(2)) } + + fn set_new_genesis() -> Weight { + DbWeight::get().writes(1) + } } diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs index 77e74436dd6710416d7418f3dca2198dced68428..0760446753a688516a2c4ce75727ebb4b3cd06a8 100644 --- a/substrate/frame/beefy/src/lib.rs +++ b/substrate/frame/beefy/src/lib.rs @@ -33,7 +33,7 @@ use frame_system::{ use log; use sp_runtime::{ generic::DigestItem, - traits::{IsMember, Member}, + traits::{IsMember, Member, One}, RuntimeAppPublic, }; use sp_session::{GetSessionNumber, GetValidatorCount}; @@ -62,7 +62,7 @@ const LOG_TARGET: &str = "runtime::beefy"; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_system::pallet_prelude::BlockNumberFor; + use frame_system::{ensure_root, pallet_prelude::BlockNumberFor}; #[pallet::config] pub trait Config: frame_system::Config { @@ -152,8 +152,8 @@ pub mod pallet { StorageMap<_, Twox64Concat, sp_consensus_beefy::ValidatorSetId, SessionIndex>; /// Block number where BEEFY consensus is enabled/started. - /// By changing this (through governance or sudo), BEEFY consensus is effectively - /// restarted from the new block number. + /// By changing this (through privileged `set_new_genesis()`), BEEFY consensus is effectively + /// restarted from the newly set block number. #[pallet::storage] #[pallet::getter(fn genesis_block)] pub(super) type GenesisBlock = @@ -174,7 +174,7 @@ pub mod pallet { fn default() -> Self { // BEEFY genesis will be first BEEFY-MANDATORY block, // use block number one instead of chain-genesis. - let genesis_block = Some(sp_runtime::traits::One::one()); + let genesis_block = Some(One::one()); Self { authorities: Vec::new(), genesis_block } } } @@ -198,6 +198,8 @@ pub mod pallet { InvalidEquivocationProof, /// A given equivocation report is valid but already previously reported. DuplicateOffenceReport, + /// Submitted configuration is invalid. + InvalidConfiguration, } #[pallet::call] @@ -265,6 +267,23 @@ pub mod pallet { )?; Ok(Pays::No.into()) } + + /// Reset BEEFY consensus by setting a new BEEFY genesis at `delay_in_blocks` blocks in the + /// future. + /// + /// Note: `delay_in_blocks` has to be at least 1. + #[pallet::call_index(2)] + #[pallet::weight(::WeightInfo::set_new_genesis())] + pub fn set_new_genesis( + origin: OriginFor, + delay_in_blocks: BlockNumberFor, + ) -> DispatchResult { + ensure_root(origin)?; + ensure!(delay_in_blocks >= One::one(), Error::::InvalidConfiguration); + let genesis_block = frame_system::Pallet::::block_number() + delay_in_blocks; + GenesisBlock::::put(Some(genesis_block)); + Ok(()) + } } #[pallet::validate_unsigned] @@ -452,4 +471,5 @@ impl IsMember for Pallet { pub trait WeightInfo { fn report_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight; + fn set_new_genesis() -> Weight; } diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs index e04dc330d0c07cb7258323b34db0086c6226ffb2..bf1b204e0260e29a2ce473416a43d0b641f8a7b9 100644 --- a/substrate/frame/beefy/src/tests.rs +++ b/substrate/frame/beefy/src/tests.rs @@ -791,3 +791,25 @@ fn valid_equivocation_reports_dont_pay_fees() { assert_eq!(post_info.pays_fee, Pays::Yes); }) } + +#[test] +fn set_new_genesis_works() { + let authorities = test_authorities(); + + new_test_ext_raw_authorities(authorities).execute_with(|| { + start_era(1); + + let new_genesis_delay = 10u64; + // the call for setting new genesis should work + assert_ok!(Beefy::set_new_genesis(RuntimeOrigin::root(), new_genesis_delay,)); + let expected = System::block_number() + new_genesis_delay; + // verify new genesis was set + assert_eq!(Beefy::genesis_block(), Some(expected)); + + // setting delay < 1 should fail + assert_err!( + Beefy::set_new_genesis(RuntimeOrigin::root(), 0u64,), + Error::::InvalidConfiguration, + ); + }); +} diff --git a/substrate/frame/benchmarking/README.md b/substrate/frame/benchmarking/README.md index dc6a184435df6e843ac81ab5746db9b48a0f4549..bf0bde2c3df5a5dcc85a6aa61f3c0af519bc3ccf 100644 --- a/substrate/frame/benchmarking/README.md +++ b/substrate/frame/benchmarking/README.md @@ -1,99 +1,86 @@ # Substrate Runtime Benchmarking Framework -This crate contains a set of utilities that can be used to benchmark and weigh FRAME pallets that -you develop for your Substrate Runtime. +This crate contains a set of utilities that can be used to benchmark and weigh FRAME pallets that you develop for your +Substrate Runtime. ## Overview -Substrate's FRAME framework allows you to develop custom logic for your blockchain that can be -included in your runtime. This flexibility is key to help you design complex and interactive -pallets, but without accurate weights assigned to dispatchables, your blockchain may become -vulnerable to denial of service (DoS) attacks by malicious actors. +Substrate's FRAME framework allows you to develop custom logic for your blockchain that can be included in your runtime. +This flexibility is key to help you design complex and interactive pallets, but without accurate weights assigned to +dispatchables, your blockchain may become vulnerable to denial of service (DoS) attacks by malicious actors. -The Substrate Runtime Benchmarking Framework is a tool you can use to mitigate DoS attacks against -your blockchain network by benchmarking the computational resources required to execute different -functions in the runtime, for example extrinsics, `on_initialize`, `verify_unsigned`, etc... +The Substrate Runtime Benchmarking Framework is a tool you can use to mitigate DoS attacks against your blockchain +network by benchmarking the computational resources required to execute different functions in the runtime, for example +extrinsics, `on_initialize`, `verify_unsigned`, etc... -The general philosophy behind the benchmarking system is: If your node can know ahead of time how -long it will take to execute an extrinsic, it can safely make decisions to include or exclude that -extrinsic based on its available resources. By doing this, it can keep the block production and -import process running smoothly. +The general philosophy behind the benchmarking system is: If your node can know ahead of time how long it will take to +execute an extrinsic, it can safely make decisions to include or exclude that extrinsic based on its available +resources. By doing this, it can keep the block production and import process running smoothly. To achieve this, we need to model how long it takes to run each function in the runtime by: * Creating custom benchmarking logic that executes a specific code path of a function. -* Executing the benchmark in the Wasm execution environment, on a specific set of hardware, with a - custom runtime configuration, etc... -* Executing the benchmark across controlled ranges of possible values that may affect the result of - the benchmark (called "components"). +* Executing the benchmark in the Wasm execution environment, on a specific set of hardware, with a custom runtime + configuration, etc... +* Executing the benchmark across controlled ranges of possible values that may affect the result of the benchmark + (called "components"). * Executing the benchmark multiple times at each point in order to isolate and remove outliers. * Using the results of the benchmark to create a linear model of the function across its components. -With this linear model, we are able to estimate ahead of time how long it takes to execute some -logic, and thus make informed decisions without actually spending any significant resources at -runtime. +With this linear model, we are able to estimate ahead of time how long it takes to execute some logic, and thus make +informed decisions without actually spending any significant resources at runtime. -Note that we assume that all extrinsics are assumed to be of linear complexity, which is why we are -able to always fit them to a linear model. Quadratic or higher complexity functions are, in general, -considered to be dangerous to the runtime as the weight of these functions may explode as the -runtime state or input becomes too complex. +Note that we assume that all extrinsics are assumed to be of linear complexity, which is why we are able to always fit +them to a linear model. Quadratic or higher complexity functions are, in general, considered to be dangerous to the +runtime as the weight of these functions may explode as the runtime state or input becomes too complex. The benchmarking framework comes with the following tools: -* [A set of macros](./src/lib.rs) (`benchmarks!`, `add_benchmark!`, etc...) to make it easy to - write, test, and add runtime benchmarks. +* [A set of macros](./src/lib.rs) (`benchmarks!`, `add_benchmark!`, etc...) to make it easy to write, test, and add + runtime benchmarks. * [A set of linear regression analysis functions](./src/analysis.rs) for processing benchmark data. -* [A CLI extension](../../utils/frame/benchmarking-cli/README.md) to make it easy to execute benchmarks on your - node. +* [A CLI extension](../../utils/frame/benchmarking-cli/README.md) to make it easy to execute benchmarks on your node. -The end-to-end benchmarking pipeline is disabled by default when compiling a node. If you want to -run benchmarks, you need to enable it by compiling with a Rust feature flag `runtime-benchmarks`. -More details about this below. +The end-to-end benchmarking pipeline is disabled by default when compiling a node. If you want to run benchmarks, you +need to enable it by compiling with a Rust feature flag `runtime-benchmarks`. More details about this below. ### Weight -Substrate represents computational resources using a generic unit of measurement called "Weight". It -defines 10^12 Weight as 1 second of computation on the physical machine used for benchmarking. This -means that the weight of a function may change based on the specific hardware used to benchmark the -runtime functions. +Substrate represents computational resources using a generic unit of measurement called "Weight". It defines 10^12 +Weight as 1 second of computation on the physical machine used for benchmarking. This means that the weight of a +function may change based on the specific hardware used to benchmark the runtime functions. -By modeling the expected weight of each runtime function, the blockchain is able to calculate how -many transactions or system level functions it will be able to execute within a certain period of -time. Often, the limiting factor for a blockchain is the fixed block production time for the -network. +By modeling the expected weight of each runtime function, the blockchain is able to calculate how many transactions or +system level functions it will be able to execute within a certain period of time. Often, the limiting factor for a +blockchain is the fixed block production time for the network. -Within FRAME, each dispatchable function must have a `#[weight]` annotation with a function that can -return the expected weight for the worst case scenario execution of that function given its inputs. -This benchmarking framework will result in a file that automatically generates those formulas for -you, which you can then use in your pallet. +Within FRAME, each dispatchable function must have a `#[weight]` annotation with a function that can return the expected +weight for the worst case scenario execution of that function given its inputs. This benchmarking framework will result +in a file that automatically generates those formulas for you, which you can then use in your pallet. ## Writing Benchmarks -Writing a runtime benchmark is much like writing a unit test for your pallet. It needs to be -carefully crafted to execute a certain logical path in your code. In tests you want to check for -various success and failure conditions, but with benchmarks you specifically look for the **most -computationally heavy** path, a.k.a the "worst case scenario". +Writing a runtime benchmark is much like writing a unit test for your pallet. It needs to be carefully crafted to +execute a certain logical path in your code. In tests you want to check for various success and failure conditions, but +with benchmarks you specifically look for the **most computationally heavy** path, a.k.a the "worst case scenario". -This means that if there are certain storage items or runtime state that may affect the complexity -of the function, for example triggering more iterations in a `for` loop, to get an accurate result, -you must set up your benchmark to trigger this. +This means that if there are certain storage items or runtime state that may affect the complexity of the function, for +example triggering more iterations in a `for` loop, to get an accurate result, you must set up your benchmark to trigger +this. -It may be that there are multiple paths your function can go down, and it is not clear which one is -the heaviest. In this case, you should just create a benchmark for each scenario! You may find that -there are paths in your code where complexity may become unbounded depending on user input. This may -be a hint that you should enforce sane boundaries for how a user can use your pallet. For example: -limiting the number of elements in a vector, limiting the number of iterations in a `for` loop, -etc... +It may be that there are multiple paths your function can go down, and it is not clear which one is the heaviest. In +this case, you should just create a benchmark for each scenario! You may find that there are paths in your code where +complexity may become unbounded depending on user input. This may be a hint that you should enforce sane boundaries for +how a user can use your pallet. For example: limiting the number of elements in a vector, limiting the number of +iterations in a `for` loop, etc... -Examples of end-to-end benchmarks can be found in the [pallets provided by Substrate](../), and the -specific details on how to use the `benchmarks!` macro can be found in [its -documentation](./src/lib.rs). +Examples of end-to-end benchmarks can be found in the [pallets provided by Substrate](../), and the specific details on +how to use the `benchmarks!` macro can be found in [its documentation](./src/lib.rs). ## Testing Benchmarks -You can test your benchmarks using the same test runtime that you created for your pallet's unit -tests. By creating your benchmarks in the `benchmarks!` macro, it automatically generates test -functions for you: +You can test your benchmarks using the same test runtime that you created for your pallet's unit tests. By creating your +benchmarks in the `benchmarks!` macro, it automatically generates test functions for you: ```rust fn test_benchmark_[benchmark_name]::() -> Result<(), &'static str> @@ -101,19 +88,18 @@ fn test_benchmark_[benchmark_name]::() -> Result<(), &'static str> Simply add these functions to a unit test and ensure that the result of the function is `Ok(())`. -> **Note:** If your test runtime and production runtime have different configurations, you may get -different results when testing your benchmark and actually running it. +> **Note:** If your test runtime and production runtime have different configurations, you may get different results +when testing your benchmark and actually running it. -In general, benchmarks returning `Ok(())` is all you need to check for since it signals the executed -extrinsic has completed successfully. However, you can optionally include a `verify` block with your -benchmark, which can additionally verify any final conditions, such as the final state of your -runtime. +In general, benchmarks returning `Ok(())` is all you need to check for since it signals the executed extrinsic has +completed successfully. However, you can optionally include a `verify` block with your benchmark, which can additionally +verify any final conditions, such as the final state of your runtime. These additional `verify` blocks will not affect the results of your final benchmarking process. -To run the tests, you need to enable the `runtime-benchmarks` feature flag. This may also mean you -need to move into your node's binary folder. For example, with the Substrate repository, this is how -you would test the Balances pallet's benchmarks: +To run the tests, you need to enable the `runtime-benchmarks` feature flag. This may also mean you need to move into +your node's binary folder. For example, with the Substrate repository, this is how you would test the Balances pallet's +benchmarks: ```bash cargo test -p pallet-balances --features runtime-benchmarks @@ -123,19 +109,20 @@ cargo test -p pallet-balances --features runtime-benchmarks > ``` > error: --features is not allowed in the root of a virtual workspace` > ``` -> To solve this, navigate to the folder of the node (`cd bin/node/cli`) or pallet (`cd frame/pallet`) and run the command there. +> To solve this, navigate to the folder of the node (`cd bin/node/cli`) or pallet (`cd frame/pallet`) and run the +> command there. -This will instance each linear component with different values. The number of values per component is set to six and can be changed with the `VALUES_PER_COMPONENT` environment variable. +This will instance each linear component with different values. The number of values per component is set to six and can +be changed with the `VALUES_PER_COMPONENT` environment variable. ## Adding Benchmarks -The benchmarks included with each pallet are not automatically added to your node. To actually -execute these benchmarks, you need to implement the `frame_benchmarking::Benchmark` trait. You can -see an example of how to do this in the [included Substrate -node](../../bin/node/runtime/src/lib.rs). +The benchmarks included with each pallet are not automatically added to your node. To actually execute these benchmarks, +you need to implement the `frame_benchmarking::Benchmark` trait. You can see an example of how to do this in the +[included Substrate node](../../bin/node/runtime/src/lib.rs). -Assuming there are already some benchmarks set up on your node, you just need to add another -instance of the `add_benchmark!` macro: +Assuming there are already some benchmarks set up on your node, you just need to add another instance of the +`add_benchmark!` macro: ```rust /// configuration for running benchmarks @@ -147,22 +134,20 @@ add_benchmark!(params, batches, pallet_balances, Balances); /// the `struct` created for your pallet by `construct_runtime!` ``` -Once you have done this, you will need to compile your node binary with the `runtime-benchmarks` -feature flag: +Once you have done this, you will need to compile your node binary with the `runtime-benchmarks` feature flag: ```bash cd bin/node/cli cargo build --profile=production --features runtime-benchmarks ``` -The production profile applies various compiler optimizations. -These optimizations slow down the compilation process *a lot*. +The production profile applies various compiler optimizations. +These optimizations slow down the compilation process *a lot*. If you are just testing things out and don't need final numbers, don't include `--profile=production`. ## Running Benchmarks -Finally, once you have a node binary with benchmarks enabled, you need to execute your various -benchmarks. +Finally, once you have a node binary with benchmarks enabled, you need to execute your various benchmarks. You can get a list of the available benchmarks by running: @@ -183,24 +168,24 @@ Then you can run a benchmark like so: --output \ # Output benchmark results into a folder or file ``` -This will output a file `pallet_name.rs` which implements the `WeightInfo` trait you should include -in your pallet. Double colons `::` will be replaced with a `_` in the output name if you specify a directory. Each blockchain should generate their own benchmark file with their custom -implementation of the `WeightInfo` trait. This means that you will be able to use these modular -Substrate pallets while still keeping your network safe for your specific configuration and +This will output a file `pallet_name.rs` which implements the `WeightInfo` trait you should include in your pallet. +Double colons `::` will be replaced with a `_` in the output name if you specify a directory. Each blockchain should +generate their own benchmark file with their custom implementation of the `WeightInfo` trait. This means that you will +be able to use these modular Substrate pallets while still keeping your network safe for your specific configuration and requirements. -The benchmarking CLI uses a Handlebars template to format the final output file. You can optionally -pass the flag `--template` pointing to a custom template that can be used instead. Within the -template, you have access to all the data provided by the `TemplateData` struct in the -[benchmarking CLI writer](../../utils/frame/benchmarking-cli/src/writer.rs). You can find the -default template used [here](../../utils/frame/benchmarking-cli/src/template.hbs). +The benchmarking CLI uses a Handlebars template to format the final output file. You can optionally pass the flag +`--template` pointing to a custom template that can be used instead. Within the template, you have access to all the +data provided by the `TemplateData` struct in the [benchmarking CLI +writer](../../utils/frame/benchmarking-cli/src/writer.rs). You can find the default template used +[here](../../utils/frame/benchmarking-cli/src/template.hbs). There are some custom Handlebars helpers included with our output generation: -* `underscore`: Add an underscore to every 3rd character from the right of a string. Primarily to be -used for delimiting large numbers. -* `join`: Join an array of strings into a space-separated string for the template. Primarily to be -used for joining all the arguments passed to the CLI. +* `underscore`: Add an underscore to every 3rd character from the right of a string. Primarily to be used for delimiting +large numbers. +* `join`: Join an array of strings into a space-separated string for the template. Primarily to be used for joining all +the arguments passed to the CLI. To get a full list of available options when running benchmarks, run: diff --git a/substrate/frame/broker/Cargo.toml b/substrate/frame/broker/Cargo.toml index edb3c5f63bbed8e1413d5a116a3479805ff6e85c..d6c5399d56eb178fe6e35f06638bf4409604ce42 100644 --- a/substrate/frame/broker/Cargo.toml +++ b/substrate/frame/broker/Cargo.toml @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] } scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } -bitvec = "1" +bitvec = { version = "1.0.0", default-features = false } sp-std = { path = "../../primitives/std", default-features = false} sp-arithmetic = { path = "../../primitives/arithmetic", default-features = false} sp-core = { path = "../../primitives/core", default-features = false} @@ -30,6 +30,7 @@ sp-io = { path = "../../primitives/io" } default = [ "std" ] std = [ + "bitvec/std", "codec/std", "frame-benchmarking?/std", "frame-support/std", diff --git a/substrate/frame/broker/README.md b/substrate/frame/broker/README.md index 65b6179863e3d9bd69f6286538ea5ff696413f5c..a3c4b251a452752172c789bcfdab97fa5da6bc9c 100644 --- a/substrate/frame/broker/README.md +++ b/substrate/frame/broker/README.md @@ -4,7 +4,7 @@ Brokerage tool for managing Polkadot Core scheduling. Properly described in RFC-0001 Agile Coretime. -## Implemnentation Specifics +## Implementation Specifics ### Core Mask Bits diff --git a/substrate/frame/broker/src/benchmarking.rs b/substrate/frame/broker/src/benchmarking.rs index 663bf2f466cf3c0090cd7b832bb38ed26da90613..d22f3936c3e2435665d869cd32edc0e042cc5f55 100644 --- a/substrate/frame/broker/src/benchmarking.rs +++ b/substrate/frame/broker/src/benchmarking.rs @@ -852,6 +852,27 @@ mod benches { } } + #[benchmark] + fn do_tick_base() -> Result<(), BenchmarkError> { + setup_and_start_sale::()?; + + advance_to::(5); + + let mut status = Status::::get().unwrap(); + status.last_committed_timeslice = 3; + Status::::put(&status); + + #[block] + { + Broker::::do_tick(); + } + + let updated_status = Status::::get().unwrap(); + assert_eq!(status, updated_status); + + Ok(()) + } + // Implements a test for each benchmark. Execute with: // `cargo test -p pallet-broker --features runtime-benchmarks`. impl_benchmark_test_suite!(Pallet, crate::mock::new_test_ext(), crate::mock::Test); diff --git a/substrate/frame/broker/src/dispatchable_impls.rs b/substrate/frame/broker/src/dispatchable_impls.rs index 8dc0c9de393eca94e64fa7db4f6b2d34f4475ca5..0b08a7b665b75ac1c860774762878012bf82a749 100644 --- a/substrate/frame/broker/src/dispatchable_impls.rs +++ b/substrate/frame/broker/src/dispatchable_impls.rs @@ -76,7 +76,7 @@ impl Pallet { last_timeslice: Self::current_timeslice(), }; let now = frame_system::Pallet::::block_number(); - let dummy_sale = SaleInfoRecord { + let new_sale = SaleInfoRecord { sale_start: now, leadin_length: Zero::zero(), price, @@ -89,7 +89,7 @@ impl Pallet { cores_sold: 0, }; Self::deposit_event(Event::::SalesStarted { price, core_count }); - Self::rotate_sale(dummy_sale, &config, &status); + Self::rotate_sale(new_sale, &config, &status); Status::::put(&status); Ok(()) } @@ -333,12 +333,8 @@ impl Pallet { region.begin = r + 1; contribution.length.saturating_dec(); - let Some(mut pool_record) = InstaPoolHistory::::get(r) else { - continue - }; - let Some(total_payout) = pool_record.maybe_payout else { - break - }; + let Some(mut pool_record) = InstaPoolHistory::::get(r) else { continue }; + let Some(total_payout) = pool_record.maybe_payout else { break }; let p = total_payout .saturating_mul(contributed_parts.into()) .checked_div(&pool_record.private_contributions.into()) diff --git a/substrate/frame/broker/src/tests.rs b/substrate/frame/broker/src/tests.rs index 3c326010dddfc7a01396064af1e5f706ed085ce1..e1b489bbe6e67801fcae1059eb09f85d72203365 100644 --- a/substrate/frame/broker/src/tests.rs +++ b/substrate/frame/broker/src/tests.rs @@ -77,7 +77,9 @@ fn drop_renewal_works() { let e = Error::::StillValid; assert_noop!(Broker::do_drop_renewal(region.core, region.begin + 3), e); advance_to(12); + assert_eq!(AllowedRenewals::::iter().count(), 1); assert_ok!(Broker::do_drop_renewal(region.core, region.begin + 3)); + assert_eq!(AllowedRenewals::::iter().count(), 0); let e = Error::::UnknownRenewal; assert_noop!(Broker::do_drop_renewal(region.core, region.begin + 3), e); }); @@ -90,7 +92,10 @@ fn drop_contribution_works() { advance_to(2); let region = Broker::do_purchase(1, u64::max_value()).unwrap(); // Place region in pool. Active in pool timeslices 4, 5, 6 = rcblocks 8, 10, 12; we - // expect the contribution record to timeout 3 timeslices following 7 = 10 + // expect the contribution record to timeout 3 timeslices following 7 = 14 + // + // Due to the contribution_timeout being configured for 3 timeslices, the contribution + // can only be discarded at timeslice 10, i.e. rcblock 20. assert_ok!(Broker::do_pool(region, Some(1), 1, Final)); assert_eq!(InstaPoolContribution::::iter().count(), 1); advance_to(19); @@ -378,6 +383,41 @@ fn instapool_partial_core_payouts_work() { }); } +#[test] +fn instapool_core_payouts_work_with_partitioned_region() { + TestExt::new().endow(1, 1000).execute_with(|| { + assert_ok!(Broker::do_start_sales(100, 1)); + advance_to(2); + let region = Broker::do_purchase(1, u64::max_value()).unwrap(); + let (region1, region2) = Broker::do_partition(region, None, 2).unwrap(); + // `region1` duration is from rcblock 8 to rcblock 12. This means that the + // coretime purchased during this time period will be purchased from `region1` + // + // `region2` duration is from rcblock 12 to rcblock 14 and during this period + // coretime will be purchased from `region2`. + assert_ok!(Broker::do_pool(region1, None, 2, Final)); + assert_ok!(Broker::do_pool(region2, None, 3, Final)); + assert_ok!(Broker::do_purchase_credit(1, 20, 1)); + advance_to(8); + assert_ok!(TestCoretimeProvider::spend_instantaneous(1, 10)); + advance_to(11); + assert_eq!(pot(), 20); + assert_eq!(revenue(), 100); + assert_ok!(Broker::do_claim_revenue(region1, 100)); + assert_eq!(pot(), 10); + assert_eq!(balance(2), 10); + advance_to(12); + assert_ok!(TestCoretimeProvider::spend_instantaneous(1, 10)); + advance_to(15); + assert_eq!(pot(), 10); + assert_ok!(Broker::do_claim_revenue(region2, 100)); + assert_eq!(pot(), 0); + // The balance of account `2` remains unchanged. + assert_eq!(balance(2), 10); + assert_eq!(balance(3), 10); + }); +} + #[test] fn initialize_with_system_paras_works() { TestExt::new().execute_with(|| { @@ -654,6 +694,79 @@ fn partition_then_interlace_works() { }); } +#[test] +fn partitioning_after_assignment_works() { + TestExt::new().endow(1, 1000).execute_with(|| { + assert_ok!(Broker::do_start_sales(100, 1)); + advance_to(2); + // We will initially allocate a task to a purchased region, and after that + // we will proceed to partition the region. + let region = Broker::do_purchase(1, u64::max_value()).unwrap(); + assert_ok!(Broker::do_assign(region, None, 1001, Provisional)); + let (_region, region1) = Broker::do_partition(region, None, 2).unwrap(); + // After the partitioning if we assign a new task to `region` the other region + // will still be assigned to `Task(1001)`. + assert_ok!(Broker::do_assign(region1, None, 1002, Provisional)); + advance_to(10); + assert_eq!( + CoretimeTrace::get(), + vec![ + ( + 6, + AssignCore { + core: 0, + begin: 8, + assignment: vec![(Task(1001), 57600),], + end_hint: None + } + ), + ( + 10, + AssignCore { + core: 0, + begin: 12, + assignment: vec![(Task(1002), 57600),], + end_hint: None + } + ), + ] + ); + }); +} + +#[test] +fn interlacing_after_assignment_works() { + TestExt::new().endow(1, 1000).execute_with(|| { + assert_ok!(Broker::do_start_sales(100, 1)); + advance_to(2); + // We will initially allocate a task to a purchased region, and after that + // we will proceed to interlace the region. + let region = Broker::do_purchase(1, u64::max_value()).unwrap(); + assert_ok!(Broker::do_assign(region, None, 1001, Provisional)); + let (region1, _region) = + Broker::do_interlace(region, None, CoreMask::from_chunk(0, 40)).unwrap(); + // Interlacing the region won't affect the assignment. The entire region will still + // be assigned to `Task(1001)`. + // + // However, after we assign a task to `region1` the `_region` won't be assigned + // to `Task(1001)` anymore. It will become idle. + assert_ok!(Broker::do_assign(region1, None, 1002, Provisional)); + advance_to(10); + assert_eq!( + CoretimeTrace::get(), + vec![( + 6, + AssignCore { + core: 0, + begin: 8, + assignment: vec![(Idle, 28800), (Task(1002), 28800)], + end_hint: None + } + ),] + ); + }); +} + #[test] fn reservations_are_limited() { TestExt::new().execute_with(|| { @@ -866,7 +979,7 @@ fn assign_should_drop_invalid_region() { advance_to(10); assert_ok!(Broker::do_assign(region, Some(1), 1001, Provisional)); region.begin = 7; - System::assert_last_event(Event::RegionDropped { region_id: region, duration: 0 }.into()); + System::assert_last_event(Event::RegionDropped { region_id: region, duration: 3 }.into()); }); } @@ -879,7 +992,7 @@ fn pool_should_drop_invalid_region() { advance_to(10); assert_ok!(Broker::do_pool(region, Some(1), 1001, Provisional)); region.begin = 7; - System::assert_last_event(Event::RegionDropped { region_id: region, duration: 0 }.into()); + System::assert_last_event(Event::RegionDropped { region_id: region, duration: 3 }.into()); }); } diff --git a/substrate/frame/broker/src/tick_impls.rs b/substrate/frame/broker/src/tick_impls.rs index 909af6caf734c9259427b3e28b98eecf0b02c7c6..7df8bd39d42fe13e6cf7375442d7ddab98c60d99 100644 --- a/substrate/frame/broker/src/tick_impls.rs +++ b/substrate/frame/broker/src/tick_impls.rs @@ -36,13 +36,14 @@ impl Pallet { /// - Request revenue information for a previous timeslice /// - Initialize an instantaneous core pool historical revenue record pub(crate) fn do_tick() -> Weight { + let mut meter = WeightMeter::new(); + meter.consume(T::WeightInfo::do_tick_base()); + let (mut status, config) = match (Status::::get(), Configuration::::get()) { (Some(s), Some(c)) => (s, c), - _ => return Weight::zero(), + _ => return meter.consumed(), }; - let mut meter = WeightMeter::new(); - if Self::process_core_count(&mut status) { meter.consume(T::WeightInfo::process_core_count(status.core_count.into())); } @@ -95,9 +96,7 @@ impl Pallet { } pub(crate) fn process_revenue() -> bool { - let Some((until, amount)) = T::Coretime::check_notify_revenue_info() else { - return false - }; + let Some((until, amount)) = T::Coretime::check_notify_revenue_info() else { return false }; let when: Timeslice = (until / T::TimeslicePeriod::get()).saturating_sub(One::one()).saturated_into(); let mut revenue = T::ConvertBalance::convert_back(amount); @@ -289,9 +288,7 @@ impl Pallet { rc_begin: RelayBlockNumberOf, core: CoreIndex, ) { - let Some(workplan) = Workplan::::take((timeslice, core)) else { - return - }; + let Some(workplan) = Workplan::::take((timeslice, core)) else { return }; let workload = Workload::::get(core); let parts_used = workplan.iter().map(|i| i.mask).fold(CoreMask::void(), |a, i| a | i); let mut workplan = workplan.into_inner(); diff --git a/substrate/frame/broker/src/utility_impls.rs b/substrate/frame/broker/src/utility_impls.rs index 99c4de32f77678f4df9568e7c5a2c442b255c4c8..2450198050b67fccc852bd06e116bcdc16628750 100644 --- a/substrate/frame/broker/src/utility_impls.rs +++ b/substrate/frame/broker/src/utility_impls.rs @@ -101,9 +101,9 @@ impl Pallet { let last_committed_timeslice = status.last_committed_timeslice; if region_id.begin <= last_committed_timeslice { + let duration = region.end.saturating_sub(region_id.begin); region_id.begin = last_committed_timeslice + 1; if region_id.begin >= region.end { - let duration = region.end.saturating_sub(region_id.begin); Self::deposit_event(Event::RegionDropped { region_id, duration }); return Ok(None) } diff --git a/substrate/frame/broker/src/weights.rs b/substrate/frame/broker/src/weights.rs index 93b568bf2a035af7542056026ab3c8238bfc6cfd..b3a151c6062c45f43f9ae1ccafa513a07f70f08f 100644 --- a/substrate/frame/broker/src/weights.rs +++ b/substrate/frame/broker/src/weights.rs @@ -18,10 +18,10 @@ //! Autogenerated weights for `pallet_broker` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-07-29, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-09-04, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-ynta1nyy-project-145-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: ``, WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` +//! HOSTNAME: `runner-pzhd7p6z-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` // Executed Command: // target/production/substrate-node @@ -32,12 +32,12 @@ // --extrinsic=* // --wasm-execution=compiled // --heap-pages=4096 -// --json-file=/builds/parity/mirrors/substrate/.git/.artifacts/bench.json +// --json-file=/builds/parity/mirrors/polkadot-sdk/.git/.artifacts/bench.json // --pallet=pallet_broker // --chain=dev -// --header=./HEADER-APACHE2 -// --output=./frame/broker/src/weights.rs -// --template=./.maintain/frame-weight-template.hbs +// --header=./substrate/HEADER-APACHE2 +// --output=./substrate/frame/broker/src/weights.rs +// --template=./substrate/.maintain/frame-weight-template.hbs #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -74,6 +74,7 @@ pub trait WeightInfo { fn process_pool() -> Weight; fn process_core_schedule() -> Weight; fn request_revenue_info_at() -> Weight; + fn do_tick_base() -> Weight; } /// Weights for `pallet_broker` using the Substrate node and recommended hardware. @@ -85,8 +86,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_448_000 picoseconds. - Weight::from_parts(3_729_000, 0) + // Minimum execution time: 3_040_000 picoseconds. + Weight::from_parts(3_344_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Broker::Reservations` (r:1 w:1) @@ -95,8 +96,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `5016` // Estimated: `7496` - // Minimum execution time: 22_537_000 picoseconds. - Weight::from_parts(23_335_000, 7496) + // Minimum execution time: 21_259_000 picoseconds. + Weight::from_parts(22_110_000, 7496) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -106,8 +107,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `6218` // Estimated: `7496` - // Minimum execution time: 21_668_000 picoseconds. - Weight::from_parts(22_442_000, 7496) + // Minimum execution time: 20_330_000 picoseconds. + Weight::from_parts(20_826_000, 7496) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -117,8 +118,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `239` // Estimated: `1526` - // Minimum execution time: 13_606_000 picoseconds. - Weight::from_parts(14_104_000, 1526) + // Minimum execution time: 13_411_000 picoseconds. + Weight::from_parts(13_960_000, 1526) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -137,12 +138,14 @@ impl WeightInfo for SubstrateWeight { /// Storage: `Broker::Workplan` (r:0 w:10) /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) /// The range of component `n` is `[0, 1000]`. - fn start_sales(_n: u32, ) -> Weight { + fn start_sales(n: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `6330` // Estimated: `8499` - // Minimum execution time: 64_012_000 picoseconds. - Weight::from_parts(67_819_922, 8499) + // Minimum execution time: 57_770_000 picoseconds. + Weight::from_parts(61_047_512, 8499) + // Standard Error: 165 + .saturating_add(Weight::from_parts(3, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(6_u64)) .saturating_add(T::DbWeight::get().writes(16_u64)) } @@ -160,8 +163,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `568` // Estimated: `2053` - // Minimum execution time: 48_110_000 picoseconds. - Weight::from_parts(49_234_000, 2053) + // Minimum execution time: 51_196_000 picoseconds. + Weight::from_parts(52_382_000, 2053) .saturating_add(T::DbWeight::get().reads(4_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -183,8 +186,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `686` // Estimated: `4698` - // Minimum execution time: 69_580_000 picoseconds. - Weight::from_parts(70_914_000, 4698) + // Minimum execution time: 71_636_000 picoseconds. + Weight::from_parts(73_679_000, 4698) .saturating_add(T::DbWeight::get().reads(6_u64)) .saturating_add(T::DbWeight::get().writes(4_u64)) } @@ -194,8 +197,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 17_687_000 picoseconds. - Weight::from_parts(18_573_000, 3550) + // Minimum execution time: 19_182_000 picoseconds. + Weight::from_parts(19_775_000, 3550) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -205,8 +208,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 19_675_000 picoseconds. - Weight::from_parts(20_234_000, 3550) + // Minimum execution time: 20_688_000 picoseconds. + Weight::from_parts(21_557_000, 3550) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -216,8 +219,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 19_426_000 picoseconds. - Weight::from_parts(20_414_000, 3550) + // Minimum execution time: 21_190_000 picoseconds. + Weight::from_parts(22_215_000, 3550) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -233,8 +236,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `740` // Estimated: `4681` - // Minimum execution time: 31_751_000 picoseconds. - Weight::from_parts(32_966_000, 4681) + // Minimum execution time: 34_591_000 picoseconds. + Weight::from_parts(36_227_000, 4681) .saturating_add(T::DbWeight::get().reads(4_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -252,8 +255,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `775` // Estimated: `5996` - // Minimum execution time: 36_709_000 picoseconds. - Weight::from_parts(38_930_000, 5996) + // Minimum execution time: 40_346_000 picoseconds. + Weight::from_parts(41_951_000, 5996) .saturating_add(T::DbWeight::get().reads(5_u64)) .saturating_add(T::DbWeight::get().writes(5_u64)) } @@ -261,20 +264,20 @@ impl WeightInfo for SubstrateWeight { /// Proof: `Broker::InstaPoolContribution` (`max_values`: None, `max_size`: Some(68), added: 2543, mode: `MaxEncodedLen`) /// Storage: `Broker::InstaPoolHistory` (r:3 w:1) /// Proof: `Broker::InstaPoolHistory` (`max_values`: None, `max_size`: Some(45), added: 2520, mode: `MaxEncodedLen`) - /// Storage: `System::Account` (r:2 w:0) + /// Storage: `System::Account` (r:2 w:2) /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) /// The range of component `m` is `[1, 3]`. fn claim_revenue(m: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `720` + // Measured: `859` // Estimated: `6196 + m * (2520 ±0)` - // Minimum execution time: 55_510_000 picoseconds. - Weight::from_parts(56_665_061, 6196) - // Standard Error: 61_729 - .saturating_add(Weight::from_parts(1_724_824, 0).saturating_mul(m.into())) + // Minimum execution time: 75_734_000 picoseconds. + Weight::from_parts(78_168_395, 6196) + // Standard Error: 63_180 + .saturating_add(Weight::from_parts(1_076_259, 0).saturating_mul(m.into())) .saturating_add(T::DbWeight::get().reads(3_u64)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(m.into()))) - .saturating_add(T::DbWeight::get().writes(3_u64)) + .saturating_add(T::DbWeight::get().writes(5_u64)) .saturating_add(Weight::from_parts(0, 2520).saturating_mul(m.into())) } /// Storage: `System::Account` (r:1 w:1) @@ -283,8 +286,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `103` // Estimated: `3593` - // Minimum execution time: 44_992_000 picoseconds. - Weight::from_parts(46_225_000, 3593) + // Minimum execution time: 46_383_000 picoseconds. + Weight::from_parts(47_405_000, 3593) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -296,8 +299,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `603` // Estimated: `3550` - // Minimum execution time: 28_207_000 picoseconds. - Weight::from_parts(28_707_000, 3550) + // Minimum execution time: 30_994_000 picoseconds. + Weight::from_parts(31_979_000, 3550) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -311,8 +314,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `601` // Estimated: `3533` - // Minimum execution time: 31_813_000 picoseconds. - Weight::from_parts(32_612_000, 3533) + // Minimum execution time: 37_584_000 picoseconds. + Weight::from_parts(44_010_000, 3533) .saturating_add(T::DbWeight::get().reads(3_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -326,10 +329,10 @@ impl WeightInfo for SubstrateWeight { /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) fn drop_history() -> Weight { // Proof Size summary in bytes: - // Measured: `829` + // Measured: `830` // Estimated: `3593` - // Minimum execution time: 38_571_000 picoseconds. - Weight::from_parts(39_493_000, 3593) + // Minimum execution time: 45_266_000 picoseconds. + Weight::from_parts(48_000_000, 3593) .saturating_add(T::DbWeight::get().reads(4_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -341,42 +344,53 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `525` // Estimated: `4698` - // Minimum execution time: 24_714_000 picoseconds. - Weight::from_parts(25_288_000, 4698) + // Minimum execution time: 25_365_000 picoseconds. + Weight::from_parts(26_920_000, 4698) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:0 w:1) - /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:0 w:1) /// The range of component `n` is `[0, 1000]`. - fn request_core_count(_n: u32, ) -> Weight { + fn request_core_count(n: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 7_258_000 picoseconds. - Weight::from_parts(7_925_570, 0) - .saturating_add(T::DbWeight::get().writes(1_u64)) + // Minimum execution time: 6_519_000 picoseconds. + Weight::from_parts(7_098_698, 0) + // Standard Error: 20 + .saturating_add(Weight::from_parts(8, 0).saturating_mul(n.into())) } - /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:0) - /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:0) + /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) /// The range of component `n` is `[0, 1000]`. - fn process_core_count(_n: u32, ) -> Weight { - // Proof Size summary in bytes: - // Measured: `97` - // Estimated: `3562` - // Minimum execution time: 7_136_000 picoseconds. - Weight::from_parts(7_788_194, 3562) + fn process_core_count(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `98` + // Estimated: `3563` + // Minimum execution time: 7_608_000 picoseconds. + Weight::from_parts(8_157_815, 3563) + // Standard Error: 26 + .saturating_add(Weight::from_parts(48, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: `Broker::InstaPoolHistory` (r:0 w:1) + /// Storage: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Proof: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Storage: `Broker::InstaPoolHistory` (r:1 w:1) /// Proof: `Broker::InstaPoolHistory` (`max_values`: None, `max_size`: Some(45), added: 2520, mode: `MaxEncodedLen`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + /// Storage: `Authorship::Author` (r:1 w:0) + /// Proof: `Authorship::Author` (`max_values`: Some(1), `max_size`: Some(32), added: 527, mode: `MaxEncodedLen`) + /// Storage: `System::Digest` (r:1 w:0) + /// Proof: `System::Digest` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) fn process_revenue() -> Weight { // Proof Size summary in bytes: - // Measured: `0` - // Estimated: `0` - // Minimum execution time: 6_049_000 picoseconds. - Weight::from_parts(6_311_000, 0) - .saturating_add(T::DbWeight::get().writes(1_u64)) + // Measured: `905` + // Estimated: `4370` + // Minimum execution time: 59_993_000 picoseconds. + Weight::from_parts(61_752_000, 4370) + .saturating_add(T::DbWeight::get().reads(5_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) } /// Storage: `Broker::InstaPoolIo` (r:3 w:3) /// Proof: `Broker::InstaPoolIo` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) @@ -393,10 +407,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `6281` // Estimated: `8499` - // Minimum execution time: 47_504_000 picoseconds. - Weight::from_parts(49_778_098, 8499) - // Standard Error: 109 - .saturating_add(Weight::from_parts(427, 0).saturating_mul(n.into())) + // Minimum execution time: 41_863_000 picoseconds. + Weight::from_parts(44_033_031, 8499) + // Standard Error: 116 + .saturating_add(Weight::from_parts(764, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(5_u64)) .saturating_add(T::DbWeight::get().writes(15_u64)) } @@ -408,8 +422,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `180` // Estimated: `3493` - // Minimum execution time: 9_573_000 picoseconds. - Weight::from_parts(10_034_000, 3493) + // Minimum execution time: 9_588_000 picoseconds. + Weight::from_parts(9_925_000, 3493) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -421,8 +435,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1423` // Estimated: `4681` - // Minimum execution time: 21_331_000 picoseconds. - Weight::from_parts(22_235_000, 4681) + // Minimum execution time: 19_308_000 picoseconds. + Weight::from_parts(20_482_000, 4681) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -430,8 +444,25 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 191_000 picoseconds. - Weight::from_parts(234_000, 0) + // Minimum execution time: 147_000 picoseconds. + Weight::from_parts(184_000, 0) + } + /// Storage: `Broker::Status` (r:1 w:1) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Configuration` (r:1 w:0) + /// Proof: `Broker::Configuration` (`max_values`: Some(1), `max_size`: Some(31), added: 526, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Storage: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Proof: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + fn do_tick_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `699` + // Estimated: `4164` + // Minimum execution time: 19_824_000 picoseconds. + Weight::from_parts(20_983_000, 4164) + .saturating_add(T::DbWeight::get().reads(4_u64)) + .saturating_add(T::DbWeight::get().writes(3_u64)) } } @@ -443,8 +474,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_448_000 picoseconds. - Weight::from_parts(3_729_000, 0) + // Minimum execution time: 3_040_000 picoseconds. + Weight::from_parts(3_344_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Broker::Reservations` (r:1 w:1) @@ -453,8 +484,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `5016` // Estimated: `7496` - // Minimum execution time: 22_537_000 picoseconds. - Weight::from_parts(23_335_000, 7496) + // Minimum execution time: 21_259_000 picoseconds. + Weight::from_parts(22_110_000, 7496) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -464,8 +495,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `6218` // Estimated: `7496` - // Minimum execution time: 21_668_000 picoseconds. - Weight::from_parts(22_442_000, 7496) + // Minimum execution time: 20_330_000 picoseconds. + Weight::from_parts(20_826_000, 7496) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -475,8 +506,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `239` // Estimated: `1526` - // Minimum execution time: 13_606_000 picoseconds. - Weight::from_parts(14_104_000, 1526) + // Minimum execution time: 13_411_000 picoseconds. + Weight::from_parts(13_960_000, 1526) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -495,12 +526,14 @@ impl WeightInfo for () { /// Storage: `Broker::Workplan` (r:0 w:10) /// Proof: `Broker::Workplan` (`max_values`: None, `max_size`: Some(1216), added: 3691, mode: `MaxEncodedLen`) /// The range of component `n` is `[0, 1000]`. - fn start_sales(_n: u32, ) -> Weight { + fn start_sales(n: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `6330` // Estimated: `8499` - // Minimum execution time: 64_012_000 picoseconds. - Weight::from_parts(67_819_922, 8499) + // Minimum execution time: 57_770_000 picoseconds. + Weight::from_parts(61_047_512, 8499) + // Standard Error: 165 + .saturating_add(Weight::from_parts(3, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(6_u64)) .saturating_add(RocksDbWeight::get().writes(16_u64)) } @@ -518,8 +551,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `568` // Estimated: `2053` - // Minimum execution time: 48_110_000 picoseconds. - Weight::from_parts(49_234_000, 2053) + // Minimum execution time: 51_196_000 picoseconds. + Weight::from_parts(52_382_000, 2053) .saturating_add(RocksDbWeight::get().reads(4_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -541,8 +574,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `686` // Estimated: `4698` - // Minimum execution time: 69_580_000 picoseconds. - Weight::from_parts(70_914_000, 4698) + // Minimum execution time: 71_636_000 picoseconds. + Weight::from_parts(73_679_000, 4698) .saturating_add(RocksDbWeight::get().reads(6_u64)) .saturating_add(RocksDbWeight::get().writes(4_u64)) } @@ -552,8 +585,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 17_687_000 picoseconds. - Weight::from_parts(18_573_000, 3550) + // Minimum execution time: 19_182_000 picoseconds. + Weight::from_parts(19_775_000, 3550) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -563,8 +596,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 19_675_000 picoseconds. - Weight::from_parts(20_234_000, 3550) + // Minimum execution time: 20_688_000 picoseconds. + Weight::from_parts(21_557_000, 3550) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -574,8 +607,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `495` // Estimated: `3550` - // Minimum execution time: 19_426_000 picoseconds. - Weight::from_parts(20_414_000, 3550) + // Minimum execution time: 21_190_000 picoseconds. + Weight::from_parts(22_215_000, 3550) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -591,8 +624,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `740` // Estimated: `4681` - // Minimum execution time: 31_751_000 picoseconds. - Weight::from_parts(32_966_000, 4681) + // Minimum execution time: 34_591_000 picoseconds. + Weight::from_parts(36_227_000, 4681) .saturating_add(RocksDbWeight::get().reads(4_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -610,8 +643,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `775` // Estimated: `5996` - // Minimum execution time: 36_709_000 picoseconds. - Weight::from_parts(38_930_000, 5996) + // Minimum execution time: 40_346_000 picoseconds. + Weight::from_parts(41_951_000, 5996) .saturating_add(RocksDbWeight::get().reads(5_u64)) .saturating_add(RocksDbWeight::get().writes(5_u64)) } @@ -619,20 +652,20 @@ impl WeightInfo for () { /// Proof: `Broker::InstaPoolContribution` (`max_values`: None, `max_size`: Some(68), added: 2543, mode: `MaxEncodedLen`) /// Storage: `Broker::InstaPoolHistory` (r:3 w:1) /// Proof: `Broker::InstaPoolHistory` (`max_values`: None, `max_size`: Some(45), added: 2520, mode: `MaxEncodedLen`) - /// Storage: `System::Account` (r:2 w:0) + /// Storage: `System::Account` (r:2 w:2) /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) /// The range of component `m` is `[1, 3]`. fn claim_revenue(m: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `720` + // Measured: `859` // Estimated: `6196 + m * (2520 ±0)` - // Minimum execution time: 55_510_000 picoseconds. - Weight::from_parts(56_665_061, 6196) - // Standard Error: 61_729 - .saturating_add(Weight::from_parts(1_724_824, 0).saturating_mul(m.into())) + // Minimum execution time: 75_734_000 picoseconds. + Weight::from_parts(78_168_395, 6196) + // Standard Error: 63_180 + .saturating_add(Weight::from_parts(1_076_259, 0).saturating_mul(m.into())) .saturating_add(RocksDbWeight::get().reads(3_u64)) .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(m.into()))) - .saturating_add(RocksDbWeight::get().writes(3_u64)) + .saturating_add(RocksDbWeight::get().writes(5_u64)) .saturating_add(Weight::from_parts(0, 2520).saturating_mul(m.into())) } /// Storage: `System::Account` (r:1 w:1) @@ -641,8 +674,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `103` // Estimated: `3593` - // Minimum execution time: 44_992_000 picoseconds. - Weight::from_parts(46_225_000, 3593) + // Minimum execution time: 46_383_000 picoseconds. + Weight::from_parts(47_405_000, 3593) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -654,8 +687,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `603` // Estimated: `3550` - // Minimum execution time: 28_207_000 picoseconds. - Weight::from_parts(28_707_000, 3550) + // Minimum execution time: 30_994_000 picoseconds. + Weight::from_parts(31_979_000, 3550) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -669,8 +702,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `601` // Estimated: `3533` - // Minimum execution time: 31_813_000 picoseconds. - Weight::from_parts(32_612_000, 3533) + // Minimum execution time: 37_584_000 picoseconds. + Weight::from_parts(44_010_000, 3533) .saturating_add(RocksDbWeight::get().reads(3_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -684,10 +717,10 @@ impl WeightInfo for () { /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) fn drop_history() -> Weight { // Proof Size summary in bytes: - // Measured: `829` + // Measured: `830` // Estimated: `3593` - // Minimum execution time: 38_571_000 picoseconds. - Weight::from_parts(39_493_000, 3593) + // Minimum execution time: 45_266_000 picoseconds. + Weight::from_parts(48_000_000, 3593) .saturating_add(RocksDbWeight::get().reads(4_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -699,42 +732,53 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `525` // Estimated: `4698` - // Minimum execution time: 24_714_000 picoseconds. - Weight::from_parts(25_288_000, 4698) + // Minimum execution time: 25_365_000 picoseconds. + Weight::from_parts(26_920_000, 4698) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:0 w:1) - /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:0 w:1) /// The range of component `n` is `[0, 1000]`. - fn request_core_count(_n: u32, ) -> Weight { + fn request_core_count(n: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 7_258_000 picoseconds. - Weight::from_parts(7_925_570, 0) - .saturating_add(RocksDbWeight::get().writes(1_u64)) + // Minimum execution time: 6_519_000 picoseconds. + Weight::from_parts(7_098_698, 0) + // Standard Error: 20 + .saturating_add(Weight::from_parts(8, 0).saturating_mul(n.into())) } - /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:0) - /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:0) + /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) /// The range of component `n` is `[0, 1000]`. - fn process_core_count(_n: u32, ) -> Weight { - // Proof Size summary in bytes: - // Measured: `97` - // Estimated: `3562` - // Minimum execution time: 7_136_000 picoseconds. - Weight::from_parts(7_788_194, 3562) + fn process_core_count(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `98` + // Estimated: `3563` + // Minimum execution time: 7_608_000 picoseconds. + Weight::from_parts(8_157_815, 3563) + // Standard Error: 26 + .saturating_add(Weight::from_parts(48, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: `Broker::InstaPoolHistory` (r:0 w:1) + /// Storage: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Proof: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Storage: `Broker::InstaPoolHistory` (r:1 w:1) /// Proof: `Broker::InstaPoolHistory` (`max_values`: None, `max_size`: Some(45), added: 2520, mode: `MaxEncodedLen`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + /// Storage: `Authorship::Author` (r:1 w:0) + /// Proof: `Authorship::Author` (`max_values`: Some(1), `max_size`: Some(32), added: 527, mode: `MaxEncodedLen`) + /// Storage: `System::Digest` (r:1 w:0) + /// Proof: `System::Digest` (`max_values`: Some(1), `max_size`: None, mode: `Measured`) fn process_revenue() -> Weight { // Proof Size summary in bytes: - // Measured: `0` - // Estimated: `0` - // Minimum execution time: 6_049_000 picoseconds. - Weight::from_parts(6_311_000, 0) - .saturating_add(RocksDbWeight::get().writes(1_u64)) + // Measured: `905` + // Estimated: `4370` + // Minimum execution time: 59_993_000 picoseconds. + Weight::from_parts(61_752_000, 4370) + .saturating_add(RocksDbWeight::get().reads(5_u64)) + .saturating_add(RocksDbWeight::get().writes(3_u64)) } /// Storage: `Broker::InstaPoolIo` (r:3 w:3) /// Proof: `Broker::InstaPoolIo` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) @@ -751,10 +795,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `6281` // Estimated: `8499` - // Minimum execution time: 47_504_000 picoseconds. - Weight::from_parts(49_778_098, 8499) - // Standard Error: 109 - .saturating_add(Weight::from_parts(427, 0).saturating_mul(n.into())) + // Minimum execution time: 41_863_000 picoseconds. + Weight::from_parts(44_033_031, 8499) + // Standard Error: 116 + .saturating_add(Weight::from_parts(764, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(5_u64)) .saturating_add(RocksDbWeight::get().writes(15_u64)) } @@ -766,8 +810,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `180` // Estimated: `3493` - // Minimum execution time: 9_573_000 picoseconds. - Weight::from_parts(10_034_000, 3493) + // Minimum execution time: 9_588_000 picoseconds. + Weight::from_parts(9_925_000, 3493) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -779,8 +823,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1423` // Estimated: `4681` - // Minimum execution time: 21_331_000 picoseconds. - Weight::from_parts(22_235_000, 4681) + // Minimum execution time: 19_308_000 picoseconds. + Weight::from_parts(20_482_000, 4681) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -788,7 +832,24 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 191_000 picoseconds. - Weight::from_parts(234_000, 0) + // Minimum execution time: 147_000 picoseconds. + Weight::from_parts(184_000, 0) + } + /// Storage: `Broker::Status` (r:1 w:1) + /// Proof: `Broker::Status` (`max_values`: Some(1), `max_size`: Some(18), added: 513, mode: `MaxEncodedLen`) + /// Storage: `Broker::Configuration` (r:1 w:0) + /// Proof: `Broker::Configuration` (`max_values`: Some(1), `max_size`: Some(31), added: 526, mode: `MaxEncodedLen`) + /// Storage: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Proof: UNKNOWN KEY `0x18194fcb5c1fcace44d2d0a004272614` (r:1 w:1) + /// Storage: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + /// Proof: UNKNOWN KEY `0xf308d869daf021a7724e69c557dd8dbe` (r:1 w:1) + fn do_tick_base() -> Weight { + // Proof Size summary in bytes: + // Measured: `699` + // Estimated: `4164` + // Minimum execution time: 19_824_000 picoseconds. + Weight::from_parts(20_983_000, 4164) + .saturating_add(RocksDbWeight::get().reads(4_u64)) + .saturating_add(RocksDbWeight::get().writes(3_u64)) } } diff --git a/substrate/frame/child-bounties/README.md b/substrate/frame/child-bounties/README.md index 695b6616b1751f087277ee8872ae6536afd560c2..cf62698dcaf91eed3adfaabc3b1530493873960d 100644 --- a/substrate/frame/child-bounties/README.md +++ b/substrate/frame/child-bounties/README.md @@ -8,7 +8,7 @@ With child bounties, a large bounty proposal can be divided into smaller chunks, for parallel execution, and for efficient governance and tracking of spent funds. A child bounty is a smaller piece of work, extracted from a parent bounty. A curator is assigned after the child bounty is created by the parent bounty curator, -to be delegated with the responsibility of assigning a payout address once +to be delegated with the responsibility of assigning a payout address once the specified set of tasks is completed. ## Interface diff --git a/substrate/frame/contracts/CHANGELOG.md b/substrate/frame/contracts/CHANGELOG.md index dcb9d6d4d2b206b97cd271195f6e6a8cb86c56a0..aca94e5b149152d5c598102c633048baa17d1276 100644 --- a/substrate/frame/contracts/CHANGELOG.md +++ b/substrate/frame/contracts/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -The semantic versioning guarantees cover the interface to the substrate runtime which +The semantic versioning guarantees cover the interface to the Substrate runtime which includes this pallet as a dependency. This module will also add storage migrations whenever changes require it. Stability with regard to offchain tooling is explicitly excluded from this guarantee: For example adding a new field to an in-storage data structure will require diff --git a/substrate/frame/contracts/Cargo.toml b/substrate/frame/contracts/Cargo.toml index 237ab9303e25058d3894d6ad8d5eef62a6b7bf93..d5c809e1bf7cf3df4b3e47c3f0995b429e392320 100644 --- a/substrate/frame/contracts/Cargo.toml +++ b/substrate/frame/contracts/Cargo.toml @@ -9,7 +9,7 @@ homepage = "https://substrate.io" repository.workspace = true description = "FRAME pallet for WASM contracts" readme = "README.md" -include = ["src/**/*", "README.md", "CHANGELOG.md"] +include = ["src/**/*", "build.rs", "README.md", "CHANGELOG.md"] [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/frame/contracts/README.md b/substrate/frame/contracts/README.md index aeb30cef32fc8cd9cdd3d65f24fcd98be151b7dd..0b6548cf6414a2bfbe253c0b18a95814b2dfb63b 100644 --- a/substrate/frame/contracts/README.md +++ b/substrate/frame/contracts/README.md @@ -9,66 +9,68 @@ The Contracts module provides functionality for the runtime to deploy and execut ## Overview -This module extends accounts based on the [`frame_support::traits::fungible`] traits to have smart-contract functionality. It can -be used with other modules that implement accounts based on [`frame_support::traits::fungible`]. These "smart-contract accounts" -have the ability to instantiate smart-contracts and make calls to other contract and non-contract accounts. +This module extends accounts based on the [`frame_support::traits::fungible`] traits to have smart-contract +functionality. It can be used with other modules that implement accounts based on [`frame_support::traits::fungible`]. +These "smart-contract accounts" have the ability to instantiate smart-contracts and make calls to other contract and +non-contract accounts. -The smart-contract code is stored once, and later retrievable via its `code_hash`. -This means that multiple smart-contracts can be instantiated from the same `code`, without replicating -the code each time. +The smart-contract code is stored once, and later retrievable via its `code_hash`. This means that multiple +smart-contracts can be instantiated from the same `code`, without replicating the code each time. -When a smart-contract is called, its associated code is retrieved via the code hash and gets executed. -This call can alter the storage entries of the smart-contract account, instantiate new smart-contracts, -or call other smart-contracts. +When a smart-contract is called, its associated code is retrieved via the code hash and gets executed. This call can +alter the storage entries of the smart-contract account, instantiate new smart-contracts, or call other smart-contracts. -Finally, when an account is reaped, its associated code and storage of the smart-contract account -will also be deleted. +Finally, when an account is reaped, its associated code and storage of the smart-contract account will also be deleted. ### Weight -Senders must specify a [`Weight`](https://paritytech.github.io/substrate/master/sp_weights/struct.Weight.html) limit with every call, as all instructions invoked by the smart-contract require weight. -Unused weight is refunded after the call, regardless of the execution outcome. +Senders must specify a [`Weight`](https://paritytech.github.io/substrate/master/sp_weights/struct.Weight.html) limit +with every call, as all instructions invoked by the smart-contract require weight. Unused weight is refunded after the +call, regardless of the execution outcome. -If the weight limit is reached, then all calls and state changes (including balance transfers) are only -reverted at the current call's contract level. For example, if contract A calls B and B runs out of weight mid-call, -then all of B's calls are reverted. Assuming correct error handling by contract A, A's other calls and state -changes still persist. +If the weight limit is reached, then all calls and state changes (including balance transfers) are only reverted at the +current call's contract level. For example, if contract A calls B and B runs out of weight mid-call, then all of B's +calls are reverted. Assuming correct error handling by contract A, A's other calls and state changes still persist. One `ref_time` `Weight` is defined as one picosecond of execution time on the runtime's reference machine. ### Revert Behaviour -Contract call failures are not cascading. When failures occur in a sub-call, they do not "bubble up", -and the call will only revert at the specific contract level. For example, if contract A calls contract B, and B -fails, A can decide how to handle that failure, either proceeding or reverting A's changes. +Contract call failures are not cascading. When failures occur in a sub-call, they do not "bubble up", and the call will +only revert at the specific contract level. For example, if contract A calls contract B, and B fails, A can decide how +to handle that failure, either proceeding or reverting A's changes. ### Off-chain Execution -In general, a contract execution needs to be deterministic so that all nodes come to the same -conclusion when executing it. To that end we disallow any instructions that could cause -indeterminism. Most notable are any floating point arithmetic. That said, sometimes contracts -are executed off-chain and hence are not subject to consensus. If code is only executed by a -single node and implicitly trusted by other actors is such a case. Trusted execution environments -come to mind. To that end we allow the execution of indeterminstic code for off-chain usages -with the following constraints: - -1. No contract can ever be instantiated from an indeterministic code. The only way to execute -the code is to use a delegate call from a deterministic contract. -2. The code that wants to use this feature needs to depend on `pallet-contracts` and use [`bare_call()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.bare_call) +In general, a contract execution needs to be deterministic so that all nodes come to the same conclusion when executing +it. To that end we disallow any instructions that could cause indeterminism. Most notable are any floating point +arithmetic. That said, sometimes contracts are executed off-chain and hence are not subject to consensus. If code is +only executed by a single node and implicitly trusted by other actors is such a case. Trusted execution environments +come to mind. To that end we allow the execution of indeterminstic code for off-chain usages with the following +constraints: + +1. No contract can ever be instantiated from an indeterministic code. The only way to execute the code is to use a +delegate call from a deterministic contract. +2. The code that wants to use this feature needs to depend on `pallet-contracts` and use +[`bare_call()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.bare_call) directly. This makes sure that by default `pallet-contracts` does not expose any indeterminism. #### How to use -An indeterministic code can be deployed on-chain by passing `Determinism::Relaxed` -to [`upload_code()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.upload_code). A deterministic contract can then delegate call into it if and only if it -is ran by using [`bare_call()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.bare_call) and passing [`Determinism::Relaxed`](https://paritytech.github.io/substrate/master/pallet_contracts/enum.Determinism.html#variant.Relaxed) to it. **Never use -this argument when the contract is called from an on-chain transaction.** +An indeterministic code can be deployed on-chain by passing `Determinism::Relaxed` to +[`upload_code()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.upload_code). +A deterministic contract can then delegate call into it if and only if it is ran by using +[`bare_call()`](https://paritytech.github.io/substrate/master/pallet_contracts/pallet/struct.Pallet.html#method.bare_call) +and passing +[`Determinism::Relaxed`](https://paritytech.github.io/substrate/master/pallet_contracts/enum.Determinism.html#variant.Relaxed) +to it. **Never use this argument when the contract is called from an on-chain transaction.** ## Interface ### Dispatchable functions -Those are documented in the [reference documentation](https://paritytech.github.io/substrate/master/pallet_contracts/index.html#dispatchable-functions). +Those are documented in the [reference +documentation](https://paritytech.github.io/substrate/master/pallet_contracts/index.html#dispatchable-functions). ### Interface exposed to contracts @@ -99,43 +101,43 @@ The documentation of all importable functions can be found ## Usage -This module executes WebAssembly smart contracts. These can potentially be written in any language -that compiles to Wasm. However, using a language that specifically targets this module -will make things a lot easier. One such language is [`ink!`](https://use.ink). It enables -writing WebAssembly-based smart-contracts in the Rust programming language. +This module executes WebAssembly smart contracts. These can potentially be written in any language that compiles to +Wasm. However, using a language that specifically targets this module will make things a lot easier. One such language +is [`ink!`](https://use.ink). It enables writing WebAssembly-based smart-contracts in the Rust programming language. ## Debugging -Contracts can emit messages to the client when called as RPC through the [`debug_message`](https://paritytech.github.io/substrate/master/pallet_contracts/api_doc/trait.Current.html#tymethod.debug_message) +Contracts can emit messages to the client when called as RPC through the +[`debug_message`](https://paritytech.github.io/substrate/master/pallet_contracts/api_doc/trait.Current.html#tymethod.debug_message) API. This is exposed in [ink!](https://use.ink) via [`ink_env::debug_message()`](https://paritytech.github.io/ink/ink_env/fn.debug_message.html). -Those messages are gathered into an internal buffer and sent to the RPC client. -It is up the the individual client if and how those messages are presented to the user. +Those messages are gathered into an internal buffer and sent to the RPC client. It is up the the individual client if +and how those messages are presented to the user. -This buffer is also printed as a debug message. In order to see these messages on the node -console the log level for the `runtime::contracts` target needs to be raised to at least -the `debug` level. However, those messages are easy to overlook because of the noise generated -by block production. A good starting point for observing them on the console is using this -command line in the root directory of the substrate repository: +This buffer is also printed as a debug message. In order to see these messages on the node console the log level for the +`runtime::contracts` target needs to be raised to at least the `debug` level. However, those messages are easy to +overlook because of the noise generated by block production. A good starting point for observing them on the console is +using this command line in the root directory of the Substrate repository: ```bash cargo run --release -- --dev -lerror,runtime::contracts=debug ``` -This raises the log level of `runtime::contracts` to `debug` and all other targets -to `error` in order to prevent them from spamming the console. +This raises the log level of `runtime::contracts` to `debug` and all other targets to `error` in order to prevent them +from spamming the console. -`--dev`: Use a dev chain spec -`--tmp`: Use temporary storage for chain data (the chain state is deleted on exit) +`--dev`: Use a dev chain spec `--tmp`: Use temporary storage for chain data (the chain state is deleted on exit) ## Host function tracing -For contract authors, it can be a helpful debugging tool to see which host functions are called, with which arguments, and what the result was. +For contract authors, it can be a helpful debugging tool to see which host functions are called, with which arguments, +and what the result was. -In order to see these messages on the node console, the log level for the `runtime::contracts::strace` target needs to be raised to the `trace` level. +In order to see these messages on the node console, the log level for the `runtime::contracts::strace` target needs to +be raised to the `trace` level. -Example: +Example: ```bash cargo run --release -- --dev -lerror,runtime::contracts::strace=trace,runtime::contracts=debug @@ -143,18 +145,16 @@ cargo run --release -- --dev -lerror,runtime::contracts::strace=trace,runtime::c ## Unstable Interfaces -Driven by the desire to have an iterative approach in developing new contract interfaces -this pallet contains the concept of an unstable interface. Akin to the rust nightly compiler -it allows us to add new interfaces but mark them as unstable so that contract languages can -experiment with them and give feedback before we stabilize those. +Driven by the desire to have an iterative approach in developing new contract interfaces this pallet contains the +concept of an unstable interface. Akin to the rust nightly compiler it allows us to add new interfaces but mark them as +unstable so that contract languages can experiment with them and give feedback before we stabilize those. In order to access interfaces marked as `#[unstable]` in [`runtime.rs`](src/wasm/runtime.rs) one need to set -`pallet_contracts::Config::UnsafeUnstableInterface` to `ConstU32`. **It should be obvious -that any production runtime should never be compiled with this feature: In addition to be -subject to change or removal those interfaces might not have proper weights associated with -them and are therefore considered unsafe**. +`pallet_contracts::Config::UnsafeUnstableInterface` to `ConstU32`. **It should be obvious that any production +runtime should never be compiled with this feature: In addition to be subject to change or removal those interfaces +might not have proper weights associated with them and are therefore considered unsafe**. -New interfaces are generally added as unstable and might go through several iterations -before they are promoted to a stable interface. +New interfaces are generally added as unstable and might go through several iterations before they are promoted to a +stable interface. License: Apache-2.0 diff --git a/substrate/frame/contracts/benchmarks/README.md b/substrate/frame/contracts/benchmarks/README.md index a621dd65d593143d48b6ca16d5820ce09eb99e3d..e4441d6bab2c425a37639830f12ca303db30d1fa 100644 --- a/substrate/frame/contracts/benchmarks/README.md +++ b/substrate/frame/contracts/benchmarks/README.md @@ -1,9 +1,8 @@ # Benchmarks -This directory contains real world ([ink!](https://use.ink), [solang](https://github.com/hyperledger/solang)) contracts which are used in macro benchmarks. -Those benchmarks are not used to determine weights but rather to compare different contract -languages and execution engines with larger wasm modules. - -Files in this directory are used by `#[extra]` benchmarks in `src/benchmarking`. The json -files are for informational purposes only and are not consumed by the benchmarks. +This directory contains real world ([ink!](https://use.ink), [solang](https://github.com/hyperledger/solang)) contracts +which are used in macro benchmarks. Those benchmarks are not used to determine weights but rather to compare different +contract languages and execution engines with larger wasm modules. +Files in this directory are used by `#[extra]` benchmarks in `src/benchmarking`. The json files are for informational +purposes only and are not consumed by the benchmarks. diff --git a/substrate/frame/contracts/build.rs b/substrate/frame/contracts/build.rs index 7817ace9c98e2e877080d05552914f39d89eec85..42bc45d563d325a605c79749f096b5048a412af5 100644 --- a/substrate/frame/contracts/build.rs +++ b/substrate/frame/contracts/build.rs @@ -53,7 +53,7 @@ fn get_latest_version() -> u16 { fn main() -> Result<(), Box> { let out_dir = std::env::var("OUT_DIR")?; let path = std::path::Path::new(&out_dir).join("migration_codegen.rs"); - let mut f = std::fs::File::create(&path)?; + let mut f = std::fs::File::create(path)?; let version = get_latest_version(); write!( f, diff --git a/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat b/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat new file mode 100644 index 0000000000000000000000000000000000000000..6591d7ede78c20e0d19c7c60c183b4c58e98d2dc --- /dev/null +++ b/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat @@ -0,0 +1,10 @@ +(module + (import "env" "memory" (memory 1 1)) + (start $start) + (func $start + (loop $inf (br $inf)) ;; just run out of gas + (unreachable) + ) + (func (export "call")) + (func (export "deploy")) +) diff --git a/substrate/frame/contracts/primitives/README.md b/substrate/frame/contracts/primitives/README.md index 12718cd86425b4820dc63ce1369c7e4d93d3bc93..c84cfbfe1a87b4fe2ca6da57d9e9fef626704b57 100644 --- a/substrate/frame/contracts/primitives/README.md +++ b/substrate/frame/contracts/primitives/README.md @@ -1,3 +1,3 @@ A crate that hosts a common definitions that are relevant for the pallet-contracts. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/contracts/proc-macro/Cargo.toml b/substrate/frame/contracts/proc-macro/Cargo.toml index ecc5bc627b70f898bd437756b4f1b0b0ecae3d59..a04f5544067096c378bbb0a78c8554bed7cb4163 100644 --- a/substrate/frame/contracts/proc-macro/Cargo.toml +++ b/substrate/frame/contracts/proc-macro/Cargo.toml @@ -17,7 +17,7 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full"] } +syn = { version = "2.0.37", features = ["full"] } [dev-dependencies] diff --git a/substrate/frame/contracts/src/benchmarking/sandbox.rs b/substrate/frame/contracts/src/benchmarking/sandbox.rs index 34974b02ea0c45305acfa67123663ddf10ce74c0..c3abbcad5f2b363bdda7ef2e7b5c38255220211e 100644 --- a/substrate/frame/contracts/src/benchmarking/sandbox.rs +++ b/substrate/frame/contracts/src/benchmarking/sandbox.rs @@ -58,7 +58,13 @@ impl From<&WasmModule> for Sandbox { .add_fuel(u64::MAX) .expect("We've set up engine to fuel consuming mode; qed"); - let entry_point = instance.get_export(&store, "call").unwrap().into_func().unwrap(); + let entry_point = instance + .start(&mut store) + .unwrap() + .get_export(&store, "call") + .unwrap() + .into_func() + .unwrap(); Self { entry_point, store } } } diff --git a/substrate/frame/contracts/src/exec.rs b/substrate/frame/contracts/src/exec.rs index 894667280b71d3a81cdb7ab0f9cc1033562ea3a3..f93e7a2b21a553405a02f3063ffa51482c943eb3 100644 --- a/substrate/frame/contracts/src/exec.rs +++ b/substrate/frame/contracts/src/exec.rs @@ -21,7 +21,7 @@ use crate::{ storage::{self, meter::Diff, WriteOutcome}, BalanceOf, CodeHash, CodeInfo, CodeInfoOf, Config, ContractInfo, ContractInfoOf, DebugBufferVec, Determinism, Error, Event, Nonce, Origin, Pallet as Contracts, Schedule, - WasmBlob, LOG_TARGET, + LOG_TARGET, }; use frame_support::{ crypto::ecdsa::ECDSAExt, @@ -318,6 +318,22 @@ pub trait Ext: sealing::Sealed { /// Returns a nonce that is incremented for every instantiated contract. fn nonce(&mut self) -> u64; + /// Increment the reference count of a of a stored code by one. + /// + /// # Errors + /// + /// [`Error::CodeNotFound`] is returned if no stored code found having the specified + /// `code_hash`. + fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError>; + + /// Decrement the reference count of a stored code by one. + /// + /// # Note + /// + /// A contract whose reference count dropped to zero isn't automatically removed. A + /// `remove_code` transaction must be submitted by the original uploader to do so. + fn decrement_refcount(code_hash: CodeHash); + /// Adds a delegate dependency to [`ContractInfo`]'s `delegate_dependencies` field. /// /// This ensures that the delegated contract is not removed while it is still in use. It @@ -381,22 +397,6 @@ pub trait Executable: Sized { gas_meter: &mut GasMeter, ) -> Result; - /// Increment the reference count of a of a stored code by one. - /// - /// # Errors - /// - /// [`Error::CodeNotFound`] is returned if no stored code found having the specified - /// `code_hash`. - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError>; - - /// Decrement the reference count of a stored code by one. - /// - /// # Note - /// - /// A contract whose reference count dropped to zero isn't automatically removed. A - /// `remove_code` transaction must be submitted by the original uploader to do so. - fn decrement_refcount(code_hash: CodeHash); - /// Execute the specified exported function and return the result. /// /// When the specified function is `Constructor` the executable is stored and its @@ -1285,10 +1285,10 @@ where info.queue_trie_for_deletion(); ContractInfoOf::::remove(&frame.account_id); - E::decrement_refcount(info.code_hash); + Self::decrement_refcount(info.code_hash); for (code_hash, deposit) in info.delegate_dependencies() { - E::decrement_refcount(*code_hash); + Self::decrement_refcount(*code_hash); frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Refund(*deposit)); @@ -1491,8 +1491,8 @@ where frame.nested_storage.charge_deposit(frame.account_id.clone(), deposit); - E::increment_refcount(hash)?; - E::decrement_refcount(prev_hash); + Self::increment_refcount(hash)?; + Self::decrement_refcount(prev_hash); Contracts::::deposit_event( vec![T::Hashing::hash_of(&frame.account_id), hash, prev_hash], Event::ContractCodeUpdated { @@ -1525,6 +1525,25 @@ where } } + fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { + >::mutate(code_hash, |existing| -> Result<(), DispatchError> { + if let Some(info) = existing { + *info.refcount_mut() = info.refcount().saturating_add(1); + Ok(()) + } else { + Err(Error::::CodeNotFound.into()) + } + }) + } + + fn decrement_refcount(code_hash: CodeHash) { + >::mutate(code_hash, |existing| { + if let Some(info) = existing { + *info.refcount_mut() = info.refcount().saturating_sub(1); + } + }); + } + fn add_delegate_dependency( &mut self, code_hash: CodeHash, @@ -1537,7 +1556,7 @@ where let deposit = T::CodeHashLockupDepositPercent::get().mul_ceil(code_info.deposit()); info.add_delegate_dependency(code_hash, deposit)?; - >::increment_refcount(code_hash)?; + Self::increment_refcount(code_hash)?; frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Charge(deposit)); @@ -1552,8 +1571,7 @@ where let info = frame.contract_info.get(&frame.account_id); let deposit = info.remove_delegate_dependency(code_hash)?; - >::decrement_refcount(*code_hash); - + Self::decrement_refcount(*code_hash); frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Refund(deposit)); @@ -1600,11 +1618,7 @@ mod tests { use pallet_contracts_primitives::ReturnFlags; use pretty_assertions::assert_eq; use sp_runtime::{traits::Hash, DispatchError}; - use std::{ - cell::RefCell, - collections::hash_map::{Entry, HashMap}, - rc::Rc, - }; + use std::{cell::RefCell, collections::hash_map::HashMap, rc::Rc}; type System = frame_system::Pallet; @@ -1625,17 +1639,16 @@ mod tests { } struct MockCtx<'a> { - ext: &'a mut dyn Ext, + ext: &'a mut MockStack<'a>, input_data: Vec, } #[derive(Clone)] struct MockExecutable { - func: Rc ExecResult + 'static>, + func: Rc Fn(MockCtx<'a>, &Self) -> ExecResult + 'static>, func_type: ExportedFunction, code_hash: CodeHash, code_info: CodeInfo, - refcount: u64, } #[derive(Default, Clone)] @@ -1664,37 +1677,11 @@ mod tests { func_type, code_hash: hash, code_info: CodeInfo::::new(ALICE), - refcount: 1, }, ); hash }) } - - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - Loader::mutate(|loader| { - match loader.map.entry(code_hash) { - Entry::Vacant(_) => Err(>::CodeNotFound)?, - Entry::Occupied(mut entry) => entry.get_mut().refcount += 1, - } - Ok(()) - }) - } - - fn decrement_refcount(code_hash: CodeHash) { - use std::collections::hash_map::Entry::Occupied; - Loader::mutate(|loader| { - let mut entry = match loader.map.entry(code_hash) { - Occupied(e) => e, - _ => panic!("code_hash does not exist"), - }; - let refcount = &mut entry.get_mut().refcount; - *refcount -= 1; - if *refcount == 0 { - entry.remove(); - } - }); - } } impl Executable for MockExecutable { @@ -1707,14 +1694,6 @@ mod tests { }) } - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - MockLoader::increment_refcount(code_hash) - } - - fn decrement_refcount(code_hash: CodeHash) { - MockLoader::decrement_refcount(code_hash); - } - fn execute>( self, ext: &mut E, @@ -1722,8 +1701,18 @@ mod tests { input_data: Vec, ) -> ExecResult { if let &Constructor = function { - Self::increment_refcount(self.code_hash).unwrap(); + E::increment_refcount(self.code_hash).unwrap(); } + // # Safety + // + // We know that we **always** call execute with a `MockStack` in this test. + // + // # Note + // + // The transmute is necessary because `execute` has to be generic over all + // `E: Ext`. However, `MockExecutable` can't be generic over `E` as it would + // constitute a cycle. + let ext = unsafe { mem::transmute(ext) }; if function == &self.func_type { (self.func)(MockCtx { ext, input_data }, &self) } else { diff --git a/substrate/frame/contracts/src/lib.rs b/substrate/frame/contracts/src/lib.rs index e22e4a3f9ff86b64c47646e6578a6aa63fdef43a..7d516fbe2496cbe5e0ad67cad4e46cae7d9ba447 100644 --- a/substrate/frame/contracts/src/lib.rs +++ b/substrate/frame/contracts/src/lib.rs @@ -103,7 +103,9 @@ pub mod weights; #[cfg(test)] mod tests; use crate::{ - exec::{AccountIdOf, ErrorOrigin, ExecError, Executable, Key, MomentOf, Stack as ExecStack}, + exec::{ + AccountIdOf, ErrorOrigin, ExecError, Executable, Ext, Key, MomentOf, Stack as ExecStack, + }, gas::GasMeter, storage::{meter::Meter as StorageMeter, ContractInfo, DeletionQueueManager}, wasm::{CodeInfo, WasmBlob}, @@ -658,8 +660,8 @@ pub mod pallet { } else { return Err(>::ContractNotFound.into()) }; - >::increment_refcount(code_hash)?; - >::decrement_refcount(contract.code_hash); + >>::increment_refcount(code_hash)?; + >>::decrement_refcount(contract.code_hash); Self::deposit_event( vec![T::Hashing::hash_of(&dest), code_hash, contract.code_hash], Event::ContractCodeUpdated { diff --git a/substrate/frame/contracts/src/tests.rs b/substrate/frame/contracts/src/tests.rs index 0c0a2f7f9327315a5991e2283e6e3f6b00aedf87..0fea2b1559509a7719e42c9461f35733e6617efc 100644 --- a/substrate/frame/contracts/src/tests.rs +++ b/substrate/frame/contracts/src/tests.rs @@ -564,7 +564,9 @@ where { let fixture_path = [ // When `CARGO_MANIFEST_DIR` is not set, Rust resolves relative paths from the root folder - std::env::var("CARGO_MANIFEST_DIR").as_deref().unwrap_or("frame/contracts"), + std::env::var("CARGO_MANIFEST_DIR") + .as_deref() + .unwrap_or("substrate/frame/contracts"), "/fixtures/", fixture_name, ".wat", @@ -860,6 +862,27 @@ fn deposit_event_max_value_limit() { }); } +// Fail out of fuel (ref_time weight) inside the start function. +#[test] +fn run_out_of_fuel_start_fun() { + let (wasm, _code_hash) = compile_module::("run_out_of_gas_start_fn").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let _ = ::Currency::set_balance(&ALICE, 1_000_000); + assert_err_ignore_postinfo!( + Contracts::instantiate_with_code( + RuntimeOrigin::signed(ALICE), + 0, + Weight::from_parts(1_000_000_000_000, u64::MAX), + None, + wasm, + vec![], + vec![], + ), + Error::::OutOfGas, + ); + }); +} + // Fail out of fuel (ref_time weight) in the engine. #[test] fn run_out_of_fuel_engine() { diff --git a/substrate/frame/contracts/src/wasm/mod.rs b/substrate/frame/contracts/src/wasm/mod.rs index 5fc65e314ad9fe6b61d6ae641bc37d6f9df17e24..77e94b16777b041d746746f2c0653bae3c3d5949 100644 --- a/substrate/frame/contracts/src/wasm/mod.rs +++ b/substrate/frame/contracts/src/wasm/mod.rs @@ -49,7 +49,7 @@ use frame_support::{ use sp_core::Get; use sp_runtime::{DispatchError, RuntimeDebug}; use sp_std::prelude::*; -use wasmi::{Instance, Linker, Memory, MemoryType, StackLimits, Store}; +use wasmi::{InstancePre, Linker, Memory, MemoryType, StackLimits, Store}; const BYTES_PER_PAGE: usize = 64 * 1024; @@ -164,7 +164,30 @@ impl WasmBlob { /// /// Applies all necessary checks before removing the code. pub fn remove(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { - Self::try_remove_code(origin, code_hash) + >::try_mutate_exists(&code_hash, |existing| { + if let Some(code_info) = existing { + ensure!(code_info.refcount == 0, >::CodeInUse); + ensure!(&code_info.owner == origin, BadOrigin); + let _ = T::Currency::release( + &HoldReason::CodeUploadDepositReserve.into(), + &code_info.owner, + code_info.deposit, + BestEffort, + ); + let deposit_released = code_info.deposit; + let remover = code_info.owner.clone(); + + *existing = None; + >::remove(&code_hash); + >::deposit_event( + vec![code_hash], + Event::CodeRemoved { code_hash, deposit_released, remover }, + ); + Ok(()) + } else { + Err(>::CodeNotFound.into()) + } + }) } /// Creates and returns an instance of the supplied code. @@ -179,7 +202,7 @@ impl WasmBlob { determinism: Determinism, stack_limits: StackLimits, allow_deprecated: AllowDeprecatedInterface, - ) -> Result<(Store, Memory, Instance), &'static str> + ) -> Result<(Store, Memory, InstancePre), &'static str> where E: Environment, { @@ -217,9 +240,7 @@ impl WasmBlob { let instance = linker .instantiate(&mut store, &contract.module) - .map_err(|_| "can't instantiate module with provided definitions")? - .ensure_no_start(&mut store) - .map_err(|_| "start function is forbidden but found in the module")?; + .map_err(|_| "can't instantiate module with provided definitions")?; Ok((store, memory, instance)) } @@ -261,45 +282,6 @@ impl WasmBlob { }) } - /// Try to remove code together with all associated information. - fn try_remove_code(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { - >::try_mutate_exists(&code_hash, |existing| { - if let Some(code_info) = existing { - ensure!(code_info.refcount == 0, >::CodeInUse); - ensure!(&code_info.owner == origin, BadOrigin); - let _ = T::Currency::release( - &HoldReason::CodeUploadDepositReserve.into(), - &code_info.owner, - code_info.deposit, - BestEffort, - ); - let deposit_released = code_info.deposit; - let remover = code_info.owner.clone(); - - *existing = None; - >::remove(&code_hash); - >::deposit_event( - vec![code_hash], - Event::CodeRemoved { code_hash, deposit_released, remover }, - ); - Ok(()) - } else { - Err(>::CodeNotFound.into()) - } - }) - } - - /// Load code with the given code hash. - fn load_code( - code_hash: CodeHash, - gas_meter: &mut GasMeter, - ) -> Result<(CodeVec, CodeInfo), DispatchError> { - let code_info = >::get(code_hash).ok_or(Error::::CodeNotFound)?; - gas_meter.charge(CodeLoadToken(code_info.code_len))?; - let code = >::get(code_hash).ok_or(Error::::CodeNotFound)?; - Ok((code, code_info)) - } - /// Create the module without checking the passed code. /// /// # Note @@ -318,12 +300,6 @@ impl WasmBlob { } impl CodeInfo { - /// Return the refcount of the module. - #[cfg(test)] - pub fn refcount(&self) -> u64 { - self.refcount - } - #[cfg(test)] pub fn new(owner: T::AccountId) -> Self { CodeInfo { @@ -335,6 +311,16 @@ impl CodeInfo { } } + /// Returns reference count of the module. + pub fn refcount(&self) -> u64 { + self.refcount + } + + /// Return mutable reference to the refcount of the module. + pub fn refcount_mut(&mut self) -> &mut u64 { + &mut self.refcount + } + /// Returns the deposit of the module. pub fn deposit(&self) -> BalanceOf { self.deposit @@ -346,29 +332,12 @@ impl Executable for WasmBlob { code_hash: CodeHash, gas_meter: &mut GasMeter, ) -> Result { - let (code, code_info) = Self::load_code(code_hash, gas_meter)?; + let code_info = >::get(code_hash).ok_or(Error::::CodeNotFound)?; + gas_meter.charge(CodeLoadToken(code_info.code_len))?; + let code = >::get(code_hash).ok_or(Error::::CodeNotFound)?; Ok(Self { code, code_info, code_hash }) } - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - >::mutate(code_hash, |existing| -> Result<(), DispatchError> { - if let Some(info) = existing { - info.refcount = info.refcount.saturating_add(1); - Ok(()) - } else { - Err(Error::::CodeNotFound.into()) - } - }) - } - - fn decrement_refcount(code_hash: CodeHash) { - >::mutate(code_hash, |existing| { - if let Some(info) = existing { - info.refcount = info.refcount.saturating_sub(1); - } - }); - } - fn execute>( self, ext: &mut E, @@ -410,25 +379,38 @@ impl Executable for WasmBlob { .add_fuel(fuel_limit) .expect("We've set up engine to fuel consuming mode; qed"); - let exported_func = instance - .get_export(&store, function.identifier()) - .and_then(|export| export.into_func()) - .ok_or_else(|| { - log::error!(target: LOG_TARGET, "failed to find entry point"); - Error::::CodeRejected - })?; + // Sync this frame's gas meter with the engine's one. + let process_result = |mut store: Store>, result| { + let engine_consumed_total = + store.fuel_consumed().expect("Fuel metering is enabled; qed"); + let gas_meter = store.data_mut().ext().gas_meter_mut(); + gas_meter.charge_fuel(engine_consumed_total)?; + store.into_data().to_execution_result(result) + }; + // Start function should already see the correct refcount in case it will be ever inspected. if let &ExportedFunction::Constructor = function { - WasmBlob::::increment_refcount(self.code_hash)?; + E::increment_refcount(self.code_hash)?; } - let result = exported_func.call(&mut store, &[], &mut []); - let engine_consumed_total = store.fuel_consumed().expect("Fuel metering is enabled; qed"); - // Sync this frame's gas meter with the engine's one. - let gas_meter = store.data_mut().ext().gas_meter_mut(); - gas_meter.charge_fuel(engine_consumed_total)?; - - store.into_data().to_execution_result(result) + // Any abort in start function (includes `return` + `terminate`) will make us skip the + // call into the subsequent exported function. This means that calling `return` returns data + // from the whole contract execution. + match instance.start(&mut store) { + Ok(instance) => { + let exported_func = instance + .get_export(&store, function.identifier()) + .and_then(|export| export.into_func()) + .ok_or_else(|| { + log::error!(target: LOG_TARGET, "failed to find entry point"); + Error::::CodeRejected + })?; + + let result = exported_func.call(&mut store, &[], &mut []); + process_result(store, result) + }, + Err(err) => process_result(store, Err(err)), + } } fn code_hash(&self) -> &CodeHash { @@ -740,7 +722,10 @@ mod tests { fn nonce(&mut self) -> u64 { 995 } - + fn increment_refcount(_code_hash: CodeHash) -> Result<(), DispatchError> { + Ok(()) + } + fn decrement_refcount(_code_hash: CodeHash) {} fn add_delegate_dependency( &mut self, code: CodeHash, @@ -748,7 +733,6 @@ mod tests { self.delegate_dependencies.borrow_mut().insert(code); Ok(()) } - fn remove_delegate_dependency( &mut self, code: &CodeHash, @@ -790,11 +774,20 @@ mod tests { executable.execute(ext.borrow_mut(), entry_point, input_data) } - /// Execute the supplied code. + /// Execute the `call` function within the supplied code. fn execute>(wat: &str, input_data: Vec, ext: E) -> ExecResult { execute_internal(wat, input_data, ext, &ExportedFunction::Call, true, false) } + /// Execute the `deploy` function within the supplied code. + fn execute_instantiate>( + wat: &str, + input_data: Vec, + ext: E, + ) -> ExecResult { + execute_internal(wat, input_data, ext, &ExportedFunction::Constructor, true, false) + } + /// Execute the supplied code with disabled unstable functions. /// /// In our test config unstable functions are disabled so that we can test them. @@ -1878,32 +1871,47 @@ mod tests { assert_ok!(execute(CODE_VALUE_TRANSFERRED, vec![], MockExt::default())); } - const START_FN_ILLEGAL: &str = r#" + const START_FN_DOES_RUN: &str = r#" (module - (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "seal0" "seal_deposit_event" (func $seal_deposit_event (param i32 i32 i32 i32))) (import "env" "memory" (memory 1 1)) (start $start) (func $start - (unreachable) + (call $seal_deposit_event + (i32.const 0) ;; Pointer to the start of topics buffer + (i32.const 0) ;; The length of the topics buffer. + (i32.const 0) ;; Pointer to the start of the data buffer + (i32.const 13) ;; Length of the buffer + ) ) - (func (export "call") - (unreachable) - ) + (func (export "call")) - (func (export "deploy") - (unreachable) - ) + (func (export "deploy")) - (data (i32.const 8) "\01\02\03\04") + (data (i32.const 0) "\00\01\2A\00\00\00\00\00\00\00\E5\14\00") ) "#; #[test] - fn start_fn_illegal() { - let output = execute(START_FN_ILLEGAL, vec![], MockExt::default()); - assert_err!(output, >::CodeRejected,); + fn start_fn_does_run_on_call() { + let mut ext = MockExt::default(); + execute(START_FN_DOES_RUN, vec![], &mut ext).unwrap(); + assert_eq!( + ext.events[0].1, + [0x00_u8, 0x01, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00] + ); + } + + #[test] + fn start_fn_does_run_on_deploy() { + let mut ext = MockExt::default(); + execute_instantiate(START_FN_DOES_RUN, vec![], &mut ext).unwrap(); + assert_eq!( + ext.events[0].1, + [0x00_u8, 0x01, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00] + ); } const CODE_TIMESTAMP_NOW: &str = r#" diff --git a/substrate/frame/contracts/src/wasm/runtime.rs b/substrate/frame/contracts/src/wasm/runtime.rs index 4bc00388f726fb2f4057b6e527157ec6080439fa..4fd52b471a0c6b41cabd364d530d2989891591d7 100644 --- a/substrate/frame/contracts/src/wasm/runtime.rs +++ b/substrate/frame/contracts/src/wasm/runtime.rs @@ -608,7 +608,7 @@ impl<'a, E: Ext + 'a> Runtime<'a, E> { let mut bound_checked = memory .get(ptr..ptr + D::max_encoded_len() as usize) .ok_or_else(|| Error::::OutOfBounds)?; - let decoded = D::decode_all_with_depth_limit(MAX_DECODE_NESTING, &mut bound_checked) + let decoded = D::decode_with_depth_limit(MAX_DECODE_NESTING, &mut bound_checked) .map_err(|_| DispatchError::from(Error::::DecodingFailed))?; Ok(decoded) } diff --git a/substrate/frame/core-fellowship/README.md b/substrate/frame/core-fellowship/README.md index 3c9b1f63e08961e7f1a0634e04018ed6b0fc85b2..97718eef7a32ca69319fc106c2284aee4deab02a 100644 --- a/substrate/frame/core-fellowship/README.md +++ b/substrate/frame/core-fellowship/README.md @@ -1,3 +1,3 @@ # Core Fellowship -Logic specific to the core Polkadot Fellowship. \ No newline at end of file +Logic specific to the core Polkadot Fellowship. diff --git a/substrate/frame/democracy/src/tests.rs b/substrate/frame/democracy/src/tests.rs index e5cfcc5b4002963387ef78126630ea7433ed63ab..9b885fb5915b7a3d92f28d7a7220f5c5ddf7d7f5 100644 --- a/substrate/frame/democracy/src/tests.rs +++ b/substrate/frame/democracy/src/tests.rs @@ -111,8 +111,7 @@ impl pallet_preimage::Config for Test { type WeightInfo = (); type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = ConstU64<0>; - type ByteDeposit = ConstU64<0>; + type Consideration = (); } impl pallet_scheduler::Config for Test { diff --git a/substrate/frame/election-provider-multi-phase/src/lib.rs b/substrate/frame/election-provider-multi-phase/src/lib.rs index f26a6f40d42677584d548fd2e396c5b2cac230d6..8b6e0827c715834de9101dd1f061c9d362b1705c 100644 --- a/substrate/frame/election-provider-multi-phase/src/lib.rs +++ b/substrate/frame/election-provider-multi-phase/src/lib.rs @@ -149,7 +149,8 @@ //! while this binary lives in the Polkadot repository, this particular subcommand of it can work //! against any substrate-based chain. //! -//! See the `staking-miner` documentation in the Polkadot repository for more information. +//! See the [`staking-miner`](https://github.com/paritytech/staking-miner-v2) docs for more +//! information. //! //! ## Feasible Solution (correct solution) //! @@ -278,8 +279,8 @@ use unsigned::VoterOf; pub use weights::WeightInfo; pub use signed::{ - BalanceOf, NegativeImbalanceOf, PositiveImbalanceOf, SignedSubmission, SignedSubmissionOf, - SignedSubmissions, SubmissionIndicesOf, + BalanceOf, GeometricDepositBase, NegativeImbalanceOf, PositiveImbalanceOf, SignedSubmission, + SignedSubmissionOf, SignedSubmissions, SubmissionIndicesOf, }; pub use unsigned::{Miner, MinerConfig}; @@ -571,6 +572,7 @@ pub mod pallet { use frame_election_provider_support::{InstantElectionProvider, NposSolver}; use frame_support::{pallet_prelude::*, traits::EstimateCallFee}; use frame_system::pallet_prelude::*; + use sp_runtime::traits::Convert; #[pallet::config] pub trait Config: frame_system::Config + SendTransactionTypes> { @@ -648,10 +650,6 @@ pub mod pallet { #[pallet::constant] type SignedRewardBase: Get>; - /// Base deposit for a signed solution. - #[pallet::constant] - type SignedDepositBase: Get>; - /// Per-byte deposit for a signed solution. #[pallet::constant] type SignedDepositByte: Get>; @@ -667,6 +665,10 @@ pub mod pallet { #[pallet::constant] type MaxWinners: Get; + /// Something that calculates the signed deposit base based on the signed submissions queue + /// size. + type SignedDepositBase: Convert>; + /// The maximum number of electing voters and electable targets to put in the snapshot. /// At the moment, snapshots are only over a single block, but once multi-block elections /// are introduced they will take place over multiple blocks. diff --git a/substrate/frame/election-provider-multi-phase/src/mock.rs b/substrate/frame/election-provider-multi-phase/src/mock.rs index 05d151e51ecc5beb5cc3e5f883ac91ada10318c4..d4659eba5661a3059c8ab55b453ab8c9561f4e77 100644 --- a/substrate/frame/election-provider-multi-phase/src/mock.rs +++ b/substrate/frame/election-provider-multi-phase/src/mock.rs @@ -16,7 +16,7 @@ // limitations under the License. use super::*; -use crate::{self as multi_phase, unsigned::MinerConfig}; +use crate::{self as multi_phase, signed::GeometricDepositBase, unsigned::MinerConfig}; use frame_election_provider_support::{ bounds::{DataProviderBounds, ElectionBounds}, data_provider, onchain, ElectionDataProvider, NposSolution, SequentialPhragmen, @@ -44,8 +44,8 @@ use sp_npos_elections::{ use sp_runtime::{ bounded_vec, testing::Header, - traits::{BlakeTwo256, IdentityLookup}, - BuildStorage, PerU16, + traits::{BlakeTwo256, Convert, IdentityLookup}, + BuildStorage, PerU16, Percent, }; use std::sync::Arc; @@ -283,7 +283,11 @@ parameter_types! { pub static UnsignedPhase: BlockNumber = 5; pub static SignedMaxSubmissions: u32 = 5; pub static SignedMaxRefunds: u32 = 1; - pub static SignedDepositBase: Balance = 5; + // for tests only. if `EnableVariableDepositBase` is true, the deposit base will be calculated + // by `Multiphase::DepositBase`. Otherwise the deposit base is `SignedFixedDeposit`. + pub static EnableVariableDepositBase: bool = false; + pub static SignedFixedDeposit: Balance = 5; + pub static SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); pub static SignedDepositByte: Balance = 0; pub static SignedDepositWeight: Balance = 0; pub static SignedRewardBase: Balance = 7; @@ -393,7 +397,7 @@ impl crate::Config for Runtime { type OffchainRepeat = OffchainRepeat; type MinerTxPriority = MinerTxPriority; type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; + type SignedDepositBase = Self; type SignedDepositByte = (); type SignedDepositWeight = (); type SignedMaxWeight = SignedMaxWeight; @@ -414,6 +418,18 @@ impl crate::Config for Runtime { type ElectionBounds = ElectionsBounds; } +impl Convert> for Runtime { + /// returns the geometric increase deposit fee if `EnableVariableDepositBase` is set, otherwise + /// the fee is `SignedFixedDeposit`. + fn convert(queue_len: usize) -> Balance { + if !EnableVariableDepositBase::get() { + SignedFixedDeposit::get() + } else { + GeometricDepositBase::::convert(queue_len) + } + } +} + impl frame_system::offchain::SendTransactionTypes for Runtime where RuntimeCall: From, @@ -553,8 +569,14 @@ impl ExtBuilder { ::set(count); self } + pub fn signed_base_deposit(self, base: u64, variable: bool, increase: Percent) -> Self { + ::set(variable); + ::set(base); + ::set(increase); + self + } pub fn signed_deposit(self, base: u64, byte: u64, weight: u64) -> Self { - ::set(base); + ::set(base); ::set(byte); ::set(weight); self diff --git a/substrate/frame/election-provider-multi-phase/src/signed.rs b/substrate/frame/election-provider-multi-phase/src/signed.rs index 76068ba99d36c97a9b1060cea4e3147e894da4dd..a5fe8ce55582064078882ff125e3f945ac1b895c 100644 --- a/substrate/frame/election-provider-multi-phase/src/signed.rs +++ b/substrate/frame/election-provider-multi-phase/src/signed.rs @@ -17,6 +17,8 @@ //! The signed phase implementation. +use core::marker::PhantomData; + use crate::{ unsigned::MinerConfig, Config, ElectionCompute, Pallet, QueuedSolution, RawSolution, ReadySolution, SignedSubmissionIndices, SignedSubmissionNextIndex, SignedSubmissionsMap, @@ -32,8 +34,8 @@ use sp_arithmetic::traits::SaturatedConversion; use sp_core::bounded::BoundedVec; use sp_npos_elections::ElectionScore; use sp_runtime::{ - traits::{Saturating, Zero}, - RuntimeDebug, + traits::{Convert, Saturating, Zero}, + FixedPointNumber, FixedPointOperand, FixedU128, Percent, RuntimeDebug, }; use sp_std::{ cmp::Ordering, @@ -348,6 +350,32 @@ impl SignedSubmissions { } } +/// Type that can be used to calculate the deposit base for signed submissions. +/// +/// The deposit base is calculated as a geometric progression based on the number of signed +/// submissions in the queue. The size of the queue represents the progression term. +pub struct GeometricDepositBase { + _marker: (PhantomData, PhantomData, PhantomData), +} + +impl Convert for GeometricDepositBase +where + Balance: FixedPointOperand, + Fixed: Get, + Inc: Get, +{ + // Calculates the base deposit as a geometric progression based on the number of signed + // submissions. + // + // The nth term is obtained by calculating `base * (1 + increase_factor)^nth`. Example: factor + // 5, with initial deposit of 1000 and 10% of increase factor is 1000 * (1 + 0.1)^5. + fn convert(queue_len: usize) -> Balance { + let increase_factor: FixedU128 = FixedU128::from_u32(1) + Inc::get().into(); + + increase_factor.saturating_pow(queue_len).saturating_mul_int(Fixed::get()) + } +} + impl Pallet { /// `Self` accessor for `SignedSubmission`. pub fn signed_submissions() -> SignedSubmissions { @@ -520,14 +548,14 @@ impl Pallet { size: SolutionOrSnapshotSize, ) -> BalanceOf { let encoded_len: u32 = raw_solution.encoded_size().saturated_into(); - let encoded_len: BalanceOf = encoded_len.into(); + let encoded_len_balance: BalanceOf = encoded_len.into(); let feasibility_weight = Self::solution_weight_of(raw_solution, size); - let len_deposit = T::SignedDepositByte::get().saturating_mul(encoded_len); + let len_deposit = T::SignedDepositByte::get().saturating_mul(encoded_len_balance); let weight_deposit = T::SignedDepositWeight::get() .saturating_mul(feasibility_weight.ref_time().saturated_into()); - T::SignedDepositBase::get() + T::SignedDepositBase::convert(Self::signed_submissions().len()) .saturating_add(len_deposit) .saturating_add(weight_deposit) } @@ -541,6 +569,7 @@ mod tests { Phase, }; use frame_support::{assert_noop, assert_ok, assert_storage_noop}; + use sp_runtime::Percent; #[test] fn cannot_submit_too_early() { @@ -779,6 +808,56 @@ mod tests { }) } + #[test] + fn geometric_deposit_queue_size_works() { + let constant = vec![1000; 10]; + // geometric progression with 10% increase in each iteration for 10 terms. + let progression_10 = vec![1000, 1100, 1210, 1331, 1464, 1610, 1771, 1948, 2143, 2357]; + let progression_40 = vec![1000, 1400, 1960, 2744, 3841, 5378, 7529, 10541, 14757, 20661]; + + let check_progressive_base_fee = |expected: &Vec| { + for s in 0..SignedMaxSubmissions::get() { + let account = 99 + s as u64; + Balances::make_free_balance_be(&account, 10000000); + let mut solution = raw_solution(); + solution.score.minimal_stake -= s as u128; + + assert_ok!(MultiPhase::submit(RuntimeOrigin::signed(account), Box::new(solution))); + assert_eq!(balances(&account).1, expected[s as usize]) + } + }; + + ExtBuilder::default() + .signed_max_submission(10) + .signed_base_deposit(1000, true, Percent::from_percent(0)) + .build_and_execute(|| { + roll_to_signed(); + assert!(MultiPhase::current_phase().is_signed()); + + check_progressive_base_fee(&constant); + }); + + ExtBuilder::default() + .signed_max_submission(10) + .signed_base_deposit(1000, true, Percent::from_percent(10)) + .build_and_execute(|| { + roll_to_signed(); + assert!(MultiPhase::current_phase().is_signed()); + + check_progressive_base_fee(&progression_10); + }); + + ExtBuilder::default() + .signed_max_submission(10) + .signed_base_deposit(1000, true, Percent::from_percent(40)) + .build_and_execute(|| { + roll_to_signed(); + assert!(MultiPhase::current_phase().is_signed()); + + check_progressive_base_fee(&progression_40); + }); + } + #[test] fn call_fee_refund_is_limited_by_signed_max_refunds() { ExtBuilder::default().build_and_execute(|| { diff --git a/substrate/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs b/substrate/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs index 501f9f89ab7a9f57b4a5990d9139be98f95d65a8..ec646c3119789ec1ea01ef08fa5514037060dc9d 100644 --- a/substrate/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs +++ b/substrate/frame/election-provider-multi-phase/test-staking-e2e/src/mock.rs @@ -32,7 +32,7 @@ use sp_runtime::{ }, testing, traits::Zero, - transaction_validity, BuildStorage, PerU16, Perbill, + transaction_validity, BuildStorage, PerU16, Perbill, Percent, }; use sp_staking::{ offence::{DisableStrategy, OffenceDetails, OnOffenceHandler}, @@ -47,7 +47,8 @@ use frame_election_provider_support::{ SequentialPhragmen, Weight, }; use pallet_election_provider_multi_phase::{ - unsigned::MinerConfig, Call, ElectionCompute, QueuedSolution, SolutionAccuracyOf, + unsigned::MinerConfig, Call, ElectionCompute, GeometricDepositBase, QueuedSolution, + SolutionAccuracyOf, }; use pallet_staking::StakerStatus; use parking_lot::RwLock; @@ -182,6 +183,9 @@ parameter_types! { pub static TransactionPriority: transaction_validity::TransactionPriority = 1; #[derive(Debug)] pub static MaxWinners: u32 = 100; + pub static MaxVotesPerVoter: u32 = 16; + pub static SignedFixedDeposit: Balance = 1; + pub static SignedDepositIncreaseFactor: Percent = Percent::from_percent(10); pub static ElectionBounds: frame_election_provider_support::bounds::ElectionBounds = ElectionBoundsBuilder::default() .voters_count(1_000.into()).targets_count(1_000.into()).build(); } @@ -199,7 +203,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type MinerConfig = Self; type SignedMaxSubmissions = ConstU32<10>; type SignedRewardBase = (); - type SignedDepositBase = (); + type SignedDepositBase = + GeometricDepositBase; type SignedDepositByte = (); type SignedMaxRefunds = ConstU32<3>; type SignedDepositWeight = (); diff --git a/substrate/frame/election-provider-support/solution-type/Cargo.toml b/substrate/frame/election-provider-support/solution-type/Cargo.toml index 7693da0372d990a51c55f8e8040d62656769a4f0..39e535c6c3ee6add48d9099ce5638bcf5bcff7dd 100644 --- a/substrate/frame/election-provider-support/solution-type/Cargo.toml +++ b/substrate/frame/election-provider-support/solution-type/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "2.0.31", features = ["full", "visit"] } +syn = { version = "2.0.37", features = ["full", "visit"] } quote = "1.0.28" proc-macro2 = "1.0.56" proc-macro-crate = "1.1.3" diff --git a/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml b/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml index 69ce42559a61e95b171034481d0fd5fafaea7b19..6ac09dd45c60138ad1e69d48f232e4baba692ac6 100644 --- a/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml +++ b/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml @@ -13,7 +13,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } honggfuzz = "0.5" rand = { version = "0.8", features = ["std", "small_rng"] } diff --git a/substrate/frame/elections-phragmen/README.md b/substrate/frame/elections-phragmen/README.md index 26b3f260da5635de07949d7449800e3d34c1a480..f5ed609f2129d33788652e5cf33f138471505454 100644 --- a/substrate/frame/elections-phragmen/README.md +++ b/substrate/frame/elections-phragmen/README.md @@ -1,64 +1,58 @@ -# Phragmén Election Module. +# Phragmén Election Module An election module based on sequential phragmen. -### Term and Round - -The election happens in _rounds_: every `N` blocks, all previous members are retired and a new -set is elected (which may or may not have an intersection with the previous set). Each round -lasts for some number of blocks defined by `TermDuration` storage item. The words _term_ and -_round_ can be used interchangeably in this context. - -`TermDuration` might change during a round. This can shorten or extend the length of the round. -The next election round's block number is never stored but rather always checked on the fly. -Based on the current block number and `TermDuration`, the condition `BlockNumber % TermDuration -== 0` being satisfied will always trigger a new election round. - -### Voting - -Voters can vote for any set of the candidates by providing a list of account ids. Invalid votes -(voting for non-candidates) are ignored during election. Yet, a voter _might_ vote for a future -candidate. Voters reserve a bond as they vote. Each vote defines a `value`. This amount is -locked from the account of the voter and indicates the weight of the vote. Voters can update -their votes at any time by calling `vote()` again. This keeps the bond untouched but can -optionally change the locked `value`. After a round, votes are kept and might still be valid for -further rounds. A voter is responsible for calling `remove_voter` once they are done to have -their bond back and remove the lock. - -Voters also report other voters as being defunct to earn their bond. A voter is defunct once all -of the candidates that they have voted for are neither a valid candidate anymore nor a member. -Upon reporting, if the target voter is actually defunct, the reporter will be rewarded by the -voting bond of the target. The target will lose their bond and get removed. If the target is not -defunct, the reporter is slashed and removed. To prevent being reported, voters should manually -submit a `remove_voter()` as soon as they are in the defunct state. - -### Candidacy and Members - -Candidates also reserve a bond as they submit candidacy. A candidate cannot take their candidacy -back. A candidate can end up in one of the below situations: - - **Winner**: A winner is kept as a _member_. They must still have a bond in reserve and they - are automatically counted as a candidate for the next election. - - **Runner-up**: Runners-up are the best candidates immediately after the winners. The number - of runners_up to keep is configurable. Runners-up are used, in order that they are elected, - as replacements when a candidate is kicked by `[remove_member]`, or when an active member - renounces their candidacy. Runners are automatically counted as a candidate for the next - election. - - **Loser**: Any of the candidate who are not a winner are left as losers. A loser might be an - _outgoing member or runner_, meaning that they are an active member who failed to keep their - spot. An outgoing will always lose their bond. - -##### Renouncing candidacy. - -All candidates, elected or not, can renounce their candidacy. A call to [`Module::renounce_candidacy`] -will always cause the candidacy bond to be refunded. - -Note that with the members being the default candidates for the next round and votes persisting -in storage, the election system is entirely stable given no further input. This means that if -the system has a particular set of candidates `C` and voters `V` that lead to a set of members -`M` being elected, as long as `V` and `C` don't remove their candidacy and votes, `M` will keep -being re-elected at the end of each round. - -### Module Information +## Term and Round + +The election happens in _rounds_: every `N` blocks, all previous members are retired and a new set is elected (which may +or may not have an intersection with the previous set). Each round lasts for some number of blocks defined by +`TermDuration` storage item. The words _term_ and _round_ can be used interchangeably in this context. + +`TermDuration` might change during a round. This can shorten or extend the length of the round. The next election +round's block number is never stored but rather always checked on the fly. Based on the current block number and +`TermDuration`, the condition `BlockNumber % TermDuration == 0` being satisfied will always trigger a new election +round. + +## Voting + +Voters can vote for any set of the candidates by providing a list of account ids. Invalid votes (voting for +non-candidates) are ignored during election. Yet, a voter _might_ vote for a future candidate. Voters reserve a bond as +they vote. Each vote defines a `value`. This amount is locked from the account of the voter and indicates the weight of +the vote. Voters can update their votes at any time by calling `vote()` again. This keeps the bond untouched but can +optionally change the locked `value`. After a round, votes are kept and might still be valid for further rounds. A voter +is responsible for calling `remove_voter` once they are done to have their bond back and remove the lock. + +Voters also report other voters as being defunct to earn their bond. A voter is defunct once all of the candidates that +they have voted for are neither a valid candidate anymore nor a member. Upon reporting, if the target voter is actually +defunct, the reporter will be rewarded by the voting bond of the target. The target will lose their bond and get +removed. If the target is not defunct, the reporter is slashed and removed. To prevent being reported, voters should +manually submit a `remove_voter()` as soon as they are in the defunct state. + +## Candidacy and Members + +Candidates also reserve a bond as they submit candidacy. A candidate cannot take their candidacy back. A candidate can +end up in one of the below situations: + - **Winner**: A winner is kept as a _member_. They must still have a bond in reserve and they are automatically + counted as a candidate for the next election. + - **Runner-up**: Runners-up are the best candidates immediately after the winners. The number of runners_up to keep is + configurable. Runners-up are used, in order that they are elected, as replacements when a candidate is kicked by + `[remove_member]`, or when an active member renounces their candidacy. Runners are automatically counted as a + candidate for the next election. + - **Loser**: Any of the candidate who are not a winner are left as losers. A loser might be an _outgoing member or + runner_, meaning that they are an active member who failed to keep their spot. An outgoing will always lose their + bond. + +### Renouncing candidacy + +All candidates, elected or not, can renounce their candidacy. A call to [`Module::renounce_candidacy`] will always cause +the candidacy bond to be refunded. + +Note that with the members being the default candidates for the next round and votes persisting in storage, the election +system is entirely stable given no further input. This means that if the system has a particular set of candidates `C` +and voters `V` that lead to a set of members `M` being elected, as long as `V` and `C` don't remove their candidacy and +votes, `M` will keep being re-elected at the end of each round. + +## Module Information - [`election_sp_phragmen::Config`](https://docs.rs/pallet-elections-phragmen/latest/pallet_elections_phragmen/trait.Config.html) - [`Call`](https://docs.rs/pallet-elections-phragmen/latest/pallet_elections_phragmen/enum.Call.html) diff --git a/substrate/frame/examples/Cargo.toml b/substrate/frame/examples/Cargo.toml index b072416b6121da41123b1e6276351306d128c6ab..9c47d7442111fdfa0313af5b9d1ae2048ae90b12 100644 --- a/substrate/frame/examples/Cargo.toml +++ b/substrate/frame/examples/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true license = "Apache-2.0" homepage = "https://substrate.io" repository.workspace = true -description = "The single package with various examples for frame pallets" +description = "The single package with examples of various types of FRAME pallets" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/frame/examples/basic/README.md b/substrate/frame/examples/basic/README.md index 2af50573f8075cffe03e8509cc5740ea3ebb6a0f..be787d1b6ecc1d3b20302c78a346d8f291ff4e0f 100644 --- a/substrate/frame/examples/basic/README.md +++ b/substrate/frame/examples/basic/README.md @@ -9,7 +9,7 @@ Run `cargo doc --package pallet-example-basic --open` to view this pallet's docu **This pallet serves as an example and is not meant to be used in production.** -### Documentation Guidelines: +## Documentation Guidelines diff --git a/substrate/frame/examples/basic/src/lib.rs b/substrate/frame/examples/basic/src/lib.rs index 426e9b7ec648c25dda54b27b498cbdd1d92b51d6..31d20e07f5f74ea18815b37ef4e93ec4ed6136bc 100644 --- a/substrate/frame/examples/basic/src/lib.rs +++ b/substrate/frame/examples/basic/src/lib.rs @@ -15,258 +15,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! //! # Basic Example Pallet //! -//! -//! The Example: A simple example of a FRAME pallet demonstrating -//! concepts, APIs and structures common to most FRAME runtimes. -//! -//! Run `cargo doc --package pallet-example-basic --open` to view this pallet's documentation. +//! A pallet demonstrating concepts, APIs and structures common to most FRAME runtimes. //! //! **This pallet serves as an example and is not meant to be used in production.** //! -//! ### Documentation Guidelines: -//! -//! -//!
    -//!
  • Documentation comments (i.e. /// comment) - should -//! accompany pallet functions and be restricted to the pallet interface, -//! not the internals of the pallet implementation. Only state inputs, -//! outputs, and a brief description that mentions whether calling it -//! requires root, but without repeating the source code details. -//! Capitalize the first word of each documentation comment and end it with -//! a full stop. See -//! Generic example of annotating source code with documentation comments
  • -//! -//!
  • Self-documenting code - Try to refactor code to be self-documenting.
  • -//! -//!
  • Code comments - Supplement complex code with a brief explanation, not every line of -//! code.
  • -//! -//!
  • Identifiers - surround by backticks (i.e. INHERENT_IDENTIFIER, -//! InherentType, u64)
  • -//! -//!
  • Usage scenarios - should be simple doctests. The compiler should ensure they stay -//! valid.
  • -//! -//!
  • Extended tutorials - should be moved to external files and refer to.
  • -//! -//!
  • Mandatory - include all of the sections/subsections where MUST is specified.
  • -//! -//!
  • Optional - optionally include sections/subsections where CAN is specified.
  • -//!
-//! -//! ### Documentation Template:
-//! -//! Copy and paste this template from frame/examples/basic/src/lib.rs into file -//! `frame//src/lib.rs` of your own custom pallet and complete it. -//!

-//! // Add heading with custom pallet name
-//!
-//! \#  Pallet
-//!
-//! // Add simple description
-//!
-//! // Include the following links that shows what trait needs to be implemented to use the pallet
-//! // and the supported dispatchables that are documented in the Call enum.
-//!
-//! - \[`Config`]
-//! - \[`Call`]
-//! - \[`Pallet`]
-//!
-//! \## Overview
-//!
-//! 
-//! // Short description of pallet's purpose.
-//! // Links to Traits that should be implemented.
-//! // What this pallet is for.
-//! // What functionality the pallet provides.
-//! // When to use the pallet (use case examples).
-//! // How it is used.
-//! // Inputs it uses and the source of each input.
-//! // Outputs it produces.
-//!
-//! 
-//! 
-//!
-//! \## Terminology
-//!
-//! // Add terminology used in the custom pallet. Include concepts, storage items, or actions that
-//! you think // deserve to be noted to give context to the rest of the documentation or pallet
-//! usage. The author needs to // use some judgment about what is included. We don't want a list of
-//! every storage item nor types - the user // can go to the code for that. For example, "transfer
-//! fee" is obvious and should not be included, but // "free balance" and "reserved balance" should
-//! be noted to give context to the pallet. // Please do not link to outside resources. The
-//! reference docs should be the ultimate source of truth.
-//!
-//! 
-//!
-//! \## Goals
-//!
-//! // Add goals that the custom pallet is designed to achieve.
-//!
-//! 
-//!
-//! \### Scenarios
-//!
-//! 
-//!
-//! \#### 
-//!
-//! // Describe requirements prior to interacting with the custom pallet.
-//! // Describe the process of interacting with the custom pallet for this scenario and public API
-//! functions used.
-//!
-//! \## Interface
-//!
-//! \### Supported Origins
-//!
-//! // What origins are used and supported in this pallet (root, signed, none)
-//! // i.e. root when \`ensure_root\` used
-//! // i.e. none when \`ensure_none\` used
-//! // i.e. signed when \`ensure_signed\` used
-//!
-//! \`inherent\` 
-//!
-//! 
-//! 
-//!
-//! \### Types
-//!
-//! // Type aliases. Include any associated types and where the user would typically define them.
-//!
-//! \`ExampleType\` 
-//!
-//! 
-//!
-//! // Reference documentation of aspects such as `storageItems` and `dispatchable` functions should
-//! // only be included in the  Rustdocs for Substrate and not repeated in the
-//! // README file.
-//!
-//! \### Dispatchable Functions
-//!
-//! 
-//!
-//! // A brief description of dispatchable functions and a link to the rustdoc with their actual
-//! documentation.
-//!
-//! // MUST have link to Call enum
-//! // MUST have origin information included in function doc
-//! // CAN have more info up to the user
-//!
-//! \### Public Functions
-//!
-//! 
-//!
-//! // A link to the rustdoc and any notes about usage in the pallet, not for specific functions.
-//! // For example, in the Balances Pallet: "Note that when using the publicly exposed functions,
-//! // you (the runtime developer) are responsible for implementing any necessary checks
-//! // (e.g. that the sender is the signer) before calling a function that will affect storage."
-//!
-//! 
-//!
-//! // It is up to the writer of the respective pallet (with respect to how much information to
-//! provide).
-//!
-//! \#### Public Inspection functions - Immutable (getters)
-//!
-//! // Insert a subheading for each getter function signature
-//!
-//! \##### \`example_getter_name()\`
-//!
-//! // What it returns
-//! // Why, when, and how often to call it
-//! // When it could panic or error
-//! // When safety issues to consider
-//!
-//! \#### Public Mutable functions (changing state)
-//!
-//! // Insert a subheading for each setter function signature
-//!
-//! \##### \`example_setter_name(origin, parameter_name: T::ExampleType)\`
-//!
-//! // What state it changes
-//! // Why, when, and how often to call it
-//! // When it could panic or error
-//! // When safety issues to consider
-//! // What parameter values are valid and why
-//!
-//! \### Storage Items
-//!
-//! // Explain any storage items included in this pallet
-//!
-//! \### Digest Items
-//!
-//! // Explain any digest items included in this pallet
-//!
-//! \### Inherent Data
-//!
-//! // Explain what inherent data (if any) is defined in the pallet and any other related types
-//!
-//! \### Events:
-//!
-//! // Insert events for this pallet if any
-//!
-//! \### Errors:
-//!
-//! // Explain what generates errors
-//!
-//! \## Usage
-//!
-//! // Insert 2-3 examples of usage and code snippets that show how to
-//! // use  Pallet in a custom pallet.
-//!
-//! \### Prerequisites
-//!
-//! // Show how to include necessary imports for  and derive
-//! // your pallet configuration trait with the `INSERT_CUSTOM_PALLET_NAME` trait.
-//!
-//! \```rust
-//! use ;
-//!
-//! pub trait Config: ::Config { }
-//! \```
-//!
-//! \### Simple Code Snippet
-//!
-//! // Show a simple example (e.g. how to query a public getter function of
-//! )
-//!
-//! \### Example from FRAME
-//!
-//! // Show a usage example in an actual runtime
-//!
-//! // See:
-//! // - Substrate TCR 
-//! // - Substrate Kitties 
-//!
-//! \## Genesis Config
-//!
-//! 
-//!
-//! \## Dependencies
+//! > Made with *Substrate*, for *Polkadot*.
 //!
-//! // Dependencies on other FRAME pallets and the genesis config should be mentioned,
-//! // but not the Rust Standard Library.
-//! // Genesis configuration modifications that may be made to incorporate this pallet
-//! // Interaction with other pallets
+//! [![github]](https://github.com/paritytech/polkadot-sdk/tree/master/substrate/frame/examples/basic)
+//! [![polkadot]](https://polkadot.network)
 //!
-//! 
+//! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white
+//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github
 //!
-//! \## Related Pallets
+//! ## Pallet API
 //!
-//! // Interaction with other pallets in the form of a bullet point list
+//! See the [`pallet`] module for more information about the interfaces this pallet exposes,
+//! including its configuration trait, dispatchables, storage items, events and errors.
 //!
-//! \## References
+//! ## Overview
 //!
-//! 
+//! This pallet provides basic examples of using:
 //!
-//! // Links to reference material, if applicable. For example, Phragmen, W3F research, etc.
-//! // that the implementation is based on.
-//! 

+//! - A custom weight calculator able to classify a call's dispatch class (see: +//! [`frame_support::dispatch::DispatchClass`]) +//! - Pallet hooks to implement some custom logic that's executed before and after a block is +//! imported (see: [`frame_support::traits::Hooks`]) +//! - Inherited weight annotation for pallet calls, used to create less repetition for calls that +//! use the [`Config::WeightInfo`] trait to calculate call weights. This can also be overridden, +//! as demonstrated by [`Call::set_dummy`]. +//! - A private function that performs a storage update. +//! - A simple signed extension implementation (see: [`sp_runtime::traits::SignedExtension`]) which +//! increases the priority of the [`Call::set_dummy`] if it's present and drops any transaction +//! with an encoded length higher than 200 bytes. // Ensure we're `no_std` when compiling for Wasm. #![cfg_attr(not(feature = "std"), no_std)] @@ -381,7 +163,8 @@ pub mod pallet { #[pallet::pallet] pub struct Pallet(_); - // Pallet implements [`Hooks`] trait to define some logic to execute in some context. + // This pallet implements the [`frame_support::traits::Hooks`] trait to define some logic to + // execute in some context. #[pallet::hooks] impl Hooks> for Pallet { // `on_initialize` is executed at the beginning of the block before any extrinsic are @@ -686,7 +469,7 @@ impl Pallet { // sender of the transaction (if signed) are also provided. // // The full list of hooks that can be added to a signed extension can be found -// [here](https://crates.parity.io/sp_runtime/traits/trait.SignedExtension.html). +// [here](https://paritytech.github.io/polkadot-sdk/master/sp_runtime/traits/trait.SignedExtension.html). // // The signed extensions are aggregated in the runtime file of a substrate chain. All extensions // should be aggregated in a tuple and passed to the `CheckedExtrinsic` and `UncheckedExtrinsic` diff --git a/substrate/frame/examples/split/README.md b/substrate/frame/examples/split/README.md index 413ce9b913cb9850acb0a910297a1aa46202427c..9c4d5443ca06d04815a4392e797cdc32398c504a 100644 --- a/substrate/frame/examples/split/README.md +++ b/substrate/frame/examples/split/README.md @@ -3,7 +3,7 @@ A simple example of a FRAME pallet demonstrating the ability to split sections across multiple files. -Note that this is purely experimental at this point. +Note that this is purely experimental at this point. Run `cargo doc --package pallet-example-split --open` to view this pallet's documentation. diff --git a/substrate/frame/examples/src/lib.rs b/substrate/frame/examples/src/lib.rs index d1cd32bb50f263429567b4872ca5049f08586ac3..a7084fc6ef9bcb465952ec1203323b3d6db99322 100644 --- a/substrate/frame/examples/src/lib.rs +++ b/substrate/frame/examples/src/lib.rs @@ -17,24 +17,27 @@ //! # FRAME Pallet Examples //! -//! This crate contains examples of FRAME pallets. It is not intended to be used in production. +//! This crate contains a collection of simple examples of FRAME pallets, demonstrating useful +//! features in action. It is not intended to be used in production. //! //! ## Pallets //! -//! - [**`pallet_example_basic`**](./basic): A simple example of a FRAME pallet demonstrating -//! concepts, APIs and structures common to most FRAME runtimes. +//! - [`pallet_example_basic`]: This pallet demonstrates concepts, APIs and structures common to +//! most FRAME runtimes. //! -//! - [**`pallet_example_offchain_worker`**](./offchain-worker): A simple example of a FRAME pallet -//! demonstrating concepts, APIs and structures common to most offchain workers. +//! - [`pallet_example_offchain_worker`]: This pallet demonstrates concepts, APIs and structures +//! common to most offchain workers. //! -//! - [**`pallet-default-config-example`**](./default-config): A simple example of a FRAME pallet -//! demonstrating the simpler way to implement `Config` trait of pallets. +//! - [`pallet_default_config_example`]: This pallet demonstrates different ways to implement the +//! `Config` trait of pallets. //! -//! - [**`pallet-dev-mode`**](./dev-mode): A simple example of a FRAME pallet demonstrating the ease -//! of requirements for a pallet in dev mode. +//! - [`pallet_dev_mode`]: This pallet demonstrates the ease of requirements for a pallet in "dev +//! mode". //! -//! - [**`pallet-example-kitchensink`**](./kitchensink): A simple example of a FRAME pallet -//! demonstrating a catalog of the the FRAME macros and their various syntax options. +//! - [`pallet_example_kitchensink`]: This pallet demonstrates a catalog of all FRAME macros in use +//! and their various syntax options. //! -//! - [**`pallet-example-split`**](./split): A simple example of a FRAME pallet demonstrating the -//! ability to split sections across multiple files. +//! - [`pallet_example_split`]: A simple example of a FRAME pallet demonstrating the ability to +//! split sections across multiple files. +//! +//! **Tip**: Use `cargo doc --package --open` to view each pallet's documentation. diff --git a/substrate/frame/executive/README.md b/substrate/frame/executive/README.md index c14c3912b082d940aad7e0ee848c718915242a5e..96a412a4537a3657563550558f1828d209b4b92b 100644 --- a/substrate/frame/executive/README.md +++ b/substrate/frame/executive/README.md @@ -1,13 +1,13 @@ # Executive Module -The Executive module acts as the orchestration layer for the runtime. It dispatches incoming -extrinsic calls to the respective modules in the runtime. +The Executive module acts as the orchestration layer for the runtime. It dispatches incoming extrinsic calls to the +respective modules in the runtime. ## Overview -The executive module is not a typical pallet providing functionality around a specific feature. -It is a cross-cutting framework component for the FRAME. It works in conjunction with the -[FRAME System module](https://docs.rs/frame-system/latest/frame_system/) to perform these cross-cutting functions. +The executive module is not a typical pallet providing functionality around a specific feature. It is a cross-cutting +framework component for the FRAME. It works in conjunction with the [FRAME System +module](https://docs.rs/frame-system/latest/frame_system/) to perform these cross-cutting functions. The Executive module provides functions to: @@ -26,7 +26,8 @@ The Executive module provides the following implementations: ## Usage -The default Substrate node template declares the [`Executive`](https://docs.rs/frame-executive/latest/frame_executive/struct.Executive.html) type in its library. +The default Substrate node template declares the +[`Executive`](https://docs.rs/frame-executive/latest/frame_executive/struct.Executive.html) type in its library. ### Example @@ -46,9 +47,8 @@ pub type Executive = executive::Executive< ### Custom `OnRuntimeUpgrade` logic -You can add custom logic that should be called in your runtime on a runtime upgrade. This is -done by setting an optional generic parameter. The custom logic will be called before -the on runtime upgrade logic of all modules is called. +You can add custom logic that should be called in your runtime on a runtime upgrade. This is done by setting an optional +generic parameter. The custom logic will be called before the on runtime upgrade logic of all modules is called. ```rust # diff --git a/substrate/frame/fast-unstake/Cargo.toml b/substrate/frame/fast-unstake/Cargo.toml index a2aa769620cb21885f6537e47164cee3325ed2fa..85548c6600db827f316549b91c4a120be406745d 100644 --- a/substrate/frame/fast-unstake/Cargo.toml +++ b/substrate/frame/fast-unstake/Cargo.toml @@ -27,7 +27,7 @@ frame-election-provider-support = { path = "../election-provider-support", defau frame-benchmarking = { path = "../benchmarking", default-features = false, optional = true} -docify = "0.2.1" +docify = "0.2.4" [dev-dependencies] pallet-staking-reward-curve = { path = "../staking/reward-curve" } diff --git a/substrate/frame/glutton/README.md b/substrate/frame/glutton/README.md index 8ad4f79171820a48fe0d9d94f6793a931840e6e6..89dbe26ec7a9d7eb0ad379f0526a770b4ba45a28 100644 --- a/substrate/frame/glutton/README.md +++ b/substrate/frame/glutton/README.md @@ -4,6 +4,9 @@ # Glutton Pallet -The `Glutton` pallet gets the name from its property to consume vast amounts of resources. It can be used to push para-chains and their relay-chains to the limits. This is good for testing out theoretical limits in a practical way. +The `Glutton` pallet gets the name from its property to consume vast amounts of resources. It can be used to push +para-chains and their relay-chains to the limits. This is good for testing out theoretical limits in a practical way. -The `Glutton` can be set to consume a fraction of the available unused weight of a chain. It accomplishes this by utilizing the `on_idle` hook and consuming a specific ration of the remaining weight. The rations can be set via `set_compute` and `set_storage`. Initially the `Glutton` needs to be initialized once with `initialize_pallet`. +The `Glutton` can be set to consume a fraction of the available unused weight of a chain. It accomplishes this by +utilizing the `on_idle` hook and consuming a specific ration of the remaining weight. The rations can be set via +`set_compute` and `set_storage`. Initially the `Glutton` needs to be initialized once with `initialize_pallet`. diff --git a/substrate/frame/grandpa/README.md b/substrate/frame/grandpa/README.md index 84b181a8b31e1c102215f72d6a2d59724a9af596..5978931c5a85f554b0a1e036f840a71beaa7302a 100644 --- a/substrate/frame/grandpa/README.md +++ b/substrate/frame/grandpa/README.md @@ -9,4 +9,4 @@ finality notifications. For full integration with GRANDPA, the `GrandpaApi` should be implemented. The necessary items are re-exported via the `fg_primitives` crate. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/identity/README.md b/substrate/frame/identity/README.md index a67c259e2537af20118e194dbb31b66bfdbb1353..0203656eff46b6eb49726704a535e68dccad60f2 100644 --- a/substrate/frame/identity/README.md +++ b/substrate/frame/identity/README.md @@ -28,27 +28,27 @@ no state-bloat attack is viable. ### Dispatchable Functions #### For general users -* `set_identity` - Set the associated identity of an account; a small deposit is reserved if not +- `set_identity` - Set the associated identity of an account; a small deposit is reserved if not already taken. -* `clear_identity` - Remove an account's associated identity; the deposit is returned. -* `request_judgement` - Request a judgement from a registrar, paying a fee. -* `cancel_request` - Cancel the previous request for a judgement. +- `clear_identity` - Remove an account's associated identity; the deposit is returned. +- `request_judgement` - Request a judgement from a registrar, paying a fee. +- `cancel_request` - Cancel the previous request for a judgement. #### For general users with sub-identities -* `set_subs` - Set the sub-accounts of an identity. -* `add_sub` - Add a sub-identity to an identity. -* `remove_sub` - Remove a sub-identity of an identity. -* `rename_sub` - Rename a sub-identity of an identity. -* `quit_sub` - Remove a sub-identity of an identity (called by the sub-identity). +- `set_subs` - Set the sub-accounts of an identity. +- `add_sub` - Add a sub-identity to an identity. +- `remove_sub` - Remove a sub-identity of an identity. +- `rename_sub` - Rename a sub-identity of an identity. +- `quit_sub` - Remove a sub-identity of an identity (called by the sub-identity). #### For registrars -* `set_fee` - Set the fee required to be paid for a judgement to be given by the registrar. -* `set_fields` - Set the fields that a registrar cares about in their judgements. -* `provide_judgement` - Provide a judgement to an identity. +- `set_fee` - Set the fee required to be paid for a judgement to be given by the registrar. +- `set_fields` - Set the fields that a registrar cares about in their judgements. +- `provide_judgement` - Provide a judgement to an identity. #### For super-users -* `add_registrar` - Add a new registrar to the system. -* `kill_identity` - Forcibly remove the associated identity; the deposit is lost. +- `add_registrar` - Add a new registrar to the system. +- `kill_identity` - Forcibly remove the associated identity; the deposit is lost. [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html diff --git a/substrate/frame/indices/README.md b/substrate/frame/indices/README.md index 243392780db28e9e6b941a5bb1e7d3c79d78887d..ba4b9679a294b5969c05f0efe7a2f48645540f22 100644 --- a/substrate/frame/indices/README.md +++ b/substrate/frame/indices/README.md @@ -1,4 +1,4 @@ An index is a short form of an address. This module handles allocation of indices for a newly created accounts. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/insecure-randomness-collective-flip/README.md b/substrate/frame/insecure-randomness-collective-flip/README.md index ef02e4b5c8445132be938e7b84dddc845551b49e..4f02782fa65910068ea9d0345b06156d06f8ca57 100644 --- a/substrate/frame/insecure-randomness-collective-flip/README.md +++ b/substrate/frame/insecure-randomness-collective-flip/README.md @@ -1,25 +1,27 @@ # DO NOT USE IN PRODUCTION -The produced values do not fulfill the cryptographic requirements for random numbers. Should not be used for high-stake production use-cases. +The produced values do not fulfill the cryptographic requirements for random numbers. Should not be used for high-stake +production use-cases. # Randomness Module -The Randomness Collective Flip module provides a [`random`](https://docs.rs/pallet-insecure-randomness-collective-flip/latest/pallet_insecure_randomness_collective_flip/struct.Module.html#method.random) -function that generates low-influence random values based on the block hashes from the previous -`81` blocks. Low-influence randomness can be useful when defending against relatively weak -adversaries. Using this pallet as a randomness source is advisable primarily in low-security -situations like testing. +The Randomness Collective Flip module provides a +[`random`](https://docs.rs/pallet-insecure-randomness-collective-flip/latest/pallet_insecure_randomness_collective_flip/struct.Module.html#method.random) +function that generates low-influence random values based on the block hashes from the previous `81` blocks. +Low-influence randomness can be useful when defending against relatively weak adversaries. Using this pallet as a +randomness source is advisable primarily in low-security situations like testing. ## Public Functions -See the [`Module`](https://docs.rs/pallet-insecure-randomness-collective-flip/latest/pallet_insecure_randomness_collective_flip/struct.Module.html) struct for details of publicly available functions. +See the +[`Module`](https://docs.rs/pallet-insecure-randomness-collective-flip/latest/pallet_insecure_randomness_collective_flip/struct.Module.html) +struct for details of publicly available functions. ## Usage ### Prerequisites -Import the Randomness Collective Flip module and derive your module's configuration trait from -the system trait. +Import the Randomness Collective Flip module and derive your module's configuration trait from the system trait. ### Example - Get random seed for the current block diff --git a/substrate/frame/message-queue/src/lib.rs b/substrate/frame/message-queue/src/lib.rs index 9ded84bb035c5f1adf964e9748dc3805fad3ff61..7c38dec4b08052f0847204447938bbdac725e422 100644 --- a/substrate/frame/message-queue/src/lib.rs +++ b/substrate/frame/message-queue/src/lib.rs @@ -1092,7 +1092,7 @@ impl Pallet { origin.clone(), page_index, page.first_index, - payload.deref(), + payload, weight, overweight_limit, ) { @@ -1242,7 +1242,7 @@ impl Pallet { if let Some((_, processed, message)) = page.peek_index(i.try_into().expect("std-only code")) { - let msg = String::from_utf8_lossy(message.deref()); + let msg = String::from_utf8_lossy(message); if processed { page_info.push('*'); } diff --git a/substrate/frame/multisig/README.md b/substrate/frame/multisig/README.md index 5f320377d3454e5e2d09f9346ef6539a141ac76c..e9095198585b9a4fc3cc0ea5e8379ad99c12bcac 100644 --- a/substrate/frame/multisig/README.md +++ b/substrate/frame/multisig/README.md @@ -18,10 +18,10 @@ not available or desired. ### Dispatchable Functions -* `as_multi` - Approve and if possible dispatch a call from a composite origin formed from a +- `as_multi` - Approve and if possible dispatch a call from a composite origin formed from a number of signed origins. -* `approve_as_multi` - Approve a call from a composite origin. -* `cancel_as_multi` - Cancel a call from a composite origin. +- `approve_as_multi` - Approve a call from a composite origin. +- `cancel_as_multi` - Cancel a call from a composite origin. [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html diff --git a/substrate/frame/nft-fractionalization/README.md b/substrate/frame/nft-fractionalization/README.md index 180eef22cc46f5e1237975244b582b3be03f2200..3f83ae9d150d59415b0cc3dd3d9d8121b9390e7e 100644 --- a/substrate/frame/nft-fractionalization/README.md +++ b/substrate/frame/nft-fractionalization/README.md @@ -1,6 +1,8 @@ -### Lock NFT +# Lock NFT Lock an NFT from `pallet-nfts` and mint fungible assets from `pallet-assets`. -The NFT gets locked by putting a system-level attribute named `Locked`. This prevents the NFT from being transferred further. -The NFT becomes unlocked when the `Locked` attribute is removed. In order to unify the fungible asset and unlock the NFT, an account must hold the full issuance of the asset the NFT was fractionalised into. Holding less of the fungible asset will not allow the unlocking of the NFT. +The NFT gets locked by putting a system-level attribute named `Locked`. This prevents the NFT from being transferred +further. The NFT becomes unlocked when the `Locked` attribute is removed. In order to unify the fungible asset and +unlock the NFT, an account must hold the full issuance of the asset the NFT was fractionalised into. Holding less of the +fungible asset will not allow the unlocking of the NFT. diff --git a/substrate/frame/nfts/README.md b/substrate/frame/nfts/README.md index 7de4b9440e7f50a30aee080601df44537defc282..93ccf29498525f80af2f415476dab3805c40cffb 100644 --- a/substrate/frame/nfts/README.md +++ b/substrate/frame/nfts/README.md @@ -13,9 +13,11 @@ The NFTs pallet provides functionality for non-fungible tokens' management, incl * Attributes Management * NFT Burning -To use it in your runtime, you need to implement [`nfts::Config`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/trait.Config.html). +To use it in your runtime, you need to implement +[`nfts::Config`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/trait.Config.html). -The supported dispatchable functions are documented in the [`nfts::Call`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/enum.Call.html) enum. +The supported dispatchable functions are documented in the +[`nfts::Call`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/enum.Call.html) enum. ### Terminology @@ -24,8 +26,9 @@ The supported dispatchable functions are documented in the [`nfts::Call`](https: * **NFT transfer:** The action of sending an item from one account to another. * **Atomic swap:** The action of exchanging items between accounts without needing a 3rd party service. * **NFT burning:** The destruction of an item. -* **Non-fungible token (NFT):** An item for which each unit has unique characteristics. There is exactly - one instance of such an item in existence and there is exactly one owning account (though that owning account could be a proxy account or multi-sig account). +* **Non-fungible token (NFT):** An item for which each unit has unique characteristics. There is exactly one instance of + such an item in existence and there is exactly one owning account (though that owning account could be a proxy account + or multi-sig account). * **Soul Bound NFT:** An item that is non-transferable from the account which it is minted into. ### Goals @@ -35,10 +38,8 @@ The NFTs pallet in Substrate is designed to make the following possible: * Allow accounts to permissionlessly create nft collections. * Allow a named (permissioned) account to mint and burn unique items within a collection. * Move items between accounts permissionlessly. -* Allow a named (permissioned) account to freeze and unfreeze items within a - collection or the entire collection. -* Allow the owner of an item to delegate the ability to transfer the item to some - named third-party. +* Allow a named (permissioned) account to freeze and unfreeze items within a collection or the entire collection. +* Allow the owner of an item to delegate the ability to transfer the item to some named third-party. * Allow third-parties to store information in an NFT _without_ owning it (Eg. save game state). ## Interface @@ -71,7 +72,8 @@ The NFTs pallet in Substrate is designed to make the following possible: * `clear_all_transfer_approvals`: Clears all transfer approvals set by calling the `approve_transfer`. * `lock_collection`: Prevent all items within a collection from being transferred (making them all `soul bound`). * `lock_item_properties`: Lock item's metadata or attributes. -* `transfer_ownership`: Alter the owner of a collection, moving all associated deposits. (Ownership of individual items will not be affected.) +* `transfer_ownership`: Alter the owner of a collection, moving all associated deposits. (Ownership of individual items + will not be affected.) * `set_team`: Alter the permissioned accounts of a collection. * `set_collection_max_supply`: Change the max supply of a collection. * `update_mint_settings`: Update the minting settings for collection. @@ -94,8 +96,8 @@ The NFTs pallet in Substrate is designed to make the following possible: * `force_collection_config`: Change collection's config. * `force_set_attribute`: Set an attribute. -Please refer to the [`Call`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/enum.Call.html) enum -and its associated variants for documentation on each function. +Please refer to the [`Call`](https://paritytech.github.io/substrate/master/pallet_nfts/pallet/enum.Call.html) enum and +its associated variants for documentation on each function. ## Related Modules diff --git a/substrate/frame/nicks/README.md b/substrate/frame/nicks/README.md index 768043ffb9bf96964abbc60a2b39e124143d1e9c..2b05f32d33448e9041b041653dcb1ffd8d3fba26 100644 --- a/substrate/frame/nicks/README.md +++ b/substrate/frame/nicks/README.md @@ -14,10 +14,10 @@ have not been designed to be economically secure. Do not use this pallet as-is i ### Dispatchable Functions -* `set_name` - Set the associated name of an account; a small deposit is reserved if not already +- `set_name` - Set the associated name of an account; a small deposit is reserved if not already taken. -* `clear_name` - Remove an account's associated name; the deposit is returned. -* `kill_name` - Forcibly remove the associated name; the deposit is lost. +- `clear_name` - Remove an account's associated name; the deposit is returned. +- `kill_name` - Forcibly remove the associated name; the deposit is lost. [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html diff --git a/substrate/frame/nis/README.md b/substrate/frame/nis/README.md index 0c3f0c383a16cf9cb3bc92e201c2844acc21e1c4..032df7d01868f8643284985985fc98b3f6ccc0ac 100644 --- a/substrate/frame/nis/README.md +++ b/substrate/frame/nis/README.md @@ -2,4 +2,4 @@ Provides a non-interactiove variant of staking. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/nomination-pools/runtime-api/README.md b/substrate/frame/nomination-pools/runtime-api/README.md index af90b31733b0b306eec13f43ccbd6bb6a857f1ac..499af052a73ef06b332becc9b900857947c87d15 100644 --- a/substrate/frame/nomination-pools/runtime-api/README.md +++ b/substrate/frame/nomination-pools/runtime-api/README.md @@ -1,3 +1,3 @@ Runtime API definition for nomination-pools pallet. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index c4bebc5a1d0304070b47bbc546dd7c56e90ecf63..485cdada7173b2c583b3cf49ab89ac05c9e026dd 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -3134,6 +3134,10 @@ impl Pallet { // reward math rounds down, we might accumulate some dust here. let pending_rewards_lt_leftover_bal = RewardPool::::current_balance(id) >= pools_members_pending_rewards.get(&id).copied().unwrap_or_default(); + + // this is currently broken in Kusama, a fix is being worked on in + // . until it is fixed, log a + // warning instead of panicing with an `ensure` statement. if !pending_rewards_lt_leftover_bal { log::warn!( "pool {:?}, sum pending rewards = {:?}, remaining balance = {:?}", @@ -3142,10 +3146,6 @@ impl Pallet { RewardPool::::current_balance(id) ); } - ensure!( - pending_rewards_lt_leftover_bal, - "The sum of the pending rewards must be less than the leftover balance." - ); Ok(()) })?; diff --git a/substrate/frame/offences/README.md b/substrate/frame/offences/README.md index 454c7effaf36cc599516feb5d047606415ee68d5..e7cf302fb8a0e821c18bb90414d789b456b690e4 100644 --- a/substrate/frame/offences/README.md +++ b/substrate/frame/offences/README.md @@ -2,4 +2,4 @@ Tracks reported offences -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/offences/benchmarking/README.md b/substrate/frame/offences/benchmarking/README.md index cbfe91d73a6a7cc5118f4e56a806f4c766c426d1..95892a8f344fb30048f44f6590f88a005d7db0e5 100644 --- a/substrate/frame/offences/benchmarking/README.md +++ b/substrate/frame/offences/benchmarking/README.md @@ -1,3 +1,3 @@ Offences pallet benchmarking. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/paged-list/Cargo.toml b/substrate/frame/paged-list/Cargo.toml index f3165a2f5f92025d605447bb0a5d7d10841b528a..eee099fff5f062b54085094050f55990018a7890 100644 --- a/substrate/frame/paged-list/Cargo.toml +++ b/substrate/frame/paged-list/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] } -docify = "0.2.1" +docify = "0.2.4" scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } frame-benchmarking = { path = "../benchmarking", default-features = false, optional = true} diff --git a/substrate/frame/preimage/src/benchmarking.rs b/substrate/frame/preimage/src/benchmarking.rs index b719c28be641bb3d57086798e1cd694c826dc317..d0c3404f40a911cf32d620858b97652b242f2be8 100644 --- a/substrate/frame/preimage/src/benchmarking.rs +++ b/substrate/frame/preimage/src/benchmarking.rs @@ -18,7 +18,7 @@ //! Preimage pallet benchmarking. use super::*; -use frame_benchmarking::v1::{account, benchmarks, whitelist_account, BenchmarkError}; +use frame_benchmarking::v1::{account, benchmarks, whitelisted_caller, BenchmarkError}; use frame_support::assert_ok; use frame_system::RawOrigin; use sp_runtime::traits::Bounded; @@ -26,11 +26,9 @@ use sp_std::{prelude::*, vec}; use crate::Pallet as Preimage; -const SEED: u32 = 0; - -fn funded_account(name: &'static str, index: u32) -> T::AccountId { - let caller: T::AccountId = account(name, index, SEED); - T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); +fn funded_account() -> T::AccountId { + let caller: T::AccountId = whitelisted_caller(); + T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value() / 2u32.into()); caller } @@ -49,8 +47,7 @@ benchmarks! { // Expensive note - will reserve. note_preimage { let s in 0 .. MAX_SIZE; - let caller = funded_account::("caller", 0); - whitelist_account!(caller); + let caller = funded_account::(); let (preimage, hash) = sized_preimage_and_hash::(s); }: _(RawOrigin::Signed(caller), preimage) verify { @@ -59,8 +56,7 @@ benchmarks! { // Cheap note - will not reserve since it was requested. note_requested_preimage { let s in 0 .. MAX_SIZE; - let caller = funded_account::("caller", 0); - whitelist_account!(caller); + let caller = funded_account::(); let (preimage, hash) = sized_preimage_and_hash::(s); assert_ok!(Preimage::::request_preimage( T::ManagerOrigin::try_successful_origin() @@ -89,8 +85,7 @@ benchmarks! { // Expensive unnote - will unreserve. unnote_preimage { - let caller = funded_account::("caller", 0); - whitelist_account!(caller); + let caller = funded_account::(); let (preimage, hash) = preimage_and_hash::(); assert_ok!(Preimage::::note_preimage(RawOrigin::Signed(caller.clone()).into(), preimage)); }: _(RawOrigin::Signed(caller), hash) @@ -115,16 +110,15 @@ benchmarks! { // Expensive request - will unreserve the noter's deposit. request_preimage { let (preimage, hash) = preimage_and_hash::(); - let noter = funded_account::("noter", 0); - whitelist_account!(noter); + let noter = funded_account::(); assert_ok!(Preimage::::note_preimage(RawOrigin::Signed(noter.clone()).into(), preimage)); }: _( T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - let deposit = T::BaseDeposit::get() + T::ByteDeposit::get() * MAX_SIZE.into(); - let s = RequestStatus::Requested { deposit: Some((noter, deposit)), count: 1, len: Some(MAX_SIZE) }; - assert_eq!(StatusFor::::get(&hash), Some(s)); + let ticket = TicketOf::::new(¬er, Footprint { count: 1, size: MAX_SIZE as u64 }).unwrap(); + let s = RequestStatus::Requested { maybe_ticket: Some((noter, ticket)), count: 1, maybe_len: Some(MAX_SIZE) }; + assert_eq!(RequestStatusFor::::get(&hash), Some(s)); } // Cheap request - would unreserve the deposit but none was held. request_no_deposit_preimage { @@ -138,8 +132,8 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - let s = RequestStatus::Requested { deposit: None, count: 2, len: Some(MAX_SIZE) }; - assert_eq!(StatusFor::::get(&hash), Some(s)); + let s = RequestStatus::Requested { maybe_ticket: None, count: 2, maybe_len: Some(MAX_SIZE) }; + assert_eq!(RequestStatusFor::::get(&hash), Some(s)); } // Cheap request - the preimage is not yet noted, so deposit to unreserve. request_unnoted_preimage { @@ -148,8 +142,8 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - let s = RequestStatus::Requested { deposit: None, count: 1, len: None }; - assert_eq!(StatusFor::::get(&hash), Some(s)); + let s = RequestStatus::Requested { maybe_ticket: None, count: 1, maybe_len: None }; + assert_eq!(RequestStatusFor::::get(&hash), Some(s)); } // Cheap request - the preimage is already requested, so just a counter bump. request_requested_preimage { @@ -163,8 +157,8 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - let s = RequestStatus::Requested { deposit: None, count: 2, len: None }; - assert_eq!(StatusFor::::get(&hash), Some(s)); + let s = RequestStatus::Requested { maybe_ticket: None, count: 2, maybe_len: None }; + assert_eq!(RequestStatusFor::::get(&hash), Some(s)); } // Expensive unrequest - last reference and it's noted, so will destroy the preimage. @@ -184,7 +178,7 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - assert_eq!(StatusFor::::get(&hash), None); + assert_eq!(RequestStatusFor::::get(&hash), None); } // Cheap unrequest - last reference, but it's not noted. unrequest_unnoted_preimage { @@ -198,7 +192,7 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - assert_eq!(StatusFor::::get(&hash), None); + assert_eq!(RequestStatusFor::::get(&hash), None); } // Cheap unrequest - not the last reference. unrequest_multi_referenced_preimage { @@ -217,9 +211,38 @@ benchmarks! { T::ManagerOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?, hash ) verify { - let s = RequestStatus::Requested { deposit: None, count: 1, len: None }; - assert_eq!(StatusFor::::get(&hash), Some(s)); + let s = RequestStatus::Requested { maybe_ticket: None, count: 1, maybe_len: None }; + assert_eq!(RequestStatusFor::::get(&hash), Some(s)); + } + + ensure_updated { + let n in 1..MAX_HASH_UPGRADE_BULK_COUNT; + + let caller = funded_account::(); + let hashes = (0..n).map(|i| insert_old_unrequested::(i)).collect::>(); + }: _(RawOrigin::Signed(caller), hashes) + verify { + assert_eq!(RequestStatusFor::::iter_keys().count(), n as usize); + #[allow(deprecated)] + let c = StatusFor::::iter_keys().count(); + assert_eq!(c, 0); } impl_benchmark_test_suite!(Preimage, crate::mock::new_test_ext(), crate::mock::Test); } + +fn insert_old_unrequested(s: u32) -> ::Hash { + let acc = account("old", s, 0); + T::Currency::make_free_balance_be(&acc, BalanceOf::::max_value() / 2u32.into()); + + // The preimage size does not matter here as it is not touched. + let preimage = s.to_le_bytes(); + let hash = ::Hashing::hash(&preimage[..]); + + #[allow(deprecated)] + StatusFor::::insert( + &hash, + OldRequestStatus::Unrequested { deposit: (acc, 123u32.into()), len: preimage.len() as u32 }, + ); + hash +} diff --git a/substrate/frame/preimage/src/lib.rs b/substrate/frame/preimage/src/lib.rs index 5ab1e7643b2e7150c1a2f166f4ffcabf69fef882..4dda0207e62c816882b0e4ffcc38476452c274ea 100644 --- a/substrate/frame/preimage/src/lib.rs +++ b/substrate/frame/preimage/src/lib.rs @@ -37,7 +37,10 @@ mod mock; mod tests; pub mod weights; -use sp_runtime::traits::{BadOrigin, Hash, Saturating}; +use sp_runtime::{ + traits::{BadOrigin, Hash, Saturating}, + Perbill, +}; use sp_std::{borrow::Cow, prelude::*}; use codec::{Decode, Encode, MaxEncodedLen}; @@ -46,8 +49,8 @@ use frame_support::{ ensure, pallet_prelude::Get, traits::{ - Currency, Defensive, FetchResult, Hash as PreimageHash, PreimageProvider, - PreimageRecipient, QueryPreimage, ReservableCurrency, StorePreimage, + Consideration, Currency, Defensive, FetchResult, Footprint, Hash as PreimageHash, + PreimageProvider, PreimageRecipient, QueryPreimage, ReservableCurrency, StorePreimage, }, BoundedSlice, BoundedVec, }; @@ -61,7 +64,7 @@ pub use pallet::*; /// A type to note whether a preimage is owned by a user or the system. #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] -pub enum RequestStatus { +pub enum OldRequestStatus { /// The associated preimage has not yet been requested by the system. The given deposit (if /// some) is being held until either it becomes requested or the user retracts the preimage. Unrequested { deposit: (AccountId, Balance), len: u32 }, @@ -71,13 +74,31 @@ pub enum RequestStatus { Requested { deposit: Option<(AccountId, Balance)>, count: u32, len: Option }, } +/// A type to note whether a preimage is owned by a user or the system. +#[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] +pub enum RequestStatus { + /// The associated preimage has not yet been requested by the system. The given deposit (if + /// some) is being held until either it becomes requested or the user retracts the preimage. + Unrequested { ticket: (AccountId, Ticket), len: u32 }, + /// There are a non-zero number of outstanding requests for this hash by this chain. If there + /// is a preimage registered, then `len` is `Some` and it may be removed iff this counter + /// becomes zero. + Requested { maybe_ticket: Option<(AccountId, Ticket)>, count: u32, maybe_len: Option }, +} + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; +type TicketOf = ::Consideration; /// Maximum size of preimage we can store is 4mb. const MAX_SIZE: u32 = 4 * 1024 * 1024; +/// Hard-limit on the number of hashes that can be passed to `ensure_updated`. +/// +/// Exists only for benchmarking purposes. +pub const MAX_HASH_UPGRADE_BULK_COUNT: u32 = 1024; #[frame_support::pallet] +#[allow(deprecated)] pub mod pallet { use super::*; @@ -93,17 +114,15 @@ pub mod pallet { type WeightInfo: weights::WeightInfo; /// Currency type for this pallet. + // TODO#1569: Remove. type Currency: ReservableCurrency; /// An origin that can request a preimage be placed on-chain without a deposit or fee, or /// manage existing preimages. type ManagerOrigin: EnsureOrigin; - /// The base deposit for placing a preimage on chain. - type BaseDeposit: Get>; - - /// The per-byte deposit for placing a preimage on chain. - type ByteDeposit: Get>; + /// A means of providing some cost while data is stored on-chain. + type Consideration: Consideration; } #[pallet::pallet] @@ -135,18 +154,35 @@ pub mod pallet { Requested, /// The preimage request cannot be removed since no outstanding requests exist. NotRequested, + /// More than `MAX_HASH_UPGRADE_BULK_COUNT` hashes were requested to be upgraded at once. + TooMany, + /// Too few hashes were requested to be upgraded (i.e. zero). + TooFew, + } + + /// A reason for this pallet placing a hold on funds. + #[pallet::composite_enum] + pub enum HoldReason { + /// The funds are held as storage deposit for a preimage. + Preimage, } /// The request status of a given hash. + #[deprecated = "RequestStatusFor"] #[pallet::storage] pub(super) type StatusFor = - StorageMap<_, Identity, T::Hash, RequestStatus>>; + StorageMap<_, Identity, T::Hash, OldRequestStatus>>; + + /// The request status of a given hash. + #[pallet::storage] + pub(super) type RequestStatusFor = + StorageMap<_, Identity, T::Hash, RequestStatus>>; #[pallet::storage] pub(super) type PreimageFor = StorageMap<_, Identity, (T::Hash, u32), BoundedVec>>; - #[pallet::call] + #[pallet::call(weight = T::WeightInfo)] impl Pallet { /// Register a preimage on-chain. /// @@ -173,7 +209,6 @@ pub mod pallet { /// - `hash`: The hash of the preimage to be removed from the store. /// - `len`: The length of the preimage of `hash`. #[pallet::call_index(1)] - #[pallet::weight(T::WeightInfo::unnote_preimage())] pub fn unnote_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { let maybe_sender = Self::ensure_signed_or_manager(origin)?; Self::do_unnote_preimage(&hash, maybe_sender) @@ -184,7 +219,6 @@ pub mod pallet { /// If the preimage requests has already been provided on-chain, we unreserve any deposit /// a user may have paid, and take the control of the preimage out of their hands. #[pallet::call_index(2)] - #[pallet::weight(T::WeightInfo::request_preimage())] pub fn request_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { T::ManagerOrigin::ensure_origin(origin)?; Self::do_request_preimage(&hash); @@ -195,15 +229,81 @@ pub mod pallet { /// /// NOTE: THIS MUST NOT BE CALLED ON `hash` MORE TIMES THAN `request_preimage`. #[pallet::call_index(3)] - #[pallet::weight(T::WeightInfo::unrequest_preimage())] pub fn unrequest_preimage(origin: OriginFor, hash: T::Hash) -> DispatchResult { T::ManagerOrigin::ensure_origin(origin)?; Self::do_unrequest_preimage(&hash) } + + /// Ensure that the a bulk of pre-images is upgraded. + /// + /// The caller pays no fee if at least 90% of pre-images were successfully updated. + #[pallet::call_index(4)] + #[pallet::weight(T::WeightInfo::ensure_updated(hashes.len() as u32))] + pub fn ensure_updated( + origin: OriginFor, + hashes: Vec, + ) -> DispatchResultWithPostInfo { + ensure_signed(origin)?; + ensure!(hashes.len() > 0, Error::::TooFew); + ensure!(hashes.len() <= MAX_HASH_UPGRADE_BULK_COUNT as usize, Error::::TooMany); + + let updated = hashes.iter().map(Self::do_ensure_updated).filter(|b| *b).count() as u32; + let ratio = Perbill::from_rational(updated, hashes.len() as u32); + + let pays: Pays = (ratio < Perbill::from_percent(90)).into(); + Ok(pays.into()) + } } } impl Pallet { + fn do_ensure_updated(h: &T::Hash) -> bool { + #[allow(deprecated)] + let r = match StatusFor::::take(h) { + Some(r) => r, + None => return false, + }; + let n = match r { + OldRequestStatus::Unrequested { deposit: (who, amount), len } => { + // unreserve deposit + T::Currency::unreserve(&who, amount); + // take consideration + let Ok(ticket) = + T::Consideration::new(&who, Footprint::from_parts(1, len as usize)) + .defensive_proof("Unexpected inability to take deposit after unreserved") + else { + return true + }; + RequestStatus::Unrequested { ticket: (who, ticket), len } + }, + OldRequestStatus::Requested { deposit: maybe_deposit, count, len: maybe_len } => { + let maybe_ticket = if let Some((who, deposit)) = maybe_deposit { + // unreserve deposit + T::Currency::unreserve(&who, deposit); + // take consideration + if let Some(len) = maybe_len { + let Ok(ticket) = + T::Consideration::new(&who, Footprint::from_parts(1, len as usize)) + .defensive_proof( + "Unexpected inability to take deposit after unreserved", + ) + else { + return true + }; + Some((who, ticket)) + } else { + None + } + } else { + None + }; + RequestStatus::Requested { maybe_ticket, count, maybe_len } + }, + }; + RequestStatusFor::::insert(h, n); + true + } + /// Ensure that the origin is either the `ManagerOrigin` or a signed origin. fn ensure_signed_or_manager( origin: T::RuntimeOrigin, @@ -230,26 +330,29 @@ impl Pallet { let len = preimage.len() as u32; ensure!(len <= MAX_SIZE, Error::::TooBig); + Self::do_ensure_updated(&hash); // We take a deposit only if there is a provided depositor and the preimage was not // previously requested. This also allows the tx to pay no fee. - let status = match (StatusFor::::get(hash), maybe_depositor) { - (Some(RequestStatus::Requested { count, deposit, .. }), _) => - RequestStatus::Requested { count, deposit, len: Some(len) }, + let status = match (RequestStatusFor::::get(hash), maybe_depositor) { + (Some(RequestStatus::Requested { maybe_ticket, count, .. }), _) => + RequestStatus::Requested { maybe_ticket, count, maybe_len: Some(len) }, (Some(RequestStatus::Unrequested { .. }), Some(_)) => return Err(Error::::AlreadyNoted.into()), - (Some(RequestStatus::Unrequested { len, deposit }), None) => - RequestStatus::Requested { deposit: Some(deposit), count: 1, len: Some(len) }, - (None, None) => RequestStatus::Requested { count: 1, len: Some(len), deposit: None }, + (Some(RequestStatus::Unrequested { ticket, len }), None) => RequestStatus::Requested { + maybe_ticket: Some(ticket), + count: 1, + maybe_len: Some(len), + }, + (None, None) => + RequestStatus::Requested { maybe_ticket: None, count: 1, maybe_len: Some(len) }, (None, Some(depositor)) => { - let length = preimage.len() as u32; - let deposit = T::BaseDeposit::get() - .saturating_add(T::ByteDeposit::get().saturating_mul(length.into())); - T::Currency::reserve(depositor, deposit)?; - RequestStatus::Unrequested { deposit: (depositor.clone(), deposit), len } + let ticket = + T::Consideration::new(depositor, Footprint::from_parts(1, len as usize))?; + RequestStatus::Unrequested { ticket: (depositor.clone(), ticket), len } }, }; let was_requested = matches!(status, RequestStatus::Requested { .. }); - StatusFor::::insert(hash, status); + RequestStatusFor::::insert(hash, status); let _ = Self::insert(&hash, preimage) .defensive_proof("Unable to insert. Logic error in `note_bytes`?"); @@ -264,15 +367,19 @@ impl Pallet { // If the preimage already exists before the request is made, the deposit for the preimage is // returned to the user, and removed from their management. fn do_request_preimage(hash: &T::Hash) { - let (count, len, deposit) = - StatusFor::::get(hash).map_or((1, None, None), |x| match x { - RequestStatus::Requested { mut count, len, deposit } => { + Self::do_ensure_updated(&hash); + let (count, maybe_len, maybe_ticket) = + RequestStatusFor::::get(hash).map_or((1, None, None), |x| match x { + RequestStatus::Requested { maybe_ticket, mut count, maybe_len } => { count.saturating_inc(); - (count, len, deposit) + (count, maybe_len, maybe_ticket) }, - RequestStatus::Unrequested { deposit, len } => (1, Some(len), Some(deposit)), + RequestStatus::Unrequested { ticket, len } => (1, Some(len), Some(ticket)), }); - StatusFor::::insert(hash, RequestStatus::Requested { count, len, deposit }); + RequestStatusFor::::insert( + hash, + RequestStatus::Requested { maybe_ticket, count, maybe_len }, + ); if count == 1 { Self::deposit_event(Event::Requested { hash: *hash }); } @@ -288,24 +395,25 @@ impl Pallet { hash: &T::Hash, maybe_check_owner: Option, ) -> DispatchResult { - match StatusFor::::get(hash).ok_or(Error::::NotNoted)? { - RequestStatus::Requested { deposit: Some((owner, deposit)), count, len } => { + Self::do_ensure_updated(&hash); + match RequestStatusFor::::get(hash).ok_or(Error::::NotNoted)? { + RequestStatus::Requested { maybe_ticket: Some((owner, ticket)), count, maybe_len } => { ensure!(maybe_check_owner.map_or(true, |c| c == owner), Error::::NotAuthorized); - T::Currency::unreserve(&owner, deposit); - StatusFor::::insert( + let _ = ticket.drop(&owner); + RequestStatusFor::::insert( hash, - RequestStatus::Requested { deposit: None, count, len }, + RequestStatus::Requested { maybe_ticket: None, count, maybe_len }, ); Ok(()) }, - RequestStatus::Requested { deposit: None, .. } => { + RequestStatus::Requested { maybe_ticket: None, .. } => { ensure!(maybe_check_owner.is_none(), Error::::NotAuthorized); Self::do_unrequest_preimage(hash) }, - RequestStatus::Unrequested { deposit: (owner, deposit), len } => { + RequestStatus::Unrequested { ticket: (owner, ticket), len } => { ensure!(maybe_check_owner.map_or(true, |c| c == owner), Error::::NotAuthorized); - T::Currency::unreserve(&owner, deposit); - StatusFor::::remove(hash); + let _ = ticket.drop(&owner); + RequestStatusFor::::remove(hash); Self::remove(hash, len); Self::deposit_event(Event::Cleared { hash: *hash }); @@ -316,25 +424,32 @@ impl Pallet { /// Clear a preimage request. fn do_unrequest_preimage(hash: &T::Hash) -> DispatchResult { - match StatusFor::::get(hash).ok_or(Error::::NotRequested)? { - RequestStatus::Requested { mut count, len, deposit } if count > 1 => { + Self::do_ensure_updated(&hash); + match RequestStatusFor::::get(hash).ok_or(Error::::NotRequested)? { + RequestStatus::Requested { mut count, maybe_len, maybe_ticket } if count > 1 => { count.saturating_dec(); - StatusFor::::insert(hash, RequestStatus::Requested { count, len, deposit }); + RequestStatusFor::::insert( + hash, + RequestStatus::Requested { maybe_ticket, count, maybe_len }, + ); }, - RequestStatus::Requested { count, len, deposit } => { + RequestStatus::Requested { count, maybe_len, maybe_ticket } => { debug_assert!(count == 1, "preimage request counter at zero?"); - match (len, deposit) { + match (maybe_len, maybe_ticket) { // Preimage was never noted. - (None, _) => StatusFor::::remove(hash), + (None, _) => RequestStatusFor::::remove(hash), // Preimage was noted without owner - just remove it. (Some(len), None) => { Self::remove(hash, len); - StatusFor::::remove(hash); + RequestStatusFor::::remove(hash); Self::deposit_event(Event::Cleared { hash: *hash }); }, // Preimage was noted with owner - move to unrequested so they can get refund. - (Some(len), Some(deposit)) => { - StatusFor::::insert(hash, RequestStatus::Unrequested { deposit, len }); + (Some(len), Some(ticket)) => { + RequestStatusFor::::insert( + hash, + RequestStatus::Unrequested { ticket, len }, + ); }, } }, @@ -359,8 +474,10 @@ impl Pallet { fn len(hash: &T::Hash) -> Option { use RequestStatus::*; - match StatusFor::::get(hash) { - Some(Requested { len: Some(len), .. }) | Some(Unrequested { len, .. }) => Some(len), + Self::do_ensure_updated(&hash); + match RequestStatusFor::::get(hash) { + Some(Requested { maybe_len: Some(len), .. }) | Some(Unrequested { len, .. }) => + Some(len), _ => None, } } @@ -380,7 +497,8 @@ impl PreimageProvider for Pallet { } fn preimage_requested(hash: &T::Hash) -> bool { - matches!(StatusFor::::get(hash), Some(RequestStatus::Requested { .. })) + Self::do_ensure_updated(hash); + matches!(RequestStatusFor::::get(hash), Some(RequestStatus::Requested { .. })) } fn get_preimage(hash: &T::Hash) -> Option> { @@ -423,7 +541,8 @@ impl> QueryPreimage for Pallet { } fn is_requested(hash: &T::Hash) -> bool { - matches!(StatusFor::::get(hash), Some(RequestStatus::Requested { .. })) + Self::do_ensure_updated(&hash); + matches!(RequestStatusFor::::get(hash), Some(RequestStatus::Requested { .. })) } fn request(hash: &T::Hash) { diff --git a/substrate/frame/preimage/src/migration.rs b/substrate/frame/preimage/src/migration.rs index 0f3337e39bf50ff742ef8a1d658e86f61939b856..821cb01bbaae5784352bec53eecc89ea65dfd3d6 100644 --- a/substrate/frame/preimage/src/migration.rs +++ b/substrate/frame/preimage/src/migration.rs @@ -37,7 +37,7 @@ mod v0 { use super::*; #[derive(Clone, Eq, PartialEq, Encode, Decode, TypeInfo, MaxEncodedLen, RuntimeDebug)] - pub enum RequestStatus { + pub enum OldRequestStatus { Unrequested(Option<(AccountId, Balance)>), Requested(u32), } @@ -55,7 +55,7 @@ mod v0 { Pallet, Identity, ::Hash, - RequestStatus<::AccountId, BalanceOf>, + OldRequestStatus<::AccountId, BalanceOf>, >; /// Returns the number of images or `None` if the storage is corrupted. @@ -127,21 +127,22 @@ pub mod v1 { } let status = match status { - v0::RequestStatus::Unrequested(deposit) => match deposit { - Some(deposit) => RequestStatus::Unrequested { deposit, len }, + v0::OldRequestStatus::Unrequested(deposit) => match deposit { + Some(deposit) => OldRequestStatus::Unrequested { deposit, len }, // `None` depositor becomes system-requested. None => - RequestStatus::Requested { deposit: None, count: 1, len: Some(len) }, + OldRequestStatus::Requested { deposit: None, count: 1, len: Some(len) }, }, - v0::RequestStatus::Requested(count) if count == 0 => { + v0::OldRequestStatus::Requested(count) if count == 0 => { log::error!(target: TARGET, "preimage has counter of zero: {:?}", hash); continue }, - v0::RequestStatus::Requested(count) => - RequestStatus::Requested { deposit: None, count, len: Some(len) }, + v0::OldRequestStatus::Requested(count) => + OldRequestStatus::Requested { deposit: None, count, len: Some(len) }, }; log::trace!(target: TARGET, "Moving preimage {:?} with len {}", hash, len); + #[allow(deprecated)] crate::StatusFor::::insert(hash, status); crate::PreimageFor::::insert(&(hash, len), preimage); @@ -176,6 +177,7 @@ pub mod v1 { pub fn image_count() -> Option { // Use iter_values() to ensure that the values are decodable. let images = crate::PreimageFor::::iter_values().count() as u32; + #[allow(deprecated)] let status = crate::StatusFor::::iter_values().count() as u32; if images == status { @@ -189,6 +191,7 @@ pub mod v1 { #[cfg(test)] #[cfg(feature = "try-runtime")] mod test { + #![allow(deprecated)] use super::*; use crate::mock::{Test as T, *}; @@ -203,19 +206,19 @@ mod test { // Case 1: Unrequested without deposit let (p, h) = preimage::(128); v0::PreimageFor::::insert(h, p); - v0::StatusFor::::insert(h, v0::RequestStatus::Unrequested(None)); + v0::StatusFor::::insert(h, v0::OldRequestStatus::Unrequested(None)); // Case 2: Unrequested with deposit let (p, h) = preimage::(1024); v0::PreimageFor::::insert(h, p); - v0::StatusFor::::insert(h, v0::RequestStatus::Unrequested(Some((1, 1)))); + v0::StatusFor::::insert(h, v0::OldRequestStatus::Unrequested(Some((1, 1)))); // Case 3: Requested by 0 (invalid) let (p, h) = preimage::(8192); v0::PreimageFor::::insert(h, p); - v0::StatusFor::::insert(h, v0::RequestStatus::Requested(0)); + v0::StatusFor::::insert(h, v0::OldRequestStatus::Requested(0)); // Case 4: Requested by 10 let (p, h) = preimage::(65536); v0::PreimageFor::::insert(h, p); - v0::StatusFor::::insert(h, v0::RequestStatus::Requested(10)); + v0::StatusFor::::insert(h, v0::OldRequestStatus::Requested(10)); assert_eq!(v0::image_count::(), Some(4)); assert_eq!(v1::image_count::(), None, "V1 storage should be corrupted"); @@ -234,14 +237,14 @@ mod test { assert_eq!(crate::PreimageFor::::get(&(h, 128)), Some(p)); assert_eq!( crate::StatusFor::::get(h), - Some(RequestStatus::Requested { deposit: None, count: 1, len: Some(128) }) + Some(OldRequestStatus::Requested { deposit: None, count: 1, len: Some(128) }) ); // Case 2: Unrequested with deposit becomes unrequested let (p, h) = preimage::(1024); assert_eq!(crate::PreimageFor::::get(&(h, 1024)), Some(p)); assert_eq!( crate::StatusFor::::get(h), - Some(RequestStatus::Unrequested { deposit: (1, 1), len: 1024 }) + Some(OldRequestStatus::Unrequested { deposit: (1, 1), len: 1024 }) ); // Case 3: Requested by 0 should be skipped let (_, h) = preimage::(8192); @@ -252,7 +255,7 @@ mod test { assert_eq!(crate::PreimageFor::::get(&(h, 65536)), Some(p)); assert_eq!( crate::StatusFor::::get(h), - Some(RequestStatus::Requested { deposit: None, count: 10, len: Some(65536) }) + Some(OldRequestStatus::Requested { deposit: None, count: 10, len: Some(65536) }) ); }); } diff --git a/substrate/frame/preimage/src/mock.rs b/substrate/frame/preimage/src/mock.rs index 2fb9f36dec454c85b2e1cfca1b853ec7ac507b20..2e7051a99a497008c8339251975ff66508ff15f3 100644 --- a/substrate/frame/preimage/src/mock.rs +++ b/substrate/frame/preimage/src/mock.rs @@ -22,13 +22,13 @@ use super::*; use crate as pallet_preimage; use frame_support::{ ord_parameter_types, - traits::{ConstU32, ConstU64, Everything}, + traits::{fungible::HoldConsideration, ConstU32, ConstU64, Everything}, weights::constants::RocksDbWeight, }; use frame_system::EnsureSignedBy; use sp_core::H256; use sp_runtime::{ - traits::{BlakeTwo256, IdentityLookup}, + traits::{BlakeTwo256, Convert, IdentityLookup}, BuildStorage, }; @@ -80,22 +80,28 @@ impl pallet_balances::Config for Test { type MaxReserves = ConstU32<50>; type ReserveIdentifier = [u8; 8]; type FreezeIdentifier = (); - type MaxFreezes = (); + type MaxFreezes = ConstU32<1>; type RuntimeHoldReason = (); - type MaxHolds = (); + type MaxHolds = ConstU32<2>; } ord_parameter_types! { pub const One: u64 = 1; } +pub struct ConvertDeposit; +impl Convert for ConvertDeposit { + fn convert(a: Footprint) -> u64 { + a.count * 2 + a.size + } +} + impl Config for Test { type WeightInfo = (); type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureSignedBy; - type BaseDeposit = ConstU64<2>; - type ByteDeposit = ConstU64<1>; + type Consideration = HoldConsideration; } pub fn new_test_ext() -> sp_io::TestExternalities { @@ -110,3 +116,20 @@ pub fn new_test_ext() -> sp_io::TestExternalities { pub fn hashed(data: impl AsRef<[u8]>) -> H256 { BlakeTwo256::hash(data.as_ref()) } + +/// Insert an un-migrated preimage. +pub fn insert_old_unrequested( + s: u32, + acc: T::AccountId, +) -> ::Hash { + // The preimage size does not matter here as it is not touched. + let preimage = s.to_le_bytes(); + let hash = ::Hashing::hash(&preimage[..]); + + #[allow(deprecated)] + StatusFor::::insert( + &hash, + OldRequestStatus::Unrequested { deposit: (acc, 123u32.into()), len: preimage.len() as u32 }, + ); + hash +} diff --git a/substrate/frame/preimage/src/tests.rs b/substrate/frame/preimage/src/tests.rs index fa621c8f5589e693a10d4c6694af177953375df9..a473a0ae8e25f51aeb5d825eae497bba68cfc4b2 100644 --- a/substrate/frame/preimage/src/tests.rs +++ b/substrate/frame/preimage/src/tests.rs @@ -24,12 +24,11 @@ use crate::mock::*; use frame_support::{ assert_err, assert_noop, assert_ok, assert_storage_noop, - traits::{Bounded, BoundedInline, Hash as PreimageHash}, + traits::{fungible::InspectHold, Bounded, BoundedInline, Hash as PreimageHash}, StorageNoopGuard, }; -use pallet_balances::Error as BalancesError; use sp_core::{blake2_256, H256}; -use sp_runtime::bounded_vec; +use sp_runtime::{bounded_vec, TokenError}; /// Returns one `Inline`, `Lookup` and `Legacy` item each with different data and hash. pub fn make_bounded_values() -> (Bounded>, Bounded>, Bounded>) { @@ -52,7 +51,7 @@ pub fn make_bounded_values() -> (Bounded>, Bounded>, Bounded::AlreadyNoted + Error::::AlreadyNoted, ); assert_noop!( Preimage::note_preimage(RuntimeOrigin::signed(0), vec![2]), - BalancesError::::InsufficientBalance + TokenError::FundsUnavailable, ); }); } @@ -171,7 +170,7 @@ fn request_note_order_makes_no_difference() { assert_ok!(Preimage::request_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::note_preimage(RuntimeOrigin::signed(2), vec![1])); ( - StatusFor::::iter().collect::>(), + RequestStatusFor::::iter().collect::>(), PreimageFor::::iter().collect::>(), ) }); @@ -180,7 +179,7 @@ fn request_note_order_makes_no_difference() { assert_ok!(Preimage::request_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::unnote_preimage(RuntimeOrigin::signed(2), hashed([1]))); let other_way = ( - StatusFor::::iter().collect::>(), + RequestStatusFor::::iter().collect::>(), PreimageFor::::iter().collect::>(), ); assert_eq!(one_way, other_way); @@ -207,7 +206,7 @@ fn request_user_note_order_makes_no_difference() { assert_ok!(Preimage::request_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::note_preimage(RuntimeOrigin::signed(2), vec![1])); ( - StatusFor::::iter().collect::>(), + RequestStatusFor::::iter().collect::>(), PreimageFor::::iter().collect::>(), ) }); @@ -216,7 +215,7 @@ fn request_user_note_order_makes_no_difference() { assert_ok!(Preimage::request_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::unnote_preimage(RuntimeOrigin::signed(2), hashed([1]))); let other_way = ( - StatusFor::::iter().collect::>(), + RequestStatusFor::::iter().collect::>(), PreimageFor::::iter().collect::>(), ); assert_eq!(one_way, other_way); @@ -249,13 +248,14 @@ fn unrequest_preimage_works() { fn user_noted_then_requested_preimage_is_refunded_once_only() { new_test_ext().execute_with(|| { assert_ok!(Preimage::note_preimage(RuntimeOrigin::signed(2), vec![1; 3])); + assert_eq!(Balances::balance_on_hold(&(), &2), 5); assert_ok!(Preimage::note_preimage(RuntimeOrigin::signed(2), vec![1])); + assert_eq!(Balances::balance_on_hold(&(), &2), 8); assert_ok!(Preimage::request_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::unrequest_preimage(RuntimeOrigin::signed(1), hashed([1]))); assert_ok!(Preimage::unnote_preimage(RuntimeOrigin::signed(2), hashed([1]))); - // Still have reserve from `vec[1; 3]`. - assert_eq!(Balances::reserved_balance(2), 5); - assert_eq!(Balances::free_balance(2), 95); + // Still have hold from `vec[1; 3]`. + assert_eq!(Balances::balance_on_hold(&(), &2), 5); }); } @@ -270,7 +270,7 @@ fn noted_preimage_use_correct_map() { assert_eq!(PreimageFor::::iter().count(), 8); // All are present - assert_eq!(StatusFor::::iter().count(), 8); + assert_eq!(RequestStatusFor::::iter().count(), 8); // Now start removing them again... for i in 0..7 { @@ -287,7 +287,7 @@ fn noted_preimage_use_correct_map() { assert_eq!(PreimageFor::::iter().count(), 0); // All are gone - assert_eq!(StatusFor::::iter().count(), 0); + assert_eq!(RequestStatusFor::::iter().count(), 0); }); } @@ -327,20 +327,20 @@ fn query_and_store_preimage_workflow() { Preimage::request(&hash); // It is requested thrice. assert!(matches!( - StatusFor::::get(&hash).unwrap(), + RequestStatusFor::::get(&hash).unwrap(), RequestStatus::Requested { count: 3, .. } )); // It can be realized and decoded correctly. assert_eq!(Preimage::realize::>(&bound).unwrap(), (data.clone(), Some(len))); assert!(matches!( - StatusFor::::get(&hash).unwrap(), + RequestStatusFor::::get(&hash).unwrap(), RequestStatus::Requested { count: 2, .. } )); // Dropping should unrequest. Preimage::drop(&bound); assert!(matches!( - StatusFor::::get(&hash).unwrap(), + RequestStatusFor::::get(&hash).unwrap(), RequestStatus::Requested { count: 1, .. } )); @@ -381,7 +381,7 @@ fn query_preimage_request_works() { assert!(::len(&hash).is_none()); assert_noop!(::fetch(&hash, None), DispatchError::Unavailable); // But there is only one entry in the map. - assert_eq!(StatusFor::::iter().count(), 1); + assert_eq!(RequestStatusFor::::iter().count(), 1); // Un-request the preimage. ::unrequest(&hash); @@ -392,7 +392,7 @@ fn query_preimage_request_works() { // It is not requested anymore. assert!(!::is_requested(&hash)); // And there is no entry in the map. - assert_eq!(StatusFor::::iter().count(), 0); + assert_eq!(RequestStatusFor::::iter().count(), 0); }); } @@ -413,7 +413,7 @@ fn query_preimage_hold_and_drop_work() { assert!(::is_requested(&legacy.hash())); // There are two values requested in total. - assert_eq!(StatusFor::::iter().count(), 2); + assert_eq!(RequestStatusFor::::iter().count(), 2); // Cleanup by dropping both. ::drop(&lookup); @@ -422,7 +422,7 @@ fn query_preimage_hold_and_drop_work() { assert!(!::is_requested(&legacy.hash())); // There are no values requested anymore. - assert_eq!(StatusFor::::iter().count(), 0); + assert_eq!(RequestStatusFor::::iter().count(), 0); }); } @@ -489,3 +489,27 @@ fn store_preimage_bound_too_large_errors() { assert_ok!(::bound(data.clone())); }); } + +#[test] +fn ensure_updated_works() { + #![allow(deprecated)] + new_test_ext().execute_with(|| { + let alice = 2; + + for i in 0..100 { + let hashes = + (0..100).map(|j| insert_old_unrequested::(j, alice)).collect::>(); + let old = hashes.iter().take(i).cloned().collect::>(); + let bad = vec![hashed([0; 32]); 100 - i]; + + let hashes = [old.as_slice(), bad.as_slice()].concat(); + let res = Preimage::ensure_updated(RuntimeOrigin::signed(alice), hashes).unwrap(); + + // Alice pays a fee when less than 90% of the hashes are new. + assert_eq!(res.pays_fee, (i < 90).into()); + + assert_eq!(RequestStatusFor::::iter().count(), i); + assert_eq!(StatusFor::::iter().count(), 100 - i); + } + }); +} diff --git a/substrate/frame/preimage/src/weights.rs b/substrate/frame/preimage/src/weights.rs index 41e58a1027800b15672da71f7c17b624471c3ebd..c11ab74c1e551a0fcdd6e239a657ab5df0809aad 100644 --- a/substrate/frame/preimage/src/weights.rs +++ b/substrate/frame/preimage/src/weights.rs @@ -15,32 +15,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Autogenerated weights for pallet_preimage +//! Autogenerated weights for `pallet_preimage` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-06-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-09-06, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` -//! HOSTNAME: `runner-e8ezs4ez-project-145-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` -//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 +//! HOSTNAME: `runner-mia4uyug-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` +//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` // Executed Command: -// ./target/production/substrate +// target/production/substrate-node // benchmark // pallet -// --chain=dev // --steps=50 // --repeat=20 -// --pallet=pallet_preimage -// --no-storage-info -// --no-median-slopes -// --no-min-squares // --extrinsic=* -// --execution=wasm // --wasm-execution=compiled // --heap-pages=4096 -// --output=./frame/preimage/src/weights.rs -// --header=./HEADER-APACHE2 -// --template=./.maintain/frame-weight-template.hbs +// --json-file=/builds/parity/mirrors/polkadot-sdk/.git/.artifacts/bench.json +// --pallet=pallet_preimage +// --chain=dev +// --header=./substrate/HEADER-APACHE2 +// --output=./substrate/frame/preimage/src/weights.rs +// --template=./substrate/.maintain/frame-weight-template.hbs #![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unused_parens)] @@ -50,7 +47,7 @@ use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; use core::marker::PhantomData; -/// Weight functions needed for pallet_preimage. +/// Weight functions needed for `pallet_preimage`. pub trait WeightInfo { fn note_preimage(s: u32, ) -> Weight; fn note_requested_preimage(s: u32, ) -> Weight; @@ -64,319 +61,410 @@ pub trait WeightInfo { fn unrequest_preimage() -> Weight; fn unrequest_unnoted_preimage() -> Weight; fn unrequest_multi_referenced_preimage() -> Weight; + fn ensure_updated(n: u32, ) -> Weight; } -/// Weights for pallet_preimage using the Substrate node and recommended hardware. +/// Weights for `pallet_preimage` using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `143` + // Measured: `42` // Estimated: `3556` - // Minimum execution time: 30_479_000 picoseconds. - Weight::from_parts(23_381_775, 3556) - // Standard Error: 2 - .saturating_add(Weight::from_parts(1_670, 0).saturating_mul(s.into())) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 15_936_000 picoseconds. + Weight::from_parts(16_271_000, 3556) + // Standard Error: 1 + .saturating_add(Weight::from_parts(1_916, 0).saturating_mul(s.into())) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_requested_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 16_104_000 picoseconds. - Weight::from_parts(18_393_879, 3556) + // Minimum execution time: 16_468_000 picoseconds. + Weight::from_parts(17_031_000, 3556) // Standard Error: 2 - .saturating_add(Weight::from_parts(1_669, 0).saturating_mul(s.into())) - .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(Weight::from_parts(1_948, 0).saturating_mul(s.into())) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_no_deposit_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 15_652_000 picoseconds. - Weight::from_parts(22_031_627, 3556) - // Standard Error: 2 - .saturating_add(Weight::from_parts(1_672, 0).saturating_mul(s.into())) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 16_342_000 picoseconds. + Weight::from_parts(16_535_000, 3556) + // Standard Error: 1 + .saturating_add(Weight::from_parts(1_906, 0).saturating_mul(s.into())) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unnote_preimage() -> Weight { // Proof Size summary in bytes: - // Measured: `289` + // Measured: `172` // Estimated: `3556` - // Minimum execution time: 37_148_000 picoseconds. - Weight::from_parts(40_247_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 31_047_000 picoseconds. + Weight::from_parts(34_099_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unnote_no_deposit_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 19_909_000 picoseconds. - Weight::from_parts(21_572_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 32_559_000 picoseconds. + Weight::from_parts(36_677_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_preimage() -> Weight { // Proof Size summary in bytes: - // Measured: `188` + // Measured: `172` // Estimated: `3556` - // Minimum execution time: 17_602_000 picoseconds. - Weight::from_parts(18_899_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 27_887_000 picoseconds. + Weight::from_parts(30_303_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_no_deposit_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 11_253_000 picoseconds. - Weight::from_parts(11_667_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 17_256_000 picoseconds. + Weight::from_parts(19_481_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_unnoted_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `42` // Estimated: `3556` - // Minimum execution time: 14_152_000 picoseconds. - Weight::from_parts(14_652_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 22_344_000 picoseconds. + Weight::from_parts(23_868_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_requested_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 8_267_000 picoseconds. - Weight::from_parts(8_969_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 10_542_000 picoseconds. + Weight::from_parts(11_571_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unrequest_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 18_429_000 picoseconds. - Weight::from_parts(18_946_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 29_054_000 picoseconds. + Weight::from_parts(32_996_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn unrequest_unnoted_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 7_910_000 picoseconds. - Weight::from_parts(8_272_000, 3556) - .saturating_add(T::DbWeight::get().reads(1_u64)) + // Minimum execution time: 10_775_000 picoseconds. + Weight::from_parts(11_937_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn unrequest_multi_referenced_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 7_936_000 picoseconds. - Weight::from_parts(8_504_000, 3556) + // Minimum execution time: 10_696_000 picoseconds. + Weight::from_parts(11_717_000, 3556) + .saturating_add(T::DbWeight::get().reads(2_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: `Preimage::StatusFor` (r:1024 w:1024) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:0 w:1024) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 1024]`. + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_452_000 picoseconds. + Weight::from_parts(2_641_000, 3593) + // Standard Error: 19_797 + .saturating_add(Weight::from_parts(15_620_946, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes(1_u64)) + .saturating_add(T::DbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) } } -// For backwards compatibility and tests +// For backwards compatibility and tests. impl WeightInfo for () { - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: - // Measured: `143` + // Measured: `42` // Estimated: `3556` - // Minimum execution time: 30_479_000 picoseconds. - Weight::from_parts(23_381_775, 3556) - // Standard Error: 2 - .saturating_add(Weight::from_parts(1_670, 0).saturating_mul(s.into())) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 15_936_000 picoseconds. + Weight::from_parts(16_271_000, 3556) + // Standard Error: 1 + .saturating_add(Weight::from_parts(1_916, 0).saturating_mul(s.into())) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_requested_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 16_104_000 picoseconds. - Weight::from_parts(18_393_879, 3556) + // Minimum execution time: 16_468_000 picoseconds. + Weight::from_parts(17_031_000, 3556) // Standard Error: 2 - .saturating_add(Weight::from_parts(1_669, 0).saturating_mul(s.into())) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(Weight::from_parts(1_948, 0).saturating_mul(s.into())) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) /// The range of component `s` is `[0, 4194304]`. fn note_no_deposit_preimage(s: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 15_652_000 picoseconds. - Weight::from_parts(22_031_627, 3556) - // Standard Error: 2 - .saturating_add(Weight::from_parts(1_672, 0).saturating_mul(s.into())) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 16_342_000 picoseconds. + Weight::from_parts(16_535_000, 3556) + // Standard Error: 1 + .saturating_add(Weight::from_parts(1_906, 0).saturating_mul(s.into())) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unnote_preimage() -> Weight { // Proof Size summary in bytes: - // Measured: `289` + // Measured: `172` // Estimated: `3556` - // Minimum execution time: 37_148_000 picoseconds. - Weight::from_parts(40_247_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 31_047_000 picoseconds. + Weight::from_parts(34_099_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unnote_no_deposit_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 19_909_000 picoseconds. - Weight::from_parts(21_572_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 32_559_000 picoseconds. + Weight::from_parts(36_677_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_preimage() -> Weight { // Proof Size summary in bytes: - // Measured: `188` + // Measured: `172` // Estimated: `3556` - // Minimum execution time: 17_602_000 picoseconds. - Weight::from_parts(18_899_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 27_887_000 picoseconds. + Weight::from_parts(30_303_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_no_deposit_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 11_253_000 picoseconds. - Weight::from_parts(11_667_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 17_256_000 picoseconds. + Weight::from_parts(19_481_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_unnoted_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `42` // Estimated: `3556` - // Minimum execution time: 14_152_000 picoseconds. - Weight::from_parts(14_652_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 22_344_000 picoseconds. + Weight::from_parts(23_868_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn request_requested_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 8_267_000 picoseconds. - Weight::from_parts(8_969_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 10_542_000 picoseconds. + Weight::from_parts(11_571_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) - /// Storage: Preimage PreimageFor (r:0 w:1) - /// Proof: Preimage PreimageFor (max_values: None, max_size: Some(4194344), added: 4196819, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// Storage: `Preimage::PreimageFor` (r:0 w:1) + /// Proof: `Preimage::PreimageFor` (`max_values`: None, `max_size`: Some(4194344), added: 4196819, mode: `MaxEncodedLen`) fn unrequest_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `144` // Estimated: `3556` - // Minimum execution time: 18_429_000 picoseconds. - Weight::from_parts(18_946_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 29_054_000 picoseconds. + Weight::from_parts(32_996_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn unrequest_unnoted_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 7_910_000 picoseconds. - Weight::from_parts(8_272_000, 3556) - .saturating_add(RocksDbWeight::get().reads(1_u64)) + // Minimum execution time: 10_775_000 picoseconds. + Weight::from_parts(11_937_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: Preimage StatusFor (r:1 w:1) - /// Proof: Preimage StatusFor (max_values: None, max_size: Some(91), added: 2566, mode: MaxEncodedLen) + /// Storage: `Preimage::StatusFor` (r:1 w:0) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:1 w:1) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) fn unrequest_multi_referenced_preimage() -> Weight { // Proof Size summary in bytes: // Measured: `106` // Estimated: `3556` - // Minimum execution time: 7_936_000 picoseconds. - Weight::from_parts(8_504_000, 3556) + // Minimum execution time: 10_696_000 picoseconds. + Weight::from_parts(11_717_000, 3556) + .saturating_add(RocksDbWeight::get().reads(2_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: `Preimage::StatusFor` (r:1024 w:1024) + /// Proof: `Preimage::StatusFor` (`max_values`: None, `max_size`: Some(91), added: 2566, mode: `MaxEncodedLen`) + /// Storage: `System::Account` (r:1 w:1) + /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) + /// Storage: `Preimage::RequestStatusFor` (r:0 w:1024) + /// Proof: `Preimage::RequestStatusFor` (`max_values`: None, `max_size`: Some(75), added: 2550, mode: `MaxEncodedLen`) + /// The range of component `n` is `[0, 1024]`. + fn ensure_updated(n: u32, ) -> Weight { + // Proof Size summary in bytes: + // Measured: `193 + n * (91 ±0)` + // Estimated: `3593 + n * (2566 ±0)` + // Minimum execution time: 2_452_000 picoseconds. + Weight::from_parts(2_641_000, 3593) + // Standard Error: 19_797 + .saturating_add(Weight::from_parts(15_620_946, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(RocksDbWeight::get().writes(1_u64)) + .saturating_add(RocksDbWeight::get().writes((2_u64).saturating_mul(n.into()))) + .saturating_add(Weight::from_parts(0, 2566).saturating_mul(n.into())) } } diff --git a/substrate/frame/ranked-collective/README.md b/substrate/frame/ranked-collective/README.md index b5fe65ef34920f3eecd6f595a0c3144ca9450d2a..e9624159d2fdabbe5c83b34cd6d0fe14772b98af 100644 --- a/substrate/frame/ranked-collective/README.md +++ b/substrate/frame/ranked-collective/README.md @@ -1,4 +1,4 @@ -# Ranked collective system. +# Ranked collective system This is a membership pallet providing a `Tally` implementation ready for use with polling systems such as the Referenda pallet. Members each have a rank, with zero being the lowest. diff --git a/substrate/frame/recovery/README.md b/substrate/frame/recovery/README.md index 31416c65c46a53e9aca2223321c1ee8a27c4eda8..7e2dd7a23619ac893061d0730f7813bdc7e33e98 100644 --- a/substrate/frame/recovery/README.md +++ b/substrate/frame/recovery/README.md @@ -16,11 +16,11 @@ friends are needed to give another account access to the recoverable account. The recovery process for each recoverable account can be configured by the account owner. They are able to choose: -* `friends` - The list of friends that the account owner trusts to protect the +- `friends` - The list of friends that the account owner trusts to protect the recovery process for their account. -* `threshold` - The number of friends that need to approve a recovery process for +- `threshold` - The number of friends that need to approve a recovery process for the account to be successfully recovered. -* `delay_period` - The minimum number of blocks after the beginning of the recovery +- `delay_period` - The minimum number of blocks after the beginning of the recovery process that need to pass before the account can be successfully recovered. There is a configurable deposit that all users need to pay to create a recovery @@ -84,20 +84,20 @@ It is important to note that this is a powerful pallet that can compromise the security of an account if used incorrectly. Some recommended practices for users of this pallet are: -* Configure a significant `delay_period` for your recovery process: As long as you +- Configure a significant `delay_period` for your recovery process: As long as you have access to your recoverable account, you need only check the blockchain once every `delay_period` blocks to ensure that no recovery attempt is successful against your account. Using off-chain notification systems can help with this, but ultimately, setting a large `delay_period` means that even the most skilled attacker will need to wait this long before they can access your account. -* Use a high threshold of approvals: Setting a value of 1 for the threshold means +- Use a high threshold of approvals: Setting a value of 1 for the threshold means that any of your friends would be able to recover your account. They would simply need to start a recovery process and approve their own process. Similarly, a threshold of 2 would mean that any 2 friends could work together to gain access to your account. The only way to prevent against these kinds of attacks is to choose a high threshold of approvals and select from a diverse friend group that would not be able to reasonably coordinate with one another. -* Reset your configuration over time: Since the entire deposit of creating a +- Reset your configuration over time: Since the entire deposit of creating a recovery configuration is returned to the user, the only cost of updating your recovery configuration is the transaction fees for the calls. Thus, it is strongly encouraged to regularly update your recovery configuration @@ -110,25 +110,25 @@ of this pallet are: #### For General Users -* `create_recovery` - Create a recovery configuration for your account and make it recoverable. -* `initiate_recovery` - Start the recovery process for a recoverable account. +- `create_recovery` - Create a recovery configuration for your account and make it recoverable. +- `initiate_recovery` - Start the recovery process for a recoverable account. #### For Friends of a Recoverable Account -* `vouch_recovery` - As a `friend` of a recoverable account, vouch for a recovery attempt on the account. +- `vouch_recovery` - As a `friend` of a recoverable account, vouch for a recovery attempt on the account. #### For a User Who Successfully Recovered an Account -* `claim_recovery` - Claim access to the account that you have successfully completed the recovery process for. -* `as_recovered` - Send a transaction as an account that you have recovered. See other functions below. +- `claim_recovery` - Claim access to the account that you have successfully completed the recovery process for. +- `as_recovered` - Send a transaction as an account that you have recovered. See other functions below. #### For the Recoverable Account -* `close_recovery` - Close an active recovery process for your account and reclaim the recovery deposit. -* `remove_recovery` - Remove the recovery configuration from the account, making it un-recoverable. +- `close_recovery` - Close an active recovery process for your account and reclaim the recovery deposit. +- `remove_recovery` - Remove the recovery configuration from the account, making it un-recoverable. #### For Super Users -* `set_recovered` - The ROOT origin is able to skip the recovery process and directly allow +- `set_recovered` - The ROOT origin is able to skip the recovery process and directly allow one account to access another. License: Apache-2.0 diff --git a/substrate/frame/referenda/src/mock.rs b/substrate/frame/referenda/src/mock.rs index e44167ed561c591236554bc86b7be50d0709d88b..c23fa4609d4340dc92e712a439ab07b8b4ab7605 100644 --- a/substrate/frame/referenda/src/mock.rs +++ b/substrate/frame/referenda/src/mock.rs @@ -89,8 +89,7 @@ impl pallet_preimage::Config for Test { type WeightInfo = (); type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = (); - type ByteDeposit = (); + type Consideration = (); } impl pallet_scheduler::Config for Test { type RuntimeEvent = RuntimeEvent; diff --git a/substrate/frame/remark/README.md b/substrate/frame/remark/README.md index f2341d6a0eaec45287957feab73a0199ddc17dff..5224f1b2882071eec72243cf3ef0ea706e57e52c 100644 --- a/substrate/frame/remark/README.md +++ b/substrate/frame/remark/README.md @@ -1,6 +1,6 @@ # Remark Storage Pallet -Allows storing arbitrary data off chain. +Allows storing arbitrary data off chain. License: Apache-2.0 diff --git a/substrate/frame/root-offences/README.md b/substrate/frame/root-offences/README.md index c582158721816d6e16ee3d2f1a93d5daa555a1da..b4e8381df2e71e275bae10de1f6effde748cbc13 100644 --- a/substrate/frame/root-offences/README.md +++ b/substrate/frame/root-offences/README.md @@ -2,4 +2,4 @@ Pallet that allows the root to create an offence. -NOTE: This pallet should only be used for testing purposes. \ No newline at end of file +NOTE: This pallet should only be used for testing purposes. diff --git a/substrate/frame/root-testing/README.md b/substrate/frame/root-testing/README.md index 637430445a22f6046f6a20bd6fc8ba58901373bc..aa231e3ef20a06e1afd7d03939d82b4b06853dc9 100644 --- a/substrate/frame/root-testing/README.md +++ b/substrate/frame/root-testing/README.md @@ -2,4 +2,4 @@ Pallet that contains extrinsics that can be usefull in testing. -NOTE: This pallet should only be used for testing purposes and should not be used in production runtimes! \ No newline at end of file +NOTE: This pallet should only be used for testing purposes and should not be used in production runtimes! diff --git a/substrate/frame/safe-mode/src/mock.rs b/substrate/frame/safe-mode/src/mock.rs index 337b6076f84b994bde73691deb55cc80fe141747..635ee0cfedc025ec4796d95dfbdef2cd50275349 100644 --- a/substrate/frame/safe-mode/src/mock.rs +++ b/substrate/frame/safe-mode/src/mock.rs @@ -122,7 +122,10 @@ impl InstanceFilter for ProxyType { match self { ProxyType::Any => true, ProxyType::JustTransfer => { - matches!(c, RuntimeCall::Balances(pallet_balances::Call::transfer { .. })) + matches!( + c, + RuntimeCall::Balances(pallet_balances::Call::transfer_allow_death { .. }) + ) }, ProxyType::JustUtility => matches!(c, RuntimeCall::Utility { .. }), } diff --git a/substrate/frame/safe-mode/src/tests.rs b/substrate/frame/safe-mode/src/tests.rs index 1e2eb343aa2f7fd26a20f89b9b31078d1ce1fb78..ca1d7eb1d93400b0bad97c45ac6aee00e15e2aa9 100644 --- a/substrate/frame/safe-mode/src/tests.rs +++ b/substrate/frame/safe-mode/src/tests.rs @@ -605,7 +605,7 @@ fn fails_when_explicit_origin_required() { } fn call_transfer() -> RuntimeCall { - RuntimeCall::Balances(pallet_balances::Call::transfer { dest: 1, value: 1 }) + RuntimeCall::Balances(pallet_balances::Call::transfer_allow_death { dest: 1, value: 1 }) } fn signed(who: u64) -> RuntimeOrigin { diff --git a/substrate/frame/salary/README.md b/substrate/frame/salary/README.md index 25c1be0e805d777be200a8ff8dae88199ce0935e..ec3882dd09788b3c4b02ad915d1233d0e18ae64a 100644 --- a/substrate/frame/salary/README.md +++ b/substrate/frame/salary/README.md @@ -1,3 +1,3 @@ # Salary -Make periodic payment to members of a ranked collective according to rank. \ No newline at end of file +Make periodic payment to members of a ranked collective according to rank. diff --git a/substrate/frame/scheduler/Cargo.toml b/substrate/frame/scheduler/Cargo.toml index 3ad49ef5e662935c66488aa3826a9502e4899a72..e81b4dbeff78c53e645dd4988b86411193127ac7 100644 --- a/substrate/frame/scheduler/Cargo.toml +++ b/substrate/frame/scheduler/Cargo.toml @@ -20,7 +20,7 @@ sp-io = { path = "../../primitives/io", default-features = false} sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} sp-weights = { path = "../../primitives/weights", default-features = false} -docify = "0.2.1" +docify = "0.2.4" [dev-dependencies] pallet-preimage = { path = "../preimage" } diff --git a/substrate/frame/scheduler/README.md b/substrate/frame/scheduler/README.md index bdd2c2226c88e8318c54f9e61521cc6a0417a47b..6aec2ddb0e4325e8ec4bbe3987ad2ee16a12cd52 100644 --- a/substrate/frame/scheduler/README.md +++ b/substrate/frame/scheduler/README.md @@ -23,12 +23,12 @@ then those filter will not be used when dispatching the schedule call. ### Dispatchable Functions -* `schedule` - schedule a dispatch, which may be periodic, to occur at a +- `schedule` - schedule a dispatch, which may be periodic, to occur at a specified block and with a specified priority. -* `cancel` - cancel a scheduled dispatch, specified by block number and +- `cancel` - cancel a scheduled dispatch, specified by block number and index. -* `schedule_named` - augments the `schedule` interface with an additional +- `schedule_named` - augments the `schedule` interface with an additional `Vec` parameter that can be used for identification. -* `cancel_named` - the named complement to the cancel function. +- `cancel_named` - the named complement to the cancel function. License: Apache 2.0 diff --git a/substrate/frame/scheduler/src/mock.rs b/substrate/frame/scheduler/src/mock.rs index 28e334958d924f927737a033f9e732fba66b4581..b6eb1d044fa2321935b0bbd74ae6e89b1114bf3c 100644 --- a/substrate/frame/scheduler/src/mock.rs +++ b/substrate/frame/scheduler/src/mock.rs @@ -100,7 +100,7 @@ frame_support::construct_runtime!( System: frame_system::{Pallet, Call, Config, Storage, Event}, Logger: logger::{Pallet, Call, Event}, Scheduler: scheduler::{Pallet, Call, Storage, Event}, - Preimage: pallet_preimage::{Pallet, Call, Storage, Event}, + Preimage: pallet_preimage::{Pallet, Call, Storage, Event, HoldReason}, } ); @@ -155,8 +155,7 @@ impl pallet_preimage::Config for Test { type WeightInfo = (); type Currency = (); type ManagerOrigin = EnsureRoot; - type BaseDeposit = (); - type ByteDeposit = (); + type Consideration = (); } pub struct TestWeightInfo; diff --git a/substrate/frame/session/README.md b/substrate/frame/session/README.md index 09132470d4433e71141b8efea9de9c5cc3eddef5..fa7c9b3f98348e4c72fac4fd44b94c492d656c09 100644 --- a/substrate/frame/session/README.md +++ b/substrate/frame/session/README.md @@ -1,7 +1,7 @@ # Session Pallet -The Session module allows validators to manage their session keys, provides a function for changing -the session length, and handles session rotation. +The Session module allows validators to manage their session keys, provides a function for changing the session length, +and handles session rotation. - [`session::Trait`](https://docs.rs/pallet-session/latest/pallet_session/trait.Config.html) - [`Call`](https://docs.rs/pallet-session/latest/pallet_session/enum.Call.html) @@ -12,34 +12,31 @@ the session length, and handles session rotation. ### Terminology -- **Session:** A session is a period of time that has a constant set of validators. Validators can only join -or exit the validator set at a session change. It is measured in block numbers. The block where a session is -ended is determined by the `ShouldEndSession` trait. When the session is ending, a new validator set -can be chosen by `OnSessionEnding` implementations. -- **Session key:** A session key is actually several keys kept together that provide the various signing -functions required by network authorities/validators in pursuit of their duties. -- **Validator ID:** Every account has an associated validator ID. For some simple staking systems, this -may just be the same as the account ID. For staking systems using a stash/controller model, -the validator ID would be the stash account ID of the controller. -- **Session key configuration process:** Session keys are set using `set_keys` for use not in -the next session, but the session after next. They are stored in `NextKeys`, a mapping between -the caller's `ValidatorId` and the session keys provided. `set_keys` allows users to set their -session key prior to being selected as validator. -It is a public call since it uses `ensure_signed`, which checks that the origin is a signed account. -As such, the account ID of the origin stored in `NextKeys` may not necessarily be associated with -a block author or a validator. The session keys of accounts are removed once their account balance is zero. -- **Session length:** This pallet does not assume anything about the length of each session. -Rather, it relies on an implementation of `ShouldEndSession` to dictate a new session's start. -This pallet provides the `PeriodicSessions` struct for simple periodic sessions. -- **Session rotation configuration:** Configure as either a 'normal' (rewardable session where rewards are -applied) or 'exceptional' (slashable) session rotation. -- **Session rotation process:** At the beginning of each block, the `on_initialize` function -queries the provided implementation of `ShouldEndSession`. If the session is to end the newly -activated validator IDs and session keys are taken from storage and passed to the -`SessionHandler`. The validator set supplied by `SessionManager::new_session` and the corresponding session -keys, which may have been registered via `set_keys` during the previous session, are written -to storage where they will wait one session before being passed to the `SessionHandler` -themselves. +- **Session:** A session is a period of time that has a constant set of validators. Validators can only join or exit the +validator set at a session change. It is measured in block numbers. The block where a session is ended is determined by +the `ShouldEndSession` trait. When the session is ending, a new validator set can be chosen by `OnSessionEnding` +implementations. +- **Session key:** A session key is actually several keys kept together that provide the various signing functions +required by network authorities/validators in pursuit of their duties. +- **Validator ID:** Every account has an associated validator ID. For some simple staking systems, this may just be the +same as the account ID. For staking systems using a stash/controller model, the validator ID would be the stash account +ID of the controller. +- **Session key configuration process:** Session keys are set using `set_keys` for use not in the next session, but the +session after next. They are stored in `NextKeys`, a mapping between the caller's `ValidatorId` and the session keys +provided. `set_keys` allows users to set their session key prior to being selected as validator. It is a public call +since it uses `ensure_signed`, which checks that the origin is a signed account. As such, the account ID of the origin +stored in `NextKeys` may not necessarily be associated with a block author or a validator. The session keys of accounts +are removed once their account balance is zero. +- **Session length:** This pallet does not assume anything about the length of each session. Rather, it relies on an +implementation of `ShouldEndSession` to dictate a new session's start. This pallet provides the `PeriodicSessions` +struct for simple periodic sessions. +- **Session rotation configuration:** Configure as either a 'normal' (rewardable session where rewards are applied) or +'exceptional' (slashable) session rotation. +- **Session rotation process:** At the beginning of each block, the `on_initialize` function queries the provided +implementation of `ShouldEndSession`. If the session is to end the newly activated validator IDs and session keys are +taken from storage and passed to the `SessionHandler`. The validator set supplied by `SessionManager::new_session` and +the corresponding session keys, which may have been registered via `set_keys` during the previous session, are written +to storage where they will wait one session before being passed to the `SessionHandler` themselves. ### Goals @@ -57,8 +54,8 @@ The Session pallet is designed to make the following possible: ### Public Functions -- `rotate_session` - Change to the next session. Register the new authority set. Queue changes -for next session rotation. +- `rotate_session` - Change to the next session. Register the new authority set. Queue changes for next session +rotation. - `disable_index` - Disable a validator by index. - `disable` - Disable a validator by Validator ID @@ -66,7 +63,8 @@ for next session rotation. ### Example from the FRAME -The [Staking pallet](https://docs.rs/pallet-staking/latest/pallet_staking/) uses the Session pallet to get the validator set. +The [Staking pallet](https://docs.rs/pallet-staking/latest/pallet_staking/) uses the Session pallet to get the validator +set. ```rust use pallet_session as session; diff --git a/substrate/frame/session/benchmarking/README.md b/substrate/frame/session/benchmarking/README.md index d034a9ec73284fee91c5a2a8c397f5d4bc551a0e..e097f03f34a8e777df3fa149451d1624684a688b 100644 --- a/substrate/frame/session/benchmarking/README.md +++ b/substrate/frame/session/benchmarking/README.md @@ -1,3 +1,3 @@ Benchmarks for the Session Pallet. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/society/Cargo.toml b/substrate/frame/society/Cargo.toml index 1510e17b8b57025f5476742b229fdac0c1e9e02b..d38875836aa254a345ecf577146f5ff2c919c208 100644 --- a/substrate/frame/society/Cargo.toml +++ b/substrate/frame/society/Cargo.toml @@ -34,8 +34,6 @@ sp-io = { path = "../../primitives/io" } [features] default = [ "std" ] -# Enable `VersionedMigration` for migrations using this feature. -experimental = [ "frame-support/experimental" ] std = [ "codec/std", "frame-benchmarking?/std", diff --git a/substrate/frame/society/README.md b/substrate/frame/society/README.md index 80998618664297f05d4dd87405b03cbfad1c4600..c091d6c97d41b8409a0092a92ee938788b9b02b9 100644 --- a/substrate/frame/society/README.md +++ b/substrate/frame/society/README.md @@ -11,16 +11,16 @@ and maintain a membership society. ### User Types At any point, a user in the society can be one of a: -* Bidder - A user who has submitted intention of joining the society. -* Candidate - A user who will be voted on to join the society. -* Suspended Candidate - A user who failed to win a vote. -* Member - A user who is a member of the society. -* Suspended Member - A member of the society who has accumulated too many strikes +- Bidder - A user who has submitted intention of joining the society. +- Candidate - A user who will be voted on to join the society. +- Suspended Candidate - A user who failed to win a vote. +- Member - A user who is a member of the society. +- Suspended Member - A member of the society who has accumulated too many strikes or failed their membership challenge. Of the non-suspended members, there is always a: -* Head - A member who is exempt from suspension. -* Defender - A member whose membership is under question and voted on again. +- Head - A member who is exempt from suspension. +- Defender - A member whose membership is under question and voted on again. Of the non-suspended members of the society, a random set of them are chosen as "skeptics". The mechanics of skeptics is explained in the @@ -201,28 +201,28 @@ future payouts slashed. #### For General Users -* `bid` - A user can make a bid to join the membership society by reserving a deposit. -* `unbid` - A user can withdraw their bid for entry, the deposit is returned. +- `bid` - A user can make a bid to join the membership society by reserving a deposit. +- `unbid` - A user can withdraw their bid for entry, the deposit is returned. #### For Members -* `vouch` - A member can place a bid on behalf of a user to join the membership society. -* `unvouch` - A member can revoke their vouch for a user. -* `vote` - A member can vote to approve or reject a candidate's request to join the society. -* `defender_vote` - A member can vote to approve or reject a defender's continued membership +- `vouch` - A member can place a bid on behalf of a user to join the membership society. +- `unvouch` - A member can revoke their vouch for a user. +- `vote` - A member can vote to approve or reject a candidate's request to join the society. +- `defender_vote` - A member can vote to approve or reject a defender's continued membership to the society. -* `payout` - A member can claim their first matured payment. -* `unfound` - Allow the founder to unfound the society when they are the only member. +- `payout` - A member can claim their first matured payment. +- `unfound` - Allow the founder to unfound the society when they are the only member. #### For Super Users -* `found` - The founder origin can initiate this society. Useful for bootstrapping the Society +- `found` - The founder origin can initiate this society. Useful for bootstrapping the Society pallet on an already running chain. -* `judge_suspended_member` - The suspension judgement origin is able to make +- `judge_suspended_member` - The suspension judgement origin is able to make judgement on a suspended member. -* `judge_suspended_candidate` - The suspension judgement origin is able to +- `judge_suspended_candidate` - The suspension judgement origin is able to make judgement on a suspended candidate. -* `set_max_membership` - The ROOT origin can update the maximum member count for the society. +- `set_max_membership` - The ROOT origin can update the maximum member count for the society. The max membership count must be greater than 1. License: Apache-2.0 diff --git a/substrate/frame/society/src/migrations.rs b/substrate/frame/society/src/migrations.rs index b50b0e088a6e882b6c9a8060c658c5abd08cbf87..553eea1a7952f5f2e88fe43f0e972826abd206fd 100644 --- a/substrate/frame/society/src/migrations.rs +++ b/substrate/frame/society/src/migrations.rs @@ -95,7 +95,6 @@ impl< /// [`VersionUncheckedMigrateToV2`] wrapped in a [`frame_support::migrations::VersionedMigration`], /// ensuring the migration is only performed when on-chain version is 0. -#[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV2 = frame_support::migrations::VersionedMigration< 0, diff --git a/substrate/frame/staking/README.md b/substrate/frame/staking/README.md index ccb9901a6796ed3e97a139806770531ebef9f5ae..387b94b6a681f0e2dacd5c5644ee3795ea8019e7 100644 --- a/substrate/frame/staking/README.md +++ b/substrate/frame/staking/README.md @@ -8,25 +8,24 @@ The Staking module is used to manage funds at stake by network maintainers. ## Overview -The Staking module is the means by which a set of network maintainers (known as _authorities_ in -some contexts and _validators_ in others) are chosen based upon those who voluntarily place -funds under deposit. Under deposit, those funds are rewarded under normal operation but are held -at pain of _slash_ (expropriation) should the staked maintainer be found not to be discharging -its duties properly. +The Staking module is the means by which a set of network maintainers (known as _authorities_ in some contexts and +_validators_ in others) are chosen based upon those who voluntarily place funds under deposit. Under deposit, those +funds are rewarded under normal operation but are held at pain of _slash_ (expropriation) should the staked maintainer +be found not to be discharging its duties properly. ### Terminology -- Staking: The process of locking up funds for some time, placing them at risk of slashing - (loss) in order to become a rewarded maintainer of the network. -- Validating: The process of running a node to actively maintain the network, either by - producing blocks or guaranteeing finality of the chain. -- Nominating: The process of placing staked funds behind one or more validators in order to - share in any reward, and punishment, they take. +- Staking: The process of locking up funds for some time, placing them at risk of slashing (loss) in order to become a + rewarded maintainer of the network. +- Validating: The process of running a node to actively maintain the network, either by producing blocks or guaranteeing + finality of the chain. +- Nominating: The process of placing staked funds behind one or more validators in order to share in any reward, and + punishment, they take. - Stash account: The account holding an owner's funds used for staking. - Controller account: The account that controls an owner's funds for staking. -- Era: A (whole) number of sessions, which is the period that the validator set (and each - validator's active nominator set) is recalculated and where rewards are paid out. +- Era: A (whole) number of sessions, which is the period that the validator set (and each validator's active nominator + set) is recalculated and where rewards are paid out. - Slash: The punishment of a staker by reducing its funds. ### Goals @@ -42,91 +41,90 @@ The staking system in Substrate NPoS is designed to make the following possible: #### Staking -Almost any interaction with the Staking module requires a process of _**bonding**_ (also known -as being a _staker_). To become *bonded*, a fund-holding account known as the _stash account_, -which holds some or all of the funds that become frozen in place as part of the staking process, -is paired with an active **controller** account, which issues instructions on how they shall be -used. +Almost any interaction with the Staking module requires a process of _**bonding**_ (also known as being a _staker_). To +become *bonded*, a fund-holding account known as the _stash account_, which holds some or all of the funds that become +frozen in place as part of the staking process, is paired with an active **controller** account, which issues +instructions on how they shall be used. -An account pair can become bonded using the [`bond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.bond) call. +An account pair can become bonded using the +[`bond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.bond) call. Stash accounts can update their associated controller back to their stash account using the -[`set_controller`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.set_controller) -call. +[`set_controller`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.set_controller) call. -Note: Controller accounts are being deprecated in favor of proxy accounts, so it is no longer -possible to set a unique address for a stash's controller. +Note: Controller accounts are being deprecated in favor of proxy accounts, so it is no longer possible to set a unique +address for a stash's controller. -There are three possible roles that any staked account pair can be in: `Validator`, `Nominator` -and `Idle` (defined in [`StakerStatus`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.StakerStatus.html)). There are three +There are three possible roles that any staked account pair can be in: `Validator`, `Nominator` and `Idle` (defined in +[`StakerStatus`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.StakerStatus.html)). There are three corresponding instructions to change between roles, namely: [`validate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.validate), -[`nominate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.nominate), and [`chill`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.chill). +[`nominate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.nominate), and +[`chill`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.chill). #### Validating -A **validator** takes the role of either validating blocks or ensuring their finality, -maintaining the veracity of the network. A validator should avoid both any sort of malicious -misbehavior and going offline. Bonded accounts that state interest in being a validator do NOT -get immediately chosen as a validator. Instead, they are declared as a _candidate_ and they -_might_ get elected at the _next era_ as a validator. The result of the election is determined -by nominators and their votes. +A **validator** takes the role of either validating blocks or ensuring their finality, maintaining the veracity of the +network. A validator should avoid both any sort of malicious misbehavior and going offline. Bonded accounts that state +interest in being a validator do NOT get immediately chosen as a validator. Instead, they are declared as a _candidate_ +and they _might_ get elected at the _next era_ as a validator. The result of the election is determined by nominators +and their votes. An account can become a validator candidate via the [`validate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.validate) call. #### Nomination -A **nominator** does not take any _direct_ role in maintaining the network, instead, it votes on -a set of validators to be elected. Once interest in nomination is stated by an account, it -takes effect at the next election round. The funds in the nominator's stash account indicate the -_weight_ of its vote. Both the rewards and any punishment that a validator earns are shared -between the validator and its nominators. This rule incentivizes the nominators to NOT vote for -the misbehaving/offline validators as much as possible, simply because the nominators will also -lose funds if they vote poorly. +A **nominator** does not take any _direct_ role in maintaining the network, instead, it votes on a set of validators to +be elected. Once interest in nomination is stated by an account, it takes effect at the next election round. The funds +in the nominator's stash account indicate the _weight_ of its vote. Both the rewards and any punishment that a validator +earns are shared between the validator and its nominators. This rule incentivizes the nominators to NOT vote for the +misbehaving/offline validators as much as possible, simply because the nominators will also lose funds if they vote +poorly. -An account can become a nominator via the [`nominate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.nominate) call. +An account can become a nominator via the +[`nominate`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.nominate) call. #### Rewards and Slash -The **reward and slashing** procedure is the core of the Staking module, attempting to _embrace -valid behavior_ while _punishing any misbehavior or lack of availability_. +The **reward and slashing** procedure is the core of the Staking module, attempting to _embrace valid behavior_ while +_punishing any misbehavior or lack of availability_. -Rewards must be claimed for each era before it gets too old by `$HISTORY_DEPTH` using the -`payout_stakers` call. Any account can call `payout_stakers`, which pays the reward to the -validator as well as its nominators. Only the [`Config::MaxNominatorRewardedPerValidator`] -biggest stakers can claim their reward. This is to limit the i/o cost to mutate storage for each -nominator's account. +Rewards must be claimed for each era before it gets too old by `$HISTORY_DEPTH` using the `payout_stakers` call. Any +account can call `payout_stakers`, which pays the reward to the validator as well as its nominators. Only the +[`Config::MaxNominatorRewardedPerValidator`] biggest stakers can claim their reward. This is to limit the i/o cost to +mutate storage for each nominator's account. -Slashing can occur at any point in time, once misbehavior is reported. Once slashing is -determined, a value is deducted from the balance of the validator and all the nominators who -voted for this validator (values are deducted from the _stash_ account of the slashed entity). +Slashing can occur at any point in time, once misbehavior is reported. Once slashing is determined, a value is deducted +from the balance of the validator and all the nominators who voted for this validator (values are deducted from the +_stash_ account of the slashed entity). Slashing logic is further described in the documentation of the `slashing` module. -Similar to slashing, rewards are also shared among a validator and its associated nominators. -Yet, the reward funds are not always transferred to the stash account and can be configured. See -[Reward Calculation](https://docs.rs/pallet-staking/latest/pallet_staking/#reward-calculation) for more details. +Similar to slashing, rewards are also shared among a validator and its associated nominators. Yet, the reward funds are +not always transferred to the stash account and can be configured. See [Reward +Calculation](https://docs.rs/pallet-staking/latest/pallet_staking/#reward-calculation) for more details. #### Chilling -Finally, any of the roles above can choose to step back temporarily and just chill for a while. -This means that if they are a nominator, they will not be considered as voters anymore and if -they are validators, they will no longer be a candidate for the next election. +Finally, any of the roles above can choose to step back temporarily and just chill for a while. This means that if they +are a nominator, they will not be considered as voters anymore and if they are validators, they will no longer be a +candidate for the next election. -An account can step back via the [`chill`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.chill) call. +An account can step back via the +[`chill`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.chill) call. ### Session managing -The module implement the trait `SessionManager`. Which is the only API to query new validator -set and allowing these validator set to be rewarded once their era is ended. +The module implement the trait `SessionManager`. Which is the only API to query new validator set and allowing these +validator set to be rewarded once their era is ended. ## Interface ### Dispatchable Functions -The dispatchable functions of the Staking module enable the steps needed for entities to accept -and change their role, alongside some helper functions to get/set the metadata of the module. +The dispatchable functions of the Staking module enable the steps needed for entities to accept and change their role, +alongside some helper functions to get/set the metadata of the module. ### Public Functions @@ -134,7 +132,7 @@ The Staking module contains many public storage items and (im)mutable functions. ## Usage -### Example: Rewarding a validator by id. +### Example: Rewarding a validator by id ```rust use pallet_staking::{self as staking}; @@ -169,7 +167,8 @@ pub mod pallet { ### Era payout The era payout is computed using yearly inflation curve defined at -[`T::RewardCurve`](https://docs.rs/pallet-staking/latest/pallet_staking/trait.Config.html#associatedtype.RewardCurve) as such: +[`T::RewardCurve`](https://docs.rs/pallet-staking/latest/pallet_staking/trait.Config.html#associatedtype.RewardCurve) as +such: ```nocompile staker_payout = yearly_inflation(npos_token_staked / total_tokens) * total_tokens / era_per_year @@ -184,37 +183,38 @@ The remaining reward is send to the configurable end-point ### Reward Calculation -Validators and nominators are rewarded at the end of each era. The total reward of an era is -calculated using the era duration and the staking rate (the total amount of tokens staked by -nominators and validators, divided by the total token supply). It aims to incentivize toward a -defined staking rate. The full specification can be found +Validators and nominators are rewarded at the end of each era. The total reward of an era is calculated using the era +duration and the staking rate (the total amount of tokens staked by nominators and validators, divided by the total +token supply). It aims to incentivize toward a defined staking rate. The full specification can be found [here](https://research.web3.foundation/en/latest/polkadot/economics/1-token-economics.html#inflation-model). -Total reward is split among validators and their nominators depending on the number of points -they received during the era. Points are added to a validator using +Total reward is split among validators and their nominators depending on the number of points they received during the +era. Points are added to a validator using [`reward_by_ids`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.reward_by_ids) or [`reward_by_indices`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.reward_by_indices). [`Module`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Module.html) implements -[`pallet_authorship::EventHandler`](https://docs.rs/pallet-authorship/latest/pallet_authorship/trait.EventHandler.html) to add reward -points to block producer and block producer of referenced uncles. +[`pallet_authorship::EventHandler`](https://docs.rs/pallet-authorship/latest/pallet_authorship/trait.EventHandler.html) +to add reward points to block producer and block producer of referenced uncles. The validator and its nominator split their reward as following: The validator can declare an amount, named -[`commission`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.ValidatorPrefs.html#structfield.commission), that does not get shared -with the nominators at each reward payout through its -[`ValidatorPrefs`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.ValidatorPrefs.html). This value gets deducted from the total reward -that is paid to the validator and its nominators. The remaining portion is split among the -validator and all of the nominators that nominated the validator, proportional to the value -staked behind this validator (_i.e._ dividing the +[`commission`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.ValidatorPrefs.html#structfield.commission), +that does not get shared with the nominators at each reward payout through its +[`ValidatorPrefs`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.ValidatorPrefs.html). This value gets +deducted from the total reward that is paid to the validator and its nominators. The remaining portion is split among +the validator and all of the nominators that nominated the validator, proportional to the value staked behind this +validator (_i.e._ dividing the [`own`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html#structfield.own) or [`others`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html#structfield.others) by -[`total`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html#structfield.total) in [`Exposure`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html)). +[`total`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html#structfield.total) in +[`Exposure`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Exposure.html)). All entities who receive a reward have the option to choose their reward destination through the [`Payee`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.Payee.html) storage item (see -[`set_payee`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.set_payee)), to be one of the following: +[`set_payee`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.set_payee)), to be one of the +following: - Controller account, (obviously) not increasing the staked value. - Stash account, not increasing the staked value. @@ -225,32 +225,33 @@ All entities who receive a reward have the option to choose their reward destina Any funds already placed into stash can be the target of the following operations: The controller account can free a portion (or all) of the funds using the -[`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond) call. Note that the funds are not immediately -accessible. Instead, a duration denoted by [`BondingDuration`](https://docs.rs/pallet-staking/latest/pallet_staking/trait.Config.html#associatedtype.BondingDuration) -(in number of eras) must pass until the funds can actually be removed. Once the -`BondingDuration` is over, the [`withdraw_unbonded`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.withdraw_unbonded) +[`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond) call. Note that the funds +are not immediately accessible. Instead, a duration denoted by +[`BondingDuration`](https://docs.rs/pallet-staking/latest/pallet_staking/trait.Config.html#associatedtype.BondingDuration) +(in number of eras) must pass until the funds can actually be removed. Once the `BondingDuration` is over, the +[`withdraw_unbonded`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.withdraw_unbonded) call can be used to actually withdraw the funds. -Note that there is a limitation to the number of fund-chunks that can be scheduled to be -unlocked in the future via [`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond). In case this maximum -(`MAX_UNLOCKING_CHUNKS`) is reached, the bonded account _must_ first wait until a successful -call to `withdraw_unbonded` to remove some of the chunks. +Note that there is a limitation to the number of fund-chunks that can be scheduled to be unlocked in the future via +[`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond). In case this maximum +(`MAX_UNLOCKING_CHUNKS`) is reached, the bonded account _must_ first wait until a successful call to `withdraw_unbonded` +to remove some of the chunks. ### Election Algorithm -The current election algorithm is implemented based on Phragmén. The reference implementation -can be found [here](https://github.com/w3f/consensus/tree/master/NPoS). +The current election algorithm is implemented based on Phragmén. The reference implementation can be found +[here](https://github.com/w3f/consensus/tree/master/NPoS). -The election algorithm, aside from electing the validators with the most stake value and votes, -tries to divide the nominator votes among candidates in an equal manner. To further assure this, -an optional post-processing can be applied that iteratively normalizes the nominator staked -values until the total difference among votes of a particular nominator are less than a -threshold. +The election algorithm, aside from electing the validators with the most stake value and votes, tries to divide the +nominator votes among candidates in an equal manner. To further assure this, an optional post-processing can be applied +that iteratively normalizes the nominator staked values until the total difference among votes of a particular nominator +are less than a threshold. ## GenesisConfig -The Staking module depends on the [`GenesisConfig`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.GenesisConfig.html). The -`GenesisConfig` is optional and allow to set some initial stakers. +The Staking module depends on the +[`GenesisConfig`](https://docs.rs/pallet-staking/latest/pallet_staking/struct.GenesisConfig.html). The `GenesisConfig` +is optional and allow to set some initial stakers. ## Related Modules diff --git a/substrate/frame/staking/reward-curve/Cargo.toml b/substrate/frame/staking/reward-curve/Cargo.toml index e1b97d278da1ab1a8d7469a5a7d1a305ebe10bf1..484afb6136bf0674b2fab4dd9ba2137d50382a41 100644 --- a/substrate/frame/staking/reward-curve/Cargo.toml +++ b/substrate/frame/staking/reward-curve/Cargo.toml @@ -18,7 +18,7 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit"] } +syn = { version = "2.0.37", features = ["full", "visit"] } [dev-dependencies] sp-runtime = { path = "../../../primitives/runtime" } diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index e0f5c9558781856e97730d93ab38c769242acdd4..3d9b1157ca87afaaea80b260ecb8d71179ffe4df 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -236,11 +236,12 @@ impl Pallet { let mut total_imbalance = PositiveImbalanceOf::::zero(); // We can now make total validator payout: - if let Some(imbalance) = + if let Some((imbalance, dest)) = Self::make_payout(&ledger.stash, validator_staking_payout + validator_commission_payout) { Self::deposit_event(Event::::Rewarded { stash: ledger.stash, + dest, amount: imbalance.peek(), }); total_imbalance.subsume(imbalance); @@ -259,11 +260,14 @@ impl Pallet { let nominator_reward: BalanceOf = nominator_exposure_part * validator_leftover_payout; // We can now make nominator payout: - if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { + if let Some((imbalance, dest)) = Self::make_payout(&nominator.who, nominator_reward) { // Note: this logic does not count payouts for `RewardDestination::None`. nominator_payout_count += 1; - let e = - Event::::Rewarded { stash: nominator.who.clone(), amount: imbalance.peek() }; + let e = Event::::Rewarded { + stash: nominator.who.clone(), + dest, + amount: imbalance.peek(), + }; Self::deposit_event(e); total_imbalance.subsume(imbalance); } @@ -293,9 +297,11 @@ impl Pallet { /// Actually make a payment to a staker. This uses the currency's reward function /// to pay the right payee for the given staker account. - fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { - let dest = Self::payee(stash); - match dest { + fn make_payout( + stash: &T::AccountId, + amount: BalanceOf, + ) -> Option<(PositiveImbalanceOf, RewardDestination)> { + let maybe_imbalance = match Self::payee(stash) { RewardDestination::Controller => Self::bonded(stash) .map(|controller| T::Currency::deposit_creating(&controller, amount)), RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), @@ -311,7 +317,8 @@ impl Pallet { RewardDestination::Account(dest_account) => Some(T::Currency::deposit_creating(&dest_account, amount)), RewardDestination::None => None, - } + }; + maybe_imbalance.map(|imbalance| (imbalance, Self::payee(stash))) } /// Plan a new session potentially trigger a new era. diff --git a/substrate/frame/staking/src/pallet/mod.rs b/substrate/frame/staking/src/pallet/mod.rs index c6c75326b80cdef42afa9d35dd844b2f91e3b50d..0bcf932d90b44d9233515bb312e470fcd4ef1480 100644 --- a/substrate/frame/staking/src/pallet/mod.rs +++ b/substrate/frame/staking/src/pallet/mod.rs @@ -664,8 +664,12 @@ pub mod pallet { /// The era payout has been set; the first balance is the validator-payout; the second is /// the remainder from the maximum amount of reward. EraPaid { era_index: EraIndex, validator_payout: BalanceOf, remainder: BalanceOf }, - /// The nominator has been rewarded by this amount. - Rewarded { stash: T::AccountId, amount: BalanceOf }, + /// The nominator has been rewarded by this amount to this destination. + Rewarded { + stash: T::AccountId, + dest: RewardDestination, + amount: BalanceOf, + }, /// A staker (validator or nominator) has been slashed by the given amount. Slashed { staker: T::AccountId, amount: BalanceOf }, /// A slash for the given validator, for the given percentage of their stake, at the given diff --git a/substrate/frame/staking/src/tests.rs b/substrate/frame/staking/src/tests.rs index fd7dabac74d8de2e17753040720e04b5476a9b5a..78183cfde9296bdec25327b39c8fd6691a403296 100644 --- a/substrate/frame/staking/src/tests.rs +++ b/substrate/frame/staking/src/tests.rs @@ -3759,6 +3759,16 @@ fn test_payout_stakers() { ); assert!(RewardOnUnbalanceWasCalled::get()); + // `Rewarded` events are being executed. + assert!(matches!( + staking_events_since_last_call().as_slice(), + &[ + .., + Event::Rewarded { stash: 1037, dest: RewardDestination::Controller, amount: 108 }, + Event::Rewarded { stash: 1036, dest: RewardDestination::Controller, amount: 108 } + ] + )); + // Top 64 nominators of validator 11 automatically paid out, including the validator // Validator payout goes to controller. assert!(Balances::free_balance(&11) > balance); diff --git a/substrate/frame/sudo/Cargo.toml b/substrate/frame/sudo/Cargo.toml index a75a0c504f98363f403d46c2dc8f4b2ffbe61986..a4934346d5dec87d4c6f8df9a924bb1781b0039e 100644 --- a/substrate/frame/sudo/Cargo.toml +++ b/substrate/frame/sudo/Cargo.toml @@ -22,6 +22,8 @@ sp-io = { path = "../../primitives/io", default-features = false} sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} +docify = "0.2.4" + [dev-dependencies] sp-core = { path = "../../primitives/core" } diff --git a/substrate/frame/sudo/README.md b/substrate/frame/sudo/README.md index 886dc5981778d2c05dba38f6f711e23e9d5e7b50..371f89e5348c6745f92c8762a577fdb570bb9498 100644 --- a/substrate/frame/sudo/README.md +++ b/substrate/frame/sudo/README.md @@ -16,8 +16,8 @@ Only one account can be the sudo key at a time. Only the sudo key can call the dispatchable functions from the Sudo module. -* `sudo` - Make a `Root` call to a dispatchable function. -* `set_key` - Assign a new account to be the sudo key. +- `sudo` - Make a `Root` call to a dispatchable function. +- `set_key` - Assign a new account to be the sudo key. ## Usage @@ -68,7 +68,7 @@ You need to set an initial superuser account as the sudo `key`. ## Related Modules -* [Democracy](https://docs.rs/pallet-democracy/latest/pallet_democracy/) +- [Democracy](https://docs.rs/pallet-democracy/latest/pallet_democracy/) [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html diff --git a/substrate/frame/sudo/src/lib.rs b/substrate/frame/sudo/src/lib.rs index f735469558c70b18c3cb6ea93a77c39979b4cb6a..0c869bec7f076f7cc2e240adda1eeb77b8409277 100644 --- a/substrate/frame/sudo/src/lib.rs +++ b/substrate/frame/sudo/src/lib.rs @@ -7,7 +7,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,41 +15,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Sudo Pallet -//! -//! - [`Config`] -//! - [`Call`] -//! -//! ## Overview -//! -//! The Sudo pallet allows for a single account (called the "sudo key") -//! to execute dispatchable functions that require a `Root` call -//! or designate a new account to replace them as the sudo key. -//! Only one account can be the sudo key at a time. -//! -//! ## Interface +//! > Made with *Substrate*, for *Polkadot*. //! -//! ### Dispatchable Functions +//! [![github]](https://github.com/paritytech/polkadot-sdk/tree/master/substrate/frame/sudo) +//! [![polkadot]](https://polkadot.network) //! -//! Only the sudo key can call the dispatchable functions from the Sudo pallet. +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white //! -//! * `sudo` - Make a `Root` call to a dispatchable function. -//! * `set_key` - Assign a new account to be the sudo key. +//! # Sudo Pallet //! -//! ## Usage +//! A pallet to provide a way to execute privileged runtime calls using a specified sudo ("superuser +//! do") account. //! -//! ### Executing Privileged Functions +//! ## Pallet API //! -//! The Sudo pallet itself is not intended to be used within other pallets. -//! Instead, you can build "privileged functions" (i.e. functions that require `Root` origin) in -//! other pallets. You can execute these privileged functions by calling `sudo` with the sudo key -//! account. Privileged functions cannot be directly executed via an extrinsic. +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. //! -//! Learn more about privileged functions and `Root` origin in the [`Origin`] type documentation. +//! ## Overview //! -//! ### Simple Code Snippet +//! In Substrate blockchains, pallets may contain dispatchable calls that can only be called at +//! the system level of the chain (i.e. dispatchables that require a `Root` origin). +//! Setting a privileged account, called the _sudo key_, allows you to make such calls as an +//! extrinisic. //! -//! This is an example of a pallet that exposes a privileged function: +//! Here's an example of a privileged function in another pallet: //! //! ``` //! #[frame_support::pallet] @@ -76,27 +67,58 @@ //! } //! } //! } -//! # fn main() {} //! ``` //! -//! ### Signed Extension +//! With the Sudo pallet configured in your chain's runtime you can execute this privileged +//! function by constructing a call using the [`sudo`](Pallet::sudo) dispatchable. +//! +//! To use this pallet in your runtime, a sudo key must be specified in the [`GenesisConfig`] of +//! the pallet. You can change this key at anytime once your chain is live using the +//! [`set_key`](Pallet::set_key) dispatchable, however only one sudo key can be set at a +//! time. The pallet also allows you to make a call using +//! [`sudo_unchecked_weight`](Pallet::sudo_unchecked_weight), which allows the sudo account to +//! execute a call with a custom weight. +//! +//!
+//! Note: this pallet is not meant to be used inside other pallets. It is only
+//! meant to be used by constructing runtime calls from outside the runtime.
+//! 
+//! +//! This pallet also defines a [`SignedExtension`](sp_runtime::traits::SignedExtension) called +//! [`CheckOnlySudoAccount`] to ensure that only signed transactions by the sudo account are +//! accepted by the transaction pool. The intended use of this signed extension is to prevent other +//! accounts from spamming the transaction pool for the initial phase of a chain, during which +//! developers may only want a sudo account to be able to make transactions. +//! +//! Learn more about the `Root` origin in the [`RawOrigin`](frame_system::RawOrigin) type +//! documentation. //! -//! The Sudo pallet defines the following extension: +//! ### Examples //! -//! - [`CheckOnlySudoAccount`]: Ensures that the signed transactions are only valid if they are -//! signed by sudo account. +//! 1. You can make a privileged runtime call using `sudo` with an account that matches the sudo +//! key. +#![doc = docify::embed!("src/tests.rs", sudo_basics)] //! -//! ## Genesis Config +//! 2. Only an existing sudo key can set a new one. +#![doc = docify::embed!("src/tests.rs", set_key_basics)] //! -//! The Sudo pallet depends on the [`GenesisConfig`]. -//! You need to set an initial superuser account as the sudo `key`. +//! 3. You can also make non-privileged calls using `sudo_as`. +#![doc = docify::embed!("src/tests.rs", sudo_as_emits_events_correctly)] //! -//! ## Related Pallets +//! ## Low Level / Implementation Details //! -//! * [Democracy](../pallet_democracy/index.html) +//! This pallet checks that the caller of its dispatchables is a signed account and ensures that the +//! caller matches the sudo key in storage. +//! A caller of this pallet's dispatchables does not pay any fees to dispatch a call. If the account +//! making one of these calls is not the sudo key, the pallet returns a [`Error::RequireSudo`] +//! error. //! -//! [`Origin`]: https://docs.substrate.io/main-docs/build/origins/ +//! Once an origin is verified, sudo calls use `dispatch_bypass_filter` from the +//! [`UnfilteredDispatchable`](frame_support::traits::UnfilteredDispatchable) trait to allow call +//! execution without enforcing any further origin checks. +#![deny(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] use sp_runtime::{traits::StaticLookup, DispatchResult}; @@ -261,12 +283,21 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A sudo just took place. \[result\] - Sudid { sudo_result: DispatchResult }, - /// The \[sudoer\] just switched identity; the old key is supplied if one existed. - KeyChanged { old_sudoer: Option }, - /// A sudo just took place. \[result\] - SudoAsDone { sudo_result: DispatchResult }, + /// A sudo call just took place. + Sudid { + /// The result of the call made by the sudo user. + sudo_result: DispatchResult, + }, + /// The sudo key has been updated. + KeyChanged { + /// The old sudo key if one was previously set. + old_sudoer: Option, + }, + /// A [sudo_as](Pallet::sudo_as) call just took place. + SudoAsDone { + /// The result of the call made by the sudo user. + sudo_result: DispatchResult, + }, } #[pallet::error] diff --git a/substrate/frame/sudo/src/tests.rs b/substrate/frame/sudo/src/tests.rs index c854fed8f0736e7aedb523ccd6e49745dfca21b1..6963ba2e6a05133b719e546b63f2cd0c722aa264 100644 --- a/substrate/frame/sudo/src/tests.rs +++ b/substrate/frame/sudo/src/tests.rs @@ -34,6 +34,7 @@ fn test_setup_works() { }); } +#[docify::export] #[test] fn sudo_basics() { // Configure a default test environment and set the root `key` to 1. @@ -134,6 +135,7 @@ fn sudo_unchecked_weight_emits_events_correctly() { }) } +#[docify::export] #[test] fn set_key_basics() { new_test_ext(1).execute_with(|| { @@ -195,6 +197,7 @@ fn sudo_as_basics() { }); } +#[docify::export] #[test] fn sudo_as_emits_events_correctly() { new_test_ext(1).execute_with(|| { diff --git a/substrate/frame/sudo/src/weights.rs b/substrate/frame/sudo/src/weights.rs index 6a0197d1469b4ac65d786255c3973e2e575a62d4..0cdd0c8a81f2c94dc8a967d128f21b00bd0f22e2 100644 --- a/substrate/frame/sudo/src/weights.rs +++ b/substrate/frame/sudo/src/weights.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Autogenerated weights for pallet_sudo +//! Autogenerated weights for pallet_sudo. //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev //! DATE: 2023-06-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` diff --git a/substrate/frame/support/Cargo.toml b/substrate/frame/support/Cargo.toml index 864e3bd9ad3077142692aac6d22f3ec063d49825..5cb5d6d12ab784c5fabd205de84bdba227f3edec 100644 --- a/substrate/frame/support/Cargo.toml +++ b/substrate/frame/support/Cargo.toml @@ -42,8 +42,8 @@ sp-core-hashing-proc-macro = { path = "../../primitives/core/hashing/proc-macro" k256 = { version = "0.13.1", default-features = false, features = ["ecdsa"] } environmental = { version = "1.1.4", default-features = false } sp-genesis-builder = { path = "../../primitives/genesis-builder", default-features=false} -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } -docify = "0.2.1" +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } +docify = "0.2.4" static_assertions = "1.1.0" aquamarine = { version = "0.3.2" } diff --git a/substrate/frame/support/README.md b/substrate/frame/support/README.md index 2282870aca05ca8393e5d6bef1ddb059e0466a72..dd2a07346dda1c411cd32c0ab04518dec0c643ac 100644 --- a/substrate/frame/support/README.md +++ b/substrate/frame/support/README.md @@ -1,3 +1,3 @@ Support code for the runtime. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/support/procedural/Cargo.toml b/substrate/frame/support/procedural/Cargo.toml index 9ca86b9fccb1bc23bd6a311fcb89eb353898573c..6381e430f2baae2dbb036e8e424ca0c6fc9ff60d 100644 --- a/substrate/frame/support/procedural/Cargo.toml +++ b/substrate/frame/support/procedural/Cargo.toml @@ -17,11 +17,11 @@ proc-macro = true [dependencies] derive-syn-parse = "0.1.5" Inflector = "0.11.4" -cfg-expr = "0.15.4" +cfg-expr = "0.15.5" itertools = "0.10.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full"] } +syn = { version = "2.0.37", features = ["full"] } frame-support-procedural-tools = { path = "tools" } proc-macro-warning = { version = "0.4.2", default-features = false } macro_magic = { version = "0.4.2", features = ["proc_support"] } diff --git a/substrate/frame/support/procedural/tools/Cargo.toml b/substrate/frame/support/procedural/tools/Cargo.toml index 46c264256513c78ea857f4a8ac10f5c369881ec5..7589fa353d16a19650b41fcedf7cf67d28535606 100644 --- a/substrate/frame/support/procedural/tools/Cargo.toml +++ b/substrate/frame/support/procedural/tools/Cargo.toml @@ -15,5 +15,5 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit", "extra-traits"] } +syn = { version = "2.0.37", features = ["full", "visit", "extra-traits"] } frame-support-procedural-tools-derive = { path = "derive" } diff --git a/substrate/frame/support/procedural/tools/derive/Cargo.toml b/substrate/frame/support/procedural/tools/derive/Cargo.toml index da01e6f4f6af5f81b5e609dc08e181c451e920c7..5bf67d43d06ed96502f20341b3b736bba3d621ba 100644 --- a/substrate/frame/support/procedural/tools/derive/Cargo.toml +++ b/substrate/frame/support/procedural/tools/derive/Cargo.toml @@ -17,4 +17,4 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = { version = "1.0.28", features = ["proc-macro"] } -syn = { version = "2.0.31", features = ["proc-macro", "full", "extra-traits", "parsing"] } +syn = { version = "2.0.37", features = ["proc-macro", "full", "extra-traits", "parsing"] } diff --git a/substrate/frame/support/src/dispatch.rs b/substrate/frame/support/src/dispatch.rs index eb1fc524200b7129a7d8f749c639e8710834b345..75e94827fea8a142d5038d4685ede440b7b02c5c 100644 --- a/substrate/frame/support/src/dispatch.rs +++ b/substrate/frame/support/src/dispatch.rs @@ -138,6 +138,15 @@ impl From for PostDispatchInfo { } } +impl From for Pays { + fn from(b: bool) -> Self { + match b { + true => Self::Yes, + false => Self::No, + } + } +} + /// A generalized group of dispatch types. /// /// NOTE whenever upgrading the enum make sure to also update diff --git a/substrate/frame/support/src/migrations.rs b/substrate/frame/support/src/migrations.rs index 6fc1834f01adf61198366d7c01f1eb71332965dc..8f490030c25b2b53b2c7ec6b3b3debdb61e7b5e8 100644 --- a/substrate/frame/support/src/migrations.rs +++ b/substrate/frame/support/src/migrations.rs @@ -24,9 +24,7 @@ use sp_core::Get; use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; use sp_std::marker::PhantomData; -/// EXPERIMENTAL: The API of this feature may change. -/// -/// Make it easier to write versioned runtime upgrades. +/// Handles storage migration pallet versioning. /// /// [`VersionedMigration`] allows developers to write migrations without worrying about checking and /// setting storage versions. Instead, the developer wraps their migration in this struct which @@ -69,14 +67,12 @@ use sp_std::marker::PhantomData; /// // other migrations... /// ); /// ``` -#[cfg(feature = "experimental")] pub struct VersionedMigration { _marker: PhantomData<(Inner, Pallet, Weight)>, } /// A helper enum to wrap the pre_upgrade bytes like an Option before passing them to post_upgrade. /// This enum is used rather than an Option to make the API clearer to the developer. -#[cfg(feature = "experimental")] #[derive(codec::Encode, codec::Decode)] pub enum VersionedPostUpgradeData { /// The migration ran, inner vec contains pre_upgrade data. @@ -91,7 +87,6 @@ pub enum VersionedPostUpgradeData { /// version of the pallets storage matches `From`, and after the upgrade set the on-chain storage to /// `To`. If the versions do not match, it writes a log notifying the developer that the migration /// is a noop. -#[cfg(feature = "experimental")] impl< const FROM: u16, const TO: u16, diff --git a/substrate/frame/support/src/storage/generator/map.rs b/substrate/frame/support/src/storage/generator/map.rs index 90fac4b41c7592c29b7f38a781a39e3be186fd94..1d2511e324dc654dbbcbe4c37521d25511fc148c 100644 --- a/substrate/frame/support/src/storage/generator/map.rs +++ b/substrate/frame/support/src/storage/generator/map.rs @@ -21,7 +21,6 @@ use crate::{ Never, }; use codec::{Decode, Encode, EncodeLike, FullCodec, FullEncode}; -use sp_std::borrow::Borrow; #[cfg(not(feature = "std"))] use sp_std::prelude::*; @@ -297,7 +296,7 @@ impl> storage::StorageMap let ret = f(&mut val); if ret.is_ok() { match G::from_query_to_optional_value(val) { - Some(ref val) => unhashed::put(final_key.as_ref(), &val.borrow()), + Some(ref val) => unhashed::put(final_key.as_ref(), &val), None => unhashed::kill(final_key.as_ref()), } } @@ -314,7 +313,7 @@ impl> storage::StorageMap let ret = f(&mut val); if ret.is_ok() { match val { - Some(ref val) => unhashed::put(final_key.as_ref(), &val.borrow()), + Some(ref val) => unhashed::put(final_key.as_ref(), &val), None => unhashed::kill(final_key.as_ref()), } } diff --git a/substrate/frame/support/src/traits.rs b/substrate/frame/support/src/traits.rs index f669046f858f44d8013311abdf778b4f7be56d49..10ae5d83b5ac958fca06be636d38bb0fe4020cb4 100644 --- a/substrate/frame/support/src/traits.rs +++ b/substrate/frame/support/src/traits.rs @@ -90,8 +90,8 @@ pub use hooks::{ pub mod schedule; mod storage; pub use storage::{ - Incrementable, Instance, PartialStorageInfoTrait, StorageInfo, StorageInfoTrait, - StorageInstance, TrackedStorageKey, WhitelistedStorageKeys, + Consideration, Footprint, Incrementable, Instance, LinearStoragePrice, PartialStorageInfoTrait, + StorageInfo, StorageInfoTrait, StorageInstance, TrackedStorageKey, WhitelistedStorageKeys, }; mod dispatch; @@ -111,9 +111,8 @@ pub use preimages::{Bounded, BoundedInline, FetchResult, Hash, QueryPreimage, St mod messages; pub use messages::{ - EnqueueMessage, EnqueueWithOrigin, ExecuteOverweightError, Footprint, HandleMessage, - NoopServiceQueues, ProcessMessage, ProcessMessageError, QueuePausedQuery, ServiceQueues, - TransformOrigin, + EnqueueMessage, EnqueueWithOrigin, ExecuteOverweightError, HandleMessage, NoopServiceQueues, + ProcessMessage, ProcessMessageError, QueuePausedQuery, ServiceQueues, TransformOrigin, }; mod safe_mode; diff --git a/substrate/frame/support/src/traits/messages.rs b/substrate/frame/support/src/traits/messages.rs index 36fa7957dff7c3891053eddc1f33b29da20366de..0db163e072b179100a44c0c39dcc76b35c02b997 100644 --- a/substrate/frame/support/src/traits/messages.rs +++ b/substrate/frame/support/src/traits/messages.rs @@ -17,6 +17,7 @@ //! Traits for managing message queuing and handling. +use super::storage::Footprint; use codec::{Decode, Encode, FullCodec, MaxEncodedLen}; use scale_info::TypeInfo; use sp_core::{ConstU32, Get, TypedGet}; @@ -115,13 +116,6 @@ impl ServiceQueues for NoopServiceQueues { } } -/// The resource footprint of a queue. -#[derive(Default, Copy, Clone, Eq, PartialEq, RuntimeDebug)] -pub struct Footprint { - pub count: u64, - pub size: u64, -} - /// Can enqueue messages for multiple origins. pub trait EnqueueMessage { /// The maximal length any enqueued message may have. diff --git a/substrate/frame/support/src/traits/preimages.rs b/substrate/frame/support/src/traits/preimages.rs index 3e78116202b4f0d0a3d85561a06bb095e56b2815..bf08a286dd7cfb617e3271256e06fbb9a27e15c0 100644 --- a/substrate/frame/support/src/traits/preimages.rs +++ b/substrate/frame/support/src/traits/preimages.rs @@ -18,6 +18,7 @@ //! Stuff for dealing with 32-byte hashed preimages. use codec::{Decode, Encode, EncodeLike, MaxEncodedLen}; +use scale_info::TypeInfo; use sp_core::{RuntimeDebug, H256}; use sp_io::hashing::blake2_256; use sp_runtime::{traits::ConstU32, DispatchError}; @@ -29,9 +30,7 @@ pub type BoundedInline = crate::BoundedVec>; /// The maximum we expect a single legacy hash lookup to be. const MAX_LEGACY_LEN: u32 = 1_000_000; -#[derive( - Encode, Decode, MaxEncodedLen, Clone, Eq, PartialEq, scale_info::TypeInfo, RuntimeDebug, -)] +#[derive(Encode, Decode, MaxEncodedLen, Clone, Eq, PartialEq, TypeInfo, RuntimeDebug)] #[codec(mel_bound())] pub enum Bounded { /// A Blake2 256 hash with no preimage length. We diff --git a/substrate/frame/support/src/traits/storage.rs b/substrate/frame/support/src/traits/storage.rs index 64eddf51b7fabb160dfd9b09c2cd3c98bf7c317b..e0ce1c0fbd317fff284a01c43b3c25bca21ead76 100644 --- a/substrate/frame/support/src/traits/storage.rs +++ b/substrate/frame/support/src/traits/storage.rs @@ -17,9 +17,16 @@ //! Traits for encoding data related to pallet's storage items. +use codec::{Encode, FullCodec, MaxEncodedLen}; +use core::marker::PhantomData; use impl_trait_for_tuples::impl_for_tuples; +use scale_info::TypeInfo; pub use sp_core::storage::TrackedStorageKey; -use sp_runtime::{traits::Saturating, RuntimeDebug}; +use sp_core::Get; +use sp_runtime::{ + traits::{Convert, Member, Saturating}, + DispatchError, RuntimeDebug, +}; use sp_std::{collections::btree_set::BTreeSet, prelude::*}; /// An instance of a pallet in the storage. @@ -127,6 +134,89 @@ impl WhitelistedStorageKeys for Tuple { } } +/// The resource footprint of a bunch of blobs. We assume only the number of blobs and their total +/// size in bytes matter. +#[derive(Default, Copy, Clone, Eq, PartialEq, RuntimeDebug)] +pub struct Footprint { + /// The number of blobs. + pub count: u64, + /// The total size of the blobs in bytes. + pub size: u64, +} + +impl Footprint { + pub fn from_parts(items: usize, len: usize) -> Self { + Self { count: items as u64, size: len as u64 } + } + + pub fn from_encodable(e: impl Encode) -> Self { + Self::from_parts(1, e.encoded_size()) + } +} + +/// A storage price that increases linearly with the number of elements and their size. +pub struct LinearStoragePrice(PhantomData<(Base, Slope, Balance)>); +impl Convert for LinearStoragePrice +where + Base: Get, + Slope: Get, + Balance: From + sp_runtime::Saturating, +{ + fn convert(a: Footprint) -> Balance { + let s: Balance = (a.count.saturating_mul(a.size)).into(); + s.saturating_mul(Slope::get()).saturating_add(Base::get()) + } +} + +/// Some sort of cost taken from account temporarily in order to offset the cost to the chain of +/// holding some data [`Footprint`] in state. +/// +/// The cost may be increased, reduced or dropped entirely as the footprint changes. +/// +/// A single ticket corresponding to some particular datum held in storage. This is an opaque +/// type, but must itself be stored and generally it should be placed alongside whatever data +/// the ticket was created for. +/// +/// While not technically a linear type owing to the need for `FullCodec`, *this should be +/// treated as one*. Don't type to duplicate it, and remember to drop it when you're done with +/// it. +#[must_use] +pub trait Consideration: Member + FullCodec + TypeInfo + MaxEncodedLen { + /// Create a ticket for the `new` footprint attributable to `who`. This ticket *must* ultimately + /// be consumed through `update` or `drop` once the footprint changes or is removed. + fn new(who: &AccountId, new: Footprint) -> Result; + + /// Optionally consume an old ticket and alter the footprint, enforcing the new cost to `who` + /// and returning the new ticket (or an error if there was an issue). + /// + /// For creating tickets and dropping them, you can use the simpler `new` and `drop` instead. + fn update(self, who: &AccountId, new: Footprint) -> Result; + + /// Consume a ticket for some `old` footprint attributable to `who` which should now been freed. + fn drop(self, who: &AccountId) -> Result<(), DispatchError>; + + /// Consume a ticket for some `old` footprint attributable to `who` which should be sacrificed. + /// + /// This is infallible. In the general case (and it is left unimplemented), then it is + /// equivalent to the consideration never being dropped. Cases which can handle this properly + /// should implement, but it *MUST* rely on the loss of the consideration to the owner. + fn burn(self, _: &AccountId) { + let _ = self; + } +} + +impl Consideration for () { + fn new(_: &A, _: Footprint) -> Result { + Ok(()) + } + fn update(self, _: &A, _: Footprint) -> Result<(), DispatchError> { + Ok(()) + } + fn drop(self, _: &A) -> Result<(), DispatchError> { + Ok(()) + } +} + macro_rules! impl_incrementable { ($($type:ty),+) => { $( @@ -165,3 +255,25 @@ where } impl_incrementable!(u8, u16, u32, u64, u128, i8, i16, i32, i64, i128); + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::ConstU64; + + #[test] + fn linear_storage_price_works() { + type Linear = LinearStoragePrice, ConstU64<3>, u64>; + let p = |count, size| Linear::convert(Footprint { count, size }); + + assert_eq!(p(0, 0), 7); + assert_eq!(p(0, 1), 7); + assert_eq!(p(1, 0), 7); + + assert_eq!(p(1, 1), 10); + assert_eq!(p(8, 1), 31); + assert_eq!(p(1, 8), 31); + + assert_eq!(p(u64::MAX, u64::MAX), u64::MAX); + } +} diff --git a/substrate/frame/support/src/traits/tokens/fungible/freeze.rs b/substrate/frame/support/src/traits/tokens/fungible/freeze.rs index bc8f22b959cb99166300a9f09d7fe1a43185e89b..8b542ee4c6060e143ee58902a38e39cca38b109b 100644 --- a/substrate/frame/support/src/traits/tokens/fungible/freeze.rs +++ b/substrate/frame/support/src/traits/tokens/fungible/freeze.rs @@ -18,7 +18,13 @@ //! The traits for putting freezes within a single fungible token class. use scale_info::TypeInfo; -use sp_runtime::DispatchResult; +use sp_arithmetic::{ + traits::{CheckedAdd, CheckedSub}, + ArithmeticError, +}; +use sp_runtime::{DispatchResult, TokenError}; + +use crate::{ensure, traits::tokens::Fortitude}; /// Trait for inspecting a fungible asset which can be frozen. Freezing is essentially setting a /// minimum balance bellow which the total balance (inclusive of any funds placed on hold) may not @@ -65,4 +71,53 @@ pub trait Mutate: Inspect { /// Remove an existing lock. fn thaw(id: &Self::Id, who: &AccountId) -> DispatchResult; + + /// Attempt to alter the amount frozen under the given `id` to `amount`. + /// + /// Fail if the account of `who` has fewer freezable funds than `amount`, unless `fortitude` is + /// `Fortitude::Force`. + fn set_frozen( + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + fortitude: Fortitude, + ) -> DispatchResult { + let force = fortitude == Fortitude::Force; + ensure!(force || Self::balance_freezable(who) >= amount, TokenError::FundsUnavailable); + Self::set_freeze(id, who, amount) + } + + /// Attempt to set the amount frozen under the given `id` to `amount`, iff this would increase + /// the amount frozen under `id`. Do nothing otherwise. + /// + /// Fail if the account of `who` has fewer freezable funds than `amount`, unless `fortitude` is + /// `Fortitude::Force`. + fn ensure_frozen( + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + fortitude: Fortitude, + ) -> DispatchResult { + let force = fortitude == Fortitude::Force; + ensure!(force || Self::balance_freezable(who) >= amount, TokenError::FundsUnavailable); + Self::extend_freeze(id, who, amount) + } + + /// Decrease the amount which is being frozen for a particular freeze, failing in the case of + /// underflow. + fn decrease_frozen(id: &Self::Id, who: &AccountId, amount: Self::Balance) -> DispatchResult { + let a = Self::balance_frozen(id, who) + .checked_sub(&amount) + .ok_or(ArithmeticError::Underflow)?; + Self::set_freeze(id, who, a) + } + + /// Increase the amount which is being frozen for a particular freeze, failing in the case that + /// too little balance is available for being frozen. + fn increase_frozen(id: &Self::Id, who: &AccountId, amount: Self::Balance) -> DispatchResult { + let a = Self::balance_frozen(id, who) + .checked_add(&amount) + .ok_or(ArithmeticError::Overflow)?; + Self::set_frozen(id, who, a, Fortitude::Polite) + } } diff --git a/substrate/frame/support/src/traits/tokens/fungible/hold.rs b/substrate/frame/support/src/traits/tokens/fungible/hold.rs index 15c046fbe9a2e99364b990dacc1f6fd8942457cf..6da652d2998dfc18dd7da5754da7faf22abab839 100644 --- a/substrate/frame/support/src/traits/tokens/fungible/hold.rs +++ b/substrate/frame/support/src/traits/tokens/fungible/hold.rs @@ -92,11 +92,11 @@ pub trait Inspect: super::Inspect { who: &AccountId, amount: Self::Balance, ) -> DispatchResult { - ensure!(Self::hold_available(reason, who), TokenError::CannotCreateHold); ensure!( amount <= Self::reducible_balance(who, Protect, Force), TokenError::FundsUnavailable ); + ensure!(Self::hold_available(reason, who), TokenError::CannotCreateHold); Ok(()) } @@ -242,6 +242,41 @@ pub trait Mutate: Ok(actual) } + /// Hold or release funds in the account of `who` to bring the balance on hold for `reason` to + /// exactly `amount`. + fn set_on_hold( + reason: &Self::Reason, + who: &AccountId, + amount: Self::Balance, + ) -> DispatchResult { + let current_amount = Self::balance_on_hold(reason, who); + if current_amount < amount { + Self::hold(reason, who, amount - current_amount) + } else if current_amount > amount { + Self::release(reason, who, current_amount - amount, Precision::Exact).map(|_| ()) + } else { + Ok(()) + } + } + + /// Release all funds in the account of `who` on hold for `reason`. + /// + /// The actual amount released is returned with `Ok`. + /// + /// If `precision` is `BestEffort`, then the amount actually unreserved and returned as the + /// inner value of `Ok` may be smaller than the `amount` passed. + /// + /// NOTE! The inner of the `Ok` result variant returns the *actual* amount released. This is the + /// opposite of the `ReservableCurrency::unreserve()` result, which gives the amount not able + /// to be released! + fn release_all( + reason: &Self::Reason, + who: &AccountId, + precision: Precision, + ) -> Result { + Self::release(reason, who, Self::balance_on_hold(reason, who), precision) + } + /// Attempt to decrease the balance of `who` which is held for the given `reason` by `amount`. /// /// If `precision` is `BestEffort`, then as much as possible is reduced, up to `amount`, and the @@ -271,6 +306,25 @@ pub trait Mutate: Ok(amount) } + /// Attempt to decrease the balance of `who` which is held for the given `reason` to zero. + /// + /// If `precision` is `BestEffort`, then as much as possible is reduced, up to `amount`, and the + /// amount of tokens reduced is returned. Otherwise, if the total amount can be reduced, then it + /// is and the amount returned, and if not, then nothing changes and `Err` is returned. + /// + /// If `force` is `Force`, then locks/freezes will be ignored. This should only be used when + /// conducting slashing or other activity which materially disadvantages the account holder + /// since it could provide a means of circumventing freezes. + fn burn_all_held( + reason: &Self::Reason, + who: &AccountId, + precision: Precision, + force: Fortitude, + ) -> Result { + let amount = Self::balance_on_hold(reason, who); + Self::burn_held(reason, who, amount, precision, force) + } + /// Transfer held funds into a destination account. /// /// If `mode` is `OnHold`, then the destination account must already exist and the assets diff --git a/substrate/frame/support/src/traits/tokens/fungible/mod.rs b/substrate/frame/support/src/traits/tokens/fungible/mod.rs index 2205fd5dc589b0f0d85cb9542bc8f79af1a954d1..61b75fd6563c84659ca19adeb39217feb91524a8 100644 --- a/substrate/frame/support/src/traits/tokens/fungible/mod.rs +++ b/substrate/frame/support/src/traits/tokens/fungible/mod.rs @@ -45,6 +45,15 @@ mod imbalance; mod item_of; mod regular; +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_support_procedural::{CloneNoBound, EqNoBound, PartialEqNoBound, RuntimeDebugNoBound}; +use scale_info::TypeInfo; +use sp_std::marker::PhantomData; + +use super::{ + Fortitude::{Force, Polite}, + Precision::BestEffort, +}; pub use freeze::{Inspect as InspectFreeze, Mutate as MutateFreeze}; pub use hold::{ Balanced as BalancedHold, Inspect as InspectHold, Mutate as MutateHold, @@ -55,3 +64,182 @@ pub use item_of::ItemOf; pub use regular::{ Balanced, DecreaseIssuance, Dust, IncreaseIssuance, Inspect, Mutate, Unbalanced, }; +use sp_arithmetic::traits::Zero; +use sp_core::Get; +use sp_runtime::{traits::Convert, DispatchError}; + +use crate::{ + ensure, + traits::{Consideration, Footprint}, +}; + +/// Consideration method using a `fungible` balance frozen as the cost exacted for the footprint. +/// +/// The aggregate amount frozen under `R::get()` for any account which has multiple tickets, +/// is the *cumulative* amounts of each ticket's footprint (each individually determined by `D`). +#[derive( + CloneNoBound, + EqNoBound, + PartialEqNoBound, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, + RuntimeDebugNoBound, +)] +#[scale_info(skip_type_params(A, F, R, D))] +#[codec(mel_bound())] +pub struct FreezeConsideration(F::Balance, PhantomData (A, R, D)>) +where + F: MutateFreeze; +impl< + A: 'static, + F: 'static + MutateFreeze, + R: 'static + Get, + D: 'static + Convert, + > Consideration for FreezeConsideration +{ + fn new(who: &A, footprint: Footprint) -> Result { + let new = D::convert(footprint); + F::increase_frozen(&R::get(), who, new)?; + Ok(Self(new, PhantomData)) + } + fn update(self, who: &A, footprint: Footprint) -> Result { + let new = D::convert(footprint); + if self.0 > new { + F::decrease_frozen(&R::get(), who, self.0 - new)?; + } else if new > self.0 { + F::increase_frozen(&R::get(), who, new - self.0)?; + } + Ok(Self(new, PhantomData)) + } + fn drop(self, who: &A) -> Result<(), DispatchError> { + F::decrease_frozen(&R::get(), who, self.0).map(|_| ()) + } +} + +/// Consideration method using a `fungible` balance frozen as the cost exacted for the footprint. +#[derive( + CloneNoBound, + EqNoBound, + PartialEqNoBound, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, + RuntimeDebugNoBound, +)] +#[scale_info(skip_type_params(A, F, R, D))] +#[codec(mel_bound())] +pub struct HoldConsideration(F::Balance, PhantomData (A, R, D)>) +where + F: MutateHold; +impl< + A: 'static, + F: 'static + MutateHold, + R: 'static + Get, + D: 'static + Convert, + > Consideration for HoldConsideration +{ + fn new(who: &A, footprint: Footprint) -> Result { + let new = D::convert(footprint); + F::hold(&R::get(), who, new)?; + Ok(Self(new, PhantomData)) + } + fn update(self, who: &A, footprint: Footprint) -> Result { + let new = D::convert(footprint); + if self.0 > new { + F::release(&R::get(), who, self.0 - new, BestEffort)?; + } else if new > self.0 { + F::hold(&R::get(), who, new - self.0)?; + } + Ok(Self(new, PhantomData)) + } + fn drop(self, who: &A) -> Result<(), DispatchError> { + F::release(&R::get(), who, self.0, BestEffort).map(|_| ()) + } + fn burn(self, who: &A) { + let _ = F::burn_held(&R::get(), who, self.0, BestEffort, Force); + } +} + +/// Basic consideration method using a `fungible` balance frozen as the cost exacted for the +/// footprint. +/// +/// NOTE: This is an optimized implementation, which can only be used for systems where each +/// account has only a single active ticket associated with it since individual tickets do not +/// track the specific balance which is frozen. If you are uncertain then use `FreezeConsideration` +/// instead, since this works in all circumstances. +#[derive( + CloneNoBound, + EqNoBound, + PartialEqNoBound, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, + RuntimeDebugNoBound, +)] +#[scale_info(skip_type_params(A, Fx, Rx, D))] +#[codec(mel_bound())] +pub struct LoneFreezeConsideration(PhantomData (A, Fx, Rx, D)>); +impl< + A: 'static, + Fx: 'static + MutateFreeze, + Rx: 'static + Get, + D: 'static + Convert, + > Consideration for LoneFreezeConsideration +{ + fn new(who: &A, footprint: Footprint) -> Result { + ensure!(Fx::balance_frozen(&Rx::get(), who).is_zero(), DispatchError::Unavailable); + Fx::set_frozen(&Rx::get(), who, D::convert(footprint), Polite).map(|_| Self(PhantomData)) + } + fn update(self, who: &A, footprint: Footprint) -> Result { + Fx::set_frozen(&Rx::get(), who, D::convert(footprint), Polite).map(|_| Self(PhantomData)) + } + fn drop(self, who: &A) -> Result<(), DispatchError> { + Fx::thaw(&Rx::get(), who).map(|_| ()) + } +} + +/// Basic consideration method using a `fungible` balance placed on hold as the cost exacted for the +/// footprint. +/// +/// NOTE: This is an optimized implementation, which can only be used for systems where each +/// account has only a single active ticket associated with it since individual tickets do not +/// track the specific balance which is frozen. If you are uncertain then use `FreezeConsideration` +/// instead, since this works in all circumstances. +#[derive( + CloneNoBound, + EqNoBound, + PartialEqNoBound, + Encode, + Decode, + TypeInfo, + MaxEncodedLen, + RuntimeDebugNoBound, +)] +#[scale_info(skip_type_params(A, Fx, Rx, D))] +#[codec(mel_bound())] +pub struct LoneHoldConsideration(PhantomData (A, Fx, Rx, D)>); +impl< + A: 'static, + F: 'static + MutateHold, + R: 'static + Get, + D: 'static + Convert, + > Consideration for LoneHoldConsideration +{ + fn new(who: &A, footprint: Footprint) -> Result { + ensure!(F::balance_on_hold(&R::get(), who).is_zero(), DispatchError::Unavailable); + F::set_on_hold(&R::get(), who, D::convert(footprint)).map(|_| Self(PhantomData)) + } + fn update(self, who: &A, footprint: Footprint) -> Result { + F::set_on_hold(&R::get(), who, D::convert(footprint)).map(|_| Self(PhantomData)) + } + fn drop(self, who: &A) -> Result<(), DispatchError> { + F::release_all(&R::get(), who, BestEffort).map(|_| ()) + } + fn burn(self, who: &A) { + let _ = F::burn_all_held(&R::get(), who, BestEffort, Force); + } +} diff --git a/substrate/frame/support/src/traits/tokens/fungibles/freeze.rs b/substrate/frame/support/src/traits/tokens/fungibles/freeze.rs index 88f083b0a210a7b82f1999329b10143c41c7c598..b07d20d6c41ce48c3f4face0ea4a213a3023f5e3 100644 --- a/substrate/frame/support/src/traits/tokens/fungibles/freeze.rs +++ b/substrate/frame/support/src/traits/tokens/fungibles/freeze.rs @@ -17,8 +17,13 @@ //! The traits for putting freezes within a single fungible token class. +use crate::{ensure, traits::tokens::Fortitude}; use scale_info::TypeInfo; -use sp_runtime::DispatchResult; +use sp_arithmetic::{ + traits::{CheckedAdd, CheckedSub}, + ArithmeticError, +}; +use sp_runtime::{DispatchResult, TokenError}; /// Trait for inspecting a fungible asset which can be frozen. Freezing is essentially setting a /// minimum balance below which the total balance (inclusive of any funds placed on hold) may not @@ -75,4 +80,71 @@ pub trait Mutate: Inspect { /// Remove an existing lock. fn thaw(asset: Self::AssetId, id: &Self::Id, who: &AccountId) -> DispatchResult; + + /// Attempt to alter the amount frozen under the given `id` to `amount`. + /// + /// Fail if the account of `who` has fewer freezable funds than `amount`, unless `fortitude` is + /// `Fortitude::Force`. + fn set_frozen( + asset: Self::AssetId, + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + fortitude: Fortitude, + ) -> DispatchResult { + let force = fortitude == Fortitude::Force; + ensure!( + force || Self::balance_freezable(asset.clone(), who) >= amount, + TokenError::FundsUnavailable + ); + Self::set_freeze(asset, id, who, amount) + } + + /// Attempt to set the amount frozen under the given `id` to `amount`, iff this would increase + /// the amount frozen under `id`. Do nothing otherwise. + /// + /// Fail if the account of `who` has fewer freezable funds than `amount`, unless `fortitude` is + /// `Fortitude::Force`. + fn ensure_frozen( + asset: Self::AssetId, + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + fortitude: Fortitude, + ) -> DispatchResult { + let force = fortitude == Fortitude::Force; + ensure!( + force || Self::balance_freezable(asset.clone(), who) >= amount, + TokenError::FundsUnavailable + ); + Self::extend_freeze(asset, id, who, amount) + } + + /// Decrease the amount which is being frozen for a particular lock, failing in the case of + /// underflow. + fn decrease_frozen( + asset: Self::AssetId, + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + ) -> DispatchResult { + let a = Self::balance_frozen(asset.clone(), id, who) + .checked_sub(&amount) + .ok_or(ArithmeticError::Underflow)?; + Self::set_frozen(asset, id, who, a, Fortitude::Polite) + } + + /// Increase the amount which is being frozen for a particular lock, failing in the case that + /// too little balance is available for being frozen. + fn increase_frozen( + asset: Self::AssetId, + id: &Self::Id, + who: &AccountId, + amount: Self::Balance, + ) -> DispatchResult { + let a = Self::balance_frozen(asset.clone(), id, who) + .checked_add(&amount) + .ok_or(ArithmeticError::Overflow)?; + Self::set_frozen(asset, id, who, a, Fortitude::Polite) + } } diff --git a/substrate/frame/support/src/traits/tokens/fungibles/hold.rs b/substrate/frame/support/src/traits/tokens/fungibles/hold.rs index 4f86704df1685227c865b4c7cd1792ef8ca9cb99..1efd1594213c1a80eeccdd4fdf0c9f01c812794c 100644 --- a/substrate/frame/support/src/traits/tokens/fungibles/hold.rs +++ b/substrate/frame/support/src/traits/tokens/fungibles/hold.rs @@ -301,7 +301,8 @@ pub trait Mutate: Ok(actual) } - /// Attempt to decrease the balance of `who` which is held for the given `reason` by `amount`. + /// Attempt to decrease the `asset` balance of `who` which is held for the given `reason` by + /// `amount`. /// /// If `precision` is true, then as much as possible is reduced, up to `amount`, and the /// amount of tokens reduced is returned. Otherwise, if the total amount can be reduced, then it @@ -334,6 +335,27 @@ pub trait Mutate: Ok(amount) } + /// Attempt to decrease the `asset` balance of `who` which is held for the given `reason` to + /// zero. + /// + /// If `precision` is `BestEffort`, then as much as possible is reduced, up to `amount`, and the + /// amount of tokens reduced is returned. Otherwise, if the total amount can be reduced, then it + /// is and the amount returned, and if not, then nothing changes and `Err` is returned. + /// + /// If `force` is `Force`, then locks/freezes will be ignored. This should only be used when + /// conducting slashing or other activity which materially disadvantages the account holder + /// since it could provide a means of circumventing freezes. + fn burn_all_held( + asset: Self::AssetId, + reason: &Self::Reason, + who: &AccountId, + precision: Precision, + force: Fortitude, + ) -> Result { + let amount = Self::balance_on_hold(asset.clone(), reason, who); + Self::burn_held(asset, reason, who, amount, precision, force) + } + /// Transfer held funds into a destination account. /// /// If `mode` is `OnHold`, then the destination account must already exist and the assets diff --git a/substrate/frame/support/src/traits/tokens/misc.rs b/substrate/frame/support/src/traits/tokens/misc.rs index baf3fd5f354646153679d81b6b3349762f62b5c0..84bbe3e8d9c838fac2d0ec81e9a283955166bfd3 100644 --- a/substrate/frame/support/src/traits/tokens/misc.rs +++ b/substrate/frame/support/src/traits/tokens/misc.rs @@ -231,7 +231,16 @@ impl Balance for T { } diff --git a/substrate/frame/support/test/tests/pallet.rs b/substrate/frame/support/test/tests/pallet.rs index 88aad32225a0bbc47987bbc587abfa2036911ada..1898246470c73fee4bd046d737d90580015989b1 100644 --- a/substrate/frame/support/test/tests/pallet.rs +++ b/substrate/frame/support/test/tests/pallet.rs @@ -1386,7 +1386,7 @@ fn metadata() { } } - let readme = "Support code for the runtime.\n\nLicense: Apache-2.0"; + let readme = "Support code for the runtime.\n\nLicense: Apache-2.0\n"; let expected_pallet_doc = vec![" Pallet documentation", readme, readme]; let pallets = vec![ @@ -1889,7 +1889,7 @@ fn metadata_ir_pallet_runtime_docs() { .find(|pallet| pallet.name == "Example") .expect("Pallet should be present"); - let readme = "Support code for the runtime.\n\nLicense: Apache-2.0"; + let readme = "Support code for the runtime.\n\nLicense: Apache-2.0\n"; let expected = vec![" Pallet documentation", readme, readme]; assert_eq!(pallet.docs, expected); } @@ -1919,7 +1919,7 @@ fn extrinsic_metadata_ir_types() { #[test] fn test_pallet_runtime_docs() { let docs = crate::pallet::Pallet::::pallet_documentation_metadata(); - let readme = "Support code for the runtime.\n\nLicense: Apache-2.0"; + let readme = "Support code for the runtime.\n\nLicense: Apache-2.0\n"; let expected = vec![" Pallet documentation", readme, readme]; assert_eq!(docs, expected); } diff --git a/substrate/frame/support/test/tests/versioned_migration.rs b/substrate/frame/support/test/tests/versioned_migration.rs index a0766f6bec254b7bf5d5d38b82b2f6d3ff67b211..03dbd948df92a73f0567338e575000fca11ceb8d 100644 --- a/substrate/frame/support/test/tests/versioned_migration.rs +++ b/substrate/frame/support/test/tests/versioned_migration.rs @@ -17,7 +17,7 @@ //! Tests for [`VersionedMigration`] -#![cfg(all(feature = "experimental", feature = "try-runtime"))] +#![cfg(feature = "try-runtime")] use frame_support::{ construct_runtime, derive_impl, diff --git a/substrate/frame/system/README.md b/substrate/frame/system/README.md index 30b2ea73720cf1fa7af62f5cd82c32fda3f0e3ac..c15281d365ba7482de57e6ce372f19b2861322bb 100644 --- a/substrate/frame/system/README.md +++ b/substrate/frame/system/README.md @@ -1,20 +1,20 @@ # System Module -The System module provides low-level access to core types and cross-cutting utilities. -It acts as the base layer for other pallets to interact with the Substrate framework components. +The System module provides low-level access to core types and cross-cutting utilities. It acts as the base layer for +other pallets to interact with the Substrate framework components. - [`system::Config`](https://docs.rs/frame-system/latest/frame_system/pallet/trait.Config.html) ## Overview -The System module defines the core data types used in a Substrate runtime. -It also provides several utility functions (see [`Pallet`](https://docs.rs/frame-system/latest/frame_system/pallet/struct.Pallet.html)) for other FRAME pallets. +The System module defines the core data types used in a Substrate runtime. It also provides several utility functions +(see [`Pallet`](https://docs.rs/frame-system/latest/frame_system/pallet/struct.Pallet.html)) for other FRAME pallets. -In addition, it manages the storage items for extrinsics data, indexes, event records, and digest items, -among other things that support the execution of the current block. +In addition, it manages the storage items for extrinsics data, indexes, event records, and digest items, among other +things that support the execution of the current block. -It also handles low-level tasks like depositing logs, basic set up and take down of -temporary storage entries, and access to previous block hashes. +It also handles low-level tasks like depositing logs, basic set up and take down of temporary storage entries, and +access to previous block hashes. ## Interface @@ -24,26 +24,22 @@ The System module does not implement any dispatchable functions. ### Public Functions -See the [`Pallet`](https://docs.rs/frame-system/latest/frame_system/pallet/struct.Pallet.html) struct for details of publicly available functions. +See the [`Pallet`](https://docs.rs/frame-system/latest/frame_system/pallet/struct.Pallet.html) struct for details of +publicly available functions. ### Signed Extensions The System module defines the following extensions: - - [`CheckWeight`]: Checks the weight and length of the block and ensure that it does not - exceed the limits. - - [`CheckNonce`]: Checks the nonce of the transaction. Contains a single payload of type - `T::Nonce`. + - [`CheckWeight`]: Checks the weight and length of the block and ensure that it does not exceed the limits. + - [`CheckNonce`]: Checks the nonce of the transaction. Contains a single payload of type `T::Nonce`. - [`CheckEra`]: Checks the era of the transaction. Contains a single payload of type `Era`. - - [`CheckGenesis`]: Checks the provided genesis hash of the transaction. Must be a part of the - signed payload of the transaction. - - [`CheckSpecVersion`]: Checks that the runtime version is the same as the one used to sign the - transaction. - - [`CheckTxVersion`]: Checks that the transaction version is the same as the one used to sign the + - [`CheckGenesis`]: Checks the provided genesis hash of the transaction. Must be a part of the signed payload of the transaction. + - [`CheckSpecVersion`]: Checks that the runtime version is the same as the one used to sign the transaction. + - [`CheckTxVersion`]: Checks that the transaction version is the same as the one used to sign the transaction. -Lookup the runtime aggregator file (e.g. `node/runtime`) to see the full list of signed -extensions included in a chain. +Lookup the runtime aggregator file (e.g. `node/runtime`) to see the full list of signed extensions included in a chain. ## Usage diff --git a/substrate/frame/system/benchmarking/README.md b/substrate/frame/system/benchmarking/README.md index 9718db58b37e9ae1c81b5b627fe27e51129cf418..811207fd8c0f916f31d08d27fc53499572b6bef4 100644 --- a/substrate/frame/system/benchmarking/README.md +++ b/substrate/frame/system/benchmarking/README.md @@ -1 +1 @@ -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/system/benchmarking/res/README.md b/substrate/frame/system/benchmarking/res/README.md index 43bb2b5c283ef895211ed06d8e17fdcb651ab02f..6eca9229531a34c76ffcc12879518200a6cc9a4b 100644 --- a/substrate/frame/system/benchmarking/res/README.md +++ b/substrate/frame/system/benchmarking/res/README.md @@ -2,4 +2,5 @@ These runtimes are used for benchmarking the `set_code` intrinsic. **Don't use them in production environments!** -To update the just copy the new runtime from `target/release/wbuild/kitchensink-runtime/kitchensink_runtime.compact.compressed.wasm` to here. +To update the just copy the new runtime from +`target/release/wbuild/kitchensink-runtime/kitchensink_runtime.compact.compressed.wasm` to here. diff --git a/substrate/frame/system/rpc/runtime-api/README.md b/substrate/frame/system/rpc/runtime-api/README.md index ab46c22a8be3345b13dc4674f3234cb8f511851b..d418cad5a342e6bcf634def7dde600f7f6646b92 100644 --- a/substrate/frame/system/rpc/runtime-api/README.md +++ b/substrate/frame/system/rpc/runtime-api/README.md @@ -4,4 +4,4 @@ This API should be imported and implemented by the runtime, of a node that wants to use the custom RPC extension adding System access methods. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/frame/timestamp/Cargo.toml b/substrate/frame/timestamp/Cargo.toml index a39c79892d19add53bd4649c05f7319161b88ce4..291f6b1cf590ac0bdb751b8d5af9b6c0e22ab0e3 100644 --- a/substrate/frame/timestamp/Cargo.toml +++ b/substrate/frame/timestamp/Cargo.toml @@ -27,6 +27,8 @@ sp-std = { path = "../../primitives/std", default-features = false} sp-storage = { path = "../../primitives/storage", default-features = false} sp-timestamp = { path = "../../primitives/timestamp", default-features = false} +docify = "0.2.4" + [dev-dependencies] sp-core = { path = "../../primitives/core" } sp-io = { path = "../../primitives/io" } diff --git a/substrate/frame/timestamp/README.md b/substrate/frame/timestamp/README.md index 1546377ee67432bd2b2b42556aba096fc224185e..69dba60550e4b83df0719a274a31dd3be2efe6b3 100644 --- a/substrate/frame/timestamp/README.md +++ b/substrate/frame/timestamp/README.md @@ -22,16 +22,16 @@ because of cumulative calculation errors and hence should be avoided. ### Dispatchable Functions -* `set` - Sets the current time. +- `set` - Sets the current time. ### Public functions -* `get` - Gets the current time for the current block. If this function is called prior to +- `get` - Gets the current time for the current block. If this function is called prior to setting the timestamp, it will return the timestamp of the previous block. ### Config Getters -* `MinimumPeriod` - Gets the minimum (and advised) period between blocks for the chain. +- `MinimumPeriod` - Gets the minimum (and advised) period between blocks for the chain. ## Usage @@ -78,6 +78,6 @@ the Timestamp module for session management. ## Related Modules -* [Session](https://docs.rs/pallet-session/latest/pallet_session/) +- [Session](https://docs.rs/pallet-session/latest/pallet_session/) License: Apache-2.0 diff --git a/substrate/frame/timestamp/src/lib.rs b/substrate/frame/timestamp/src/lib.rs index 4eb95941d782881778c493ea28f88492438c8be4..ad055bab004f99a49710a2e479a54910b249cc00 100644 --- a/substrate/frame/timestamp/src/lib.rs +++ b/substrate/frame/timestamp/src/lib.rs @@ -15,53 +15,40 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Timestamp Pallet -//! -//! The Timestamp pallet provides functionality to get and set the on-chain time. -//! -//! - [`Config`] -//! - [`Call`] -//! - [`Pallet`] -//! -//! ## Overview -//! -//! The Timestamp pallet allows the validators to set and validate a timestamp with each block. -//! -//! It uses inherents for timestamp data, which is provided by the block author and -//! validated/verified by other validators. The timestamp can be set only once per block and must be -//! set each block. There could be a constraint on how much time must pass before setting the new -//! timestamp. +//! > Made with *Substrate*, for *Polkadot*. //! -//! **NOTE:** The Timestamp pallet is the recommended way to query the on-chain time instead of -//! using an approach based on block numbers. The block number based time measurement can cause -//! issues because of cumulative calculation errors and hence should be avoided. +//! [![github]](https://github.com/paritytech/polkadot-sdk/substrate/frame/timestamp) +//! [![polkadot]](https://polkadot.network) //! -//! ## Interface +//! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github //! -//! ### Dispatchable Functions -//! -//! * `set` - Sets the current time. -//! -//! ### Public functions +//! # Timestamp Pallet //! -//! * `get` - Gets the current time for the current block. If this function is called prior to -//! setting the timestamp, it will return the timestamp of the previous block. +//! A pallet that provides a way for consensus systems to set and check the onchain time. //! -//! ### Config Getters +//! ## Pallet API //! -//! * `MinimumPeriod` - Gets the minimum (and advised) period between blocks for the chain. +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. +//! +//! ## Overview //! -//! ## Usage +//! The Timestamp pallet is designed to create a consensus-based time source. This helps ensure that +//! nodes maintain a synchronized view of time that all network participants can agree on. //! -//! The following example shows how to use the Timestamp pallet in your custom pallet to query the -//! current timestamp. +//! It defines an _acceptable range_ using a configurable constant to specify how much time must +//! pass before setting the new timestamp. Validator nodes in the network must verify that the +//! timestamp falls within this acceptable range and reject blocks that do not. //! -//! ### Prerequisites +//! > **Note:** The timestamp set by this pallet is the recommended way to query the onchain time +//! > instead of using block numbers alone. Measuring time with block numbers can cause cumulative +//! > calculation errors if depended upon in time critical operations and hence should generally be +//! > avoided. //! -//! Import the Timestamp pallet into your custom pallet and derive the pallet configuration -//! trait from the timestamp trait. +//! ## Example //! -//! ### Get current timestamp +//! To get the current time for the current block in another pallet: //! //! ``` //! use pallet_timestamp::{self as timestamp}; @@ -83,7 +70,7 @@ //! #[pallet::weight(0)] //! pub fn get_time(origin: OriginFor) -> DispatchResult { //! let _sender = ensure_signed(origin)?; -//! let _now = >::get(); +//! let _now = timestamp::Pallet::::get(); //! Ok(()) //! } //! } @@ -91,15 +78,52 @@ //! # fn main() {} //! ``` //! -//! ### Example from the FRAME +//! If [`Pallet::get`] is called prior to setting the timestamp, it will return the timestamp of +//! the previous block. //! -//! The [Session pallet](https://github.com/paritytech/substrate/blob/master/frame/session/src/lib.rs) uses -//! the Timestamp pallet for session management. +//! ## Low Level / Implementation Details //! -//! ## Related Pallets +//! A timestamp is added to the chain using an _inherent extrinsic_ that only a block author can +//! submit. Inherents are a special type of extrinsic in Substrate chains that will always be +//! included in a block. //! -//! * [Session](../pallet_session/index.html) - +//! To provide inherent data to the runtime, this pallet implements +//! [`ProvideInherent`](frame_support::inherent::ProvideInherent). It will only create an inherent +//! if the [`Call::set`] dispatchable is called, using the +//! [`inherent`](frame_support::pallet_macros::inherent) macro which enables validator nodes to call +//! into the runtime to check that the timestamp provided is valid. +//! The implementation of [`ProvideInherent`](frame_support::inherent::ProvideInherent) specifies a +//! constant called `MAX_TIMESTAMP_DRIFT_MILLIS` which is used to determine the acceptable range for +//! a valid timestamp. If a block author sets a timestamp to anything that is more than this +//! constant, a validator node will reject the block. +//! +//! The pallet also ensures that a timestamp is set at the start of each block by running an +//! assertion in the `on_finalize` runtime hook. See [`frame_support::traits::Hooks`] for more +//! information about how hooks work. +//! +//! Because inherents are applied to a block in the order they appear in the runtime +//! construction, the index of this pallet in +//! [`construct_runtime`](frame_support::construct_runtime) must always be less than any other +//! pallet that depends on it. +//! +//! The [`Config::OnTimestampSet`] configuration trait can be set to another pallet we want to +//! notify that the timestamp has been updated, as long as it implements [`OnTimestampSet`]. +//! Examples are the Babe and Aura pallets. +//! This pallet also implements [`Time`] and [`UnixTime`] so it can be used to configure other +//! pallets that require these types (e.g. in Staking pallet). +//! +//! ## Panics +//! +//! There are 3 cases where this pallet could cause the runtime to panic. +//! +//! 1. If no timestamp is set at the end of a block. +//! +//! 2. If a timestamp is set more than once per block: +#![doc = docify::embed!("src/tests.rs", double_timestamp_should_fail)] +//! +//! 3. If a timestamp is set before the [`Config::MinimumPeriod`] is elapsed: +#![doc = docify::embed!("src/tests.rs", block_period_minimum_enforced)] +#![deny(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] mod benchmarking; @@ -123,10 +147,9 @@ pub mod pallet { use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; - /// The pallet configuration trait #[pallet::config] pub trait Config: frame_system::Config { - /// Type used for expressing timestamp. + /// Type used for expressing a timestamp. type Moment: Parameter + Default + AtLeast32Bit @@ -135,14 +158,17 @@ pub mod pallet { + MaxEncodedLen + scale_info::StaticTypeInfo; - /// Something which can be notified when the timestamp is set. Set this to `()` if not - /// needed. + /// Something which can be notified (e.g. another pallet) when the timestamp is set. + /// + /// This can be set to `()` if it is not needed. type OnTimestampSet: OnTimestampSet; - /// The minimum period between blocks. Beware that this is different to the *expected* - /// period that the block production apparatus provides. Your chosen consensus system will - /// generally work with this to determine a sensible block time. e.g. For Aura, it will be - /// double this period on default settings. + /// The minimum period between blocks. + /// + /// Be aware that this is different to the *expected* period that the block production + /// apparatus provides. Your chosen consensus system will generally work with this to + /// determine a sensible block time. For example, in the Aura pallet it will be double this + /// period on default settings. #[pallet::constant] type MinimumPeriod: Get; @@ -153,23 +179,31 @@ pub mod pallet { #[pallet::pallet] pub struct Pallet(_); - /// Current time for the current block. + /// The current time for the current block. #[pallet::storage] #[pallet::getter(fn now)] pub type Now = StorageValue<_, T::Moment, ValueQuery>; - /// Did the timestamp get updated in this block? + /// Whether the timestamp has been updated in this block. + /// + /// This value is updated to `true` upon successful submission of a timestamp by a node. + /// It is then checked at the end of each block execution in the `on_finalize` hook. #[pallet::storage] pub(super) type DidUpdate = StorageValue<_, bool, ValueQuery>; #[pallet::hooks] impl Hooks> for Pallet { - /// dummy `on_initialize` to return the weight used in `on_finalize`. + /// A dummy `on_initialize` to return the amount of weight that `on_finalize` requires to + /// execute. fn on_initialize(_n: BlockNumberFor) -> Weight { // weight of `on_finalize` T::WeightInfo::on_finalize() } + /// At the end of block execution, the `on_finalize` hook checks that the timestamp was + /// updated. Upon success, it removes the boolean value from storage. If the value resolves + /// to `false`, the pallet will panic. + /// /// ## Complexity /// - `O(1)` fn on_finalize(_n: BlockNumberFor) { @@ -185,13 +219,17 @@ pub mod pallet { /// phase, if this call hasn't been invoked by that time. /// /// The timestamp should be greater than the previous one by the amount specified by - /// `MinimumPeriod`. + /// [`Config::MinimumPeriod`]. + /// + /// The dispatch origin for this call must be _None_. /// - /// The dispatch origin for this call must be `Inherent`. + /// This dispatch class is _Mandatory_ to ensure it gets executed in the block. Be aware + /// that changing the complexity of this call could result exhausting the resources in a + /// block to execute any other calls. /// /// ## Complexity /// - `O(1)` (Note that implementations of `OnTimestampSet` must also be `O(1)`) - /// - 1 storage read and 1 storage mutation (codec `O(1)`). (because of `DidUpdate::take` in + /// - 1 storage read and 1 storage mutation (codec `O(1)` because of `DidUpdate::take` in /// `on_finalize`) /// - 1 event handler `on_timestamp_set`. Must be `O(1)`. #[pallet::call_index(0)] @@ -216,6 +254,14 @@ pub mod pallet { } } + /// To check the inherent is valid, we simply take the max value between the current timestamp + /// and the current timestamp plus the [`Config::MinimumPeriod`]. + /// We also check that the timestamp has not already been set in this block. + /// + /// ## Errors: + /// - [`InherentError::TooFarInFuture`]: If the timestamp is larger than the current timestamp + + /// minimum drift period. + /// - [`InherentError::TooEarly`]: If the timestamp is less than the current + minimum period. #[pallet::inherent] impl ProvideInherent for Pallet { type Call = Call; @@ -285,9 +331,9 @@ impl Pallet { } impl Time for Pallet { + /// A type that represents a unit of time. type Moment = T::Moment; - /// Before the first set of now with inherent the value returned is zero. fn now() -> Self::Moment { Self::now() } diff --git a/substrate/frame/timestamp/src/tests.rs b/substrate/frame/timestamp/src/tests.rs index 317631eeb704843140db261925d7ed109c614211..cc49d8a3296e831ae01fd60ab28c0cf7fafbf9b5 100644 --- a/substrate/frame/timestamp/src/tests.rs +++ b/substrate/frame/timestamp/src/tests.rs @@ -30,6 +30,7 @@ fn timestamp_works() { }); } +#[docify::export] #[test] #[should_panic(expected = "Timestamp must be updated only once in the block")] fn double_timestamp_should_fail() { @@ -39,6 +40,7 @@ fn double_timestamp_should_fail() { }); } +#[docify::export] #[test] #[should_panic( expected = "Timestamp must increment by at least between sequential blocks" diff --git a/substrate/frame/tips/README.md b/substrate/frame/tips/README.md index d885ce770f79533c9b203f73bcf25458b644852b..1960172c497a7e8a894065bb7272936a24d7c0f6 100644 --- a/substrate/frame/tips/README.md +++ b/substrate/frame/tips/README.md @@ -11,7 +11,7 @@ entered where any remaining members can declare their tip amounts also. After th countdown period, the median of all declared tips is paid to the reported beneficiary, along with any finders fee, in case of a public (and bonded) original report. -### Terminology +## Terminology - **Tipping:** The process of gathering declarations of amounts to tip and taking the median amount to be transferred from the treasury to a beneficiary account. @@ -30,4 +30,4 @@ any finders fee, in case of a public (and bonded) original report. - `tip_new` - Report an item worthy of a tip and declare a specific amount to tip. - `tip` - Declare or redeclare an amount to tip for a particular reason. - `close_tip` - Close and pay out a tip. -- `slash_tip` - Remove and slash an already-open tip. \ No newline at end of file +- `slash_tip` - Remove and slash an already-open tip. diff --git a/substrate/frame/transaction-payment/Cargo.toml b/substrate/frame/transaction-payment/Cargo.toml index 1db63bd134ad87aeae6f1243d6dc19d21bb2db2c..5c65ebd4c730a4dfd2195cd39cf7c9a55d32af31 100644 --- a/substrate/frame/transaction-payment/Cargo.toml +++ b/substrate/frame/transaction-payment/Cargo.toml @@ -26,7 +26,7 @@ sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" pallet-balances = { path = "../balances" } [features] diff --git a/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml b/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml index edf05a0533178c0dfc436c3ff2919023902a837f..d3f040e9893fab35d397ddabbfc0eeea957436ab 100644 --- a/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml +++ b/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml @@ -30,7 +30,7 @@ scale-info = { version = "2.5.0", default-features = false, features = ["derive" serde = { version = "1.0.188", optional = true } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" sp-storage = { path = "../../../primitives/storage", default-features = false} diff --git a/substrate/frame/transaction-storage/README.md b/substrate/frame/transaction-storage/README.md index 0ed3ba279c2a590c4a4f10e669b9fbb74d6947ef..1066968469d4b7c629ac7243b9e7e20febdbdbab 100644 --- a/substrate/frame/transaction-storage/README.md +++ b/substrate/frame/transaction-storage/README.md @@ -2,8 +2,9 @@ Indexes transactions and manages storage proofs. -Allows storing arbitrary data on the chain. Data is automatically removed after `StoragePeriod` blocks, unless the storage is renewed. -Validators must submit proof of storing a random chunk of data for block `N - StoragePeriod` when producing block `N`. +Allows storing arbitrary data on the chain. Data is automatically removed after `StoragePeriod` blocks, unless the +storage is renewed. Validators must submit proof of storing a random chunk of data for block `N - StoragePeriod` when +producing block `N`. # Running a chain @@ -15,8 +16,9 @@ Start with generating a chain spec. cargo run --release -- build-spec --chain=local > sc_init.json ``` -Edit the json chain spec file to customise the chain. The storage chain genesis params are configured in the `transactionStorage` section. -Note that `storagePeriod` is specified in blocks and changing it also requires code changes at the moment. +Edit the json chain spec file to customise the chain. The storage chain genesis params are configured in the +`transactionStorage` section. Note that `storagePeriod` is specified in blocks and changing it also requires code +changes at the moment. Build a raw spec from the init spec. @@ -31,11 +33,11 @@ cargo run --release -- --chain=sc.json -d /tmp/alice --storage-chain --keep-bloc cargo run --release -- --chain=sc.json -d /tmp/bob --storage-chain --keep-blocks=100800 --ipfs-server --validator --bob ``` -`--storage-chain` enables transaction indexing. -`--keep-blocks=100800` enables block pruning. The value here should be greater or equal than the storage period. -`--ipfs-server` enables serving stored content over IPFS. +`--storage-chain` enables transaction indexing. `--keep-blocks=100800` enables block pruning. The value here should be +greater or equal than the storage period. `--ipfs-server` enables serving stored content over IPFS. -Once the network is started, any other joining nodes need to sync with `--sync=fast`. Regular sync will fail because block pruning removes old blocks. The chain does not keep full block history. +Once the network is started, any other joining nodes need to sync with `--sync=fast`. Regular sync will fail because +block pruning removes old blocks. The chain does not keep full block history. ```bash cargo run --release -- --chain=sc.json -d /tmp/charlie --storage-chain --keep-blocks=100800 --ipfs-server --validator --charlie --sync=fast @@ -43,7 +45,8 @@ cargo run --release -- --chain=sc.json -d /tmp/charlie --storage-chain --keep-bl # Making transactions -To store data use the `transactionStorage.store` extrinsic. And IPFS CID can be generated from the Blake2-256 hash of the data. +To store data use the `transactionStorage.store` extrinsic. And IPFS CID can be generated from the Blake2-256 hash of +the data. ```js const util_crypto = require('@polkadot/util-crypto'); @@ -76,7 +79,8 @@ ipfs block get /ipfs/ > kitten.jpeg ``` To renew data and prevent it from being disposed after the storage period, use `transactionStorage.renew(block, index)` -where `block` is the block number of the previous store or renew transction, and index is the index of that transaction in the block. +where `block` is the block number of the previous store or renew transction, and index is the index of that transaction +in the block. License: Apache-2.0 diff --git a/substrate/frame/tx-pause/Cargo.toml b/substrate/frame/tx-pause/Cargo.toml index 356693d90f04f897c6b98e647e99dff3ed80533f..6d96cb8abe79924b6b23c2f509bc63f08a6bb631 100644 --- a/substrate/frame/tx-pause/Cargo.toml +++ b/substrate/frame/tx-pause/Cargo.toml @@ -7,7 +7,6 @@ license = "Apache-2.0" homepage = "https://substrate.io" repository.workspace = true description = "FRAME transaction pause pallet" -readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/frame/tx-pause/src/mock.rs b/substrate/frame/tx-pause/src/mock.rs index 706b0a558ba7eb07e5fbf4f11509fdc4282fef34..60c5fc1eced5c3b7e71eaac3a606d763ea57b175 100644 --- a/substrate/frame/tx-pause/src/mock.rs +++ b/substrate/frame/tx-pause/src/mock.rs @@ -120,7 +120,10 @@ impl InstanceFilter for ProxyType { match self { ProxyType::Any => true, ProxyType::JustTransfer => { - matches!(c, RuntimeCall::Balances(pallet_balances::Call::transfer { .. })) + matches!( + c, + RuntimeCall::Balances(pallet_balances::Call::transfer_allow_death { .. }) + ) }, ProxyType::JustUtility => matches!(c, RuntimeCall::Utility { .. }), } diff --git a/substrate/frame/tx-pause/src/tests.rs b/substrate/frame/tx-pause/src/tests.rs index 48b70f71ccb0837c0cff4ff0ffaa3ffd89b72323..a71ff3439d902b47036632f923f944e7dd68317d 100644 --- a/substrate/frame/tx-pause/src/tests.rs +++ b/substrate/frame/tx-pause/src/tests.rs @@ -32,7 +32,7 @@ fn can_pause_specific_call() { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer") + full_name::(b"Balances", b"transfer_allow_death") )); assert_err!( @@ -69,7 +69,7 @@ fn can_unpause_specific_call() { new_test_ext().execute_with(|| { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_err!( call_transfer(2, 1).dispatch(RuntimeOrigin::signed(2)), @@ -78,7 +78,7 @@ fn can_unpause_specific_call() { assert_ok!(TxPause::unpause( RuntimeOrigin::signed(mock::UnpauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_ok!(call_transfer(4, 1).dispatch(RuntimeOrigin::signed(0))); }); @@ -92,7 +92,7 @@ fn can_filter_balance_in_batch_when_paused() { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_ok!(batch_call.clone().dispatch(RuntimeOrigin::signed(0))); @@ -111,7 +111,7 @@ fn can_filter_balance_in_proxy_when_paused() { new_test_ext().execute_with(|| { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_ok!(Proxy::add_proxy(RuntimeOrigin::signed(1), 2, ProxyType::JustTransfer, 0)); @@ -152,7 +152,7 @@ fn fails_to_pause_unpausable_call_when_other_call_is_paused() { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_ok!(call_transfer_keep_alive(3, 1).dispatch(RuntimeOrigin::signed(3))); @@ -181,13 +181,13 @@ fn fails_to_pause_already_paused_pallet() { new_test_ext().execute_with(|| { assert_ok!(TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), )); assert_noop!( TxPause::pause( RuntimeOrigin::signed(mock::PauseOrigin::get()), - full_name::(b"Balances", b"transfer"), + full_name::(b"Balances", b"transfer_allow_death"), ), Error::::IsPaused ); @@ -208,7 +208,7 @@ fn fails_to_unpause_not_paused_pallet() { } pub fn call_transfer(dest: u64, value: u64) -> RuntimeCall { - RuntimeCall::Balances(pallet_balances::Call::transfer { dest, value }) + RuntimeCall::Balances(pallet_balances::Call::transfer_allow_death { dest, value }) } pub fn call_transfer_keep_alive(dest: u64, value: u64) -> RuntimeCall { diff --git a/substrate/frame/uniques/README.md b/substrate/frame/uniques/README.md index 6cdbcf79f1c95442cd9ec602ffb482f61ba0945f..538fd9e70a26f263689ce4bcd8162ab416425bd3 100644 --- a/substrate/frame/uniques/README.md +++ b/substrate/frame/uniques/README.md @@ -13,9 +13,11 @@ The Uniques module provides functionality for non-fungible tokens' management, i * Attributes Management * Item Burning -To use it in your runtime, you need to implement [`uniques::Config`](https://paritytech.github.io/substrate/master/pallet_uniques/pallet/trait.Config.html). +To use it in your runtime, you need to implement +[`uniques::Config`](https://paritytech.github.io/substrate/master/pallet_uniques/pallet/trait.Config.html). -The supported dispatchable functions are documented in the [`uniques::Call`](https://paritytech.github.io/substrate/master/pallet_uniques/pallet/enum.Call.html) enum. +The supported dispatchable functions are documented in the +[`uniques::Call`](https://paritytech.github.io/substrate/master/pallet_uniques/pallet/enum.Call.html) enum. ### Terminology @@ -23,8 +25,8 @@ The supported dispatchable functions are documented in the [`uniques::Call`](htt * **Item minting:** The action of creating a new item within a collection. * **Item transfer:** The action of sending an item from one account to another. * **Item burning:** The destruction of an item. -* **Non-fungible token (NFT):** An item for which each unit has unique characteristics. There is exactly - one instance of such an item in existence and there is exactly one owning account. +* **Non-fungible token (NFT):** An item for which each unit has unique characteristics. There is exactly one instance of + such an item in existence and there is exactly one owning account. ### Goals @@ -33,10 +35,8 @@ The Uniques pallet in Substrate is designed to make the following possible: * Allow accounts to permissionlessly create NFT collections. * Allow a named (permissioned) account to mint and burn unique items within a collection. * Move items between accounts permissionlessly. -* Allow a named (permissioned) account to freeze and unfreeze unique items within a - collection or the entire collection. -* Allow the owner of an item to delegate the ability to transfer the item to some - named third-party. +* Allow a named (permissioned) account to freeze and unfreeze unique items within a collection or the entire collection. +* Allow the owner of an item to delegate the ability to transfer the item to some named third-party. ## Interface diff --git a/substrate/frame/utility/README.md b/substrate/frame/utility/README.md index db19b0cf8cf9ed28b23f043b44be3da4e6a1dbc1..00fff76cd626bcb119f971edb910c033024b54c8 100644 --- a/substrate/frame/utility/README.md +++ b/substrate/frame/utility/README.md @@ -27,10 +27,10 @@ filtered by any proxy. ### Dispatchable Functions #### For batch dispatch -* `batch` - Dispatch multiple calls from the sender's origin. +- `batch` - Dispatch multiple calls from the sender's origin. #### For pseudonymal dispatch -* `as_derivative` - Dispatch a call from a derivative signed origin. +- `as_derivative` - Dispatch a call from a derivative signed origin. [`Call`]: ./enum.Call.html [`Config`]: ./trait.Config.html diff --git a/substrate/frame/whitelist/src/mock.rs b/substrate/frame/whitelist/src/mock.rs index d91f43b33af91bd8fe605bbfae6e7f4f0c09fb63..e6ddbf02bb2bf882624806df823cb49879c57314 100644 --- a/substrate/frame/whitelist/src/mock.rs +++ b/substrate/frame/whitelist/src/mock.rs @@ -90,8 +90,7 @@ impl pallet_preimage::Config for Test { type RuntimeEvent = RuntimeEvent; type Currency = Balances; type ManagerOrigin = EnsureRoot; - type BaseDeposit = ConstU64<1>; - type ByteDeposit = ConstU64<1>; + type Consideration = (); type WeightInfo = (); } diff --git a/substrate/primitives/api/Cargo.toml b/substrate/primitives/api/Cargo.toml index c5611b22017cb4720c9942476933519a4c517a83..95b5dde371394952a202575fdc61153f1c229640 100644 --- a/substrate/primitives/api/Cargo.toml +++ b/substrate/primitives/api/Cargo.toml @@ -23,7 +23,7 @@ sp-version = { path = "../version", default-features = false} sp-state-machine = { path = "../state-machine", default-features = false, optional = true} sp-trie = { path = "../trie", default-features = false, optional = true} hash-db = { version = "0.16.0", optional = true } -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } sp-metadata-ir = { path = "../metadata-ir", default-features = false, optional = true} log = { version = "0.4.17", default-features = false } diff --git a/substrate/primitives/api/README.md b/substrate/primitives/api/README.md index 1cf9437373c77f878b85d8271c87fdf77bf452ec..ee0e402f9d7f08fb01bdac7ea0cc63b4d158008a 100644 --- a/substrate/primitives/api/README.md +++ b/substrate/primitives/api/README.md @@ -14,4 +14,4 @@ api, the [`ApiExt`] trait, the [`CallApiAt`] trait and the [`ConstructRuntimeApi On a meta level this implies, the client calls the generated API from the client perspective. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/api/proc-macro/Cargo.toml b/substrate/primitives/api/proc-macro/Cargo.toml index 6819d139f45073ad26f2f095e6e5b1f4c0090a52..de5ddcf9dac09fcc8d2e5f8d1edbfc4abd5dcaf6 100644 --- a/substrate/primitives/api/proc-macro/Cargo.toml +++ b/substrate/primitives/api/proc-macro/Cargo.toml @@ -17,7 +17,7 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "fold", "extra-traits", "visit"] } +syn = { version = "2.0.37", features = ["full", "fold", "extra-traits", "visit"] } proc-macro2 = "1.0.56" blake2 = { version = "0.10.4", default-features = false } proc-macro-crate = "1.1.3" diff --git a/substrate/primitives/application-crypto/README.md b/substrate/primitives/application-crypto/README.md index c86e33552f60605d5ca8e9db7e48621f24060f2f..a686b746581249c5e705eb6301c5c11dc8ffce80 100644 --- a/substrate/primitives/application-crypto/README.md +++ b/substrate/primitives/application-crypto/README.md @@ -1,3 +1,3 @@ Traits and macros for constructing application specific strongly typed crypto wrappers. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/arithmetic/README.md b/substrate/primitives/arithmetic/README.md index e6e52c2a82696ba42e756458fff5839d2d88c09f..b5dcfdb29446fe573715af60b219b8642be7c537 100644 --- a/substrate/primitives/arithmetic/README.md +++ b/substrate/primitives/arithmetic/README.md @@ -1,3 +1,3 @@ Minimal fixed point arithmetic primitives and types for runtime. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/authority-discovery/README.md b/substrate/primitives/authority-discovery/README.md index 65c2e22dde004483fdcff27d9a4b4392753bff3a..3b48ddc61c51cac1a162b6576cdb15132af2972d 100644 --- a/substrate/primitives/authority-discovery/README.md +++ b/substrate/primitives/authority-discovery/README.md @@ -1,3 +1,3 @@ Runtime Api to help discover authorities. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/block-builder/README.md b/substrate/primitives/block-builder/README.md index 433197d3be9e4104606809bab2834e16e8bcf065..952c94798d3a2c2a6ebb100bc4a7c99829017cff 100644 --- a/substrate/primitives/block-builder/README.md +++ b/substrate/primitives/block-builder/README.md @@ -1,3 +1,3 @@ The block builder runtime api. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/blockchain/Cargo.toml b/substrate/primitives/blockchain/Cargo.toml index 418f945898572101d34657d6338e45cc725c486e..33db09ce0ac2f41897957a43b8b74497b90e93a1 100644 --- a/substrate/primitives/blockchain/Cargo.toml +++ b/substrate/primitives/blockchain/Cargo.toml @@ -19,7 +19,7 @@ futures = "0.3.21" log = "0.4.17" parking_lot = "0.12.1" schnellru = "0.2.1" -thiserror = "1.0.30" +thiserror = "1.0.48" sp-api = { path = "../api" } sp-consensus = { path = "../consensus/common" } sp-database = { path = "../database" } diff --git a/substrate/primitives/blockchain/README.md b/substrate/primitives/blockchain/README.md index 8298bfd7ae60a5d9a6be9c24ae4c7831e72f2bd0..a0a5b2edce518005673f6066ac7e27f74e35367b 100644 --- a/substrate/primitives/blockchain/README.md +++ b/substrate/primitives/blockchain/README.md @@ -1,3 +1,3 @@ Substrate blockchain traits and primitives. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/aura/README.md b/substrate/primitives/consensus/aura/README.md index 0f360ae67eb28a9145ad4c83980f6dff9a9d22f1..725c6fc6db496c7b995c967d80d3585415e005cc 100644 --- a/substrate/primitives/consensus/aura/README.md +++ b/substrate/primitives/consensus/aura/README.md @@ -1,3 +1,3 @@ Primitives for Aura. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/babe/README.md b/substrate/primitives/consensus/babe/README.md index 54bae05fd6db72a167bc8fd687e2d6a91a8a47ba..59f8d925af7541a6b583be3f61dd34c507482365 100644 --- a/substrate/primitives/consensus/babe/README.md +++ b/substrate/primitives/consensus/babe/README.md @@ -1,3 +1,3 @@ Primitives for BABE. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/common/Cargo.toml b/substrate/primitives/consensus/common/Cargo.toml index 284e00b272e6cc7e8c896cf594a0dff224c9be4e..e8f6b806f8c6403296e5cd95907b3a74c4e097f2 100644 --- a/substrate/primitives/consensus/common/Cargo.toml +++ b/substrate/primitives/consensus/common/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] async-trait = "0.1.57" futures = { version = "0.3.21", features = ["thread-pool"] } log = "0.4.17" -thiserror = "1.0.30" +thiserror = "1.0.48" sp-core = { path = "../../core" } sp-inherents = { path = "../../inherents" } sp-runtime = { path = "../../runtime" } diff --git a/substrate/primitives/consensus/common/README.md b/substrate/primitives/consensus/common/README.md index 963bb0fbdba4af4de706c5303003f8036016291d..f61a00c42c840159e921a06982d963e4fd350522 100644 --- a/substrate/primitives/consensus/common/README.md +++ b/substrate/primitives/consensus/common/README.md @@ -1,7 +1,7 @@ -Common utilities for building and using consensus engines in substrate. +Common utilities for building and using consensus engines in Substrate. Much of this crate is _unstable_ and thus the API is likely to undergo change. Implementors of traits should not rely on the interfaces to remain the same. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/grandpa/README.md b/substrate/primitives/consensus/grandpa/README.md index 77a7abca2eef3fa2da9b8c2580c1c0ae7f1d184b..d357904cd1f11d4d7b649988a84cd63768c542e7 100644 --- a/substrate/primitives/consensus/grandpa/README.md +++ b/substrate/primitives/consensus/grandpa/README.md @@ -1,3 +1,3 @@ Primitives for GRANDPA integration, suitable for WASM compilation. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/pow/README.md b/substrate/primitives/consensus/pow/README.md index 881864377649804e610fddecce9e9a3eb8624856..390190c5d18384a8164cc1e3370b4d8cdb0248fb 100644 --- a/substrate/primitives/consensus/pow/README.md +++ b/substrate/primitives/consensus/pow/README.md @@ -1,3 +1,3 @@ Primitives for Substrate Proof-of-Work (PoW) consensus. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/consensus/slots/README.md b/substrate/primitives/consensus/slots/README.md index f451c32888a47843c1f679a17d8db06d23811272..3052131721ad8deb1ab73f5fc98e89cec4f24761 100644 --- a/substrate/primitives/consensus/slots/README.md +++ b/substrate/primitives/consensus/slots/README.md @@ -1,3 +1,3 @@ Primitives for slots-based consensus engines. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index efcaad1a6f610795d4bc15cde487070633a2854b..9f1f0127d7cdcb365810c4c7351aa55fdafc2081 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -13,7 +13,6 @@ documentation = "https://docs.rs/sp-core" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -arrayvec = { version = "0.7.2", default-features = false } codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive","max-encoded-len"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } log = { version = "0.4.17", default-features = false } @@ -39,7 +38,7 @@ sp-storage = { path = "../storage", default-features = false} sp-externalities = { path = "../externalities", optional = true} futures = { version = "0.3.21", optional = true } dyn-clonable = { version = "0.9.0", optional = true } -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } tracing = { version = "0.1.29", optional = true } bitflags = "1.3" paste = "1.0.7" @@ -58,7 +57,7 @@ sp-runtime-interface = { path = "../runtime-interface", default-features = false # bls crypto w3f-bls = { version = "0.1.3", default-features = false, optional = true} # bandersnatch crypto -bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "3119f51", default-features = false, optional = true } +bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "f4fe253", default-features = false, optional = true } [dev-dependencies] criterion = "0.4.0" @@ -76,7 +75,6 @@ bench = false default = [ "std" ] std = [ "array-bytes", - "arrayvec/std", "bandersnatch_vrfs/getrandom", "blake2/std", "bounded-collections/std", diff --git a/substrate/primitives/core/hashing/proc-macro/Cargo.toml b/substrate/primitives/core/hashing/proc-macro/Cargo.toml index ef17473a6f39e8f9171ba3c57df7cb19890f4114..64b46ab9c19ef142d3c79f78f7ace6f5f6fa29b0 100644 --- a/substrate/primitives/core/hashing/proc-macro/Cargo.toml +++ b/substrate/primitives/core/hashing/proc-macro/Cargo.toml @@ -17,5 +17,5 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "parsing"] } +syn = { version = "2.0.37", features = ["full", "parsing"] } sp-core-hashing = { path = "..", default-features = false} diff --git a/substrate/primitives/core/src/bandersnatch.rs b/substrate/primitives/core/src/bandersnatch.rs index 01f3538188a462ed465356fb4be75eab7223c78c..832ef6c77bb3dbb71e94200e27a0e7379510769c 100644 --- a/substrate/primitives/core/src/bandersnatch.rs +++ b/substrate/primitives/core/src/bandersnatch.rs @@ -53,9 +53,8 @@ const SEED_SERIALIZED_LEN: usize = 32; // Short-Weierstrass form serialized sizes. const PUBLIC_SERIALIZED_LEN: usize = 33; const SIGNATURE_SERIALIZED_LEN: usize = 65; +const RING_SIGNATURE_SERIALIZED_LEN: usize = 755; const PREOUT_SERIALIZED_LEN: usize = 33; -const PEDERSEN_SIGNATURE_SERIALIZED_LEN: usize = 163; -const RING_PROOF_SERIALIZED_LEN: usize = 592; // Max size of serialized ring-vrf context params. // @@ -69,7 +68,7 @@ const RING_PROOF_SERIALIZED_LEN: usize = 592; // 2048 → 295 KB // NOTE: This is quite big but looks like there is an upcoming fix // in the backend. -const RING_CONTEXT_SERIALIZED_LEN: usize = 147752; +const RING_CONTEXT_SERIALIZED_LEN: usize = 147748; /// Bandersnatch public key. #[cfg_attr(feature = "full_crypto", derive(Hash))] @@ -278,7 +277,7 @@ impl TraitPair for Pair { let mut raw = [0; PUBLIC_SERIALIZED_LEN]; public .serialize_compressed(raw.as_mut_slice()) - .expect("key buffer length is good; qed"); + .expect("serialization length is constant and checked by test; qed"); Public::unchecked_from(raw) } @@ -356,7 +355,7 @@ pub mod vrf { let mut bytes = [0; PREOUT_SERIALIZED_LEN]; self.0 .serialize_compressed(bytes.as_mut_slice()) - .expect("preout serialization can't fail"); + .expect("serialization length is constant and checked by test; qed"); bytes.encode() } } @@ -506,7 +505,7 @@ pub mod vrf { } fn vrf_output(&self, input: &Self::VrfInput) -> Self::VrfOutput { - let output = self.secret.0.vrf_preout(&input.0); + let output = self.secret.vrf_preout(&input.0); VrfOutput(output) } } @@ -539,24 +538,26 @@ pub mod vrf { #[cfg(feature = "full_crypto")] impl Pair { fn vrf_sign_gen(&self, data: &VrfSignData) -> VrfSignature { - let ios: Vec<_> = data - .inputs - .iter() - .map(|i| self.secret.clone().0.vrf_inout(i.0.clone())) - .collect(); + let ios = core::array::from_fn(|i| { + let input = data.inputs[i].0.clone(); + self.secret.vrf_inout(input) + }); - let signature: ThinVrfSignature = - self.secret.sign_thin_vrf(data.transcript.clone(), ios.as_slice()); + let thin_signature: ThinVrfSignature = + self.secret.sign_thin_vrf(data.transcript.clone(), &ios); - let mut sign_bytes = [0; SIGNATURE_SERIALIZED_LEN]; - signature - .signature - .serialize_compressed(sign_bytes.as_mut_slice()) - .expect("serialization can't fail"); - - let outputs: Vec<_> = signature.preoutputs.into_iter().map(VrfOutput).collect(); + let outputs: Vec<_> = thin_signature.preouts.into_iter().map(VrfOutput).collect(); let outputs = VrfIosVec::truncate_from(outputs); - VrfSignature { signature: Signature(sign_bytes), outputs } + + let mut signature = + VrfSignature { signature: Signature([0; SIGNATURE_SERIALIZED_LEN]), outputs }; + + thin_signature + .proof + .serialize_compressed(signature.signature.0.as_mut_slice()) + .expect("serialization length is constant and checked by test; qed"); + + signature } /// Generate an arbitrary number of bytes from the given `context` and VRF `input`. @@ -566,7 +567,7 @@ pub mod vrf { input: &VrfInput, ) -> [u8; N] { let transcript = Transcript::new_labeled(context); - let inout = self.secret.clone().0.vrf_inout(input.0.clone()); + let inout = self.secret.vrf_inout(input.0.clone()); inout.vrf_output_bytes(transcript) } } @@ -581,30 +582,23 @@ pub mod vrf { return false }; - let Ok(preouts) = signature - .outputs - .iter() - .map(|o| o.0.clone()) - .collect::>() - .into_inner() - else { - return false - }; + let preouts: [bandersnatch_vrfs::VrfPreOut; N] = + core::array::from_fn(|i| signature.outputs[i].0.clone()); // Deserialize only the proof, the rest has already been deserialized // This is another hack used because backend signature type is generic over // the number of ios. - let Ok(signature) = + let Ok(proof) = ThinVrfSignature::<0>::deserialize_compressed(signature.signature.as_ref()) - .map(|s| s.signature) + .map(|s| s.proof) else { return false }; - let signature = ThinVrfSignature { signature, preoutputs: preouts }; + let signature = ThinVrfSignature { proof, preouts }; let inputs = data.inputs.iter().map(|i| i.0.clone()); - signature.verify_thin_vrf(data.transcript.clone(), inputs, &public).is_ok() + public.verify_thin_vrf(data.transcript.clone(), inputs, &signature).is_ok() } } @@ -627,7 +621,7 @@ pub mod vrf { pub mod ring_vrf { use super::{vrf::*, *}; pub use bandersnatch_vrfs::ring::{RingProof, RingProver, RingVerifier, KZG}; - use bandersnatch_vrfs::{CanonicalDeserialize, PedersenVrfSignature, PublicKey}; + use bandersnatch_vrfs::{CanonicalDeserialize, PublicKey}; /// Context used to produce ring signatures. #[derive(Clone)] @@ -649,7 +643,7 @@ pub mod ring_vrf { let mut pks = Vec::with_capacity(public_keys.len()); for public_key in public_keys { let pk = PublicKey::deserialize_compressed(public_key.as_slice()).ok()?; - pks.push(pk.0 .0.into()); + pks.push(pk.0.into()); } let prover_key = self.0.prover_key(pks); @@ -662,7 +656,7 @@ pub mod ring_vrf { let mut pks = Vec::with_capacity(public_keys.len()); for public_key in public_keys { let pk = PublicKey::deserialize_compressed(public_key.as_slice()).ok()?; - pks.push(pk.0 .0.into()); + pks.push(pk.0.into()); } let verifier_key = self.0.verifier_key(pks); @@ -676,7 +670,7 @@ pub mod ring_vrf { let mut buf = Box::new([0; RING_CONTEXT_SERIALIZED_LEN]); self.0 .serialize_compressed(buf.as_mut_slice()) - .expect("preout serialization can't fail"); + .expect("serialization length is constant and checked by test; qed"); buf.encode() } } @@ -711,10 +705,8 @@ pub mod ring_vrf { pub struct RingVrfSignature { /// VRF (pre)outputs. pub outputs: VrfIosVec, - /// Pedersen VRF signature. - pub signature: [u8; PEDERSEN_SIGNATURE_SERIALIZED_LEN], - /// Ring proof. - pub ring_proof: [u8; RING_PROOF_SERIALIZED_LEN], + /// Ring signature. + pub signature: [u8; RING_SIGNATURE_SERIALIZED_LEN], } #[cfg(feature = "full_crypto")] @@ -741,31 +733,27 @@ pub mod ring_vrf { data: &VrfSignData, prover: &RingProver, ) -> RingVrfSignature { - let ios: Vec<_> = data - .inputs - .iter() - .map(|i| self.secret.clone().0.vrf_inout(i.0.clone())) - .collect(); + let ios = core::array::from_fn(|i| { + let input = data.inputs[i].0.clone(); + self.secret.vrf_inout(input) + }); let ring_signature: bandersnatch_vrfs::RingVrfSignature = - self.secret.sign_ring_vrf(data.transcript.clone(), ios.as_slice(), prover); + bandersnatch_vrfs::RingProver { ring_prover: prover, secret: &self.secret } + .sign_ring_vrf(data.transcript.clone(), &ios); - let outputs: Vec<_> = ring_signature.preoutputs.into_iter().map(VrfOutput).collect(); + let outputs: Vec<_> = ring_signature.preouts.into_iter().map(VrfOutput).collect(); let outputs = VrfIosVec::truncate_from(outputs); - let mut signature = [0; PEDERSEN_SIGNATURE_SERIALIZED_LEN]; - ring_signature - .signature - .serialize_compressed(signature.as_mut_slice()) - .expect("ped-signature serialization can't fail"); + let mut signature = + RingVrfSignature { outputs, signature: [0; RING_SIGNATURE_SERIALIZED_LEN] }; - let mut ring_proof = [0; RING_PROOF_SERIALIZED_LEN]; ring_signature - .ring_proof - .serialize_compressed(ring_proof.as_mut_slice()) - .expect("ring-proof serialization can't fail"); + .proof + .serialize_compressed(signature.signature.as_mut_slice()) + .expect("serialization length is constant and checked by test; qed"); - RingVrfSignature { outputs, signature, ring_proof } + signature } } @@ -774,7 +762,7 @@ pub mod ring_vrf { /// /// The signature is verifiable if it has been produced by a member of the ring /// from which the [`RingVerifier`] has been constructed. - pub fn verify(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { + pub fn ring_vrf_verify(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { const _: () = assert!(MAX_VRF_IOS == 3, "`MAX_VRF_IOS` expected to be 3"); let preouts_len = self.outputs.len(); if preouts_len != data.inputs.len() { @@ -782,43 +770,37 @@ pub mod ring_vrf { } // Workaround to overcome backend signature generic over the number of IOs. match preouts_len { - 0 => self.verify_gen::<0>(data, verifier), - 1 => self.verify_gen::<1>(data, verifier), - 2 => self.verify_gen::<2>(data, verifier), - 3 => self.verify_gen::<3>(data, verifier), + 0 => self.ring_vrf_verify_gen::<0>(data, verifier), + 1 => self.ring_vrf_verify_gen::<1>(data, verifier), + 2 => self.ring_vrf_verify_gen::<2>(data, verifier), + 3 => self.ring_vrf_verify_gen::<3>(data, verifier), _ => unreachable!(), } } - fn verify_gen(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { - let Ok(preoutputs) = self - .outputs - .iter() - .map(|o| o.0.clone()) - .collect::>() - .into_inner() - else { - return false - }; - - let Ok(signature) = - PedersenVrfSignature::deserialize_compressed(self.signature.as_slice()) + fn ring_vrf_verify_gen( + &self, + data: &VrfSignData, + verifier: &RingVerifier, + ) -> bool { + let Ok(vrf_signature) = + bandersnatch_vrfs::RingVrfSignature::<0>::deserialize_compressed( + self.signature.as_slice(), + ) else { return false }; - let Ok(ring_proof) = RingProof::deserialize_compressed(self.ring_proof.as_slice()) - else { - return false - }; + let preouts: [bandersnatch_vrfs::VrfPreOut; N] = + core::array::from_fn(|i| self.outputs[i].0.clone()); - let ring_signature = - bandersnatch_vrfs::RingVrfSignature { signature, preoutputs, ring_proof }; + let signature = + bandersnatch_vrfs::RingVrfSignature { proof: vrf_signature.proof, preouts }; let inputs = data.inputs.iter().map(|i| i.0.clone()); - ring_signature - .verify_ring_vrf(data.transcript.clone(), inputs, verifier) + bandersnatch_vrfs::RingVerifier(verifier) + .verify_ring_vrf(data.transcript.clone(), inputs, &signature) .is_ok() } } @@ -840,16 +822,41 @@ mod tests { } #[test] - fn assumptions_sanity_check() { - // Backend - let ring_ctx = RingContext::new_testing(); - let pair = SecretKey::from_seed(DEV_SEED); - let public = pair.to_public(); + fn backend_assumptions_sanity_check() { + let kzg = KZG::testing_kzg_setup([0; 32], RING_DOMAIN_SIZE as u32); + assert_eq!(kzg.max_keyset_size(), RING_DOMAIN_SIZE - 257); + assert_eq!(kzg.compressed_size(), RING_CONTEXT_SERIALIZED_LEN); + + let pks: Vec<_> = (0..16) + .map(|i| SecretKey::from_seed(&[i as u8; 32]).to_public().0.into()) + .collect(); - assert_eq!(public.0.size_of_serialized(), PUBLIC_SERIALIZED_LEN); - assert_eq!(ring_ctx.max_keyset_size(), RING_DOMAIN_SIZE - 257); + let secret = SecretKey::from_seed(&[0u8; 32]); - // Wrapper + let public = secret.to_public(); + assert_eq!(public.compressed_size(), PUBLIC_SERIALIZED_LEN); + + let input = VrfInput::new(b"foo", &[]); + let preout = secret.vrf_preout(&input.0); + assert_eq!(preout.compressed_size(), PREOUT_SERIALIZED_LEN); + + let prover_key = kzg.prover_key(pks); + let ring_prover = kzg.init_ring_prover(prover_key, 0); + + let data = VrfSignData::new_unchecked(b"mydata", &[b"tdata"], None); + + let thin_signature: bandersnatch_vrfs::ThinVrfSignature<0> = + secret.sign_thin_vrf(data.transcript.clone(), &[]); + assert_eq!(thin_signature.compressed_size(), SIGNATURE_SERIALIZED_LEN); + + let ring_signature: bandersnatch_vrfs::RingVrfSignature<0> = + bandersnatch_vrfs::RingProver { ring_prover: &ring_prover, secret: &secret } + .sign_ring_vrf(data.transcript.clone(), &[]); + assert_eq!(ring_signature.compressed_size(), RING_SIGNATURE_SERIALIZED_LEN); + } + + #[test] + fn max_vrf_ios_bound_respected() { let inputs: Vec<_> = (0..MAX_VRF_IOS - 1).map(|_| VrfInput::new(b"", &[])).collect(); let mut sign_data = VrfSignData::new(b"", &[b""], inputs).unwrap(); let res = sign_data.push_vrf_input(VrfInput::new(b"", b"")); @@ -987,7 +994,7 @@ mod tests { let signature = pair.ring_vrf_sign(&data, &prover); let verifier = ring_ctx.verifier(&pks).unwrap(); - assert!(signature.verify(&data, &verifier)); + assert!(signature.ring_vrf_verify(&data, &verifier)); } #[test] @@ -1006,7 +1013,7 @@ mod tests { let signature = pair.ring_vrf_sign(&data, &prover); let verifier = ring_ctx.verifier(&pks).unwrap(); - assert!(!signature.verify(&data, &verifier)); + assert!(!signature.ring_vrf_verify(&data, &verifier)); } #[test] @@ -1062,10 +1069,8 @@ mod tests { let bytes = expected.encode(); - let expected_len = data.inputs.len() * PREOUT_SERIALIZED_LEN + - PEDERSEN_SIGNATURE_SERIALIZED_LEN + - RING_PROOF_SERIALIZED_LEN + - 1; + let expected_len = + data.inputs.len() * PREOUT_SERIALIZED_LEN + RING_SIGNATURE_SERIALIZED_LEN + 1; assert_eq!(bytes.len(), expected_len); let decoded = RingVrfSignature::decode(&mut bytes.as_slice()).unwrap(); diff --git a/substrate/primitives/core/src/traits.rs b/substrate/primitives/core/src/traits.rs index 40137053ab7527d5191bc2889642549989d3be30..9815c84f3396a69401be386a7a5d0b627bddd761 100644 --- a/substrate/primitives/core/src/traits.rs +++ b/substrate/primitives/core/src/traits.rs @@ -91,7 +91,7 @@ pub struct RuntimeCode<'a> { /// /// If `None` are given, the default value of the executor will be used. pub heap_pages: Option, - /// The SCALE encoded hash of `code`. + /// The hash of `code`. /// /// The hashing algorithm isn't that important, as long as all runtime /// code instances use the same. diff --git a/substrate/primitives/crypto/ec-utils/src/lib.rs b/substrate/primitives/crypto/ec-utils/src/lib.rs index c1877dd5b5d727b7b63104b2d3215e9969f41fe8..22e24e61f7b357d338e4cd85f411eb8ae43e82e5 100644 --- a/substrate/primitives/crypto/ec-utils/src/lib.rs +++ b/substrate/primitives/crypto/ec-utils/src/lib.rs @@ -21,6 +21,9 @@ #![warn(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] +use sp_runtime_interface::runtime_interface; +use sp_std::vec::Vec; + pub mod bls12_377; pub mod bls12_381; pub mod bw6_761; @@ -28,8 +31,6 @@ pub mod ed_on_bls12_377; pub mod ed_on_bls12_381_bandersnatch; mod utils; -use sp_runtime_interface::runtime_interface; - /// Interfaces for working with elliptic curves related types from within the runtime. /// All type are (de-)serialized through the wrapper types from the ark-scale trait, /// with ark_scale::{ArkScale, ArkScaleProjective}; diff --git a/substrate/primitives/database/README.md b/substrate/primitives/database/README.md index cd0677eb9eb44ef3d26f1cfecdb1bdbbd371536a..26f58ea8d458669ae5e80853ec6cd03160708748 100644 --- a/substrate/primitives/database/README.md +++ b/substrate/primitives/database/README.md @@ -1,3 +1,3 @@ The main database trait, allowing Substrate to store data persistently. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/debug-derive/Cargo.toml b/substrate/primitives/debug-derive/Cargo.toml index 292c07e092f9a466feb270adc8f296fd484b7815..9d3930ac2572025245201aa099f87cfc07a0413f 100644 --- a/substrate/primitives/debug-derive/Cargo.toml +++ b/substrate/primitives/debug-derive/Cargo.toml @@ -18,7 +18,7 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.37" proc-macro2 = "1.0.56" [features] diff --git a/substrate/primitives/externalities/README.md b/substrate/primitives/externalities/README.md index 3141b2609e6375063dc42b089489451973393cba..543b63ecbdb6a9a34709e3a3b989b0b88b8d6800 100644 --- a/substrate/primitives/externalities/README.md +++ b/substrate/primitives/externalities/README.md @@ -6,4 +6,4 @@ access the node from the runtime via the runtime interfaces. This crate exposes the main [`Externalities`] trait. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/genesis-builder/Cargo.toml b/substrate/primitives/genesis-builder/Cargo.toml index fb92f0223d06b930a31d7249299e08fac15f804e..8f083e2d7d54ca1396d46b22e002b706fce2f406 100644 --- a/substrate/primitives/genesis-builder/Cargo.toml +++ b/substrate/primitives/genesis-builder/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] sp-api = { path = "../api", default-features = false} sp-runtime = { path = "../runtime", default-features = false} sp-std = { path = "../std", default-features = false} -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } [features] default = [ "std" ] diff --git a/substrate/primitives/inherents/Cargo.toml b/substrate/primitives/inherents/Cargo.toml index d3ac94aa5fb34ba1f74f10434ca8482e1051525f..aa0aa95b3f8d060c909c23fb3f6bbdfeac3643ad 100644 --- a/substrate/primitives/inherents/Cargo.toml +++ b/substrate/primitives/inherents/Cargo.toml @@ -18,7 +18,7 @@ async-trait = { version = "0.1.57", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } impl-trait-for-tuples = "0.2.2" -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } sp-runtime = { path = "../runtime", default-features = false, optional = true} sp-std = { path = "../std", default-features = false} diff --git a/substrate/primitives/inherents/README.md b/substrate/primitives/inherents/README.md index 78aa625fe85692b1d53164415a5b2086d5704186..20ceb529469182438beb29a921d117c029cbd197 100644 --- a/substrate/primitives/inherents/README.md +++ b/substrate/primitives/inherents/README.md @@ -14,4 +14,4 @@ A module can also just check given inherents. For using a module as inherent pro to be registered by the `construct_runtime!` macro. The macro documentation gives more information on how that is done. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/io/Cargo.toml b/substrate/primitives/io/Cargo.toml index 4793938617e067a6364a4cefda78c4c57a51ab1a..ab9d26ec26f49fc4ad0097f3be71dd0e6a7c675c 100644 --- a/substrate/primitives/io/Cargo.toml +++ b/substrate/primitives/io/Cargo.toml @@ -33,7 +33,7 @@ tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.28", default-features = false} # Required for backwards compatibility reason, but only used for verifying when `UseDalekExt` is set. -ed25519-dalek = { version = "2.0.0", default-features = false, optional = true } +ed25519-dalek = { version = "2.0", default-features = false, optional = true } [build-dependencies] rustversion = "1.0.6" diff --git a/substrate/primitives/io/README.md b/substrate/primitives/io/README.md index a24370cc566b359b2b3aeab195d185dec34f7e78..5e252eeacb0929bf46179e8115c3924c3ee4cacf 100644 --- a/substrate/primitives/io/README.md +++ b/substrate/primitives/io/README.md @@ -1,3 +1,3 @@ -I/O host interface for substrate runtime. +I/O host interface for Substrate runtime. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/keyring/README.md b/substrate/primitives/keyring/README.md index 1610f237df97a8a96f5017f0c15c686661de36e4..555a35f09bc3af22db63c953feac7220ade2ca2d 100644 --- a/substrate/primitives/keyring/README.md +++ b/substrate/primitives/keyring/README.md @@ -1,3 +1,3 @@ Support code for the runtime. A set of test accounts. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/npos-elections/README.md b/substrate/primitives/npos-elections/README.md index 6881fc6418f3a93e476b097fd049e3fc354be0c7..e65f22ca271a24b341760f2d257465195a64a888 100644 --- a/substrate/primitives/npos-elections/README.md +++ b/substrate/primitives/npos-elections/README.md @@ -1,30 +1,29 @@ -A set of election algorithms to be used with a substrate runtime, typically within the staking -sub-system. Notable implementation include: +# sp-npos-elections -- [`seq_phragmen`]: Implements the Phragmén Sequential Method. An un-ranked, relatively fast - election method that ensures PJR, but does not provide a constant factor approximation of the - maximin problem. -- [`phragmms`]: Implements a hybrid approach inspired by Phragmén which is executed faster but - it can achieve a constant factor approximation of the maximin problem, similar to that of the - MMS algorithm. -- [`balance_solution`]: Implements the star balancing algorithm. This iterative process can push - a solution toward being more `balanced`, which in turn can increase its score. +A set of election algorithms to be used with a Substrate runtime, typically within the staking sub-system. Notable +implementation include: -### Terminology +- [`seq_phragmen`]: Implements the Phragmén Sequential Method. An un-ranked, relatively fast election method that + ensures PJR, but does not provide a constant factor approximation of the maximin problem. +- [`phragmms`]: Implements a hybrid approach inspired by Phragmén which is executed faster but it can achieve a constant + factor approximation of the maximin problem, similar to that of the MMS algorithm. +- [`balance_solution`]: Implements the star balancing algorithm. This iterative process can push a solution toward being + more `balanced`, which in turn can increase its score. -This crate uses context-independent words, not to be confused with staking. This is because the -election algorithms of this crate, while designed for staking, can be used in other contexts as -well. +## Terminology -`Voter`: The entity casting some votes to a number of `Targets`. This is the same as `Nominator` -in the context of staking. `Target`: The entities eligible to be voted upon. This is the same as -`Validator` in the context of staking. `Edge`: A mapping from a `Voter` to a `Target`. +This crate uses context-independent words, not to be confused with staking. This is because the election algorithms of +this crate, while designed for staking, can be used in other contexts as well. + +`Voter`: The entity casting some votes to a number of `Targets`. This is the same as `Nominator` in the context of +staking. `Target`: The entities eligible to be voted upon. This is the same as `Validator` in the context of staking. +`Edge`: A mapping from a `Voter` to a `Target`. The goal of an election algorithm is to provide an `ElectionResult`. A data composed of: -- `winners`: A flat list of identifiers belonging to those who have won the election, usually - ordered in some meaningful way. They are zipped with their total backing stake. -- `assignment`: A mapping from each voter to their winner-only targets, zipped with a ration - denoting the amount of support given to that particular target. +- `winners`: A flat list of identifiers belonging to those who have won the election, usually ordered in some meaningful + way. They are zipped with their total backing stake. +- `assignment`: A mapping from each voter to their winner-only targets, zipped with a ration denoting the amount of + support given to that particular target. ```rust // the winners. @@ -44,16 +43,14 @@ let election_result = ElectionResult { winners, assignments }; ``` -The `Assignment` field of the election result is voter-major, i.e. it is from the perspective of -the voter. The struct that represents the opposite is called a `Support`. This struct is usually -accessed in a map-like manner, i.e. keyed by voters, therefore it is stored as a mapping called -`SupportMap`. +The `Assignment` field of the election result is voter-major, i.e. it is from the perspective of the voter. The struct +that represents the opposite is called a `Support`. This struct is usually accessed in a map-like manner, i.e. keyed by +voters, therefore it is stored as a mapping called `SupportMap`. -Moreover, the support is built from absolute backing values, not ratios like the example above. -A struct similar to `Assignment` that has stake value instead of ratios is called an -`StakedAssignment`. +Moreover, the support is built from absolute backing values, not ratios like the example above. A struct similar to +`Assignment` that has stake value instead of ratios is called an `StakedAssignment`. More information can be found at: https://arxiv.org/abs/2004.12990 -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/npos-elections/fuzzer/Cargo.toml b/substrate/primitives/npos-elections/fuzzer/Cargo.toml index 86927786c58f2d2a3604db28f4cfb7c40083f30a..eeb9deebb71e9e68c7c07edce5f11e91053d4854 100644 --- a/substrate/primitives/npos-elections/fuzzer/Cargo.toml +++ b/substrate/primitives/npos-elections/fuzzer/Cargo.toml @@ -14,7 +14,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } honggfuzz = "0.5" rand = { version = "0.8", features = ["std", "small_rng"] } sp-npos-elections = { path = ".." } diff --git a/substrate/primitives/offchain/README.md b/substrate/primitives/offchain/README.md index a8620d3bb9d5b2a6d45e27404891cece853eafc8..5c239d2e0e733d239ee37751aa17b419c4551ab6 100644 --- a/substrate/primitives/offchain/README.md +++ b/substrate/primitives/offchain/README.md @@ -1,3 +1,3 @@ The Offchain Worker runtime api primitives. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/panic-handler/README.md b/substrate/primitives/panic-handler/README.md index c08396960f4c953abda24049359cbb6649dbc7d2..f4f974ea6e1cfee0e3317a1af2ab744f8c842de1 100644 --- a/substrate/primitives/panic-handler/README.md +++ b/substrate/primitives/panic-handler/README.md @@ -7,4 +7,4 @@ given URL. By default, the panic handler aborts the process by calling [`std::process::exit`]. This can temporarily be disabled by using an [`AbortGuard`]. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/rpc/Cargo.toml b/substrate/primitives/rpc/Cargo.toml index 21447dafb050d90542380eaecd0ce5f2bb847dc0..4739c96740e9ff1c72ed0aa8000437283d4000ac 100644 --- a/substrate/primitives/rpc/Cargo.toml +++ b/substrate/primitives/rpc/Cargo.toml @@ -18,4 +18,4 @@ serde = { version = "1.0.188", features = ["derive"] } sp-core = { path = "../core" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" diff --git a/substrate/primitives/rpc/README.md b/substrate/primitives/rpc/README.md index 8a9c17edd4755be7c9b040bde1019ea85fabb1be..4d48fc56aed9e9e0d150de7ed4189551d908ccab 100644 --- a/substrate/primitives/rpc/README.md +++ b/substrate/primitives/rpc/README.md @@ -1,3 +1,3 @@ Substrate RPC primitives and utilities. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/runtime-interface/README.md b/substrate/primitives/runtime-interface/README.md index 49e13f1b2e7436db5301781cf3a9ff300b6081dd..f6dfea945ddb44d6a5dbe7b93f869991c0c0aa83 100644 --- a/substrate/primitives/runtime-interface/README.md +++ b/substrate/primitives/runtime-interface/README.md @@ -1,43 +1,42 @@ Substrate runtime interface -This crate provides types, traits and macros around runtime interfaces. A runtime interface is -a fixed interface between a Substrate runtime and a Substrate node. For a native runtime the -interface maps to a direct function call of the implementation. For a wasm runtime the interface -maps to an external function call. These external functions are exported by the wasm executor -and they map to the same implementation as the native calls. +This crate provides types, traits and macros around runtime interfaces. A runtime interface is a fixed interface between +a Substrate runtime and a Substrate node. For a native runtime the interface maps to a direct function call of the +implementation. For a wasm runtime the interface maps to an external function call. These external functions are +exported by the wasm executor and they map to the same implementation as the native calls. # Using a type in a runtime interface -Any type that should be used in a runtime interface as argument or return value needs to -implement [`RIType`]. The associated type [`FFIType`](https:/docs.rs/sp-runtime-interface/latest/sp_runtime_interface/trait.RIType.html#associatedtype.FFIType) -is the type that is used in the FFI function to represent the actual type. For example `[T]` is -represented by an `u64`. The slice pointer and the length will be mapped to an `u64` value. -For more information see this [table](https:/docs.rs/sp-runtime-interface/latest/sp_runtime_interface/#ffi-type-and-conversion). -The FFI function definition is used when calling from the wasm runtime into the node. +Any type that should be used in a runtime interface as argument or return value needs to implement [`RIType`]. The +associated type +[`FFIType`](https:/docs.rs/sp-runtime-interface/latest/sp_runtime_interface/trait.RIType.html#associatedtype.FFIType) is +the type that is used in the FFI function to represent the actual type. For example `[T]` is represented by an `u64`. +The slice pointer and the length will be mapped to an `u64` value. For more information see this +[table](https:/docs.rs/sp-runtime-interface/latest/sp_runtime_interface/#ffi-type-and-conversion). The FFI function +definition is used when calling from the wasm runtime into the node. Traits are used to convert from a type to the corresponding [`RIType::FFIType`](https:/docs.rs/sp-runtime-interface/latest/sp_runtime_interface/trait.RIType.html#associatedtype.FFIType). -Depending on where and how a type should be used in a function signature, a combination of the -following traits need to be implemented: +Depending on where and how a type should be used in a function signature, a combination of the following traits need to +be implemented: 1. Pass as function argument: [`wasm::IntoFFIValue`] and [`host::FromFFIValue`] 2. As function return value: [`wasm::FromFFIValue`] and [`host::IntoFFIValue`] 3. Pass as mutable function argument: [`host::IntoPreallocatedFFIValue`] -The traits are implemented for most of the common types like `[T]`, `Vec`, arrays and -primitive types. +The traits are implemented for most of the common types like `[T]`, `Vec`, arrays and primitive types. -For custom types, we provide the [`PassBy`](https://docs.rs/sp-runtime-interface/latest/sp_runtime_interface/pass_by#PassBy) trait and strategies that define -how a type is passed between the wasm runtime and the node. Each strategy also provides a derive -macro to simplify the implementation. +For custom types, we provide the +[`PassBy`](https://docs.rs/sp-runtime-interface/latest/sp_runtime_interface/pass_by#PassBy) trait and strategies that +define how a type is passed between the wasm runtime and the node. Each strategy also provides a derive macro to +simplify the implementation. # Performance -To not waste any more performance when calling into the node, not all types are SCALE encoded -when being passed as arguments between the wasm runtime and the node. For most types that -are raw bytes like `Vec`, `[u8]` or `[u8; N]` we pass them directly, without SCALE encoding -them in front of. The implementation of [`RIType`] each type provides more information on how -the data is passed. +To not waste any more performance when calling into the node, not all types are SCALE encoded when being passed as +arguments between the wasm runtime and the node. For most types that are raw bytes like `Vec`, `[u8]` or `[u8; N]` +we pass them directly, without SCALE encoding them in front of. The implementation of [`RIType`] each type provides more +information on how the data is passed. # Declaring a runtime interface @@ -57,9 +56,10 @@ For more information on declaring a runtime interface, see # FFI type and conversion -The following table documents how values of types are passed between the wasm and -the host side and how they are converted into the corresponding type. +The following table documents how values of types are passed between the wasm and the host side and how they are +converted into the corresponding type. + | Type | FFI type | Conversion | |----|----|----| | `u8` | `u8` | `Identity` | diff --git a/substrate/primitives/runtime-interface/proc-macro/Cargo.toml b/substrate/primitives/runtime-interface/proc-macro/Cargo.toml index e01343d6dc0a937887203e19a5d2e93e4f88e7a0..5569e31c936027b41dbbf7b5e176571db3fdf081 100644 --- a/substrate/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/substrate/primitives/runtime-interface/proc-macro/Cargo.toml @@ -20,4 +20,4 @@ Inflector = "0.11.4" proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit", "fold", "extra-traits"] } +syn = { version = "2.0.37", features = ["full", "visit", "fold", "extra-traits"] } diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index 7050c27bc849758599d8fd2086f8f4fa7da93220..fa3c3ae2e6ea032447de8e0fe158e845ead2b9ec 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -32,7 +32,7 @@ sp-weights = { path = "../weights", default-features = false} [dev-dependencies] rand = "0.8.5" -serde_json = "1.0.85" +serde_json = "1.0.107" zstd = { version = "0.12.4", default-features = false } sp-api = { path = "../api" } sp-state-machine = { path = "../state-machine" } diff --git a/substrate/primitives/runtime/README.md b/substrate/primitives/runtime/README.md index 1515cd8e2961b5a72fee1d39297e51305d16b0a6..2755690e4b32234b93948b0fbd352ce811392de7 100644 --- a/substrate/primitives/runtime/README.md +++ b/substrate/primitives/runtime/README.md @@ -1,3 +1,3 @@ Runtime Modules shared primitive types. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/session/README.md b/substrate/primitives/session/README.md index 2d1f9d9bc1d5b3cb77125be91d847b97c276833c..9ad3e27470343557795dff292b3f0c758dbf1999 100644 --- a/substrate/primitives/session/README.md +++ b/substrate/primitives/session/README.md @@ -1,3 +1,3 @@ Substrate core types around sessions. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/staking/README.md b/substrate/primitives/staking/README.md index 892e1379d9a53d9677bcc13f5f0ba42b61ae6a0b..3e0ea0ba1fa20a9589b68e6cf8ff0d19da53d7ba 100644 --- a/substrate/primitives/staking/README.md +++ b/substrate/primitives/staking/README.md @@ -1,4 +1,4 @@ A crate which contains primitives that are useful for implementation that uses staking approaches in general. Definitions related to sessions, slashing, etc go here. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/state-machine/Cargo.toml b/substrate/primitives/state-machine/Cargo.toml index 3ab21308c3c87b7081cc81fb0f90823d630efb7b..ec5d9b5ea14e5b930130445c02c35f6a64ee9733 100644 --- a/substrate/primitives/state-machine/Cargo.toml +++ b/substrate/primitives/state-machine/Cargo.toml @@ -20,14 +20,14 @@ log = { version = "0.4.17", default-features = false } parking_lot = { version = "0.12.1", optional = true } rand = { version = "0.8.5", optional = true } smallvec = "1.11.0" -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } tracing = { version = "0.1.29", optional = true } sp-core = { path = "../core", default-features = false} sp-externalities = { path = "../externalities", default-features = false} sp-panic-handler = { path = "../panic-handler", optional = true} sp-std = { path = "../std", default-features = false} sp-trie = { path = "../trie", default-features = false} -trie-db = { version = "0.27.1", default-features = false } +trie-db = { version = "0.28.0", default-features = false } [dev-dependencies] array-bytes = "6.1" diff --git a/substrate/primitives/state-machine/README.md b/substrate/primitives/state-machine/README.md index aa244da62d50fc4ee549f1431d8f1dada8b48051..91d706f860e1de1cae8a51651e532b363411c57f 100644 --- a/substrate/primitives/state-machine/README.md +++ b/substrate/primitives/state-machine/README.md @@ -1,3 +1,3 @@ Substrate state machine implementation. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/state-machine/src/backend.rs b/substrate/primitives/state-machine/src/backend.rs index 2a25bdc54d9491faaf500f760aae7550dd1dadd3..ea9cd442d70ba55056d2ebdc27fa1eb26d0ba9c9 100644 --- a/substrate/primitives/state-machine/src/backend.rs +++ b/substrate/primitives/state-machine/src/backend.rs @@ -30,7 +30,7 @@ use sp_core::storage::{ChildInfo, StateVersion, TrackedStorageKey}; #[cfg(feature = "std")] use sp_core::traits::RuntimeCode; use sp_std::vec::Vec; -use sp_trie::PrefixedMemoryDB; +use sp_trie::{MerkleValue, PrefixedMemoryDB}; /// A struct containing arguments for iterating over the storage. #[derive(Default)] @@ -195,7 +195,17 @@ pub trait Backend: sp_std::fmt::Debug { /// Get keyed storage value hash or None if there is nothing associated. fn storage_hash(&self, key: &[u8]) -> Result, Self::Error>; - /// Get keyed child storage or None if there is nothing associated. + /// Get the merkle value or None if there is nothing associated. + fn closest_merkle_value(&self, key: &[u8]) -> Result>, Self::Error>; + + /// Get the child merkle value or None if there is nothing associated. + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Self::Error>; + + /// Get child keyed child storage or None if there is nothing associated. fn child_storage( &self, child_info: &ChildInfo, diff --git a/substrate/primitives/state-machine/src/trie_backend.rs b/substrate/primitives/state-machine/src/trie_backend.rs index cc7132181f90a0ceab30a66b2d78a54798031b25..afa80addd2bae1bb8694ded9c84f4b11a673f65f 100644 --- a/substrate/primitives/state-machine/src/trie_backend.rs +++ b/substrate/primitives/state-machine/src/trie_backend.rs @@ -30,7 +30,6 @@ use codec::Codec; use hash_db::HashDB; use hash_db::Hasher; use sp_core::storage::{ChildInfo, StateVersion}; -use sp_trie::PrefixedMemoryDB; #[cfg(feature = "std")] use sp_trie::{ cache::{LocalTrieCache, TrieCache}, @@ -39,6 +38,7 @@ use sp_trie::{ }; #[cfg(not(feature = "std"))] use sp_trie::{Error, NodeCodec}; +use sp_trie::{MerkleValue, PrefixedMemoryDB}; use trie_db::TrieCache as TrieCacheT; #[cfg(not(feature = "std"))] use trie_db::{node::NodeOwned, CachedValue}; @@ -405,6 +405,18 @@ where self.essence.child_storage(child_info, key) } + fn closest_merkle_value(&self, key: &[u8]) -> Result>, Self::Error> { + self.essence.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.essence.child_closest_merkle_value(child_info, key) + } + fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { let (is_cached, mut cache) = access_cache(&self.next_storage_key_cache, Option::take) .map(|cache| (cache.last_key == key, cache)) diff --git a/substrate/primitives/state-machine/src/trie_backend_essence.rs b/substrate/primitives/state-machine/src/trie_backend_essence.rs index 4bb51f4a134370795da03d0bbfefc347d72f12cd..ad7aeab899c8887e9fcaca2af1577b51f7533b1d 100644 --- a/substrate/primitives/state-machine/src/trie_backend_essence.rs +++ b/substrate/primitives/state-machine/src/trie_backend_essence.rs @@ -32,11 +32,12 @@ use sp_std::{boxed::Box, marker::PhantomData, vec::Vec}; #[cfg(feature = "std")] use sp_trie::recorder::Recorder; use sp_trie::{ - child_delta_trie_root, delta_trie_root, empty_child_trie_root, read_child_trie_hash, - read_child_trie_value, read_trie_value, + child_delta_trie_root, delta_trie_root, empty_child_trie_root, + read_child_trie_first_descedant_value, read_child_trie_hash, read_child_trie_value, + read_trie_first_descedant_value, read_trie_value, trie_types::{TrieDBBuilder, TrieError}, - DBValue, KeySpacedDB, NodeCodec, PrefixedMemoryDB, Trie, TrieCache, TrieDBRawIterator, - TrieRecorder, + DBValue, KeySpacedDB, MerkleValue, NodeCodec, PrefixedMemoryDB, Trie, TrieCache, + TrieDBRawIterator, TrieRecorder, }; #[cfg(feature = "std")] use std::{collections::HashMap, sync::Arc}; @@ -574,6 +575,39 @@ where }) } + /// Get the closest merkle value at given key. + pub fn closest_merkle_value(&self, key: &[u8]) -> Result>> { + let map_e = |e| format!("Trie lookup error: {}", e); + + self.with_recorder_and_cache(None, |recorder, cache| { + read_trie_first_descedant_value::, _>(self, &self.root, key, recorder, cache) + .map_err(map_e) + }) + } + + /// Get the child closest merkle value at given key. + pub fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result>> { + let Some(child_root) = self.child_root(child_info)? else { return Ok(None) }; + + let map_e = |e| format!("Trie lookup error: {}", e); + + self.with_recorder_and_cache(Some(child_root), |recorder, cache| { + read_child_trie_first_descedant_value::, _>( + child_info.keyspace(), + self, + &child_root, + key, + recorder, + cache, + ) + .map_err(map_e) + }) + } + /// Create a raw iterator over the storage. pub fn raw_iter(&self, args: IterArgs) -> Result> { let root = if let Some(child_info) = args.child_info.as_ref() { diff --git a/substrate/primitives/statement-store/README.md b/substrate/primitives/statement-store/README.md index 2cde0d669eefa1de0a71b874fdeb5acecf2c9cb8..f224f6ce43192ed86f99935fc356b4241caa0fd8 100644 --- a/substrate/primitives/statement-store/README.md +++ b/substrate/primitives/statement-store/README.md @@ -1,18 +1,35 @@ +# Statement store + Statement store is an off-chain data-store for signed statements accessible via RPC and OCW. -Nodes hold a number of statements with a proof of authenticity owing to an account ID. OCWs can place items in the data-store (with valid signatures) for any accounts whose keys they control. Users can also submit pre-signed statements via RPC. Statements can also be submitted from on-chain logic through an on-chain event. +Nodes hold a number of statements with a proof of authenticity owing to an account ID. OCWs can place items in the +data-store (with valid signatures) for any accounts whose keys they control. Users can also submit pre-signed statements +via RPC. Statements can also be submitted from on-chain logic through an on-chain event. -A new system event `NewStatement` is added to the runtime. This event allows any account on-chain to declare that they want to make a statement for the store. Within the node store and for broadcasting, the statement would be accompanied with the hash of the block and index of the event within it, essentially taking the place of a real signature. +A new system event `NewStatement` is added to the runtime. This event allows any account on-chain to declare that they +want to make a statement for the store. Within the node store and for broadcasting, the statement would be accompanied +with the hash of the block and index of the event within it, essentially taking the place of a real signature. -Statements comprise an optional proof of authenticity (e.g. a signature) and a number of fields. For statements without a proof, nodes would gossip statements randomly with a rate-limiter to minimise the chance of being overrun by a misbehaving node. These will generally be disregarded by nodes unless they are gossiped by several different peers or if a peer pays for it somehow (e.g. gossiping something valuable). +Statements comprise an optional proof of authenticity (e.g. a signature) and a number of fields. For statements without +a proof, nodes would gossip statements randomly with a rate-limiter to minimise the chance of being overrun by a +misbehaving node. These will generally be disregarded by nodes unless they are gossiped by several different peers or if +a peer pays for it somehow (e.g. gossiping something valuable). -Each field is effectively a key/value pair. Fields must be sorted and the same field type may not be repeated. Depending on which keys are present, clients may index the message for ease of retrieval. +Each field is effectively a key/value pair. Fields must be sorted and the same field type may not be repeated. Depending +on which keys are present, clients may index the message for ease of retrieval. Formally, `Statement` is equivalent to the type `Vec` and `Field` is the SCALE-encoded enumeration: -- 0: `AuthenticityProof(Proof)`: The signature of the message. For cryptography where the public key cannot be derived from the signature together with the message data, then this will also include the signer's public key. The message data is all fields of the messages fields except the signature concatenated together *without the length prefix that a `Vec` would usually imply*. This is so that the signature can be derived without needing to re-encode the statement. -- 1: `DecryptionKey([u8; 32])`: The decryption key identifier which should be used to decrypt the statement's data. In the absence of this field `Data` should be treated as not encrypted. -- 2: `Priority(u32)`: Priority specifier. Higher priority statements should be kept around at the cost of lower priority statements if multiple statements from the same sender are competing for persistence or transport. Nodes should disregard when considering unsigned statements. -- 3: `Channel([u8; 32])`: The channel identifier. Only one message of a given channel should be retained at once (the one of highest priority). Nodes should disregard when considering unsigned statements. +- 0: `AuthenticityProof(Proof)`: The signature of the message. For cryptography where the public key cannot be derived + from the signature together with the message data, then this will also include the signer's public key. The message + data is all fields of the messages fields except the signature concatenated together *without the length prefix that a + `Vec` would usually imply*. This is so that the signature can be derived without needing to re-encode the statement. +- 1: `DecryptionKey([u8; 32])`: The decryption key identifier which should be used to decrypt the statement's data. In + the absence of this field `Data` should be treated as not encrypted. +- 2: `Priority(u32)`: Priority specifier. Higher priority statements should be kept around at the cost of lower priority + statements if multiple statements from the same sender are competing for persistence or transport. Nodes should + disregard when considering unsigned statements. +- 3: `Channel([u8; 32])`: The channel identifier. Only one message of a given channel should be retained at once (the + one of highest priority). Nodes should disregard when considering unsigned statements. - 4: `Topic1([u8; 32]))`: First topic identifier. - 5: `Topic2([u8; 32]))`: Second topic identifier. - 6: `Topic3([u8; 32]))`: Third topic identifier. @@ -25,7 +42,7 @@ Formally, `Statement` is equivalent to the type `Vec` and `Field` is the - 2: `Secp256k1Ecdsa { signature: [u8; 65], signer: [u8; 33] )` - 3: `OnChain { who: [u8; 32], block_hash: [u8; 32], event_index: u64 }` -### Potential uses +## Potential uses Potential use-cases are various and include: - ring-signature aggregation; diff --git a/substrate/primitives/std/README.md b/substrate/primitives/std/README.md index 6dddd8fbbdd9d2fade460195ac2d9cee0eaf35ad..e186ccecf7b9f806b89b2e46990e227692b8789e 100644 --- a/substrate/primitives/std/README.md +++ b/substrate/primitives/std/README.md @@ -1,4 +1,4 @@ Lowest-abstraction level for the Substrate runtime: just exports useful primitives from std or client/alloc to be used with any code that depends on the runtime. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/storage/README.md b/substrate/primitives/storage/README.md index c33144fc4f66202804594d6cf0cc6886b5cad339..ac922f139ffef26a2ea725bd173e81b63347e66c 100644 --- a/substrate/primitives/storage/README.md +++ b/substrate/primitives/storage/README.md @@ -1,3 +1,3 @@ Primitive types for storage related stuff. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/timestamp/Cargo.toml b/substrate/primitives/timestamp/Cargo.toml index 7de2a7d904d09432b3ecfbf0e0e16a1f14db1d89..44b0fdd831c029fed9f1df4acbcd18705d1782f1 100644 --- a/substrate/primitives/timestamp/Cargo.toml +++ b/substrate/primitives/timestamp/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-trait = { version = "0.1.57", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } sp-inherents = { path = "../inherents", default-features = false} sp-runtime = { path = "../runtime", default-features = false} sp-std = { path = "../std", default-features = false} diff --git a/substrate/primitives/timestamp/README.md b/substrate/primitives/timestamp/README.md index a61a776912c93b4ab55525cba2d30146466ff905..41649cdfc8e2c1e091e55101daac57d843b402ef 100644 --- a/substrate/primitives/timestamp/README.md +++ b/substrate/primitives/timestamp/README.md @@ -1,3 +1,3 @@ Substrate core types and inherents for timestamps. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/timestamp/src/lib.rs b/substrate/primitives/timestamp/src/lib.rs index eeec73efbc8bee691b43c7cbfd06f4a8399fe674..d1bd2a3446e6b65ea2bf7a35942cae5f10f599be 100644 --- a/substrate/primitives/timestamp/src/lib.rs +++ b/substrate/primitives/timestamp/src/lib.rs @@ -140,7 +140,7 @@ pub enum InherentError { error("The time since the last timestamp is lower than the minimum period.") )] TooEarly, - /// The block timestamp is too far in the future + /// The block timestamp is too far in the future. #[cfg_attr(feature = "std", error("The timestamp of the block is too far in the future."))] TooFarInFuture, } diff --git a/substrate/primitives/tracing/README.md b/substrate/primitives/tracing/README.md index d66bb90016c7184e7178d437ab6944e4e36faee2..4582a34d918e8d259824521e5b7691e37f928238 100644 --- a/substrate/primitives/tracing/README.md +++ b/substrate/primitives/tracing/README.md @@ -12,4 +12,4 @@ Additionally, we have a const: `WASM_TRACE_IDENTIFIER`, which holds a span name to signal that the 'actual' span name and target should be retrieved instead from the associated Fields mentioned above. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/transaction-pool/README.md b/substrate/primitives/transaction-pool/README.md index 417565ebfce00c6a9961efa5def9bf327c37dd7f..209d23e9980fa5f733b4c3f570d663acfa9042da 100644 --- a/substrate/primitives/transaction-pool/README.md +++ b/substrate/primitives/transaction-pool/README.md @@ -1,3 +1,3 @@ Transaction pool primitives types & Runtime API. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/trie/Cargo.toml b/substrate/primitives/trie/Cargo.toml index 31eb009328c1490301427fdb98f4ff0bf181fec8..0b54514f6003d9c10f96640ff61172cee5bf8d2b 100644 --- a/substrate/primitives/trie/Cargo.toml +++ b/substrate/primitives/trie/Cargo.toml @@ -27,9 +27,9 @@ memory-db = { version = "0.32.0", default-features = false } nohash-hasher = { version = "0.2.0", optional = true } parking_lot = { version = "0.12.1", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } tracing = { version = "0.1.29", optional = true } -trie-db = { version = "0.27.0", default-features = false } +trie-db = { version = "0.28.0", default-features = false } trie-root = { version = "0.18.0", default-features = false } sp-core = { path = "../core", default-features = false} sp-std = { path = "../std", default-features = false} @@ -38,7 +38,7 @@ schnellru = { version = "0.2.1", optional = true } [dev-dependencies] array-bytes = "6.1" criterion = "0.4.0" -trie-bench = "0.37.0" +trie-bench = "0.38.0" trie-standardmap = "0.16.0" sp-runtime = { path = "../runtime" } diff --git a/substrate/primitives/trie/README.md b/substrate/primitives/trie/README.md index 634ba4bdead26d4a77e4c284866bf0cb37b3d93c..e82080da5ff899a3df6480da373612aa7b7b6c27 100644 --- a/substrate/primitives/trie/README.md +++ b/substrate/primitives/trie/README.md @@ -1,3 +1,3 @@ Utility functions to interact with Substrate's Base-16 Modified Merkle Patricia tree ("trie"). -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/trie/src/lib.rs b/substrate/primitives/trie/src/lib.rs index 94155458569bff5dcb880aaae5c49899877a4e01..1a1ed670454dce46f72ef16ff033af1336eadf8a 100644 --- a/substrate/primitives/trie/src/lib.rs +++ b/substrate/primitives/trie/src/lib.rs @@ -44,7 +44,6 @@ pub use storage_proof::{CompactProof, StorageProof}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; -pub use trie_db::proof::VerifyError; use trie_db::proof::{generate_proof, verify_proof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ @@ -53,6 +52,7 @@ pub use trie_db::{ CError, DBValue, Query, Recorder, Trie, TrieCache, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieDBRawIterator, TrieLayout, TrieMut, TrieRecorder, }; +pub use trie_db::{proof::VerifyError, MerkleValue}; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; @@ -295,6 +295,25 @@ pub fn read_trie_value( + db: &DB, + root: &TrieHash, + key: &[u8], + recorder: Option<&mut dyn TrieRecorder>>, + cache: Option<&mut dyn TrieCache>, +) -> Result>>, Box>> +where + DB: hash_db::HashDBRef, +{ + TrieDBBuilder::::new(db, root) + .with_optional_cache(cache) + .with_optional_recorder(recorder) + .build() + .lookup_first_descendant(key) +} + /// Read a value from the trie with given Query. pub fn read_trie_value_with< L: TrieLayout, @@ -397,6 +416,27 @@ where .get_hash(key) } +/// Read the [`trie_db::MerkleValue`] of the node that is the closest descendant for +/// the provided child key. +pub fn read_child_trie_first_descedant_value( + keyspace: &[u8], + db: &DB, + root: &TrieHash, + key: &[u8], + recorder: Option<&mut dyn TrieRecorder>>, + cache: Option<&mut dyn TrieCache>, +) -> Result>>, Box>> +where + DB: hash_db::HashDBRef, +{ + let db = KeySpacedDB::new(db, keyspace); + TrieDBBuilder::::new(&db, &root) + .with_optional_recorder(recorder) + .with_optional_cache(cache) + .build() + .lookup_first_descendant(key) +} + /// Read a value from the child trie with given query. pub fn read_child_trie_value_with( keyspace: &[u8], diff --git a/substrate/primitives/trie/src/recorder.rs b/substrate/primitives/trie/src/recorder.rs index 728dc836205b57f9d9ce7b8fd8190c0ef8eac681..154cee3f37dcba79c9eea34c3bec4bfc2fdcc0b7 100644 --- a/substrate/primitives/trie/src/recorder.rs +++ b/substrate/primitives/trie/src/recorder.rs @@ -379,6 +379,17 @@ impl>> trie_db::TrieRecord // that the value doesn't exist in the trie. self.update_recorded_keys(full_key, RecordedForKey::Value); }, + TrieAccess::InlineValue { full_key } => { + tracing::trace!( + target: LOG_TARGET, + key = ?sp_core::hexdisplay::HexDisplay::from(&full_key), + "Recorded inline value access for key", + ); + + // A value was accessed that is stored inline a node and we recorded all trie nodes + // to access this value. + self.update_recorded_keys(full_key, RecordedForKey::Value); + }, }; self.encoded_size_estimation.fetch_add(encoded_size_update, Ordering::Relaxed); diff --git a/substrate/primitives/version/Cargo.toml b/substrate/primitives/version/Cargo.toml index 3002566f74ff78aba61b326b52a67cd15db307eb..1ab51a08bbe3ca476e58ef2501a8d3ea61eaa3c6 100644 --- a/substrate/primitives/version/Cargo.toml +++ b/substrate/primitives/version/Cargo.toml @@ -19,7 +19,7 @@ impl-serde = { version = "0.4.0", default-features = false, optional = true } parity-wasm = { version = "0.45", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } serde = { version = "1.0.188", default-features = false, features = ["derive", "alloc"], optional = true } -thiserror = { version = "1.0.30", optional = true } +thiserror = { version = "1.0.48", optional = true } sp-core-hashing-proc-macro = { path = "../core/hashing/proc-macro" } sp-runtime = { path = "../runtime", default-features = false} sp-std = { path = "../std", default-features = false} diff --git a/substrate/primitives/version/README.md b/substrate/primitives/version/README.md index 84f0ae57d9dbeef1c84e4e01a8769b5f886e4501..8decaaa85f8c218c842d1fb81b35a4eb625b8d7c 100644 --- a/substrate/primitives/version/README.md +++ b/substrate/primitives/version/README.md @@ -1,3 +1,3 @@ Version module for the Substrate runtime; Provides a function that returns the runtime version. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/primitives/version/proc-macro/Cargo.toml b/substrate/primitives/version/proc-macro/Cargo.toml index 25b5e7906f620e2fb9955f0e0f63487577707b07..cc28b8f176b886901d55f17505c57e9a0c4a15dd 100644 --- a/substrate/primitives/version/proc-macro/Cargo.toml +++ b/substrate/primitives/version/proc-macro/Cargo.toml @@ -19,7 +19,7 @@ proc-macro = true codec = { package = "parity-scale-codec", version = "3.6.1", features = [ "derive" ] } proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "fold", "extra-traits", "visit"] } +syn = { version = "2.0.37", features = ["full", "fold", "extra-traits", "visit"] } [dev-dependencies] sp-version = { path = ".." } diff --git a/substrate/primitives/wasm-interface/README.md b/substrate/primitives/wasm-interface/README.md index 7e6c46581ae43c19b3b9b73d3d2b7975017c1a38..2e584adeb83cb4fe38034eb1aa0ad59365d98c15 100644 --- a/substrate/primitives/wasm-interface/README.md +++ b/substrate/primitives/wasm-interface/README.md @@ -1,3 +1,3 @@ Types and traits for interfacing between the host and the wasm runtime. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/scripts/ci/docker/subkey.Dockerfile.README.md b/substrate/scripts/ci/docker/subkey.Dockerfile.README.md index 30a5e8212150e1035075893def8b0f13f6678fa2..fe3359d01e811ab5a0ad41385660cea593acd4e7 100644 --- a/substrate/scripts/ci/docker/subkey.Dockerfile.README.md +++ b/substrate/scripts/ci/docker/subkey.Dockerfile.README.md @@ -1,8 +1,11 @@ -# The `subkey` program is a key management utility for Substrate-based blockchains. You can use the `subkey` program to perform the following tasks: +# Subkey + +The `subkey` program is a key management utility for Substrate-based blockchains. You can use the `subkey` program to +perform the following tasks * Generate and inspect cryptographically-secure public and private key pairs. * Restore keys from secret phrases and raw seeds. * Sign and verify signatures on messages. * Sign and verify signatures for encoded transactions. * Derive hierarchical deterministic child key pairs. -* [Documentation](https://docs.substrate.io/reference/command-line-tools/subkey/) \ No newline at end of file +* [Documentation](https://docs.substrate.io/reference/command-line-tools/subkey/) diff --git a/substrate/scripts/ci/docker/substrate.Dockerfile.README.md b/substrate/scripts/ci/docker/substrate.Dockerfile.README.md index 557fd8f835d73ac5628f33f1ea086b174803e69c..9e97701e92f6411658b73d1370be2f5c1ac005f0 100644 --- a/substrate/scripts/ci/docker/substrate.Dockerfile.README.md +++ b/substrate/scripts/ci/docker/substrate.Dockerfile.README.md @@ -1 +1 @@ -# Substrate Docker Image \ No newline at end of file +# Substrate Docker Image diff --git a/substrate/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md b/substrate/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md index 50f54a107e93398f121ac4504c49ec117db3d36f..2267cfbefb7fb6ddbf773da92e34d004e242c8e6 100644 --- a/substrate/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md +++ b/substrate/scripts/ci/monitoring/grafana-dashboards/README_dashboard.md @@ -1,4 +1,4 @@ -## Substrate Dashboard +# Substrate Dashboard Shared templated Grafana dashboards. diff --git a/substrate/scripts/ci/node-template-release/Cargo.toml b/substrate/scripts/ci/node-template-release/Cargo.toml index 6c4cee8ece335b106239bdf5c91372f187269812..c0e02758724676086011982c9cec9723c52eb2a3 100644 --- a/substrate/scripts/ci/node-template-release/Cargo.toml +++ b/substrate/scripts/ci/node-template-release/Cargo.toml @@ -11,7 +11,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } flate2 = "1.0" fs_extra = "1.3" glob = "0.3" diff --git a/substrate/test-utils/client/Cargo.toml b/substrate/test-utils/client/Cargo.toml index 91c30f5a8ecef0ffb525ba536c5763701d90975f..12863ac184c1192b3ac688ed12496ca5a6fc6928 100644 --- a/substrate/test-utils/client/Cargo.toml +++ b/substrate/test-utils/client/Cargo.toml @@ -18,7 +18,7 @@ async-trait = "0.1.57" codec = { package = "parity-scale-codec", version = "3.6.1" } futures = "0.3.21" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-client-api = { path = "../../client/api" } sc-client-db = { path = "../../client/db", default-features = false, features = [ "test-helpers", diff --git a/substrate/test-utils/runtime/Cargo.toml b/substrate/test-utils/runtime/Cargo.toml index 4f587d55e79da2d918f1a5aaa81d30496c927f7c..727d46cb8f53de54ab14704580614e2b41a34ec5 100644 --- a/substrate/test-utils/runtime/Cargo.toml +++ b/substrate/test-utils/runtime/Cargo.toml @@ -40,7 +40,7 @@ pallet-timestamp = { path = "../../frame/timestamp", default-features = false} sp-consensus-grandpa = { path = "../../primitives/consensus/grandpa", default-features = false, features = ["serde"] } sp-trie = { path = "../../primitives/trie", default-features = false} sp-transaction-pool = { path = "../../primitives/transaction-pool", default-features = false} -trie-db = { version = "0.27.0", default-features = false } +trie-db = { version = "0.28.0", default-features = false } sc-service = { path = "../../client/service", default-features = false, features = ["test-helpers"], optional = true} sp-state-machine = { path = "../../primitives/state-machine", default-features = false} sp-externalities = { path = "../../primitives/externalities", default-features = false} @@ -49,7 +49,7 @@ sp-externalities = { path = "../../primitives/externalities", default-features = array-bytes = { version = "6.1", optional = true } log = { version = "0.4.17", default-features = false } serde = { version = "1.0.188", features = ["alloc", "derive"], default-features = false } -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } [dev-dependencies] futures = "0.3.21" diff --git a/substrate/utils/build-script-utils/README.md b/substrate/utils/build-script-utils/README.md index 1c184f67326e3c3170b1a92767deffbdcbfa9c70..f857d04710b182836cba71a7b40d3b762c2bddb9 100644 --- a/substrate/utils/build-script-utils/README.md +++ b/substrate/utils/build-script-utils/README.md @@ -1,3 +1,3 @@ Crate with utility functions for `build.rs` scripts. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/utils/build-script-utils/src/version.rs b/substrate/utils/build-script-utils/src/version.rs index 4ee5376ed322d1a3fc3b1d5eb1677f4f93a3a3ea..309c6d71d77b1478211875fa8c06e372470c4ae0 100644 --- a/substrate/utils/build-script-utils/src/version.rs +++ b/substrate/utils/build-script-utils/src/version.rs @@ -41,6 +41,7 @@ pub fn generate_cargo_keys() { } }; + println!("cargo:rustc-env=SUBSTRATE_CLI_COMMIT_HASH={commit}"); println!("cargo:rustc-env=SUBSTRATE_CLI_IMPL_VERSION={}", get_version(&commit)) } diff --git a/substrate/utils/fork-tree/README.md b/substrate/utils/fork-tree/README.md index fef7db57f68f21ed590bc6df8e3a3444dc544476..ba573dfff41d98427bacb772e892b2308beeb3bf 100644 --- a/substrate/utils/fork-tree/README.md +++ b/substrate/utils/fork-tree/README.md @@ -1,4 +1,4 @@ Utility library for managing tree-like ordered data with logic for pruning the tree while finalizing nodes. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index 096e4be82d4bb9aa444f8b5ff38920cc7bfabfe5..9ba22e24faacda98a9ca9a0c076623097de0f3d1 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" chrono = "0.4" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.6.1" } comfy-table = { version = "7.0.1", default-features = false } handlebars = "4.2.2" @@ -27,8 +27,8 @@ log = "0.4.17" rand = { version = "0.8.4", features = ["small_rng"] } rand_pcg = "0.3.1" serde = "1.0.188" -serde_json = "1.0.85" -thiserror = "1.0.30" +serde_json = "1.0.107" +thiserror = "1.0.48" thousands = "0.2.0" frame-benchmarking = { path = "../../../frame/benchmarking" } frame-support = { path = "../../../frame/support" } diff --git a/substrate/utils/frame/benchmarking-cli/README.md b/substrate/utils/frame/benchmarking-cli/README.md index e6a48b61fd22741f42466dae5d8ee6eccd56d035..27673ea9580d7a24bb9af4a538dda77ebf6a1ccc 100644 --- a/substrate/utils/frame/benchmarking-cli/README.md +++ b/substrate/utils/frame/benchmarking-cli/README.md @@ -1,10 +1,11 @@ # The Benchmarking CLI -This crate contains commands to benchmark various aspects of Substrate and the hardware. -All commands are exposed by the Substrate node but can be exposed by any Substrate client. -The goal is to have a comprehensive suite of benchmarks that cover all aspects of Substrate and the hardware that its running on. +This crate contains commands to benchmark various aspects of Substrate and the hardware. +All commands are exposed by the Substrate node but can be exposed by any Substrate client. +The goal is to have a comprehensive suite of benchmarks that cover all aspects of Substrate and the hardware that its +running on. -Invoking the root benchmark command prints a help menu: +Invoking the root benchmark command prints a help menu: ```sh $ cargo run --profile=production -- benchmark @@ -25,10 +26,12 @@ SUBCOMMANDS: storage Benchmark the storage speed of a chain snapshot ``` -All examples use the `production` profile for correctness which makes the compilation *very* slow; for testing you can use `--release`. -For the final results the `production` profile and reference hardware should be used, otherwise the results are not comparable. +All examples use the `production` profile for correctness which makes the compilation *very* slow; for testing you can +use `--release`. +For the final results the `production` profile and reference hardware should be used, otherwise the results are not +comparable. -The sub-commands are explained in depth here: +The sub-commands are explained in depth here: - [block] Compare the weight of a historic block to its actual resource usage - [machine] Gauges the speed of the hardware - [overhead] Creates weight files for the *Block*- and *Extrinsic*-base weights diff --git a/substrate/utils/frame/benchmarking-cli/src/block/README.md b/substrate/utils/frame/benchmarking-cli/src/block/README.md index 7e99f0df9d43b9ccfd65b61c650c2846fb168605..e03f60e76276ff9a2c8f5c6f81c3d6f0fbdd8dae 100644 --- a/substrate/utils/frame/benchmarking-cli/src/block/README.md +++ b/substrate/utils/frame/benchmarking-cli/src/block/README.md @@ -1,65 +1,62 @@ # The `benchmark block` command -The whole benchmarking process in Substrate aims to predict the resource usage of an unexecuted block. -This command measures how accurate this prediction was by executing a block and comparing the predicted weight to its actual resource usage. -It can be used to measure the accuracy of the pallet benchmarking. +The whole benchmarking process in Substrate aims to predict the resource usage of an unexecuted block. This command +measures how accurate this prediction was by executing a block and comparing the predicted weight to its actual resource +usage. It can be used to measure the accuracy of the pallet benchmarking. -In the following it will be explained once for Polkadot and once for Substrate. +In the following it will be explained once for Polkadot and once for Substrate. ## Polkadot # 1 (Also works for Kusama, Westend and Rococo) -Suppose you either have a synced Polkadot node or downloaded a snapshot from [Polkachu]. -This example uses a pruned ParityDB snapshot from the 2022-4-19 with the last block being 9939462. -For pruned snapshots you need to know the number of the last block (to be improved [here]). -Pruned snapshots normally store the last 256 blocks, archive nodes can use any block range. +Suppose you either have a synced Polkadot node or downloaded a snapshot from [Polkachu]. This example uses a pruned +ParityDB snapshot from the 2022-4-19 with the last block being 9939462. For pruned snapshots you need to know the number +of the last block (to be improved [here]). Pruned snapshots normally store the last 256 blocks, archive nodes can use +any block range. -In this example we will benchmark just the last 10 blocks: +In this example we will benchmark just the last 10 blocks: ```sh cargo run --profile=production -- benchmark block --from 9939453 --to 9939462 --db paritydb ``` Output: ```pre -Block 9939453 with 2 tx used 4.57% of its weight ( 26,458,801 of 579,047,053 ns) -Block 9939454 with 3 tx used 4.80% of its weight ( 28,335,826 of 590,414,831 ns) -Block 9939455 with 2 tx used 4.76% of its weight ( 27,889,567 of 586,484,595 ns) -Block 9939456 with 2 tx used 4.65% of its weight ( 27,101,306 of 582,789,723 ns) -Block 9939457 with 2 tx used 4.62% of its weight ( 26,908,882 of 582,789,723 ns) -Block 9939458 with 2 tx used 4.78% of its weight ( 28,211,440 of 590,179,467 ns) -Block 9939459 with 4 tx used 4.78% of its weight ( 27,866,077 of 583,260,451 ns) -Block 9939460 with 3 tx used 4.72% of its weight ( 27,845,836 of 590,462,629 ns) -Block 9939461 with 2 tx used 4.58% of its weight ( 26,685,119 of 582,789,723 ns) -Block 9939462 with 2 tx used 4.60% of its weight ( 26,840,938 of 583,697,101 ns) +Block 9939453 with 2 tx used 4.57% of its weight ( 26,458,801 of 579,047,053 ns) +Block 9939454 with 3 tx used 4.80% of its weight ( 28,335,826 of 590,414,831 ns) +Block 9939455 with 2 tx used 4.76% of its weight ( 27,889,567 of 586,484,595 ns) +Block 9939456 with 2 tx used 4.65% of its weight ( 27,101,306 of 582,789,723 ns) +Block 9939457 with 2 tx used 4.62% of its weight ( 26,908,882 of 582,789,723 ns) +Block 9939458 with 2 tx used 4.78% of its weight ( 28,211,440 of 590,179,467 ns) +Block 9939459 with 4 tx used 4.78% of its weight ( 27,866,077 of 583,260,451 ns) +Block 9939460 with 3 tx used 4.72% of its weight ( 27,845,836 of 590,462,629 ns) +Block 9939461 with 2 tx used 4.58% of its weight ( 26,685,119 of 582,789,723 ns) +Block 9939462 with 2 tx used 4.60% of its weight ( 26,840,938 of 583,697,101 ns) ``` ### Output Interpretation (Only results from reference hardware are relevant) -Each block is executed multiple times and the results are averaged. -The percent number is the interesting part and indicates how much weight was used as compared to how much was predicted. -The closer to 100% this is without exceeding 100%, the better. -If it exceeds 100%, the block is marked with "**OVER WEIGHT!**" to easier spot them. This is not good since then the benchmarking under-estimated the weight. -This would mean that an honest validator would possibly not be able to keep up with importing blocks since users did not pay for enough weight. -If that happens the validator could lag behind the chain and get slashed for missing deadlines. -It is therefore important to investigate any overweight blocks. +Each block is executed multiple times and the results are averaged. The percent number is the interesting part and +indicates how much weight was used as compared to how much was predicted. The closer to 100% this is without exceeding +100%, the better. If it exceeds 100%, the block is marked with "**OVER WEIGHT!**" to easier spot them. This is not good +since then the benchmarking under-estimated the weight. This would mean that an honest validator would possibly not be +able to keep up with importing blocks since users did not pay for enough weight. If that happens the validator could lag +behind the chain and get slashed for missing deadlines. It is therefore important to investigate any overweight blocks. -In this example you can see an unexpected result; only < 5% of the weight was used! -The measured blocks can be executed much faster than predicted. -This means that the benchmarking process massively over-estimated the execution time. -Since they are off by so much, it is an issue [polkadot#5192]. +In this example you can see an unexpected result; only < 5% of the weight was used! The measured blocks can be executed +much faster than predicted. This means that the benchmarking process massively over-estimated the execution time. Since +they are off by so much, it is an issue [`polkadot#5192`]. The ideal range for these results would be 85-100%. ## Polkadot # 2 -Let's take a more interesting example where the blocks use more of their predicted weight. -Every day when validators pay out rewards, the blocks are nearly full. -Using an archive node here is the easiest. +Let's take a more interesting example where the blocks use more of their predicted weight. Every day when validators pay +out rewards, the blocks are nearly full. Using an archive node here is the easiest. -The Polkadot blocks TODO-TODO for example contain large batch transactions for staking payout. +The Polkadot blocks TODO-TODO for example contain large batch transactions for staking payout. ```sh cargo run --profile=production -- benchmark block --from TODO --to TODO --db paritydb @@ -71,21 +68,20 @@ TODO ## Substrate -It is also possible to try the procedure in Substrate, although it's a bit boring. +It is also possible to try the procedure in Substrate, although it's a bit boring. -First you need to create some blocks with either a local or dev chain. -This example will use the standard development spec. -Pick a non existing directory where the chain data will be stored, eg `/tmp/dev`. +First you need to create some blocks with either a local or dev chain. This example will use the standard development +spec. Pick a non existing directory where the chain data will be stored, eg `/tmp/dev`. ```sh cargo run --profile=production -- --dev -d /tmp/dev ``` -You should see after some seconds that it started to produce blocks: +You should see after some seconds that it started to produce blocks: ```pre … ✨ Imported #1 (0x801d…9189) … ``` -You can now kill the node with `Ctrl+C`. Then measure how long it takes to execute these blocks: +You can now kill the node with `Ctrl+C`. Then measure how long it takes to execute these blocks: ```sh cargo run --profile=production -- benchmark block --from 1 --to 1 --dev -d /tmp/dev --pruning archive ``` @@ -94,9 +90,8 @@ This will benchmark the first block. If you killed the node at a later point, yo Block 1 with 1 tx used 72.04% of its weight ( 4,945,664 of 6,864,702 ns) ``` -In this example the block used ~72% of its weight. -The benchmarking therefore over-estimated the effort to execute the block. -Since this block is empty, its not very interesting. +In this example the block used ~72% of its weight. The benchmarking therefore over-estimated the effort to execute the +block. Since this block is empty, its not very interesting. ## Arguments diff --git a/substrate/utils/frame/benchmarking-cli/src/machine/README.md b/substrate/utils/frame/benchmarking-cli/src/machine/README.md index f22a8ea54b81dbf9f4336809fde0671cda2aed11..d3c9a0ec84062dd26acf5cf5b20b89e4c6bb95d0 100644 --- a/substrate/utils/frame/benchmarking-cli/src/machine/README.md +++ b/substrate/utils/frame/benchmarking-cli/src/machine/README.md @@ -1,17 +1,17 @@ # The `benchmark machine` command -Different Substrate chains can have different hardware requirements. -It is therefore important to be able to quickly gauge if a piece of hardware fits a chains' requirements. -The `benchmark machine` command archives this by measuring key metrics and making them comparable. +Different Substrate chains can have different hardware requirements. +It is therefore important to be able to quickly gauge if a piece of hardware fits a chains' requirements. +The `benchmark machine` command archives this by measuring key metrics and making them comparable. -Invoking the command looks like this: +Invoking the command looks like this: ```sh cargo run --profile=production -- benchmark machine --dev ``` ## Output -The output on reference hardware: +The output on reference hardware: ```pre +----------+----------------+---------------+--------------+-------------------+ @@ -29,37 +29,49 @@ The output on reference hardware: +----------+----------------+---------------+--------------+-------------------+ ``` -The *score* is the average result of each benchmark. It always adheres to "higher is better". +The *score* is the average result of each benchmark. It always adheres to "higher is better". -The *category* indicate which part of the hardware was benchmarked: +The *category* indicate which part of the hardware was benchmarked: - **CPU** Processor intensive task - **Memory** RAM intensive task - **Disk** Hard drive intensive task -The *function* is the concrete benchmark that was run: -- **BLAKE2-256** The throughput of the [Blake2-256] cryptographic hashing function with 32 KiB input. The [blake2_256 function] is used in many places in Substrate. The throughput of a hash function strongly depends on the input size, therefore we settled to use a fixed input size for comparable results. -- **SR25519 Verify** Sr25519 is an optimized version of the [Curve25519] signature scheme. Signature verification is used by Substrate when verifying extrinsics and blocks. +The *function* is the concrete benchmark that was run: +- **BLAKE2-256** The throughput of the [Blake2-256] cryptographic hashing function with 32 KiB input. The [blake2_256 + function] is used in many places in Substrate. The throughput of a hash function strongly depends on the input size, + therefore we settled to use a fixed input size for comparable results. +- **SR25519 Verify** Sr25519 is an optimized version of the [Curve25519] signature scheme. Signature verification is + used by Substrate when verifying extrinsics and blocks. - **Copy** The throughput of copying memory from one place in the RAM to another. -- **Seq Write** The throughput of writing data to the storage location sequentially. It is important that the same disk is used that will later-on be used to store the chain data. -- **Rnd Write** The throughput of writing data to the storage location in a random order. This is normally much slower than the sequential write. +- **Seq Write** The throughput of writing data to the storage location sequentially. It is important that the same disk + is used that will later-on be used to store the chain data. +- **Rnd Write** The throughput of writing data to the storage location in a random order. This is normally much slower + than the sequential write. -The *score* needs to reach the *minimum* in order to pass the benchmark. This can be reduced with the `--tolerance` flag. +The *score* needs to reach the *minimum* in order to pass the benchmark. This can be reduced with the `--tolerance` +flag. -The *result* indicated if a specific benchmark was passed by the machine or not. The percent number is the relative score reached to the *minimum* that is needed. The `--tolerance` flag is taken into account for this decision. For example a benchmark that passes even with 95% since the *tolerance* was set to 10% would look like this: `✅ Pass ( 95.0 %)`. +The *result* indicated if a specific benchmark was passed by the machine or not. The percent number is the relative +score reached to the *minimum* that is needed. The `--tolerance` flag is taken into account for this decision. For +example a benchmark that passes even with 95% since the *tolerance* was set to 10% would look like this: `✅ Pass ( 95.0 +%)`. ## Interpretation -Ideally all results show a `Pass` and the program exits with code 0. Currently some of the benchmarks can fail even on reference hardware; they are still being improved to make them more deterministic. -Make sure to run nothing else on the machine when benchmarking it. +Ideally all results show a `Pass` and the program exits with code 0. Currently some of the benchmarks can fail even on +reference hardware; they are still being improved to make them more deterministic. +Make sure to run nothing else on the machine when benchmarking it. You can re-run them multiple times to get more reliable results. ## Arguments -- `--tolerance` A percent number to reduce the *minimum* requirement. This should be used to ignore outliers of the benchmarks. The default value is 10%. +- `--tolerance` A percent number to reduce the *minimum* requirement. This should be used to ignore outliers of the + benchmarks. The default value is 10%. - `--verify-duration` How long the verification benchmark should run. - `--disk-duration` How long the *read* and *write* benchmarks should run each. - `--allow-fail` Always exit the program with code 0. -- `--chain` / `--dev` Specify the chain config to use. This will be used to compare the results with the requirements of the chain (WIP). +- `--chain` / `--dev` Specify the chain config to use. This will be used to compare the results with the requirements of + the chain (WIP). - [`--base-path`] License: Apache-2.0 diff --git a/substrate/utils/frame/benchmarking-cli/src/overhead/README.md b/substrate/utils/frame/benchmarking-cli/src/overhead/README.md index 390bc09e41701ed5b3fe03308ea3857d5ba41ed1..648908010ba04cf6b19cf2aa50c0ea68491a1e28 100644 --- a/substrate/utils/frame/benchmarking-cli/src/overhead/README.md +++ b/substrate/utils/frame/benchmarking-cli/src/overhead/README.md @@ -1,21 +1,21 @@ # The `benchmark overhead` command -Each time an extrinsic or a block is executed, a fixed weight is charged as "execution overhead". -This is necessary since the weight that is calculated by the pallet benchmarks does not include this overhead. -The exact overhead to can vary per Substrate chain and needs to be calculated per chain. -This command calculates the exact values of these overhead weights for any Substrate chain that supports it. +Each time an extrinsic or a block is executed, a fixed weight is charged as "execution overhead". This is necessary +since the weight that is calculated by the pallet benchmarks does not include this overhead. The exact overhead to can +vary per Substrate chain and needs to be calculated per chain. This command calculates the exact values of these +overhead weights for any Substrate chain that supports it. ## How does it work? -The benchmark consists of two parts; the [`BlockExecutionWeight`] and the [`ExtrinsicBaseWeight`]. -Both are executed sequentially when invoking the command. +The benchmark consists of two parts; the [`BlockExecutionWeight`] and the [`ExtrinsicBaseWeight`]. Both are executed +sequentially when invoking the command. ## BlockExecutionWeight -The block execution weight is defined as the weight that it takes to execute an *empty block*. -It is measured by constructing an empty block and measuring its executing time. -The result are written to a `block_weights.rs` file which is created from a template. -The file will contain the concrete weight value and various statistics about the measurements. For example: +The block execution weight is defined as the weight that it takes to execute an *empty block*. It is measured by +constructing an empty block and measuring its executing time. The result are written to a `block_weights.rs` file which +is created from a template. The file will contain the concrete weight value and various statistics about the +measurements. For example: ```rust /// Time to execute an empty block. /// Calculated by multiplying the *Average* with `1` and adding `0`. @@ -34,16 +34,17 @@ pub const BlockExecutionWeight: Weight = Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(3_532_484), 0); ``` -In this example it takes 3.5 ms to execute an empty block. That means that it always takes at least 3.5 ms to execute *any* block. -This constant weight is therefore added to each block to ensure that Substrate budgets enough time to execute it. +In this example it takes 3.5 ms to execute an empty block. That means that it always takes at least 3.5 ms to execute +*any* block. This constant weight is therefore added to each block to ensure that Substrate budgets enough time to +execute it. ## ExtrinsicBaseWeight -The extrinsic base weight is defined as the weight that it takes to execute an *empty* extrinsic. -An *empty* extrinsic is also called a *NO-OP*. It does nothing and is the equivalent to the empty block form above. -The benchmark now constructs a block which is filled with only NO-OP extrinsics. -This block is then executed many times and the weights are measured. -The result is divided by the number of extrinsics in that block and the results are written to `extrinsic_weights.rs`. +The extrinsic base weight is defined as the weight that it takes to execute an *empty* extrinsic. An *empty* extrinsic +is also called a *NO-OP*. It does nothing and is the equivalent to the empty block form above. The benchmark now +constructs a block which is filled with only NO-OP extrinsics. This block is then executed many times and the weights +are measured. The result is divided by the number of extrinsics in that block and the results are written to +`extrinsic_weights.rs`. The relevant section in the output file looks like this: ```rust @@ -64,8 +65,9 @@ pub const ExtrinsicBaseWeight: Weight = Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(67_745), 0); ``` -In this example it takes 67.7 µs to execute a NO-OP extrinsic. That means that it always takes at least 67.7 µs to execute *any* extrinsic. -This constant weight is therefore added to each extrinsic to ensure that Substrate budgets enough time to execute it. +In this example it takes 67.7 µs to execute a NO-OP extrinsic. That means that it always takes at least 67.7 µs to +execute *any* extrinsic. This constant weight is therefore added to each extrinsic to ensure that Substrate budgets +enough time to execute it. ## Invocation @@ -106,15 +108,18 @@ The complete command for Polkadot looks like this: cargo run --profile=production -- benchmark overhead --chain=polkadot-dev --wasm-execution=compiled --weight-path=runtime/polkadot/constants/src/weights/ ``` -This will overwrite the the [block_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/block_weights.rs) and [extrinsic_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/extrinsic_weights.rs) files in the Polkadot runtime directory. -You can try the same for *Rococo* and to see that the results slightly differ. +This will overwrite the the +[block_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/block_weights.rs) +and +[extrinsic_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/extrinsic_weights.rs) +files in the Polkadot runtime directory. You can try the same for *Rococo* and to see that the results slightly differ. 👉 It is paramount to use `--profile=production` and `--wasm-execution=compiled` as the results are otherwise useless. ## Output Interpretation -Lower is better. The less weight the execution overhead needs, the better. -Since the weights of the overhead is charged per extrinsic and per block, a larger weight results in less extrinsics per block. -Minimizing this is important to have a large transaction throughput. +Lower is better. The less weight the execution overhead needs, the better. Since the weights of the overhead is charged +per extrinsic and per block, a larger weight results in less extrinsics per block. Minimizing this is important to have +a large transaction throughput. ## Arguments @@ -132,7 +137,10 @@ Minimizing this is important to have a large transaction throughput. License: Apache-2.0 -[`ExtrinsicBaseWeight`]: https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/extrinsic_weights.rs#L26 -[`BlockExecutionWeight`]: https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/block_weights.rs#L26 +[`ExtrinsicBaseWeight`]: + https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/extrinsic_weights.rs#L26 +[`BlockExecutionWeight`]: + https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/block_weights.rs#L26 -[System::Remark]: https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/system/src/lib.rs#L382 +[System::Remark]: + https://github.com/paritytech/substrate/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/system/src/lib.rs#L382 diff --git a/substrate/utils/frame/benchmarking-cli/src/shared/README.md b/substrate/utils/frame/benchmarking-cli/src/shared/README.md index 08e25b0e08f769ad016d3422fb9215ca44a84634..0b1128bca3a6283dd3420c9675327ef7d0bffee8 100644 --- a/substrate/utils/frame/benchmarking-cli/src/shared/README.md +++ b/substrate/utils/frame/benchmarking-cli/src/shared/README.md @@ -10,7 +10,10 @@ Contains code that is shared among multiple sub-commands. - `--weight-path` Set the file or directory to write the weight files to. - `--db` The database backend to use. This depends on your snapshot. - `--pruning` Set the pruning mode of the node. Some benchmarks require you to set this to `archive`. -- `--base-path` The location on the disk that should be used for the benchmarks. You can try this on different disks or even on a mounted RAM-disk. It is important to use the same location that will later-on be used to store the chain data to get the correct results. -- `--header` Optional file header which will be prepended to the weight output file. Can be used for adding LICENSE headers. +- `--base-path` The location on the disk that should be used for the benchmarks. You can try this on different disks or + even on a mounted RAM-disk. It is important to use the same location that will later-on be used to store the chain + data to get the correct results. +- `--header` Optional file header which will be prepended to the weight output file. Can be used for adding LICENSE + headers. License: Apache-2.0 diff --git a/substrate/utils/frame/benchmarking-cli/src/storage/README.md b/substrate/utils/frame/benchmarking-cli/src/storage/README.md index f61b7ba1bddd0cd324be80dd41d95a456dda9eed..95c83d2edbc5c68a00589951b0b83acd7a4adbb3 100644 --- a/substrate/utils/frame/benchmarking-cli/src/storage/README.md +++ b/substrate/utils/frame/benchmarking-cli/src/storage/README.md @@ -1,17 +1,19 @@ # The `benchmark storage` command -The cost of storage operations in a Substrate chain depends on the current chain state. -It is therefore important to regularly update these weights as the chain grows. -This sub-command measures the cost of storage operations for a concrete snapshot. +The cost of storage operations in a Substrate chain depends on the current chain state. +It is therefore important to regularly update these weights as the chain grows. +This sub-command measures the cost of storage operations for a concrete snapshot. -For the Substrate node it looks like this (for debugging you can use `--release`): +For the Substrate node it looks like this (for debugging you can use `--release`): ```sh cargo run --profile=production -- benchmark storage --dev --state-version=1 ``` -Running the command on Substrate itself is not verify meaningful, since the genesis state of the `--dev` chain spec is used. +Running the command on Substrate itself is not verify meaningful, since the genesis state of the `--dev` chain spec is +used. -The output for the Polkadot client with a recent chain snapshot will give you a better impression. A recent snapshot can be downloaded from [Polkachu]. +The output for the Polkadot client with a recent chain snapshot will give you a better impression. A recent snapshot can +be downloaded from [Polkachu]. Then run (remove the `--db=paritydb` if you have a RocksDB snapshot): ```sh cargo run --profile=production -- benchmark storage --dev --state-version=0 --db=paritydb --weight-path runtime/polkadot/constants/src/weights @@ -20,8 +22,8 @@ cargo run --profile=production -- benchmark storage --dev --state-version=0 --db This takes a while since reads and writes all keys from the snapshot: ```pre # The 'read' benchmark -Preparing keys from block BlockId::Number(9939462) -Reading 1379083 keys +Preparing keys from block BlockId::Number(9939462) +Reading 1379083 keys Time summary [ns]: Total: 19668919930 Min: 6450, Max: 1217259 @@ -31,11 +33,11 @@ Value size summary: Total: 265702275 Min: 1, Max: 1381859 Average: 192, Median: 80, Stddev: 3427.53 -Percentiles 99th, 95th, 75th: 3368, 383, 80 +Percentiles 99th, 95th, 75th: 3368, 383, 80 # The 'write' benchmark -Preparing keys from block BlockId::Number(9939462) -Writing 1379083 keys +Preparing keys from block BlockId::Number(9939462) +Writing 1379083 keys Time summary [ns]: Total: 98393809781 Min: 12969, Max: 13282577 @@ -49,12 +51,13 @@ Percentiles 99th, 95th, 75th: 3368, 383, 80 Writing weights to "paritydb_weights.rs" ``` -You will see that the [paritydb_weights.rs] files was modified and now contains new weights. -The exact command for Polkadot can be seen at the top of the file. -This uses the most recent block from your snapshot which is printed at the top. -The value size summary tells us that the pruned Polkadot chain state is ~253 MiB in size. -Reading a value on average takes (in this examples) 14.3 µs and writing 71.3 µs. -The interesting part in the generated weight file tells us the weight constants and some statistics about the measurements: +You will see that the [paritydb_weights.rs] files was modified and now contains new weights. The exact command for +Polkadot can be seen at the top of the file. +This uses the most recent block from your snapshot which is printed at the top. +The value size summary tells us that the pruned Polkadot chain state is ~253 MiB in size. +Reading a value on average takes (in this examples) 14.3 µs and writing 71.3 µs. +The interesting part in the generated weight file tells us the weight constants and some statistics about the +measurements: ```rust /// Time to read one storage item. /// Calculated by multiplying the *Average* of all values with `1.1` and adding `0`. @@ -90,7 +93,8 @@ write: 71_347 * constants::WEIGHT_REF_TIME_PER_NANOS, ## Arguments - `--db` Specify which database backend to use. This greatly influences the results. -- `--state-version` Set the version of the state encoding that this snapshot uses. Should be set to `1` for Substrate `--dev` and `0` for Polkadot et al. Using the wrong version can corrupt the snapshot. +- `--state-version` Set the version of the state encoding that this snapshot uses. Should be set to `1` for Substrate + `--dev` and `0` for Polkadot et al. Using the wrong version can corrupt the snapshot. - [`--mul`](../shared/README.md#arguments) - [`--add`](../shared/README.md#arguments) - [`--metric`](../shared/README.md#arguments) @@ -103,4 +107,5 @@ License: Apache-2.0 [Polkachu]: https://polkachu.com/snapshots -[paritydb_weights.rs]: https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/paritydb_weights.rs#L60 +[paritydb_weights.rs]: + https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/paritydb_weights.rs#L60 diff --git a/substrate/utils/frame/frame-utilities-cli/Cargo.toml b/substrate/utils/frame/frame-utilities-cli/Cargo.toml index d191506a2acbbc5d2e1626b46eb5d048dd0cf6e9..5a3365dc900381f25e4f1e634ae2c8e8005707c3 100644 --- a/substrate/utils/frame/frame-utilities-cli/Cargo.toml +++ b/substrate/utils/frame/frame-utilities-cli/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/substrate-frame-cli" readme = "README.md" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } frame-support = { path = "../../../frame/support" } frame-system = { path = "../../../frame/system" } sc-cli = { path = "../../../client/cli" } diff --git a/substrate/utils/frame/frame-utilities-cli/README.md b/substrate/utils/frame/frame-utilities-cli/README.md index b1e4f869af7580687e663c5bf2e3c89a1badd35a..54467a3ad7704451f4064f8334ea374b8c7f6bca 100644 --- a/substrate/utils/frame/frame-utilities-cli/README.md +++ b/substrate/utils/frame/frame-utilities-cli/README.md @@ -1,3 +1,3 @@ frame-system CLI utilities -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml b/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml index 43005ca5c4a3121680c32bb39d4a4c4c90afa352..e1490aa363ca763f3cc2d587c9355eb119e7a98d 100644 --- a/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml +++ b/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -14,4 +14,4 @@ kitchensink-runtime = { path = "../../../../bin/node/runtime" } generate-bags = { path = ".." } # third-party -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } diff --git a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml index d82377593c7161455279adeee4a81859df28558b..8ce9d06c2d11e650bb6687de70e628da3d6d4e22 100644 --- a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml +++ b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -19,7 +19,7 @@ serde = { version = "1", features = ["derive"] } sp-core = { path = "../../../../primitives/core" } sp-state-machine = { path = "../../../../primitives/state-machine" } sp-trie = { path = "../../../../primitives/trie" } -trie-db = "0.27.0" +trie-db = "0.28.0" jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } diff --git a/substrate/utils/frame/rpc/support/README.md b/substrate/utils/frame/rpc/support/README.md index ca5750612931ce385a2251b86a350afca5dd97da..72e39b761183eaf65905ad47f2aa1f3f958abed4 100644 --- a/substrate/utils/frame/rpc/support/README.md +++ b/substrate/utils/frame/rpc/support/README.md @@ -1,4 +1,4 @@ Combines [sc_rpc_api::state::StateClient] with [frame_support::storage::generator] traits to provide strongly typed chain state queries over rpc. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/utils/frame/rpc/system/README.md b/substrate/utils/frame/rpc/system/README.md index 38986983d93c5b3794f93c56d50840de9a4ead18..4eb4a74ceceb6da883d5995ec92c90644f5e6912 100644 --- a/substrate/utils/frame/rpc/system/README.md +++ b/substrate/utils/frame/rpc/system/README.md @@ -1,3 +1,3 @@ System FRAME specific RPC methods. -License: Apache-2.0 \ No newline at end of file +License: Apache-2.0 diff --git a/substrate/utils/frame/try-runtime/cli/Cargo.toml b/substrate/utils/frame/try-runtime/cli/Cargo.toml index 7d1c204422ae3b72d67ba991db198f6755b2df4d..3f693ca6c82d86e0c853bd6436a059164cac7388 100644 --- a/substrate/utils/frame/try-runtime/cli/Cargo.toml +++ b/substrate/utils/frame/try-runtime/cli/Cargo.toml @@ -35,12 +35,12 @@ frame-try-runtime = { path = "../../../../frame/try-runtime", optional = true} substrate-rpc-client = { path = "../../rpc/client" } async-trait = "0.1.57" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.4", features = ["derive"] } hex = { version = "0.4.3", default-features = false } log = "0.4.17" parity-scale-codec = "3.6.1" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" zstd = { version = "0.12.4", default-features = false } [dev-dependencies] diff --git a/substrate/utils/prometheus/README.md b/substrate/utils/prometheus/README.md index 9dd0882105c698e382c8d35b31244ca06e3127e1..507b9b24fb7e66040a3820fa65d144e6618fa796 100644 --- a/substrate/utils/prometheus/README.md +++ b/substrate/utils/prometheus/README.md @@ -2,15 +2,19 @@ ## Introduction -[Prometheus](https://prometheus.io/) is one of the most widely used monitoring tools for managing highly available services supported by [Cloud Native Computing Foundation](https://www.cncf.io/). By providing Prometheus metrics in Substrate, node operators can easily adopt widely used display/alert tools such -as [Grafana](https://grafana.com/) and [Alertmanager](https://prometheus.io/docs/alerting/alertmanager/). Easy access to such monitoring tools will benefit parachain developers/operators and validators to have much higher availability of their services. +[Prometheus](https://prometheus.io/) is one of the most widely used monitoring tools for managing highly available +services supported by [Cloud Native Computing Foundation](https://www.cncf.io/). By providing Prometheus metrics in +Substrate, node operators can easily adopt widely used display/alert tools such as [Grafana](https://grafana.com/) and +[Alertmanager](https://prometheus.io/docs/alerting/alertmanager/). Easy access to such monitoring tools will benefit +parachain developers/operators and validators to have much higher availability of their services. Metrics will be served under `/metrics` on TCP port 9615 by default. ## Quick Start - + 1. From the root of the repository start Substrate `cargo run --release`. 2. In another terminal run `curl localhost:9615/metrics` to retrieve the metrics. -To learn how to configure Prometheus see the Prometheus [Getting Started](https://prometheus.io/docs/prometheus/latest/getting_started/) guide. \ No newline at end of file +To learn how to configure Prometheus see the Prometheus [Getting +Started](https://prometheus.io/docs/prometheus/latest/getting_started/) guide. diff --git a/substrate/utils/prometheus/src/lib.rs b/substrate/utils/prometheus/src/lib.rs index 581666635ab54698d0426bc80007e245ee8568fe..ed1f9137aec4efad97218b2340e49436a05fd577 100644 --- a/substrate/utils/prometheus/src/lib.rs +++ b/substrate/utils/prometheus/src/lib.rs @@ -111,10 +111,16 @@ async fn init_prometheus_with_listener( } }); - let server = Server::builder(listener).serve(service); + let (signal, on_exit) = tokio::sync::oneshot::channel::<()>(); + let server = Server::builder(listener).serve(service).with_graceful_shutdown(async { + let _ = on_exit.await; + }); let result = server.await.map_err(Into::into); + // Gracefully shutdown server, otherwise the server does not stop if it has open connections + let _ = signal.send(()); + result } diff --git a/substrate/utils/wasm-builder/README.md b/substrate/utils/wasm-builder/README.md index b1ccb1b35b10e9e1bc42cd1d6726a82895fc1a7c..db32f5cbc955cc36b056c612caa65c1ef45ca683 100644 --- a/substrate/utils/wasm-builder/README.md +++ b/substrate/utils/wasm-builder/README.md @@ -1,14 +1,15 @@ # Wasm builder is a utility for building a project as a Wasm binary -The Wasm builder is a tool that integrates the process of building the WASM binary of your project into the main -`cargo` build process. +The Wasm builder is a tool that integrates the process of building the WASM binary of your project into the main `cargo` +build process. ## Project setup A project that should be compiled as a Wasm binary needs to: 1. Add a `build.rs` file. -2. Add `wasm-builder` as dependency into `build-dependencies` (can be made optional and only enabled when `std` feature is used). +2. Add `wasm-builder` as dependency into `build-dependencies` (can be made optional and only enabled when `std` feature + is used). The `build.rs` file needs to contain the following code: @@ -35,43 +36,40 @@ As the final step, you need to add the following to your project: include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); ``` -This will include the generated Wasm binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. -The former is a compact Wasm binary and the latter is the Wasm binary as being generated by the compiler. -Both variables have `Option<&'static [u8]>` as type. +This will include the generated Wasm binary as two constants `WASM_BINARY` and `WASM_BINARY_BLOATY`. The former is a +compact Wasm binary and the latter is the Wasm binary as being generated by the compiler. Both variables have +`Option<&'static [u8]>` as type. ### Features -Wasm builder supports to enable cargo features while building the Wasm binary. By default it will -enable all features in the wasm build that are enabled for the native build except the -`default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm` -feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the -Wasm binary. If this feature is not present, it will not be enabled. +Wasm builder supports to enable cargo features while building the Wasm binary. By default it will enable all features in +the wasm build that are enabled for the native build except the `default` and `std` features. Besides that, wasm builder +supports the special `runtime-wasm` feature. This `runtime-wasm` feature will be enabled by the wasm builder when it +compiles the Wasm binary. If this feature is not present, it will not be enabled. ## Environment variables By using environment variables, you can configure which Wasm binaries are built and how: -- `SKIP_WASM_BUILD` - Skips building any Wasm binary. This is useful when only native should be recompiled. - If this is the first run and there doesn't exist a Wasm binary, this will set both - variables to `None`. -- `WASM_BUILD_TYPE` - Sets the build type for building Wasm binaries. Supported values are `release` or `debug`. - By default the build type is equal to the build type used by the main build. -- `FORCE_WASM_BUILD` - Can be set to force a Wasm build. On subsequent calls the value of the variable - needs to change. As wasm-builder instructs `cargo` to watch for file changes - this environment variable should only be required in certain circumstances. +- `SKIP_WASM_BUILD` - Skips building any Wasm binary. This is useful when only native should be recompiled. If this is + the first run and there doesn't exist a Wasm binary, this will set both variables to `None`. +- `WASM_BUILD_TYPE` - Sets the build type for building Wasm binaries. Supported values are `release` or `debug`. By + default the build type is equal to the build type used by the main build. +- `FORCE_WASM_BUILD` - Can be set to force a Wasm build. On subsequent calls the value of the variable needs to change. + As wasm-builder instructs `cargo` to watch for file changes this environment variable should only + be required in certain circumstances. - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm binary. - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build. -- `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path needs - to be absolute. -- `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The - format needs to be the same as used by cargo, e.g. `nightly-2020-02-20`. -- `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to prevent network access. Useful in offline environments. +- `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path needs to be absolute. +- `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The format needs to be the same + as used by cargo, e.g. `nightly-2020-02-20`. +- `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to prevent network access. + Useful in offline environments. -Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`. -Where `PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will -be `NODE_RUNTIME`. +Each project can be skipped individually by using the environment variable `SKIP_PROJECT_NAME_WASM_BUILD`. Where +`PROJECT_NAME` needs to be replaced by the name of the cargo project, e.g. `node-runtime` will be `NODE_RUNTIME`. -## Prerequisites: +## Prerequisites Wasm builder requires the following prerequisites for building the Wasm binary: @@ -81,9 +79,8 @@ or - rust stable and version at least 1.68.0 + `wasm32-unknown-unknown` toolchain -If a specific rust is installed with `rustup`, it is important that the wasm target is -installed as well. For example if installing the rust from 20.02.2020 using `rustup -install nightly-2020-02-20`, the wasm target needs to be installed as well `rustup target add -wasm32-unknown-unknown --toolchain nightly-2020-02-20`. +If a specific rust is installed with `rustup`, it is important that the wasm target is installed as well. For example if +installing the rust from 20.02.2020 using `rustup install nightly-2020-02-20`, the wasm target needs to be installed as +well `rustup target add wasm32-unknown-unknown --toolchain nightly-2020-02-20`. License: Apache-2.0 diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs index 208b56077669e19d5d61c490cbcc22bfa4bff471..e0a30644b231adce3eb539a1e5226781415df17d 100644 --- a/substrate/utils/wasm-builder/src/builder.rs +++ b/substrate/utils/wasm-builder/src/builder.rs @@ -203,7 +203,10 @@ fn generate_crate_skip_build_env_name() -> String { /// Checks if the build of the WASM binary should be skipped. fn check_skip_build() -> bool { env::var(crate::SKIP_BUILD_ENV).is_ok() || - env::var(generate_crate_skip_build_env_name()).is_ok() + env::var(generate_crate_skip_build_env_name()).is_ok() || + // If we are running in docs.rs, let's skip building. + // https://docs.rs/about/builds#detecting-docsrs + env::var("DOCS_RS").is_ok() } /// Provide a dummy WASM binary if there doesn't exist one. diff --git a/substrate/zombienet/0001-basic-warp-sync/README.md b/substrate/zombienet/0001-basic-warp-sync/README.md index 7550ca89236fbe6e50f901498376db324e594283..1f8bddee640cc08b115b1a81d25db228c833592f 100644 --- a/substrate/zombienet/0001-basic-warp-sync/README.md +++ b/substrate/zombienet/0001-basic-warp-sync/README.md @@ -35,8 +35,8 @@ Once the zombienet is stopped, the database snapshot (`{alice,bob}/data/chains/local_testnet/db/` dirs) was created using the following commands: ```bash -mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/ -cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/ +mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/ +cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/ cp -r db-test-gen/bob/data/chains/local_testnet/db/full db-snapshot/bob/data/chains/local_testnet/db/ ``` @@ -72,7 +72,8 @@ Chain spec was simply built with: substrate build-spec --chain=local > chain-spec.json ``` -Please note that `chain-spec.json` committed into repository is `raw` version produced by `zombienet` during database snapshot generation. Zombienet applies some modifications to plain versions of chain-spec. +Please note that `chain-spec.json` committed into repository is `raw` version produced by `zombienet` during database +snapshot generation. Zombienet applies some modifications to plain versions of chain-spec. # Run the test Test can be run with the following command: @@ -91,7 +92,7 @@ index 23fb13cfb0..89f8646291 100644 --- a/bin/node/runtime/src/constants.rs +++ b/bin/node/runtime/src/constants.rs @@ -63,7 +63,7 @@ pub mod time { - + // NOTE: Currently it is not possible to change the epoch duration after the chain has started. // Attempting to do so will brick block production. - pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 10 * MINUTES; diff --git a/substrate/zombienet/0002-validators-warp-sync/README.md b/substrate/zombienet/0002-validators-warp-sync/README.md index 662360fbf3c68adc545767963de1eeeb4f624087..99fcb73138045ab62e945fcbd1eb757b4f6e5dcc 100644 --- a/substrate/zombienet/0002-validators-warp-sync/README.md +++ b/substrate/zombienet/0002-validators-warp-sync/README.md @@ -1,4 +1,3 @@ -Refer to ../0001-basic-warp-sync/README.md for more details. This test is nearly a clone. We want to warp-sync validators and make sure they can build blocks. -0001-basic-warp-sync chainspec (copied) and database are reused in this test. - - +Refer to ../0001-basic-warp-sync/README.md for more details. This test is nearly a clone. We want to warp-sync +validators and make sure they can build blocks. 0001-basic-warp-sync chainspec (copied) and database are reused in this +test. diff --git a/substrate/zombienet/0003-block-building-warp-sync/README.md b/substrate/zombienet/0003-block-building-warp-sync/README.md index 311d3550f766395f1294d37ecec57a2ae8f3a872..08dc730a58b6d7c43c076d3f64277eee22d526c8 100644 --- a/substrate/zombienet/0003-block-building-warp-sync/README.md +++ b/substrate/zombienet/0003-block-building-warp-sync/README.md @@ -1,4 +1,3 @@ -Refer to ../0001-basic-warp-sync/README.md for more details. This test is nearly a clone. We want to warp-sync full nodes in the presence of validators. +Refer to ../0001-basic-warp-sync/README.md for more details. This test is nearly a clone. We want to warp-sync full nodes +in the presence of validators. 0001-basic-warp-sync chainspec (copied) and database are reused in this test. - -