PRDoc new schema (#1946)

## Overview

This PR brings in the new version of prdoc v0.0.6 and allows:
- local schema
- local config
- local template

It also fixes the existing prdoc files to match the new schema.

## todo

- [x] add a brief doc/tldr to help contributors get started
- [x] test CI
- [x] finalize schema
- [x] publish the next `prdoc` cli version (v0.0.7 or above)

---------

Co-authored-by: Egor_P <egor@parity.io>
Co-authored-by: Bastian Köcher <git@kchr.de>

.github/review-bot.yml

 rules:
   - name: CI files
     condition:
-      include: 
+      include:
         - ^\.gitlab-ci\.yml
         - ^docker/.*
         - ^\.github/.*
@@ -19,12 +19,12 @@ rules:
   - name: Audit rules
     type: basic
     condition:
-      include: 
+      include:
         - ^polkadot/runtime\/(kusama|polkadot|common)\/.*
         - ^polkadot/primitives/src\/.+\.rs$
         - ^substrate/primitives/.*
         - ^substrate/frame/.*
-      exclude: 
+      exclude:
         - ^polkadot/runtime\/(kusama|polkadot)\/src\/weights\/.+\.rs$
         - ^substrate\/frame\/.+\.md$
     minApprovals: 1
@@ -80,7 +80,7 @@ rules:
   # if there are any changes in the bridges subtree (in case of backport changes back to bridges repo)
   - name: Bridges subtree files
     type: basic
-    condition: 
+    condition:
       include:
         - ^bridges/.*
     minApprovals: 1
@@ -91,7 +91,7 @@ rules:
 
   - name: FRAME coders substrate
     condition:
-      include: 
+      include:
         - ^substrate/frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*))
     type: "and"
     reviewers:
@@ -105,7 +105,7 @@ rules:
   # Protection of THIS file
   - name: Review Bot
     condition:
-      include: 
+      include:
         - review-bot\.yml
     type: "and"
     reviewers:

.github/workflows/check-prdoc.yml

@@ -6,26 +6,27 @@ on:
   merge_group:
 
 env:
-  IMAGE: paritytech/prdoc:v0.0.5
+  IMAGE: docker.io/paritytech/prdoc:v0.0.7
   API_BASE: https://api.github.com/repos
   REPO: ${{ github.repository }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   GITHUB_PR: ${{ github.event.pull_request.number }}
-  MOUNT: /prdoc
   ENGINE: docker
+  PRDOC_DOC: https://github.com/paritytech/polkadot-sdk/blob/master/docs/prdoc.md
 
 jobs:
   check-prdoc:
     runs-on: ubuntu-latest
     steps:
+      # we cannot show the version in this step (ie before checking out the repo)
+      # due to https://github.com/paritytech/prdoc/issues/15
       - name: Skip merge queue
         if: ${{ contains(github.ref, 'gh-readonly-queue') }}
         run: exit 0
       - name: Pull image
         run: |
           echo "Pulling $IMAGE"
-          docker pull $IMAGE
-          docker run --rm $IMAGE --version
+          $ENGINE pull $IMAGE
 
       - name: Check if PRdoc is required
         id: get-labels
@@ -36,18 +37,29 @@ jobs:
           echo "Labels: ${labels}"
           echo "labels=${labels}" >> "$GITHUB_OUTPUT"
 
-      - name: No PRdoc required
+      - name: Checkout repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
+
+      - name: Check PRDoc version
+        run: |
+          $ENGINE run --rm -v $PWD:/repo $IMAGE --version
+
+      - name: Early exit if PR is silent
         if: ${{ contains(steps.get-labels.outputs.labels, 'R0') }}
         run: |
-          echo "PR detected as silent, no PRdoc is required, exiting..."
+          hits=$(find prdoc -name "pr_$GITHUB_PR*.prdoc" | wc -l)
+          if (( hits > 0 )); then
+            echo "PR detected as silent, but a PRDoc was found, checking it as information"
+            $ENGINE run --rm -v $PWD:/repo $IMAGE check -n ${GITHUB_PR} || echo "Ignoring failure"
+          else
+            echo "PR detected as silent, no PRDoc found, exiting..."
+          fi
+          echo "If you want to add a PRDoc, please refer to $PRDOC_DOC"
           exit 0
 
-      - name: Checkout repo
-        if: ${{ !contains(steps.get-labels.outputs.labels, 'R0') }}
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
-
       - name: PRdoc check for PR#${{ github.event.pull_request.number }}
         if: ${{ !contains(steps.get-labels.outputs.labels, 'R0') }}
         run: |
-          echo "Checking for PR#${GITHUB_PR} in $MOUNT"
-          $ENGINE run --rm -v $PWD/prdoc:/doc $IMAGE check -n ${GITHUB_PR} || true
+          echo "Checking for PR#${GITHUB_PR}"
+          echo "You can find more information about PRDoc at $PRDOC_DOC"
+          $ENGINE run --rm -v $PWD:/repo $IMAGE check -n ${GITHUB_PR}

.prdoc.toml

+# Config file for prdoc, see https://github.com/paritytech/prdoc
+
+version = 1
+schema = "prdoc/schema_user.json"
+output_dir = "prdoc"
+prdoc_folders = ["prdoc"]
+template = "prdoc/.template.prdoc"

docs/CONTRIBUTING.md

@@ -93,22 +93,12 @@ The reviewers are also responsible to check:
 
 All Pull Requests must contain proper title & description.
 
-Some Pull Requests can be exempt of `prdoc` documentation, those
-must be labelled with
+Some Pull Requests can be exempt of `prdoc` documentation, those must be labelled with
 [`R0-silent`](https://github.com/paritytech/labels/blob/main/ruled_labels/specs_polkadot-sdk.yaml#L89-L91).
 
 Non "silent" PRs must come with documentation in the form of a `.prdoc` file.
-A `.prdoc` documentation is made of a text file (YAML) named `/prdoc/pr_NNNN.prdoc` where `NNNN` is the PR number.
-For convenience, those file can also contain a short description/title: `/prdoc/pr_NNNN_pr-foobar.prdoc`.
 
-The CI automation checks for the presence and validity of a `prdoc` in the `/prdoc` folder.
-Those files need to comply with a specific [schema](https://github.com/paritytech/prdoc/blob/master/schema_user.json). It
-is highly recommended to [make your editor aware](https://github.com/paritytech/prdoc#schemas) of the schema as it is
-self-described and will assist you in writing correct content.
-
-This schema is also embedded in the
-[prdoc](https://github.com/paritytech/prdoc) utility that can also be used to generate and check the validity of a
-`prdoc` locally.
+See more about `prdoc` [here](./prdoc.md)
 
 ## Helping out
 

docs/DOCUMENTATION_GUIDELINES.md

@@ -225,7 +225,7 @@ For the top-level pallet docs, consider the following template:
 //!
 //! ## Pallet API
 //!
-//! <Reminder: inside the [`pallet`] module, a template that leads the reader to the relevant items is auto-generated. There is no need to repeat 
+//! <Reminder: inside the [`pallet`] module, a template that leads the reader to the relevant items is auto-generated. There is no need to repeat
 //! things like "See Config trait for ...", which are generated inside [`pallet`] here anyways. You can use the line below as-is:>
 //!
 //! See the [`pallet`] module for more information about the interfaces this pallet exposes, including its
@@ -349,3 +349,7 @@ Consider the fact that, similar to dispatchables, these docs will be part of the
 and might be used by wallets and explorers.
 
 Specifically for `error`, explain why the error has happened, and what can be done in order to avoid it.
+
+## Documenting Changes/PR
+
+See [PRDoc](./prdoc.md).

docs/prdoc.md

+# PRDoc
+
+## Intro
+
+With the merge of [PR #1946](https://github.com/paritytech/polkadot-sdk/pull/1946), a new method for
+documenting changes has been introduced: `prdoc`. The [prdoc repository](https://github.com/paritytech/prdoc)
+contains more documentation and tooling.
+
+The current document describes how to quickly get started authoring `PRDoc` files.
+
+## Requirements
+
+When creating a PR, the author needs to decides with the `R0` label whether the change (PR) should
+appear in the release notes or not.
+
+Labelling a PR with `R0` means that no `PRDoc` is required.
+
+A PR without the `R0` label **does** require a valid `PRDoc` file to be introduced in the PR.
+
+## PRDoc how-to
+
+A `.prdoc` file is a YAML file with a defined structure (ie JSON Schema).
+
+For significant changes, a `.prdoc` file is mandatory and the file must meet the following
+requirements:
+- file named `pr_NNNN.prdoc` where `NNNN` is the PR number.
+  For convenience, those file can also contain a short description: `pr_NNNN_foobar.prdoc`.
+- located under the [`prdoc` folder](https://github.com/paritytech/polkadot-sdk/tree/master/prdoc) of the repository
+- compliant with the [JSON schema](https://json-schema.org/) defined in `prdoc/schema_user.json`
+
+Those requirements can be fulfilled manually without any tooling but a text editor.
+
+## Tooling
+
+Users might find the following helpers convenient:
+- Setup VSCode to be aware of the prdoc schema: see [using VSCode](https://github.com/paritytech/prdoc#using-vscode)
+- Using the `prdoc` cli to:
+  - generate a `PRDoc` file from a [template defined in the Polkadot SDK
+    repo](https://github.com/paritytech/polkadot-sdk/blob/master/prdoc/.template.prdoc) simply providing a PR number
+  - check the validity of one or more `PRDoc` files
+
+## `prdoc` cli usage
+
+The `prdoc` cli documentation can be found at https://github.com/paritytech/prdoc#prdoc
+
+tldr:
+- `prdoc generate <NNNN>`
+- `prdoc check -n <NNNN>`
+
+where <NNNN> is the PR number.
+
+## Pick an audience
+
+While describing a PR, the author needs to consider which audience(s) need to be addressed.
+The list of valid audiences is described and documented in the JSON schema as follow:
+
+- `Node Dev`: Those who build around the client side code. Alternative client builders, SMOLDOT, those who consume RPCs.
+   These are people who are oblivious to the runtime changes. They only care about the meta-protocol, not the protocol
+   itself.
+
+- `Runtime Dev`: All of those who rely on the runtime. A parachain team that is using a pallet. A DApp that is using a
+   pallet. These are people who care about the protocol (WASM), not the meta-protocol (client).
+
+- `Node Operator`: Those who don't write any code and only run code.
+
+- `Runtime User`: Anyone using the runtime. This can be a token holder or a dev writing a front end for a chain.
+
+## Tips
+
+The PRDoc schema is defined in each repo and usually is quite restrictive.
+You cannot simply add a new property to a `PRDoc` file unless the Schema allows it.

prdoc/.template.prdoc

+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: ...
+
+doc:
+  - audience: Node Dev
+    description: |
+      ...
+
+crates: [ ]

prdoc/pr_1226.prdoc

 title: Removed deprecated `Balances::transfer` and `Balances::set_balance_deprecated` functions.
 
 doc:
-  - audience: Builder
-    description: The Balances pallet's dispatchables `set_balance_deprecated` and `transfer` were deprecated in [paritytech/substrate#12951](https://github.com/paritytech/substrate/pull/12951) and have now been removed.
-    notes: 
-      - Use `set_balance_deprecated` instead `force_set_balance` and `transfer_allow_death` instead of `transfer`.
-
-migrations:
-  db: []
+  - audience: Runtime User
+    description: |
+      The Balances pallet's dispatchables `set_balance_deprecated` and `transfer` were deprecated in [paritytech/substrate#12951](https://github.com/paritytech/substrate/pull/12951) and have now been removed.
 
-  runtime: []
+      notes:
+      - Use `set_balance_deprecated` instead `force_set_balance` and `transfer_allow_death` instead of `transfer`.
 
 crates:
   - name: pallet-balances
-
-host_functions: []

prdoc/pr_1234.prdoc

@@ -4,17 +4,10 @@
 title: Introduce XcmFeesToAccount fee manager
 
 doc:
-  - audience: Builder
+  - audience: Runtime User
     description: |
       Now all XCM sending, unless done by the system for the system, will be charged delivery fees.
       All runtimes are now configured to send these delivery fees to a treasury account.
       The fee formula is `delivery_fee_factor * (base_fee + encoded_msg_len * per_byte_fee)`.
 
-migrations:
-  db: []
-
-  runtime: []
-
-crates: []
-
-host_functions: []
+crates: [ ]

prdoc/pr_1255.prdoc

@@ -4,19 +4,18 @@
 title: Fix for Reward Deficit in the pool
 
 doc:
-  - audience: Core Dev
-    description: Instead of fragile calculation of current balance by looking at free balance - ED, Nomination Pool now freezes ED in the pool reward account to restrict an account from going below minimum balance. This also has a nice side effect that if ED changes, we know how much is the imbalance in ED frozen in the pool and the current required ED. A pool operator can diligently top up the pool with the deficit in ED or vice versa, withdraw the excess they transferred to the pool.
-    notes:
+  - audience: Runtime Dev
+    description: |
+      Instead of fragile calculation of current balance by looking at free balance - ED, Nomination Pool now freezes ED in the pool reward account to restrict an account from going below minimum balance. This also has a nice side effect that if ED changes, we know how much is the imbalance in ED frozen in the pool and the current required ED. A pool operator can diligently top up the pool with the deficit in ED or vice versa, withdraw the excess they transferred to the pool.
+
+      notes:
       - Introduces new call `adjust_pool_deposit` that allows to top up the deficit or withdraw the excess deposit for the pool.
       - Switch to using Fungible trait from Currency trait.
 
 migrations:
-  db: []
-
   runtime:
-    - { pallet: "pallet-nomination-pools", description: "One time migration of freezing ED from each of the existing pools."}
+    - reference: pallet-nomination-pools
+      description: One time migration of freezing ED from each of the existing pools.
 
 crates:
   - name: pallet-nomination-pools
-
-host_functions: []
\ No newline at end of file

prdoc/pr_1408_prodc-introduction.prdoc

@@ -2,18 +2,11 @@
 title: PRdoc check
 
 doc:
-  - audience: Core Dev
+  - audience: Node Dev
     description: |
       This PRdoc is an **example**.
 
       This PR brings support and automated checks for documentation in the form of a
       [`prdoc`](https://github.com/paritytech/prdoc/) file.
 
-migrations:
-  db: []
-
-  runtime: []
-
-crates: []
-
-host_functions: []
+crates: [ ]

prdoc/pr_1818.prdoc

 title: FRAME pallets warning for unchecked weight witness
 
 doc:
-  - audience: Core Dev
+  - audience: Runtime Dev
     description: |
       FRAME pallets now emit a warning when a call uses a function argument that starts with an underscore in its weight declaration.
 
-migrations:
-  db: [ ]
-  runtime: [ ]
-
-host_functions: []
-
 crates:
-- name: "frame-support-procedural"
-  semver: minor
+  - name: frame-support-procedural

prdoc/pr_1873.prdoc

 title: Message Queue use proper overweight limit
 
 doc:
-  - audience: Core Dev
+  - audience: Node Dev
     description: |
       Changed the overweight cutoff limit from the full `Config::ServiceWeight` to a lower value that is calculated based on the weight of the functions being called.
 
-migrations:
-  db: []
-
-  runtime: []
-
-crates: ["pallet-message-queue", patch]
-
-host_functions: []
+crates:
+  - name: pallet-message-queue

prdoc/pr_1913.prdoc

@@ -7,13 +7,6 @@ doc:
       If experiencing stability issues caused by BEEFY, it can be disabled using `--no-beefy` flag.
       BEEFY doesn't (yet) support warp sync. So, attempting to Warp sync as a validator will throw an error.
 
-migrations:
-  db: []
-
-  runtime: []
-
 crates:
   - name: polkadot-cli
   - name: polkadot-service
-
-host_functions: []

prdoc/pr_1921.prdoc

 title: Fix para-scheduler migration
 
 doc:
-  - audience: Core Dev
+  - audience: Runtime Dev
     description: |
       Changing the `MigrateToV1` migration in the `ParachainScheduler` pallet to be truly idempotent. It is achieved by wrapping it in a `VersionedMigration`.
 
 migrations:
-  db: []
-
   runtime:
-    - pallet: "ParachainScheduler"
+    - reference: ParachainScheduler
       description: Non-critical fixup for `MigrateToV1`.
 
 crates:
-  - name: "polkadot-runtime-parachains"
-    semver: patch
-
-host_functions: []
+  - name: polkadot-runtime-parachains

prdoc/pr_1946_prdoc_new_schema.prdoc

+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: New PRDoc Schema
+
+doc:
+  - audience: Node Dev
+    description:  &desc |
+      The new version of prdoc and the new schema is activated in this PR.
+
+  - audience: Runtime Dev
+    description: *desc
+
+crates: []

prdoc/schema_user.json

+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "$id": "https://raw.githubusercontent.com/paritytech/prdoc/master/prdoc_schema_user.json",
+    "version": {
+      "major": 1,
+      "minor": 0,
+      "patch": 0,
+      "timestamp": 20230817152351
+    },
+    "title": "Polkadot SDK PRDoc Schema",
+    "description": "JSON Schema definition for the Polkadot SDK PR documentation",
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "title": {
+        "title": "Title of the change",
+        "type": "string",
+        "description": "Title for the PR. This is what will show up in the release notes.\nif needed, you may provide a different title override for each audience in the `doc` property."
+      },
+
+      "doc": {
+        "type": "array",
+        "title": "Documentation adapted to the audience(s)",
+        "description": "Description of the PR. Provide a description for each relevant audience.\nSee the `audience` property for more documentation about audiences",
+        "items": {
+          "$ref": "#/$defs/doc"
+        },
+        "minItems": 1
+      },
+
+      "crates": {
+        "title": "Crates",
+        "description": "You have the option to provide a hint about the crates that have noticeable changes.\n This is used during the crate publishing to crates.io and to help users understand the impact of the changes introduced in your PR.",
+        "type": "array",
+        "items": {
+          "$ref": "#/$defs/crate"
+        }
+      },
+
+      "migrations": {
+        "title": "Migrations (DB & Runtime)",
+        "description": "It is important for users to be aware of migrations.\nMake sure to mention any migrations in the appropriate sub-properties:\n- db\n- runtime",
+        "type": "object",
+        "properties": {
+          "db": {
+            "type": "array",
+            "nullable": false,
+            "title": "Database Migration",
+            "description": "List of the Database Migrations or empty array: []",
+            "items": {
+              "$ref": "#/$defs/migration_db"
+            },
+            "minItems": 0,
+            "required": [
+              "name",
+              "description"
+            ]
+          },
+          "runtime": {
+            "type": "array",
+            "title": "Runtime Migration",
+            "nullable": false,
+            "description": "List of the Runtime Migrations or empty array: []",
+            "minItems": 0,
+            "items": {
+              "$ref": "#/$defs/migration_runtime"
+            },
+            "required": [
+              "db",
+              "runtime"
+            ]
+          }
+        }
+      },
+      "host_functions": {
+        "title": "Host Functions",
+        "description": "List of the host functions involved in this PR.",
+        "type": "array",
+        "items": {
+          "$ref": "#/$defs/host_function"
+        }
+      }
+    },
+    "required": [
+      "title",
+      "doc",
+      "crates"
+    ],
+    "$defs": {
+      "audience": {
+        "description": "You may pick one or more audiences and address those users with appropriate documentation, information and warning related to the PR.",
+        "oneOf": [
+          {"const": "Node Dev",
+          "title": "Node Dev",
+          "description": "Those who build around the client side code. Alternative client builders, SMOLDOT, those who consume RPCs. These are people who are oblivious to the runtime changes. They only care about the meta-protocol, not the protocol itself."},
+
+          {"const": "Runtime Dev",
+          "title": "Runtime Dev",
+          "description": "All of those who rely on the runtime. A parachain team that is using a pallet. A DApp that is using a pallet. These are people who care about the protocol (WASM), not the meta-protocol (client)."},
+
+          {"const": "Node Operator",
+          "title": "Node Operator",
+          "description": "Those who don't write any code and only run code."},
+
+          {"const": "Runtime User",
+          "title": "Runtime User",
+          "description": "Anyone using the runtime. This can be a token holder or a dev writing a front end for a chain."}
+        ]
+      },
+      "crate": {
+        "type": "object",
+        "description": "You have the option here to provide a hint about a crate that has changed to help with the publishing of crates.",
+        "additionalProperties": false,
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "note": {
+            "type": "string"
+          }
+        }
+      },
+      "migration_db": {
+        "type": "object",
+        "description": "This property allows the documentation of database migrations.",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "name",
+          "description"
+        ]
+      },
+      "migration_runtime": {
+        "type": "object",
+        "description": "This property allows the documentation of runtime migrations.",
+        "properties": {
+          "reference": {
+            "title": "Migration reference",
+            "description": "Reference to the runtime migration",
+            "type": "string"
+          },
+          "description": {
+            "type": "string"
+          }
+        },
+        "additionalProperties": false,
+        "required": [
+          "description"
+        ]
+      },
+      "doc": {
+        "type": "object",
+        "description": "You have the the option to provide different description of your PR for different audiences.",
+        "additionalProperties": false,
+        "properties": {
+          "audience": {
+            "description": "The selected audience",
+            "$ref": "#/$defs/audience"
+          },
+          "title": {
+            "type": "string",
+            "title": "Title for the audience",
+            "description": "Optional title override for the PR and for the current audience"
+          },
+          "description": {
+            "title": "Description for the audience",
+            "description": "Description of the change",
+            "type": "string"
+          }
+        }
+      },
+      "array_of_strings": {
+        "description": "An array of strings that can be empty",
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
+      "host_function": {
+        "type": "object",
+        "additionalProperties": false,
+        "title": "Host Functions",
+        "description": "List of host functions and their descriptions",
+        "properties": {
+          "name": {
+            "title": "Host function name",
+            "description": "Name or identifier to find the host function in the codebase",
+            "type": "string"
+          },
+          "description": {
+            "title": "Host function description",
+            "description": "Short description of the host function",
+            "type": "string"
+          },
+          "notes": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "name",
+          "description"
+        ]
+      }
+    }
+  }