Github workflow to automate release draft creation (#3978)

This PR introduces the github flow which will create a release draft
automatically when the rc tag is pushed. The flow contains the following
steps:

- Gets the info about rust version used to build the node
- Builds the runtimes using `srtool`
- Extracts the info about each runtime 
- Aggregates the changelog from the prdocs
- Creates the release draft containing all the info related to the
release (changelog, runtimes, rust versions)
- Attaches the runtimes to the draft
- Posts the message to the RelEng internal channel to inform that the
build is done.

Related to the #3295

---------

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

.github/scripts/common/lib.sh

@@ -369,7 +369,7 @@ function relative_parent() {
 # used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as:
 # podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan
 function find_runtimes() {
-    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs'))
+    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches '[frame_support::runtime]!' -- '*lib.rs'))
     re=".*-runtime$"
     JSON=$(jq --null-input '{ "include": [] }')
 
@@ -434,3 +434,13 @@ check_release_id() {
   fi
 
 }
+
+# Get latest release tag
+#
+# input: none
+# output: latest_release_tag
+get_latest_release_tag() {
+    TOKEN="Authorization: Bearer $GITHUB_TOKEN"
+    latest_release_tag=$(curl -s -H "$TOKEN" $api_base/paritytech/polkadot-sdk/releases/latest | jq -r '.tag_name')
+    printf $latest_release_tag
+}

.github/workflows/release-30_publish_release_draft.yml

+name: Release - Publish draft
+
+on:
+  push:
+    tags:
+      # Catches v1.2.3 and v1.2.3-rc1
+      - v[0-9]+.[0-9]+.[0-9]+*
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Current release/rc version
+
+jobs:
+  get-rust-versions:
+    runs-on: ubuntu-latest
+    outputs:
+      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
+    steps:
+      - id: get-rust-versions
+        run: |
+          RUST_STABLE_VERSION=$(curl -sS https://raw.githubusercontent.com/paritytech/scripts/master/dockerfiles/ci-unified/Dockerfile | grep -oP 'ARG RUST_STABLE_VERSION=\K[^ ]+')
+          echo "stable=$RUST_STABLE_VERSION" >> $GITHUB_OUTPUT
+
+  build-runtimes:
+    uses: "./.github/workflows/srtool.yml"
+    with:
+       excluded_runtimes: "substrate-test bp cumulus-test kitchensink minimal-template parachain-template penpal polkadot-test seedling shell frame-try sp solochain-template"
+
+  publish-release-draft:
+    runs-on: ubuntu-latest
+    needs: [get-rust-versions, build-runtimes]
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Prepare tooling
+        run: |
+          URL=https://github.com/chevdor/tera-cli/releases/download/v0.2.4/tera-cli_linux_amd64.deb
+          wget $URL -O tera.deb
+          sudo dpkg -i tera.deb
+          tera --version
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Prepare draft
+        id: draft
+        env:
+          RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ASSET_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/asset-hub-rococo-runtime/asset-hub-rococo-srtool-digest.json
+          ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json
+          BRIDGE_HUB_ROCOCO_DIGEST: ${{ github.workspace}}/bridge-hub-rococo-runtime/bridge-hub-rococo-srtool-digest.json
+          BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json
+          COLLECTIVES_WESTEND_DIGEST: ${{ github.workspace}}/collectives-westend-runtime/collectives-westend-srtool-digest.json
+          CONTRACTS_ROCOCO_DIGEST: ${{ github.workspace}}/contracts-rococo-runtime/contracts-rococo-srtool-digest.json
+          CORETIME_ROCOCO_DIGEST: ${{ github.workspace}}/coretime-rococo-runtime/coretime-rococo-srtool-digest.json
+          CORETIME_WESTEND_DIGEST: ${{ github.workspace}}/coretime-westend-runtime/coretime-westend-srtool-digest.json
+          GLUTTON_WESTEND_DIGEST: ${{ github.workspace}}/glutton-westend-runtime/glutton-westend-srtool-digest.json
+          PEOPLE_ROCOCO_DIGEST: ${{ github.workspace}}/people-rococo-runtime/people-rococo-srtool-digest.json
+          PEOPLE_WESTEND_DIGEST: ${{ github.workspace}}/people-westend-runtime/people-westend-srtool-digest.json
+          ROCOCO_DIGEST: ${{ github.workspace}}/rococo-runtime/rococo-srtool-digest.json
+          WESTEND_DIGEST: ${{ github.workspace}}/westend-runtime/westend-srtool-digest.json
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          export REF1=$(get_latest_release_tag)
+          if [[ -z "${{ inputs.version }}" ]]; then
+            export REF2="${{ github.ref }}"
+          else
+            export REF2="${{ inputs.version }}"
+          fi
+          echo "REL_TAG=$REF2" >> $GITHUB_ENV
+          export VERSION=$(echo "$REF2" | sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+).*$/\1/')
+
+          ./scripts/release/build-changelogs.sh
+
+          echo "Checking the folder state"
+          pwd
+          ls -la scripts/release
+
+      - name: Archive artifact context.json
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        with:
+          name: release-notes-context
+          path: |
+            scripts/release/context.json
+            **/*-srtool-digest.json
+
+      - name: Create draft release
+        id: create-release
+        uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.REL_TAG }}
+          release_name: Polkadot ${{ env.REL_TAG }}
+          body_path: ${{ github.workspace}}/scripts/release/RELEASE_DRAFT.md
+          draft: true
+
+  publish-runtimes:
+    needs: [ build-runtimes, publish-release-draft ]
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.build-runtimes.outputs.published_runtimes) }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Download artifacts
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+
+      - name: Get runtime info
+        env:
+          JSON: release-notes-context/${{ matrix.chain }}-runtime/${{ matrix.chain }}-srtool-digest.json
+        run: |
+          >>$GITHUB_ENV echo ASSET=$(find ${{ matrix.chain }}-runtime -name '*.compact.compressed.wasm')
+          >>$GITHUB_ENV echo SPEC=$(<${JSON} jq -r .runtimes.compact.subwasm.core_version.specVersion)
+
+      - name: Upload compressed ${{ matrix.chain }} v${{ env.SPEC }} wasm
+        if: ${{ matrix.chain  != 'rococo-parachain' }}
+        uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }}
+          asset_path: ${{ env.ASSET }}
+          asset_name: ${{ matrix.chain }}_runtime-v${{ env.SPEC }}.compact.compressed.wasm
+          asset_content_type: application/wasm
+
+  post_to_matrix:
+    runs-on: ubuntu-latest
+    needs: publish-release-draft
+    strategy:
+      matrix:
+        channel:
+          - name: "Team: RelEng Internal"
+            room: '!GvAyzgCDgaVrvibaAF:parity.io'
+
+    steps:
+      - name: Send Matrix message to ${{ matrix.channel.name }}
+        uses: s3krit/matrix-message-action@70ad3fb812ee0e45ff8999d6af11cafad11a6ecf # v0.0.3
+        with:
+          room_id: ${{ matrix.channel.room }}
+          access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }}
+          server: m.parity.io
+          message: |
+            **New version of polkadot tagged**: ${{ github.ref }}<br/>
+            Draft release created: ${{ needs.publish-release-draft.outputs.release_url }}

.github/workflows/srtool.yml

@@ -12,6 +12,13 @@ on:
       - release-v[0-9]+.[0-9]+.[0-9]+*
       - release-cumulus-v[0-9]+*
       - release-polkadot-v[0-9]+*
+  workflow_call:
+    inputs:
+      excluded_runtimes:
+        type: string
+    outputs:
+      published_runtimes:
+        value: ${{ jobs.find-runtimes.outputs.runtime }}
 
   schedule:
     - cron: "00 02 * * 1" # 2AM weekly on monday
@@ -39,7 +46,7 @@ jobs:
 
         - name: Scan runtimes
           env:
-            EXCLUDED_RUNTIMES: "substrate-test"
+            EXCLUDED_RUNTIMES: ${{ inputs.excluded_runtimes }}:"substrate-test"
           run: |
             . ./.github/scripts/common/lib.sh
 
@@ -85,16 +92,6 @@ jobs:
           echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}"
           echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}"
 
-      # it takes a while to build the runtime, so let's save the artifact as soon as we have it
-      - name: Archive Artifacts for ${{ matrix.chain }}
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          name: ${{ matrix.chain }}-runtime
-          path: |
-            ${{ steps.srtool_build.outputs.wasm }}
-            ${{ steps.srtool_build.outputs.wasm_compressed }}
-            ${{ matrix.chain }}-srtool-digest.json
-
       # We now get extra information thanks to subwasm
       - name: Install subwasm
         run: |
@@ -125,7 +122,7 @@ jobs:
             tee ${{ matrix.chain }}-diff.txt
 
       - name: Archive Subwasm results
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         with:
           name: ${{ matrix.chain }}-runtime
           path: |
@@ -133,3 +130,6 @@ jobs:
             ${{ matrix.chain }}-compressed-info.json
             ${{ matrix.chain }}-metadata.json
             ${{ matrix.chain }}-diff.txt
+            ${{ steps.srtool_build.outputs.wasm }}
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+            ${{ matrix.chain }}-srtool-digest.json

prdoc/1.9.0/pr_3513.prdoc

@@ -5,7 +5,13 @@ title: Fix call enum's metadata regression
 
 doc:
   - audience: Runtime Dev
-  - audience: Runtime User
+    description: |
+      This PR fixes an issue where in the metadata of all FRAME-based chains, the documentation for
+      all extrinsic/call/transactions has been replaced by a single line saying "see Pallet::name".
+
+      Many wallets display the metadata of transactions to the user before signing, and this bug
+      might have affected this process.
+  - audience: [Runtime User, Runtime Dev]
     description: |
       This PR fixes an issue where in the metadata of all FRAME-based chains, the documentation for
       all extrinsic/call/transactions has been replaced by a single line saying "see Pallet::name".

scripts/release/build-changelogs.sh

@@ -2,11 +2,16 @@
 
 export PRODUCT=polkadot
 export VERSION=${VERSION:-1.5.0}
+export ENGINE=${ENGINE:-docker}
+export REF1=${REF1:-'HEAD'}
+export REF2=${REF2}
+export RUSTC_STABLE=${RUSTC_STABLE:-'1.0'}
+export RUSTC_NIGHTLY=${RUSTC_NIGHTLY:-'1.0'}
 
 PROJECT_ROOT=`git rev-parse --show-toplevel`
 echo $PROJECT_ROOT
 
-TMP=$(mktemp -d)
+TMP=${TMP:-$(mktemp -d)}
 TEMPLATE_AUDIENCE="${PROJECT_ROOT}/scripts/release/templates/audience.md.tera"
 TEMPLATE_CHANGELOG="${PROJECT_ROOT}/scripts/release/templates/changelog.md.tera"
 
@@ -21,7 +26,7 @@ OUTPUT="${TMP}/changelogs/$PRODUCT/$VERSION"
 echo -e "OUTPUT: \t\t$OUTPUT"
 mkdir -p $OUTPUT
 
-prdoc load -d "$PROJECT_ROOT/prdoc/$VERSION" --json > $DATA_JSON
+$ENGINE run --rm -v ${PROJECT_ROOT}:/repo paritytech/prdoc load -d "prdoc/$VERSION" --json > $DATA_JSON
 # ls -al $DATA_JSON
 
 cat $DATA_JSON | jq ' { "prdoc" : .}' > $CONTEXT_JSON
@@ -48,8 +53,58 @@ for audience in "${audiences[@]}"; do
     echo "Processing audience: $audience ($audience_id)"
     export TARGET_AUDIENCE=$audience
     tera -t "${TEMPLATE_AUDIENCE}" --env --env-key env "${CONTEXT_JSON}" > "$OUTPUT/relnote_${audience_id}.md"
-    cat "$OUTPUT/relnote_${audience_id}.md" >> "$OUTPUT/relnote_combined.md"
+    cat "$OUTPUT/relnote_${audience_id}.md" >> "$PROJECT_ROOT/scripts/release/templates/changelog.md"
 done
 
 # Show the files
 tree -s -h -c $OUTPUT/
+
+ASSET_HUB_ROCOCO_DIGEST=${ASSET_HUB_ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/asset-hub-rococo-srtool-digest.json"}
+ASSET_HUB_WESTEND_DIGEST=${ASSET_HUB_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/asset-hub-westend-srtool-digest.json"}
+BRIDGE_HUB_ROCOCO_DIGEST=${BRIDGE_HUB_ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/bridge-hub-rococo-srtool-digest.json"}
+BRIDGE_HUB_WESTEND_DIGEST=${BRIDGE_HUB_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/bridge-hub-westend-srtool-digest.json"}
+COLLECTIVES_WESTEND_DIGEST=${COLLECTIVES_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/collectives-westend-srtool-digest.json"}
+CONTRACTS_ROCOCO_DIGEST=${CONTRACTS_ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/contracts-rococo-srtool-digest.json"}
+CORETIME_ROCOCO_DIGEST=${CORETIME_ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/coretime-rococo-srtool-digest.json"}
+CORETIME_WESTEND_DIGEST=${CORETIME_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/coretime-westend-srtool-digest.json"}
+GLUTTON_WESTEND_DIGEST=${GLUTTON_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/glutton-westend-srtool-digest.json"}
+PEOPLE_ROCOCO_DIGEST=${PEOPLE_ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/people-rococo-srtool-digest.json"}
+PEOPLE_WESTEND_DIGEST=${PEOPLE_WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/people-westend-srtool-digest.json"}
+ROCOCO_DIGEST=${ROCOCO_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/rococo-srtool-digest.json"}
+WESTEND_DIGEST=${WESTEND_DIGEST:-"$PROJECT_ROOT/scripts/release/digests/westend-srtool-digest.json"}
+
+jq \
+      --slurpfile srtool_asset_hub_rococo $ASSET_HUB_ROCOCO_DIGEST \
+      --slurpfile srtool_asset_hub_westend $ASSET_HUB_WESTEND_DIGEST \
+      --slurpfile srtool_bridge_hub_rococo $BRIDGE_HUB_ROCOCO_DIGEST \
+      --slurpfile srtool_bridge_hub_westend $BRIDGE_HUB_WESTEND_DIGEST \
+      --slurpfile srtool_collectives_westend $COLLECTIVES_WESTEND_DIGEST \
+      --slurpfile srtool_contracts_rococo $CONTRACTS_ROCOCO_DIGEST \
+      --slurpfile srtool_coretime_rococo $CORETIME_ROCOCO_DIGEST\
+      --slurpfile srtool_coretime_westend $CORETIME_WESTEND_DIGEST \
+      --slurpfile srtool_glutton_westend $GLUTTON_WESTEND_DIGEST \
+      --slurpfile srtool_people_rococ $PEOPLE_ROCOCO_DIGEST \
+      --slurpfile srtool_people_westend $PEOPLE_WESTEND_DIGEST \
+      --slurpfile srtool_rococo $ROCOCO_DIGEST \
+      --slurpfile srtool_westend $WESTEND_DIGEST \
+      -n '{
+          srtool: [
+            { order: 10, name: "Westend", data: $srtool_westend[0] },
+            { order: 11, name: "Westend AssetHub", data: $srtool_asset_hub_westend[0] },
+            { order: 12, name: "Westend BridgeHub", data: $srtool_bridge_hub_westend[0] },
+            { order: 13, name: "Westend Collectives", data: $srtool_collectives_westend[0] },
+            { order: 14, name: "Westend Coretime", data: $srtool_coretime_westend[0] },
+            { order: 15, name: "Westend Glutton", data: $srtool_glutton_westend[0] },
+            { order: 16, name: "Westend People", data: $srtool_people_westend[0] },
+            { order: 17, name: "Rococo", data: $srtool_rococo[0] },
+            { order: 18, name: "Rococo AssetHub", data: $srtool_asset_hub_rococo[0] },
+            { order: 19, name: "Rococo BridgeHub", data: $srtool_bridge_hub_rococo[0] },
+            { order: 20, name: "Rococo Contracts", data: $srtool_contracts_rococo[0] },
+            { order: 21, name: "Rococo Coretime", data: $srtool_coretime_rococo[0] },
+            { order: 22, name: "Rococo People", data: $srtool_people_rococ[0] }
+      ] }' > "$PROJECT_ROOT/scripts/release/context.json"
+
+RELEASE_DIR="$PROJECT_ROOT/scripts/release/"
+pushd $RELEASE_DIR >/dev/null
+tera --env --env-key env --include-path templates --template templates/template.md.tera context.json > RELEASE_DRAFT.md
+popd >/dev/null

scripts/release/digests/.gitignore

+*.json

scripts/release/templates/changes.md.tera

+{# This include generates the section showing the changes #}
+## Changelog
+
+{% include "changelog.md" -%}

scripts/release/templates/compiler.md.tera

+## Rust compiler versions
+
+This release was built and tested against the following versions of `rustc`.
+Other versions may work.
+
+- Rust Stable: `{{ env.RUSTC_STABLE }}`

scripts/release/templates/runtime.md.tera

+{# This macro shows one runtime #}
+{%- macro runtime(runtime) -%}
+
+### {{ runtime.name | title }}
+
+{%- if runtime.data.runtimes.compressed.subwasm.compression.compressed %}
+{%- set compressed = "Yes" %}
+{%- else %}
+{%- set compressed = "No" %}
+{%- endif %}
+
+{%- set comp_ratio = 100 - (runtime.data.runtimes.compressed.subwasm.compression.size_compressed /
+runtime.data.runtimes.compressed.subwasm.compression.size_decompressed *100) %}
+
+```
+🏋️ Runtime Size:          {{ runtime.data.runtimes.compressed.subwasm.size | filesizeformat }} ({{
+runtime.data.runtimes.compressed.subwasm.size }} bytes)
+🔥 Core Version:          {{ runtime.data.runtimes.compressed.subwasm.core_version.specName }}-{{runtime.data.runtimes.compressed.subwasm.core_version.specVersion }} ({{runtime.data.runtimes.compressed.subwasm.core_version.implName }}-{{runtime.data.runtimes.compressed.subwasm.core_version.implVersion }}.tx{{runtime.data.runtimes.compressed.subwasm.core_version.transactionVersion }}.au{{runtime.data.runtimes.compressed.subwasm.core_version.authoringVersion }})
+🗜 Compressed:            {{ compressed }}: {{ comp_ratio | round(method="ceil", precision=2) }}%
+🎁 Metadata version:      V{{ runtime.data.runtimes.compressed.subwasm.metadata_version }}
+🗳️ system.setCode hash:   {{ runtime.data.runtimes.compressed.subwasm.proposal_hash }}
+🗳️ authorizeUpgrade hash: {{ runtime.data.runtimes.compressed.subwasm.parachain_authorize_upgrade_hash }}
+🗳️ Blake2-256 hash:       {{ runtime.data.runtimes.compressed.subwasm.blake2_256 }}
+📦 IPFS:                  {{ runtime.data.runtimes.compressed.subwasm.ipfs_hash }}
+```
+{%- endmacro runtime %}

scripts/release/templates/runtimes.md.tera

+{# This include shows the list and details of the runtimes #}
+{%- import "runtime.md.tera" as m_r -%}
+
+{# ---  #}
+
+## Runtimes
+
+{% set rtm = srtool[0] -%}
+
+The information about the runtimes included in this release can be found below.
+The runtimes have been built using [{{ rtm.data.gen }}](https://github.com/paritytech/srtool) and `{{ rtm.data.rustc }}`.
+
+{%- for runtime in srtool | sort(attribute="name") %}
+{%- set HIDE_VAR = "HIDE_SRTOOL_" ~ runtime.name | upper %}
+{%- if not env is containing(HIDE_VAR) %}
+
+{{ m_r::runtime(runtime=runtime) }}
+{%- endif %}
+{%- endfor %}

scripts/release/templates/template.md.tera

+{# This is the entry point of the template -#}
+
+This release contains the changes from `{{ env.REF1 | replace(from="refs/tags/", to="") }}` to `{{ env.REF2 | replace(from="refs/tags/", to="") }}`.
+
+{% include "changes.md.tera" -%}
+
+{% include "compiler.md.tera" -%}
+
+{% include "runtimes.md.tera" -%}