[CI] Publish draft release redux (#1493)

* Add publish-draft-release Github action
* Remove publish_draft_release.sh gitlab job

polkadot/.github/workflows/publish-draft-release.yml

+name: Publish draft release
+
+on:
+  push:
+    tags:
+      - v**.**.**
+
+jobs:
+  get-rust-versions:
+    runs-on: ubuntu-latest
+    container:
+      image: paritytech/ci-linux:production
+    outputs:
+      rustc-stable: ${{ steps.get-rust-versions.outputs.stable }}
+      rustc-nightly: ${{ steps.get-rust-versions.outputs.nightly }}
+    steps:
+      - id: get-rust-versions
+        run: |
+          echo "::set-output name=stable::$(rustc +stable --version)"
+          echo "::set-output name=nightly::$(rustc +nightly --version)"
+
+  publish-draft-release:
+    runs-on: ubuntu-latest
+    needs: get-rust-versions
+    outputs:
+      release_url: ${{ steps.create-release.outputs.html_url }}
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        path: polkadot
+    - name: Set up Ruby 2.7
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.7
+    - name: Generate release text
+      env:
+        RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
+        RUSTC_NIGHTLY: ${{ needs.get-rust-versions.outputs.rustc-nightly }}
+      run: |
+        gem install changelogerator git toml
+        ruby $GITHUB_WORKSPACE/polkadot/scripts/github/generate_release_text.rb > release_text.md
+    - uses: actions/upload-artifact@v2
+      with:
+        name: release_text
+        path: release_text.md
+    - name: Create draft release
+      id: create-release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ github.ref }}
+        release_name: Polkadot ${{ github.ref }}
+        body_path: ./release_text.md
+        draft: true
+
+  post_to_matrix:
+    runs-on: ubuntu-latest
+    needs: publish-draft-release
+    steps:
+      - name: Internal polkadot channel
+        uses: s3krit/matrix-message-action@v0.0.2
+        with:
+          room_id: ${{ secrets.INTERNAL_POLKADOT_MATRIX_ROOM_ID }}
+          access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}
+          message: "**New version of polkadot tagged**: ${{ github.ref }}<br/>Gav: Draft release created: ${{ needs.publish-draft-release.outputs.release_url }}"
+          server: "matrix.parity.io"

polkadot/.gitlab-ci.yml

@@ -94,17 +94,6 @@ check-line-width:
   interruptible:                   true
   allow_failure:                   true
 
-publish-draft-release:
-  stage:                           test
-  only:
-    - tags
-    - /^v[0-9]+\.[0-9]+\.[0-9]+.*$/        # i.e. v1.0.1, v2.1.0rc1
-  script:
-    - apt-get -y update; apt-get -y install jq
-    - ./scripts/gitlab/publish_draft_release.sh
-  interruptible:                   true
-  allow_failure:                   true
-
 test-deterministic-wasm:
   stage:                           test
   <<:                              *docker-env

polkadot/scripts/github/generate_release_text.rb

+# frozen_string_literal: true
+
+require 'changelogerator'
+require 'git'
+require 'erb'
+require 'toml'
+
+version = ENV['GITHUB_REF']
+token = ENV['GITHUB_TOKEN']
+
+polkadot_path = ENV['GITHUB_WORKSPACE'] + '/polkadot/'
+pg = Git.open(polkadot_path)
+
+# Generate an ERB renderer based on the template .erb file
+renderer = ERB.new(
+  File.read(ENV['GITHUB_WORKSPACE'] + '/polkadot/scripts/github/polkadot_release.erb'),
+  trim_mode: '<>'
+)
+
+# get last polkadot version. Use handy Gem::Version for sorting by version
+last_version = pg
+              .tags
+              .map(&:name)
+              .grep(/^v\d+\.\d+\.\d+.*$/)
+              .sort_by { |v| Gem::Version.new(v.slice(1...)) }[-2]
+
+polkadot_cl = Changelog.new(
+  'paritytech/polkadot', version, last_version, token: token
+)
+
+# Get prev and cur substrate SHAs - parse the old and current Cargo.lock for
+# polkadot and extract the sha that way.
+prev_cargo = TOML::Parser.new(pg.show("#{last_version}:Cargo.lock")).parsed
+current_cargo = TOML::Parser.new(pg.show("#{version}:Cargo.lock")).parsed
+
+substrate_prev_sha = prev_cargo['package']
+                    .find { |p| p['name'] == 'sc-cli' }['source']
+                    .split('#').last
+
+substrate_cur_sha = current_cargo['package']
+                    .find { |p| p['name'] == 'sc-cli' }['source']
+                    .split('#').last
+
+substrate_cl = Changelog.new(
+  'paritytech/substrate', substrate_prev_sha, substrate_cur_sha,
+  token: token,
+  prefix: true
+)
+
+all_changes = polkadot_cl.changes + substrate_cl.changes
+
+# Set all the variables needed for a release
+
+misc_changes = Changelog.changes_with_label(all_changes, 'B1-releasenotes')
+client_changes = Changelog.changes_with_label(all_changes, 'B5-clientnoteworthy')
+runtime_changes = Changelog.changes_with_label(all_changes, 'B7-runtimenoteworthy')
+
+release_priority = Changelog.highest_priority_for_changes(all_changes)
+
+# Pulled from the previous Github step
+rustc_stable = ENV['RUSTC_STABLE']
+rustc_nightly = ENV['RUSTC_NIGHTLY']
+
+polkadot_runtime = File.open(polkadot_path + '/runtime/polkadot/src/lib.rs') do |f|
+  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
+end
+kusama_runtime = File.open(polkadot_path + '/runtime/kusama/src/lib.rs') do |f|
+  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
+end
+westend_runtime = File.open(polkadot_path + '/runtime/westend/src/lib.rs') do |f|
+  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
+end
+
+puts renderer.result

polkadot/scripts/github/polkadot_release.erb

+<%= print release_priority[:text] %> <%= puts " due to changes: *#{Changelog.changes_with_label(all_changes, release_priority[:label]).map(&:pretty_title).join(", ")}*" if release_priority[:priority] > 1 %>
+
+Native runtimes:
+
+- Polkadot: **<%= polkadot_runtime %>**
+- Kusama: **<%= kusama_runtime %>**
+- Westend: **<%= westend_runtime %>**
+
+This release was tested against the following versions of `rustc`. Other versions may work.
+
+- <%= rustc_stable %>
+- <%= rustc_nightly %>
+
+<% unless misc_changes.empty? %>
+## Changes
+
+<% misc_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>
+
+<% unless client_changes.empty? %>
+## Client
+
+<% client_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>
+
+<% unless runtime_changes.empty? %>
+## Runtime
+
+<% runtime_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>

polkadot/scripts/gitlab/publish_draft_release.sh

-#!/usr/bin/env bash
-
-# shellcheck source=lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
-
-# Set initial variables
-substrate_repo="https://github.com/paritytech/substrate"
-substrate_dir='./substrate'
-
-# Cloning repos to ensure freshness
-echo "[+] Cloning substrate to generate list of changes"
-git clone $substrate_repo $substrate_dir
-echo "[+] Finished cloning substrate into $substrate_dir"
-
-version="$CI_COMMIT_TAG"
-last_version=$(git tag -l | sort -V | grep -B 1 -x "$CI_COMMIT_TAG" | head -n 1)
-echo "[+] Version: $version; Previous version: $last_version"
-
-# Check that a signed tag exists on github for this version
-echo '[+] Checking tag has been signed'
-check_tag "paritytech/polkadot" "$version"
-case $? in
-  0) echo '[+] Tag found and has been signed'
-    ;;
-  1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
-    ;;
-  2) echo '[!] Tag not found. Aborting release.'; exit 1
-esac
-
-# Pull rustc version used by rust-builder for stable and nightly
-stable_rustc="$(rustc +stable --version)"
-nightly_rustc="$(rustc +nightly --version)"
-
-# Start with referencing current native runtime
-# and find any referenced PRs since last release
-# Note: Drop any changes that begin with '[contracts]' or 'contracts:'
-polkadot_spec=$(grep spec_version runtime/polkadot/src/lib.rs | tail -n 1 | grep -Eo '[0-9]+')
-echo "[+] Polkadot spec version: $polkadot_spec"
-kusama_spec=$(grep spec_version runtime/kusama/src/lib.rs | tail -n 1 | grep -Eo '[0-9]+')
-echo "[+] Kusama spec version: $kusama_spec"
-westend_spec=$(grep spec_version runtime/westend/src/lib.rs | tail -n 1 | grep -Eo '[0-9]+')
-echo "[+] Westend spec version: $westend_spec"
-release_text="Native runtimes:
-- Polkadot: **$polkadot_spec**
-- Kusama: **$kusama_spec**
-- Westend: **$westend_spec**
-
-This release was built with the following versions of \`rustc\`. Other versions may work.
-- $stable_rustc
-- $nightly_rustc
-"
-
-declare -a misc_changes
-declare -a runtime_changes
-declare -a client_changes
-
-# Following variables are for tracking the priority of the release (i.e.,
-# how important it is for the user to upgrade).
-# It's frustrating that we need to make an array of indexes (in this case the
-# labels), but it's necessary to maintain the correct order. Labels and
-# descriptions *must* be kept in lockstep
-
-priority_labels=(
-  'C1-low'
-  'C3-medium'
-  'C7-high'
-  'C9-critical'
-)
-
-declare -A priority_descriptions=(
-['C1-low']="Upgrade priority: **Low** (upgrade at your convenience)"
-['C3-medium']="Upgrade priority: **Medium** (timely upgrade recommended)"
-['C7-high']="Upgrade priority:❗ **HIGH** ❗ Please upgrade your node as soon as possible"
-['C9-critical']="Upgrade priority: ❗❗ **URGENT** ❗❗ PLEASE UPGRADE IMMEDIATELY"
-)
-
-# We don't actually take any action on C1-low, so we can start at medium
-# But set C1-low as the default
-max_label=1
-priority="${priority_descriptions['C1-low']}"
-declare -a priority_changes
-
-# Iterate through every PR
-while IFS= read -r line; do
-  pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
-
-  # Release priority check:
-  # For each PR, we look for every label equal to or higher than the current highest
-  # I.e., if there has already been a PR marked as 'medium', we only need
-  # to look for priorities medium or above. If we find one, we set the
-  # priority to that level.
-  for ((index=max_label; index<${#priority_labels[@]}; index++)) ; do
-    cur_label="${priority_labels[$index]}"
-    echo "[+] Checking #$pr_id for presence of $cur_label label"
-    if has_label 'paritytech/polkadot' "$pr_id" "$cur_label" ; then
-      echo "[+] #$pr_id has label $cur_label. Setting max."
-      prev_label="$max_label"
-      max_label="$index"
-      priority="${priority_descriptions[$cur_label]}"
-
-      # If it's not an increase in priority, we just append the PR to the list
-      if [ "$prev_label" == "$max_label" ]; then
-        priority_changes+=("${line/\* /}")
-      fi
-      # If the priority has increased, we override previous changes with new changes
-      if [ "$prev_label" != "$max_label" ]; then
-        priority_changes=("${line/\* /}")
-      fi
-
-      # Append priority to change
-      # Skip first 3 chars
-      note=${cur_label:3}
-      # And capitalise
-      line=" \`${note^}\` $line"
-    fi
-  done
-
-  # If the PR is labelled silent, we can do an early continue to save a little work
-  if has_label 'paritytech/polkadot' "$pr_id" 'B0-silent'; then
-    continue
-  fi
-
-  # If the PR has a runtimenoteworthy label, add to the runtime_changes section
-  if has_label 'paritytech/polkadot' "$pr_id" 'B2-runtimenoteworthy'; then
-    runtime_changes+=("$line")
-  fi
-  # If the PR has a releasenotes label, add to the release section
-  if has_label 'paritytech/polkadot' "$pr_id" 'B1-releasenotes'; then
-    misc_changes+=("$line")
-  fi
-done <<< "$(sanitised_git_logs "$last_version" "$version" | \
-  sed '/^\[contracts\].*/d' | \
-  sed '/^contracts:.*/d' )"
-
-# Get substrate changes between last polkadot version and current
-# By grepping the Cargo.lock for a substrate crate, and grepping out the commit hash
-cur_substrate_commit=$(grep -A 2 'name = "sc-cli"' Cargo.lock | grep -E -o '[a-f0-9]{40}')
-old_substrate_commit=$(git diff "refs/tags/$last_version" Cargo.lock |\
-  grep -A 2 'name = "sc-cli"' | grep -E -o '[a-f0-9]{40}')
-pushd $substrate_dir || exit
-  git checkout master > /dev/null
-  git pull > /dev/null
-  all_substrate_changes="$(sanitised_git_logs "$old_substrate_commit" "$cur_substrate_commit" | sed 's/(#/(paritytech\/substrate#/')"
-
-  echo "[+] Iterating through substrate changes to find labelled PRs"
-  while IFS= read -r line; do
-    pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
-
-    # Basically same check as Polkadot priority
-    # We only need to check for any labels of the current priority or higher
-    for ((index=max_label; index<${#priority_labels[@]}; index++)) ; do
-      cur_label="${priority_labels[$index]}"
-      echo "[+] Checking substrate/#$pr_id for presence of $cur_label label"
-      if has_label 'paritytech/substrate' "$pr_id" "$cur_label" ; then
-        echo "[+] #$pr_id has label $cur_label. Setting max."
-        prev_label="$max_label"
-        max_label="$index"
-        priority="${priority_descriptions[$cur_label]}"
-
-        # If it's not an increase in priority, we just append
-        if [ "$prev_label" == "$max_label" ]; then
-          priority_changes+=("${line/\* /}")
-        fi
-        # If the priority has increased, we override previous changes with new changes
-        if [ "$prev_label" != "$max_label" ]; then
-          priority_changes=("${line/\* /}")
-        fi
-
-        # Append priority to change
-        # Skip first 3 chars
-        note=${cur_label:3}
-        # And capitalise
-        line=" \`${note^}\` $line"
-      fi
-    done
-
-    # Skip if the PR has the silent label - this allows us to skip a few requests
-    if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
-      continue
-    fi
-    if has_label 'paritytech/substrate' "$pr_id" 'B5-clientnoteworthy'; then
-      client_changes+=("$line")
-    fi
-    if has_label 'paritytech/substrate' "$pr_id" 'B7-runtimenoteworthy'; then
-      runtime_changes+=("$line")
-    fi
-  done <<< "$all_substrate_changes"
-popd || exit
-
-
-# Add the priorities to the *start* of the release notes
-# If polkadot and substrate priority = low, no need for list of changes
-if [ "$priority" == "${priority_descriptions['C1-low']}" ]; then
-  release_text="$priority
-
-$release_text"
-else
-  release_text="$priority - due to change(s): *${priority_changes[*]}*
-
-$release_text"
-fi
-
-# Append all notable changes to the release notes
-
-if [ "${#misc_changes[*]}" -gt 0 ] ; then
-  release_text="$release_text
-
-## Changes
-$(printf '* %s\n' "${misc_changes[@]}")"
-fi
-
-if [ "${#client_changes[*]}" -gt 0 ] ; then
-  release_text="$release_text
-
-## Client
-$(printf '* %s\n' "${client_changes[@]}")"
-fi
-
-if [ "${#runtime_changes[*]}" -gt 0 ] ; then
-  release_text="$release_text
-
-## Runtime
-$(printf '* %s\n' "${runtime_changes[@]}")"
-fi
-
-echo "[+] Release text generated: "
-echo "$release_text"
-
-echo "[+] Pushing release to github"
-# Create release on github
-release_name="Polkadot $version"
-data=$(jq -Rs --arg version "$version" \
-  --arg release_name "$release_name" \
-  --arg release_text "$release_text" \
-'{
-  "tag_name": $version,
-  "target_commitish": "master",
-  "name": $release_name,
-  "body": $release_text,
-  "draft": true,
-  "prerelease": false
-}' < /dev/null)
-
-out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/polkadot/releases")
-
-html_url=$(echo "$out" | jq -r .html_url)
-
-if [ "$html_url" == "null" ]
-then
-  echo "[!] Something went wrong posting:"
-  echo "$out"
-  # If we couldn't post, don't want to announce in Matrix
-  exit 1
-else
-  echo "[+] Release draft created: $html_url"
-fi
-
-echo '[+] Sending draft release URL to Matrix'
-
-msg_body=$(cat <<EOF
-**New version of polkadot tagged:** $CI_COMMIT_TAG.
-Gav: Draft release created: $html_url
-Build pipeline: $CI_PIPELINE_URL
-EOF
-)
-formatted_msg_body=$(cat <<EOF
-<strong>New version of polkadot tagged:</strong> $CI_COMMIT_TAG<br />
-Gav: Draft release created: $html_url <br />
-Build pipeline: $CI_PIPELINE_URL
-EOF
-)
-send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ROOM_ID" "$MATRIX_ACCESS_TOKEN"
-
-echo "[+] Done! Maybe the release worked..."