[ci] Return publish-rustdoc (#1402)

* [WIP][ci] Return publish-rustdoc

* rm files

* fix ref

* add build doc

* add comment

* move test-deterministic-wasm to test stage

* rm test-deterministic-wasm from root

* test publish

* enable pipeline

* add test_deterministic_wasm.sh for resolving conflicts

* rm unused bash script

.github/pr-custom-review.yml

@@ -19,7 +19,7 @@ rules:
     condition:
       include: .*
       # excluding files from 'Runtime files' and 'CI files' rules
-      exclude: ^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/assets/(asset-hub-kusama|asset-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/bridge-hubs/(bridge-hub-kusama|bridge-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/collectives/collectives-polkadot/src/[^/]+\.rs$|^cumulus/parachains/common/src/[^/]+\.rs$|^substrate/frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*))|^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^\.github/.*
+      exclude: ^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/assets/(asset-hub-kusama|asset-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/bridge-hubs/(bridge-hub-kusama|bridge-hub-polkadot)/src/[^/]+\.rs$|^cumulus/parachains/runtimes/collectives/collectives-polkadot/src/[^/]+\.rs$|^cumulus/parachains/common/src/[^/]+\.rs$|^substrate/frame/(?!.*(nfts/.*|uniques/.*|babe/.*|grandpa/.*|beefy|merkle-mountain-range/.*|contracts/.*|election|nomination-pools/.*|staking/.*|aura/.*))|^polkadot/runtime/(kusama|polkadot)/src/[^/]+\.rs$|^\.gitlab-ci\.yml|^docker/.*|^\.github/.*|^\.gitlab/.*|^\.config/nextest.toml|^\.cargo/.*
     min_approvals: 2
     teams:
       - core-devs

.gitlab-ci.yml

@@ -197,8 +197,6 @@ default:
   extends: .build-refs
 
 include:
-  # weights jobs
-  # - gitlab/pipeline/weights.yml
   # check jobs
   - .gitlab/pipeline/check.yml
   # test jobs
@@ -211,11 +209,10 @@ include:
   - .gitlab/pipeline/publish.yml
   # zombienet jobs
   - .gitlab/pipeline/zombienet.yml
-  # # timestamp handler
+  # timestamp handler
   - project: parity/infrastructure/ci_cd/shared
     ref: v0.2
     file: /common/timestamp.yml
-
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.

.gitlab/pipeline/build.yml

@@ -105,20 +105,18 @@ build-rustdoc:
     - .run-immediately
   variables:
     SKIP_WASM_BUILD: 1
-  # artifacts:
-  #   name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
-  #   when: on_success
-  #   expire_in: 1 days
-  #   paths:
-  #     - ./crate-docs/
+  artifacts:
+    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
+    when: on_success
+    expire_in: 1 days
+    paths:
+      - ./crate-docs/
   script:
     # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
     # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
     - time cargo doc --features try-runtime,experimental --workspace --no-deps
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
-    # FIXME: remove me after CI image gets nonroot
-    - chown -R nonroot:nonroot ./crate-docs
     # Inject Simple Analytics (https://www.simpleanalytics.com/) privacy preserving tracker into
     # all .html files
     - |

.gitlab/pipeline/check.yml

@@ -103,7 +103,6 @@ test-rust-feature-propagation:
     - zepter lint propagate-feature --feature try-runtime --left-side-feature-missing=ignore --workspace --feature-enables-dep="try-runtime:frame-try-runtime" --locked
     - zepter lint propagate-feature --feature runtime-benchmarks --left-side-feature-missing=ignore --workspace --feature-enables-dep="runtime-benchmarks:frame-benchmarking" --locked
     - zepter lint propagate-feature --feature std --left-side-feature-missing=ignore --workspace --locked
-  allow_failure: true # Experimental
 
 # More info can be found here: https://github.com/paritytech/polkadot/pull/5865
 .check-runtime-migration:

.gitlab/pipeline/publish.yml

 # This file is part of .gitlab-ci.yml
 # Here are all jobs that are executed during "publish" stage
 
+publish-rustdoc:
+  stage: publish
+  extends: .kubernetes-env
+  variables:
+    CI_IMAGE: node:18
+    GIT_DEPTH: 100
+    RUSTDOCS_DEPLOY_REFS: "master"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
+  needs:
+    - job: build-rustdoc
+      artifacts: true
+  script:
+    # If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
+    # exit immediately.
+    # Putting spaces at the front and back to ensure we are not matching just any substring, but the
+    # whole space-separated value.
+    # setup ssh
+    - eval $(ssh-agent)
+    - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
+    - mkdir ~/.ssh && touch ~/.ssh/known_hosts
+    - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    # Set git config
+    - git config user.email "devops-team@parity.io"
+    - git config user.name "${GITHUB_USER}"
+    - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
+    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+    - git fetch origin gh-pages
+    # Save README and docs
+    - cp -r ./crate-docs/ /tmp/doc/
+    - cp README.md /tmp/doc/
+    # we don't need to commit changes because we copy docs to /tmp
+    - git checkout gh-pages --force
+    # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS
+    - which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud
+    - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME}
+    # Ensure the destination dir doesn't exist.
+    - rm -rf ${CI_COMMIT_REF_NAME}
+    - mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
+    # Upload files
+    - git add --all
+    # `git commit` has an exit code of > 0 if there is nothing to commit.
+    # This causes GitLab to exit immediately and marks this job failed.
+    # We don't want to mark the entire job failed if there's nothing to
+    # publish though, hence the `|| true`.
+    - git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
+      echo "___Nothing to commit___"
+    - git push origin gh-pages --force
+  after_script:
+    - rm -rf .git/ ./*
+
 # cumulus
 
 .build-push-image:

.gitlab/pipeline/test.yml

@@ -231,8 +231,6 @@ test-node-metrics:
     - time cargo test --profile testnet
       --locked
       --features=runtime-metrics -p polkadot-node-metrics > artifacts/log.txt
-  # FIXME!
-  allow_failure: true
 
 test-deterministic-wasm:
   stage: test
@@ -244,7 +242,15 @@ test-deterministic-wasm:
     - job: test-frame-ui
       artifacts: false
   script:
-    - .gitlab/test_deterministic_wasm.sh
+    # build runtime
+    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime
+    # make checksum
+    - sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
+    - cargo clean
+    # build again
+    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime
+    # confirm checksum
+    - sha256sum -c checksum.sha256
 
 cargo-check-benches:
   stage: test

.gitlab/test_deterministic_wasm.sh

-#!/usr/bin/env bash
-set -e
-
-#shellcheck source=../common/lib.sh
-source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.github/scripts/common/lib.sh"
-
-# build runtime
-WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime
-# make checksum
-sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
-
-cargo clean
-
-# build again
-WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime
-# confirm checksum
-sha256sum -c checksum.sha256