diff --git a/.editorconfig b/.editorconfig
index 47fde53b690b5b86037ede8cbf0337db8a472d7d..2b40ec32fac3e935b4f85e70bf339224d5e8f8b0 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -14,3 +14,9 @@ indent_style=space
indent_size=2
tab_width=8
end_of_line=lf
+
+[*.sh]
+indent_style=space
+indent_size=2
+tab_width=8
+end_of_line=lf
diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md
new file mode 100644
index 0000000000000000000000000000000000000000..6067dbf12fa70ab43a3065e59e7e68375b721049
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/release.md
@@ -0,0 +1,9 @@
+---
+title: Release failure for {{ ref }}
+---
+
+Pipeline for release {{ ref }} failed. Please investigate.
+
+If the pipeline has failed before pushing to crates.io, delete the release tag
+and fix the release as necessary, retagging after complete. If the pipeline has
+failed after pushing to crates.io, create a new tag incrementing the version.
diff --git a/.github/workflows/check-gitlab-pipeline.yml b/.github/workflows/check-gitlab-pipeline.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c87f17c2f732e447cacf297cfe41e7b266a4bc99
--- /dev/null
+++ b/.github/workflows/check-gitlab-pipeline.yml
@@ -0,0 +1,30 @@
+# A github action to track the status of the gitlab pipeline for tagged
+# releases, and cancel the release/create a new issue if it fails
+
+name: Monitor gitlab pipeline status
+
+on:
+ push:
+ tags:
+ - v*
+ - ci-release-*
+
+jobs:
+ monitor:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Monitor pipeline
+ run: env; ./.maintain/github/check_gitlab_pipeline.sh
+ id: monitor_pipeline
+ env:
+ TAGGER: ${{ github.event.pusher.name }}
+ - name: Create Issue
+ if: failure()
+ uses: JasonEtco/create-an-issue@v2
+ with:
+ filename: .github/ISSUE_TEMPLATE/release.md
+ assignees: ${{ github.event.pusher.name }}
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml
new file mode 100644
index 0000000000000000000000000000000000000000..08aa94417c047d4e93645e4b6d0949ee2df29777
--- /dev/null
+++ b/.github/workflows/release-bot.yml
@@ -0,0 +1,18 @@
+name: Pushes release updates to a pre-defined Matrix room
+on:
+ release:
+ types:
+ - edited
+ - prereleased
+ - published
+jobs:
+ ping_matrix:
+ runs-on: ubuntu-latest
+ steps:
+ - name: send message
+ uses: s3krit/matrix-message-action@v0.0.2
+ with:
+ room_id: ${{ secrets.MATRIX_ROOM_ID }}
+ access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}
+ message: "**${{github.event.repository.full_name}}:** A release has been ${{github.event.action}}
Release version [${{github.event.release.tag_name}}](${{github.event.release.html_url}})
***Description:***
${{github.event.release.body}}
"
+ server: "matrix.parity.io"
diff --git a/.github/workflows/release-tagging.yml b/.github/workflows/release-tagging.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c55fc13a626e0be0aef571cfae18a5c5210e7bef
--- /dev/null
+++ b/.github/workflows/release-tagging.yml
@@ -0,0 +1,20 @@
+# Github action to ensure the `release` tag always tracks latest release
+
+name: Retag release
+
+on:
+ release:
+ types: [ published ]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Set Git tag
+ uses: s3krit/walking-tag-action@master
+ with:
+ TAG_NAME: release
+ TAG_MESSAGE: Latest release
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
index aadaa13912c19dbc056245784442d8b4c9de2a82..6398c09fe796278130978c2a8598ee9270399388 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,5 @@ rls*.log
*.rej
**/wip/*.stderr
.local
+**/hfuzz_target/
+**/hfuzz_workspace/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 17080a80814b6ebba97f7d349cac60c322c23cd4..bfe202f08e1d854b2a8d8adf1a9450de1eb4f6c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,10 +5,26 @@
# pipelines can be triggered manually in the web
# setting DEPLOY_TAG will only deploy the tagged image
+# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
+# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
+#
+# my-example-job:
+# stage: test # One of the stages listed below this job (required)
+# image: parity/tools:latest # Any docker image (required)
+# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
+# dependencies:
+# - build-rust-doc-release # Any jobs that are required to run before this job (optional)
+# variables:
+# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
+# script:
+# - echo "List of shell commands to run in your job"
+# - echo "You can also just specify a script here, like so:"
+# - ./.maintain/gitlab/my_amazing_script.sh
stages:
- test
- build
+ - post-build-test
- publish
- kubernetes
- flaming-fir
@@ -22,9 +38,17 @@ variables:
# CARGO_HOME: "/ci-cache/${CI_PROJECT_NAME}/cargo/${CI_JOB_NAME}"
# CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
REGISTRY: registry.parity.io/parity/infrastructure/scripts
+ SCCACHE_DIR: "/ci-cache/${CI_PROJECT_NAME}/sccache"
+ CARGO_INCREMENTAL: 0
CI_SERVER_NAME: "GitLab CI"
DOCKER_OS: "debian:stretch"
ARCH: "x86_64"
+ # FIXME set to release
+ CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.10"
+ CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder sp-arithmetic-fuzzer"
+ CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner"
+ WASM_BINDGEN_TEST_TIMEOUT: 120
+ CHROMEDRIVER_ARGS: "--log-level=INFO --whitelisted-ips=127.0.0.1"
.collect-artifacts: &collect-artifacts
@@ -66,7 +90,7 @@ variables:
dependencies: []
interruptible: true
tags:
- - ci3
+ - k2
.build-only: &build-only
only:
@@ -79,31 +103,34 @@ variables:
#### stage: test
-# build-linux-substrate: &build-binary
-# stage: test
-# <<: *collect-artifacts
-# <<: *docker-env
-# # <<: *build-only
-# except:
-# variables:
-# - $DEPLOY_TAG
-# before_script:
-# - mkdir -p ./artifacts/substrate/
-# script:
-# - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
-# - cp ${CARGO_TARGET_DIR}/release/substrate ./artifacts/substrate/.
-# - echo -n "Substrate version = "
-# - if [ "${CI_COMMIT_TAG}" ]; then
-# echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
-# else
-# ./artifacts/substrate/substrate --version |
-# sed -n -r 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
-# tee ./artifacts/substrate/VERSION;
-# fi
-# - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
-# - printf '\n# building node-template\n\n'
-# - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
-# - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
+
+cargo-audit:
+ stage: test
+ <<: *docker-env
+ except:
+ - /^[0-9]+$/
+ script:
+ - cargo audit
+ allow_failure: true
+
+
+cargo-check-benches:
+ stage: test
+ <<: *docker-env
+ script:
+ - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all
+ - sccache -s
+
+
+cargo-check-subkey:
+ stage: test
+ <<: *docker-env
+ except:
+ - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
+ script:
+ - cd ./bin/utils/subkey
+ - BUILD_DUMMY_WASM_BINARY=1 time cargo check --release
+ - sccache -s
#### stage: build
@@ -126,102 +153,54 @@ test-linux-stable: &test-linux
variables:
- $DEPLOY_TAG
script:
- - rustup +nightly show
- # - cargo clean
- # - rm -rf $CARGO_HOME/*
- - RUSTC_WRAPPER=""
- - which cargo
- - echo $PATH
- - env
- # - git checkout 7b00a5cb2a80be68bcda7b68ad9b183646b6924b
- # - echo $CARGO_TARGET_DIR
- - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked --no-fail-fast |
+ - WASM_BUILD_NO_COLOR=1 time cargo test --all --release --verbose --locked |&
tee output.log
- # after_script:
- # - echo "___Collecting warnings for check_warnings job___"
- # - awk '/^warning:/,/^$/ { print }' output.log > ${CI_COMMIT_SHORT_SHA}_warnings.log
- # artifacts:
- # name: $CI_COMMIT_SHORT_SHA
- # expire_in: 24 hrs
- # paths:
- # - ${CI_COMMIT_SHORT_SHA}_warnings.log
-
-# node-exits:
-# stage: build
-# <<: *docker-env
-# variables:
-# GIT_STRATEGY: none
-# CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/build-linux-substrate"
-# needs:
-# - build-linux-substrate
-# except:
-# - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
-# script:
-# - ${CARGO_TARGET_DIR}/release/substrate --dev &
-# - PID=$!
-# # Let the chain running for 60 seconds
-# - sleep 60
-# # Send `SIGINT` and give the process 30 seconds to end
-# - kill -INT $PID
-# - timeout 30 tail --pid=$PID -f /dev/null
-
-# test-linux-stable-int:
-# <<: *test-linux
-# except:
-# refs:
-# - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
-# variables:
-# - $DEPLOY_TAG
-# variables:
-# CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/build-linux-substrate"
-# script:
-# - echo "___Logs will be partly shown at the end in case of failure.___"
-# - echo "___Full log will be saved to the job artifacts only in case of failure.___"
-# - WASM_BUILD_NO_COLOR=1 RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
-# time cargo test -p node-cli --release --verbose --locked -- --ignored
-# &> ${CI_COMMIT_SHORT_SHA}_int_failure.log
-# - sccache -s
-# after_script:
-# - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
-# artifacts:
-# name: $CI_COMMIT_SHORT_SHA
-# when: on_failure
-# expire_in: 24 hrs
-# paths:
-# - ${CI_COMMIT_SHORT_SHA}_int_failure.log
-
-
-# trigger the pipeline
-# control-trigger:
-# stage: publish
-# needs:
-# - build-linux-substrate
-# variables:
-# CARGO_TARGET_DIR: ""
-# SUBSTRATE_VERSION: tbdn
-# SUBSTRATE_COMMIT: $CI_COMMIT_SHA
-# SUBSTRATE_JOB_ID: $CI_JOB_ID
-# SUBSTRATE_JOB_URL: $CI_JOB_URL
-# trigger:
-# project: parity/srml-contracts-waterfall
-# branch: "28"
-# strategy: depend
-
-# check_warnings:
-# stage: publish
-# <<: *docker-env
-# except:
-# variables:
-# - $DEPLOY_TAG
-# variables:
-# GIT_STRATEGY: none
-# needs:
-# - test-linux-stable
-# script:
-# - if [ -s ${CI_COMMIT_SHORT_SHA}_warnings.log ]; then
-# cat ${CI_COMMIT_SHORT_SHA}_warnings.log;
-# exit 1;
-# else
-# echo "___No warnings___";
-# fi
-# allow_failure: true
+
+
+#### stage: build
+
+build-linux-substrate: &build-binary
+ stage: build
+ <<: *collect-artifacts
+ <<: *docker-env
+ <<: *build-only
+ before_script:
+ - mkdir -p ./artifacts/substrate/
+ except:
+ variables:
+ - $DEPLOY_TAG
+ script:
+ - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
+ - mv ./target/release/substrate ./artifacts/substrate/.
+ - echo -n "Substrate version = "
+ - if [ "${CI_COMMIT_TAG}" ]; then
+ echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
+ else
+ ./artifacts/substrate/substrate --version |
+ sed -n -r 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
+ tee ./artifacts/substrate/VERSION;
+ fi
+ - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
+ - printf '\n# building node-template\n\n'
+ - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
+ - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
+ - sccache -s
+
+build-linux-subkey:
+ <<: *build-binary
+ before_script:
+ - mkdir -p ./artifacts/subkey
+ script:
+ - cd ./bin/utils/subkey
+ - BUILD_DUMMY_WASM_BINARY=1 time cargo build --release --verbose
+ - cd -
+ - mv ./target/release/subkey ./artifacts/subkey/.
+ - echo -n "Subkey version = "
+ - ./artifacts/subkey/subkey --version |
+ sed -n -r 's/^subkey ([0-9.]+.*)/\1/p' |
+ tee ./artifacts/subkey/VERSION;
+ - sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
+ - cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
+ - sccache -s
+
+#### stage: publish
diff --git a/.maintain/Dockerfile b/.maintain/Dockerfile
index 7cba85c544afc2c8cc1ff56401b2172a01d30364..21a41720f7d65ffd41a35cb1fa425e5999c9e289 100644
--- a/.maintain/Dockerfile
+++ b/.maintain/Dockerfile
@@ -1,7 +1,7 @@
# Note: We don't use Alpine and its packaged Rust/Cargo because they're too often out of date,
# preventing them from being used to build Substrate/Polkadot.
-FROM phusion/baseimage:0.10.2 as builder
+FROM phusion/baseimage:0.11 as builder
LABEL maintainer="chevdor@gmail.com"
LABEL description="This is the build stage for Substrate. Here we create the binary."
@@ -20,13 +20,12 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
export PATH="$PATH:$HOME/.cargo/bin" && \
rustup toolchain install nightly && \
rustup target add wasm32-unknown-unknown --toolchain nightly && \
- rustup default nightly && \
rustup default stable && \
cargo build "--$PROFILE"
# ===== SECOND STAGE ======
-FROM phusion/baseimage:0.10.2
+FROM phusion/baseimage:0.11
LABEL maintainer="chevdor@gmail.com"
LABEL description="This is the 2nd stage: a very small image where we copy the Substrate binary."
ARG PROFILE=release
@@ -34,11 +33,16 @@ ARG PROFILE=release
RUN mv /usr/share/ca* /tmp && \
rm -rf /usr/share/* && \
mv /tmp/ca-certificates /usr/share/ && \
- mkdir -p /root/.local/share/Polkadot && \
- ln -s /root/.local/share/Polkadot /data && \
- useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate
+ useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate && \
+ mkdir -p /substrate/.local/share/substrate && \
+ chown -R substrate:substrate /substrate/.local && \
+ ln -s /substrate/.local/share/substrate /data
COPY --from=builder /substrate/target/$PROFILE/substrate /usr/local/bin
+COPY --from=builder /substrate/target/$PROFILE/subkey /usr/local/bin
+COPY --from=builder /substrate/target/$PROFILE/node-rpc-client /usr/local/bin
+COPY --from=builder /substrate/target/$PROFILE/node-template /usr/local/bin
+COPY --from=builder /substrate/target/$PROFILE/chain-spec-builder /usr/local/bin
# checks
RUN ldd /usr/local/bin/substrate && \
@@ -49,7 +53,7 @@ RUN rm -rf /usr/lib/python* && \
rm -rf /usr/bin /usr/sbin /usr/share/man
USER substrate
-EXPOSE 30333 9933 9944
+EXPOSE 30333 9933 9944 9615
VOLUME ["/data"]
CMD ["/usr/local/bin/substrate"]
diff --git a/.maintain/deny.toml b/.maintain/deny.toml
new file mode 100644
index 0000000000000000000000000000000000000000..8cc7635d5049be795e994b0ba740d9273b2f520f
--- /dev/null
+++ b/.maintain/deny.toml
@@ -0,0 +1,193 @@
+# This template contains all of the possible sections and their default values
+
+# Note that all fields that take a lint level have these possible values:
+# * deny - An error will be produced and the check will fail
+# * warn - A warning will be produced, but the check will not fail
+# * allow - No warning or error will be produced, though in some cases a note
+# will be
+
+# The values provided in this template are the default values that will be used
+# when any section or field is not specified in your own configuration
+
+# If 1 or more target triples (and optionally, target_features) are specified,
+# only the specified targets will be checked when running `cargo deny check`.
+# This means, if a particular package is only ever used as a target specific
+# dependency, such as, for example, the `nix` crate only being used via the
+# `target_family = "unix"` configuration, that only having windows targets in
+# this list would mean the nix crate, as well as any of its exclusive
+# dependencies not shared by any other crates, would be ignored, as the target
+# list here is effectively saying which targets you are building for.
+targets = [
+ # The triple can be any string, but only the target triples built in to
+ # rustc (as of 1.40) can be checked against actual config expressions
+ #{ triple = "x86_64-unknown-linux-musl" },
+ # You can also specify which target_features you promise are enabled for a
+ # particular target. target_features are currently not validated against
+ # the actual valid features supported by the target architecture.
+ #{ triple = "wasm32-unknown-unknown", features = ["atomics"] },
+]
+
+# This section is considered when running `cargo deny check advisories`
+# More documentation for the advisories section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html
+[advisories]
+# The path where the advisory database is cloned/fetched into
+db-path = "~/.cargo/advisory-db"
+# The url of the advisory database to use
+db-url = "https://github.com/rustsec/advisory-db"
+# The lint level for security vulnerabilities
+vulnerability = "deny"
+# The lint level for unmaintained crates
+unmaintained = "warn"
+# The lint level for crates that have been yanked from their source registry
+yanked = "warn"
+# The lint level for crates with security notices. Note that as of
+# 2019-12-17 there are no security notice advisories in
+# https://github.com/rustsec/advisory-db
+notice = "warn"
+# A list of advisory IDs to ignore. Note that ignored advisories will still
+# output a note when they are encountered.
+ignore = [
+ #"RUSTSEC-0000-0000",
+]
+# Threshold for security vulnerabilities, any vulnerability with a CVSS score
+# lower than the range specified will be ignored. Note that ignored advisories
+# will still output a note when they are encountered.
+# * None - CVSS Score 0.0
+# * Low - CVSS Score 0.1 - 3.9
+# * Medium - CVSS Score 4.0 - 6.9
+# * High - CVSS Score 7.0 - 8.9
+# * Critical - CVSS Score 9.0 - 10.0
+#severity-threshold =
+
+# This section is considered when running `cargo deny check licenses`
+# More documentation for the licenses section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html
+[licenses]
+# The lint level for crates which do not have a detectable license
+unlicensed = "deny"
+# List of explictly allowed licenses
+# See https://spdx.org/licenses/ for list of possible licenses
+# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
+allow = [
+ #"MIT",
+ #"Apache-2.0",
+ #"Apache-2.0 WITH LLVM-exception",
+]
+# List of explictly disallowed licenses
+# See https://spdx.org/licenses/ for list of possible licenses
+# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
+deny = [
+ #"Nokia",
+]
+# Lint level for licenses considered copyleft
+copyleft = "allow"
+# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses
+# * both - The license will be approved if it is both OSI-approved *AND* FSF
+# * either - The license will be approved if it is either OSI-approved *OR* FSF
+# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF
+# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved
+# * neither - This predicate is ignored and the default lint level is used
+allow-osi-fsf-free = "either"
+# Lint level used when no other predicates are matched
+# 1. License isn't in the allow or deny lists
+# 2. License isn't copyleft
+# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither"
+default = "deny"
+# The confidence threshold for detecting a license from license text.
+# The higher the value, the more closely the license text must be to the
+# canonical license text of a valid SPDX license file.
+# [possible values: any between 0.0 and 1.0].
+confidence-threshold = 0.9
+# Allow 1 or more licenses on a per-crate basis, so that particular licenses
+# aren't accepted for every possible crate as with the normal allow list
+exceptions = [
+ # Each entry is the crate and version constraint, and its specific allow
+ # list
+ #{ allow = ["Zlib"], name = "adler32", version = "*" },
+]
+
+# Some crates don't have (easily) machine readable licensing information,
+# adding a clarification entry for it allows you to manually specify the
+# licensing information
+[[licenses.clarify]]
+# The name of the crate the clarification applies to
+name = "ring"
+# THe optional version constraint for the crate
+#version = "*"
+# The SPDX expression for the license requirements of the crate
+expression = "OpenSSL"
+# One or more files in the crate's source used as the "source of truth" for
+# the license expression. If the contents match, the clarification will be used
+# when running the license check, otherwise the clarification will be ignored
+# and the crate will be checked normally, which may produce warnings or errors
+# depending on the rest of your configuration
+license-files = [
+ # Each entry is a crate relative path, and the (opaque) hash of its contents
+ { path = "LICENSE", hash = 0xbd0eed23 }
+]
+[[licenses.clarify]]
+name = "webpki"
+expression = "ISC"
+license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
+
+[licenses.private]
+# If true, ignores workspace crates that aren't published, or are only
+# published to private registries
+ignore = false
+# One or more private registries that you might publish crates to, if a crate
+# is only published to private registries, and ignore is true, the crate will
+# not have its license(s) checked
+registries = [
+ #"https://sekretz.com/registry
+]
+
+# This section is considered when running `cargo deny check bans`.
+# More documentation about the 'bans' section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html
+[bans]
+# Lint level for when multiple versions of the same crate are detected
+multiple-versions = "warn"
+# The graph highlighting used when creating dotgraphs for crates
+# with multiple versions
+# * lowest-version - The path to the lowest versioned duplicate is highlighted
+# * simplest-path - The path to the version with the fewest edges is highlighted
+# * all - Both lowest-version and simplest-path are used
+highlight = "lowest-version"
+# List of crates that are allowed. Use with care!
+allow = [
+ #{ name = "ansi_term", version = "=0.11.0" },
+]
+# List of crates to deny
+deny = [
+ { name = "parity-util-mem", version = "<0.6" }
+ # Each entry the name of a crate and a version range. If version is
+ # not specified, all versions will be matched.
+]
+# Certain crates/versions that will be skipped when doing duplicate detection.
+skip = [
+ #{ name = "ansi_term", version = "=0.11.0" },
+]
+# Similarly to `skip` allows you to skip certain crates during duplicate
+# detection. Unlike skip, it also includes the entire tree of transitive
+# dependencies starting at the specified crate, up to a certain depth, which is
+# by default infinite
+skip-tree = [
+ #{ name = "ansi_term", version = "=0.11.0", depth = 20 },
+]
+
+# This section is considered when running `cargo deny check sources`.
+# More documentation about the 'sources' section can be found here:
+# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html
+[sources]
+# Lint level for what to happen when a crate from a crate registry that is not
+# in the allow list is encountered
+unknown-registry = "deny"
+# Lint level for what to happen when a crate from a git repository that is not
+# in the allow list is encountered
+unknown-git = "warn"
+# List of URLs for allowed crate registries. Defaults to the crates.io index
+# if not specified. If it is specified but empty, no registries are allowed.
+allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+# List of URLs for allowed Git repositories
+allow-git = []
diff --git a/.maintain/github/check_gitlab_pipeline.sh b/.maintain/github/check_gitlab_pipeline.sh
new file mode 100755
index 0000000000000000000000000000000000000000..e7aaff15bf6a85b49c8d613efe64e1451a0800be
--- /dev/null
+++ b/.maintain/github/check_gitlab_pipeline.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+SUBSTRATE_API_BASEURL="https://gitlab.parity.io/api/v4/projects/145"
+
+TAG_NAME=$(echo "$GITHUB_REF" | sed -E 's_refs/tags/(.*)_\1_')
+PIPELINE_ID=$(curl -s $SUBSTRATE_API_BASEURL/pipelines | jq -r "map(select(.ref==\"$TAG_NAME\")) | .[0] | .id")
+if [ "$PIPELINE_ID" == "null" ]; then
+ echo "[!] Pipeline for $TAG_NAME not found. Exiting."
+ exit 1
+fi
+
+echo "[+] Pipeline path: https://gitlab.parity.io/parity/substrate/pipelines/$PIPELINE_ID"
+
+# 130 minute job max
+for (( c=0; c < 130; c++ )); do
+ out=$(curl -s "$SUBSTRATE_API_BASEURL/pipelines/$PIPELINE_ID" | jq -r .status)
+ case $out in
+ "success")
+ echo "[+] Pipeline $PIPELINE_ID for $TAG_NAME succeeded!"
+ exit 0
+ ;;
+ "failed")
+ echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME failed. Cannot proceed. Check job output on gitlab!"
+ exit 1
+ ;;
+ "cancelled")
+ echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME was cancelled. Cannot proceed!"
+ exit 1
+ ;;
+ "running")
+ echo "[-] Pipeline $PIPELINE_ID for $TAG_NAME still in progress..."
+ esac
+ sleep 60
+done
+# If we reach here, we timed out after 30 minutes
+echo "[!] Pipeline $PIPELINE_ID for $TAG_NAME timed out! Cannot proceed"
+echo "::set-output name=pipeline_status::timedout"
+exit 1
diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh
new file mode 100755
index 0000000000000000000000000000000000000000..eb4c6440f46c9fcafbd6a9ea9d78102518a4e619
--- /dev/null
+++ b/.maintain/gitlab/check_polkadot_companion_build.sh
@@ -0,0 +1,110 @@
+#!/bin/sh
+#
+# check if a pr is compatible with polkadot companion pr or master if not
+# available
+#
+# to override one that was just mentioned mark companion pr in the body of the
+# polkadot pr like
+#
+# polkadot companion: paritytech/polkadot#567
+#
+
+
+github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
+# use github api v3 in order to access the data without authentication
+github_header="Accept: application/vnd.github.v3+json"
+
+boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
+boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
+
+
+
+boldcat <<-EOT
+
+
+check_polkadot_companion_build
+==============================
+
+this job checks if there is a string in the description of the pr like
+
+polkadot companion: paritytech/polkadot#567
+
+
+it will then run cargo check from this polkadot's branch with substrate code
+from this pull request. in absence of that string it will check if a polkadot
+pr is mentioned and will use the last one instead. if none of the above can be
+found it will check if polkadot has a branch of the exact same name than the
+substrate's branch. if it can't find anything, it will uses master instead
+
+
+EOT
+
+
+SUBSTRATE_PATH=$(pwd)
+
+# Clone the current Polkadot master branch into ./polkadot.
+git clone --depth 1 https://github.com/paritytech/polkadot.git
+
+cd polkadot
+
+# either it's a pull request then check for a companion otherwise use
+# polkadot:master
+if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null
+then
+ boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
+
+ pr_data_file="$(mktemp)"
+ # get the last reference to a pr in polkadot
+ curl -sSL -H "${github_header}" -o "${pr_data_file}" \
+ "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}"
+
+ pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")"
+
+ pr_companion="$(echo "${pr_body}" | sed -n -r \
+ -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \
+ -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+ | tail -n 1)"
+
+ if [ -z "${pr_companion}" ]
+ then
+ pr_companion="$(echo "${pr_body}" | sed -n -r \
+ -e 's;^.*paritytech/polkadot/#([0-9]+).*$;\1;p' \
+ -e 's;^.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+ | tail -n 1)"
+ fi
+
+ if [ "${pr_companion}" ]
+ then
+ boldprint "companion pr specified/detected: #${pr_companion}"
+ git fetch --depth 1 origin refs/pull/${pr_companion}/head:pr/${pr_companion}
+ git checkout pr/${pr_companion}
+ else
+ pr_ref="$(grep -Po '"ref"\s*:\s*"\K(?!master)[^"]*' "${pr_data_file}")"
+ if git fetch --depth 1 origin "$pr_ref":branch/"$pr_ref" 2>/dev/null
+ then
+ boldprint "companion branch detected: $pr_ref"
+ git checkout branch/"$pr_ref"
+ else
+ boldprint "no companion branch found - building polkadot:master"
+ fi
+ fi
+ rm -f "${pr_data_file}"
+else
+ boldprint "this is not a pull request - building polkadot:master"
+fi
+
+# Make sure we override the crates in native and wasm build
+# patching the git path as described in the link below did not test correctly
+# https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html
+mkdir .cargo
+echo "paths = [ \"$SUBSTRATE_PATH\" ]" > .cargo/config
+
+mkdir -p target/debug/wbuild/.cargo
+cp .cargo/config target/debug/wbuild/.cargo/config
+
+# package, others are updated along the way.
+cargo update
+
+# Test Polkadot pr or master branch with this Substrate commit.
+time cargo test --all --release --verbose
+
diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5387e68f25cbba284433b8946f20f6eacfc76b9f
--- /dev/null
+++ b/.maintain/gitlab/check_polkadot_companion_status.sh
@@ -0,0 +1,100 @@
+#!/bin/sh
+#
+# check for a polkadot companion pr and ensure it has approvals and is
+# mergeable
+#
+
+github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls"
+github_api_polkadot_pull_url="https://api.github.com/repos/paritytech/polkadot/pulls"
+# use github api v3 in order to access the data without authentication
+github_header="Accept: application/vnd.github.v3+json"
+
+boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
+boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
+
+
+
+boldcat <<-EOT
+
+
+check_polkadot_companion_status
+===============================
+
+this job checks if there is a string in the description of the pr like
+
+polkadot companion: paritytech/polkadot#567
+
+or any other polkadot pr is mentioned in this pr's description and checks its
+status.
+
+
+EOT
+
+
+if ! [ "${CI_COMMIT_REF_NAME}" -gt 0 2>/dev/null ]
+then
+ boldprint "this doesn't seem to be a pull request"
+ exit 1
+fi
+
+boldprint "this is pull request no ${CI_COMMIT_REF_NAME}"
+
+pr_body="$(curl -H "${github_header}" -s ${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME} \
+ | sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p')"
+
+# get companion if explicitly specified
+pr_companion="$(echo "${pr_body}" | sed -n -r \
+ -e 's;^.*polkadot companion: paritytech/polkadot#([0-9]+).*$;\1;p' \
+ -e 's;^.*polkadot companion: https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+ | tail -n 1)"
+
+# get companion mentioned in the description
+if [ -z "${pr_companion}" ]
+then
+ pr_companion="$(echo "${pr_body}" | sed -n -r \
+ 's;^.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \
+ | tail -n 1)"
+fi
+
+if [ -z "${pr_companion}" ]
+then
+ boldprint "no companion pr found"
+ exit 0
+fi
+
+boldprint "companion pr: #${pr_companion}"
+
+# check the status of that pull request - needs to be
+# mergable and approved
+
+curl -H "${github_header}" -sS -o companion_pr.json \
+ ${github_api_polkadot_pull_url}/${pr_companion}
+
+if jq -e .merged < companion_pr.json >/dev/null
+then
+ boldprint "polkadot pr #${pr_companion} already merged"
+ exit 0
+fi
+
+if jq -e '.mergeable' < companion_pr.json >/dev/null
+then
+ boldprint "polkadot pr #${pr_companion} mergeable"
+else
+ boldprint "polkadot pr #${pr_companion} not mergeable"
+ exit 1
+fi
+
+curl -H "${github_header}" -sS -o companion_pr_reviews.json \
+ ${github_api_polkadot_pull_url}/${pr_companion}/reviews
+
+if [ -n "$(jq -r -e '.[].state | select(. == "CHANGES_REQUESTED")' < companion_pr_reviews.json)" ] && \
+ [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews.json)" ]
+then
+ boldprint "polkadot pr #${pr_companion} not APPROVED"
+ exit 1
+fi
+
+boldprint "polkadot pr #${pr_companion} state APPROVED"
+exit 0
+
+
diff --git a/.maintain/gitlab/check_runtime.sh b/.maintain/gitlab/check_runtime.sh
index 8c0720843803b61e2e2c71649c531f3d858eb57b..5b7e25e3afc4eabef01d984047a98d675ab09757 100755
--- a/.maintain/gitlab/check_runtime.sh
+++ b/.maintain/gitlab/check_runtime.sh
@@ -30,8 +30,11 @@ github_label () {
boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
git log --graph --oneline --decorate=short -n 10
-boldprint "make sure the master branch is available in shallow clones"
+boldprint "make sure the master branch and release tag are available in shallow clones"
git fetch --depth=${GIT_DEPTH:-100} origin master
+git fetch --depth=${GIT_DEPTH:-100} origin release
+git tag -f release FETCH_HEAD
+git log -n1 release
boldprint "check if the wasm sources changed"
@@ -54,9 +57,9 @@ fi
# consensus-critical logic that has changed. the runtime wasm blobs must be
# rebuilt.
-add_spec_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+add_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-sub_spec_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+sub_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
@@ -79,9 +82,9 @@ else
# check for impl_version updates: if only the impl versions changed, we assume
# there is no consensus-critical logic that has changed.
- add_impl_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+ add_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
- sub_impl_version="$(git diff origin/master...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+ sub_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
diff --git a/.maintain/gitlab/check_signed.sh b/.maintain/gitlab/check_signed.sh
new file mode 100755
index 0000000000000000000000000000000000000000..7c4cc47baba38fa41214ed0fefc7e09b75a69e7d
--- /dev/null
+++ b/.maintain/gitlab/check_signed.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+version="$CI_COMMIT_TAG"
+
+echo '[+] Checking tag has been signed'
+check_tag "paritytech/substrate" "$version"
+case $? in
+ 0) echo '[+] Tag found and has been signed'; exit 0
+ ;;
+ 1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
+ ;;
+ 2) echo '[!] Tag not found. Aborting release.'; exit 1
+esac
diff --git a/.maintain/gitlab/generate_changelog.sh b/.maintain/gitlab/generate_changelog.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ba2a507e4cac694f918aa2c88cf1585991faa373
--- /dev/null
+++ b/.maintain/gitlab/generate_changelog.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+version="$2"
+last_version="$1"
+
+all_changes="$(sanitised_git_logs "$last_version" "$version")"
+runtime_changes=""
+api_changes=""
+client_changes=""
+changes=""
+
+while IFS= read -r line; do
+ pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
+
+ # Skip if the PR has the silent label - this allows us to skip a few requests
+ if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
+ continue
+ fi
+ if has_label 'paritytech/substrate' "$pr_id" 'B1-runtimenoteworthy'; then
+ runtime_changes="$runtime_changes
+$line"
+ fi
+ if has_label 'paritytech/substrate' "$pr_id" 'B1-clientnoteworthy'; then
+ client_changes="$client_changes
+$line"
+ fi
+ if has_label 'paritytech/substrate' "$pr_id" 'B1-apinoteworthy' ; then
+ api_changes="$api_changes
+$line"
+ continue
+ fi
+done <<< "$all_changes"
+
+# Make the substrate section if there are any substrate changes
+if [ -n "$runtime_changes" ] ||
+ [ -n "$api_changes" ] ||
+ [ -n "$client_changes" ]; then
+ changes=$(cat << EOF
+Substrate changes
+-----------------
+
+EOF
+)
+ if [ -n "$runtime_changes" ]; then
+ changes="$changes
+
+Runtime
+-------
+$runtime_changes"
+ fi
+ if [ -n "$client_changes" ]; then
+ changes="$changes
+
+Client
+------
+$client_changes"
+ fi
+ if [ -n "$api_changes" ]; then
+ changes="$changes
+
+API
+---
+$api_changes"
+ fi
+ release_text="$release_text
+
+$changes"
+fi
+
+echo "$changes"
diff --git a/.maintain/gitlab/lib.sh b/.maintain/gitlab/lib.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ecc9a5f54288cd4636a05cb7f2a2d0535e40ebe9
--- /dev/null
+++ b/.maintain/gitlab/lib.sh
@@ -0,0 +1,93 @@
+#!/bin/sh
+
+api_base="https://api.github.com/repos"
+
+# Function to take 2 git tags/commits and get any lines from commit messages
+# that contain something that looks like a PR reference: e.g., (#1234)
+sanitised_git_logs(){
+ git --no-pager log --pretty=format:"%s" "$1..$2" |
+ # Only find messages referencing a PR
+ grep -E '\(#[0-9]+\)' |
+ # Strip any asterisks
+ sed 's/^* //g' |
+ # And add them all back
+ sed 's/^/* /g'
+}
+
+# Returns the last published release on github
+# Note: we can't just use /latest because that ignores prereleases
+# repo: 'organization/repo'
+# Usage: last_github_release "$repo"
+last_github_release(){
+ i=0
+ # Iterate over releases until we find the last release that's not just a draft
+ while [ $i -lt 29 ]; do
+ out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$1/releases" | jq ".[$i]")
+ echo "$out"
+ # Ugh when echoing to jq, we need to translate newlines into spaces :/
+ if [ "$(echo "$out" | tr '\r\n' ' ' | jq '.draft')" = "false" ]; then
+ echo "$out" | tr '\r\n' ' ' | jq '.tag_name'
+ return
+ else
+ i=$((i + 1))
+ fi
+ done
+}
+
+# Checks whether a tag on github has been verified
+# repo: 'organization/repo'
+# tagver: 'v1.2.3'
+# Usage: check_tag $repo $tagver
+check_tag () {
+ repo=$1
+ tagver=$2
+ tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
+ tag_sha=$(echo "$tag_out" | jq -r .object.sha)
+ object_url=$(echo "$tag_out" | jq -r .object.url)
+ if [ "$tag_sha" = "null" ]; then
+ return 2
+ fi
+ verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
+ if [ "$verified_str" = "true" ]; then
+ # Verified, everything is good
+ return 0
+ else
+ # Not verified. Bad juju.
+ return 1
+ fi
+}
+
+# Checks whether a given PR has a given label.
+# repo: 'organization/repo'
+# pr_id: 12345
+# label: B1-silent
+# Usage: has_label $repo $pr_id $label
+has_label(){
+ repo="$1"
+ pr_id="$2"
+ label="$3"
+ out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
+ [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ]
+}
+
+# Formats a message into a JSON string for posting to Matrix
+# message: 'any plaintext message'
+# formatted_message: 'optional message formatted in html'
+# Usage: structure_message $content $formatted_content (optional)
+structure_message() {
+ if [ -z "$2" ]; then
+ body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
+ else
+ body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
+ fi
+ echo "$body"
+}
+
+# Post a message to a matrix room
+# body: '{body: "JSON string produced by structure_message"}'
+# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
+# access_token: see https://matrix.org/docs/guides/client-server-api/
+# Usage: send_message $body (json formatted) $room_id $access_token
+send_message() {
+curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
+}
diff --git a/.maintain/gitlab/publish_draft_release.sh b/.maintain/gitlab/publish_draft_release.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c5813718a69f27636b2729d8bc2cc7fbf05561fa
--- /dev/null
+++ b/.maintain/gitlab/publish_draft_release.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+version="$CI_COMMIT_TAG"
+
+# Note that this is not the last *tagged* version, but the last *published* version
+last_version=$(last_github_release 'paritytech/substrate')
+
+release_text="$(./generate_release_text.sh "$last_version" "$version")"
+
+echo "[+] Pushing release to github"
+# Create release on github
+release_name="Substrate $version"
+data=$(jq -Rs --arg version "$version" \
+ --arg release_name "$release_name" \
+ --arg release_text "$release_text" \
+'{
+ "tag_name": $version,
+ "target_commitish": "master",
+ "name": $release_name,
+ "body": $release_text,
+ "draft": true,
+ "prerelease": false
+}' < /dev/null)
+
+out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases")
+
+html_url=$(echo "$out" | jq -r .html_url)
+
+if [ "$html_url" == "null" ]
+then
+ echo "[!] Something went wrong posting:"
+ echo "$out"
+else
+ echo "[+] Release draft created: $html_url"
+fi
+
+echo '[+] Sending draft release URL to Matrix'
+
+msg_body=$(cat <Release pipeline for Substrate $version complete.
+Draft release created: $html_url
+EOF
+)
+send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ROOM_ID" "$MATRIX_ACCESS_TOKEN"
+
+echo "[+] Done! Maybe the release worked..."
diff --git a/.maintain/monitoring/grafana-dashboards/README_dashboard.md b/.maintain/monitoring/grafana-dashboards/README_dashboard.md
new file mode 100644
index 0000000000000000000000000000000000000000..37bebc6f8eaae0e9430a612d7bc9676ac666e648
--- /dev/null
+++ b/.maintain/monitoring/grafana-dashboards/README_dashboard.md
@@ -0,0 +1,14 @@
+## Substrate Dashboard
+
+Shared templated Grafana dashboards.
+
+To import the dashboards follow the [Grafana
+documentation](https://grafana.com/docs/grafana/latest/reference/export_import/).
+You can see an example setup [here](../../../.maintain/sentry-node).
+
+#### Required labels on Prometheus metrics
+
+- `instance` referring to a single scrape target (see [Prometheus docs for
+ details](https://prometheus.io/docs/concepts/jobs_instances/)).
+
+- `network` referring to the Blockchain network e.g. Kusama.
diff --git a/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json b/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json
new file mode 100644
index 0000000000000000000000000000000000000000..629b22617b22a970eb1f439179f62711d1bac9b4
--- /dev/null
+++ b/.maintain/monitoring/grafana-dashboards/substrate-dashboard.json
@@ -0,0 +1,1650 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "$$hashKey": "object:15",
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "iteration": 1586424254170,
+ "links": [
+ {
+ "icon": "external link",
+ "tags": [],
+ "targetBlank": true,
+ "title": "With love from ColmenaLabs",
+ "tooltip": "",
+ "type": "link",
+ "url": "https://colmenalabs.org"
+ },
+ {
+ "icon": "external link",
+ "tags": [],
+ "targetBlank": true,
+ "title": "Polkastats.io",
+ "tooltip": "",
+ "type": "link",
+ "url": "https://polkastats.io"
+ }
+ ],
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[10m])/rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[1m])",
+ "intervalFactor": 1,
+ "legendFormat": "rate[10m] / rate[1m]",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Relative Block Production Speed",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 15,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_peers_count{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Peers count",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "cacheTimeout": null,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 12,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 17,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pluginVersion": "6.4.1",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "scalar([[metric_namespace]]_block_height{status=\"best\",instance=\"[[instance]]\",network=\"[[network]]\"})-scalar([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"})",
+ "intervalFactor": 2,
+ "legendFormat": "[[hostname]]",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Diff -> ( Best Block - Finalized )",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "cacheTimeout": null,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 18,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 18,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "rate([[metric_namespace]]_block_height{status=\"finalized\",instance=\"[[instance]]\",network=\"[[network]]\"}[10m])*60",
+ "intervalFactor": 10,
+ "legendFormat": "{{instance}} Blocks / minute",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Block rate",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 6
+ },
+ "hiddenSeries": false,
+ "id": 10,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "increase([[metric_namespace]]_block_height{instance=\"[[instance]]\",network=\"[[network]]\",status=~\"finalized|sync_target\"}[1m])",
+ "intervalFactor": 5,
+ "legendFormat": "{{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Blocks Av per min",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 6
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_block_height{instance=\"[[instance]]\",network=\"[[network]]\",status=~\"finalized|sync_target\"}",
+ "legendFormat": "{{instance}} {{status}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Block Finalized",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 12,
+ "y": 6
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_block_height{status=\"best\",instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Block height",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "Prometheus",
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 18,
+ "y": 6
+ },
+ "hiddenSeries": false,
+ "id": 20,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "data": "",
+ "expr": "[[metric_namespace]]_ready_transactions_number{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "hide": false,
+ "legendFormat": "{{instance}}",
+ "refId": "A",
+ "target": "txcount",
+ "type": "timeseries"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "TXs Count",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 23,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sync_extra_finality_proofs_active{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} active",
+ "refId": "A"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_finality_proofs_failed{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} failed",
+ "refId": "B"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_finality_proofs_importing{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} importing",
+ "refId": "C"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_finality_proofs_pending{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} pending",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Sync Proof",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 22,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sync_extra_justifications_active{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} active",
+ "refId": "A"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_justifications_failed{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} failed",
+ "refId": "B"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_justifications_importing{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} importing",
+ "refId": "C"
+ },
+ {
+ "expr": "[[metric_namespace]]_sync_extra_justifications_pending{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} pending",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Sync justifications",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 12,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_connections{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} connections",
+ "refId": "A"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_is_major_syncing{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} syncing",
+ "refId": "B"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_kbuckets_num_nodes{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} num_nodes",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "sub_libp2p",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 18,
+ "y": 12
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"FRNK\",direction=\"in\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} FRNK in",
+ "refId": "A"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"FRNK\",direction=\"out\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} FRNK out",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "libp2p_notifications",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 0,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 28,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_cpu_usage_percentage{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "CPU usage %",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 6,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 27,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_memory_usage_bytes{instance=\"[[instance]]\",network=\"[[network]]\"}",
+ "legendFormat": "{{instance}} Mem bytes",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "Memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 2,
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 12,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 25,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_network_per_sec_bytes",
+ "hide": false,
+ "legendFormat": "{{instance}}",
+ "refId": "A"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total",
+ "hide": true,
+ "legendFormat": "{{instance}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "libp2p_network_per_sec_bytes",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "cacheTimeout": null,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": null,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 6,
+ "w": 6,
+ "x": 18,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 29,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pluginVersion": "6.5.2",
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot1\",direction=\"in\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} dot1 in",
+ "refId": "B"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot2\",direction=\"in\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} dot2 in",
+ "refId": "C"
+ },
+ {
+ "expr": "[[metric_namespace]]_sub_libp2p_notifications_total{instance=\"[[instance]]\",network=\"[[network]]\",protocol=\"dot2\",direction=\"out\"}",
+ "hide": false,
+ "legendFormat": "{{instance}} dot2 out",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeFrom": null,
+ "timeRegions": [],
+ "timeShift": null,
+ "title": "libp2p_notifications",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "refresh": "5s",
+ "schemaVersion": 22,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "allValue": null,
+ "current": {
+ "selected": true,
+ "text": "substrate",
+ "value": "substrate"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": null,
+ "multi": false,
+ "name": "metric_namespace",
+ "options": [
+ {
+ "selected": true,
+ "text": "substrate",
+ "value": "substrate"
+ },
+ {
+ "selected": false,
+ "text": "polkadot",
+ "value": "polkadot"
+ }
+ ],
+ "query": "substrate, polkadot",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": true,
+ "text": "dev",
+ "value": "dev"
+ },
+ "datasource": "Prometheus",
+ "definition": "label_values(network)",
+ "hide": 0,
+ "includeAll": false,
+ "index": -1,
+ "label": null,
+ "multi": false,
+ "name": "network",
+ "options": [],
+ "query": "label_values(network)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": null,
+ "current": {
+ "selected": false,
+ "text": "validator-a:9615",
+ "value": "validator-a:9615"
+ },
+ "datasource": "Prometheus",
+ "definition": "label_values(instance)",
+ "hide": 0,
+ "includeAll": false,
+ "index": -1,
+ "label": null,
+ "multi": false,
+ "name": "instance",
+ "options": [],
+ "query": "label_values(instance)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tags": [],
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ]
+ },
+ "timezone": "",
+ "title": "Substrate Dashboard",
+ "uid": "ColmenaLabs",
+ "variables": {
+ "list": []
+ },
+ "version": 2
+}
diff --git a/.maintain/node-template-release/Cargo.toml b/.maintain/node-template-release/Cargo.toml
index 606def19bb99f1002058d1d868181cbc9c0fbb56..dd3166d58ddf463580474a02ea70efdcdbf00228 100644
--- a/.maintain/node-template-release/Cargo.toml
+++ b/.maintain/node-template-release/Cargo.toml
@@ -16,3 +16,6 @@ git2 = "0.8"
flate2 = "1.0"
[workspace]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
diff --git a/.maintain/sentry-node/docker-compose.yml b/.maintain/sentry-node/docker-compose.yml
index 0ca42613b4576e4405d14c78e154fc8ed6b614b1..376538dde5786e15d1524dc8f75072aa4c1cb59d 100644
--- a/.maintain/sentry-node/docker-compose.yml
+++ b/.maintain/sentry-node/docker-compose.yml
@@ -19,6 +19,9 @@
# - validator-a: localhost:9944
# - validator-b: localhost:9945
# - sentry-a: localhost:9946
+# - grafana: localhost:3001
+# - prometheus: localhost:9090
+
version: "3.7"
services:
@@ -26,6 +29,7 @@ services:
validator-a:
ports:
- "9944:9944"
+ - "9615:9615"
volumes:
- ../../target/release/substrate:/usr/local/bin/substrate
image: parity/substrate
@@ -58,6 +62,7 @@ services:
- "--no-telemetry"
- "--rpc-cors"
- "all"
+ - "--prometheus-external"
sentry-a:
image: parity/substrate
@@ -77,13 +82,12 @@ services:
- "--chain=local"
- "--port"
- "30333"
- - "--charlie"
- - "--bootnodes"
+ - "--sentry"
- "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR"
- - "--bootnodes"
- - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk"
- "--reserved-nodes"
- "/dns4/validator-a/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR"
+ - "--bootnodes"
+ - "/dns4/validator-b/tcp/30333/p2p/QmSVnNf9HwVMT1Y4cK1P6aoJcEZjmoTXpjKBmAABLMnZEk"
- "--no-telemetry"
- "--rpc-cors"
- "all"
@@ -92,7 +96,7 @@ services:
- "--unsafe-rpc-external"
- "--log"
- "sub-authority-discovery=trace"
- - "--sentry"
+ - "--prometheus-external"
validator-b:
image: parity/substrate
@@ -125,12 +129,43 @@ services:
- "--unsafe-rpc-external"
- "--log"
- "sub-authority-discovery=trace"
+ - "--prometheus-external"
ui:
image: polkadot-js/apps
ports:
- "3000:80"
+ prometheus:
+ image: prom/prometheus
+ networks:
+ - network-a
+ - internet
+ ports:
+ - "9090:9090"
+ links:
+ - validator-a:validator-a
+ - sentry-a:sentry-a
+ - validator-b:validator-b
+ volumes:
+ - ./prometheus/:/etc/prometheus/
+ restart: always
+
+ grafana:
+ image: grafana/grafana
+ user: "104"
+ depends_on:
+ - prometheus
+ networks:
+ - network-a
+ - internet
+ ports:
+ - 3001:3000
+ volumes:
+ - ./grafana/provisioning/:/etc/grafana/provisioning
+ - ../monitoring/grafana-dashboards/:/etc/grafana/provisioning/dashboard-definitions
+ restart: always
+
networks:
network-a:
internet:
diff --git a/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml b/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml
new file mode 100644
index 0000000000000000000000000000000000000000..ad9164fd8ea01023a2b5736fdea9bfc0259f1eed
--- /dev/null
+++ b/.maintain/sentry-node/grafana/provisioning/dashboards/dashboards.yml
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+- name: 'Prometheus'
+ orgId: 1
+ folder: ''
+ type: file
+ disableDeletion: false
+ editable: false
+ options:
+ path: /etc/grafana/provisioning/dashboard-definitions
diff --git a/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml b/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c02bb38b3d378112480b0772855e79b0f02cb02f
--- /dev/null
+++ b/.maintain/sentry-node/grafana/provisioning/datasources/datasource.yml
@@ -0,0 +1,50 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+ - name: Prometheus
+ orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+ # name of the datasource. Required
+- name: Prometheus
+ # datasource type. Required
+ type: prometheus
+ # access mode. direct or proxy. Required
+ access: proxy
+ # org id. will default to orgId 1 if not specified
+ orgId: 1
+ # url
+ url: http://prometheus:9090
+ # database password, if used
+ password:
+ # database user, if used
+ user:
+ # database name, if used
+ database:
+ # enable/disable basic auth
+ basicAuth: false
+ # basic auth username, if used
+ basicAuthUser:
+ # basic auth password, if used
+ basicAuthPassword:
+ # enable/disable with credentials headers
+ withCredentials:
+ # mark as default datasource. Max one per org
+ isDefault: true
+ #