diff --git a/polkadot/.editorconfig b/polkadot/.editorconfig
index f511aad460790b8fdf756eab15a7f2b3def16531..dda39da09e90232714823183d17c918ec70f082c 100644
--- a/polkadot/.editorconfig
+++ b/polkadot/.editorconfig
@@ -14,3 +14,9 @@ indent_style=space
indent_size=2
tab_width=8
end_of_line=lf
+
+[*.sh]
+indent_style=space
+indent_size=2
+tab_width=8
+end_of_line=lf
diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml
index 11312ad649595b5ddcf3d42c664bd8368566e7ba..1f2ef6ded2a7ff86d38a907aaf45a25ce65e623b 100644
--- a/polkadot/.gitlab-ci.yml
+++ b/polkadot/.gitlab-ci.yml
@@ -99,17 +99,7 @@ test-deterministic-wasm:
<<: *docker-env
except:
script:
- # build runtime
- - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
- # make checksum
- - sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
- # clean up – FIXME: can we reuse some of the artifacts?
- - cargo clean
- # build again
- - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
- # confirm checksum
- - sha256sum -c checksum.sha256
-
+ - ./scripts/gitlab/test_deterministic_wasm.sh
test-linux-stable: &test
stage: test
@@ -123,7 +113,7 @@ test-linux-stable: &test
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
TARGET: native
script:
- - time cargo test --all --release --verbose --locked --features runtime-benchmarks
+ - ./scripts/gitlab/test_linux_stable.sh
- sccache -s
check-web-wasm: &test
@@ -134,14 +124,7 @@ check-web-wasm: &test
script:
# WASM support is in progress. As more and more crates support WASM, we
# should add entries here. See https://github.com/paritytech/polkadot/issues/625
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml
- - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser
+ - ./scripts/gitlab/check_web_wasm.sh
- sccache -s
check-runtime-benchmarks: &test
@@ -151,7 +134,7 @@ check-runtime-benchmarks: &test
<<: *compiler_info
script:
# Check that the node will compile with `runtime-benchmarks` feature flag.
- - time cargo check --features runtime-benchmarks
+ - ./scripts/gitlab/check_runtime_benchmarks.sh
- sccache -s
build-wasm-release:
diff --git a/polkadot/scripts/gitlab/check_runtime.sh b/polkadot/scripts/gitlab/check_runtime.sh
index 30fdcd455c1d2c8dc623463dd1f83118083e1d8c..0c635c88246c58076a48fd960216b8967a73aef7 100755
--- a/polkadot/scripts/gitlab/check_runtime.sh
+++ b/polkadot/scripts/gitlab/check_runtime.sh
@@ -34,13 +34,13 @@ boldprint "make sure the master branch is available in shallow clones"
git fetch --depth="${GIT_DEPTH:-100}" origin master
runtimes=(
- "kusama"
- "polkadot"
- "westend"
+ "kusama"
+ "polkadot"
+ "westend"
)
common_dirs=(
- "common"
+ "common"
)
# Helper function to join elements in an array with a multi-char delimiter
@@ -52,176 +52,178 @@ runtime_regex="^runtime/$(join_by '|^runtime/' "${runtimes[@]}" "${common_dirs[@
boldprint "check if the wasm sources changed since ${LATEST_TAG}"
if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \
- | grep -E -q -e "$runtime_regex"
+ | grep -E -q -e "$runtime_regex"
then
- boldprint "no changes to any runtime source code detected"
- # continue checking if Cargo.lock was updated with a new substrate reference
- # and if that change includes a {spec|impl}_version update.
-
- SUBSTRATE_REFS_CHANGED="$(
- git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
- | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l
- )"
-
- # check Cargo.lock for substrate ref change
- case "${SUBSTRATE_REFS_CHANGED}" in
- (0)
- boldprint "substrate refs not changed in Cargo.lock"
- exit 0
- ;;
- (2)
- boldprint "substrate refs updated since ${LATEST_TAG}"
- ;;
- (*)
- boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}"
- exit 1
- esac
-
-
- SUBSTRATE_PREV_REF="$(
- git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
- | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
- )"
-
- SUBSTRATE_NEW_REF="$(
- git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
- | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
- )"
-
-
- boldcat <<-EOT
- previous substrate commit id ${SUBSTRATE_PREV_REF}
- new substrate commit id ${SUBSTRATE_NEW_REF}
- EOT
-
- # okay so now need to fetch the substrate repository and check whether spec_version or impl_version has changed there
- SUBSTRATE_CLONE_DIR="$(mktemp -t -d substrate-XXXXXX)"
- trap 'rm -rf "${SUBSTRATE_CLONE_DIR}"' INT QUIT TERM ABRT EXIT
-
- git clone --depth="${GIT_DEPTH:-100}" --no-tags \
- "${SUBSTRATE_REPO}" "${SUBSTRATE_CLONE_DIR}"
-
- # check if there are changes to the spec|impl versions
- git -C "${SUBSTRATE_CLONE_DIR}" diff \
- "${SUBSTRATE_PREV_REF}..${SUBSTRATE_NEW_REF}" "${SUBSTRATE_VERSIONS_FILE}" \
- | grep -E '^[\+\-][[:space:]]+(spec|impl)_version: +([0-9]+),$' || exit 0
-
- boldcat <<-EOT
- spec_version or or impl_version have changed in substrate after updating Cargo.lock
- please make sure versions are bumped in polkadot accordingly
- EOT
-
- # Now check if any of the substrate changes have been tagged D2-breaksapi
- (
- cd "${SUBSTRATE_CLONE_DIR}"
- substrate_changes="$(sanitised_git_logs "${SUBSTRATE_PREV_REF}" "${SUBSTRATE_NEW_REF}")"
- echo "$substrate_changes" | while read -r line; do
- pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
-
- if has_label 'paritytech/substrate' "$pr_id" 'D2-breaksapi'; then
- boldprint "Substrate change labelled with D2-breaksapi. Labelling..."
- github_label "D2-breaksapi"
- exit 1
- fi
- done
- )
+ boldprint "no changes to any runtime source code detected"
+ # continue checking if Cargo.lock was updated with a new substrate reference
+ # and if that change includes a {spec|impl}_version update.
+
+ SUBSTRATE_REFS_CHANGED="$(
+ git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
+ | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l
+ )"
+
+ # check Cargo.lock for substrate ref change
+ case "${SUBSTRATE_REFS_CHANGED}" in
+ (0)
+ boldprint "substrate refs not changed in Cargo.lock"
+ exit 0
+ ;;
+ (2)
+ boldprint "substrate refs updated since ${LATEST_TAG}"
+ ;;
+ (*)
+ boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}"
+ exit 1
+ esac
+
+
+ SUBSTRATE_PREV_REF="$(
+ git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
+ | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
+ )"
+
+ SUBSTRATE_NEW_REF="$(
+ git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
+ | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
+ )"
+
+
+ boldcat <<EOT
+previous substrate commit id ${SUBSTRATE_PREV_REF}
+new substrate commit id ${SUBSTRATE_NEW_REF}
+EOT
+
+ # okay so now need to fetch the substrate repository and check whether spec_version or impl_version has changed there
+ SUBSTRATE_CLONE_DIR="$(mktemp -t -d substrate-XXXXXX)"
+ trap 'rm -rf "${SUBSTRATE_CLONE_DIR}"' INT QUIT TERM ABRT EXIT
+
+ git clone --depth="${GIT_DEPTH:-100}" --no-tags \
+ "${SUBSTRATE_REPO}" "${SUBSTRATE_CLONE_DIR}"
+
+ # check if there are changes to the spec|impl versions
+ git -C "${SUBSTRATE_CLONE_DIR}" diff \
+ "${SUBSTRATE_PREV_REF}..${SUBSTRATE_NEW_REF}" "${SUBSTRATE_VERSIONS_FILE}" \
+ | grep -E '^[\+\-][[:space:]]+(spec|impl)_version: +([0-9]+),$' || exit 0
+
+ boldcat <<EOT
+spec_version or or impl_version have changed in substrate after updating Cargo.lock
+please make sure versions are bumped in polkadot accordingly
+EOT
+
+ # Now check if any of the substrate changes have been tagged D2-breaksapi
+ (
+ cd "${SUBSTRATE_CLONE_DIR}"
+ substrate_changes="$(sanitised_git_logs "${SUBSTRATE_PREV_REF}" "${SUBSTRATE_NEW_REF}")"
+ echo "$substrate_changes" | while read -r line; do
+ pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
+
+ if has_label 'paritytech/substrate' "$pr_id" 'D2-breaksapi'; then
+ boldprint "Substrate change labelled with D2-breaksapi. Labelling..."
+ github_label "D2-breaksapi"
+ exit 1
+ fi
+ done
+ )
fi
+failed_runtime_checks=()
+
# Iterate over each runtime defined at the start of the script
for RUNTIME in "${runtimes[@]}"
do
- # Check if there were changes to this specific runtime or common directories.
- # If not, we can skip to the next runtime
- regex="^runtime/$(join_by '|^runtime/' "$RUNTIME" "${common_dirs[@]}")"
- if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \
- | grep -E -q -e "$regex"; then
- continue
- fi
-
- # check for spec_version updates: if the spec versions changed, then there is
- # consensus-critical logic that has changed. the runtime wasm blobs must be
- # rebuilt.
+ # Check if there were changes to this specific runtime or common directories.
+ # If not, we can skip to the next runtime
+ regex="^runtime/$(join_by '|^runtime/' "$RUNTIME" "${common_dirs[@]}")"
+ if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \
+ | grep -E -q -e "$regex"; then
+ continue
+ fi
- add_spec_version="$(
- git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
- | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p"
- )"
- sub_spec_version="$(
- git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
- | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p"
- )"
+ # check for spec_version updates: if the spec versions changed, then there is
+ # consensus-critical logic that has changed. the runtime wasm blobs must be
+ # rebuilt.
+ add_spec_version="$(
+ git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
+ | sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p"
+ )"
+ sub_spec_version="$(
+ git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
+ | sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p"
+ )"
- # see if the version and the binary blob changed
- if [ "${add_spec_version}" != "${sub_spec_version}" ]
- then
- if git diff --name-only "origin/master...${CI_COMMIT_SHA}" \
- | grep -E -q -e "$regex"
- then
- # add label breaksapi only if this pr altered the runtime sources
- github_label "D2-breaksapi"
- fi
+ # see if the version and the binary blob changed
+ if [ "${add_spec_version}" != "${sub_spec_version}" ]
+ then
- boldcat <<-EOT
- ## RUNTIME: ${RUNTIME} ##
+ if git diff --name-only "origin/master...${CI_COMMIT_SHA}" \
+ | grep -E -q -e "$regex"
+ then
+ # add label breaksapi only if this pr altered the runtime sources
+ github_label "D2-breaksapi"
+ fi
- changes to the ${RUNTIME} runtime sources and changes in the spec version.
+ boldcat <<EOT
+## RUNTIME: ${RUNTIME} ##
- spec_version: ${sub_spec_version} -> ${add_spec_version}
+changes to the ${RUNTIME} runtime sources and changes in the spec version.
- EOT
- continue
+spec_version: ${sub_spec_version} -> ${add_spec_version}
- else
- # check for impl_version updates: if only the impl versions changed, we assume
- # there is no consensus-critical logic that has changed.
+EOT
+ continue
- add_impl_version="$(
- git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
- | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p'
- )"
- sub_impl_version="$(
- git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
- | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p'
- )"
+ else
+ # check for impl_version updates: if only the impl versions changed, we assume
+ # there is no consensus-critical logic that has changed.
+ add_impl_version="$(
+ git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
+ | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p'
+ )"
+ sub_impl_version="$(
+ git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
+ | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p'
+ )"
- # see if the impl version changed
- if [ "${add_impl_version}" != "${sub_impl_version}" ]
- then
- boldcat <<-EOT
- ## RUNTIME: ${RUNTIME} ##
+ # see if the impl version changed
+ if [ "${add_impl_version}" != "${sub_impl_version}" ]
+ then
+ boldcat <<EOT
- changes to the ${RUNTIME} runtime sources and changes in the impl version.
+## RUNTIME: ${RUNTIME} ##
- impl_version: ${sub_impl_version} -> ${add_impl_version}
+changes to the ${RUNTIME} runtime sources and changes in the impl version.
- EOT
- continue
- fi
+impl_version: ${sub_impl_version} -> ${add_impl_version}
+EOT
+ continue
+ fi
- boldcat <<-EOT
- wasm source files changed or the spec version in the substrate reference in
- the Cargo.lock but not the spec/impl version. If changes made do not alter
- logic, just bump 'impl_version'. If they do change logic, bump
- 'spec_version'.
+ failed_runtime_checks+=("$RUNTIME")
+ fi
+done
- source file directories:
- - runtime
+if [ ${#failed_runtime_checks} -gt 0 ]; then
+ boldcat <<EOT
+wasm source files changed or the spec version in the substrate reference in
+the Cargo.lock but not the spec/impl version. If changes made do not alter
+logic, just bump 'impl_version'. If they do change logic, bump
+'spec_version'.
- versions file: ${RUNTIME}
+source file directories:
+- runtime
- EOT
+version files: ${failed_runtime_checks[@]}
+EOT
- exit 1
- fi
-done
+ exit 1
+fi
exit 0
-
-# vim: noexpandtab
diff --git a/polkadot/scripts/gitlab/check_runtime_benchmarks.sh b/polkadot/scripts/gitlab/check_runtime_benchmarks.sh
new file mode 100755
index 0000000000000000000000000000000000000000..6714279d5e58e635d27f5bc657a74724df7df89c
--- /dev/null
+++ b/polkadot/scripts/gitlab/check_runtime_benchmarks.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+skip_if_companion_pr
+
+time cargo check --features runtime-benchmarks
diff --git a/polkadot/scripts/gitlab/check_web_wasm.sh b/polkadot/scripts/gitlab/check_web_wasm.sh
new file mode 100755
index 0000000000000000000000000000000000000000..056e90d1e37d3f899313fd2615b4927b43f152b1
--- /dev/null
+++ b/polkadot/scripts/gitlab/check_web_wasm.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+#shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+skip_if_companion_pr
+
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml
+time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser
diff --git a/polkadot/scripts/gitlab/lib.sh b/polkadot/scripts/gitlab/lib.sh
index ab2e7c86f6766f9227a909f9db507c0809483f2e..993cab35e3406af7a1f0e49650d1f13b4ea23754 100755
--- a/polkadot/scripts/gitlab/lib.sh
+++ b/polkadot/scripts/gitlab/lib.sh
@@ -96,3 +96,18 @@ curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.
# Pretty-printing functions
boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
+
+skip_if_companion_pr() {
+ url="https://api.github.com/repos/paritytech/polkadot/pulls/${CI_COMMIT_REF_NAME}"
+ echo "[+] API URL: $url"
+
+ pr_title=$(curl -sSL -H "Authorization: token ${GITHUB_PR_TOKEN}" "$url" | jq -r .title)
+ echo "[+] PR title: $pr_title"
+
+ if echo "$pr_title" | grep -qi '^companion'; then
+ echo "[!] PR is a companion PR. Build is already done in substrate"
+ exit 0
+ else
+ echo "[+] PR is not a companion PR. Proceeding test"
+ fi
+}
diff --git a/polkadot/scripts/gitlab/test_deterministic_wasm.sh b/polkadot/scripts/gitlab/test_deterministic_wasm.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f2bf2a4f05f13a6f03d22c8d15d5c8e0e6a9258e
--- /dev/null
+++ b/polkadot/scripts/gitlab/test_deterministic_wasm.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+#shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+skip_if_companion_pr
+
+# build runtime
+WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
+# make checksum
+sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
+# clean up - FIXME: can we reuse some of the artifacts?
+cargo clean
+# build again
+WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
+# confirm checksum
+sha256sum -c checksum.sha256
diff --git a/polkadot/scripts/gitlab/test_linux_stable.sh b/polkadot/scripts/gitlab/test_linux_stable.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f523e24434ab5b47867b56fb8f2131b7c8aaed5e
--- /dev/null
+++ b/polkadot/scripts/gitlab/test_linux_stable.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+#shellcheck source=lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
+
+skip_if_companion_pr
+
+time cargo test --all --release --verbose --locked --features runtime-benchmarks