Run bridges zombienet tests on CI (#2439)

Brridges zombienet tests are non-standard - zombienet currently missing
multiple relay chains support (see e.g.
https://github.com/paritytech/zombienet/pull/796), so we need to go live
with two relay networks, their parachains + custom test runner (which
e.g. doesn't shutdown net when its tests are finished and instead waits
for both networks tests to complete). So we are stuck with native
zombienet provider => this PR is an attempt to gather everything in a
single docker container and run tests there ~Draft, because it is far
from finishing - what I want now is to see how it works on CI~

.gitlab/pipeline/build.yml

@@ -382,3 +382,21 @@ build-subkey-linux:
 #   after_script: [""]
 #   tags:
 #     - osx
+
+# bridges
+
+# we need some non-binary artifacts in our bridges+zombienet image
+prepare-bridges-zombienet-artifacts:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  before_script:
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/bridges
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/cumulus/zombienet
+  script:
+    - cp -r bridges/zombienet ./artifacts/bridges-polkadot-sdk/bridges/zombienet
+    - cp -r cumulus/scripts ./artifacts/bridges-polkadot-sdk/cumulus/scripts
+    - cp -r cumulus/zombienet/bridge-hubs ./artifacts/bridges-polkadot-sdk/cumulus/zombienet/bridge-hubs

.gitlab/pipeline/publish.yml

@@ -66,6 +66,8 @@ publish-rustdoc:
 # note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE
+  extends:
+    - .zombienet-refs
   variables:
     DOCKERFILE: "" # docker/path-to.Dockerfile
     IMAGE_NAME: "" # docker.io/paritypr/image_name
@@ -77,6 +79,7 @@ publish-rustdoc:
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
       --build-arg IMAGE_NAME="${IMAGE_NAME}"
+      --build-arg ZOMBIENET_IMAGE="${ZOMBIENET_IMAGE}"
       --tag "$IMAGE_NAME:${DOCKER_IMAGES_VERSION}"
       --file ${DOCKERFILE} .
     - echo "$PARITYPR_PASS" |
@@ -163,3 +166,22 @@ build-push-image-substrate-pr:
   variables:
     DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/substrate"
+
+# unlike other images, bridges+zombienet image is based on Zombienet image that pulls required binaries
+# from other fresh images (polkadot and cumulus)
+build-push-image-bridges-zombienet-tests:
+  stage: publish
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .build-push-image
+  needs:
+    - job: build-linux-stable
+      artifacts: true
+    - job: build-linux-stable-cumulus
+      artifacts: true
+    - job: prepare-bridges-zombienet-artifacts
+      artifacts: true
+  variables:
+    DOCKERFILE: "docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile"
+    IMAGE_NAME: "docker.io/paritypr/bridges-zombienet-tests"

.gitlab/pipeline/zombienet.yml

@@ -10,3 +10,5 @@ include:
   - .gitlab/pipeline/zombienet/cumulus.yml
   # polkadot tests
   - .gitlab/pipeline/zombienet/polkadot.yml
+  # bridges tests
+  - .gitlab/pipeline/zombienet/bridges.yml

.gitlab/pipeline/zombienet/bridges.yml

+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage for bridges
+
+# common settings for all zombienet jobs
+.zombienet-bridges-common:
+  before_script:
+    # Exit if the job is not merge queue
+    # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi
+    - echo "Zombienet Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${GH_DIR}"
+    - echo "${LOCAL_DIR}"
+    - ls "${LOCAL_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${BRIDGES_ZOMBIENET_TESTS_IMAGE}":${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage: zombienet
+  image: "${BRIDGES_ZOMBIENET_TESTS_IMAGE}:${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}"
+  needs:
+    - job: build-push-image-bridges-zombienet-tests
+      artifacts: true
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  variables:
+    BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG: ${DOCKER_IMAGES_VERSION}
+    BRIDGES_ZOMBIENET_TESTS_IMAGE: "docker.io/paritypr/bridges-zombienet-tests"
+    GH_DIR: "https://github.com/paritytech/polkadot-sdk/tree/${CI_COMMIT_SHA}/bridges/zombienet"
+    LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/bridges/zombienet"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+    RUN_IN_CONTAINER: "1"
+  artifacts:
+    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}_zombienet_bridge_tests"
+    when: always
+    expire_in: 2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    # copy logs of tests runner (run-tests.sh)
+    - cp -r /tmp/bridges-zombienet-tests.*/tmp.*/tmp.* ./zombienet-logs/
+    # copy logs of all nodes
+    - cp /tmp/zombie*/logs/* ./zombienet-logs/
+# following lines are causing spurious test failures ("At least one of the nodes fails to start")
+#  retry: 2
+#  tags:
+#    - zombienet-polkadot-integration-test
+
+zombienet-bridges-0001-asset-transfer-works:
+  extends:
+    - .zombienet-bridges-common
+  script:
+    - /home/nonroot/bridges-polkadot-sdk/bridges/zombienet/run-tests.sh --docker
+    - echo "Done"

bridges/zombienet/run-tests.sh

 #!/bin/bash
 #set -eu
+set -x
 shopt -s nullglob
 
-trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+trap "trap - SIGINT SIGTERM EXIT && kill -- -$$" SIGINT SIGTERM EXIT
+
+# whether to use paths for zombienet+bridges tests container or for local testing
+ZOMBIENET_DOCKER_PATHS=0
+while [ $# -ne 0 ]
+do
+    arg="$1"
+    case "$arg" in
+        --docker)
+            ZOMBIENET_DOCKER_PATHS=1
+            ;;
+    esac
+    shift
+done
 
 # assuming that we'll be using native provide && all processes will be executing locally
 # (we need absolute paths here, because they're used when scripts are called by zombienet from tmp folders)
 export POLKADOT_SDK_FOLDER=`realpath $(dirname "$0")/../..`
 export BRIDGE_TESTS_FOLDER=$POLKADOT_SDK_FOLDER/bridges/zombienet/tests
-export POLKADOT_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot
-export POLKADOT_PARACHAIN_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot-parachain
-export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=$POLKADOT_PARACHAIN_BINARY_PATH
-export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=$POLKADOT_PARACHAIN_BINARY_PATH
-export ZOMBIENET_BINARY_PATH=~/local_bridge_testing/bin/zombienet-linux
+
+# set pathc to binaries
+if [ "$ZOMBIENET_DOCKER_PATHS" -eq 1 ]; then
+    export POLKADOT_BINARY_PATH=/usr/local/bin/polkadot
+    export POLKADOT_PARACHAIN_BINARY_PATH=/usr/local/bin/polkadot-parachain
+    export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=/usr/local/bin/polkadot-parachain
+    export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=/usr/local/bin/polkadot-parachain
+
+    export SUBSTRATE_RELAY_PATH=/usr/local/bin/substrate-relay
+    export ZOMBIENET_BINARY_PATH=/usr/local/bin/zombie
+else
+    export POLKADOT_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot
+    export POLKADOT_PARACHAIN_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot-parachain
+    export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=$POLKADOT_PARACHAIN_BINARY_PATH
+    export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=$POLKADOT_PARACHAIN_BINARY_PATH
+
+    export SUBSTRATE_RELAY_PATH=~/local_bridge_testing/bin/substrate-relay
+    export ZOMBIENET_BINARY_PATH=~/local_bridge_testing/bin/zombienet-linux
+fi
+
+# check if `wait` supports -p flag
+if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH_5_1=1; else IS_BASH_5_1=0; fi
 
 # check if `wait` supports -p flag
 if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH_5_1=1; else IS_BASH_5_1=0; fi
@@ -21,13 +52,17 @@ if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH
 export LANE_ID="00000002"
 
 # tests configuration
-ALL_TESTS_FOLDER=`mktemp -d`
+ALL_TESTS_FOLDER=`mktemp -d /tmp/bridges-zombienet-tests.XXXXX`
 
 function start_coproc() {
     local command=$1
     local name=$2
     local coproc_log=`mktemp -p $TEST_FOLDER`
     coproc COPROC {
+        # otherwise zombienet uses some hardcoded paths
+        unset RUN_IN_CONTAINER
+        unset ZOMBIENET_IMAGE
+
         $command >$coproc_log 2>&1
     }
     TEST_COPROCS[$COPROC_PID, 0]=$name
@@ -90,6 +125,7 @@ do
             echo "=== Shutting down. Log of failed process below                    ==="
             echo "====================================================================="
             echo $coproc_stdout
+
             exit 1
         fi
 

cumulus/scripts/bridges_common.sh

 #!/bin/bash
 
-function ensure_binaries() {
-    if [[ ! -f ~/local_bridge_testing/bin/polkadot ]]; then
-        echo "  Required polkadot binary '~/local_bridge_testing/bin/polkadot' does not exist!"
-        echo "  You need to build it and copy to this location!"
-        echo "  Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)"
-        exit 1
-    fi
-    if [[ ! -f ~/local_bridge_testing/bin/polkadot-parachain ]]; then
-        echo "  Required polkadot-parachain binary '~/local_bridge_testing/bin/polkadot-parachain' does not exist!"
-        echo "  You need to build it and copy to this location!"
-        echo "  Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)"
-        exit 1
-    fi
+function relayer_path() {
+    local default_path=~/local_bridge_testing/bin/substrate-relay
+    local path="${SUBSTRATE_RELAY_PATH:-$default_path}"
+    echo "$path"
 }
 
 function ensure_relayer() {
-    if [[ ! -f ~/local_bridge_testing/bin/substrate-relay ]]; then
-        echo "  Required substrate-relay binary '~/local_bridge_testing/bin/substrate-relay' does not exist!"
+    local path=$(relayer_path)
+    if [[ ! -f "$path" ]]; then
+        echo "  Required substrate-relay binary '$path' does not exist!"
         echo "  You need to build it and copy to this location!"
         echo "  Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)"
         exit 1
     fi
+
+    echo $path
 }
 
 function ensure_polkadot_js_api() {

cumulus/scripts/bridges_rococo_westend.sh

@@ -132,10 +132,10 @@ LANE_ID="00000002"
 XCM_VERSION=3
 
 function init_ro_wnd() {
-    ensure_relayer
+    local relayer_path=$(ensure_relayer)
 
     RUST_LOG=runtime=trace,rpc=trace,bridge=trace \
-        ~/local_bridge_testing/bin/substrate-relay init-bridge rococo-to-bridge-hub-westend \
+        $relayer_path init-bridge rococo-to-bridge-hub-westend \
 	--source-host localhost \
 	--source-port 9942 \
 	--source-version-mode Auto \
@@ -146,10 +146,10 @@ function init_ro_wnd() {
 }
 
 function init_wnd_ro() {
-    ensure_relayer
+    local relayer_path=$(ensure_relayer)
 
     RUST_LOG=runtime=trace,rpc=trace,bridge=trace \
-        ~/local_bridge_testing/bin/substrate-relay init-bridge westend-to-bridge-hub-rococo \
+        $relayer_path init-bridge westend-to-bridge-hub-rococo \
         --source-host localhost \
         --source-port 9945 \
         --source-version-mode Auto \
@@ -160,10 +160,10 @@ function init_wnd_ro() {
 }
 
 function run_relay() {
-    ensure_relayer
+    local relayer_path=$(ensure_relayer)
 
     RUST_LOG=runtime=trace,rpc=trace,bridge=trace \
-        ~/local_bridge_testing/bin/substrate-relay relay-headers-and-messages bridge-hub-rococo-bridge-hub-westend \
+        $relayer_path relay-headers-and-messages bridge-hub-rococo-bridge-hub-westend \
         --rococo-host localhost \
         --rococo-port 9942 \
         --rococo-version-mode Auto \

docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile

+# this image is built on top of existing Zombienet image
+ARG ZOMBIENET_IMAGE
+# this image uses substrate-relay image built elsewhere
+ARG SUBSTRATE_RELAY_IMAGE=docker.io/paritytech/substrate-relay:v2023-11-07-rococo-westend-initial-relayer
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+ARG IMAGE_NAME
+
+# we need `substrate-relay` binary, built elsewhere
+FROM ${SUBSTRATE_RELAY_IMAGE} as relay-builder
+
+# the base image is the zombienet image - we are planning to run zombienet tests using native
+# provider here
+FROM ${ZOMBIENET_IMAGE}
+
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="${IMAGE_NAME}" \
+	io.parity.image.description="Bridges Zombienet tests." \
+	io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://github.com/paritytech/polkadot-sdk/bridges/zombienet"
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+USER root
+
+# for native provider to work (TODO: fix in zn docker?)
+RUN apt-get update && apt-get install -y procps sudo
+RUN yarn global add @polkadot/api-cli
+
+# add polkadot binary to the docker image
+COPY ./artifacts/polkadot /usr/local/bin/
+COPY ./artifacts/polkadot-execute-worker /usr/local/bin/
+COPY ./artifacts/polkadot-prepare-worker /usr/local/bin/
+# add polkadot-parachain binary to the docker image
+COPY ./artifacts/polkadot-parachain /usr/local/bin
+# copy substrate-relay to the docker image
+COPY --from=relay-builder /home/user/substrate-relay /usr/local/bin/
+# we need bridges zombienet runner and tests
+RUN	mkdir -p /home/nonroot/bridges-polkadot-sdk
+COPY ./artifacts/bridges-polkadot-sdk /home/nonroot/bridges-polkadot-sdk
+# also prepare `generate_hex_encoded_call` for running
+RUN set -eux; \
+	cd /home/nonroot/bridges-polkadot-sdk/cumulus/scripts/generate_hex_encoded_call; \
+	npm install
+
+# check if executable works in this container
+USER nonroot
+RUN /usr/local/bin/polkadot --version
+RUN /usr/local/bin/polkadot-parachain --version
+RUN /usr/local/bin/substrate-relay --version
+
+# https://polkadot.js.org/apps/?rpc=ws://127.0.0.1:{PORT}#/explorer
+EXPOSE 9942 9910 8943 9945 9010 8945
+
+ENTRYPOINT ["/bin/bash", "-c", "/home/nonroot/bridges-polkadot-sdk/bridges/zombienet/run-tests.sh"]