test.yml

# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
# the job can be found in check.yml
.run-immediately:
  needs:
    - job: job-starter
      artifacts: false

test-linux-stable:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .pipeline-stopper-artifacts
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  parallel: 3
  script:
    # Build all but only execute 'runtime' tests.
    - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
    # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged
    # "upgrade_version_checks_should_work" is currently failing
    - |
      time cargo nextest run \
        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
        --workspace \
        --locked \
        --release \
        --no-fail-fast \
        --features try-runtime,experimental,riscv,ci-only-tests \
        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
    # Upload tests results to Elasticsearch
    - echo "Upload test results to Elasticsearch"
    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
    - |
      curl -v -XPOST --http1.1 \
      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
      -H 'Content-Type: application/json' \
      -d @target/nextest/default/junit.json || echo "failed to upload junit report"
    # run runtime-api tests with `enable-staging-api` feature on the 1st node
    - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
  artifacts:
    when: always
    paths:
      - target/nextest/default/junit.xml
    reports:
      junit: target/nextest/default/junit.xml
  timeout: 90m

test-linux-oldkernel-stable:
  extends: test-linux-stable
  tags:
    - oldkernel-vm

# https://github.com/paritytech/ci_cd/issues/864
test-linux-stable-runtime-benchmarks:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .pipeline-stopper-artifacts
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  script:
    - time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet

# can be used to run all tests
# test-linux-stable-all:
#   stage: test
#   extends:
#     - .docker-env
#     - .common-refs
#     - .run-immediately
#   variables:
#     RUST_TOOLCHAIN: stable
#     # Enable debug assertions since we are running optimized builds for testing
#     # but still want to have debug assertions.
#     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
#   parallel: 3
#   script:
#     # Build all but only execute 'runtime' tests.
#     - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
#     - |
#       time cargo nextest run \
#         --workspace \
#         --locked \
#         --release \
#         --no-fail-fast \
#         --features runtime-benchmarks,try-runtime \
#         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
#     # todo: add flacky-test collector

# takes about 1,5h without cache
# can be used to check that nextest works correctly
# test-linux-stable-polkadot:
#   stage: test
#   timeout: 2h
#   extends:
#     - .docker-env
#     - .common-refs
#     - .run-immediately
#     - .collect-artifacts-short
#   variables:
#     RUST_TOOLCHAIN: stable
#     # Enable debug assertions since we are running optimized builds for testing
#     # but still want to have debug assertions.
#     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
#   script:
#     - mkdir -p artifacts
#     - time cargo test --workspace
#       --locked
#       --profile testnet
#       --features=runtime-benchmarks,runtime-metrics,try-runtime --
#       --skip upgrade_version_checks_should_work

test-doc:
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: test-rustdoc
      artifacts: false
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  script:
    - time cargo test --doc --workspace

test-rustdoc:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  variables:
    SKIP_WASM_BUILD: 1
  script:
    - time cargo doc --workspace --all-features --no-deps

cargo-check-all-benches:
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: cargo-hfuzz
      artifacts: false
  script:
    - time cargo check --all --benches

test-node-metrics:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts-short
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
  script:
    # Build the required workers.
    - cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
    - mkdir -p artifacts
    - time cargo test --profile testnet
      --locked
      --features=runtime-metrics -p polkadot-node-metrics > artifacts/log.txt

test-deterministic-wasm:
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: test-frame-ui
      artifacts: false
  script:
    # build runtime
    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p westend-runtime -p rococo-runtime
    # make checksum
    - sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
    - cargo clean
    # build again
    - WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p westend-runtime -p rococo-runtime
    # confirm checksum
    - sha256sum -c checksum.sha256

cargo-check-benches:
  stage: test
  artifacts:
    expire_in: 10 days
  variables:
    CI_JOB_NAME: "cargo-check-benches"
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .collect-artifacts
    - .pipeline-stopper-artifacts
  before_script:
    # TODO: DON'T FORGET TO CHANGE FOR PROD VALUES!!!
    # merges in the master branch on PRs. skip if base is not master
    - 'if [ $CI_COMMIT_REF_NAME != "master" ]; then
      BASE=$(curl -s -H "Authorization: Bearer ${GITHUB_PR_TOKEN}" https://api.github.com/repos/paritytech-stg/polkadot-sdk/pulls/${CI_COMMIT_REF_NAME} | jq -r .base.ref);
      printf "Merging base branch %s\n" "${BASE:=master}";
      if [ $BASE != "master" ]; then
      echo "$BASE is not master, skipping merge";
      else
      git config user.email "[email protected]";
      git fetch origin "refs/heads/${BASE}";
      git merge --verbose --no-edit FETCH_HEAD;
      fi
      fi'
    - !reference [.forklift-cache, before_script]
  parallel: 2
  script:
    - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
    # this job is executed in parallel on two runners
    - echo "___Running benchmarks___";
    - case ${CI_NODE_INDEX} in
      1)
      SKIP_WASM_BUILD=1 time cargo check --locked --benches --all;
      cargo run --locked --release -p node-bench -- ::trie::read::small --json
      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json;
      echo "___Cache could be uploaded___";
      ;;
      2)
      cargo run --locked --release -p node-bench -- ::node::import::sr25519::transfer_keep_alive::paritydb::small --json
      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::sr25519::transfer_keep_alive::paritydb::small.json
      ;;
      esac

node-bench-regression-guard:
  # it's not belong to `build` semantically, but dag jobs can't depend on each other
  # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
  # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
  stage: build
  extends:
    - .docker-env
    - .common-refs
  needs:
    # this is a DAG
    - job: cargo-check-benches
      artifacts: true
    # polls artifact from master to compare with current result
    # need to specify both parallel jobs from master because of the bug
    # https://gitlab.com/gitlab-org/gitlab/-/issues/39063
    - project: $CI_PROJECT_PATH
      job: "cargo-check-benches 1/2"
      ref: master
      artifacts: true
    - project: $CI_PROJECT_PATH
      job: "cargo-check-benches 2/2"
      ref: master
      artifacts: true
  variables:
    CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
  before_script: [""]
  script:
    - if [ $(ls -la artifacts/benches/ | grep master | wc -l) == 0 ]; then
      echo "Couldn't find master artifacts";
      exit 1;
      fi
    - echo "------- IMPORTANT -------"
    - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
    - echo "In case of this job failure, check your pipeline's cargo-check-benches"
    - "node-bench-regression-guard --reference artifacts/benches/master-*
      --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
  after_script: [""]

# if this fails run `bot update-ui` in the Pull Request or "./scripts/update-ui-tests.sh" locally
# see ./docs/contributor/CONTRIBUTING.md#ui-tests
test-frame-ui:
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: test-frame-examples-compile-to-wasm
      artifacts: false
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-C debug-assertions -D warnings"
    RUST_BACKTRACE: 1
    WASM_BUILD_NO_COLOR: 1
    WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
    # Ensure we run the UI tests.
    RUN_UI_TESTS: 1
  script:
    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
    - time cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
    - cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true

# This job runs all benchmarks defined in the `/bin/node/runtime` once to check that there are no errors.
quick-benchmarks:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-C debug-assertions -D warnings"
    RUST_BACKTRACE: "full"
    WASM_BUILD_NO_COLOR: 1
    WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
  script:
    - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1

test-frame-examples-compile-to-wasm:
  # into one job
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: test-full-crypto-feature
      artifacts: false
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-C debug-assertions"
    RUST_BACKTRACE: 1
  script:
    - cd ./substrate/frame/examples/offchain-worker/
    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
    - cd ../basic
    - cargo build --locked --target=wasm32-unknown-unknown --no-default-features
  # FIXME
  allow_failure: true

test-linux-stable-int:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .pipeline-stopper-artifacts
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-C debug-assertions -D warnings"
    RUST_BACKTRACE: 1
    WASM_BUILD_NO_COLOR: 1
    WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
    # Ensure we run the UI tests.
    RUN_UI_TESTS: 1
  script:
    - WASM_BUILD_NO_COLOR=1
      time cargo test -p staging-node-cli --release --locked -- --ignored

# more information about this job can be found here:
# https://github.com/paritytech/substrate/pull/6916
check-tracing:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    - .pipeline-stopper-artifacts
  script:
    # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
    - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
    - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing

# more information about this job can be found here:
# https://github.com/paritytech/substrate/pull/3778
test-full-crypto-feature:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  variables:
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
    RUSTFLAGS: "-C debug-assertions"
    RUST_BACKTRACE: 1
  script:
    - cd substrate/primitives/core/
    - time cargo build --locked --no-default-features --features full_crypto
    - cd ../application-crypto
    - time cargo build --locked --no-default-features --features full_crypto

cargo-check-each-crate:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    # - .collect-artifacts
  variables:
    RUSTFLAGS: "-D warnings"
    # $CI_JOB_NAME is set manually so that cache could be shared for all jobs
    # "cargo-check-each-crate I/N" jobs
    CI_JOB_NAME: cargo-check-each-crate
  timeout: 2h
  script:
    - PYTHONUNBUFFERED=x time .gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
  parallel: 6

cargo-check-each-crate-macos:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
    # - .collect-artifacts
  before_script:
    # skip timestamp script, the osx bash doesn't support printf %()T
    - !reference [.job-switcher, before_script]
    - !reference [.rust-info-script, script]
    - !reference [.pipeline-stopper-vars, script]
  variables:
    SKIP_WASM_BUILD: 1
  script:
    # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
    # - time ./scripts/ci/gitlab/check-each-crate.py 1 1
    - time cargo check --workspace --locked
  timeout: 2h
  tags:
    - osx

cargo-hfuzz:
  stage: test
  extends:
    - .docker-env
    - .common-refs
  # DAG
  needs:
    - job: check-tracing
      artifacts: false
  variables:
    # max 10s per iteration, 60s per file
    HFUZZ_RUN_ARGS: >
      --exit_upon_crash
      --exit_code_upon_crash 1
      --timeout 10
      --run_time 60
    # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
    # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
    # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
    # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
    HFUZZ_BUILD_ARGS: >
      --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
      --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
  artifacts:
    name: "hfuzz-$CI_COMMIT_SHORT_SHA"
    expire_in: 7 days
    when: on_failure
    paths:
      - substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
  script:
    - cd ./substrate/primitives/arithmetic/fuzzer
    - cargo hfuzz build
    - for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); do
      cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; done

# cf https://github.com/paritytech/polkadot-sdk/issues/1652
test-syscalls:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  variables:
    SKIP_WASM_BUILD: 1
  script:
    - cargo build --locked --profile production --target x86_64-unknown-linux-musl --bin polkadot-execute-worker --bin polkadot-prepare-worker
    - cd polkadot/scripts/list-syscalls
    - ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-execute-worker --only-used-syscalls | diff -u execute-worker-syscalls -
    - ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-prepare-worker --only-used-syscalls | diff -u prepare-worker-syscalls -
  after_script:
    - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
      printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
      fi
  allow_failure: false # this rarely triggers in practice

subsystem-regression-tests:
  stage: test
  extends:
    - .docker-env
    - .common-refs
    - .run-immediately
  script:
    - cargo bench --profile=testnet -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
    - cargo bench --profile=testnet -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
  tags:
    - benchmark
  allow_failure: true