.gitlab-ci.yml

# Gitlab-CI Workflow
# stages:
#   build:
#     - Runs on commits on master or tags that match the pattern v1.0, v2.1rc1
#   deploy-staging:
#     - Runs on commits on master or tags that match the pattern v1.0, v2.1rc1 (continues deployment)
#   deploy-production:
#     - Runs on tags that match the pattern v1.0, v2.1rc1 (manual deployment)

variables:
  CONTAINER_REPO:                  "docker.io/parity/substrate-api-sidecar"
  HELM_NAMESPACE:                  "substrate-api-sidecar"
  HELM_RELEASE_NAME_1:             "kusama-sidecar"
  HELM_RELEASE_NAME_2:             "polkadot-sidecar"
  HELM_CUSTOM_VALUES_FILE_1:       "values-kusama.yaml"
  HELM_CUSTOM_VALUES_FILE_2:       "values-polkadot.yaml"
  HELM_REPO:                       "helm/"
  DOCKERFILE_DIRECTORY:            "./"
  CI_IMAGE:                        "quay.io/buildah/stable:v1.27"
  NODE_IMAGE:                      "node:18"

default:
  retry:
    max: 2
    when:
      - runner_system_failure
      - unknown_failure
      - api_failure
  interruptible:                   true
  before_script:
    - |-
      if [[ $CI_COMMIT_TAG =~ ^v[0-9]+\.[0-9]+.*$ ]]; then
        export DOCKER_IMAGE_TAG="${CI_COMMIT_TAG}"
        export BUILD_LATEST_IMAGE="true"
      else
        export DOCKER_IMAGE_TAG="${CI_COMMIT_SHORT_SHA}-beta"
      fi

stages:
  - test
  - build
  - deploy-staging
  - deploy-production
  - benchmark
  - check-benchmark
  - push-benchmark

.collect-artifacts:                &collect-artifacts
  artifacts:
    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
    when:                          on_success
    expire_in:                     1 days
    paths:
      - ./artifacts/

.test-refs:                     &test-refs
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs

.test-refs-manual:              &test-refs-manual
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
      when: manual

.publish-refs:                     &publish-refs
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1

.dockerize:                        &dockerize
  stage:                           build
  image:                           $CI_IMAGE
  tags:
    - kubernetes-parity-build

.kubernetes-env:                   &kubernetes-env
  image:                           $CI_IMAGE
  tags:
    - kubernetes-parity-build

.deploy-k8s:                       &deploy-k8s
  image:                           paritytech/kubetools:3.5.3
  script:
    - |-
      kubectl get cm helm-custom-values -n $HELM_NAMESPACE -o jsonpath='{.data.values-kusama\.yaml}' > values-kusama.yaml
      kubectl get cm helm-custom-values -n $HELM_NAMESPACE -o jsonpath='{.data.values-polkadot\.yaml}' > values-polkadot.yaml
    - helm upgrade
        --install
        --atomic
        --timeout 120s
        --namespace $HELM_NAMESPACE
        --set image.repository="${CONTAINER_REPO}"
        --set image.tag="${DOCKER_IMAGE_TAG}"
        --values ${HELM_CUSTOM_VALUES_FILE_1}
        ${HELM_RELEASE_NAME_1} ${HELM_REPO}
    - helm upgrade
        --install
        --atomic
        --timeout 120s
        --namespace $HELM_NAMESPACE
        --set image.repository="${CONTAINER_REPO}"
        --set image.tag="${DOCKER_IMAGE_TAG}"
        --values ${HELM_CUSTOM_VALUES_FILE_2}
        ${HELM_RELEASE_NAME_2} ${HELM_REPO}
  tags:
    - kubernetes-parity-build

.e2e-template:                     &e2e-template
  stage:                           test
  <<:                              *kubernetes-env
  variables:
    CI_IMAGE:                      $NODE_IMAGE
    CHAIN_NAME:                    ""
    CHAIN_URL:                     "wss://${CHAIN_NAME}-testing-rpc.parity-build.parity.io"
  script:
    - yarn --immutable
    - yarn test:latest-e2e-tests --chain $CHAIN_NAME --local $CHAIN_URL

e2e-westend:
  <<:                              *e2e-template
  variables:
    CI_IMAGE:                      $NODE_IMAGE
    CHAIN_NAME:                    "westend"

e2e-kusama:
  <<:                              *e2e-template
  variables:
    CI_IMAGE:                      $NODE_IMAGE
    CHAIN_NAME:                    "kusama"

e2e-polkadot:
  <<:                              *e2e-template
  variables:
    CI_IMAGE:                      $NODE_IMAGE
    CHAIN_NAME:                    "polkadot"

build:
  <<:                              *dockerize
  <<:                              *publish-refs
  script:
    - echo building "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
    - if [[ $BUILD_LATEST_IMAGE ]]; then
        buildah bud
        --format=docker
        --tag "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
        --tag "$CONTAINER_REPO:latest" "$DOCKERFILE_DIRECTORY";
      else
        buildah bud
        --format=docker
        --tag "$CONTAINER_REPO:$DOCKER_IMAGE_TAG" "$DOCKERFILE_DIRECTORY";
      fi
    - echo ${Docker_Hub_Pass_Parity} |
        buildah login --username ${Docker_Hub_User_Parity} --password-stdin docker.io
    - echo pushing "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
    - if [[ $BUILD_LATEST_IMAGE ]]; then
        buildah push --format=v2s2 "$CONTAINER_REPO:$DOCKER_IMAGE_TAG";
        buildah push --format=v2s2 "$CONTAINER_REPO:latest";
      else
        buildah push --format=v2s2 "$CONTAINER_REPO:$DOCKER_IMAGE_TAG";
      fi

# checks that dockerimage can be built without publishing
build-pr:
  <<:                              *dockerize
  <<:                              *test-refs
  script:
    - echo building "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
    - if [[ $BUILD_LATEST_IMAGE ]]; then
        buildah bud
        --format=docker
        --tag "$CONTAINER_REPO:$DOCKER_IMAGE_TAG"
        --tag "$CONTAINER_REPO:latest" "$DOCKERFILE_DIRECTORY";
      else
        buildah bud
        --format=docker
        --tag "$CONTAINER_REPO:$DOCKER_IMAGE_TAG" "$DOCKERFILE_DIRECTORY";
      fi

deploy-staging:
  stage:                           deploy-staging
  <<:                              *deploy-k8s
  <<:                              *publish-refs
  environment:
    name:                          parity-stg

deploy-production:
  stage:                           deploy-production
  <<:                              *deploy-k8s
  environment:
    name:                          parity-chains
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
      when: manual

.benchmark-template:               &benchmark-template
  stage: benchmark
  <<:                              *collect-artifacts
  <<:                              *kubernetes-env
  before_script:                   []
  variables:
    POLKADOT_ADDRESS:              "ws://substrate-api-sidecar-benchmark-node.nodes:9944"
    BENCHMARK_DURATION:            "15m"                                   # Tests duration
    BENCHMARK_THREADS:             4                                       # Total number of threads to use
    BENCHMARK_CONNECTIONS:         12                                      # Total number of HTTP connections to keep open with each thread handling N = connections/threads
    BENCHMARK_TIMEOUT:             "120s"                                  # Record a timeout if a response is not received within this amount of time
    BENCHMARK_OPTS:                "--latency"                             # Additional options, --latency:     print detailed latency statistics
    BENCHMARK_SCRIPT:              "./scripts/ci/benchmarks/lightweight-bench.lua" # https://github.com/wg/wrk/blob/master/SCRIPTING
    CI_IMAGE:                      "paritytech/node-wrk:latest"
  script:
    - export SAS_SUBSTRATE_URL=$POLKADOT_ADDRESS
    - yarn
    - yarn build
    - yarn run main &
    - sleep 10
    - wrk -d${BENCHMARK_DURATION}
          -t${BENCHMARK_THREADS}
          -c${BENCHMARK_CONNECTIONS}
          --timeout ${BENCHMARK_TIMEOUT}
          ${BENCHMARK_OPTS}
          -s ${BENCHMARK_SCRIPT} "http://127.0.0.1:8080" > result.txt
    - cat result.txt
    - mkdir artifacts/
    - mv result.txt artifacts/
  tags:
    - kubernetes-parity-build

benchmark:
  <<:                              *benchmark-template
  <<:                              *publish-refs
  needs:
    - build

# manual step to run benchmarks in PR pipeline
benchmark-manual-pr:
  <<:                              *benchmark-template
  <<:                              *test-refs-manual
  needs:
    - build-pr

check-benchmark:
  stage: check-benchmark
  <<:                              *publish-refs
  <<:                              *kubernetes-env
  <<:                              *collect-artifacts
  needs:
    - job:                         benchmark
      artifacts:                   true
  variables:
    GITHUB_REPO:                   "paritytech/substrate-api-sidecar"
    CI_IMAGE:                      "paritytech/benchmarks:latest"
    THRESHOLD:                     60000
    GITHUB_TOKEN:                  $GITHUB_PR_TOKEN
  script:
    - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}')
    - check_single_bench_result -g $GITHUB_REPO
                                -c $THRESHOLD
                                -v $RESULT

push-benchmark:
  stage: push-benchmark
  <<:                              *publish-refs
  <<:                              *kubernetes-env
  needs:
    - job:                         check-benchmark
      artifacts:                   true
  variables:
    PROMETHEUS_URL:                "https://pushgateway.parity-build.parity.io"
    CI_IMAGE:                      "paritytech/benchmarks:latest"
  script:
    - export RESULT=$(cat artifacts/result.txt | grep AvgRequestTime | awk '{print $2}')
    - push_bench_result -t common
                        -p $CI_PROJECT_NAME
                        -n sidecar
                        -r $RESULT
                        -u ms
                        -s $PROMETHEUS_URL
    - push_bench_result -t specific
                        -p $CI_PROJECT_NAME
                        -n sidecar
                        -r $RESULT
                        -l 'commit="'$CI_COMMIT_SHORT_SHA'"'
                        -u ms
                        -s $PROMETHEUS_URL