CI: build a test artifact (#527)

* CI: build a test artifact

* CI: try building before testing

* CI: try proper bin name

* CI: preempty cache to benchmark properly

* CI: run build after test

* CI: change target source

* CI: try build before test once again

* CI: chore

* CI: prepopulate target cache

* CI: conditional build; aws sync

* CI: mock building to test publishing

* CI: lacks permissions in the other job

* CI: undebug

cumulus/.gitlab-ci.yml

@@ -7,54 +7,84 @@
 
 stages:
   - test
-  - build
+  - publish
 
 variables:                         &default-vars
   GIT_STRATEGY:                    fetch
-  GIT_DEPTH:                       3
+  GIT_DEPTH:                       100
   CARGO_INCREMENTAL:               0
   CARGO_TARGET_DIR:                "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
-  CI_SERVER_NAME:                  "GitLab CI"
+  CI_IMAGE:                        "paritytech/ci-linux:production"
+  DOCKER_OS:                       "debian:stretch"
+  ARCH:                            "x86_64"
+
+.rust-info-script:                 &rust-info-script
+  - rustup show
+  - cargo --version
+  - rustup +nightly show
+  - cargo +nightly --version
+  - bash --version
+  - sccache -s
+
+.publish-refs:                     &publish-refs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
 .docker-env:                       &docker-env
-  image:                           paritytech/ci-linux:production
+  image:                           "${CI_IMAGE}"
   before_script:
-    - cargo -vV
-    - rustc -vV
-    - rustup show
-    - cargo --version
+    - *rust-info-script
+    - ./scripts/ci/pre_cache.sh
     - sccache -s
-  only:
-    - master
-    - /^v[0-9]+\.[0-9]+.*$/        # i.e. v1.0, v2.1rc1
-    - schedules
-    - web
-    - /^[0-9]+$/                   # PRs
-  dependencies:                    []
-  interruptible:                   true
   retry:
     max: 2
     when:
       - runner_system_failure
       - unknown_failure
       - api_failure
+  interruptible:                   true
   tags:
     - linux-docker
 
-#### stage:                        test
+.kubernetes-env:                   &kubernetes-env
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - unknown_failure
+      - api_failure
+  interruptible:                   true
+  tags:
+    - kubernetes-parity-build
 
-cargo-audit:
-  stage:                           test
-  <<:                              *docker-env
-  except:
-    - /^[0-9]+$/
-  script:
-    - cargo audit
-  allow_failure:                   true
+.collect-artifacts:                &collect-artifacts
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          on_success
+    expire_in:                     28 days
+    paths:
+      - ./artifacts/
+
+#### stage:                        test
 
 test-linux-stable:
   stage:                           test
   <<:                              *docker-env
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "tags"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    # It doesn't make sense to build on every commit, so we build on tags
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+      variables:
+        ARE_WE_RELEASING_YET:      maybe!
+    # web and schedule triggers can be provided with the non-empty variable ARE_WE_RELEASING_YET
+    # to run building and publishing the binary.
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+  <<:                              *collect-artifacts
   variables:
     <<:                            *default-vars
     # Enable debug assertions since we are running optimized builds for testing
@@ -62,4 +92,44 @@ test-linux-stable:
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
   script:
     - time cargo test --all --release --locked
+    # It's almost free to produce a binary here, please refrain from using it in production since
+    # it goes with the debug assertions.
+    - if [ "${ARE_WE_RELEASING_YET}" ]; then
+        echo "___Building a binary___";
+        time cargo build --release --locked --bin polkadot-collator;
+        echo "___Packing the artifacts___";
+        mkdir -p ./artifacts;
+        mv ${CARGO_TARGET_DIR}/release/polkadot-collator ./artifacts/.;
+        echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___";
+        echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION;
+      else
+        exit 0;
+      fi
     - sccache -s
+
+#### stage:                        publish
+
+publish-s3:
+  stage:                           publish
+  <<:                              *kubernetes-env
+  image:                           paritytech/awscli:latest
+  <<:                              *publish-refs
+  variables:
+    GIT_STRATEGY:                  none
+    BUCKET:                        "releases.parity.io"
+    PREFIX:                        "cumulus/${ARCH}-${DOCKER_OS}"
+  before_script:
+    # Job will fail if no artifacts were provided by test-linux-stable job. It's only possible for
+    # this test to fail if the pipeline was triggered by web or schedule trigger without supplying
+    # a nono-empty ARE_WE_RELEASING_YET variable.
+    - test -e ./artifacts/polkadot-collator ||
+        ( echo "___No artifacts were provided by the previous job, please check the build there___"; exit 1 )
+  script:
+    - echo "___Publishing a binary with debug assertions!___"
+    - echo "___VERSION = $(cat ./artifacts/VERSION) ___"
+    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
+    - echo "___Updating objects in latest path___"
+    - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
+  after_script:
+    - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
+        --recursive --human-readable --summarize

cumulus/scripts/ci/pre_cache.sh

+#!/bin/bash
+
+set -u
+
+# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
+# create such directory and
+# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists
+
+# cache lives in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}
+
+function prepopulate {
+  if [[ ! -d $1 ]]; then
+    mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME";
+    FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
+      -type d -name "$CI_JOB_NAME"  -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
+    if [[ -d $FRESH_CACHE ]]; then
+      echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
+      time cp -r "$FRESH_CACHE" "$1";
+    else
+      echo "_____No such $2 dir, proceeding from scratch_____";
+    fi
+  else
+    echo "____No need to prepopulate $2 cache____";
+  fi
+}
+
+# CARGO_HOME cache is still broken so would be handled some other way.
+prepopulate "$CARGO_TARGET_DIR" targets