diff --git a/cumulus/.gitlab-ci.yml b/cumulus/.gitlab-ci.yml
index ee22b3cd61aee9bf0af24e6085098634b0c976cd..95c1a76fdd26b0c19d99f00e528c187b02054ec5 100644
--- a/cumulus/.gitlab-ci.yml
+++ b/cumulus/.gitlab-ci.yml
@@ -7,54 +7,84 @@
stages:
- test
- - build
+ - publish
variables: &default-vars
GIT_STRATEGY: fetch
- GIT_DEPTH: 3
+ GIT_DEPTH: 100
CARGO_INCREMENTAL: 0
CARGO_TARGET_DIR: "/ci-cache/${CI_PROJECT_NAME}/targets/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}"
- CI_SERVER_NAME: "GitLab CI"
+ CI_IMAGE: "paritytech/ci-linux:production"
+ DOCKER_OS: "debian:stretch"
+ ARCH: "x86_64"
+
+.rust-info-script: &rust-info-script
+ - rustup show
+ - cargo --version
+ - rustup +nightly show
+ - cargo +nightly --version
+ - bash --version
+ - sccache -s
+
+.publish-refs: &publish-refs
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "web"
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
.docker-env: &docker-env
- image: paritytech/ci-linux:production
+ image: "${CI_IMAGE}"
before_script:
- - cargo -vV
- - rustc -vV
- - rustup show
- - cargo --version
+ - *rust-info-script
+ - ./scripts/ci/pre_cache.sh
- sccache -s
- only:
- - master
- - /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- - schedules
- - web
- - /^[0-9]+$/ # PRs
- dependencies: []
- interruptible: true
retry:
max: 2
when:
- runner_system_failure
- unknown_failure
- api_failure
+ interruptible: true
tags:
- linux-docker
-#### stage: test
+.kubernetes-env: &kubernetes-env
+ retry:
+ max: 2
+ when:
+ - runner_system_failure
+ - unknown_failure
+ - api_failure
+ interruptible: true
+ tags:
+ - kubernetes-parity-build
-cargo-audit:
- stage: test
- <<: *docker-env
- except:
- - /^[0-9]+$/
- script:
- - cargo audit
- allow_failure: true
+.collect-artifacts: &collect-artifacts
+ artifacts:
+ name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+ when: on_success
+ expire_in: 28 days
+ paths:
+ - ./artifacts/
+
+#### stage: test
test-linux-stable:
stage: test
<<: *docker-env
+ rules:
+ - if: $CI_COMMIT_REF_NAME == "master"
+ - if: $CI_COMMIT_REF_NAME == "tags"
+ - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+ # It doesn't make sense to build on every commit, so we build on tags
+ - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
+ variables:
+ ARE_WE_RELEASING_YET: maybe!
+ # web and schedule triggers can be provided with the non-empty variable ARE_WE_RELEASING_YET
+ # to run building and publishing the binary.
+ - if: $CI_PIPELINE_SOURCE == "web"
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ <<: *collect-artifacts
variables:
<<: *default-vars
# Enable debug assertions since we are running optimized builds for testing
@@ -62,4 +92,44 @@ test-linux-stable:
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
script:
- time cargo test --all --release --locked
+ # It's almost free to produce a binary here, please refrain from using it in production since
+ # it goes with the debug assertions.
+ - if [ "${ARE_WE_RELEASING_YET}" ]; then
+ echo "___Building a binary___";
+ time cargo build --release --locked --bin polkadot-collator;
+ echo "___Packing the artifacts___";
+ mkdir -p ./artifacts;
+ mv ${CARGO_TARGET_DIR}/release/polkadot-collator ./artifacts/.;
+ echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___";
+ echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION;
+ else
+ exit 0;
+ fi
- sccache -s
+
+#### stage: publish
+
+publish-s3:
+ stage: publish
+ <<: *kubernetes-env
+ image: paritytech/awscli:latest
+ <<: *publish-refs
+ variables:
+ GIT_STRATEGY: none
+ BUCKET: "releases.parity.io"
+ PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
+ before_script:
+ # Job will fail if no artifacts were provided by test-linux-stable job. It's only possible for
+ # this test to fail if the pipeline was triggered by web or schedule trigger without supplying
+ # a nono-empty ARE_WE_RELEASING_YET variable.
+ - test -e ./artifacts/polkadot-collator ||
+ ( echo "___No artifacts were provided by the previous job, please check the build there___"; exit 1 )
+ script:
+ - echo "___Publishing a binary with debug assertions!___"
+ - echo "___VERSION = $(cat ./artifacts/VERSION) ___"
+ - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
+ - echo "___Updating objects in latest path___"
+ - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
+ after_script:
+ - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
+ --recursive --human-readable --summarize
diff --git a/cumulus/scripts/ci/pre_cache.sh b/cumulus/scripts/ci/pre_cache.sh
new file mode 100755
index 0000000000000000000000000000000000000000..c25d73587bfa9898623df67df5433ceefe5830fb
--- /dev/null
+++ b/cumulus/scripts/ci/pre_cache.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -u
+
+# if there is no directory for this $CI_COMMIT_REF_NAME/$CI_JOB_NAME
+# create such directory and
+# copy recursively all the files from the newest dir which has $CI_JOB_NAME, if it exists
+
+# cache lives in /ci-cache/${CI_PROJECT_NAME}/${2}/${CI_COMMIT_REF_NAME}/${CI_JOB_NAME}
+
+function prepopulate {
+ if [[ ! -d $1 ]]; then
+ mkdir -p "/ci-cache/$CI_PROJECT_NAME/$2/$CI_COMMIT_REF_NAME";
+ FRESH_CACHE=$(find "/ci-cache/$CI_PROJECT_NAME/$2" -mindepth 2 -maxdepth 2 \
+ -type d -name "$CI_JOB_NAME" -exec stat --printf="%Y\t%n\n" {} \; |sort -n -r |head -1 |cut -f2);
+ if [[ -d $FRESH_CACHE ]]; then
+ echo "____Using" "$FRESH_CACHE" "to prepopulate the cache____";
+ time cp -r "$FRESH_CACHE" "$1";
+ else
+ echo "_____No such $2 dir, proceeding from scratch_____";
+ fi
+ else
+ echo "____No need to prepopulate $2 cache____";
+ fi
+}
+
+# CARGO_HOME cache is still broken so would be handled some other way.
+prepopulate "$CARGO_TARGET_DIR" targets