Skip to content
Snippets Groups Projects
Commit 68321183 authored by Denis_P's avatar Denis_P :field_hockey: Committed by GitHub
Browse files

CI: optimizations (#2885) 

* CI: do not publish unnecessary things to s3 and to release registry

* CI: remove the build-linux-release job

* CI: publish less

* CI: workspace = all

* CI: more logs for the pipeline trigger

* CI: chore

* CI: nicer

* CI: new bucket for bins

* CI: no need pushing every master image to product registry

* CI: default variables in image jobs

* CI: optimize the bin build
parent c25e79d8
Branches gh-readonly-queue/master/pr-3580-e659c4b3f744b1f04ec210e448485048b6eafa64
No related merge requests found
Hide whitespace changes
Inline Side-by-side
polkadot/.gitlab-ci.yml
+ 51
− 83
View file @ 68321183
......@@ -109,23 +109,39 @@ test-deterministic-wasm:
script:
- ./scripts/gitlab/test_deterministic_wasm.sh
test-linux-stable:
test-build-linux-stable:
stage: test
<<: *rules-test
<<: *docker-env
<<: *compiler-info
<<: *collect-artifacts
variables:
RUST_TOOLCHAIN: stable
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
TARGET: native
artifacts:
paths:
- ./target/release/polkadot
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
script:
- ./scripts/gitlab/test_linux_stable.sh
# we're using the bin built here, instead of having a parallel `build-linux-release`
- time cargo build --release --verbose --bin polkadot
- sccache -s
# pack-artifacts
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
- EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
- cp -r scripts/docker/* ./artifacts
check-web-wasm:
stage: test
......@@ -148,43 +164,6 @@ check-runtime-benchmarks:
- ./scripts/gitlab/check_runtime_benchmarks.sh
- sccache -s
.pack-artifacts: &pack-artifacts
- mkdir -p ./artifacts
- VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
- mv ./target/release/polkadot ./artifacts/.
- sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
- if [ "${CI_COMMIT_TAG}" ]; then
EXTRATAG="latest";
else
EXTRATAG="$(./artifacts/polkadot --version |
sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
fi
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
- echo -n ${VERSION} > ./artifacts/VERSION
- echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
- cp -r scripts/docker/* ./artifacts
build-linux-release:
stage: test
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
RUSTFLAGS: "-Cdebug-assertions=y"
script:
- time cargo build --release --verbose
- sccache -s
- *pack-artifacts
build-adder-collator:
stage: test
<<: *collect-artifacts
......@@ -213,7 +192,7 @@ check-transaction-versions:
<<: *rules-test
<<: *docker-env
needs:
- job: test-linux-stable
- job: test-build-linux-stable
artifacts: true
before_script:
- apt-get -y update; apt-get -y install jq lsof
......@@ -235,19 +214,16 @@ generate-impl-guide:
.build-push-image: &build-push-image
<<: *kubernetes-env
image: quay.io/buildah/stable
variables:
variables: &image-variables
GIT_STRATEGY: none
# scripts/docker/Dockerfile
DOCKERFILE: Dockerfile
IMAGE_NAME: docker.io/parity/polkadot
DOCKER_USER: ${Docker_Hub_User_Parity}
DOCKER_PASS: ${Docker_Hub_Pass_Parity}
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
before_script: &check-versions
- test -s ./artifacts/VERSION || exit 1
- test -s ./artifacts/EXTRATAG || exit 1
- VERSION="$(cat ./artifacts/VERSION)"
- EXTRATAG="$(cat ./artifacts/EXTRATAG)"
- echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
- echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
script:
- test "$DOCKER_USER" -a "$DOCKER_PASS" ||
( echo "no docker credentials provided"; exit 1 )
......@@ -280,6 +256,11 @@ generate-impl-guide:
publish-polkadot-image:
stage: build
<<: *build-push-image
variables:
<<: *image-variables
# scripts/docker/Dockerfile
DOCKERFILE: Dockerfile
IMAGE_NAME: docker.io/paritypr/synth-wave
rules:
# Don't run on releases - this is handled by the Github Action here:
# .github/workflows/publish-docker-release.yml
......@@ -288,15 +269,14 @@ publish-polkadot-image:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
IMAGE_NAME: docker.io/paritypr/synth-wave
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
variables:
<<: *image-variables
IMAGE_NAME: docker.io/parity/rococo
DOCKER_USER: ${Docker_Hub_User_Parity}
DOCKER_PASS: ${Docker_Hub_Pass_Parity}
needs:
- job: build-linux-release
- job: test-build-linux-stable
artifacts: true
publish-adder-collator-image:
......@@ -304,15 +284,13 @@ publish-adder-collator-image:
stage: build
<<: *build-push-image
variables:
<<: *image-variables
# scripts/docker/collator.Dockerfile
DOCKERFILE: collator.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander
DOCKER_USER: ${PARITYPR_USER}
DOCKER_PASS: ${PARITYPR_PASS}
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
# FIXME: remove me after merging
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
needs:
- job: build-adder-collator
......@@ -332,51 +310,41 @@ publish-adder-collator-image:
publish-s3-release: &publish-s3
stage: publish
needs:
- job: build-linux-release
- job: test-build-linux-stable
artifacts: true
<<: *kubernetes-env
image: paritytech/awscli:latest
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "polkadot/${ARCH}-${DOCKER_OS}"
PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}"
rules:
# publishing binaries nightly
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
before_script:
- *check-versions
script:
- echo "uploading objects to https://${BUCKET}/${PREFIX}/${VERSION}"
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/${VERSION}/
- echo "update objects at https://${BUCKET}/${PREFIX}/${EXTRATAG}"
- echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
- aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
- echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
- find ./artifacts -type f | while read file; do
name="${file#./artifacts/}";
aws s3api copy-object
--copy-source ${BUCKET}/${PREFIX}/${VERSION}/${name}
--bucket ${BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
name="${file#./artifacts/}";
aws s3api copy-object
--copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
--bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
done
- |
cat <<-EOM
|
| polkadot binary paths:
|
| - https://${BUCKET}/${PREFIX}/${EXTRATAG}/polkadot
| - https://${BUCKET}/${PREFIX}/${VERSION}/polkadot
| - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
| - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
|
EOM
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/
- aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
--recursive --human-readable --summarize
publish-s3-adder-collator:
<<: *publish-s3
variables:
PREFIX: "rococo/${ARCH}-${DOCKER_OS}"
needs:
- job: build-adder-collator
artifacts: true
#### stage: deploy
deploy-polkasync-kusama:
......@@ -404,7 +372,7 @@ trigger-simnet:
needs:
- job: publish-polkadot-image
- job: publish-adder-collator-image
# `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here,
# `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here,
# i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
# `collator.env` bears adder-collator unique build tag. In non-triggered builds it
# can be called by `master` tag.
......
polkadot/scripts/gitlab/test_linux_stable.sh
+ 1
− 1
View file @ 68321183
......@@ -4,4 +4,4 @@ set -e
#shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
time cargo test --all --release --verbose --locked --features=runtime-benchmarks
time cargo test --workspace --release --verbose --locked --features=runtime-benchmarks
polkadot/scripts/gitlab/trigger_pipeline.sh
+ 14
− 8
View file @ 68321183
#!/bin/bash
set -eu
# API trigger another project's pipeline
echo "Triggering Simnet pipeline."
curl --silent \
-X POST \
-F "token=${CI_JOB_TOKEN}" \
......@@ -12,13 +15,14 @@ curl --silent \
-F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
-F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
"https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
tee pipeline
tee pipeline;
PIPELINE_ID=$(cat pipeline | jq ".id")
echo "\nWaiting on ${PIPELINE_ID} status..."
PIPELINE_URL=$(cat pipeline | jq ".web_url")
echo
echo "Simnet pipeline ${PIPELINE_URL} was successfully triggered."
echo "Now we're polling it to obtain the distinguished status."
# This part polls for the triggered pipeline status, the native
# `trigger` job does not return this status via API.
# This is a workaround for a Gitlab bug, waits here until
# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
# The timeout is 360 curls with 8 sec interval, roughly an hour.
......@@ -30,17 +34,19 @@ function get_status() {
jq --raw-output ".status";
}
echo "Waiting on ${PIPELINE_ID} status..."
for i in $(seq 1 360); do
STATUS=$(get_status);
echo "Triggered pipeline status is ${STATUS}";
if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
echo "Busy...";
echo;
elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
exit 1;
echo "Something's broken in: ${PIPELINE_URL}"; exit 1;
elif [[ ${STATUS} =~ ^(success)$ ]]; then
exit 0;
echo "Look how green it is: ${PIPELINE_URL}"; exit 0;
else
exit 1;
echo "Something else has happened in ${PIPELINE_URL}"; exit 1;
fi
sleep 8;
done
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment