diff --git a/polkadot/.github/workflows/check-licenses.yml b/polkadot/.github/workflows/check-licenses.yml index a4c8d5d974246234c0e5c470308b95c657d51515..522037b6827c16fb1ed7a5e526807a3f7b808fd0 100644 --- a/polkadot/.github/workflows/check-licenses.yml +++ b/polkadot/.github/workflows/check-licenses.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout sources - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3 - uses: actions/setup-node@v3.7.0 with: node-version: '18.x' diff --git a/polkadot/.github/workflows/release-40_publish-rc-image.yml b/polkadot/.github/workflows/release-40_publish-rc-image.yml new file mode 100644 index 0000000000000000000000000000000000000000..a821eaa033fde7f1f5dbbdff732be346fd9ed07a --- /dev/null +++ b/polkadot/.github/workflows/release-40_publish-rc-image.yml @@ -0,0 +1,132 @@ +name: Release - Publish RC Container image +# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277 + +on: + workflow_dispatch: + inputs: + release_id: + description: | + Release ID. + You can find it using the command: + curl -s \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ + jq '.[] | { name: .name, id: .id }' + required: true + type: string + registry: + description: "Container registry" + required: true + type: string + default: docker.io + owner: + description: Owner of the container image repo + required: true + type: string + default: parity + +env: + RELEASE_ID: ${{ inputs.release_id }} + ENGINE: docker + REGISTRY: ${{ inputs.registry }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} + REPO: ${{ github.repository }} + ARTIFACT_FOLDER: release-artifacts + +jobs: + fetch-artifacts: + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Fetch all artifacts + run: | + . ./scripts/ci/common/lib.sh + fetch_release_artifacts + + - name: Cache the artifacts + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + key: artifacts-${{ github.sha }} + path: | + ${ARTIFACT_FOLDER}/**/* + + build-container: + runs-on: ubuntu-latest + needs: fetch-artifacts + + strategy: + matrix: + binary: ["polkadot", "staking-miner"] + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Get artifacts from cache + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 + with: + key: artifacts-${{ github.sha }} + path: | + ${ARTIFACT_FOLDER}/**/* + + - name: Check sha256 ${{ matrix.binary }} + working-directory: ${ARTIFACT_FOLDER} + run: | + . ../scripts/ci/common/lib.sh + + echo "Checking binary ${{ matrix.binary }}" + check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR" + + - name: Check GPG ${{ matrix.binary }} + working-directory: ${ARTIFACT_FOLDER} + run: | + . ../scripts/ci/common/lib.sh + import_gpg_keys + check_gpg ${{ matrix.binary }} + + - name: Fetch commit and tag + id: fetch_refs + run: | + release=release-${{ inputs.release_id }} && \ + echo "release=${release}" >> $GITHUB_OUTPUT + + commit=$(git rev-parse --short HEAD) && \ + echo "commit=${commit}" >> $GITHUB_OUTPUT + + tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ + [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ + echo "No tag, doing without" + + - name: Build Injected Container image for ${{ matrix.binary }} + env: + BIN_FOLDER: ${ARTIFACT_FOLDER} + BINARY: ${{ matrix.binary }} + TAGS: ${{join(steps.fetch_refs.outputs.*, ',')}} + run: | + echo "Building container for ${{ matrix.binary }}" + ./scripts/ci/dockerfiles/build-injected.sh + + - name: Login to Dockerhub + uses: docker/login-action@v2 + with: + username: ${{ inputs.owner }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push Container image for ${{ matrix.binary }} + id: docker_push + env: + BINARY: ${{ matrix.binary }} + run: | + $ENGINE images | grep ${BINARY} + $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} + + - name: Check version for the published image for ${{ matrix.binary }} + env: + BINARY: ${{ matrix.binary }} + RELEASE_TAG: ${{ steps.fetch_refs.outputs.release }} + run: | + echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" + $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version diff --git a/polkadot/.github/workflows/release-50_publish-docker-release.yml b/polkadot/.github/workflows/release-50_publish-docker-release.yml index a6bf19162a463e21964338e0592246eebb7c370b..81e5caa718f3e251f8eeded4210d2d4448ecb036 100644 --- a/polkadot/.github/workflows/release-50_publish-docker-release.yml +++ b/polkadot/.github/workflows/release-50_publish-docker-release.yml @@ -30,7 +30,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true - file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile + file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile tags: | parity/polkadot:latest parity/polkadot:${{ github.event.release.tag_name }} diff --git a/polkadot/.github/workflows/release-51_publish-docker-manual.yml b/polkadot/.github/workflows/release-51_publish-docker-manual.yml index 0c973d33b71c0305599b45d3644921767ce5b6a9..919769f8700d1ac70f0d831a5f6645aff86f42f2 100644 --- a/polkadot/.github/workflows/release-51_publish-docker-manual.yml +++ b/polkadot/.github/workflows/release-51_publish-docker-manual.yml @@ -37,7 +37,7 @@ jobs: uses: docker/build-push-action@v4 with: push: true - file: scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile + file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile tags: | parity/polkadot:latest parity/polkadot:${{ github.event.inputs.version }} diff --git a/polkadot/.gitignore b/polkadot/.gitignore index 0c6913dac34065229d84a1780db609407b93451d..61ef9e91a55e7007926b35e3cba2aea95788830b 100644 --- a/polkadot/.gitignore +++ b/polkadot/.gitignore @@ -10,3 +10,7 @@ polkadot.* !polkadot.service .DS_Store .env + +artifacts +release-artifacts +release.json diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml index 5056012e588e26e7795e01c4e1730a1d714efce9..5a84bbfeba85f0bd7a208aa833357ac4c9e1065b 100644 --- a/polkadot/.gitlab-ci.yml +++ b/polkadot/.gitlab-ci.yml @@ -159,31 +159,39 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .build-push-image: + variables: + CI_IMAGE: "${BUILDAH_IMAGE}" + + REGISTRY: "docker.io" + DOCKER_OWNER: "paritypr" + DOCKER_USER: "${PARITYPR_USER}" + DOCKER_PASS: "${PARITYPR_PASS}" + IMAGE: "${REGISTRY}/${DOCKER_OWNER}/${IMAGE_NAME}" + + ENGINE: "${BUILDAH_COMMAND}" + BUILDAH_FORMAT: "docker" + SKIP_IMAGE_VALIDATION: 1 + + PROJECT_ROOT: "." + BIN_FOLDER: "./artifacts" + VCS_REF: "${CI_COMMIT_SHA}" + before_script: - !reference [.common-before-script, before_script] - test -s ./artifacts/VERSION || exit 1 - test -s ./artifacts/EXTRATAG || exit 1 - - VERSION="$(cat ./artifacts/VERSION)" + - export VERSION="$(cat ./artifacts/VERSION)" - EXTRATAG="$(cat ./artifacts/EXTRATAG)" - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" script: - test "$DOCKER_USER" -a "$DOCKER_PASS" || ( echo "no docker credentials provided"; exit 1 ) - - cd ./artifacts - - $BUILDAH_COMMAND build - --format=docker - --build-arg VCS_REF="${CI_COMMIT_SHA}" - --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" - --build-arg IMAGE_NAME="${IMAGE_NAME}" - --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:$EXTRATAG" - --file ${DOCKERFILE} . - # The job will success only on the protected branch + - TAGS="${VERSION},${EXTRATAG}" scripts/ci/dockerfiles/build-injected.sh - echo "$DOCKER_PASS" | - buildah login --username "$DOCKER_USER" --password-stdin docker.io + buildah login --username "$DOCKER_USER" --password-stdin "${REGISTRY}" - $BUILDAH_COMMAND info - - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION" - - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$EXTRATAG" + - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE:$VERSION" + - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE:$EXTRATAG" after_script: - buildah logout --all diff --git a/polkadot/doc/docker.md b/polkadot/doc/docker.md index e8b7fa74732e7202046088985cde1573a51257d5..f20c2d001edd4b9a75261573b7955e5146b28002 100644 --- a/polkadot/doc/docker.md +++ b/polkadot/doc/docker.md @@ -1,43 +1,58 @@ -# Using Docker +# Using Containers + +The following commands should work no matter if you use Docker or Podman. In general, Podman is recommended. All commands are "engine neutral" so you can use the container engine of your choice while still being able to copy/paste the commands below. + +Let's start defining Podman as our engine: +``` +ENGINE=podman +``` + +If you prefer to stick with Docker, use: +``` +ENGINE=docker +``` ## The easiest way -The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use the latest official release of the Polkadot binary, pulled from our package repository. +The easiest/faster option to run Polkadot in Docker is to use the latest release images. These are small images that use the latest official release of the Polkadot binary, pulled from our Debian package. -**_Following examples are running on westend chain and without SSL. They can be used to quick start and learn how Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do not expose RPC and WS ports, if they are not correctly configured._** +**_The following examples are running on westend chain and without SSL. They can be used to quick start and learn how Polkadot needs to be configured. Please find out how to secure your node, if you want to operate it on the internet. Do not expose RPC and WS ports, if they are not correctly configured._** Let's first check the version we have. The first time you run this command, the Polkadot docker image will be downloaded. This takes a bit of time and bandwidth, be patient: ```bash -docker run --rm -it parity/polkadot:latest --version +$ENGINE run --rm -it parity/polkadot:latest --version ``` You can also pass any argument/flag that Polkadot supports: ```bash -docker run --rm -it parity/polkadot:latest --chain westend --name "PolkaDocker" +$ENGINE run --rm -it parity/polkadot:latest --chain westend --name "PolkaDocker" ``` ## Examples -Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local directory to the Polkadot user that is used by the container. Set user id 1000 and group id 1000, by running `chown 1000.1000 /my/local/folder -R` if you use a bind mount. - -To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. +Once you are done experimenting and picking the best node name :) you can start Polkadot as daemon, exposes the Polkadot ports and mount a volume that will keep your blockchain data locally. Make sure that you set the ownership of your local directory to the Polkadot user that is used by the container. -```bash -docker run -d -p 30333:30333 -p 9933:9933 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --rpc-external --rpc-cors all -``` +Set user id 1000 and group id 1000, by running `chown 1000.1000 /my/local/folder -R` if you use a bind mount. -Additionally if you want to have custom node name you can add the `--name "YourName"` at the end +To start a Polkadot node on default rpc port 9933 and default p2p port 30333 use the following command. If you want to connect to rpc port 9933, then must add Polkadot startup parameter: `--rpc-external`. ```bash -docker run -d -p 30333:30333 -p 9933:9933 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --rpc-external --rpc-cors all --name "PolkaDocker" +$ENGINE run -d -p 30333:30333 -p 9933:9933 \ + -v /my/local/folder:/polkadot \ + parity/polkadot:latest \ + --chain westend --rpc-external --rpc-cors all \ + --name "PolkaDocker ``` If you also want to expose the webservice port 9944 use the following command: ```bash -docker run -d -p 30333:30333 -p 9933:9933 -p 9944:9944 -v /my/local/folder:/polkadot parity/polkadot:latest --chain westend --ws-external --rpc-external --rpc-cors all --name "PolkaDocker" +$ENGINE run -d -p 30333:30333 -p 9933:9933 -p 9944:9944 \ + -v /my/local/folder:/polkadot \ + parity/polkadot:latest \ + --chain westend --ws-external --rpc-external --rpc-cors all --name "PolkaDocker" ``` ## Using Docker compose @@ -55,17 +70,19 @@ services: - 30333:30333 # p2p port - 9933:9933 # rpc port - 9944:9944 # ws port + - 9615:9615 # Prometheus port volumes: - /my/local/folder:/polkadot command: [ "--name", "PolkaDocker", "--ws-external", "--rpc-external", + "--prometheus-external", "--rpc-cors", "all" ] ``` -With following docker-compose.yml you can set up a node and use polkadot-js-apps as the front end on port 80. After starting the node use a browser and enter your Docker host IP in the URL field: _<http://[YOUR_DOCKER_HOST_IP>_ +With following `docker-compose.yml` you can set up a node and use polkadot-js-apps as the front end on port 80. After starting the node use a browser and enter your Docker host IP in the URL field: _<http://[YOUR_DOCKER_HOST_IP]>_ ```bash version: '2' @@ -78,10 +95,12 @@ services: - 30333:30333 # p2p port - 9933:9933 # rpc port - 9944:9944 # ws port + - 9615:9615 # Prometheus port command: [ "--name", "PolkaDocker", "--ws-external", "--rpc-external", + "--prometheus-external", "--rpc-cors", "all" ] @@ -100,27 +119,30 @@ Chain syncing will utilize all available memory and CPU power your server has to If running on a low resource VPS, use `--memory` and `--cpus` to limit the resources used. E.g. To allow a maximum of 512MB memory and 50% of 1 CPU, use `--cpus=".5" --memory="512m"`. Read more about limiting a container's resources [here](https://docs.docker.com/config/containers/resource_constraints). -Start a shell session with the daemon: -```bash -docker exec -it $(docker ps -q) bash; -``` +## Build your own image -Check the current version: +There are 3 options to build a polkadot container image: +- using the builder image +- using the injected "Debian" image +- using the generic injected image -```bash -polkadot --version -``` +### Builder image -## Build your own image +To get up and running with the smallest footprint on your system, you may use an existing Polkadot Container image. -To get up and running with the smallest footprint on your system, you may use the Polkadot Docker image. -You can build it yourself (it takes a while...) in the shell session of the daemon: +You may also build a polkadot container image yourself (it takes a while...) using the container specs `scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile`. -```bash -cd scripts/ci/dockerfiles/polkadot -./build.sh -``` +### Debian injected + +The Debian injected image is how the official polkadot container image is produced. It relies on the Debian package that is published upon each release. The Debian injected image is usually available a few minutes after a new release is published. +It has the benefit of relying on the GPG signatures embedded in the Debian package. + +### Generic injected + +For simple testing purposes, the easiest option for polkadot and also random binaries, is to use the `binary_injected.Dockerfile` container spec. This option is less secure since the injected binary is not checked at all but it has the benefit to be simple. This option requires to already have a valid `polkadot` binary, compiled for Linux. + +This binary is then simply copied inside the `parity/base-bin` image. ## Reporting issues @@ -128,8 +150,8 @@ If you run into issues with Polkadot when using docker, please run the following (replace the tag with the appropriate one if you do not use latest): ```bash -docker run --rm -it parity/polkadot:latest --version +$ENGINE run --rm -it parity/polkadot:latest --version ``` This will show you the Polkadot version as well as the git commit ref that was used to build your container. -Just paste that in the issue you create. +You can now paste the version information in a [new issue](https://github.com/paritytech/polkadot/issues/new/choose). diff --git a/polkadot/scripts/ci/common/lib.sh b/polkadot/scripts/ci/common/lib.sh index 2e94feb150ce366bbbd85dd2420daca491ed1454..00abe9a1d8d48b19694976010819c8f9b9f06fa0 100755 --- a/polkadot/scripts/ci/common/lib.sh +++ b/polkadot/scripts/ci/common/lib.sh @@ -193,3 +193,73 @@ check_bootnode(){ echo " Bootnode appears unreachable" return 1 } + +# Assumes the ENV are set: +# - RELEASE_ID +# - GITHUB_TOKEN +# - REPO in the form paritytech/polkadot +fetch_release_artifacts() { + echo "Release ID : $RELEASE_ID" + echo "Repo : $REPO" + echo "ARTIFACT_FOLDER: $ARTIFACT_FOLDER" + + curl -L -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${REPO}/releases/$RELEASE_ID > release.json + + # Get Asset ids + ids=($(jq -r '.assets[].id' < release.json )) + count=$(jq '.assets|length' < release.json ) + + # Fetch artifacts + mkdir -p ${ARTIFACT_FOLDER} + pushd ${ARTIFACT_FOLDER} > /dev/null + + iter=1 + for id in "${ids[@]}" + do + echo " - $iter/$count: downloading asset id: $id..." + curl -s -OJ -L -H "Accept: application/octet-stream" \ + -H "Authorization: Token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${REPO}/releases/assets/$id" + iter=$((iter + 1)) + done + + ls -al --color + popd > /dev/null +} + +# Check the checksum for a given binary +function check_sha256() { + echo "Checking SHA256 for $1" + shasum -qc $1.sha256 +} + +# Import GPG keys of the release team members +# This is done in parallel as it can take a while sometimes +function import_gpg_keys() { + GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"} + SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798" + WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2" + EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3" + MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF" + MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF" + + echo "Importing GPG keys from $GPG_KEYSERVER in parallel" + for key in $SEC $WILL $EGOR $MARA $MORGAN; do + ( + echo "Importing GPG key $key" + gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key + echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust; + ) & + done + wait +} + +# Check the GPG signature for a given binary +function check_gpg() { + echo "Checking GPG Signature for $1" + gpg --no-tty --verify -q $1.asc $1 +} diff --git a/polkadot/scripts/ci/dockerfiles/adder-collator/build-injected.sh b/polkadot/scripts/ci/dockerfiles/adder-collator/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..9a1857bc7ab4714d01d5cc337d5444bf85916449 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/adder-collator/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=adder-collator,undying-collator +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/polkadot/scripts/ci/dockerfiles/adder-collator/test-build.sh b/polkadot/scripts/ci/dockerfiles/adder-collator/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..171e0309f807971eaa79133f3de3b8d125787cbe --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/adder-collator/test-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# TODO: Switch to /bin/bash when the image is built from parity/base-bin + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which adder-collator)" /export' + +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which undying-collator)" /export' + +./build-injected.sh $TMP diff --git a/polkadot/scripts/ci/dockerfiles/binary_injected.Dockerfile b/polkadot/scripts/ci/dockerfiles/binary_injected.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..cee81a2eb8ae06412109d942b5ca3d322c0cdf1e --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/binary_injected.Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/parity/base-bin + +# This file allows building a Generic container image +# based on one or multiple pre-built Linux binaries. +# Some defaults are set to polkadot but all can be overriden. + +SHELL ["/bin/bash", "-c"] + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +# That can be a single one or a comma separated list +ARG BINARY=polkadot + +ARG BIN_FOLDER=. +ARG DOC_URL=https://github.com/paritytech/polkadot +ARG DESCRIPTION="Polkadot: a platform for web3" +ARG AUTHORS="devops-team@parity.io" +ARG VENDOR="Parity Technologies" + +LABEL io.parity.image.authors=${AUTHORS} \ + io.parity.image.vendor="${VENDOR}" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="${DOC_URL}" \ + io.parity.image.description="${DESCRIPTION}" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/binary_injected.Dockerfile" + +USER root +WORKDIR /app + +# add polkadot binary to docker image +# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/ +COPY entrypoint.sh . +COPY "bin/*" "/usr/local/bin/" +RUN chmod -R a+rx "/usr/local/bin" + +USER parity +ENV BINARY=${BINARY} + +# ENTRYPOINT +ENTRYPOINT ["/app/entrypoint.sh"] + +# We call the help by default +CMD ["--help"] diff --git a/polkadot/scripts/ci/dockerfiles/build-injected.sh b/polkadot/scripts/ci/dockerfiles/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..d0e7fee3646e439343767b8d5ee4645f0061412f --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/build-injected.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -e + +# This script allows building a Container Image from a Linux +# binary that is injected into a base-image. + +ENGINE=${ENGINE:-podman} + +if [ "$ENGINE" == "podman" ]; then + PODMAN_FLAGS="--format docker" +else + PODMAN_FLAGS="" +fi + +CONTEXT=$(mktemp -d) +REGISTRY=${REGISTRY:-docker.io} + +# The following line ensure we know the project root +PROJECT_ROOT=${PROJECT_ROOT:-$(git rev-parse --show-toplevel)} +DOCKERFILE=${DOCKERFILE:-$PROJECT_ROOT/scripts/ci/dockerfiles/binary_injected.Dockerfile} +VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)") + +#n The following VAR have default that can be overriden +DOCKER_OWNER=${DOCKER_OWNER:-parity} + +# We may get 1..n binaries, comma separated +BINARY=${BINARY:-polkadot} +IFS=',' read -r -a BINARIES <<< "$BINARY" + +VERSION=${VERSION:-$VERSION_TOML} +BIN_FOLDER=${BIN_FOLDER:-.} + +IMAGE=${IMAGE:-${REGISTRY}/${DOCKER_OWNER}/${BINARIES[0]}} +DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY}" +DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT} + +VCS_REF=${VCS_REF:-01234567} + +# Build the image +echo "Using engine: $ENGINE" +echo "Using Dockerfile: $DOCKERFILE" +echo "Using context: $CONTEXT" +echo "Building ${IMAGE}:latest container image for ${BINARY} v${VERSION} from ${BIN_FOLDER} hang on!" +echo "BIN_FOLDER=$BIN_FOLDER" +echo "CONTEXT=$CONTEXT" + +# We need all binaries and resources available in the Container build "CONTEXT" +mkdir -p $CONTEXT/bin +for bin in "${BINARIES[@]}" +do + echo "Copying $BIN_FOLDER/$bin to context: $CONTEXT/bin" + cp "$BIN_FOLDER/$bin" "$CONTEXT/bin" +done + +cp "$PROJECT_ROOT/scripts/ci/dockerfiles/entrypoint.sh" "$CONTEXT" + +echo "Building image: ${IMAGE}" + +TAGS=${TAGS[@]:-latest} +IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" +TAG_ARGS=" " + +echo "The image ${IMAGE} will be tagged with ${TAG_ARRAY[*]}" +for tag in "${TAG_ARRAY[@]}"; do + TAG_ARGS+="--tag ${IMAGE}:${tag} " +done + +echo "$TAG_ARGS" + +# time \ +$ENGINE build \ + ${PODMAN_FLAGS} \ + --build-arg VCS_REF="${VCS_REF}" \ + --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + --build-arg IMAGE_NAME="${IMAGE}" \ + --build-arg BINARY="${BINARY}" \ + --build-arg BIN_FOLDER="${BIN_FOLDER}" \ + --build-arg DESCRIPTION="${DESCRIPTION}" \ + ${TAG_ARGS} \ + -f "${DOCKERFILE}" \ + ${CONTEXT} + +echo "Your Container image for ${IMAGE} is ready" +$ENGINE images + +if [[ -z "${SKIP_IMAGE_VALIDATION}" ]]; then + echo "Check the image ${IMAGE}:${TAG_ARRAY[0]}" + $ENGINE run --rm -i "${IMAGE}:${TAG_ARRAY[0]}" --version + + echo "Query binaries" + $ENGINE run --rm -i --entrypoint /bin/bash "${IMAGE}:${TAG_ARRAY[0]}" -c 'echo BINARY: $BINARY' +fi diff --git a/polkadot/scripts/ci/dockerfiles/collator_injected.Dockerfile b/polkadot/scripts/ci/dockerfiles/collator_injected.Dockerfile deleted file mode 100644 index 91b8cb0057bfff17c1a02c1ae65418a8a73487a3..0000000000000000000000000000000000000000 --- a/polkadot/scripts/ci/dockerfiles/collator_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# this file copies from scripts/ci/dockerfiles/Dockerfile and changes only the binary name -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Injected adder-collator Docker image" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/collator_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/adder-collator to /data - useradd -m -u 1000 -U -s /bin/sh -d /adder-collator adder-collator && \ - mkdir -p /data /adder-collator/.local/share && \ - chown -R adder-collator:adder-collator /data && \ - ln -s /data /adder-collator/.local/share/polkadot - -# add adder-collator binary to docker image -COPY ./adder-collator /usr/local/bin -COPY ./undying-collator /usr/local/bin - -USER adder-collator - -# check if executable works in this container -RUN /usr/local/bin/adder-collator --version -RUN /usr/local/bin/undying-collator --version - -EXPOSE 30333 9933 9944 -VOLUME ["/adder-collator"] - -ENTRYPOINT ["/usr/local/bin/adder-collator"] diff --git a/polkadot/scripts/ci/dockerfiles/entrypoint.sh b/polkadot/scripts/ci/dockerfiles/entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..eaa815faf6a45307047f07768826aa44f71afac0 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Sanity check +if [ -z "$BINARY" ] +then + echo "BINARY ENV not defined, this should never be the case. Aborting..." + exit 1 +fi + +# If the user built the image with multiple binaries, +# we consider the first one to be the canonical one +# To start with another binary, the user can either: +# - use the --entrypoint option +# - pass the ENV BINARY with a single binary +IFS=',' read -r -a BINARIES <<< "$BINARY" +BIN0=${BINARIES[0]} +echo "Starting binary $BIN0" +$BIN0 $@ diff --git a/polkadot/scripts/ci/dockerfiles/malus/build-injected.sh b/polkadot/scripts/ci/dockerfiles/malus/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..99bd5fde1d5a87dec433a9662575eaf31f1ccf54 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/malus/build-injected.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=malus,polkadot-execute-worker,polkadot-prepare-worker +export BIN_FOLDER=$1 +# export TAGS=... + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/polkadot/scripts/ci/dockerfiles/malus/test-build.sh b/polkadot/scripts/ci/dockerfiles/malus/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..3114e9e2adf12c84663568f5fddf91e690621287 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/malus/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + paritypr/malus:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/polkadot/scripts/ci/dockerfiles/malus_injected.Dockerfile b/polkadot/scripts/ci/dockerfiles/malus_injected.Dockerfile deleted file mode 100644 index fa429b5f142a1deee17eba7a98a213f58e29cdf0..0000000000000000000000000000000000000000 --- a/polkadot/scripts/ci/dockerfiles/malus_injected.Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM debian:bullseye-slim - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Malus - the nemesis of polkadot" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/malus.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - ca-certificates \ - curl \ - libssl1.1 \ - tini && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user - groupadd --gid 10000 nonroot && \ - useradd --home-dir /home/nonroot \ - --create-home \ - --shell /bin/bash \ - --gid nonroot \ - --groups nonroot \ - --uid 10000 nonroot - - -# add malus binary to docker image -COPY ./malus ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin - -USER nonroot - -# check if executable works in this container -RUN /usr/local/bin/malus --version - -# Tini allows us to avoid several Docker edge cases, see https://github.com/krallin/tini. -ENTRYPOINT ["tini", "--", "/bin/bash"] diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/README.md b/polkadot/scripts/ci/dockerfiles/polkadot/README.md index 9ddf324bb29c38f61290b1b5582005859bd351dd..e331d8984c2cee5e4b79a851809e303751e3bb57 100644 --- a/polkadot/scripts/ci/dockerfiles/polkadot/README.md +++ b/polkadot/scripts/ci/dockerfiles/polkadot/README.md @@ -1,7 +1,9 @@ # Self built Docker image The Polkadot repo contains several options to build Docker images for Polkadot. + This folder contains a self-contained image that does not require a Linux pre-built binary. + Instead, building the image is possible on any host having docker installed and will build Polkadot inside Docker. That also means that no Rust toolchain is required on the host machine for the build to succeed. diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/build-injected.sh b/polkadot/scripts/ci/dockerfiles/polkadot/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..22774c7b712244a79503ae96ae87a5fffb7315fd --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/polkadot/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot,polkadot-execute-worker,polkadot-prepare-worker +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/build.sh b/polkadot/scripts/ci/dockerfiles/polkadot/build.sh deleted file mode 100755 index d00c9108bd8c35737d848639da3ac510e748b49e..0000000000000000000000000000000000000000 --- a/polkadot/scripts/ci/dockerfiles/polkadot/build.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -pushd . - -# The following line ensure we run from the project root -PROJECT_ROOT=`git rev-parse --show-toplevel` -cd $PROJECT_ROOT - -# Find the current version from Cargo.toml -VERSION=`grep "^version" ./cli/Cargo.toml | egrep -o "([0-9\.]+-?[0-9]+)"` -GITUSER=parity -GITREPO=polkadot - -# Build the image -echo "Building ${GITUSER}/${GITREPO}:latest docker image, hang on!" -time docker build \ - -f ./scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile \ - -t ${GITUSER}/${GITREPO}:latest \ - -t ${GITUSER}/${GITREPO}:v${VERSION} \ - . - -# Show the list of available images for this repo -echo "Your Docker image for $GITUSER/$GITREPO is ready" -docker images | grep ${GITREPO} - -popd diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/docker-compose.yml b/polkadot/scripts/ci/dockerfiles/polkadot/docker-compose.yml index 978191af88c19db027b032667e8f689ea642a57e..524b1164796ac3f11f4fe30ec48a98e50e23c90f 100644 --- a/polkadot/scripts/ci/dockerfiles/polkadot/docker-compose.yml +++ b/polkadot/scripts/ci/dockerfiles/polkadot/docker-compose.yml @@ -1,23 +1,22 @@ version: '3' services: polkadot: + image: parity/polkadot:latest + ports: - "127.0.0.1:30333:30333/tcp" - "127.0.0.1:9933:9933/tcp" - image: parity/polkadot:latest + - "127.0.0.1:9944:9944/tcp" + - "127.0.0.1:9615:9615/tcp" + volumes: - "polkadot-data:/data" + command: | --unsafe-rpc-external --unsafe-ws-external --rpc-cors all --prometheus-external - ports: - - "30333:30333" - - "9933:9933" - - "9944:9944" - - "9615:9615" - volumes: polkadot-data: diff --git a/polkadot/scripts/ci/dockerfiles/polkadot_Dockerfile.README.md b/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md similarity index 100% rename from polkadot/scripts/ci/dockerfiles/polkadot_Dockerfile.README.md rename to polkadot/scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile b/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile index 6e31298432f7bfb42448eafc943eafcdd2e8c764..f263c836bbfe780820b0da1a0387f1b6e72be46f 100644 --- a/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +++ b/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile @@ -7,7 +7,7 @@ COPY . /polkadot RUN cargo build --locked --release # This is the 2nd stage: a very small image where we copy the Polkadot binary." -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/parity/base-bin:latest LABEL description="Multistage Docker image for Polkadot: a platform for web3" \ io.parity.image.type="builder" \ diff --git a/polkadot/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile b/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile similarity index 95% rename from polkadot/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile rename to polkadot/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile index 74b5c7f48f884957c06d64f802e2c3f2cf57bd97..e2c72dcfe2e92411317ce86c0fb965b6246f52ea 100644 --- a/polkadot/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile +++ b/polkadot/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile @@ -11,7 +11,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/polkadot" \ io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/polkadot/scripts/ci/dockerfiles/polkadot/test-build.sh b/polkadot/scripts/ci/dockerfiles/polkadot/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..d2d904561cb559d0dbb6d62e94cefa8d01213a06 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/polkadot/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot -c \ + 'cp "$(which polkadot)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/polkadot/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile b/polkadot/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile deleted file mode 100644 index aebbbdcf1b7f5321e82c7ad662bfaf52088b2fdd..0000000000000000000000000000000000000000 --- a/polkadot/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Polkadot: a platform for web3" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/polkadot to /data - useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ - mkdir -p /data /polkadot/.local/share && \ - chown -R polkadot:polkadot /data && \ - ln -s /data /polkadot/.local/share/polkadot - -# add polkadot binary to docker image -COPY ./polkadot ./polkadot-execute-worker ./polkadot-prepare-worker /usr/local/bin - -USER polkadot - -# check if executable works in this container -RUN /usr/local/bin/polkadot --version -RUN /usr/local/bin/polkadot-execute-worker --version -RUN /usr/local/bin/polkadot-prepare-worker --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot"] - -ENTRYPOINT ["/usr/local/bin/polkadot"] diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/README.md b/polkadot/scripts/ci/dockerfiles/staking-miner/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3610e11303167b1b3b8deb4ef2a9c37a60c77a1f --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/staking-miner/README.md @@ -0,0 +1,37 @@ +# staking-miner container image + +## Build using the Builder + +``` +./build.sh +``` + +## Build the injected Image + +You first need a valid Linux binary to inject. Let's assume this binary is located in `BIN_FOLDER`. + +``` +./build-injected.sh "$BIN_FOLDER" +``` + +## Test + +Here is how to test the image. We can generate a valid seed but the staking-miner will quickly notice that our +account is not funded and "does not exist". + +You may pass any ENV supported by the binary and must provide at least a few such as `SEED` and `URI`: +``` +ENV SEED="" +ENV URI="wss://rpc.polkadot.io:443" +ENV RUST_LOG="info" +``` + +``` +export SEED=$(subkey generate -n polkadot --output-type json | jq -r .secretSeed) +podman run --rm -it \ + -e URI="wss://rpc.polkadot.io:443" \ + -e RUST_LOG="info" \ + -e SEED \ + localhost/parity/staking-miner \ + dry-run seq-phragmen +``` diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/build-injected.sh b/polkadot/scripts/ci/dockerfiles/staking-miner/build-injected.sh new file mode 100755 index 0000000000000000000000000000000000000000..536636df6a918d252af9db49bed0fed8b493d473 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/staking-miner/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=staking-miner +export BIN_FOLDER=$1 + +$PROJECT_ROOT/scripts/ci/dockerfiles/build-injected.sh diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/build.sh b/polkadot/scripts/ci/dockerfiles/staking-miner/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..67c82afcd2ce52ea002ef4df8761e471f5d9d10b --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/staking-miner/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` +ENGINE=podman + +echo "Building the staking-miner using the Builder image" +echo "PROJECT_ROOT=$PROJECT_ROOT" +$ENGINE build -t staking-miner -f staking-miner_builder.Dockerfile "$PROJECT_ROOT" diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile b/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile index a1932095fd4cab0130009ebf00953fbf09955142..0ae77f36c79d0888c29c2f8211b21fc120db6037 100644 --- a/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile +++ b/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile @@ -4,17 +4,17 @@ FROM paritytech/ci-linux:production as builder ARG VCS_REF ARG BUILD_DATE ARG IMAGE_NAME="staking-miner" -ARG PROFILE=release +ARG PROFILE=production LABEL description="This is the build stage. Here we create the binary." WORKDIR /app COPY . /app -RUN cargo build --locked --$PROFILE --package staking-miner +RUN cargo build --locked --profile $PROFILE --package staking-miner # ===== SECOND STAGE ====== -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/parity/base-bin:latest LABEL description="This is the 2nd stage: a very small image where we copy the binary." LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ @@ -28,13 +28,10 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ ARG PROFILE=release COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin -RUN useradd -u 1000 -U -s /bin/sh miner && \ - rm -rf /usr/bin /usr/sbin - # show backtraces ENV RUST_BACKTRACE 1 -USER miner +USER parity ENV SEED="" ENV URI="wss://rpc.polkadot.io" diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile b/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile deleted file mode 100644 index 4901ab4a3736ea6fa1622ae6b8bb3216e95bb1b5..0000000000000000000000000000000000000000 --- a/polkadot/scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - useradd -u 1000 -U -s /bin/sh miner - -# add binary to docker image -COPY ./staking-miner /usr/local/bin - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/polkadot/scripts/ci/dockerfiles/staking-miner/test-build.sh b/polkadot/scripts/ci/dockerfiles/staking-miner/test-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..0ce74e2df296d267bdc1f4daba701ca57acdbaa1 --- /dev/null +++ b/polkadot/scripts/ci/dockerfiles/staking-miner/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/staking-miner -c \ + 'cp "$(which staking-miner)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml index dafca393cd4f0d94e5644d85425229e05e55363d..845ac797010822167a78ebdb1fde3476b11e9d06 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/build.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/build.yml @@ -39,7 +39,6 @@ build-linux-stable: - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION - - cp -r scripts/* ./artifacts build-test-collators: stage: build @@ -64,7 +63,6 @@ build-test-collators: - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-malus: stage: build @@ -88,7 +86,6 @@ build-malus: - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-staking-miner: stage: build @@ -110,7 +107,6 @@ build-staking-miner: - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts build-rustdoc: stage: build diff --git a/polkadot/scripts/ci/gitlab/pipeline/publish.yml b/polkadot/scripts/ci/gitlab/pipeline/publish.yml index d9a0dff95767cbcef8194356244ab1c97cf20c9e..c224094125e375e527b11557b4e584a75c2ea44d 100644 --- a/polkadot/scripts/ci/gitlab/pipeline/publish.yml +++ b/polkadot/scripts/ci/gitlab/pipeline/publish.yml @@ -19,20 +19,16 @@ publish-polkadot-debug-image: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile - DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile - IMAGE_NAME: docker.io/paritypr/polkadot-debug + IMAGE_NAME: "polkadot-debug" + BINARY: "polkadot,polkadot-execute-worker,polkadot-prepare-worker" needs: - job: build-linux-stable artifacts: true after_script: + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env + - echo "PARACHAINS_IMAGE_NAME=${IMAGE}" > ./artifacts/parachains.env - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env artifacts: reports: @@ -48,20 +44,15 @@ publish-test-collators-image: - .build-push-image - .zombienet-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/collator_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/colander + IMAGE_NAME: "colander" + BINARY: "adder-collator,undying-collator" needs: - job: build-test-collators artifacts: true after_script: - - buildah logout --all + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job - - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env + - echo "COLLATOR_IMAGE_NAME=${IMAGE}" > ./artifacts/collator.env - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env artifacts: reports: @@ -76,20 +67,15 @@ publish-malus-image: - .build-push-image - .zombienet-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - # scripts/ci/dockerfiles/malus_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/malus + IMAGE_NAME: "malus" + BINARY: "malus,polkadot-execute-worker,polkadot-prepare-worker" needs: - job: build-malus artifacts: true after_script: - - buildah logout "$IMAGE_NAME" + - !reference [.build-push-image, after_script] # pass artifacts to the zombienet-tests job - - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env + - echo "MALUS_IMAGE_NAME=${IMAGE}" > ./artifacts/malus.env - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env artifacts: reports: @@ -103,13 +89,11 @@ publish-staking-miner-image: - .build-push-image - .publish-refs variables: - CI_IMAGE: ${BUILDAH_IMAGE} - # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - IMAGE_NAME: docker.io/paritytech/staking-miner - GIT_STRATEGY: none - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} + IMAGE_NAME: "staking-miner" + BINARY: "staking-miner" + DOCKER_OWNER: "paritytech" + DOCKER_USER: "${Docker_Hub_User_Parity}" + DOCKER_PASS: "${Docker_Hub_Pass_Parity}" needs: - job: build-staking-miner artifacts: true @@ -122,11 +106,11 @@ publish-polkadot-image-description: DOCKER_PASSWORD: ${Docker_Hub_Pass_Parity} DOCKERHUB_REPOSITORY: parity/polkadot SHORT_DESCRIPTION: "Polkadot Official Docker Image" - README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/dockerfiles/polkadot_Dockerfile.README.md + README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md rules: - if: $CI_COMMIT_REF_NAME == "master" changes: - - scripts/ci/dockerfiles/polkadot_Dockerfile.README.md + - scripts/ci/dockerfiles/polkadot/polkadot_Dockerfile.README.md - if: $CI_PIPELINE_SOURCE == "schedule" when: never script: diff --git a/polkadot/utils/staking-miner/README.md b/polkadot/utils/staking-miner/README.md index 4148677ee7cacb2fcda1a0fb835fef5837550443..b7f70de573b05549273b7b140a9d9b8a724c2c42 100644 --- a/polkadot/utils/staking-miner/README.md +++ b/polkadot/utils/staking-miner/README.md @@ -28,8 +28,9 @@ There are 2 options to build a staking-miner Docker image: ### Building the injected image First build the binary as documented [above](#building). -You may then inject the binary into a Docker base image from the root of the Polkadot repository: +You may then inject the binary into a Docker base image: `parity/base-bin` (running the command from the root of the Polkadot repository): ``` +TODO: UPDATE THAT docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile target/release ``` @@ -39,6 +40,7 @@ Unlike the injected image that requires a Linux pre-built binary, this option do The trade-off however is that it takes a little longer to build and this option is less ideal for CI tasks. You may build the multi-stage image the root of the Polkadot repository with: ``` +TODO: UPDATE THAT docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . ``` @@ -51,7 +53,7 @@ While it won't prevent a malicious actor to read your `SEED` if they gain access # The following line starts with an extra space on purpose: SEED=0x1234... -docker run --rm -it \ +docker run --rm -i \ --name staking-miner \ --read-only \ -e RUST_LOG=info \