From 86dcca2820a0c858715dfe56a23ddb30c7c24642 Mon Sep 17 00:00:00 2001
From: Peter Goodspeed-Niklaus <peter.r.goodspeedniklaus@gmail.com>
Date: Thu, 6 Feb 2020 11:53:01 +0100
Subject: [PATCH] BROKEN: rewrite dockerfile for improved caching
- build stage depends on an image which already has Rust present
- perform all stage-initalization steps before copying any data,
preserving maximum caching
- build all deps in a separate cached stage so changes only require
rebuilding the affected code
- build each module independently in reverse topo order so unaffected
build stages can in principle retain their cache
Unfortunately, the steps above still don't work: despite the presence
of log lines like
```
---> 104821d12748
Step 32/54 : RUN cargo build --$PROFILE -p polkadot-primitives
---> Running in a78deef9950c
Finished release [optimized] target(s) in 0.97s
Removing intermediate container a78deef9950c
```
we still end up with errors like
```
Step 37/54 : RUN cargo build --$PROFILE -p polkadot-runtime-common
---> Running in 9f65497b7c6d
Compiling polkadot-runtime-common v0.7.17 (/polkadot/runtime/common)
error[E0432]: unresolved imports `primitives::Hash`, `primitives::parachain`
--> runtime/common/src/attestations.rs:26:18
|
26 | use primitives::{Hash, parachain::{AttestedCandidate, CandidateReceipt, Id as ParaId}};
| ^^^^ ^^^^^^^^^ could not find `parachain` in `primitives`
| |
| no `Hash` in the root
```
This optimization has now consumed more dev time than it's worth. I'm
preserving it in git just in case, but it's time to simplify.
---
docker/Dockerfile | 96 ++++++++++++++++++++++++++++++++++++++---------
1 file changed, 79 insertions(+), 17 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 0a0746f27b..80ff3a413b 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,19 +1,81 @@
-FROM phusion/baseimage:0.11 as builder
+FROM rust:buster as builder
LABEL maintainer "chevdor@gmail.com"
LABEL description="This is the build stage for Polkadot. Here we create the binary."
+RUN apt update && \
+ apt upgrade -y && \
+ apt install -y cmake pkg-config libssl-dev git clang
+
ARG PROFILE=release
WORKDIR /polkadot
-COPY . /polkadot
+# initialize required build targets
+# this is a separate build step for caching purposes
+COPY scripts/init.sh scripts/init.sh
+RUN scripts/init.sh
+
+# fetch and build dependencies
+# this is a separate build step for caching purposes
+COPY Cargo.* build.rs ./
+COPY availability-store/Cargo.toml ./availability-store/
+COPY cli/Cargo.toml cli/
+COPY collator/Cargo.toml collator/
+COPY erasure-coding/Cargo.toml ./erasure-coding/
+COPY network/Cargo.toml network/
+COPY parachain/Cargo.toml parachain/
+COPY primitives/Cargo.toml ./primitives/
+COPY rpc/Cargo.toml rpc/
+COPY runtime/common/Cargo.toml runtime/common/
+COPY runtime/kusama/Cargo.toml runtime/kusama/build.rs runtime/kusama/
+COPY runtime/polkadot/Cargo.toml runtime/polkadot/build.rs runtime/polkadot/
+COPY service/Cargo.toml service/
+COPY statement-table/Cargo.toml statement-table/
+COPY test-parachains/adder/Cargo.toml test-parachains/adder/build.rs test-parachains/adder/
+COPY test-parachains/adder/collator/Cargo.toml test-parachains/adder/collator/
+COPY test-parachains/halt/Cargo.toml test-parachains/halt/build.rs test-parachains/halt/
+COPY validation/Cargo.toml validation/
+RUN mkdir src && touch src/lib.rs && \
+ mkdir availability-store/src && touch availability-store/src/lib.rs && \
+ mkdir cli/src && touch cli/src/lib.rs && \
+ mkdir collator/src && touch collator/src/lib.rs && \
+ mkdir erasure-coding/src && touch erasure-coding/src/lib.rs && \
+ mkdir network/src && touch network/src/lib.rs && \
+ mkdir parachain/src && touch parachain/src/lib.rs && \
+ mkdir primitives/src && touch primitives/src/lib.rs && \
+ mkdir rpc/src && touch rpc/src/lib.rs && \
+ mkdir runtime/common/src && touch runtime/common/src/lib.rs && \
+ mkdir runtime/kusama/src && touch runtime/kusama/src/lib.rs && \
+ mkdir runtime/polkadot/src && touch runtime/polkadot/src/lib.rs && \
+ mkdir service/src && touch service/src/lib.rs && \
+ mkdir statement-table/src && touch statement-table/src/lib.rs && \
+ mkdir test-parachains/adder/src && touch test-parachains/adder/src/lib.rs && \
+ mkdir test-parachains/adder/collator/src && touch test-parachains/adder/collator/src/lib.rs && \
+ mkdir test-parachains/halt/src && touch test-parachains/halt/src/lib.rs && \
+ mkdir validation/src && touch validation/src/lib.rs && \
+ echo 'fn main(){}' > src/main.rs && \
+ echo 'fn main(){}' > test-parachains/adder/collator/src/main.rs && \
+ SKIP_WASM_BUILD=1 cargo build --${PROFILE} && \
+ find . -name lib.rs -o -name main.rs -delete
-RUN apt-get update && \
- apt-get upgrade -y && \
- apt-get install -y cmake pkg-config libssl-dev git clang
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
- export PATH=$PATH:$HOME/.cargo/bin && \
- scripts/init.sh && \
- cargo build --$PROFILE
+# now build the actual project
+COPY . .
+# we build each module independently, in reverse topographic order, for maximum caching
+RUN cargo build --$PROFILE -p polkadot-parachain
+RUN cargo build --$PROFILE -p halt
+RUN cargo build --$PROFILE -p adder
+RUN cargo build --$PROFILE -p polkadot-primitives
+RUN cargo build --$PROFILE -p polkadot-statement-table
+RUN cargo build --$PROFILE -p polkadot-erasure-coding
+RUN cargo build --$PROFILE -p polkadot-availability-store
+RUN cargo build --$PROFILE -p polkadot-validation
+RUN cargo build --$PROFILE -p polkadot-runtime-common
+RUN cargo build --$PROFILE -p polkadot-runtime
+RUN cargo build --$PROFILE -p polkadot-rpc
+RUN cargo build --$PROFILE -p polkadot-network
+RUN cargo build --$PROFILE -p kusama-runtime
+RUN cargo build --$PROFILE -p polkadot-service
+RUN cargo build --$PROFILE -p polkadot-cli
+RUN cargo build --$PROFILE -p polkadot
# ===== SECOND STAGE ======
@@ -24,14 +86,14 @@ ARG PROFILE=release
COPY --from=builder /polkadot/target/$PROFILE/polkadot /usr/local/bin
RUN mv /usr/share/ca* /tmp && \
- rm -rf /usr/share/* && \
- mv /tmp/ca-certificates /usr/share/ && \
- rm -rf /usr/lib/python* && \
- useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \
- mkdir -p /polkadot/.local/share/polkadot && \
- chown -R polkadot:polkadot /polkadot/.local && \
- ln -s /polkadot/.local/share/polkadot /data && \
- rm -rf /usr/bin /usr/sbin
+ rm -rf /usr/share/* && \
+ mv /tmp/ca-certificates /usr/share/ && \
+ rm -rf /usr/lib/python* && \
+ useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \
+ mkdir -p /polkadot/.local/share/polkadot && \
+ chown -R polkadot:polkadot /polkadot/.local && \
+ ln -s /polkadot/.local/share/polkadot /data && \
+ rm -rf /usr/bin /usr/sbin
USER polkadot
EXPOSE 30333 9933 9944
--
GitLab