diff --git a/Dockerfile b/Dockerfile index 9e8ce06..7a4db4c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Builder must match the runtime's glibc. debian:bookworm-slim ships glibc 2.36; -# the -bookworm cargo-chef tag is built on bookworm so the two stay in sync and -# the dynamically-linked binary loads on the slim runtime. +# the rust:1-bookworm builder is built on bookworm so the two stay in sync and the +# dynamically-linked binary loads on the slim runtime. # Pulled via mirror.gcr.io (Google's Docker Hub pull-through cache) — the homelab # buildkit's shared IP exhausts Docker Hub's anonymous quota → 429 (JEF-78). # Node stage (ADR-0025): build the Preact dashboard bundle from source. The Rust builder @@ -19,36 +19,47 @@ RUN npm ci --ignore-scripts COPY engine/web/ ./ RUN npm run build -FROM mirror.gcr.io/lukemathwalker/cargo-chef:latest-rust-1-bookworm AS chef +FROM mirror.gcr.io/library/rust:1-bookworm AS builder WORKDIR /app - -FROM chef AS planner -COPY . . -RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/usr/local/cargo/registry \ - cargo chef prepare --recipe-path recipe.json - -FROM chef AS builder # sigstore-rs pulls aws-lc-sys (the rustls crypto provider via rustls-webpki), # which is a C library built with cmake — not present in the base image. RUN apt-get update \ && apt-get install -y --no-install-recommends cmake \ && rm -rf /var/lib/apt/lists/* # Pin the C standard for aws-lc-sys' C build. A C23-defaulting gcc (which newer -# cargo-chef base images now ship) rewrites sscanf/strtol to the glibc-2.38 -# `__isoc23_*` variants; linked against the bookworm-slim runtime's glibc 2.36 those -# are undefined references and the release link fails ("undefined reference to +# base images now ship) rewrites sscanf/strtol to the glibc-2.38 `__isoc23_*` +# variants; linked against the bookworm-slim runtime's glibc 2.36 those are +# undefined references and the release link fails ("undefined reference to # __isoc23_sscanf"). gnu17 keeps the emitted libc symbols in step with the runtime # glibc. (Changing CFLAGS also reruns aws-lc-sys' build script, rebuilding a stale, # toolchain-mismatched object left in the build cache.) ENV CFLAGS=-std=gnu17 -COPY --from=planner /app/recipe.json recipe.json -# Build dependencies - this is the caching Docker layer! -RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ - --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/usr/local/cargo/registry \ - cargo chef cook --release --recipe-path recipe.json +# sccache (JEF-84) is the dep-caching layer now — it shares the rustc object cache with the +# in-cluster Redis (cluster repo charts/sccache), reached via the meshed BuildKit's own identity, +# so a workspace dep compiled by ANY repo's image build (or the CI test build) is reused here. +# cargo-chef was REMOVED: sccache + cargo-chef's `cook` fight over the shared /app/target dir and +# abort with "Failed to open file for hashing: …/lib*.rmeta" (JEF-389) — a conflict that is +# backend-independent (it fails on the local fallback too). A single plain `cargo build` compiles +# deps then workspace crates in dependency order, so every `--extern` .rmeta exists when sccache +# hashes it. sccache is a HARD GATE here — if it can't start against Redis the build FAILS (no +# fallback). The earlier arm64 failures were a sccache PORT COLLISION, not a network problem: the +# build sandboxes share a netns, so concurrent matrix builds clashed on sccache's fixed default +# port 4226 ("Address in use"). The RUN below sets a unique random SCCACHE_SERVER_PORT to avoid +# that. CARGO_INCREMENTAL=0 is required (sccache can't cache incremental); the 10s guard only +# bounds a genuinely-down Redis (a normal start is ~0.4s). The musl sccache binary is static, so +# it runs on this glibc base (mozilla/sccache ships no x86_64/aarch64 linux-gnu build). +RUN set -eux; ver=0.16.0; \ + case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ + wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ + | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ + SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 +# Opt-out for builds that CANNOT reach the in-cluster redis. The github-hosted e2e +# (scripts/e2e.sh, runs-on: ubuntu-latest) does a plain `docker build` of this Dockerfile and can +# never reach sccache-redis.dev, so the hard gate would always trip there. Default empty => sccache +# stays a HARD GATE for the real deploy build on the meshed BuildKit; the e2e passes +# `--build-arg SCCACHE_DISABLE=1` to build plain (uncached) instead of failing. +ARG SCCACHE_DISABLE="" # Build application COPY . . @@ -56,13 +67,23 @@ COPY . . # it — copy it from the node stage so `include_str!("../../../web/dist/dashboard.js")` # resolves during `cargo build` (ADR-0025). COPY --from=web /web/dist/dashboard.js engine/web/dist/dashboard.js +# BuildKit cache mounts persist the cargo registry/git + the compiled target dir across builds on +# the shared in-cluster BuildKit daemon; sccache adds the cross-repo/cross-daemon object cache on +# top. sharing=locked serialises concurrent matrix builds through one target dir (cargo can't share +# it concurrently); the ephemeral target mount means the binary is cp'd out to /app. RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ - cargo build --release - -RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \ - cp /app/target/release/protector ./protector + set -e; \ + if [ -n "$SCCACHE_DISABLE" ]; then \ + echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no redis"; unset RUSTC_WRAPPER; \ + else \ + export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ + timeout 10 sccache --start-server; \ + fi; \ + cargo build --release; \ + cp /app/target/release/protector ./protector; \ + (sccache --show-stats 2>/dev/null || true) # Slim runtime. Fixed-UID non-root user (65532) matches the chart's # securityContext.runAsUser/runAsGroup so the pod satisfies runAsNonRoot. diff --git a/agent/Dockerfile b/agent/Dockerfile index 4ec4f17..469ef77 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -20,6 +20,20 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends cmake curl ca-certificates zstd \ && rm -rf /var/lib/apt/lists/* ENV CFLAGS=-std=gnu17 +# sccache (JEF-84) is the dep-caching layer — it shares the rustc object cache with the in-cluster +# Redis (cluster repo charts/sccache), reached via the meshed BuildKit's own identity, so a +# workspace dep compiled by ANY repo's image build (or the CI test build) is reused here. sccache +# is a HARD GATE — if it can't start against Redis the build FAILS (no fallback). The build +# sandboxes share a netns, so concurrent matrix builds clash on sccache's fixed default port 4226 +# ("Address in use"); the build RUN below sets a unique random SCCACHE_SERVER_PORT to avoid that. +# CARGO_INCREMENTAL=0 is required (sccache can't cache incremental). The musl sccache binary is +# static, so it runs on this glibc base (mozilla/sccache ships no x86_64/aarch64 linux-gnu build). +RUN set -eux; ver=0.16.0; \ + case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \ + wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \ + | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache" +ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \ + SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379 # Prebuilt bpf-linker (sha256-pinned, per-arch) + nightly/rust-src for the bpf target's # build-std. Prebuilt so the image build stays light (no LLVM source build). # ARCH-AWARE: with native multi-arch builders (JEF-295/#161) this builder stage runs on @@ -53,8 +67,12 @@ RUN --mount=type=cache,target=/app/agent/target,sharing=locked \ --mount=type=cache,target=/app/agent/protector-agent-ebpf/target,sharing=locked \ --mount=type=cache,target=/usr/local/cargo/git/db \ --mount=type=cache,target=/usr/local/cargo/registry \ - cargo build --release -p protector-agent --features ebpf \ - && cp target/release/protector-agent /app/protector-agent-bin + set -e; \ + export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \ + timeout 10 sccache --start-server; \ + cargo build --release -p protector-agent --features ebpf; \ + cp target/release/protector-agent /app/protector-agent-bin; \ + sccache --show-stats # Slim runtime. The image runs as non-root; the eBPF capabilities it needs are granted # at RUNTIME by the DaemonSet's securityContext (chart), not baked in. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 420626d..a41e200 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -403,7 +403,7 @@ done kubectl config use-context "k3d-$CLUSTER" >/dev/null step "2/11 Build + import the protector image" -docker build -t "$IMAGE" "$(dirname "$0")/.." +docker build --build-arg SCCACHE_DISABLE=1 -t "$IMAGE" "$(dirname "$0")/.." k3d image import "$IMAGE" -c "$CLUSTER" step "3/11 Install cert-manager ($CM_VERSION) — the pod won't start without its serving cert"