docker: enable sccache (redis) in the Rust image builds (JEF-84)

Dockerfile

@@ -1,6 +1,6 @@
 # Builder must match the runtime's glibc. debian:bookworm-slim ships glibc 2.36;
-# the -bookworm cargo-chef tag is built on bookworm so the two stay in sync and
-# the dynamically-linked binary loads on the slim runtime.
+# the rust:1-bookworm builder is built on bookworm so the two stay in sync and the
+# dynamically-linked binary loads on the slim runtime.
 # Pulled via mirror.gcr.io (Google's Docker Hub pull-through cache) — the homelab
 # buildkit's shared IP exhausts Docker Hub's anonymous quota → 429 (JEF-78).
 # Node stage (ADR-0025): build the Preact dashboard bundle from source. The Rust builder
@@ -19,50 +19,71 @@ RUN npm ci --ignore-scripts
 COPY engine/web/ ./
 RUN npm run build
 
-FROM mirror.gcr.io/lukemathwalker/cargo-chef:latest-rust-1-bookworm AS chef
+FROM mirror.gcr.io/library/rust:1-bookworm AS builder
 WORKDIR /app
-
-FROM chef AS planner
-COPY . .
-RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \
-    --mount=type=cache,target=/usr/local/cargo/git/db \
-    --mount=type=cache,target=/usr/local/cargo/registry \
-    cargo chef prepare --recipe-path recipe.json
-
-FROM chef AS builder
 # sigstore-rs pulls aws-lc-sys (the rustls crypto provider via rustls-webpki),
 # which is a C library built with cmake — not present in the base image.
 RUN apt-get update \
     && apt-get install -y --no-install-recommends cmake \
     && rm -rf /var/lib/apt/lists/*
 # Pin the C standard for aws-lc-sys' C build. A C23-defaulting gcc (which newer
-# cargo-chef base images now ship) rewrites sscanf/strtol to the glibc-2.38
-# `__isoc23_*` variants; linked against the bookworm-slim runtime's glibc 2.36 those
-# are undefined references and the release link fails ("undefined reference to
+# base images now ship) rewrites sscanf/strtol to the glibc-2.38 `__isoc23_*`
+# variants; linked against the bookworm-slim runtime's glibc 2.36 those are
+# undefined references and the release link fails ("undefined reference to
 # __isoc23_sscanf"). gnu17 keeps the emitted libc symbols in step with the runtime
 # glibc. (Changing CFLAGS also reruns aws-lc-sys' build script, rebuilding a stale,
 # toolchain-mismatched object left in the build cache.)
 ENV CFLAGS=-std=gnu17
-COPY --from=planner /app/recipe.json recipe.json
-# Build dependencies - this is the caching Docker layer!
-RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \
-    --mount=type=cache,target=/usr/local/cargo/git/db \
-    --mount=type=cache,target=/usr/local/cargo/registry \
-    cargo chef cook --release --recipe-path recipe.json
+# sccache (JEF-84) is the dep-caching layer now — it shares the rustc object cache with the
+# in-cluster Redis (cluster repo charts/sccache), reached via the meshed BuildKit's own identity,
+# so a workspace dep compiled by ANY repo's image build (or the CI test build) is reused here.
+# cargo-chef was REMOVED: sccache + cargo-chef's `cook` fight over the shared /app/target dir and
+# abort with "Failed to open file for hashing: …/lib*.rmeta" (JEF-389) — a conflict that is
+# backend-independent (it fails on the local fallback too). A single plain `cargo build` compiles
+# deps then workspace crates in dependency order, so every `--extern` .rmeta exists when sccache
+# hashes it. sccache is a HARD GATE here — if it can't start against Redis the build FAILS (no
+# fallback). The earlier arm64 failures were a sccache PORT COLLISION, not a network problem: the
+# build sandboxes share a netns, so concurrent matrix builds clashed on sccache's fixed default
+# port 4226 ("Address in use"). The RUN below sets a unique random SCCACHE_SERVER_PORT to avoid
+# that. CARGO_INCREMENTAL=0 is required (sccache can't cache incremental); the 10s guard only
+# bounds a genuinely-down Redis (a normal start is ~0.4s). The musl sccache binary is static, so
+# it runs on this glibc base (mozilla/sccache ships no x86_64/aarch64 linux-gnu build).
+RUN set -eux; ver=0.16.0; \
+    case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \
+    wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \
+      | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache"
+ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \
+    SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379
+# Opt-out for builds that CANNOT reach the in-cluster redis. The github-hosted e2e
+# (scripts/e2e.sh, runs-on: ubuntu-latest) does a plain `docker build` of this Dockerfile and can
+# never reach sccache-redis.dev, so the hard gate would always trip there. Default empty => sccache
+# stays a HARD GATE for the real deploy build on the meshed BuildKit; the e2e passes
+# `--build-arg SCCACHE_DISABLE=1` to build plain (uncached) instead of failing.
+ARG SCCACHE_DISABLE=""
 
 # Build application
 COPY . .
 # The Preact bundle is gitignored (built, never committed), so `COPY . .` doesn't carry
 # it — copy it from the node stage so `include_str!("../../../web/dist/dashboard.js")`
 # resolves during `cargo build` (ADR-0025).
 COPY --from=web /web/dist/dashboard.js engine/web/dist/dashboard.js
+# BuildKit cache mounts persist the cargo registry/git + the compiled target dir across builds on
+# the shared in-cluster BuildKit daemon; sccache adds the cross-repo/cross-daemon object cache on
+# top. sharing=locked serialises concurrent matrix builds through one target dir (cargo can't share
+# it concurrently); the ephemeral target mount means the binary is cp'd out to /app.
 RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \
     --mount=type=cache,target=/usr/local/cargo/git/db \
     --mount=type=cache,target=/usr/local/cargo/registry \
-    cargo build --release
-
-RUN --mount=type=cache,target=/app/target,id=protector-target-v2,sharing=locked \
-    cp /app/target/release/protector ./protector
+    set -e; \
+    if [ -n "$SCCACHE_DISABLE" ]; then \
+      echo "sccache disabled (SCCACHE_DISABLE set) — plain build, no redis"; unset RUSTC_WRAPPER; \
+    else \
+      export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \
+      timeout 10 sccache --start-server; \
+    fi; \
+    cargo build --release; \
+    cp /app/target/release/protector ./protector; \
+    (sccache --show-stats 2>/dev/null || true)
 
 # Slim runtime. Fixed-UID non-root user (65532) matches the chart's
 # securityContext.runAsUser/runAsGroup so the pod satisfies runAsNonRoot.

agent/Dockerfile

@@ -20,6 +20,20 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends cmake curl ca-certificates zstd \
     && rm -rf /var/lib/apt/lists/*
 ENV CFLAGS=-std=gnu17
+# sccache (JEF-84) is the dep-caching layer — it shares the rustc object cache with the in-cluster
+# Redis (cluster repo charts/sccache), reached via the meshed BuildKit's own identity, so a
+# workspace dep compiled by ANY repo's image build (or the CI test build) is reused here. sccache
+# is a HARD GATE — if it can't start against Redis the build FAILS (no fallback). The build
+# sandboxes share a netns, so concurrent matrix builds clash on sccache's fixed default port 4226
+# ("Address in use"); the build RUN below sets a unique random SCCACHE_SERVER_PORT to avoid that.
+# CARGO_INCREMENTAL=0 is required (sccache can't cache incremental). The musl sccache binary is
+# static, so it runs on this glibc base (mozilla/sccache ships no x86_64/aarch64 linux-gnu build).
+RUN set -eux; ver=0.16.0; \
+    case "$(uname -m)" in x86_64) a=x86_64 ;; aarch64) a=aarch64 ;; *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;; esac; \
+    wget -qO- "https://github.com/mozilla/sccache/releases/download/v${ver}/sccache-v${ver}-${a}-unknown-linux-musl.tar.gz" \
+      | tar -xz -C /usr/local/bin --strip-components=1 "sccache-v${ver}-${a}-unknown-linux-musl/sccache"
+ENV RUSTC_WRAPPER=sccache CARGO_INCREMENTAL=0 \
+    SCCACHE_REDIS=redis://sccache-redis.dev.svc.cluster.local:6379
 # Prebuilt bpf-linker (sha256-pinned, per-arch) + nightly/rust-src for the bpf target's
 # build-std. Prebuilt so the image build stays light (no LLVM source build).
 # ARCH-AWARE: with native multi-arch builders (JEF-295/#161) this builder stage runs on
@@ -53,8 +67,12 @@ RUN --mount=type=cache,target=/app/agent/target,sharing=locked \
     --mount=type=cache,target=/app/agent/protector-agent-ebpf/target,sharing=locked \
     --mount=type=cache,target=/usr/local/cargo/git/db \
     --mount=type=cache,target=/usr/local/cargo/registry \
-    cargo build --release -p protector-agent --features ebpf \
-    && cp target/release/protector-agent /app/protector-agent-bin
+    set -e; \
+    export SCCACHE_SERVER_PORT=$(awk 'BEGIN{srand(); print int(20000+rand()*40000)}'); \
+    timeout 10 sccache --start-server; \
+    cargo build --release -p protector-agent --features ebpf; \
+    cp target/release/protector-agent /app/protector-agent-bin; \
+    sccache --show-stats
 
 # Slim runtime. The image runs as non-root; the eBPF capabilities it needs are granted
 # at RUNTIME by the DaemonSet's securityContext (chart), not baked in.

scripts/e2e.sh

@@ -403,7 +403,7 @@ done
 kubectl config use-context "k3d-$CLUSTER" >/dev/null
 
 step "2/11  Build + import the protector image"
-docker build -t "$IMAGE" "$(dirname "$0")/.."
+docker build --build-arg SCCACHE_DISABLE=1 -t "$IMAGE" "$(dirname "$0")/.."
 k3d image import "$IMAGE" -c "$CLUSTER"
 
 step "3/11  Install cert-manager ($CM_VERSION) — the pod won't start without its serving cert"