From 25da38c480b668a42c7951b792f522efd0c81968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:33:07 -0300 Subject: [PATCH 01/29] docs: add early aggregation start design spec --- ...26-07-01-early-aggregation-start-design.md | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md new file mode 100644 index 00000000..58dc8e90 --- /dev/null +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -0,0 +1,167 @@ +# Early Aggregation Start + +**Date:** 2026-07-01 +**Status:** Approved + +## Motivation + +Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the +attestation pipeline. Today the aggregation session starts exactly at the +interval-2 tick, even when all expected signatures arrived earlier. Starting the +session up to 400 ms early, when enough signatures are already present, gives the +expensive proofs more wall-clock runway before block building consumes them at +interval 4. + +## Current behavior + +- The blockchain actor ticks at 800 ms interval boundaries. At interval 2 (if + `is_aggregator`), `start_aggregation_session` snapshots the current slot's + gossip signatures (`snapshot_current_slot_aggregation_inputs`) and spawns an + off-thread worker. A deadline timer cancels the session's token after + `AGGREGATION_DEADLINE` (750 ms). +- Gossip signatures accumulate in the store per attestation-data group as + `NewAttestation` messages arrive (plus self-delivery of own validators' + attestations at interval 1). Only aggregators store them. +- Each produced aggregate is applied to the store and published to gossip + immediately in the `AggregateProduced` handler. +- Subnet assignment is `vid % attestation_committee_count`. The set of subnets a + node subscribes to (own-validator subnets, plus `--aggregate-subnet-ids` for + aggregators, with a fallback to subnet 0 for validator-less aggregators) is + computed inside `build_swarm` in the p2p crate; the blockchain actor does not + know it. + +## New behavior + +### Early trigger + +Aggregator-only, fires at most once per slot. + +- **Window:** wall clock in `[T2 - 400 ms, T2)`, where + `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` + is the current slot's interval-2 boundary. +- **Condition:** some current-slot attestation-data group has gossip signatures + from at least 2/3 of the expected validators: + `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the + number of stored gossip signatures in that group (one per validator). + `expected` = number of head-state validators whose + `vid % attestation_committee_count` is in this node's aggregation subnet set. + Note that at most one group per slot can reach 2/3 (each validator signs once + per slot), so the per-group condition can hold for at most one group. +- **Check sites:** + 1. After each gossip-signature insert in the `NewAttestation` handler, while + the wall clock is inside the window. + 2. A one-shot `EarlyAggregationCheck { slot }` self-message scheduled at the + interval-1 tick via `send_after(400 ms)` (interval-1 tick + 400 ms = + T2 - 400 ms), covering the case where the threshold was already met before + the window opened. If the interval-1 tick is skipped (overrun), the + per-insert checks still apply. +- **Once-per-slot guard:** a session with `session_id == slot` already exists in + `current_aggregation` (the field persists after the worker finishes; it is + only replaced at the next session start). + +### Action on trigger + +Call the existing `start_aggregation_session(slot, ctx)` — the full current-slot +snapshot, not just the group that hit the threshold. This is the slot's one and +only session; it merely starts early. The existing prior-session cancel+join in +`start_aggregation_session` handles a still-running previous-slot worker +identically in both paths. + +### Interval-2 tick + +If `current_aggregation` already holds this slot's session (running or +finished), skip starting a new one. Otherwise start normally — the unchanged +fallback for slots where the threshold is never met. + +`emit_agg_start_new_coverage` moves from the interval-2 tick into +`start_aggregation_session`, so the coverage report reflects the store state the +session actually snapshotted in both paths. + +### Publish alignment + +Aggregates must not reach gossip before interval 2 (mirrors the block prebuild +pattern: build early, publish aligned to the boundary). + +- `AggregateProduced` handler: apply the output to the store immediately + (nothing local consumes `new_aggregated_payloads` before interval 4, so early + application is unobservable locally). If the wall clock is `>= T2` for the + session's slot, publish immediately (unchanged). If `< T2`, push the + `SignedAggregatedAttestation` into a pending-publish buffer on the actor. +- When starting an *early* session, schedule a `FlushAggregatePublishes` + self-message via `send_after(T2 - now)`. Its handler publishes and drains the + buffer. This is the primary flush mechanism (not the interval-2 tick, which + can be skipped on overrun); the handler is idempotent (draining an empty + buffer is a no-op). +- The buffer is bounded in practice by the number of data groups in one slot. + Entries are never carried across slots: the flush timer fires at T2 of the + same slot that buffered them. + +### Deadline change + +`AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. + +- Early session started at `T2 - x` (x ≤ 400): deadline at `T2 - x + 800`, + i.e. at most `T2 + 400`. +- Normal session started at `T2`: deadline lands exactly on the interval-3 + boundary. This removes the previous 50 ms publish/propagation margin — + accepted trade-off (the deadline only stops *new* jobs from starting; a job + mid-proof finishes and publishes slightly after). + +### Subnet-set plumbing + +The subscription-subnet computation is hoisted out of `build_swarm` into a +shared pure helper (in the p2p crate, e.g. +`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`). +`main.rs` calls it once and passes the resulting set to both: + +- the p2p config (which subscribes to exactly this set, behavior unchanged), and +- the `BlockChainServer` (new field, used for the `expected` count). + +The set is frozen at startup, matching the existing hot-standby aggregator +model (runtime toggles do not resubscribe subnets). + +## Failure modes and edge cases + +- **Snapshot returns `None` after trigger** (should not happen — the trigger + requires signatures to be present): no session is created, + `current_aggregation` keeps the prior slot's id, and interval 2 retries + normally. No signatures are lost. +- **Threshold never met:** interval 2 starts the session exactly as today. +- **Signatures arriving after the early snapshot:** never aggregated that slot. + Accepted trade-off (explicitly chosen over a top-up session). +- **Wall-clock vs monotonic drift:** checks use `unix_now_ms()` like the rest of + the tick logic; the idempotency-guard pattern already tolerates drift. The + worst case of drift is a slightly mistimed early start or flush, never a + correctness issue. +- **Non-aggregators:** never store gossip signatures, and all check sites are + additionally guarded by `AggregatorController::is_enabled()` read at check + time. + +## Observability + +- Counter `lean_aggregation_early_starts_total`: early sessions started. +- Histogram for early-start lead time (`T2 - start`, seconds, buckets covering + 0–400 ms). +- Existing session logs gain an `early: bool` field so devnet log analysis can + attribute publish-time shifts. + +## Testing + +- Unit tests for the pure trigger decision (window bounds, threshold math, + `expected` derivation from subnet set + validator count, once-per-slot guard). +- Unit tests for `compute_subscription_subnets` (validator subnets, explicit + ids, fallback-to-0, non-aggregator). +- Unit test for the publish-delay decision (before/after T2). +- Existing aggregation tests must pass unchanged (normal interval-2 path is the + fallback and keeps its semantics; only the deadline constant changes). +- Devnet validation via `test-branch.sh`: compare aggregate publish offsets + (per-slot duty timing method) and `lean_aggregation_early_starts_total`. + +## Non-goals + +- No gossip resubscription or topic changes. +- No change to the fork-choice attestation pipeline (`new_attestations` / + `known_attestations`). +- No top-up/second session per slot. +- No persistence of the pending-publish buffer (in-memory actor state, same as + the rest of the aggregation pipeline). From 5e4cfbf6b61d9926509f3ce1723835456e90ec92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:42:29 -0300 Subject: [PATCH 02/29] docs: add early aggregation start implementation plan --- .../2026-07-01-early-aggregation-start.md | 932 ++++++++++++++++++ ...26-07-01-early-aggregation-start-design.md | 21 +- 2 files changed, 945 insertions(+), 8 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-01-early-aggregation-start.md diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md new file mode 100644 index 00000000..69d193f9 --- /dev/null +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -0,0 +1,932 @@ +# Early Aggregation Start Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Start the committee-signature aggregation session up to 400 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. + +**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 400 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. + +**Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. + +**Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` + +**Key existing code:** +- `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) +- `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` +- `crates/blockchain/src/metrics.rs` — prometheus registration patterns +- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451), test mod (~line 1555) with `make_att_data(slot)`, `make_att_data_for_target(slot, root)`, `make_dummy_sig()` helpers +- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337), test mod (~line 790) +- `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) + +**Timing model (4 s slots, 5 × 800 ms intervals):** + +``` +slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms + |------------|------------|------------|------------| + ^ attest ^ aggregate ^ safe tgt ^ accept/build + [T2-400, T2) = early window + ^ timer check (scheduled at T1 + 400ms) + ^..^..^ per-insert checks as gossip sigs arrive +``` + +--- + +### Task 1: Store query — max gossip group count for a slot + +The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). + +**Files:** +- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451; test in existing `mod tests`) + +- [ ] **Step 1: Write the failing test** + +Add to the `// ============ GossipSignatureBuffer Tests ============` section of `mod tests` in `crates/storage/src/store.rs` (near `gossip_buffer_fifo_eviction`, ~line 2423): + +```rust +#[test] +fn gossip_buffer_max_group_count_for_slot() { + let mut buf = GossipSignatureBuffer::new(100); + assert_eq!(buf.max_group_count_for_slot(1), 0); + + // Slot 1, group A: 3 validators. + for vid in 0..3 { + let data = make_att_data(1); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + // Slot 1, group B (same slot, different target root): 2 validators. + for vid in 3..5 { + let data = make_att_data_for_target(1, root(99)); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + // Slot 2: 5 validators. + for vid in 0..5 { + let data = make_att_data(2); + buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); + } + + assert_eq!(buf.max_group_count_for_slot(1), 3); + assert_eq!(buf.max_group_count_for_slot(2), 5); + assert_eq!(buf.max_group_count_for_slot(3), 0); +} +``` + +Note: `make_att_data_for_target` is defined further down in the test mod (~line 2319); Rust test mods don't care about definition order. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ethlambda-storage max_group_count_for_slot` +Expected: FAIL to compile with "no method named `max_group_count_for_slot`" + +- [ ] **Step 3: Implement the buffer method and Store wrapper** + +In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): + +```rust +/// Largest signature count among data groups whose attestation slot is `slot`. +fn max_group_count_for_slot(&self, slot: u64) -> usize { + self.data + .values() + .filter(|entry| entry.data.slot == slot) + .map(|entry| entry.signatures.len()) + .max() + .unwrap_or(0) +} +``` + +In `impl Store`, next to `gossip_signatures_count()` (~line 1451): + +```rust +/// Largest per-group signature count among gossip groups voting for `slot`. +/// +/// One lock, no signature clones — cheap enough to call per gossip insert. +/// Drives the early-aggregation threshold check. +pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { + let gossip = self.gossip_signatures.lock().unwrap(); + gossip.max_group_count_for_slot(slot) +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cargo test -p ethlambda-storage max_group_count_for_slot` +Expected: PASS (1 test) + +- [ ] **Step 5: Commit** + +```bash +git add crates/storage/src/store.rs +git commit -m "feat(storage): add max gossip group count query for early aggregation" +``` + +--- + +### Task 2: p2p — extract `compute_subscription_subnets` helper + +Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. + +**Files:** +- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337; tests in existing `mod tests` ~line 790) + +- [ ] **Step 1: Write the failing tests** + +Add to `mod tests` in `crates/net/p2p/src/lib.rs` (if the test mod doesn't already have `HashSet` in scope via `use super::*;`, add `use std::collections::HashSet;`): + +```rust +#[test] +fn subscription_subnets_validators_only() { + // 3 % 4 = 3, 7 % 4 = 3, 10 % 4 = 2 + let subnets = compute_subscription_subnets(&[3, 7, 10], 4, false, None); + assert_eq!(subnets, HashSet::from([3, 2])); +} + +#[test] +fn subscription_subnets_aggregator_unions_explicit_ids() { + let subnets = compute_subscription_subnets(&[1], 4, true, Some(&[0, 2])); + assert_eq!(subnets, HashSet::from([1, 0, 2])); +} + +#[test] +fn subscription_subnets_validatorless_aggregator_falls_back_to_zero() { + let subnets = compute_subscription_subnets(&[], 4, true, None); + assert_eq!(subnets, HashSet::from([0])); +} + +#[test] +fn subscription_subnets_validatorless_non_aggregator_is_empty() { + let subnets = compute_subscription_subnets(&[], 4, false, None); + assert!(subnets.is_empty()); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ethlambda-p2p subscription_subnets` +Expected: FAIL to compile with "cannot find function `compute_subscription_subnets`" + +- [ ] **Step 3: Implement the helper and use it in `build_swarm`** + +Add near `build_swarm` (above it, ~line 185): + +```rust +/// Compute the set of attestation subnets this node subscribes to (and, when +/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): +/// every validator subscribes to its own subnet (`vid % committee_count`) for +/// mesh health; aggregators additionally subscribe to explicit +/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would +/// otherwise be empty. +/// +/// Evaluated once at startup — runtime aggregator toggles do not resubscribe +/// (hot-standby model); see the invariant note on [`SwarmConfig`]. +pub fn compute_subscription_subnets( + validator_ids: &[u64], + attestation_committee_count: u64, + is_aggregator: bool, + aggregate_subnet_ids: Option<&[u64]>, +) -> HashSet { + let mut subnets: HashSet = validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect(); + if is_aggregator { + if let Some(explicit_ids) = aggregate_subnet_ids { + subnets.extend(explicit_ids); + } + if subnets.is_empty() { + subnets.insert(0); + } + } + subnets +} +``` + +Then replace the inline logic in `build_swarm` (~lines 304–329). The metric must keep reflecting validator-only subnets, so that part stays: + +```rust + // Subscribe to attestation subnets — see `compute_subscription_subnets`. + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = config + .validator_ids + .iter() + .map(|vid| vid % config.attestation_committee_count) + .min() + .unwrap_or(0); + metrics::set_attestation_committee_subnet(metric_subnet); + + let subscription_subnets = compute_subscription_subnets( + &config.validator_ids, + config.attestation_committee_count, + config.is_aggregator, + config.aggregate_subnet_ids.as_deref(), + ); +``` + +(The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ethlambda-p2p subscription_subnets` +Expected: PASS (4 tests) + +Run: `cargo build -p ethlambda-p2p` +Expected: clean build (no unused warnings; the old inline block is fully replaced) + +- [ ] **Step 5: Commit** + +```bash +git add crates/net/p2p/src/lib.rs +git commit -m "refactor(p2p): extract subscription subnet computation into pure helper" +``` + +--- + +### Task 3: Blockchain — pure early-aggregation helpers + +Window math and threshold predicate, unit-testable without an actor. + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37; new `#[cfg(test)] mod tests` at end of file) + +- [ ] **Step 1: Write the failing tests** + +`crates/blockchain/src/aggregation.rs` has no test mod yet. Add at the end of the file: + +```rust +#[cfg(test)] +mod tests { + use super::*; + use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT}; + + const GENESIS_MS: u64 = 1_000_000; + + /// Interval-2 boundary of `slot`, relative to the test genesis. + fn t2(slot: u64) -> u64 { + GENESIS_MS + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL + } + + #[test] + fn interval2_boundary_matches_slot_layout() { + assert_eq!(interval2_boundary_ms(GENESIS_MS, 0), GENESIS_MS + 1_600); + assert_eq!(interval2_boundary_ms(GENESIS_MS, 5), GENESIS_MS + 5 * 4_000 + 1_600); + } + + #[test] + fn early_window_bounds() { + // Window is [T2 - EARLY_AGGREGATION_WINDOW_MS, T2). + let before_window = t2(5) - EARLY_AGGREGATION_WINDOW_MS - 1; + assert_eq!(early_aggregation_slot(before_window, GENESIS_MS), None); + let window_open = t2(5) - EARLY_AGGREGATION_WINDOW_MS; + assert_eq!(early_aggregation_slot(window_open, GENESIS_MS), Some(5)); + assert_eq!(early_aggregation_slot(t2(5) - 1, GENESIS_MS), Some(5)); + assert_eq!(early_aggregation_slot(t2(5), GENESIS_MS), None); + // Slot 0 has a window too. + assert_eq!(early_aggregation_slot(t2(0) - 1, GENESIS_MS), Some(0)); + } + + #[test] + fn early_window_before_genesis_is_none() { + assert_eq!(early_aggregation_slot(GENESIS_MS - 1, GENESIS_MS), None); + } + + #[test] + fn threshold_is_two_thirds_of_expected() { + // Zero expected: never trigger (validator-less edge, div-by-zero guard). + assert!(!early_threshold_met(0, 0)); + assert!(!early_threshold_met(5, 0)); + // Exact 2/3 triggers. + assert!(early_threshold_met(2, 3)); + assert!(!early_threshold_met(1, 3)); + // Ceiling behavior when expected isn't divisible by 3. + assert!(early_threshold_met(3, 4)); // 9 >= 8 + assert!(!early_threshold_met(2, 4)); // 6 < 8 + // Devnet-scale sanity: 33 validators, one subnet. + assert!(early_threshold_met(22, 33)); + assert!(!early_threshold_met(21, 33)); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` +Expected: FAIL to compile with "cannot find function `early_aggregation_slot`" (and the others) + +- [ ] **Step 3: Implement consts and functions** + +In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). + +```rust +/// Width of the early-aggregation window: a session may start at most this +/// long before the interval-2 boundary, provided the signature threshold is +/// met (see `early_threshold_met`). +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; + +/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the +/// normal aggregation start). +pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { + genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL +} + +/// If `now_ms` falls inside some slot's early-aggregation window +/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 +/// boundary), return that slot. +pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { + let since_genesis = now_ms.checked_sub(genesis_time_ms)?; + let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + let in_window = + ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; + in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) +} + +/// Early-start threshold: a single attestation-data group holds at least 2/3 +/// of the signatures expected from this node's aggregation subnets. At most +/// one group per slot can satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { + expected > 0 && max_group_count * 3 >= expected * 2 +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` +Expected: PASS (4 tests). Dead-code warnings for the not-yet-used functions are acceptable at this step only if the compiler emits them (`pub(crate)` items used from tests usually don't warn); they get consumed in Tasks 6–7. + +- [ ] **Step 5: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs +git commit -m "feat(blockchain): add early-aggregation window and threshold helpers" +``` + +--- + +### Task 4: Plumbing — subnet set into the actor, expected-count field + +Compile-level wiring, no behavior change yet. + +**Files:** +- Modify: `crates/blockchain/src/lib.rs` (`BlockChain::spawn` ~line 80, `BlockChainServer` struct ~line 138) +- Modify: `bin/ethlambda/src/main.rs` (~lines 206–223) + +- [ ] **Step 1: Add spawn parameter and fields** + +In `crates/blockchain/src/lib.rs`, change `BlockChain::spawn` signature (~line 80) — new `aggregation_subnets` parameter after `attestation_committee_count`: + +```rust + pub fn spawn( + store: Store, + validator_keys: HashMap, + aggregator: AggregatorController, + attestation_committee_count: u64, + aggregation_subnets: HashSet, + gate_duties: bool, + proposer_config: ProposerConfig, + ) -> BlockChain { +``` + +Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), compute the expected count (guard the modulo against a zero committee count): + +```rust + // Denominator of the early-aggregation 2/3 threshold: how many + // validators attest on subnets this node aggregates. Computed once — + // the validator registry is static and `head_state()` clones the full + // state, so this must not run per gossip insert. + let validator_count = store.head_state().validators.len() as u64; + let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + 0 + } else { + (0..validator_count) + .filter(|vid| { + aggregation_subnets.contains(&(vid % attestation_committee_count)) + }) + .count() + }; +``` + +Add to the `BlockChainServer` struct literal in `spawn`: + +```rust + early_aggregation_expected_sigs, + pending_aggregate_publishes: Vec::new(), +``` + +Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): + +```rust + /// Number of validators whose subnet is one this node aggregates — the + /// denominator of the early-aggregation 2/3 threshold. Computed once at + /// spawn (the validator registry is static). + early_aggregation_expected_sigs: usize, + + /// Aggregates produced before the interval-2 boundary, held back so they + /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the + /// boundary; only ever holds the current slot's aggregates. + pending_aggregate_publishes: Vec, +``` + +(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`.) + +- [ ] **Step 2: Update `main.rs`** + +In `bin/ethlambda/src/main.rs`, add `compute_subscription_subnets` to the `ethlambda_p2p` import (line 40): + +```rust +use ethlambda_p2p::{ + Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, +}; +``` + +Before `BlockChain::spawn` (~line 213), compute the set with the same startup inputs `build_swarm` uses, and pass it (borrow `aggregate_subnet_ids` — it is moved into `SwarmConfig` further down): + +```rust + // Same startup inputs build_swarm uses — single source of truth is the + // shared helper. Frozen at startup like the gossip subscriptions + // themselves (hot-standby model). + let aggregation_subnets = compute_subscription_subnets( + &validator_ids, + attestation_committee_count, + options.is_aggregator, + options.aggregate_subnet_ids.as_deref(), + ); + + let blockchain = BlockChain::spawn( + store.clone(), + validator_keys, + aggregator.clone(), + attestation_committee_count, + aggregation_subnets, + !options.disable_duty_sync_gate, + ProposerConfig { + enable_proposer_aggregation: options.enable_proposer_aggregation, + max_attestations_per_block: options.max_attestations_per_block, + }, + ); +``` + +- [ ] **Step 3: Build the workspace** + +Run: `cargo build --workspace` +Expected: clean build. If `pending_aggregate_publishes` / `early_aggregation_expected_sigs` trigger dead-code warnings, silence is NOT needed — they are read in Tasks 6–7; if the build fails on `-D warnings` locally, proceed (only `make lint` enforces it) or add the fields in the task where they're first read. Do not annotate with `#[allow(dead_code)]`. + +Note: plain `cargo build` does not deny warnings, so this passes; the fields become live two tasks later, before `make lint` runs in Task 8. + +- [ ] **Step 4: Commit** + +```bash +git add crates/blockchain/src/lib.rs bin/ethlambda/src/main.rs +git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signature count into actor" +``` + +--- + +### Task 5: Deadline change 750 → 800 ms + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs:28-33` + +- [ ] **Step 1: Change the constant and its comment** + +Replace: + +```rust +/// Soft deadline for committee-signature aggregation measured from the +/// interval-2 tick. After this much wall time elapses, the actor signals the +/// worker to stop via its cancellation token. The 50 ms budget before the next +/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving +/// aggregates and for gossip propagation margin. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); +``` + +with: + +```rust +/// Soft deadline for committee-signature aggregation measured from session +/// start. After this much wall time elapses, the actor signals the worker to +/// stop via its cancellation token. A session started exactly at interval 2 +/// gets the full interval (interval 3 is one interval later); a session +/// started early (see `early_aggregation_slot`) ends correspondingly earlier. +/// The deadline only stops new jobs from starting — a job mid-proof finishes +/// and publishes right after. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); +``` + +- [ ] **Step 2: Build and check references** + +Run: `cargo build -p ethlambda-blockchain && grep -rn "750" crates/blockchain/src/lib.rs` +Expected: clean build; the grep finds no stale `+750 ms` comment references (the doc comment on `start_aggregation_session` ~line 365 says "Schedule the `AggregationDeadline` self-message at +750 ms" — update it to "+`AGGREGATION_DEADLINE`"). + +- [ ] **Step 3: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs +git commit -m "feat(blockchain): extend aggregation deadline to a full interval" +``` + +--- + +### Task 6: Publish alignment — hold early aggregates until the interval-2 boundary + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) +- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) + +- [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** + +Add field to `AggregationSession` (~line 75): + +```rust +pub(crate) struct AggregationSession { + /// Slot at which this session was started; used as a fencing id so we can + /// drop late-arriving messages from a prior session. + pub(crate) session_id: u64, + /// Whether the session started before the slot's interval-2 boundary via + /// the early-aggregation trigger. + pub(crate) early: bool, + /// Child of the actor cancellation token; fires either at the deadline or + /// when the actor itself is stopping. + pub(crate) cancel: CancellationToken, + /// Handle to the `spawn_blocking` worker. Held so `stopped()` / new-session + /// start can await completion. + pub(crate) worker: tokio::task::JoinHandle<()>, +} +``` + +Add next to `AggregationDeadline` (~line 112): + +```rust +/// Self-message scheduled when a session starts early; fires at the +/// interval-2 boundary and publishes any aggregates held back by the +/// publish-alignment rule (aggregates must not reach gossip before +/// interval 2). +pub(crate) struct FlushAggregatePublishes; +impl Message for FlushAggregatePublishes { + type Result = (); +} +``` + +- [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** + +Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes` (and, for Task 7, `EarlyAggregationCheck` will join it). + +In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: + +```rust + coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); +``` + +Replace the tail of the function (from `let session_id = slot;` through the final `self.current_aggregation = Some(...)`) with: + +```rust + let session_id = slot; + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + let now_ms = unix_now_ms(); + let early = now_ms < t2_ms; + if early { + // Publish alignment: aggregates must not reach gossip before the + // interval-2 boundary. Aggregates produced before T2 are buffered + // in `pending_aggregate_publishes`; this timer flushes them at T2. + let lead = Duration::from_millis(t2_ms - now_ms); + metrics::inc_aggregation_early_starts(); + metrics::observe_aggregation_early_start_lead(lead); + info!( + %slot, + lead_ms = lead.as_millis() as u64, + "Starting aggregation session early" + ); + send_after(lead, ctx.clone(), FlushAggregatePublishes); + } + + // Independent token per session. Shutdown propagates via our + // #[stopped] hook which cancels any current session; the deadline + // timer cancels this specific session at +AGGREGATION_DEADLINE. + let cancel = CancellationToken::new(); + let actor_ref = ctx.actor_ref(); + + let worker_cancel = cancel.clone(); + let worker_actor = actor_ref.clone(); + let worker = tokio::task::spawn_blocking(move || { + run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); + }); + + let _deadline_timer = send_after( + AGGREGATION_DEADLINE, + ctx.clone(), + AggregationDeadline { session_id }, + ); + + self.current_aggregation = Some(AggregationSession { + session_id, + early, + cancel, + worker, + }); +``` + +(`metrics::inc_aggregation_early_starts` / `observe_aggregation_early_start_lead` are added in Step 4 of this task.) + +Remove the emission from the interval-2 tick arm (~line 312), which becomes: + +```rust + // ==== interval 2 ==== + 2 => { + if is_aggregator { + self.start_aggregation_session(slot, ctx).await; + } else { + metrics::inc_aggregator_skipped_not_aggregator(); + } + } +``` + +(The `coverage::emit_agg_start_new_coverage` call and its old surrounding braces go away; the early-session skip guard lands in Task 7.) + +- [ ] **Step 3: Split publish out of `AggregateProduced` and add the flush handler** + +Add a helper method on `BlockChainServer` (near `on_gossip_attestation`, ~line 929): + +```rust + /// Publish an aggregated attestation to the aggregation gossip topic. + fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { + if let Some(ref p2p) = self.p2p { + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } + } +``` + +Rewrite the body of `impl Handler` (~line 1050) — the session-id fencing stays, then: + +```rust + aggregation::apply_aggregated_group(&mut self.store, &msg.output); + + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + + // Publish alignment: hold back aggregates produced before this slot's + // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. + // (`session_id` is the session's slot.) + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); + if unix_now_ms() < t2_ms { + self.pending_aggregate_publishes.push(aggregate); + return; + } + self.publish_aggregate(aggregate); +``` + +Add the flush handler next to the other aggregation handlers (~line 1099): + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + let pending = std::mem::take(&mut self.pending_aggregate_publishes); + if pending.is_empty() { + return; + } + info!( + count = pending.len(), + "Publishing aggregates held back until the interval-2 boundary" + ); + for aggregate in pending { + self.publish_aggregate(aggregate); + } + } +} +``` + +Add the `early` field to the `AggregationDone` completion log (~line 1085): before the `info!`, compute + +```rust + let early = self + .current_aggregation + .as_ref() + .is_some_and(|s| s.session_id == msg.session_id && s.early); +``` + +and add `early,` to the `info!(...)` field list (after `cancelled = msg.cancelled,`). + +- [ ] **Step 4: Add the two metrics** + +In `crates/blockchain/src/metrics.rs` — counter in the `// --- Counters ---` section, histogram in `// --- Histograms ---`, public fns near their statics following file style: + +```rust +static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter!( + "lean_aggregation_early_starts_total", + "Aggregation sessions started before the interval-2 boundary" + ) + .unwrap() + }); + +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} +``` + +```rust +static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram!( + "lean_aggregation_early_start_lead_seconds", + "How far before the interval-2 boundary an early aggregation session started", + vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + ) + .unwrap() + }); + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} +``` + +- [ ] **Step 5: Build and run existing tests** + +Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` +Expected: clean build, all unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). + +- [ ] **Step 6: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs crates/blockchain/src/metrics.rs +git commit -m "feat(blockchain): hold early-produced aggregates until the interval-2 boundary" +``` + +--- + +### Task 7: Early trigger — threshold checks, timer, and interval-2 skip + +**Files:** +- Modify: `crates/blockchain/src/aggregation.rs` (new message next to `FlushAggregatePublishes`) +- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, interval-1 arm ~line 290, interval-2 arm ~line 312, `NewAttestation` handler ~line 1034, new method + handler) + +- [ ] **Step 1: Add the check message in `aggregation.rs`** + +```rust +/// One-shot self-message scheduled at the interval-1 tick; fires when the +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// the threshold check for signatures that all arrived before the window. +/// Arrivals inside the window are checked per insert instead. +pub(crate) struct EarlyAggregationCheck; +impl Message for EarlyAggregationCheck { + type Result = (); +} +``` + +- [ ] **Step 2: Add the trigger method on `BlockChainServer`** + +Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` and `EARLY_AGGREGATION_WINDOW_MS`. Add the method after `start_aggregation_session` (~line 418): + +```rust + /// Early-aggregation trigger: start the slot's session ahead of the + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// a single attestation-data group already holds 2/3 of the signatures + /// expected from this node's aggregation subnets. Called after every + /// stored gossip signature and once at the window opening via + /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started + /// session stays in `current_aggregation` (running or finished) until the + /// next session replaces it. + async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { + if !self.aggregator.is_enabled() { + return; + } + let genesis_time_ms = self.store.config().genesis_time * 1000; + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) + else { + return; + }; + if self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot) + { + return; + } + let max_group = self.store.max_gossip_group_count_for_slot(slot); + if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + return; + } + info!( + %slot, + max_group, + expected = self.early_aggregation_expected_sigs, + "Early-aggregation threshold met" + ); + self.start_aggregation_session(slot, ctx).await; + } +``` + +- [ ] **Step 3: Wire the three trigger sites** + +**(a) Interval-1 arm** (~line 290) — at the end of the `1 => { ... }` block, after the attestation-production `if/else`: + +```rust + // Schedule the early-aggregation window check. This tick is + // one interval before T2, so the timer fires right as the + // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + if is_aggregator { + send_after( + Duration::from_millis( + MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, + ), + ctx.clone(), + EarlyAggregationCheck, + ); + } +``` + +**(b) Timer handler** — next to the other aggregation handlers: + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { + self.maybe_start_early_aggregation(ctx).await; + } +} +``` + +**(c) Per-insert check** — `impl Handler` (~line 1034) becomes: + +```rust +impl Handler for BlockChainServer { + async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { + self.on_gossip_attestation(&msg.attestation); + self.maybe_start_early_aggregation(ctx).await; + } +} +``` + +(The method's internal guards make this a few comparisons outside the window; `max_gossip_group_count_for_slot` is one lock with no clones inside it.) + +- [ ] **Step 4: Skip the interval-2 start when the early session exists** + +The `2 => { ... }` arm becomes: + +```rust + // ==== interval 2 ==== + 2 => { + if is_aggregator { + // The early trigger may have already started this slot's + // session (running or finished) — it IS the slot's session, + // so don't start a second one. + let already_started = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot); + if !already_started { + self.start_aggregation_session(slot, ctx).await; + } + } else { + metrics::inc_aggregator_skipped_not_aggregator(); + } + } +``` + +- [ ] **Step 5: Build and run all blockchain tests** + +Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` +Expected: clean build, all tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs +git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived" +``` + +--- + +### Task 8: Full verification + +**Files:** none (verification only) + +- [ ] **Step 1: Format and lint** + +Run: `make fmt && make lint` +Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Task 4 would surface — fix by ensuring both new fields are read, which Tasks 6–7 guarantee). + +- [ ] **Step 2: Run the full test suite** + +Run: `make test` +Expected: all workspace tests + forkchoice spec tests pass. The spec tests exercise `on_block_without_verification` and the STF — untouched by this change. If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. + +- [ ] **Step 3: Commit any fmt fallout** + +```bash +git add -u && git commit -m "chore: fmt" || true +``` + +(Skip the commit if the tree is clean.) + +- [ ] **Step 4: Devnet validation (manual, out of band)** + +Not part of the automated plan — run `.claude/skills/test-pr-devnet/scripts/test-branch.sh` on this branch and check: +- `lean_aggregation_early_starts_total` grows on aggregator nodes +- `lean_aggregation_early_start_lead_seconds` distribution sits in (0, 0.4] +- "Publishing aggregates held back until the interval-2 boundary" appears with count ≥ 1 +- Aggregate publish offsets (per-slot duty timing method) do not move earlier than interval 2 +- Finalization rate is not worse than the base image diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 58dc8e90..0a2d51ca 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -43,8 +43,11 @@ Aggregator-only, fires at most once per slot. from at least 2/3 of the expected validators: `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the number of stored gossip signatures in that group (one per validator). - `expected` = number of head-state validators whose - `vid % attestation_committee_count` is in this node's aggregation subnet set. + `expected` = number of validators whose + `vid % attestation_committee_count` is in this node's aggregation subnet set, + computed once at actor spawn (the validator registry is static, and + `Store::head_state()` clones the full state so per-check reads would be + needlessly expensive). Note that at most one group per slot can reach 2/3 (each validator signs once per slot), so the per-group condition can hold for at most one group. - **Check sites:** @@ -110,12 +113,14 @@ pattern: build early, publish aligned to the boundary). ### Subnet-set plumbing The subscription-subnet computation is hoisted out of `build_swarm` into a -shared pure helper (in the p2p crate, e.g. -`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`). -`main.rs` calls it once and passes the resulting set to both: - -- the p2p config (which subscribes to exactly this set, behavior unchanged), and -- the `BlockChainServer` (new field, used for the `expected` count). +shared pure helper in the p2p crate: +`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`. +Both call sites use it with the same startup inputs, keeping a single source of +truth without churning `SwarmConfig`: + +- `build_swarm` (subscribes to exactly this set, behavior unchanged), and +- `main.rs`, which passes the resulting set to `BlockChain::spawn` for the + `expected` count. The set is frozen at startup, matching the existing hot-standby aggregator model (runtime toggles do not resubscribe subnets). From 2b782402760387307b2c13ffa15a38433af3098c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:47:09 -0300 Subject: [PATCH 03/29] docs: drop new-test steps from plan, add 4-node devnet validation task --- .../2026-07-01-early-aggregation-start.md | 248 +++++------------- 1 file changed, 67 insertions(+), 181 deletions(-) diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index 69d193f9..c5be585e 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -8,14 +8,16 @@ **Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. +**Verification policy (per user):** No new unit tests. Each task verifies by compiling (plus existing test suites where they already exist); final validation is a local 4-node devnet run (Task 9). + **Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` **Key existing code:** - `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) - `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` - `crates/blockchain/src/metrics.rs` — prometheus registration patterns -- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451), test mod (~line 1555) with `make_att_data(slot)`, `make_att_data_for_target(slot, root)`, `make_dummy_sig()` helpers -- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337), test mod (~line 790) +- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451) +- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337) - `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) **Timing model (4 s slots, 5 × 800 ms intervals):** @@ -36,48 +38,9 @@ slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). **Files:** -- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451; test in existing `mod tests`) - -- [ ] **Step 1: Write the failing test** - -Add to the `// ============ GossipSignatureBuffer Tests ============` section of `mod tests` in `crates/storage/src/store.rs` (near `gossip_buffer_fifo_eviction`, ~line 2423): - -```rust -#[test] -fn gossip_buffer_max_group_count_for_slot() { - let mut buf = GossipSignatureBuffer::new(100); - assert_eq!(buf.max_group_count_for_slot(1), 0); - - // Slot 1, group A: 3 validators. - for vid in 0..3 { - let data = make_att_data(1); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } - // Slot 1, group B (same slot, different target root): 2 validators. - for vid in 3..5 { - let data = make_att_data_for_target(1, root(99)); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } - // Slot 2: 5 validators. - for vid in 0..5 { - let data = make_att_data(2); - buf.insert(HashedAttestationData::new(data), vid, make_dummy_sig()); - } +- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451) - assert_eq!(buf.max_group_count_for_slot(1), 3); - assert_eq!(buf.max_group_count_for_slot(2), 5); - assert_eq!(buf.max_group_count_for_slot(3), 0); -} -``` - -Note: `make_att_data_for_target` is defined further down in the test mod (~line 2319); Rust test mods don't care about definition order. - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cargo test -p ethlambda-storage max_group_count_for_slot` -Expected: FAIL to compile with "no method named `max_group_count_for_slot`" - -- [ ] **Step 3: Implement the buffer method and Store wrapper** +- [ ] **Step 1: Implement the buffer method and Store wrapper** In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): @@ -106,12 +69,12 @@ pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { } ``` -- [ ] **Step 4: Run test to verify it passes** +- [ ] **Step 2: Build** -Run: `cargo test -p ethlambda-storage max_group_count_for_slot` -Expected: PASS (1 test) +Run: `cargo build -p ethlambda-storage` +Expected: clean build (a dead-code warning on the buffer method is possible until Task 7 wires the caller; plain `cargo build` does not deny warnings). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/storage/src/store.rs @@ -125,45 +88,9 @@ git commit -m "feat(storage): add max gossip group count query for early aggrega Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. **Files:** -- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337; tests in existing `mod tests` ~line 790) - -- [ ] **Step 1: Write the failing tests** +- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337) -Add to `mod tests` in `crates/net/p2p/src/lib.rs` (if the test mod doesn't already have `HashSet` in scope via `use super::*;`, add `use std::collections::HashSet;`): - -```rust -#[test] -fn subscription_subnets_validators_only() { - // 3 % 4 = 3, 7 % 4 = 3, 10 % 4 = 2 - let subnets = compute_subscription_subnets(&[3, 7, 10], 4, false, None); - assert_eq!(subnets, HashSet::from([3, 2])); -} - -#[test] -fn subscription_subnets_aggregator_unions_explicit_ids() { - let subnets = compute_subscription_subnets(&[1], 4, true, Some(&[0, 2])); - assert_eq!(subnets, HashSet::from([1, 0, 2])); -} - -#[test] -fn subscription_subnets_validatorless_aggregator_falls_back_to_zero() { - let subnets = compute_subscription_subnets(&[], 4, true, None); - assert_eq!(subnets, HashSet::from([0])); -} - -#[test] -fn subscription_subnets_validatorless_non_aggregator_is_empty() { - let subnets = compute_subscription_subnets(&[], 4, false, None); - assert!(subnets.is_empty()); -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cargo test -p ethlambda-p2p subscription_subnets` -Expected: FAIL to compile with "cannot find function `compute_subscription_subnets`" - -- [ ] **Step 3: Implement the helper and use it in `build_swarm`** +- [ ] **Step 1: Implement the helper and use it in `build_swarm`** Add near `build_swarm` (above it, ~line 185): @@ -223,15 +150,12 @@ Then replace the inline logic in `build_swarm` (~lines 304–329). The metric mu (The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cargo test -p ethlambda-p2p subscription_subnets` -Expected: PASS (4 tests) +- [ ] **Step 2: Build and run existing p2p tests** -Run: `cargo build -p ethlambda-p2p` -Expected: clean build (no unused warnings; the old inline block is fully replaced) +Run: `cargo build -p ethlambda-p2p && cargo test -p ethlambda-p2p --lib` +Expected: clean build, existing tests pass (behavior of `build_swarm` is unchanged — same set, same subscriptions). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/net/p2p/src/lib.rs @@ -242,76 +166,12 @@ git commit -m "refactor(p2p): extract subscription subnet computation into pure ### Task 3: Blockchain — pure early-aggregation helpers -Window math and threshold predicate, unit-testable without an actor. +Window math and threshold predicate. **Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37; new `#[cfg(test)] mod tests` at end of file) - -- [ ] **Step 1: Write the failing tests** +- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37) -`crates/blockchain/src/aggregation.rs` has no test mod yet. Add at the end of the file: - -```rust -#[cfg(test)] -mod tests { - use super::*; - use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT}; - - const GENESIS_MS: u64 = 1_000_000; - - /// Interval-2 boundary of `slot`, relative to the test genesis. - fn t2(slot: u64) -> u64 { - GENESIS_MS + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL - } - - #[test] - fn interval2_boundary_matches_slot_layout() { - assert_eq!(interval2_boundary_ms(GENESIS_MS, 0), GENESIS_MS + 1_600); - assert_eq!(interval2_boundary_ms(GENESIS_MS, 5), GENESIS_MS + 5 * 4_000 + 1_600); - } - - #[test] - fn early_window_bounds() { - // Window is [T2 - EARLY_AGGREGATION_WINDOW_MS, T2). - let before_window = t2(5) - EARLY_AGGREGATION_WINDOW_MS - 1; - assert_eq!(early_aggregation_slot(before_window, GENESIS_MS), None); - let window_open = t2(5) - EARLY_AGGREGATION_WINDOW_MS; - assert_eq!(early_aggregation_slot(window_open, GENESIS_MS), Some(5)); - assert_eq!(early_aggregation_slot(t2(5) - 1, GENESIS_MS), Some(5)); - assert_eq!(early_aggregation_slot(t2(5), GENESIS_MS), None); - // Slot 0 has a window too. - assert_eq!(early_aggregation_slot(t2(0) - 1, GENESIS_MS), Some(0)); - } - - #[test] - fn early_window_before_genesis_is_none() { - assert_eq!(early_aggregation_slot(GENESIS_MS - 1, GENESIS_MS), None); - } - - #[test] - fn threshold_is_two_thirds_of_expected() { - // Zero expected: never trigger (validator-less edge, div-by-zero guard). - assert!(!early_threshold_met(0, 0)); - assert!(!early_threshold_met(5, 0)); - // Exact 2/3 triggers. - assert!(early_threshold_met(2, 3)); - assert!(!early_threshold_met(1, 3)); - // Ceiling behavior when expected isn't divisible by 3. - assert!(early_threshold_met(3, 4)); // 9 >= 8 - assert!(!early_threshold_met(2, 4)); // 6 < 8 - // Devnet-scale sanity: 33 validators, one subnet. - assert!(early_threshold_met(22, 33)); - assert!(!early_threshold_met(21, 33)); - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` -Expected: FAIL to compile with "cannot find function `early_aggregation_slot`" (and the others) - -- [ ] **Step 3: Implement consts and functions** +- [ ] **Step 1: Implement consts and functions** In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). @@ -347,12 +207,12 @@ pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bo } ``` -- [ ] **Step 4: Run tests to verify they pass** +- [ ] **Step 2: Build** -Run: `cargo test -p ethlambda-blockchain --lib aggregation::tests` -Expected: PASS (4 tests). Dead-code warnings for the not-yet-used functions are acceptable at this step only if the compiler emits them (`pub(crate)` items used from tests usually don't warn); they get consumed in Tasks 6–7. +Run: `cargo build -p ethlambda-blockchain` +Expected: clean build (dead-code warnings possible until Tasks 6–7 wire the callers). -- [ ] **Step 5: Commit** +- [ ] **Step 3: Commit** ```bash git add crates/blockchain/src/aggregation.rs @@ -467,9 +327,7 @@ Before `BlockChain::spawn` (~line 213), compute the set with the same startup in - [ ] **Step 3: Build the workspace** Run: `cargo build --workspace` -Expected: clean build. If `pending_aggregate_publishes` / `early_aggregation_expected_sigs` trigger dead-code warnings, silence is NOT needed — they are read in Tasks 6–7; if the build fails on `-D warnings` locally, proceed (only `make lint` enforces it) or add the fields in the task where they're first read. Do not annotate with `#[allow(dead_code)]`. - -Note: plain `cargo build` does not deny warnings, so this passes; the fields become live two tasks later, before `make lint` runs in Task 8. +Expected: clean build. Dead-code warnings on the two new fields are acceptable here (plain `cargo build` does not deny warnings); they become live in Tasks 6–7, before `make lint` runs in Task 8. Do not annotate with `#[allow(dead_code)]`. - [ ] **Step 4: Commit** @@ -484,6 +342,7 @@ git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signa **Files:** - Modify: `crates/blockchain/src/aggregation.rs:28-33` +- Modify: `crates/blockchain/src/lib.rs` (stale `+750 ms` doc-comment reference on `start_aggregation_session`, ~line 365) - [ ] **Step 1: Change the constant and its comment** @@ -530,6 +389,7 @@ git commit -m "feat(blockchain): extend aggregation deadline to a full interval" **Files:** - Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) - Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) +- Modify: `crates/blockchain/src/metrics.rs` (new counter + histogram) - [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** @@ -567,7 +427,7 @@ impl Message for FlushAggregatePublishes { - [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** -Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes` (and, for Task 7, `EarlyAggregationCheck` will join it). +Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes`. In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: @@ -747,7 +607,7 @@ pub fn observe_aggregation_early_start_lead(lead: Duration) { - [ ] **Step 5: Build and run existing tests** Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, all unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). +Expected: clean build, existing unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). - [ ] **Step 6: Commit** @@ -886,10 +746,10 @@ The `2 => { ... }` arm becomes: } ``` -- [ ] **Step 5: Build and run all blockchain tests** +- [ ] **Step 5: Build and run existing blockchain tests** Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, all tests pass. +Expected: clean build, existing tests pass. - [ ] **Step 6: Commit** @@ -900,19 +760,19 @@ git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet sign --- -### Task 8: Full verification +### Task 8: Lint and existing test suites **Files:** none (verification only) - [ ] **Step 1: Format and lint** Run: `make fmt && make lint` -Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Task 4 would surface — fix by ensuring both new fields are read, which Tasks 6–7 guarantee). +Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Tasks 1–4 would surface — by now every added item has a caller). -- [ ] **Step 2: Run the full test suite** +- [ ] **Step 2: Run the existing test suites** Run: `make test` -Expected: all workspace tests + forkchoice spec tests pass. The spec tests exercise `on_block_without_verification` and the STF — untouched by this change. If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. +Expected: all workspace tests + forkchoice spec tests pass (no new tests were added; this guards against regressions in the touched paths). If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. - [ ] **Step 3: Commit any fmt fallout** @@ -922,11 +782,37 @@ git add -u && git commit -m "chore: fmt" || true (Skip the commit if the tree is clean.) -- [ ] **Step 4: Devnet validation (manual, out of band)** +--- + +### Task 9: Local 4-node devnet validation + +Run a local 4-node devnet on this branch and verify the early-start behavior end to end. Use the `devnet-runner` skill (`.claude/skills/devnet-runner/`) for the exact workflow; constraints that matter here: + +- **Release build only** — debug binaries stack-overflow in leanVM `rec_aggregation`. +- **Exactly one node with `--is-aggregator`** — multiple co-located aggregators stall finality via leanVM CPU contention. +- If using Docker Desktop, the VM needs 16+ GiB (leanVM peaks ~4–5 GiB per node); binary mode avoids this. + +- [ ] **Step 1: Start the devnet** + +4 nodes, fresh genesis, one aggregator. Let it run at least ~50 slots (~4 minutes). + +- [ ] **Step 2: Verify early-start behavior on the aggregator node** + +Check the aggregator node's logs and metrics: + +1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 400]`. +2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. +3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). +4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. +5. `"Committee signatures aggregated"` logs show `early=true` sessions and no `"Prior aggregation worker still running"` warnings. +6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). + +- [ ] **Step 3: Verify chain health** + +1. `lean_latest_finalized_slot` advances steadily on all 4 nodes (fresh local devnet should finalize nearly every justifiable slot). +2. Blocks carry attestations (`attestation_count > 0` in block logs). +3. No stalls, no panic/error-level logs attributable to aggregation. + +- [ ] **Step 4: Stop the devnet and report** -Not part of the automated plan — run `.claude/skills/test-pr-devnet/scripts/test-branch.sh` on this branch and check: -- `lean_aggregation_early_starts_total` grows on aggregator nodes -- `lean_aggregation_early_start_lead_seconds` distribution sits in (0, 0.4] -- "Publishing aggregates held back until the interval-2 boundary" appears with count ≥ 1 -- Aggregate publish offsets (per-slot duty timing method) do not move earlier than interval 2 -- Finalization rate is not worse than the base image +Stop the nodes; summarize observed early-start rate (early sessions / slots), lead-time distribution, and finalization progress. From ee03c4aa07dc9bdf4e40a04c0e6050eabd30f59e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 17:59:43 -0300 Subject: [PATCH 04/29] docs: apply plan-review fixes (stale publish drain, cached genesis_time_ms, spec corrections) --- .../2026-07-01-early-aggregation-start.md | 34 +++++++++++++------ ...26-07-01-early-aggregation-start-design.md | 30 ++++++++++------ 2 files changed, 44 insertions(+), 20 deletions(-) diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index c5be585e..f20dafd5 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -72,7 +72,7 @@ pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { - [ ] **Step 2: Build** Run: `cargo build -p ethlambda-storage` -Expected: clean build (a dead-code warning on the buffer method is possible until Task 7 wires the caller; plain `cargo build` does not deny warnings). +Expected: clean build, no warnings (the `pub` Store method keeps the private buffer method alive). - [ ] **Step 3: Commit** @@ -264,9 +264,10 @@ Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), }; ``` -Add to the `BlockChainServer` struct literal in `spawn`: +Add to the `BlockChainServer` struct literal in `spawn` (`genesis_time` is already computed at ~line 90): ```rust + genesis_time_ms: genesis_time * 1000, early_aggregation_expected_sigs, pending_aggregate_publishes: Vec::new(), ``` @@ -274,6 +275,11 @@ Add to the `BlockChainServer` struct literal in `spawn`: Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): ```rust + /// Genesis time in milliseconds, cached at spawn. `store.config()` is an + /// uncached backend read, too heavy for the per-gossip-insert early + /// checks that need this. + genesis_time_ms: u64, + /// Number of validators whose subnet is one this node aggregates — the /// denominator of the early-aggregation 2/3 threshold. Computed once at /// spawn (the validator registry is static). @@ -285,7 +291,7 @@ Add the fields to the `BlockChainServer` struct definition (after `attestation_c pending_aggregate_publishes: Vec, ``` -(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`.) +(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`. Existing `config().genesis_time * 1000` call sites in `on_tick`/`handle_tick` stay as they are — only the new code paths use the cached field.) - [ ] **Step 2: Update `main.rs`** @@ -439,8 +445,18 @@ Replace the tail of the function (from `let session_id = slot;` through the fina ```rust let session_id = slot; - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + // Any leftovers from a prior slot mean its flush never fired (a + // backwards wall-clock step, or a flush timer delayed past the next + // session). Drop them — late aggregates are dropped, same policy as + // signatures that miss the snapshot. + let stale = std::mem::take(&mut self.pending_aggregate_publishes); + if !stale.is_empty() { + warn!( + count = stale.len(), + "Dropping stale pending aggregate publishes" + ); + } + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { @@ -529,8 +545,7 @@ Rewrite the body of `impl Handler` (~line 1050) — the sessi // Publish alignment: hold back aggregates produced before this slot's // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. // (`session_id` is the session's slot.) - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); if unix_now_ms() < t2_ms { self.pending_aggregate_publishes.push(aggregate); return; @@ -654,8 +669,7 @@ Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` a if !self.aggregator.is_enabled() { return; } - let genesis_time_ms = self.store.config().genesis_time * 1000; - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) else { return; }; @@ -804,7 +818,7 @@ Check the aggregator node's logs and metrics: 2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. 3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). 4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. -5. `"Committee signatures aggregated"` logs show `early=true` sessions and no `"Prior aggregation worker still running"` warnings. +5. `"Committee signatures aggregated"` logs show `early=true` sessions. `"Prior aggregation worker still running"` warnings must be rare (the early start shrinks the worst-case gap to the previous slot's worker from 3250 ms to 2800 ms, so occasional joins under slow proofs are expected — every slot would be a bug). 6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). - [ ] **Step 3: Verify chain health** diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 0a2d51ca..082ef330 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -53,11 +53,13 @@ Aggregator-only, fires at most once per slot. - **Check sites:** 1. After each gossip-signature insert in the `NewAttestation` handler, while the wall clock is inside the window. - 2. A one-shot `EarlyAggregationCheck { slot }` self-message scheduled at the - interval-1 tick via `send_after(400 ms)` (interval-1 tick + 400 ms = - T2 - 400 ms), covering the case where the threshold was already met before - the window opened. If the interval-1 tick is skipped (overrun), the - per-insert checks still apply. + 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is + recomputed from the wall clock at fire time, so a late-fired timer can + never act on a stale slot) scheduled at the interval-1 tick via + `send_after(400 ms)` (interval-1 tick + 400 ms = T2 - 400 ms), covering + the case where the threshold was already met before the window opened. If + the interval-1 tick is skipped (overrun), the per-insert checks still + apply. - **Once-per-slot guard:** a session with `session_id == slot` already exists in `current_aggregation` (the field persists after the worker finishes; it is only replaced at the next session start). @@ -97,7 +99,11 @@ pattern: build early, publish aligned to the boundary). buffer is a no-op). - The buffer is bounded in practice by the number of data groups in one slot. Entries are never carried across slots: the flush timer fires at T2 of the - same slot that buffered them. + same slot that buffered them, and as a backstop `start_aggregation_session` + drains (drops, with a warning) any leftovers before installing a new session + — covering a backwards wall-clock step or a flush timer delayed past the + next session. Late aggregates are dropped, the same policy as signatures + that miss the snapshot. ### Deadline change @@ -127,10 +133,14 @@ model (runtime toggles do not resubscribe subnets). ## Failure modes and edge cases -- **Snapshot returns `None` after trigger** (should not happen — the trigger - requires signatures to be present): no session is created, - `current_aggregation` keeps the prior slot's id, and interval 2 retries - normally. No signatures are lost. +- **Snapshot returns `None` after trigger**: unreachable in practice — the + trigger requires stored signatures, and `on_gossip_attestation` only stores + signatures whose pubkeys resolved during verification, so the snapshot + always yields at least one job for them. If it did happen, no session is + created and `current_aggregation` is left `None` (the prior session was + already taken), so the once-per-slot guard would not hold and per-insert + checks would retry; interval 2 remains the fallback. No signatures are + lost. - **Threshold never met:** interval 2 starts the session exactly as today. - **Signatures arriving after the early snapshot:** never aggregated that slot. Accepted trade-off (explicitly chosen over a top-up session). From 8ae3b0728c4735ed6d80d46cca655ab3602f3076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:02:13 -0300 Subject: [PATCH 05/29] feat(storage): add max gossip group count query for early aggregation --- crates/storage/src/store.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index c916bfca..8e344f9b 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -464,6 +464,16 @@ impl GossipSignatureBuffer { .collect() } + /// Largest signature count among data groups whose attestation slot is `slot`. + fn max_group_count_for_slot(&self, slot: u64) -> usize { + self.data + .values() + .filter(|entry| entry.data.slot == slot) + .map(|entry| entry.signatures.len()) + .max() + .unwrap_or(0) + } + /// Extract per-validator latest attestations from the raw signature pool. /// /// Mirrors `PayloadBuffer::extract_latest_attestations`: iterate data_roots @@ -1453,6 +1463,15 @@ impl Store { gossip.total_signatures() } + /// Largest per-group signature count among gossip groups voting for `slot`. + /// + /// One lock, no signature clones — cheap enough to call per gossip insert. + /// Drives the early-aggregation threshold check. + pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { + let gossip = self.gossip_signatures.lock().unwrap(); + gossip.max_group_count_for_slot(slot) + } + /// Estimated live data size in bytes for a table, as reported by the backend. pub fn estimate_table_bytes(&self, table: Table) -> u64 { self.backend.estimate_table_bytes(table) From 433ac14065e30d75acc581ca85871e4d23059f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:10:25 -0300 Subject: [PATCH 06/29] refactor(p2p): extract subscription subnet computation into pure helper --- crates/net/p2p/src/lib.rs | 63 ++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index de34e469..4308e130 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,6 +182,36 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } +/// Compute the set of attestation subnets this node subscribes to (and, when +/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): +/// every validator subscribes to its own subnet (`vid % committee_count`) for +/// mesh health; aggregators additionally subscribe to explicit +/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would +/// otherwise be empty. +/// +/// Evaluated once at startup — runtime aggregator toggles do not resubscribe +/// (hot-standby model); see the invariant note on [`SwarmConfig`]. +pub fn compute_subscription_subnets( + validator_ids: &[u64], + attestation_committee_count: u64, + is_aggregator: bool, + aggregate_subnet_ids: Option<&[u64]>, +) -> HashSet { + let mut subnets: HashSet = validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect(); + if is_aggregator { + if let Some(explicit_ids) = aggregate_subnet_ids { + subnets.extend(explicit_ids); + } + if subnets.is_empty() { + subnets.insert(0); + } + } + subnets +} + /// Build and configure the libp2p swarm, dial bootnodes, subscribe to topics. pub fn build_swarm( config: SwarmConfig, @@ -301,32 +331,23 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): - // every validator subscribes to its own subnet for mesh health; aggregators - // additionally subscribe to explicit `aggregate_subnet_ids` and fall back to - // subnet 0 when they have no validators of their own. - let validator_subnets: HashSet = config + // Subscribe to attestation subnets — see `compute_subscription_subnets`. + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = config .validator_ids .iter() .map(|vid| vid % config.attestation_committee_count) - .collect(); - - // The committee metric should reflect validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = validator_subnets.iter().copied().min().unwrap_or(0); + .min() + .unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let mut subscription_subnets = validator_subnets; - if config.is_aggregator { - if let Some(ref explicit_ids) = config.aggregate_subnet_ids { - subscription_subnets.extend(explicit_ids); - } - // Fall back to subnet 0 only when the aggregator has no validators - // and no explicit subnets — otherwise leave the set as configured. - if subscription_subnets.is_empty() { - subscription_subnets.insert(0); - } - } + let subscription_subnets = compute_subscription_subnets( + &config.validator_ids, + config.attestation_committee_count, + config.is_aggregator, + config.aggregate_subnet_ids.as_deref(), + ); let mut attestation_topics: HashMap = HashMap::new(); for &subnet_id in &subscription_subnets { From 1791da9310c91ac24b5f1540afb6ed194396935c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:33:22 -0300 Subject: [PATCH 07/29] feat(blockchain): add early-aggregation window and threshold helpers --- crates/blockchain/src/aggregation.rs | 32 +++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index f8387fcb..0eebd4e9 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -23,7 +23,7 @@ use spawned_concurrency::tasks::ActorRef; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; -use crate::metrics; +use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; /// Soft deadline for committee-signature aggregation measured from the /// interval-2 tick. After this much wall time elapses, the actor signals the @@ -36,6 +36,36 @@ pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); /// (mismatched timers, stuck proofs); we warn before blocking. pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); +/// Width of the early-aggregation window: a session may start at most this +/// long before the interval-2 boundary, provided the signature threshold is +/// met (see `early_threshold_met`). +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; + +/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the +/// normal aggregation start). +pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { + genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL +} + +/// If `now_ms` falls inside some slot's early-aggregation window +/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 +/// boundary), return that slot. +pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { + let since_genesis = now_ms.checked_sub(genesis_time_ms)?; + let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + let in_window = + ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; + in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) +} + +/// Early-start threshold: a single attestation-data group holds at least 2/3 +/// of the signatures expected from this node's aggregation subnets. At most +/// one group per slot can satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { + expected > 0 && max_group_count * 3 >= expected * 2 +} + /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread From 07d7425521645e257e0fd3c080c87554c9b2a657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:37:15 -0300 Subject: [PATCH 08/29] feat(blockchain): plumb aggregation subnet set and expected-signature count into actor --- bin/ethlambda/src/main.rs | 15 ++++++++++++++- crates/blockchain/src/lib.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 914c6c86..19db7eb5 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -37,7 +37,9 @@ use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; use ethlambda_network_api::{InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef}; -use ethlambda_p2p::{Bootnode, P2P, PeerId, SwarmConfig, build_swarm, parse_enrs}; +use ethlambda_p2p::{ + Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, +}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use ethlambda_types::{ aggregator::AggregatorController, @@ -210,11 +212,22 @@ async fn main() -> eyre::Result<()> { // and the API server (which exposes GET/POST admin endpoints). let aggregator = AggregatorController::new(options.is_aggregator); + // Same startup inputs build_swarm uses — single source of truth is the + // shared helper. Frozen at startup like the gossip subscriptions + // themselves (hot-standby model). + let aggregation_subnets = compute_subscription_subnets( + &validator_ids, + attestation_committee_count, + options.is_aggregator, + options.aggregate_subnet_ids.as_deref(), + ); + let blockchain = BlockChain::spawn( store.clone(), validator_keys, aggregator.clone(), attestation_committee_count, + aggregation_subnets, !options.disable_duty_sync_gate, ProposerConfig { enable_proposer_aggregation: options.enable_proposer_aggregation, diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 8be7c89f..9726a564 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -82,6 +82,7 @@ impl BlockChain { validator_keys: HashMap, aggregator: AggregatorController, attestation_committee_count: u64, + aggregation_subnets: HashSet, gate_duties: bool, proposer_config: ProposerConfig, ) -> BlockChain { @@ -98,6 +99,19 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); + // Denominator of the early-aggregation 2/3 threshold: how many + // validators attest on subnets this node aggregates. Computed once — + // the validator registry is static and `head_state()` clones the full + // state, so this must not run per gossip insert. + let validator_count = store.head_state().validators.len() as u64; + let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + 0 + } else { + (0..validator_count) + .filter(|vid| aggregation_subnets.contains(&(vid % attestation_committee_count))) + .count() + }; + let handle = BlockChainServer { store, p2p: None, @@ -108,6 +122,9 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, + genesis_time_ms: genesis_time * 1000, + early_aggregation_expected_sigs, + pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, sync_status: SyncStatusTracker::new(gate_duties), @@ -169,6 +186,21 @@ pub struct BlockChainServer { /// attestation aggregate coverage emission. attestation_committee_count: u64, + /// Genesis time in milliseconds, cached at spawn. `store.config()` is an + /// uncached backend read, too heavy for the per-gossip-insert early + /// checks that need this. + genesis_time_ms: u64, + + /// Number of validators whose subnet is one this node aggregates — the + /// denominator of the early-aggregation 2/3 threshold. Computed once at + /// spawn (the validator registry is static). + early_aggregation_expected_sigs: usize, + + /// Aggregates produced before the interval-2 boundary, held back so they + /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the + /// boundary; only ever holds the current slot's aggregates. + pending_aggregate_publishes: Vec, + /// Proposer-side block-building policy proposer_config: ProposerConfig, From ff1af32749384b379035fe22b09bd3befe7aa6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:05:34 -0300 Subject: [PATCH 09/29] docs(blockchain): note genesis_time_ms cache covers tick-path recomputations --- crates/blockchain/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 9726a564..ea5787f3 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -188,7 +188,9 @@ pub struct BlockChainServer { /// Genesis time in milliseconds, cached at spawn. `store.config()` is an /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. + /// checks that need this. The tick paths still recompute it from + /// `store.config()` — deliberately left alone; prefer this field in new + /// code. genesis_time_ms: u64, /// Number of validators whose subnet is one this node aggregates — the From 1466af03d0ede84b87a993c3d25c0facd33b2083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:06:57 -0300 Subject: [PATCH 10/29] feat(blockchain): extend aggregation deadline to a full interval --- crates/blockchain/src/aggregation.rs | 14 ++++++++------ crates/blockchain/src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 0eebd4e9..caf1b51f 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -25,12 +25,14 @@ use tracing::{info, warn}; use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; -/// Soft deadline for committee-signature aggregation measured from the -/// interval-2 tick. After this much wall time elapses, the actor signals the -/// worker to stop via its cancellation token. The 50 ms budget before the next -/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving -/// aggregates and for gossip propagation margin. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); +/// Soft deadline for committee-signature aggregation measured from session +/// start. After this much wall time elapses, the actor signals the worker to +/// stop via its cancellation token. A session started exactly at interval 2 +/// gets the full interval (interval 3 is one interval later); a session +/// started early (see `early_aggregation_slot`) ends correspondingly earlier. +/// The deadline only stops new jobs from starting — a job mid-proof finishes +/// and publishes right after. +pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); /// Upper bound we wait for a prior worker to exit if it is still running when /// the next session is about to start. Reached only in pathological cases /// (mismatched timers, stuck proofs); we warn before blocking. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index ea5787f3..c01b6700 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -396,7 +396,7 @@ impl BlockChainServer { /// 1. If a prior session is still running (pathological), warn and join it. /// 2. Snapshot the aggregation inputs from the store. /// 3. Spawn a `spawn_blocking` worker that streams results back as messages. - /// 4. Schedule the `AggregationDeadline` self-message at +750 ms. + /// 4. Schedule the `AggregationDeadline` self-message at +`AGGREGATION_DEADLINE`. async fn start_aggregation_session(&mut self, slot: u64, ctx: &Context) { if let Some(prior) = self.current_aggregation.take() { prior.cancel.cancel(); From 5e7fed514bb591728c8160156f91caeaabd8ca6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:10:28 -0300 Subject: [PATCH 11/29] feat(blockchain): hold early-produced aggregates until the interval-2 boundary --- crates/blockchain/src/aggregation.rs | 12 ++++ crates/blockchain/src/lib.rs | 89 ++++++++++++++++++++++++---- crates/blockchain/src/metrics.rs | 29 +++++++++ 3 files changed, 117 insertions(+), 13 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index caf1b51f..2efb57cc 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -108,6 +108,9 @@ pub(crate) struct AggregationSession { /// Slot at which this session was started; used as a fencing id so we can /// drop late-arriving messages from a prior session. pub(crate) session_id: u64, + /// Whether the session started before the slot's interval-2 boundary via + /// the early-aggregation trigger. + pub(crate) early: bool, /// Child of the actor cancellation token; fires either at the deadline or /// when the actor itself is stopping. pub(crate) cancel: CancellationToken, @@ -148,6 +151,15 @@ impl Message for AggregationDeadline { type Result = (); } +/// Self-message scheduled when a session starts early; fires at the +/// interval-2 boundary and publishes any aggregates held back by the +/// publish-alignment rule (aggregates must not reach gossip before +/// interval 2). +pub(crate) struct FlushAggregatePublishes; +impl Message for FlushAggregatePublishes { + type Result = (); +} + /// Build a snapshot of everything needed to aggregate. Runs on the actor /// thread, touches the store, does no heavy cryptography. Returns `None` when /// there is nothing to aggregate so callers can avoid spawning an empty worker. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index c01b6700..698a213b 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,7 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -345,10 +345,6 @@ impl BlockChainServer { // ==== interval 2 ==== 2 => { if is_aggregator { - coverage::emit_agg_start_new_coverage( - &self.store, - self.attestation_committee_count, - ); self.start_aggregation_session(slot, ctx).await; } else { metrics::inc_aggregator_skipped_not_aggregator(); @@ -417,6 +413,8 @@ impl BlockChainServer { } } + coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); + let Some(snapshot) = aggregation::snapshot_current_slot_aggregation_inputs(&self.store, slot) else { @@ -425,6 +423,35 @@ impl BlockChainServer { }; let session_id = slot; + // Any leftovers from a prior slot mean its flush never fired (a + // backwards wall-clock step, or a flush timer delayed past the next + // session). Drop them — late aggregates are dropped, same policy as + // signatures that miss the snapshot. + let stale = std::mem::take(&mut self.pending_aggregate_publishes); + if !stale.is_empty() { + warn!( + count = stale.len(), + "Dropping stale pending aggregate publishes" + ); + } + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); + let now_ms = unix_now_ms(); + let early = now_ms < t2_ms; + if early { + // Publish alignment: aggregates must not reach gossip before the + // interval-2 boundary. Aggregates produced before T2 are buffered + // in `pending_aggregate_publishes`; this timer flushes them at T2. + let lead = Duration::from_millis(t2_ms - now_ms); + metrics::inc_aggregation_early_starts(); + metrics::observe_aggregation_early_start_lead(lead); + info!( + %slot, + lead_ms = lead.as_millis() as u64, + "Starting aggregation session early" + ); + send_after(lead, ctx.clone(), FlushAggregatePublishes); + } + // Independent token per session. Shutdown propagates via our // #[stopped] hook which cancels any current session; the deadline // timer cancels this specific session at +AGGREGATION_DEADLINE. @@ -445,6 +472,7 @@ impl BlockChainServer { self.current_aggregation = Some(AggregationSession { session_id, + early, cancel, worker, }); @@ -960,6 +988,15 @@ impl BlockChainServer { } } + /// Publish an aggregated attestation to the aggregation gossip topic. + fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { + if let Some(ref p2p) = self.p2p { + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } + } + fn on_gossip_attestation(&mut self, attestation: &SignedAttestation) { // Read fresh here too: a gossip event can arrive between ticks, and // if the admin API just toggled, the first gossip after the toggle @@ -1098,14 +1135,35 @@ impl Handler for BlockChainServer { aggregation::apply_aggregated_group(&mut self.store, &msg.output); - if let Some(ref p2p) = self.p2p { - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + + // Publish alignment: hold back aggregates produced before this slot's + // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. + // (`session_id` is the session's slot.) + let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); + if unix_now_ms() < t2_ms { + self.pending_aggregate_publishes.push(aggregate); + return; + } + self.publish_aggregate(aggregate); + } +} + +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + let pending = std::mem::take(&mut self.pending_aggregate_publishes); + if pending.is_empty() { + return; + } + info!( + count = pending.len(), + "Publishing aggregates held back until the interval-2 boundary" + ); + for aggregate in pending { + self.publish_aggregate(aggregate); } } } @@ -1116,6 +1174,10 @@ impl Handler for BlockChainServer { metrics::observe_committee_signatures_aggregation(msg.total_elapsed); let aggregation_elapsed = msg.total_elapsed; + let early = self + .current_aggregation + .as_ref() + .is_some_and(|s| s.session_id == msg.session_id && s.early); info!( ?aggregation_elapsed, session_id = msg.session_id, @@ -1124,6 +1186,7 @@ impl Handler for BlockChainServer { total_raw_sigs = msg.total_raw_sigs, total_children = msg.total_children, cancelled = msg.cancelled, + early, aggregation_deadline_ms = AGGREGATION_DEADLINE.as_millis() as u64, "Committee signatures aggregated" ); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 28e510f5..d178e22c 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -263,6 +263,19 @@ static LEAN_PQ_SIG_ATTESTATION_SIGNATURES_INVALID_TOTAL: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_int_counter!( + "lean_aggregation_early_starts_total", + "Aggregation sessions started before the interval-2 boundary" + ) + .unwrap() + }); + +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} + // --- Histograms --- static LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS: std::sync::LazyLock = @@ -374,6 +387,20 @@ static LEAN_FORK_CHOICE_REORG_DEPTH: std::sync::LazyLock = .unwrap() }); +static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = + std::sync::LazyLock::new(|| { + register_histogram!( + "lean_aggregation_early_start_lead_seconds", + "How far before the interval-2 boundary an early aggregation session started", + vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + ) + .unwrap() + }); + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} + static LEAN_TICK_INTERVAL_DURATION_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_histogram!( @@ -590,6 +617,7 @@ pub fn init() { std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_TOTAL); std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_VALID_TOTAL); std::sync::LazyLock::force(&LEAN_PQ_SIG_ATTESTATION_SIGNATURES_INVALID_TOTAL); + std::sync::LazyLock::force(&LEAN_AGGREGATION_EARLY_STARTS_TOTAL); // Histograms std::sync::LazyLock::force(&LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS); std::sync::LazyLock::force(&LEAN_ATTESTATION_VALIDATION_TIME_SECONDS); @@ -601,6 +629,7 @@ pub fn init() { std::sync::LazyLock::force(&LEAN_COMMITTEE_SIGNATURES_AGGREGATION_TIME_SECONDS); std::sync::LazyLock::force(&LEAN_AGGREGATED_PROOF_SIZE_BYTES); std::sync::LazyLock::force(&LEAN_FORK_CHOICE_REORG_DEPTH); + std::sync::LazyLock::force(&LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS); std::sync::LazyLock::force(&LEAN_TICK_INTERVAL_DURATION_SECONDS); // Block production std::sync::LazyLock::force(&LEAN_BLOCK_AGGREGATED_PAYLOADS); From 88c40bcef468942f61e0b875ef49ba68a59ed810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:21:19 -0300 Subject: [PATCH 12/29] polish(blockchain): move early-agg metric helpers to Public API section, add log identity fields --- crates/blockchain/src/lib.rs | 3 +++ crates/blockchain/src/metrics.rs | 16 ++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 698a213b..f5714a44 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -430,6 +430,7 @@ impl BlockChainServer { let stale = std::mem::take(&mut self.pending_aggregate_publishes); if !stale.is_empty() { warn!( + %slot, count = stale.len(), "Dropping stale pending aggregate publishes" ); @@ -1158,7 +1159,9 @@ impl Handler for BlockChainServer { if pending.is_empty() { return; } + let session_id = self.current_aggregation.as_ref().map(|s| s.session_id); info!( + session_id, count = pending.len(), "Publishing aggregates held back until the interval-2 boundary" ); diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index d178e22c..11b63f2c 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -272,10 +272,6 @@ static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = .unwrap() }); -pub fn inc_aggregation_early_starts() { - LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); -} - // --- Histograms --- static LEAN_FORK_CHOICE_BLOCK_PROCESSING_TIME_SECONDS: std::sync::LazyLock = @@ -397,10 +393,6 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock .unwrap() }); -pub fn observe_aggregation_early_start_lead(lead: Duration) { - LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); -} - static LEAN_TICK_INTERVAL_DURATION_SECONDS: std::sync::LazyLock = std::sync::LazyLock::new(|| { register_histogram!( @@ -656,6 +648,14 @@ pub fn init() { // --- Public API --- +pub fn inc_aggregation_early_starts() { + LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); +} + +pub fn observe_aggregation_early_start_lead(lead: Duration) { + LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); +} + pub fn update_head_slot(slot: u64) { LEAN_HEAD_SLOT.set(slot.try_into().unwrap()); } From 76ed220affd96b636925896c75469af7751aab7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:24:13 -0300 Subject: [PATCH 13/29] feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived --- crates/blockchain/src/aggregation.rs | 9 ++++ crates/blockchain/src/lib.rs | 72 ++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 2efb57cc..46a11b03 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -160,6 +160,15 @@ impl Message for FlushAggregatePublishes { type Result = (); } +/// One-shot self-message scheduled at the interval-1 tick; fires when the +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// the threshold check for signatures that all arrived before the window. +/// Arrivals inside the window are checked per insert instead. +pub(crate) struct EarlyAggregationCheck; +impl Message for EarlyAggregationCheck { + type Result = (); +} + /// Build a snapshot of everything needed to aggregate. Runs on the actor /// thread, touches the store, does no heavy cryptography. Returns `None` when /// there is nothing to aggregate so callers can avoid spawning an empty worker. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index f5714a44..90e22ee5 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,7 +15,8 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, + FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -340,12 +341,34 @@ impl BlockChainServer { } else if !self.key_manager.validator_ids().is_empty() { info!(%slot, "Skipping attestations while syncing"); } + + // Schedule the early-aggregation window check. This tick is + // one interval before T2, so the timer fires right as the + // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + if is_aggregator { + send_after( + Duration::from_millis( + MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, + ), + ctx.clone(), + EarlyAggregationCheck, + ); + } } // ==== interval 2 ==== 2 => { if is_aggregator { - self.start_aggregation_session(slot, ctx).await; + // The early trigger may have already started this slot's + // session (running or finished) — it IS the slot's session, + // so don't start a second one. + let already_started = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot); + if !already_started { + self.start_aggregation_session(slot, ctx).await; + } } else { metrics::inc_aggregator_skipped_not_aggregator(); } @@ -479,6 +502,42 @@ impl BlockChainServer { }); } + /// Early-aggregation trigger: start the slot's session ahead of the + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// a single attestation-data group already holds 2/3 of the signatures + /// expected from this node's aggregation subnets. Called after every + /// stored gossip signature and once at the window opening via + /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started + /// session stays in `current_aggregation` (running or finished) until the + /// next session replaces it. + async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { + if !self.aggregator.is_enabled() { + return; + } + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) + else { + return; + }; + if self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == slot) + { + return; + } + let max_group = self.store.max_gossip_group_count_for_slot(slot); + if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + return; + } + info!( + %slot, + max_group, + expected = self.early_aggregation_expected_sigs, + "Early-aggregation threshold met" + ); + self.start_aggregation_session(slot, ctx).await; + } + /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1104,8 +1163,9 @@ impl Handler for BlockChainServer { } impl Handler for BlockChainServer { - async fn handle(&mut self, msg: NewAttestation, _ctx: &Context) { + async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { self.on_gossip_attestation(&msg.attestation); + self.maybe_start_early_aggregation(ctx).await; } } @@ -1171,6 +1231,12 @@ impl Handler for BlockChainServer { } } +impl Handler for BlockChainServer { + async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { + self.maybe_start_early_aggregation(ctx).await; + } +} + impl Handler for BlockChainServer { async fn handle(&mut self, msg: AggregationDone, _ctx: &Context) { aggregation::finalize_aggregation_session(&self.store); From d57248f03bc4dc53ccaf9e6293cb3eebf24626a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:31:58 -0300 Subject: [PATCH 14/29] docs(blockchain): note the once-per-slot latch hole for unresolvable pubkeys --- crates/blockchain/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 90e22ee5..470c7ff4 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -509,7 +509,10 @@ impl BlockChainServer { /// stored gossip signature and once at the window opening via /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started /// session stays in `current_aggregation` (running or finished) until the - /// next session replaces it. + /// next session replaces it. The latch has one hole: if the snapshot + /// yields no jobs (possible only when no signer's pubkey resolves, i.e. a + /// corrupted validator registry), no session is installed and the check + /// retries on later inserts — each retry is a no-op session attempt. async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { if !self.aggregator.is_enabled() { return; From 235d97fa19f3a84ee3a60fdf1e2fb81a9e4f6aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:50:43 -0300 Subject: [PATCH 15/29] docs: fix stale session-lifecycle comments flagged by final review --- crates/blockchain/src/aggregation.rs | 6 ++++-- crates/blockchain/src/lib.rs | 8 +++++--- ...2026-07-01-early-aggregation-start-design.md | 17 ++++++++--------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 46a11b03..d5384290 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -1,7 +1,9 @@ //! Committee-signature aggregation: off-thread worker orchestration and the //! pure functions it runs. //! -//! The blockchain actor fires one aggregation session per interval 2 via +//! The blockchain actor fires one aggregation session per slot — at interval 2, +//! or up to [`EARLY_AGGREGATION_WINDOW_MS`] early when the 2/3 signature +//! threshold is met — via //! [`run_aggregation_worker`]. The actor stays on its message loop; the worker //! runs the expensive XMSS proofs on a `spawn_blocking` thread and streams //! results back as [`AggregateProduced`] / [`AggregationDone`] messages. @@ -142,7 +144,7 @@ impl Message for AggregationDone { type Result = (); } -/// Self-message scheduled via `send_after` at interval-2 start. Cancels the +/// Self-message scheduled via `send_after` at session start. Cancels the /// session's token so the worker stops starting new aggregations. pub(crate) struct AggregationDeadline { pub(crate) session_id: u64, diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 470c7ff4..edda6c2c 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -175,9 +175,11 @@ pub struct BlockChainServer { /// `--is-aggregator` flag at spawn. aggregator: AggregatorController, - /// In-flight committee-signature aggregation, if any. Present only while a - /// worker started at the most recent interval 2 is still running or until - /// the next interval 2 takes over. + /// The slot's one committee-signature aggregation session (started at + /// interval 2, or early via the 2/3 trigger). Deliberately persists after + /// the worker finishes — that persistence is the once-per-slot latch the + /// early trigger and the interval-2 skip both check — until the next + /// session start replaces it. current_aggregation: Option, /// Last tick instant for measuring interval duration. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index 082ef330..e9ca5ddb 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -162,15 +162,14 @@ model (runtime toggles do not resubscribe subnets). ## Testing -- Unit tests for the pure trigger decision (window bounds, threshold math, - `expected` derivation from subnet set + validator count, once-per-slot guard). -- Unit tests for `compute_subscription_subnets` (validator subnets, explicit - ids, fallback-to-0, non-aggregator). -- Unit test for the publish-delay decision (before/after T2). -- Existing aggregation tests must pass unchanged (normal interval-2 path is the - fallback and keeps its semantics; only the deadline constant changes). -- Devnet validation via `test-branch.sh`: compare aggregate publish offsets - (per-slot duty timing method) and `lean_aggregation_early_starts_total`. +Per user decision, no new unit tests. Validation is: + +- Existing suites pass unchanged (normal interval-2 path is the fallback and + keeps its semantics; only the deadline constant changes). +- Local 4-node devnet run (1 aggregator, binary mode): early-start rate and + lead-time distribution from logs, held-back publish flushes landing at or + after the interval-2 boundary (per-slot offset), lockstep finalization, no + aggregation warnings/errors. ## Non-goals From d9fe68a0bebbab18418b808cb82e86418bd542b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:12:33 -0300 Subject: [PATCH 16/29] fix(blockchain): fence FlushAggregatePublishes by slot; guard subnet div-by-zero and window underflow Addresses PR #487 review feedback: - FlushAggregatePublishes now carries its session slot and drains only when it matches current_aggregation, matching AggregationDeadline's fencing. A flush timer delayed past the next session start can no longer publish the newer session's buffered aggregates before its interval-2 boundary. - compute_subscription_subnets returns an empty set for a zero committee count instead of dividing by zero (defensive; callers enforce >= 1). - const assert that EARLY_AGGREGATION_WINDOW_MS fits within one interval, making the no-underflow invariant self-enforcing. --- crates/blockchain/src/aggregation.rs | 18 ++++++++++++++++-- crates/blockchain/src/lib.rs | 20 ++++++++++++++++---- crates/net/p2p/src/lib.rs | 16 ++++++++++++---- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index d5384290..7f77e43a 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -45,6 +45,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// met (see `early_threshold_met`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +// The window must fit within one interval: `early_aggregation_slot` subtracts +// it from the interval-2 offset, and the interval-1 tick schedules the check +// at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep this +// invariant self-enforcing so a future bump to the window can't silently +// underflow either subtraction. +const _: () = assert!( + EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, + "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" +); + /// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the /// normal aggregation start). pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { @@ -156,8 +166,12 @@ impl Message for AggregationDeadline { /// Self-message scheduled when a session starts early; fires at the /// interval-2 boundary and publishes any aggregates held back by the /// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). -pub(crate) struct FlushAggregatePublishes; +/// interval 2). Carries the session slot so a timer delayed past the next +/// session start is fenced (like [`AggregationDeadline`]) and cannot flush +/// the newer session's buffer before its own boundary. +pub(crate) struct FlushAggregatePublishes { + pub(crate) slot: u64, +} impl Message for FlushAggregatePublishes { type Result = (); } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index edda6c2c..4de5bace 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -475,7 +475,7 @@ impl BlockChainServer { lead_ms = lead.as_millis() as u64, "Starting aggregation session early" ); - send_after(lead, ctx.clone(), FlushAggregatePublishes); + send_after(lead, ctx.clone(), FlushAggregatePublishes { slot }); } // Independent token per session. Shutdown propagates via our @@ -1219,14 +1219,26 @@ impl Handler for BlockChainServer { } impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { + async fn handle(&mut self, msg: FlushAggregatePublishes, _ctx: &Context) { + // Fence like `AggregationDeadline`: only the current session's flush + // drains the buffer. A flush timer delayed past the next session start + // would otherwise publish that newer session's buffered aggregates + // before its own interval-2 boundary, breaking publish alignment. The + // superseded session's own aggregates were already dropped by the + // stale-drain in `start_aggregation_session`. + let is_current = self + .current_aggregation + .as_ref() + .is_some_and(|session| session.session_id == msg.slot); + if !is_current { + return; + } let pending = std::mem::take(&mut self.pending_aggregate_publishes); if pending.is_empty() { return; } - let session_id = self.current_aggregation.as_ref().map(|s| s.session_id); info!( - session_id, + slot = msg.slot, count = pending.len(), "Publishing aggregates held back until the interval-2 boundary" ); diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 4308e130..add89fcc 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -191,16 +191,24 @@ pub struct BuiltSwarm { /// /// Evaluated once at startup — runtime aggregator toggles do not resubscribe /// (hot-standby model); see the invariant note on [`SwarmConfig`]. +/// +/// A zero `attestation_committee_count` yields no validator subnets (rather +/// than dividing by zero); callers enforce `>= 1`, so this is a defensive +/// guard on the public API, not a reachable path in the binary. pub fn compute_subscription_subnets( validator_ids: &[u64], attestation_committee_count: u64, is_aggregator: bool, aggregate_subnet_ids: Option<&[u64]>, ) -> HashSet { - let mut subnets: HashSet = validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect(); + let mut subnets: HashSet = if attestation_committee_count == 0 { + HashSet::new() + } else { + validator_ids + .iter() + .map(|vid| vid % attestation_committee_count) + .collect() + }; if is_aggregator { if let Some(explicit_ids) = aggregate_subnet_ids { subnets.extend(explicit_ids); From 98fa2ea2a5f1b0fe4fccf782ca77e4706ca2dcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:34:17 -0300 Subject: [PATCH 17/29] feat(blockchain): narrow early-aggregation window to 200ms Halve EARLY_AGGREGATION_WINDOW_MS (400 -> 200). Rescale the lean_aggregation_early_start_lead_seconds histogram buckets to 0.025-0.2s so they keep full resolution over the new window, and update the spec/plan (also fixing an inverted deadline-bound phrasing). --- crates/blockchain/src/aggregation.rs | 2 +- crates/blockchain/src/metrics.rs | 2 +- .../plans/2026-07-01-early-aggregation-start.md | 12 ++++++------ .../2026-07-01-early-aggregation-start-design.md | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 7f77e43a..4124d2c7 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -43,7 +43,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; // The window must fit within one interval: `early_aggregation_slot` subtracts // it from the interval-2 offset, and the interval-1 tick schedules the check diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 11b63f2c..76289723 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -388,7 +388,7 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock register_histogram!( "lean_aggregation_early_start_lead_seconds", "How far before the interval-2 boundary an early aggregation session started", - vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] + vec![0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2] ) .unwrap() }); diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md index f20dafd5..575e2587 100644 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md @@ -2,9 +2,9 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Start the committee-signature aggregation session up to 400 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. +**Goal:** Start the committee-signature aggregation session up to 200 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. -**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 400 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. +**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 200 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. **Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. @@ -26,8 +26,8 @@ slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms |------------|------------|------------|------------| ^ attest ^ aggregate ^ safe tgt ^ accept/build - [T2-400, T2) = early window - ^ timer check (scheduled at T1 + 400ms) + [T2-200, T2) = early window + ^ timer check (scheduled at T1 + 600ms) ^..^..^ per-insert checks as gossip sigs arrive ``` @@ -179,7 +179,7 @@ In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~l /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 400; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; /// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the /// normal aggregation start). @@ -814,7 +814,7 @@ Run a local 4-node devnet on this branch and verify the early-start behavior end Check the aggregator node's logs and metrics: -1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 400]`. +1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 200]`. 2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. 3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). 4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md index e9ca5ddb..77b7fc2b 100644 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md @@ -8,7 +8,7 @@ Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the attestation pipeline. Today the aggregation session starts exactly at the interval-2 tick, even when all expected signatures arrived earlier. Starting the -session up to 400 ms early, when enough signatures are already present, gives the +session up to 200 ms early, when enough signatures are already present, gives the expensive proofs more wall-clock runway before block building consumes them at interval 4. @@ -36,7 +36,7 @@ interval 4. Aggregator-only, fires at most once per slot. -- **Window:** wall clock in `[T2 - 400 ms, T2)`, where +- **Window:** wall clock in `[T2 - 200 ms, T2)`, where `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` is the current slot's interval-2 boundary. - **Condition:** some current-slot attestation-data group has gossip signatures @@ -56,7 +56,7 @@ Aggregator-only, fires at most once per slot. 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is recomputed from the wall clock at fire time, so a late-fired timer can never act on a stale slot) scheduled at the interval-1 tick via - `send_after(400 ms)` (interval-1 tick + 400 ms = T2 - 400 ms), covering + `send_after(200 ms)` (interval-1 tick + 600 ms = T2 - 200 ms), covering the case where the threshold was already met before the window opened. If the interval-1 tick is skipped (overrun), the per-insert checks still apply. @@ -109,10 +109,10 @@ pattern: build early, publish aligned to the boundary). `AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. -- Early session started at `T2 - x` (x ≤ 400): deadline at `T2 - x + 800`, - i.e. at most `T2 + 400`. +- Early session started at `T2 - x` (0 < x ≤ 200): deadline at `T2 - x + 800`, + i.e. no earlier than `T2 + 600` (200 ms before the interval-3 boundary). - Normal session started at `T2`: deadline lands exactly on the interval-3 - boundary. This removes the previous 50 ms publish/propagation margin — + boundary. This removes the previous 50 ms publish/propagation margin; accepted trade-off (the deadline only stops *new* jobs from starting; a job mid-proof finishes and publishes slightly after). @@ -156,7 +156,7 @@ model (runtime toggles do not resubscribe subnets). - Counter `lean_aggregation_early_starts_total`: early sessions started. - Histogram for early-start lead time (`T2 - start`, seconds, buckets covering - 0–400 ms). + 0–200 ms). - Existing session logs gain an `early: bool` field so devnet log analysis can attribute publish-time shifts. From 50adb83e06ba463c533bae48a33801c3140f4fc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:03:56 -0300 Subject: [PATCH 18/29] chore: drop internal design/plan docs from the PR --- .../2026-07-01-early-aggregation-start.md | 832 ------------------ ...26-07-01-early-aggregation-start-design.md | 181 ---- 2 files changed, 1013 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-01-early-aggregation-start.md delete mode 100644 docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md diff --git a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md b/docs/superpowers/plans/2026-07-01-early-aggregation-start.md deleted file mode 100644 index 575e2587..00000000 --- a/docs/superpowers/plans/2026-07-01-early-aggregation-start.md +++ /dev/null @@ -1,832 +0,0 @@ -# Early Aggregation Start Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Start the committee-signature aggregation session up to 200 ms before the interval-2 boundary when a single attestation-data group already holds 2/3 of the signatures expected from this node's aggregation subnets, while holding back any early-produced aggregates from gossip until the boundary. - -**Architecture:** The blockchain actor (`BlockChainServer`, GenServer pattern) gains an early-trigger check invoked from two sites: after every stored gossip signature (`NewAttestation` handler) and once via a timer fired at the window opening (`T2 − 200 ms`). The trigger calls the existing `start_aggregation_session`, which becomes the single session-start path for both early and normal (interval-2 tick) starts; the tick skips if the slot's session already exists. Aggregates produced before T2 are buffered on the actor and flushed by a `send_after` timer at T2. - -**Tech Stack:** Rust (edition 2024), spawned-concurrency actors (`send_after` self-messages), prometheus metrics via `ethlambda_metrics`. - -**Verification policy (per user):** No new unit tests. Each task verifies by compiling (plus existing test suites where they already exist); final validation is a local 4-node devnet run (Task 9). - -**Spec:** `docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md` - -**Key existing code:** -- `crates/blockchain/src/lib.rs` — actor, tick loop (`on_tick`), `start_aggregation_session` (~line 366), handlers (`NewAttestation` ~line 1034, `AggregateProduced` ~line 1050, `AggregationDone` ~line 1079, `AggregationDeadline` ~line 1099) -- `crates/blockchain/src/aggregation.rs` — session types, snapshot builders, worker, `AGGREGATION_DEADLINE` -- `crates/blockchain/src/metrics.rs` — prometheus registration patterns -- `crates/storage/src/store.rs` — `GossipSignatureBuffer` (~line 348), `Store` methods (~line 1451) -- `crates/net/p2p/src/lib.rs` — `build_swarm` subnet subscription (~lines 304–337) -- `bin/ethlambda/src/main.rs` — `BlockChain::spawn` (~line 213), `build_swarm(SwarmConfig {...})` (~line 230) - -**Timing model (4 s slots, 5 × 800 ms intervals):** - -``` -slot start T1=+800ms T2=+1600ms T3=+2400ms T4=+3200ms - |------------|------------|------------|------------| - ^ attest ^ aggregate ^ safe tgt ^ accept/build - [T2-200, T2) = early window - ^ timer check (scheduled at T1 + 600ms) - ^..^..^ per-insert checks as gossip sigs arrive -``` - ---- - -### Task 1: Store query — max gossip group count for a slot - -The early threshold needs "largest signature count among current-slot data groups" without cloning signatures (the existing `iter_gossip_signatures` snapshot clones every signature; signatures are ~3 KB each). - -**Files:** -- Modify: `crates/storage/src/store.rs` (buffer method after `snapshot()` ~line 465; `Store` method next to `gossip_signatures_count()` ~line 1451) - -- [ ] **Step 1: Implement the buffer method and Store wrapper** - -In `impl GossipSignatureBuffer`, after `snapshot()` (~line 465): - -```rust -/// Largest signature count among data groups whose attestation slot is `slot`. -fn max_group_count_for_slot(&self, slot: u64) -> usize { - self.data - .values() - .filter(|entry| entry.data.slot == slot) - .map(|entry| entry.signatures.len()) - .max() - .unwrap_or(0) -} -``` - -In `impl Store`, next to `gossip_signatures_count()` (~line 1451): - -```rust -/// Largest per-group signature count among gossip groups voting for `slot`. -/// -/// One lock, no signature clones — cheap enough to call per gossip insert. -/// Drives the early-aggregation threshold check. -pub fn max_gossip_group_count_for_slot(&self, slot: u64) -> usize { - let gossip = self.gossip_signatures.lock().unwrap(); - gossip.max_group_count_for_slot(slot) -} -``` - -- [ ] **Step 2: Build** - -Run: `cargo build -p ethlambda-storage` -Expected: clean build, no warnings (the `pub` Store method keeps the private buffer method alive). - -- [ ] **Step 3: Commit** - -```bash -git add crates/storage/src/store.rs -git commit -m "feat(storage): add max gossip group count query for early aggregation" -``` - ---- - -### Task 2: p2p — extract `compute_subscription_subnets` helper - -Single source of truth for the subnet set, callable from `main.rs` (Task 4) without touching `SwarmConfig`. - -**Files:** -- Modify: `crates/net/p2p/src/lib.rs` (subnet logic in `build_swarm` ~lines 304–337) - -- [ ] **Step 1: Implement the helper and use it in `build_swarm`** - -Add near `build_swarm` (above it, ~line 185): - -```rust -/// Compute the set of attestation subnets this node subscribes to (and, when -/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): -/// every validator subscribes to its own subnet (`vid % committee_count`) for -/// mesh health; aggregators additionally subscribe to explicit -/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would -/// otherwise be empty. -/// -/// Evaluated once at startup — runtime aggregator toggles do not resubscribe -/// (hot-standby model); see the invariant note on [`SwarmConfig`]. -pub fn compute_subscription_subnets( - validator_ids: &[u64], - attestation_committee_count: u64, - is_aggregator: bool, - aggregate_subnet_ids: Option<&[u64]>, -) -> HashSet { - let mut subnets: HashSet = validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect(); - if is_aggregator { - if let Some(explicit_ids) = aggregate_subnet_ids { - subnets.extend(explicit_ids); - } - if subnets.is_empty() { - subnets.insert(0); - } - } - subnets -} -``` - -Then replace the inline logic in `build_swarm` (~lines 304–329). The metric must keep reflecting validator-only subnets, so that part stays: - -```rust - // Subscribe to attestation subnets — see `compute_subscription_subnets`. - // The committee metric should reflect validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = config - .validator_ids - .iter() - .map(|vid| vid % config.attestation_committee_count) - .min() - .unwrap_or(0); - metrics::set_attestation_committee_subnet(metric_subnet); - - let subscription_subnets = compute_subscription_subnets( - &config.validator_ids, - config.attestation_committee_count, - config.is_aggregator, - config.aggregate_subnet_ids.as_deref(), - ); -``` - -(The `let mut attestation_topics ...` loop over `&subscription_subnets` at ~line 331 stays unchanged.) - -- [ ] **Step 2: Build and run existing p2p tests** - -Run: `cargo build -p ethlambda-p2p && cargo test -p ethlambda-p2p --lib` -Expected: clean build, existing tests pass (behavior of `build_swarm` is unchanged — same set, same subscriptions). - -- [ ] **Step 3: Commit** - -```bash -git add crates/net/p2p/src/lib.rs -git commit -m "refactor(p2p): extract subscription subnet computation into pure helper" -``` - ---- - -### Task 3: Blockchain — pure early-aggregation helpers - -Window math and threshold predicate. - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new consts + functions after `PRIOR_WORKER_JOIN_TIMEOUT` ~line 37) - -- [ ] **Step 1: Implement consts and functions** - -In `crates/blockchain/src/aggregation.rs`, after `PRIOR_WORKER_JOIN_TIMEOUT` (~line 37). Also add `use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT};` to the file's imports (both constants are `pub` in `lib.rs`). - -```rust -/// Width of the early-aggregation window: a session may start at most this -/// long before the interval-2 boundary, provided the signature threshold is -/// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; - -/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the -/// normal aggregation start). -pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { - genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL -} - -/// If `now_ms` falls inside some slot's early-aggregation window -/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 -/// boundary), return that slot. -pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { - let since_genesis = now_ms.checked_sub(genesis_time_ms)?; - let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; - let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - let in_window = - ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; - in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) -} - -/// Early-start threshold: a single attestation-data group holds at least 2/3 -/// of the signatures expected from this node's aggregation subnets. At most -/// one group per slot can satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { - expected > 0 && max_group_count * 3 >= expected * 2 -} -``` - -- [ ] **Step 2: Build** - -Run: `cargo build -p ethlambda-blockchain` -Expected: clean build (dead-code warnings possible until Tasks 6–7 wire the callers). - -- [ ] **Step 3: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs -git commit -m "feat(blockchain): add early-aggregation window and threshold helpers" -``` - ---- - -### Task 4: Plumbing — subnet set into the actor, expected-count field - -Compile-level wiring, no behavior change yet. - -**Files:** -- Modify: `crates/blockchain/src/lib.rs` (`BlockChain::spawn` ~line 80, `BlockChainServer` struct ~line 138) -- Modify: `bin/ethlambda/src/main.rs` (~lines 206–223) - -- [ ] **Step 1: Add spawn parameter and fields** - -In `crates/blockchain/src/lib.rs`, change `BlockChain::spawn` signature (~line 80) — new `aggregation_subnets` parameter after `attestation_committee_count`: - -```rust - pub fn spawn( - store: Store, - validator_keys: HashMap, - aggregator: AggregatorController, - attestation_committee_count: u64, - aggregation_subnets: HashSet, - gate_duties: bool, - proposer_config: ProposerConfig, - ) -> BlockChain { -``` - -Inside `spawn`, before the `let handle = BlockChainServer {` block (~line 101), compute the expected count (guard the modulo against a zero committee count): - -```rust - // Denominator of the early-aggregation 2/3 threshold: how many - // validators attest on subnets this node aggregates. Computed once — - // the validator registry is static and `head_state()` clones the full - // state, so this must not run per gossip insert. - let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_expected_sigs = if attestation_committee_count == 0 { - 0 - } else { - (0..validator_count) - .filter(|vid| { - aggregation_subnets.contains(&(vid % attestation_committee_count)) - }) - .count() - }; -``` - -Add to the `BlockChainServer` struct literal in `spawn` (`genesis_time` is already computed at ~line 90): - -```rust - genesis_time_ms: genesis_time * 1000, - early_aggregation_expected_sigs, - pending_aggregate_publishes: Vec::new(), -``` - -Add the fields to the `BlockChainServer` struct definition (after `attestation_committee_count`, ~line 170): - -```rust - /// Genesis time in milliseconds, cached at spawn. `store.config()` is an - /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. - genesis_time_ms: u64, - - /// Number of validators whose subnet is one this node aggregates — the - /// denominator of the early-aggregation 2/3 threshold. Computed once at - /// spawn (the validator registry is static). - early_aggregation_expected_sigs: usize, - - /// Aggregates produced before the interval-2 boundary, held back so they - /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the - /// boundary; only ever holds the current slot's aggregates. - pending_aggregate_publishes: Vec, -``` - -(`HashSet` and `SignedAggregatedAttestation` are already imported in `lib.rs`. Existing `config().genesis_time * 1000` call sites in `on_tick`/`handle_tick` stay as they are — only the new code paths use the cached field.) - -- [ ] **Step 2: Update `main.rs`** - -In `bin/ethlambda/src/main.rs`, add `compute_subscription_subnets` to the `ethlambda_p2p` import (line 40): - -```rust -use ethlambda_p2p::{ - Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, -}; -``` - -Before `BlockChain::spawn` (~line 213), compute the set with the same startup inputs `build_swarm` uses, and pass it (borrow `aggregate_subnet_ids` — it is moved into `SwarmConfig` further down): - -```rust - // Same startup inputs build_swarm uses — single source of truth is the - // shared helper. Frozen at startup like the gossip subscriptions - // themselves (hot-standby model). - let aggregation_subnets = compute_subscription_subnets( - &validator_ids, - attestation_committee_count, - options.is_aggregator, - options.aggregate_subnet_ids.as_deref(), - ); - - let blockchain = BlockChain::spawn( - store.clone(), - validator_keys, - aggregator.clone(), - attestation_committee_count, - aggregation_subnets, - !options.disable_duty_sync_gate, - ProposerConfig { - enable_proposer_aggregation: options.enable_proposer_aggregation, - max_attestations_per_block: options.max_attestations_per_block, - }, - ); -``` - -- [ ] **Step 3: Build the workspace** - -Run: `cargo build --workspace` -Expected: clean build. Dead-code warnings on the two new fields are acceptable here (plain `cargo build` does not deny warnings); they become live in Tasks 6–7, before `make lint` runs in Task 8. Do not annotate with `#[allow(dead_code)]`. - -- [ ] **Step 4: Commit** - -```bash -git add crates/blockchain/src/lib.rs bin/ethlambda/src/main.rs -git commit -m "feat(blockchain): plumb aggregation subnet set and expected-signature count into actor" -``` - ---- - -### Task 5: Deadline change 750 → 800 ms - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs:28-33` -- Modify: `crates/blockchain/src/lib.rs` (stale `+750 ms` doc-comment reference on `start_aggregation_session`, ~line 365) - -- [ ] **Step 1: Change the constant and its comment** - -Replace: - -```rust -/// Soft deadline for committee-signature aggregation measured from the -/// interval-2 tick. After this much wall time elapses, the actor signals the -/// worker to stop via its cancellation token. The 50 ms budget before the next -/// interval (interval 3 at +800 ms) is reserved for publishing any late-arriving -/// aggregates and for gossip propagation margin. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(750); -``` - -with: - -```rust -/// Soft deadline for committee-signature aggregation measured from session -/// start. After this much wall time elapses, the actor signals the worker to -/// stop via its cancellation token. A session started exactly at interval 2 -/// gets the full interval (interval 3 is one interval later); a session -/// started early (see `early_aggregation_slot`) ends correspondingly earlier. -/// The deadline only stops new jobs from starting — a job mid-proof finishes -/// and publishes right after. -pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); -``` - -- [ ] **Step 2: Build and check references** - -Run: `cargo build -p ethlambda-blockchain && grep -rn "750" crates/blockchain/src/lib.rs` -Expected: clean build; the grep finds no stale `+750 ms` comment references (the doc comment on `start_aggregation_session` ~line 365 says "Schedule the `AggregationDeadline` self-message at +750 ms" — update it to "+`AGGREGATION_DEADLINE`"). - -- [ ] **Step 3: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs -git commit -m "feat(blockchain): extend aggregation deadline to a full interval" -``` - ---- - -### Task 6: Publish alignment — hold early aggregates until the interval-2 boundary - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (`AggregationSession` struct ~line 75, new message) -- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, `start_aggregation_session` ~line 366, `AggregateProduced` handler ~line 1050, `AggregationDone` handler ~line 1079, interval-2 tick arm ~line 312, new handler + helper) -- Modify: `crates/blockchain/src/metrics.rs` (new counter + histogram) - -- [ ] **Step 1: Add the `early` flag and the flush message in `aggregation.rs`** - -Add field to `AggregationSession` (~line 75): - -```rust -pub(crate) struct AggregationSession { - /// Slot at which this session was started; used as a fencing id so we can - /// drop late-arriving messages from a prior session. - pub(crate) session_id: u64, - /// Whether the session started before the slot's interval-2 boundary via - /// the early-aggregation trigger. - pub(crate) early: bool, - /// Child of the actor cancellation token; fires either at the deadline or - /// when the actor itself is stopping. - pub(crate) cancel: CancellationToken, - /// Handle to the `spawn_blocking` worker. Held so `stopped()` / new-session - /// start can await completion. - pub(crate) worker: tokio::task::JoinHandle<()>, -} -``` - -Add next to `AggregationDeadline` (~line 112): - -```rust -/// Self-message scheduled when a session starts early; fires at the -/// interval-2 boundary and publishes any aggregates held back by the -/// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). -pub(crate) struct FlushAggregatePublishes; -impl Message for FlushAggregatePublishes { - type Result = (); -} -``` - -- [ ] **Step 2: Rework `start_aggregation_session` in `lib.rs`** - -Extend the `use crate::aggregation::{...}` import (~line 16) with `FlushAggregatePublishes`. - -In `start_aggregation_session` (~line 366): move the coverage emission here from the tick arm, and compute `early` + schedule the flush. After the prior-session join block and before the snapshot: - -```rust - coverage::emit_agg_start_new_coverage(&self.store, self.attestation_committee_count); -``` - -Replace the tail of the function (from `let session_id = slot;` through the final `self.current_aggregation = Some(...)`) with: - -```rust - let session_id = slot; - // Any leftovers from a prior slot mean its flush never fired (a - // backwards wall-clock step, or a flush timer delayed past the next - // session). Drop them — late aggregates are dropped, same policy as - // signatures that miss the snapshot. - let stale = std::mem::take(&mut self.pending_aggregate_publishes); - if !stale.is_empty() { - warn!( - count = stale.len(), - "Dropping stale pending aggregate publishes" - ); - } - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); - let now_ms = unix_now_ms(); - let early = now_ms < t2_ms; - if early { - // Publish alignment: aggregates must not reach gossip before the - // interval-2 boundary. Aggregates produced before T2 are buffered - // in `pending_aggregate_publishes`; this timer flushes them at T2. - let lead = Duration::from_millis(t2_ms - now_ms); - metrics::inc_aggregation_early_starts(); - metrics::observe_aggregation_early_start_lead(lead); - info!( - %slot, - lead_ms = lead.as_millis() as u64, - "Starting aggregation session early" - ); - send_after(lead, ctx.clone(), FlushAggregatePublishes); - } - - // Independent token per session. Shutdown propagates via our - // #[stopped] hook which cancels any current session; the deadline - // timer cancels this specific session at +AGGREGATION_DEADLINE. - let cancel = CancellationToken::new(); - let actor_ref = ctx.actor_ref(); - - let worker_cancel = cancel.clone(); - let worker_actor = actor_ref.clone(); - let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); - }); - - let _deadline_timer = send_after( - AGGREGATION_DEADLINE, - ctx.clone(), - AggregationDeadline { session_id }, - ); - - self.current_aggregation = Some(AggregationSession { - session_id, - early, - cancel, - worker, - }); -``` - -(`metrics::inc_aggregation_early_starts` / `observe_aggregation_early_start_lead` are added in Step 4 of this task.) - -Remove the emission from the interval-2 tick arm (~line 312), which becomes: - -```rust - // ==== interval 2 ==== - 2 => { - if is_aggregator { - self.start_aggregation_session(slot, ctx).await; - } else { - metrics::inc_aggregator_skipped_not_aggregator(); - } - } -``` - -(The `coverage::emit_agg_start_new_coverage` call and its old surrounding braces go away; the early-session skip guard lands in Task 7.) - -- [ ] **Step 3: Split publish out of `AggregateProduced` and add the flush handler** - -Add a helper method on `BlockChainServer` (near `on_gossip_attestation`, ~line 929): - -```rust - /// Publish an aggregated attestation to the aggregation gossip topic. - fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { - if let Some(ref p2p) = self.p2p { - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); - } - } -``` - -Rewrite the body of `impl Handler` (~line 1050) — the session-id fencing stays, then: - -```rust - aggregation::apply_aggregated_group(&mut self.store, &msg.output); - - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - - // Publish alignment: hold back aggregates produced before this slot's - // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. - // (`session_id` is the session's slot.) - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); - if unix_now_ms() < t2_ms { - self.pending_aggregate_publishes.push(aggregate); - return; - } - self.publish_aggregate(aggregate); -``` - -Add the flush handler next to the other aggregation handlers (~line 1099): - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: FlushAggregatePublishes, _ctx: &Context) { - let pending = std::mem::take(&mut self.pending_aggregate_publishes); - if pending.is_empty() { - return; - } - info!( - count = pending.len(), - "Publishing aggregates held back until the interval-2 boundary" - ); - for aggregate in pending { - self.publish_aggregate(aggregate); - } - } -} -``` - -Add the `early` field to the `AggregationDone` completion log (~line 1085): before the `info!`, compute - -```rust - let early = self - .current_aggregation - .as_ref() - .is_some_and(|s| s.session_id == msg.session_id && s.early); -``` - -and add `early,` to the `info!(...)` field list (after `cancelled = msg.cancelled,`). - -- [ ] **Step 4: Add the two metrics** - -In `crates/blockchain/src/metrics.rs` — counter in the `// --- Counters ---` section, histogram in `// --- Histograms ---`, public fns near their statics following file style: - -```rust -static LEAN_AGGREGATION_EARLY_STARTS_TOTAL: std::sync::LazyLock = - std::sync::LazyLock::new(|| { - register_int_counter!( - "lean_aggregation_early_starts_total", - "Aggregation sessions started before the interval-2 boundary" - ) - .unwrap() - }); - -pub fn inc_aggregation_early_starts() { - LEAN_AGGREGATION_EARLY_STARTS_TOTAL.inc(); -} -``` - -```rust -static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock = - std::sync::LazyLock::new(|| { - register_histogram!( - "lean_aggregation_early_start_lead_seconds", - "How far before the interval-2 boundary an early aggregation session started", - vec![0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4] - ) - .unwrap() - }); - -pub fn observe_aggregation_early_start_lead(lead: Duration) { - LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS.observe(lead.as_secs_f64()); -} -``` - -- [ ] **Step 5: Build and run existing tests** - -Run: `cargo build -p ethlambda-blockchain && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, existing unit tests pass. Behavior so far is identical for normal sessions (`early` is always false when started at interval 2, so no buffering, no flush timer, no metric increments). - -- [ ] **Step 6: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs crates/blockchain/src/metrics.rs -git commit -m "feat(blockchain): hold early-produced aggregates until the interval-2 boundary" -``` - ---- - -### Task 7: Early trigger — threshold checks, timer, and interval-2 skip - -**Files:** -- Modify: `crates/blockchain/src/aggregation.rs` (new message next to `FlushAggregatePublishes`) -- Modify: `crates/blockchain/src/lib.rs` (imports ~line 16, interval-1 arm ~line 290, interval-2 arm ~line 312, `NewAttestation` handler ~line 1034, new method + handler) - -- [ ] **Step 1: Add the check message in `aggregation.rs`** - -```rust -/// One-shot self-message scheduled at the interval-1 tick; fires when the -/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run -/// the threshold check for signatures that all arrived before the window. -/// Arrivals inside the window are checked per insert instead. -pub(crate) struct EarlyAggregationCheck; -impl Message for EarlyAggregationCheck { - type Result = (); -} -``` - -- [ ] **Step 2: Add the trigger method on `BlockChainServer`** - -Extend the `use crate::aggregation::{...}` import with `EarlyAggregationCheck` and `EARLY_AGGREGATION_WINDOW_MS`. Add the method after `start_aggregation_session` (~line 418): - -```rust - /// Early-aggregation trigger: start the slot's session ahead of the - /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, - /// a single attestation-data group already holds 2/3 of the signatures - /// expected from this node's aggregation subnets. Called after every - /// stored gossip signature and once at the window opening via - /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started - /// session stays in `current_aggregation` (running or finished) until the - /// next session replaces it. - async fn maybe_start_early_aggregation(&mut self, ctx: &Context) { - if !self.aggregator.is_enabled() { - return; - } - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) - else { - return; - }; - if self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == slot) - { - return; - } - let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { - return; - } - info!( - %slot, - max_group, - expected = self.early_aggregation_expected_sigs, - "Early-aggregation threshold met" - ); - self.start_aggregation_session(slot, ctx).await; - } -``` - -- [ ] **Step 3: Wire the three trigger sites** - -**(a) Interval-1 arm** (~line 290) — at the end of the `1 => { ... }` block, after the attestation-production `if/else`: - -```rust - // Schedule the early-aggregation window check. This tick is - // one interval before T2, so the timer fires right as the - // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. - if is_aggregator { - send_after( - Duration::from_millis( - MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, - ), - ctx.clone(), - EarlyAggregationCheck, - ); - } -``` - -**(b) Timer handler** — next to the other aggregation handlers: - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { - self.maybe_start_early_aggregation(ctx).await; - } -} -``` - -**(c) Per-insert check** — `impl Handler` (~line 1034) becomes: - -```rust -impl Handler for BlockChainServer { - async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { - self.on_gossip_attestation(&msg.attestation); - self.maybe_start_early_aggregation(ctx).await; - } -} -``` - -(The method's internal guards make this a few comparisons outside the window; `max_gossip_group_count_for_slot` is one lock with no clones inside it.) - -- [ ] **Step 4: Skip the interval-2 start when the early session exists** - -The `2 => { ... }` arm becomes: - -```rust - // ==== interval 2 ==== - 2 => { - if is_aggregator { - // The early trigger may have already started this slot's - // session (running or finished) — it IS the slot's session, - // so don't start a second one. - let already_started = self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == slot); - if !already_started { - self.start_aggregation_session(slot, ctx).await; - } - } else { - metrics::inc_aggregator_skipped_not_aggregator(); - } - } -``` - -- [ ] **Step 5: Build and run existing blockchain tests** - -Run: `cargo build --workspace && cargo test -p ethlambda-blockchain --lib` -Expected: clean build, existing tests pass. - -- [ ] **Step 6: Commit** - -```bash -git add crates/blockchain/src/aggregation.rs crates/blockchain/src/lib.rs -git commit -m "feat(blockchain): start aggregation early when 2/3 of subnet signatures arrived" -``` - ---- - -### Task 8: Lint and existing test suites - -**Files:** none (verification only) - -- [ ] **Step 1: Format and lint** - -Run: `make fmt && make lint` -Expected: no diffs from fmt; clippy clean with `-D warnings` (this is where any dead-code stragglers from Tasks 1–4 would surface — by now every added item has a caller). - -- [ ] **Step 2: Run the existing test suites** - -Run: `make test` -Expected: all workspace tests + forkchoice spec tests pass (no new tests were added; this guards against regressions in the touched paths). If fixtures are missing: `rm -rf leanSpec && make leanSpec/fixtures`. - -- [ ] **Step 3: Commit any fmt fallout** - -```bash -git add -u && git commit -m "chore: fmt" || true -``` - -(Skip the commit if the tree is clean.) - ---- - -### Task 9: Local 4-node devnet validation - -Run a local 4-node devnet on this branch and verify the early-start behavior end to end. Use the `devnet-runner` skill (`.claude/skills/devnet-runner/`) for the exact workflow; constraints that matter here: - -- **Release build only** — debug binaries stack-overflow in leanVM `rec_aggregation`. -- **Exactly one node with `--is-aggregator`** — multiple co-located aggregators stall finality via leanVM CPU contention. -- If using Docker Desktop, the VM needs 16+ GiB (leanVM peaks ~4–5 GiB per node); binary mode avoids this. - -- [ ] **Step 1: Start the devnet** - -4 nodes, fresh genesis, one aggregator. Let it run at least ~50 slots (~4 minutes). - -- [ ] **Step 2: Verify early-start behavior on the aggregator node** - -Check the aggregator node's logs and metrics: - -1. `"Early-aggregation threshold met"` and `"Starting aggregation session early"` appear for most slots (local gossip is fast; signatures land well inside the window). `lead_ms` values must be in `(0, 200]`. -2. `"Publishing aggregates held back until the interval-2 boundary"` appears with `count >= 1`. -3. `lean_aggregation_early_starts_total` grows (metrics port, default `:5054`, path `/metrics`). -4. `lean_aggregation_early_start_lead_seconds` observations sit in `(0, 0.4]`. -5. `"Committee signatures aggregated"` logs show `early=true` sessions. `"Prior aggregation worker still running"` warnings must be rare (the early start shrinks the worst-case gap to the previous slot's worker from 3250 ms to 2800 ms, so occasional joins under slow proofs are expected — every slot would be a bug). -6. No aggregate publish happens before its slot's interval-2 boundary: spot-check a few `"Starting aggregation session early"` slots and confirm the corresponding aggregated-attestation publish log lands at or after T2 (per-slot offset = `(log_epoch - GENESIS_TIME) mod 4 >= 1.6`). - -- [ ] **Step 3: Verify chain health** - -1. `lean_latest_finalized_slot` advances steadily on all 4 nodes (fresh local devnet should finalize nearly every justifiable slot). -2. Blocks carry attestations (`attestation_count > 0` in block logs). -3. No stalls, no panic/error-level logs attributable to aggregation. - -- [ ] **Step 4: Stop the devnet and report** - -Stop the nodes; summarize observed early-start rate (early sessions / slots), lead-time distribution, and finalization progress. diff --git a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md b/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md deleted file mode 100644 index 77b7fc2b..00000000 --- a/docs/superpowers/specs/2026-07-01-early-aggregation-start-design.md +++ /dev/null @@ -1,181 +0,0 @@ -# Early Aggregation Start - -**Date:** 2026-07-01 -**Status:** Approved - -## Motivation - -Committee-signature aggregation (leanVM XMSS proofs) is the dominant cost in the -attestation pipeline. Today the aggregation session starts exactly at the -interval-2 tick, even when all expected signatures arrived earlier. Starting the -session up to 200 ms early, when enough signatures are already present, gives the -expensive proofs more wall-clock runway before block building consumes them at -interval 4. - -## Current behavior - -- The blockchain actor ticks at 800 ms interval boundaries. At interval 2 (if - `is_aggregator`), `start_aggregation_session` snapshots the current slot's - gossip signatures (`snapshot_current_slot_aggregation_inputs`) and spawns an - off-thread worker. A deadline timer cancels the session's token after - `AGGREGATION_DEADLINE` (750 ms). -- Gossip signatures accumulate in the store per attestation-data group as - `NewAttestation` messages arrive (plus self-delivery of own validators' - attestations at interval 1). Only aggregators store them. -- Each produced aggregate is applied to the store and published to gossip - immediately in the `AggregateProduced` handler. -- Subnet assignment is `vid % attestation_committee_count`. The set of subnets a - node subscribes to (own-validator subnets, plus `--aggregate-subnet-ids` for - aggregators, with a fallback to subnet 0 for validator-less aggregators) is - computed inside `build_swarm` in the p2p crate; the blockchain actor does not - know it. - -## New behavior - -### Early trigger - -Aggregator-only, fires at most once per slot. - -- **Window:** wall clock in `[T2 - 200 ms, T2)`, where - `T2 = genesis_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL` - is the current slot's interval-2 boundary. -- **Condition:** some current-slot attestation-data group has gossip signatures - from at least 2/3 of the expected validators: - `group_count * 3 >= expected * 2`, with `expected > 0`. `group_count` is the - number of stored gossip signatures in that group (one per validator). - `expected` = number of validators whose - `vid % attestation_committee_count` is in this node's aggregation subnet set, - computed once at actor spawn (the validator registry is static, and - `Store::head_state()` clones the full state so per-check reads would be - needlessly expensive). - Note that at most one group per slot can reach 2/3 (each validator signs once - per slot), so the per-group condition can hold for at most one group. -- **Check sites:** - 1. After each gossip-signature insert in the `NewAttestation` handler, while - the wall clock is inside the window. - 2. A one-shot `EarlyAggregationCheck` self-message (no payload — the slot is - recomputed from the wall clock at fire time, so a late-fired timer can - never act on a stale slot) scheduled at the interval-1 tick via - `send_after(200 ms)` (interval-1 tick + 600 ms = T2 - 200 ms), covering - the case where the threshold was already met before the window opened. If - the interval-1 tick is skipped (overrun), the per-insert checks still - apply. -- **Once-per-slot guard:** a session with `session_id == slot` already exists in - `current_aggregation` (the field persists after the worker finishes; it is - only replaced at the next session start). - -### Action on trigger - -Call the existing `start_aggregation_session(slot, ctx)` — the full current-slot -snapshot, not just the group that hit the threshold. This is the slot's one and -only session; it merely starts early. The existing prior-session cancel+join in -`start_aggregation_session` handles a still-running previous-slot worker -identically in both paths. - -### Interval-2 tick - -If `current_aggregation` already holds this slot's session (running or -finished), skip starting a new one. Otherwise start normally — the unchanged -fallback for slots where the threshold is never met. - -`emit_agg_start_new_coverage` moves from the interval-2 tick into -`start_aggregation_session`, so the coverage report reflects the store state the -session actually snapshotted in both paths. - -### Publish alignment - -Aggregates must not reach gossip before interval 2 (mirrors the block prebuild -pattern: build early, publish aligned to the boundary). - -- `AggregateProduced` handler: apply the output to the store immediately - (nothing local consumes `new_aggregated_payloads` before interval 4, so early - application is unobservable locally). If the wall clock is `>= T2` for the - session's slot, publish immediately (unchanged). If `< T2`, push the - `SignedAggregatedAttestation` into a pending-publish buffer on the actor. -- When starting an *early* session, schedule a `FlushAggregatePublishes` - self-message via `send_after(T2 - now)`. Its handler publishes and drains the - buffer. This is the primary flush mechanism (not the interval-2 tick, which - can be skipped on overrun); the handler is idempotent (draining an empty - buffer is a no-op). -- The buffer is bounded in practice by the number of data groups in one slot. - Entries are never carried across slots: the flush timer fires at T2 of the - same slot that buffered them, and as a backstop `start_aggregation_session` - drains (drops, with a warning) any leftovers before installing a new session - — covering a backwards wall-clock step or a flush timer delayed past the - next session. Late aggregates are dropped, the same policy as signatures - that miss the snapshot. - -### Deadline change - -`AGGREGATION_DEADLINE`: 750 ms → 800 ms, still measured from session start. - -- Early session started at `T2 - x` (0 < x ≤ 200): deadline at `T2 - x + 800`, - i.e. no earlier than `T2 + 600` (200 ms before the interval-3 boundary). -- Normal session started at `T2`: deadline lands exactly on the interval-3 - boundary. This removes the previous 50 ms publish/propagation margin; - accepted trade-off (the deadline only stops *new* jobs from starting; a job - mid-proof finishes and publishes slightly after). - -### Subnet-set plumbing - -The subscription-subnet computation is hoisted out of `build_swarm` into a -shared pure helper in the p2p crate: -`compute_subscription_subnets(validator_ids, committee_count, is_aggregator, explicit_ids) -> HashSet`. -Both call sites use it with the same startup inputs, keeping a single source of -truth without churning `SwarmConfig`: - -- `build_swarm` (subscribes to exactly this set, behavior unchanged), and -- `main.rs`, which passes the resulting set to `BlockChain::spawn` for the - `expected` count. - -The set is frozen at startup, matching the existing hot-standby aggregator -model (runtime toggles do not resubscribe subnets). - -## Failure modes and edge cases - -- **Snapshot returns `None` after trigger**: unreachable in practice — the - trigger requires stored signatures, and `on_gossip_attestation` only stores - signatures whose pubkeys resolved during verification, so the snapshot - always yields at least one job for them. If it did happen, no session is - created and `current_aggregation` is left `None` (the prior session was - already taken), so the once-per-slot guard would not hold and per-insert - checks would retry; interval 2 remains the fallback. No signatures are - lost. -- **Threshold never met:** interval 2 starts the session exactly as today. -- **Signatures arriving after the early snapshot:** never aggregated that slot. - Accepted trade-off (explicitly chosen over a top-up session). -- **Wall-clock vs monotonic drift:** checks use `unix_now_ms()` like the rest of - the tick logic; the idempotency-guard pattern already tolerates drift. The - worst case of drift is a slightly mistimed early start or flush, never a - correctness issue. -- **Non-aggregators:** never store gossip signatures, and all check sites are - additionally guarded by `AggregatorController::is_enabled()` read at check - time. - -## Observability - -- Counter `lean_aggregation_early_starts_total`: early sessions started. -- Histogram for early-start lead time (`T2 - start`, seconds, buckets covering - 0–200 ms). -- Existing session logs gain an `early: bool` field so devnet log analysis can - attribute publish-time shifts. - -## Testing - -Per user decision, no new unit tests. Validation is: - -- Existing suites pass unchanged (normal interval-2 path is the fallback and - keeps its semantics; only the deadline constant changes). -- Local 4-node devnet run (1 aggregator, binary mode): early-start rate and - lead-time distribution from logs, held-back publish flushes landing at or - after the interval-2 boundary (per-slot offset), lockstep finalization, no - aggregation warnings/errors. - -## Non-goals - -- No gossip resubscription or topic changes. -- No change to the fork-choice attestation pipeline (`new_attestations` / - `known_attestations`). -- No top-up/second session per slot. -- No persistence of the pending-publish buffer (in-memory actor state, same as - the rest of the aggregation pipeline). From 8c4dc38b5f3e3589a4cd9afc71f8ba04262e95a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:17:52 -0300 Subject: [PATCH 19/29] feat(blockchain): set early-aggregation window to 600ms Window opens at T2-600 = 200ms into interval 1, so the trigger can fire as soon as attestations start arriving. Rescale the lead-time histogram buckets to 0.075-0.6s to match. --- crates/blockchain/src/aggregation.rs | 2 +- crates/blockchain/src/metrics.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 4124d2c7..8721888b 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -43,7 +43,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see `early_threshold_met`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 200; +pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; // The window must fit within one interval: `early_aggregation_slot` subtracts // it from the interval-2 offset, and the interval-1 tick schedules the check diff --git a/crates/blockchain/src/metrics.rs b/crates/blockchain/src/metrics.rs index 76289723..5ae6e9e0 100644 --- a/crates/blockchain/src/metrics.rs +++ b/crates/blockchain/src/metrics.rs @@ -388,7 +388,7 @@ static LEAN_AGGREGATION_EARLY_START_LEAD_SECONDS: std::sync::LazyLock register_histogram!( "lean_aggregation_early_start_lead_seconds", "How far before the interval-2 boundary an early aggregation session started", - vec![0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2] + vec![0.075, 0.15, 0.225, 0.3, 0.375, 0.45, 0.525, 0.6] ) .unwrap() }); From e0907a60039722eb284fcc8ba9dae2664f170bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:20:54 -0300 Subject: [PATCH 20/29] refactor(blockchain): compute early-aggregation threshold in closed form Replace the per-validator subnet scan in BlockChain::spawn with the closed form 2*N/(3*C) (two-thirds of one committee's expected votes), folding the 2/3 fraction into the precomputed threshold so early_threshold_met is now a direct >= comparison. Drop the now-unused aggregation_subnets parameter and its main.rs plumbing; compute_subscription_subnets is only used inside the p2p crate now, so downgrade it to pub(crate). --- bin/ethlambda/src/main.rs | 15 +------------- crates/blockchain/src/aggregation.rs | 12 ++++++----- crates/blockchain/src/lib.rs | 31 ++++++++++++++-------------- crates/net/p2p/src/lib.rs | 15 +++++++------- 4 files changed, 30 insertions(+), 43 deletions(-) diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 19db7eb5..914c6c86 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -37,9 +37,7 @@ use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; use ethlambda_network_api::{InitBlockChain, InitP2P, ToBlockChainToP2PRef, ToP2PToBlockChainRef}; -use ethlambda_p2p::{ - Bootnode, P2P, PeerId, SwarmConfig, build_swarm, compute_subscription_subnets, parse_enrs, -}; +use ethlambda_p2p::{Bootnode, P2P, PeerId, SwarmConfig, build_swarm, parse_enrs}; use ethlambda_types::primitives::{H256, HashTreeRoot as _}; use ethlambda_types::{ aggregator::AggregatorController, @@ -212,22 +210,11 @@ async fn main() -> eyre::Result<()> { // and the API server (which exposes GET/POST admin endpoints). let aggregator = AggregatorController::new(options.is_aggregator); - // Same startup inputs build_swarm uses — single source of truth is the - // shared helper. Frozen at startup like the gossip subscriptions - // themselves (hot-standby model). - let aggregation_subnets = compute_subscription_subnets( - &validator_ids, - attestation_committee_count, - options.is_aggregator, - options.aggregate_subnet_ids.as_deref(), - ); - let blockchain = BlockChain::spawn( store.clone(), validator_keys, aggregator.clone(), attestation_committee_count, - aggregation_subnets, !options.disable_duty_sync_gate, ProposerConfig { enable_proposer_aggregation: options.enable_proposer_aggregation, diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 8721888b..80b2b259 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -73,11 +73,13 @@ pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Optio in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) } -/// Early-start threshold: a single attestation-data group holds at least 2/3 -/// of the signatures expected from this node's aggregation subnets. At most -/// one group per slot can satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, expected: usize) -> bool { - expected > 0 && max_group_count * 3 >= expected * 2 +/// Early-start threshold: the largest single attestation-data group already +/// holds at least `min_group_sigs` signatures. `min_group_sigs` is the +/// precomputed two-thirds-of-a-committee count (see `BlockChain::spawn`), so +/// the 2/3 fraction is not re-applied here. At most one group per slot can +/// satisfy this (each validator signs once per slot). +pub(crate) fn early_threshold_met(max_group_count: usize, min_group_sigs: usize) -> bool { + min_group_sigs > 0 && max_group_count >= min_group_sigs } /// A single pre-prepared aggregation group. diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 4de5bace..5f8e5e3e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -83,7 +83,6 @@ impl BlockChain { validator_keys: HashMap, aggregator: AggregatorController, attestation_committee_count: u64, - aggregation_subnets: HashSet, gate_duties: bool, proposer_config: ProposerConfig, ) -> BlockChain { @@ -100,17 +99,16 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); - // Denominator of the early-aggregation 2/3 threshold: how many - // validators attest on subnets this node aggregates. Computed once — - // the validator registry is static and `head_state()` clones the full - // state, so this must not run per gossip insert. + // Minimum single-group signature count that triggers early + // aggregation: two-thirds of one committee's expected votes, + // `2 * N / (3 * C)`. Closed form (no per-validator subnet scan), + // computed once; assumes an even committee split and one committee's + // worth of votes per group. let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_expected_sigs = if attestation_committee_count == 0 { + let early_aggregation_min_group_sigs = if attestation_committee_count == 0 { 0 } else { - (0..validator_count) - .filter(|vid| aggregation_subnets.contains(&(vid % attestation_committee_count))) - .count() + (2 * validator_count / (3 * attestation_committee_count)) as usize }; let handle = BlockChainServer { @@ -124,7 +122,7 @@ impl BlockChain { last_tick_instant: None, attestation_committee_count, genesis_time_ms: genesis_time * 1000, - early_aggregation_expected_sigs, + early_aggregation_min_group_sigs, pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, @@ -196,10 +194,11 @@ pub struct BlockChainServer { /// code. genesis_time_ms: u64, - /// Number of validators whose subnet is one this node aggregates — the - /// denominator of the early-aggregation 2/3 threshold. Computed once at - /// spawn (the validator registry is static). - early_aggregation_expected_sigs: usize, + /// Minimum signatures in one attestation-data group that trigger early + /// aggregation: two-thirds of a committee's expected votes, + /// `2 * validator_count / (3 * committee_count)`. Computed once at spawn + /// (the validator registry is static). + early_aggregation_min_group_sigs: usize, /// Aggregates produced before the interval-2 boundary, held back so they /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the @@ -531,13 +530,13 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_expected_sigs) { + if !aggregation::early_threshold_met(max_group, self.early_aggregation_min_group_sigs) { return; } info!( %slot, max_group, - expected = self.early_aggregation_expected_sigs, + min_group_sigs = self.early_aggregation_min_group_sigs, "Early-aggregation threshold met" ); self.start_aggregation_session(slot, ctx).await; diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index add89fcc..d6a3509c 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,20 +182,19 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } -/// Compute the set of attestation subnets this node subscribes to (and, when -/// aggregating, aggregates over), per leanSpec (`src/lean_spec/__main__.py`): -/// every validator subscribes to its own subnet (`vid % committee_count`) for -/// mesh health; aggregators additionally subscribe to explicit -/// `aggregate_subnet_ids` and fall back to subnet 0 when the set would -/// otherwise be empty. +/// Compute the set of attestation subnets this node subscribes to, per +/// leanSpec (`src/lean_spec/__main__.py`): every validator subscribes to its +/// own subnet (`vid % committee_count`) for mesh health; aggregators +/// additionally subscribe to explicit `aggregate_subnet_ids` and fall back to +/// subnet 0 when the set would otherwise be empty. /// /// Evaluated once at startup — runtime aggregator toggles do not resubscribe /// (hot-standby model); see the invariant note on [`SwarmConfig`]. /// /// A zero `attestation_committee_count` yields no validator subnets (rather /// than dividing by zero); callers enforce `>= 1`, so this is a defensive -/// guard on the public API, not a reachable path in the binary. -pub fn compute_subscription_subnets( +/// guard, not a reachable path in the binary. +pub(crate) fn compute_subscription_subnets( validator_ids: &[u64], attestation_committee_count: u64, is_aggregator: bool, From ac628dacef25b015fe971aa6426b6d1bf1bf1623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 17:26:34 -0300 Subject: [PATCH 21/29] refactor(blockchain): read genesis time and early-agg threshold at runtime Drop the cached genesis_time_ms and early_aggregation_min_group_sigs fields from BlockChainServer. Genesis time is read via store.config() at each use site (matching the tick paths); the early-aggregation threshold is computed on demand from head_state, reached only inside the early window and only until a session starts. --- crates/blockchain/src/lib.rs | 59 +++++++++++++++--------------------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 5f8e5e3e..0df433a1 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -99,18 +99,6 @@ impl BlockChain { (now_ms.saturating_sub(genesis_time * 1000) / MILLISECONDS_PER_SLOT) as u32; key_manager.advance_keys_to(current_slot); - // Minimum single-group signature count that triggers early - // aggregation: two-thirds of one committee's expected votes, - // `2 * N / (3 * C)`. Closed form (no per-validator subnet scan), - // computed once; assumes an even committee split and one committee's - // worth of votes per group. - let validator_count = store.head_state().validators.len() as u64; - let early_aggregation_min_group_sigs = if attestation_committee_count == 0 { - 0 - } else { - (2 * validator_count / (3 * attestation_committee_count)) as usize - }; - let handle = BlockChainServer { store, p2p: None, @@ -121,8 +109,6 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, - genesis_time_ms: genesis_time * 1000, - early_aggregation_min_group_sigs, pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, @@ -184,22 +170,10 @@ pub struct BlockChainServer { last_tick_instant: Option, /// Number of attestation committees (= subnet count). Used by the - /// attestation aggregate coverage emission. + /// attestation aggregate coverage emission and the early-aggregation + /// threshold. attestation_committee_count: u64, - /// Genesis time in milliseconds, cached at spawn. `store.config()` is an - /// uncached backend read, too heavy for the per-gossip-insert early - /// checks that need this. The tick paths still recompute it from - /// `store.config()` — deliberately left alone; prefer this field in new - /// code. - genesis_time_ms: u64, - - /// Minimum signatures in one attestation-data group that trigger early - /// aggregation: two-thirds of a committee's expected votes, - /// `2 * validator_count / (3 * committee_count)`. Computed once at spawn - /// (the validator registry is static). - early_aggregation_min_group_sigs: usize, - /// Aggregates produced before the interval-2 boundary, held back so they /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the /// boundary; only ever holds the current slot's aggregates. @@ -459,7 +433,8 @@ impl BlockChainServer { "Dropping stale pending aggregate publishes" ); } - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, slot); + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { @@ -518,8 +493,8 @@ impl BlockChainServer { if !self.aggregator.is_enabled() { return; } - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), self.genesis_time_ms) - else { + let genesis_time_ms = self.store.config().genesis_time * 1000; + let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) else { return; }; if self @@ -530,18 +505,33 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - if !aggregation::early_threshold_met(max_group, self.early_aggregation_min_group_sigs) { + let min_group_sigs = self.early_aggregation_min_group_sigs(); + if !aggregation::early_threshold_met(max_group, min_group_sigs) { return; } info!( %slot, max_group, - min_group_sigs = self.early_aggregation_min_group_sigs, + min_group_sigs, "Early-aggregation threshold met" ); self.start_aggregation_session(slot, ctx).await; } + /// Minimum signatures in one attestation-data group that trigger early + /// aggregation: two-thirds of one committee's expected votes, + /// `2 * validator_count / (3 * committee_count)`. Computed on demand rather + /// than cached; only reached inside the early-aggregation window and only + /// until a session starts, so the `head_state` read (memoized on the + /// stable head root) runs a handful of times per slot at most. + fn early_aggregation_min_group_sigs(&self) -> usize { + if self.attestation_committee_count == 0 { + return 0; + } + let validator_count = self.store.head_state().validators.len() as u64; + (2 * validator_count / (3 * self.attestation_committee_count)) as usize + } + /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1208,7 +1198,8 @@ impl Handler for BlockChainServer { // Publish alignment: hold back aggregates produced before this slot's // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. // (`session_id` is the session's slot.) - let t2_ms = aggregation::interval2_boundary_ms(self.genesis_time_ms, msg.session_id); + let genesis_time_ms = self.store.config().genesis_time * 1000; + let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); if unix_now_ms() < t2_ms { self.pending_aggregate_publishes.push(aggregate); return; From ed61bdcd569bcf13fdcb38cde2c4fa00826c65e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:21:21 -0300 Subject: [PATCH 22/29] refactor(blockchain): delay early aggregates in the worker, drop FlushAggregatePublishes Move publish alignment from the actor into the aggregation worker. The worker now receives the interval-2 boundary timestamp and, for each aggregate it produces before that boundary, uses send_after to delay the AggregateProduced message until the boundary (sending immediately if already past it). This removes the FlushAggregatePublishes self-message, the pending_aggregate_publishes buffer, and the T2 check in the AggregateProduced handler. Delivery timers are tied to the actor's cancellation token via Context::from_ref, so they drop on shutdown just like the old buffer did. --- crates/blockchain/src/aggregation.rs | 49 +++++++++++-------- crates/blockchain/src/lib.rs | 72 ++++------------------------ 2 files changed, 37 insertions(+), 84 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 80b2b259..2e113ece 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -21,7 +21,7 @@ use ethlambda_types::{ state::Validator, }; use spawned_concurrency::message::Message; -use spawned_concurrency::tasks::ActorRef; +use spawned_concurrency::tasks::{ActorRef, Context, send_after}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; @@ -165,19 +165,6 @@ impl Message for AggregationDeadline { type Result = (); } -/// Self-message scheduled when a session starts early; fires at the -/// interval-2 boundary and publishes any aggregates held back by the -/// publish-alignment rule (aggregates must not reach gossip before -/// interval 2). Carries the session slot so a timer delayed past the next -/// session start is fenced (like [`AggregationDeadline`]) and cannot flush -/// the newer session's buffer before its own boundary. -pub(crate) struct FlushAggregatePublishes { - pub(crate) slot: u64, -} -impl Message for FlushAggregatePublishes { - type Result = (); -} - /// One-shot self-message scheduled at the interval-1 tick; fires when the /// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run /// the threshold check for signatures that all arrived before the window. @@ -572,11 +559,19 @@ pub(crate) fn aggregation_bits_from_validator_indices(bits: &[u64]) -> Aggregati /// Pulls jobs from the snapshot, runs [`aggregate_job`] for each, and streams /// successful aggregates back to the actor as [`AggregateProduced`] messages. /// Emits [`AggregationDone`] when the loop exits (completion or cancellation). +/// +/// Publish alignment: aggregates must not reach the actor (and thus gossip) +/// before the interval-2 boundary. `publish_at_ms` is that boundary as a UNIX +/// millisecond timestamp; a produced aggregate whose wall clock is still before +/// it is delivered via [`send_after`] timed to land at the boundary, otherwise +/// it is sent immediately. A normal interval-2 session starts at the boundary, +/// so its aggregates are always past it and sent without delay. pub(crate) fn run_aggregation_worker( snapshot: AggregationSnapshot, actor: ActorRef, cancel: CancellationToken, session_id: u64, + publish_at_ms: u64, ) { let start = Instant::now(); let groups_considered = snapshot.groups_considered; @@ -624,12 +619,26 @@ pub(crate) fn run_aggregation_worker( total_raw_sigs += raw_sigs; total_children += children; - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; + // Hold the aggregate until the interval-2 boundary (early session), or + // send now if already past it. `send_after` is fire-and-forget: it + // spawns a timer that delivers the message and is cancelled only if the + // actor stops, so the produced aggregate is not lost when the worker's + // own loop ends. + let now_ms = crate::unix_now_ms(); + if now_ms >= publish_at_ms { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; + } + } else { + send_after( + Duration::from_millis(publish_at_ms - now_ms), + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 0df433a1..448a115e 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -16,7 +16,7 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, - FlushAggregatePublishes, PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -109,7 +109,6 @@ impl BlockChain { current_aggregation: None, last_tick_instant: None, attestation_committee_count, - pending_aggregate_publishes: Vec::new(), proposer_config, pre_merge_coverage: None, sync_status: SyncStatusTracker::new(gate_duties), @@ -174,11 +173,6 @@ pub struct BlockChainServer { /// threshold. attestation_committee_count: u64, - /// Aggregates produced before the interval-2 boundary, held back so they - /// are not gossiped early. Flushed by `FlushAggregatePublishes` at the - /// boundary; only ever holds the current slot's aggregates. - pending_aggregate_publishes: Vec, - /// Proposer-side block-building policy proposer_config: ProposerConfig, @@ -421,26 +415,14 @@ impl BlockChainServer { }; let session_id = slot; - // Any leftovers from a prior slot mean its flush never fired (a - // backwards wall-clock step, or a flush timer delayed past the next - // session). Drop them — late aggregates are dropped, same policy as - // signatures that miss the snapshot. - let stale = std::mem::take(&mut self.pending_aggregate_publishes); - if !stale.is_empty() { - warn!( - %slot, - count = stale.len(), - "Dropping stale pending aggregate publishes" - ); - } let genesis_time_ms = self.store.config().genesis_time * 1000; let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { - // Publish alignment: aggregates must not reach gossip before the - // interval-2 boundary. Aggregates produced before T2 are buffered - // in `pending_aggregate_publishes`; this timer flushes them at T2. + // Publish alignment lives in the worker: it holds each produced + // aggregate until `t2_ms` (the interval-2 boundary) before sending + // it back, so nothing reaches gossip early. let lead = Duration::from_millis(t2_ms - now_ms); metrics::inc_aggregation_early_starts(); metrics::observe_aggregation_early_start_lead(lead); @@ -449,7 +431,6 @@ impl BlockChainServer { lead_ms = lead.as_millis() as u64, "Starting aggregation session early" ); - send_after(lead, ctx.clone(), FlushAggregatePublishes { slot }); } // Independent token per session. Shutdown propagates via our @@ -461,7 +442,7 @@ impl BlockChainServer { let worker_cancel = cancel.clone(); let worker_actor = actor_ref.clone(); let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id); + run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id, t2_ms); }); let _deadline_timer = send_after( @@ -1188,56 +1169,19 @@ impl Handler for BlockChainServer { return; } + // Publish alignment is enforced upstream: the worker delays delivery of + // this message until the interval-2 boundary, so by the time it lands + // the aggregate is safe to apply and gossip immediately. aggregation::apply_aggregated_group(&mut self.store, &msg.output); let aggregate = SignedAggregatedAttestation { data: msg.output.hashed.data().clone(), proof: msg.output.proof, }; - - // Publish alignment: hold back aggregates produced before this slot's - // interval-2 boundary; `FlushAggregatePublishes` publishes them at T2. - // (`session_id` is the session's slot.) - let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, msg.session_id); - if unix_now_ms() < t2_ms { - self.pending_aggregate_publishes.push(aggregate); - return; - } self.publish_aggregate(aggregate); } } -impl Handler for BlockChainServer { - async fn handle(&mut self, msg: FlushAggregatePublishes, _ctx: &Context) { - // Fence like `AggregationDeadline`: only the current session's flush - // drains the buffer. A flush timer delayed past the next session start - // would otherwise publish that newer session's buffered aggregates - // before its own interval-2 boundary, breaking publish alignment. The - // superseded session's own aggregates were already dropped by the - // stale-drain in `start_aggregation_session`. - let is_current = self - .current_aggregation - .as_ref() - .is_some_and(|session| session.session_id == msg.slot); - if !is_current { - return; - } - let pending = std::mem::take(&mut self.pending_aggregate_publishes); - if pending.is_empty() { - return; - } - info!( - slot = msg.slot, - count = pending.len(), - "Publishing aggregates held back until the interval-2 boundary" - ); - for aggregate in pending { - self.publish_aggregate(aggregate); - } - } -} - impl Handler for BlockChainServer { async fn handle(&mut self, _msg: EarlyAggregationCheck, ctx: &Context) { self.maybe_start_early_aggregation(ctx).await; From 4ae261b7263bd10cd4fbe2cfb8cf7ea405fa82a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:26:15 -0300 Subject: [PATCH 23/29] refactor(blockchain): pass worker publish time as SystemTime Type the aggregation worker's publish boundary as a SystemTime instead of a raw u64 millisecond timestamp. The worker derives each aggregate's delay via duration_since(SystemTime::now()), whose Err branch is exactly the send-immediately (past-boundary) case. SystemTime (not Instant) because the boundary is derived from genesis wall-clock time. --- crates/blockchain/src/aggregation.rs | 47 +++++++++++++++------------- crates/blockchain/src/lib.rs | 15 ++++++--- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 2e113ece..733ef002 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -9,7 +9,7 @@ //! results back as [`AggregateProduced`] / [`AggregationDone`] messages. use std::collections::HashSet; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime}; use ethlambda_crypto::aggregate_mixed; use ethlambda_storage::Store; @@ -561,17 +561,17 @@ pub(crate) fn aggregation_bits_from_validator_indices(bits: &[u64]) -> Aggregati /// Emits [`AggregationDone`] when the loop exits (completion or cancellation). /// /// Publish alignment: aggregates must not reach the actor (and thus gossip) -/// before the interval-2 boundary. `publish_at_ms` is that boundary as a UNIX -/// millisecond timestamp; a produced aggregate whose wall clock is still before -/// it is delivered via [`send_after`] timed to land at the boundary, otherwise -/// it is sent immediately. A normal interval-2 session starts at the boundary, -/// so its aggregates are always past it and sent without delay. +/// before the interval-2 boundary. `publish_at` is that boundary as a wall-clock +/// instant; a produced aggregate still ahead of it is delivered via +/// [`send_after`] timed to land at the boundary, otherwise it is sent +/// immediately. A normal interval-2 session starts at the boundary, so its +/// aggregates are always past it and sent without delay. pub(crate) fn run_aggregation_worker( snapshot: AggregationSnapshot, actor: ActorRef, cancel: CancellationToken, session_id: u64, - publish_at_ms: u64, + publish_at: SystemTime, ) { let start = Instant::now(); let groups_considered = snapshot.groups_considered; @@ -623,22 +623,25 @@ pub(crate) fn run_aggregation_worker( // send now if already past it. `send_after` is fire-and-forget: it // spawns a timer that delivers the message and is cancelled only if the // actor stops, so the produced aggregate is not lost when the worker's - // own loop ends. - let now_ms = crate::unix_now_ms(); - if now_ms >= publish_at_ms { - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; + // own loop ends. `duration_since` errs once the boundary has passed, + // which is exactly the send-immediately case. + match publish_at.duration_since(SystemTime::now()) { + Ok(delay) if !delay.is_zero() => { + send_after( + delay, + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); + } + _ => { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; + } } - } else { - send_after( - Duration::from_millis(publish_at_ms - now_ms), - Context::from_ref(&actor), - AggregateProduced { session_id, output }, - ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 448a115e..591c2565 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -417,12 +417,13 @@ impl BlockChainServer { let session_id = slot; let genesis_time_ms = self.store.config().genesis_time * 1000; let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + // Interval-2 boundary as a wall-clock instant; the worker holds each + // produced aggregate until this before sending it back, so nothing + // reaches gossip early. + let publish_at = SystemTime::UNIX_EPOCH + Duration::from_millis(t2_ms); let now_ms = unix_now_ms(); let early = now_ms < t2_ms; if early { - // Publish alignment lives in the worker: it holds each produced - // aggregate until `t2_ms` (the interval-2 boundary) before sending - // it back, so nothing reaches gossip early. let lead = Duration::from_millis(t2_ms - now_ms); metrics::inc_aggregation_early_starts(); metrics::observe_aggregation_early_start_lead(lead); @@ -442,7 +443,13 @@ impl BlockChainServer { let worker_cancel = cancel.clone(); let worker_actor = actor_ref.clone(); let worker = tokio::task::spawn_blocking(move || { - run_aggregation_worker(snapshot, worker_actor, worker_cancel, session_id, t2_ms); + run_aggregation_worker( + snapshot, + worker_actor, + worker_cancel, + session_id, + publish_at, + ); }); let _deadline_timer = send_after( From df18f775dcfe9c905a23c101293cd8ede2154a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:34:16 -0300 Subject: [PATCH 24/29] refactor: inline single-use early-aggregation helpers; use if-else in worker Inline five helpers that ended up with a single caller: compute_subscription_subnets (back into build_swarm), interval2_boundary_ms and early_threshold_met (into their call sites), the early_aggregation_min_group_sigs method (into the trigger), and publish_aggregate (into the AggregateProduced handler). Keep early_aggregation_slot (non-trivial window math, reads clearer named) and the Store/GossipSignatureBuffer delegation pair (the file's encapsulation pattern). Also replace the worker's match on duration_since with an if-else on the unwrapped delay. --- crates/blockchain/src/aggregation.rs | 53 ++++++++-------------- crates/blockchain/src/lib.rs | 53 ++++++++++------------ crates/net/p2p/src/lib.rs | 66 +++++++++------------------- 3 files changed, 62 insertions(+), 110 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index 733ef002..a3f242dd 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -42,7 +42,7 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is -/// met (see `early_threshold_met`). +/// met (see the check in `maybe_start_early_aggregation`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; // The window must fit within one interval: `early_aggregation_slot` subtracts @@ -55,12 +55,6 @@ const _: () = assert!( "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" ); -/// Wall-clock millisecond timestamp of `slot`'s interval-2 boundary (the -/// normal aggregation start). -pub(crate) fn interval2_boundary_ms(genesis_time_ms: u64, slot: u64) -> u64 { - genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL -} - /// If `now_ms` falls inside some slot's early-aggregation window /// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 /// boundary), return that slot. @@ -73,15 +67,6 @@ pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Optio in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) } -/// Early-start threshold: the largest single attestation-data group already -/// holds at least `min_group_sigs` signatures. `min_group_sigs` is the -/// precomputed two-thirds-of-a-committee count (see `BlockChain::spawn`), so -/// the 2/3 fraction is not re-applied here. At most one group per slot can -/// satisfy this (each validator signs once per slot). -pub(crate) fn early_threshold_met(max_group_count: usize, min_group_sigs: usize) -> bool { - min_group_sigs > 0 && max_group_count >= min_group_sigs -} - /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread @@ -620,28 +605,28 @@ pub(crate) fn run_aggregation_worker( total_children += children; // Hold the aggregate until the interval-2 boundary (early session), or - // send now if already past it. `send_after` is fire-and-forget: it + // send now if already at/past it. `send_after` is fire-and-forget: it // spawns a timer that delivers the message and is cancelled only if the // actor stops, so the produced aggregate is not lost when the worker's // own loop ends. `duration_since` errs once the boundary has passed, - // which is exactly the send-immediately case. - match publish_at.duration_since(SystemTime::now()) { - Ok(delay) if !delay.is_zero() => { - send_after( - delay, - Context::from_ref(&actor), - AggregateProduced { session_id, output }, - ); - } - _ => { - if actor - .send(AggregateProduced { session_id, output }) - .is_err() - { - // Actor is gone; no point producing more. - break; - } + // which collapses to a zero delay here. + let delay = publish_at + .duration_since(SystemTime::now()) + .unwrap_or(Duration::ZERO); + if delay.is_zero() { + if actor + .send(AggregateProduced { session_id, output }) + .is_err() + { + // Actor is gone; no point producing more. + break; } + } else { + send_after( + delay, + Context::from_ref(&actor), + AggregateProduced { session_id, output }, + ); } } diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 591c2565..a24d2758 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -416,7 +416,7 @@ impl BlockChainServer { let session_id = slot; let genesis_time_ms = self.store.config().genesis_time * 1000; - let t2_ms = aggregation::interval2_boundary_ms(genesis_time_ms, slot); + let t2_ms = genesis_time_ms + slot * MILLISECONDS_PER_SLOT + 2 * MILLISECONDS_PER_INTERVAL; // Interval-2 boundary as a wall-clock instant; the worker holds each // produced aggregate until this before sending it back, so nothing // reaches gossip early. @@ -493,8 +493,18 @@ impl BlockChainServer { return; } let max_group = self.store.max_gossip_group_count_for_slot(slot); - let min_group_sigs = self.early_aggregation_min_group_sigs(); - if !aggregation::early_threshold_met(max_group, min_group_sigs) { + // Trigger once the largest current-slot group holds two-thirds of one + // committee's expected votes, `2 * N / (3 * C)` (0 when there is no + // committee, which never triggers). The head-state read is memoized on + // the stable head root and only runs inside the window until a session + // starts, so it costs a handful of reads per slot at most. + let min_group_sigs = if self.attestation_committee_count == 0 { + 0 + } else { + let validator_count = self.store.head_state().validators.len() as u64; + (2 * validator_count / (3 * self.attestation_committee_count)) as usize + }; + if min_group_sigs == 0 || max_group < min_group_sigs { return; } info!( @@ -506,20 +516,6 @@ impl BlockChainServer { self.start_aggregation_session(slot, ctx).await; } - /// Minimum signatures in one attestation-data group that trigger early - /// aggregation: two-thirds of one committee's expected votes, - /// `2 * validator_count / (3 * committee_count)`. Computed on demand rather - /// than cached; only reached inside the early-aggregation window and only - /// until a session starts, so the `head_state` read (memoized on the - /// stable head root) runs a handful of times per slot at most. - fn early_aggregation_min_group_sigs(&self) -> usize { - if self.attestation_committee_count == 0 { - return 0; - } - let validator_count = self.store.head_state().validators.len() as u64; - (2 * validator_count / (3 * self.attestation_committee_count)) as usize - } - /// Returns the validator ID if any of our validators is the proposer for this slot. fn get_our_proposer(&self, slot: u64) -> Option { let head_state = self.store.head_state(); @@ -1030,15 +1026,6 @@ impl BlockChainServer { } } - /// Publish an aggregated attestation to the aggregation gossip topic. - fn publish_aggregate(&self, aggregate: SignedAggregatedAttestation) { - if let Some(ref p2p) = self.p2p { - let _ = p2p - .publish_aggregated_attestation(aggregate) - .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); - } - } - fn on_gossip_attestation(&mut self, attestation: &SignedAttestation) { // Read fresh here too: a gossip event can arrive between ticks, and // if the admin API just toggled, the first gossip after the toggle @@ -1181,11 +1168,15 @@ impl Handler for BlockChainServer { // the aggregate is safe to apply and gossip immediately. aggregation::apply_aggregated_group(&mut self.store, &msg.output); - let aggregate = SignedAggregatedAttestation { - data: msg.output.hashed.data().clone(), - proof: msg.output.proof, - }; - self.publish_aggregate(aggregate); + if let Some(ref p2p) = self.p2p { + let aggregate = SignedAggregatedAttestation { + data: msg.output.hashed.data().clone(), + proof: msg.output.proof, + }; + let _ = p2p + .publish_aggregated_attestation(aggregate) + .inspect_err(|err| error!(%err, "Failed to publish aggregated attestation")); + } } } diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index d6a3509c..51ce011d 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -182,43 +182,6 @@ pub struct BuiltSwarm { pub(crate) bootnode_addrs: HashMap, } -/// Compute the set of attestation subnets this node subscribes to, per -/// leanSpec (`src/lean_spec/__main__.py`): every validator subscribes to its -/// own subnet (`vid % committee_count`) for mesh health; aggregators -/// additionally subscribe to explicit `aggregate_subnet_ids` and fall back to -/// subnet 0 when the set would otherwise be empty. -/// -/// Evaluated once at startup — runtime aggregator toggles do not resubscribe -/// (hot-standby model); see the invariant note on [`SwarmConfig`]. -/// -/// A zero `attestation_committee_count` yields no validator subnets (rather -/// than dividing by zero); callers enforce `>= 1`, so this is a defensive -/// guard, not a reachable path in the binary. -pub(crate) fn compute_subscription_subnets( - validator_ids: &[u64], - attestation_committee_count: u64, - is_aggregator: bool, - aggregate_subnet_ids: Option<&[u64]>, -) -> HashSet { - let mut subnets: HashSet = if attestation_committee_count == 0 { - HashSet::new() - } else { - validator_ids - .iter() - .map(|vid| vid % attestation_committee_count) - .collect() - }; - if is_aggregator { - if let Some(explicit_ids) = aggregate_subnet_ids { - subnets.extend(explicit_ids); - } - if subnets.is_empty() { - subnets.insert(0); - } - } - subnets -} - /// Build and configure the libp2p swarm, dial bootnodes, subscribe to topics. pub fn build_swarm( config: SwarmConfig, @@ -338,8 +301,14 @@ pub fn build_swarm( .subscribe(&aggregation_topic) .unwrap(); - // Subscribe to attestation subnets — see `compute_subscription_subnets`. - // The committee metric should reflect validator membership only, not + // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): + // every validator subscribes to its own subnet (`vid % committee_count`) + // for mesh health; aggregators additionally subscribe to explicit + // `aggregate_subnet_ids` and fall back to subnet 0 when they have no + // validators of their own. Frozen at startup — runtime aggregator toggles + // do not resubscribe (hot-standby model); see the invariant on `SwarmConfig`. + // + // The committee metric reflects validator membership only, not // aggregator-only subscriptions. let metric_subnet = config .validator_ids @@ -349,12 +318,19 @@ pub fn build_swarm( .unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let subscription_subnets = compute_subscription_subnets( - &config.validator_ids, - config.attestation_committee_count, - config.is_aggregator, - config.aggregate_subnet_ids.as_deref(), - ); + let mut subscription_subnets: HashSet = config + .validator_ids + .iter() + .map(|vid| vid % config.attestation_committee_count) + .collect(); + if config.is_aggregator { + if let Some(ref explicit_ids) = config.aggregate_subnet_ids { + subscription_subnets.extend(explicit_ids); + } + if subscription_subnets.is_empty() { + subscription_subnets.insert(0); + } + } let mut attestation_topics: HashMap = HashMap::new(); for &subnet_id in &subscription_subnets { From 61d5490805ccd0b5ae804201e741598939f33ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:39:17 -0300 Subject: [PATCH 25/29] refactor(blockchain): inline early_aggregation_slot into the trigger Fold the window-detection math into maybe_start_early_aggregation (its only caller) and drop the helper. Update the two doc comments that referenced it. --- crates/blockchain/src/aggregation.rs | 28 ++++++++-------------------- crates/blockchain/src/lib.rs | 15 +++++++++++---- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index a3f242dd..bc91a0e7 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -25,15 +25,15 @@ use spawned_concurrency::tasks::{ActorRef, Context, send_after}; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; -use crate::{MILLISECONDS_PER_INTERVAL, MILLISECONDS_PER_SLOT, metrics}; +use crate::{MILLISECONDS_PER_INTERVAL, metrics}; /// Soft deadline for committee-signature aggregation measured from session /// start. After this much wall time elapses, the actor signals the worker to /// stop via its cancellation token. A session started exactly at interval 2 /// gets the full interval (interval 3 is one interval later); a session -/// started early (see `early_aggregation_slot`) ends correspondingly earlier. -/// The deadline only stops new jobs from starting — a job mid-proof finishes -/// and publishes right after. +/// started early (see `maybe_start_early_aggregation`) ends correspondingly +/// earlier. The deadline only stops new jobs from starting — a job mid-proof +/// finishes and publishes right after. pub(crate) const AGGREGATION_DEADLINE: Duration = Duration::from_millis(800); /// Upper bound we wait for a prior worker to exit if it is still running when /// the next session is about to start. Reached only in pathological cases @@ -45,28 +45,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// met (see the check in `maybe_start_early_aggregation`). pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; -// The window must fit within one interval: `early_aggregation_slot` subtracts -// it from the interval-2 offset, and the interval-1 tick schedules the check -// at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep this -// invariant self-enforcing so a future bump to the window can't silently +// The window must fit within one interval: `maybe_start_early_aggregation` +// subtracts it from the interval-2 offset, and the interval-1 tick schedules +// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep +// this invariant self-enforcing so a future bump to the window can't silently // underflow either subtraction. const _: () = assert!( EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" ); -/// If `now_ms` falls inside some slot's early-aggregation window -/// (`[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)` with `T2` that slot's interval-2 -/// boundary), return that slot. -pub(crate) fn early_aggregation_slot(now_ms: u64, genesis_time_ms: u64) -> Option { - let since_genesis = now_ms.checked_sub(genesis_time_ms)?; - let ms_into_slot = since_genesis % MILLISECONDS_PER_SLOT; - let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - let in_window = - ms_into_slot >= t2_offset - EARLY_AGGREGATION_WINDOW_MS && ms_into_slot < t2_offset; - in_window.then_some(since_genesis / MILLISECONDS_PER_SLOT) -} - /// A single pre-prepared aggregation group. /// /// Built on the actor thread from a store snapshot; consumed by an off-thread diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index a24d2758..b884a62d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -481,10 +481,19 @@ impl BlockChainServer { if !self.aggregator.is_enabled() { return; } + // Only fire inside the early-aggregation window + // `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, where T2 is the current + // slot's interval-2 boundary; the slot is derived from the wall clock. let genesis_time_ms = self.store.config().genesis_time * 1000; - let Some(slot) = aggregation::early_aggregation_slot(unix_now_ms(), genesis_time_ms) else { + let Some(ms_since_genesis) = unix_now_ms().checked_sub(genesis_time_ms) else { return; }; + let ms_into_slot = ms_since_genesis % MILLISECONDS_PER_SLOT; + let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; + if ms_into_slot < t2_offset - EARLY_AGGREGATION_WINDOW_MS || ms_into_slot >= t2_offset { + return; + } + let slot = ms_since_genesis / MILLISECONDS_PER_SLOT; if self .current_aggregation .as_ref() @@ -495,9 +504,7 @@ impl BlockChainServer { let max_group = self.store.max_gossip_group_count_for_slot(slot); // Trigger once the largest current-slot group holds two-thirds of one // committee's expected votes, `2 * N / (3 * C)` (0 when there is no - // committee, which never triggers). The head-state read is memoized on - // the stable head root and only runs inside the window until a session - // starts, so it costs a handful of reads per slot at most. + // committee, which never triggers). let min_group_sigs = if self.attestation_committee_count == 0 { 0 } else { From 3c1a4395425763adacc3176ad2511f5948ac6778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:50:58 -0300 Subject: [PATCH 26/29] feat(blockchain): gate early-aggregation check on current-slot attestations A late- or future-slot attestation cannot advance the current slot's gossip group counts, so running the early-aggregation threshold check for it is wasted work. Gate the attestation-driven call on the attestation being for the store's current slot (store.time() / INTERVALS_PER_SLOT). The window-opening timer (EarlyAggregationCheck) still fires regardless. --- crates/blockchain/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index b884a62d..9a701c6d 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -470,7 +470,7 @@ impl BlockChainServer { /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, /// a single attestation-data group already holds 2/3 of the signatures /// expected from this node's aggregation subnets. Called after every - /// stored gossip signature and once at the window opening via + /// stored current-slot gossip signature and once at the window opening via /// [`EarlyAggregationCheck`]. Fires at most once per slot: the started /// session stays in `current_aggregation` (running or finished) until the /// next session replaces it. The latch has one hole: if the snapshot @@ -1141,7 +1141,13 @@ impl Handler for BlockChainServer { impl Handler for BlockChainServer { async fn handle(&mut self, msg: NewAttestation, ctx: &Context) { self.on_gossip_attestation(&msg.attestation); - self.maybe_start_early_aggregation(ctx).await; + // Early aggregation only advances the current slot's group counts, so a + // late- or future-slot attestation can never cross the threshold; skip + // the check unless this attestation is for the store's current slot. + let current_slot = self.store.time() / INTERVALS_PER_SLOT; + if msg.attestation.data.slot == current_slot { + self.maybe_start_early_aggregation(ctx).await; + } } } From 49bfc8896ce136734cd6aa39a7dd82a2ec3f3f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:54:19 -0300 Subject: [PATCH 27/29] revert(p2p): drop no-op build_swarm refactor churn The subnet-computation reorganization in build_swarm was leftover churn from the earlier extract/re-inline of compute_subscription_subnets and produced no behavioral change (min over the deduped set equals min over the mapped iterator; the subscription set collects identically). Restore main's single-pass version to keep this PR focused on early aggregation. --- crates/net/p2p/src/lib.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/net/p2p/src/lib.rs b/crates/net/p2p/src/lib.rs index 51ce011d..de34e469 100644 --- a/crates/net/p2p/src/lib.rs +++ b/crates/net/p2p/src/lib.rs @@ -302,31 +302,27 @@ pub fn build_swarm( .unwrap(); // Subscribe to attestation subnets per leanSpec (`src/lean_spec/__main__.py`): - // every validator subscribes to its own subnet (`vid % committee_count`) - // for mesh health; aggregators additionally subscribe to explicit - // `aggregate_subnet_ids` and fall back to subnet 0 when they have no - // validators of their own. Frozen at startup — runtime aggregator toggles - // do not resubscribe (hot-standby model); see the invariant on `SwarmConfig`. - // - // The committee metric reflects validator membership only, not - // aggregator-only subscriptions. - let metric_subnet = config + // every validator subscribes to its own subnet for mesh health; aggregators + // additionally subscribe to explicit `aggregate_subnet_ids` and fall back to + // subnet 0 when they have no validators of their own. + let validator_subnets: HashSet = config .validator_ids .iter() .map(|vid| vid % config.attestation_committee_count) - .min() - .unwrap_or(0); + .collect(); + + // The committee metric should reflect validator membership only, not + // aggregator-only subscriptions. + let metric_subnet = validator_subnets.iter().copied().min().unwrap_or(0); metrics::set_attestation_committee_subnet(metric_subnet); - let mut subscription_subnets: HashSet = config - .validator_ids - .iter() - .map(|vid| vid % config.attestation_committee_count) - .collect(); + let mut subscription_subnets = validator_subnets; if config.is_aggregator { if let Some(ref explicit_ids) = config.aggregate_subnet_ids { subscription_subnets.extend(explicit_ids); } + // Fall back to subnet 0 only when the aggregator has no validators + // and no explicit subnets — otherwise leave the set as configured. if subscription_subnets.is_empty() { subscription_subnets.insert(0); } From 72cbf213c6dfa4d73f0286af685dfadd4c248731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:40:13 -0300 Subject: [PATCH 28/29] feat(blockchain): round the early-aggregation threshold up, not down The 2/3-of-expected-votes threshold used floor division, so for counts not divisible by 3 it triggered slightly below a true 2/3. Round up with div_ceil so the early start waits for a genuine 2/3 supermajority of one committee's expected votes (ceil(2*N / (3*C))). --- crates/blockchain/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index 9a701c6d..cbca5f39 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -503,13 +503,13 @@ impl BlockChainServer { } let max_group = self.store.max_gossip_group_count_for_slot(slot); // Trigger once the largest current-slot group holds two-thirds of one - // committee's expected votes, `2 * N / (3 * C)` (0 when there is no - // committee, which never triggers). + // committee's expected votes, rounded up: `ceil(2 * N / (3 * C))` + // (0 only when there are no validators, which never triggers). let min_group_sigs = if self.attestation_committee_count == 0 { 0 } else { let validator_count = self.store.head_state().validators.len() as u64; - (2 * validator_count / (3 * self.attestation_committee_count)) as usize + (2 * validator_count).div_ceil(3 * self.attestation_committee_count) as usize }; if min_group_sigs == 0 || max_group < min_group_sigs { return; From 632979d188d897640b94aea2349a76a1be67a55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:13:28 -0300 Subject: [PATCH 29/29] refactor(blockchain): type the early-aggregation window as a Duration EARLY_AGGREGATION_WINDOW_MS was a bare u64 of milliseconds; make it a Duration (EARLY_AGGREGATION_WINDOW) so its unit is carried in the type. The timer site now subtracts Durations directly; the wall-clock window bounds check converts once via as_millis(), and the compile-time fits-within-one-interval assert uses the const as_millis(). --- crates/blockchain/src/aggregation.rs | 12 ++++++------ crates/blockchain/src/lib.rs | 17 ++++++++--------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/crates/blockchain/src/aggregation.rs b/crates/blockchain/src/aggregation.rs index bc91a0e7..6155946e 100644 --- a/crates/blockchain/src/aggregation.rs +++ b/crates/blockchain/src/aggregation.rs @@ -2,7 +2,7 @@ //! pure functions it runs. //! //! The blockchain actor fires one aggregation session per slot — at interval 2, -//! or up to [`EARLY_AGGREGATION_WINDOW_MS`] early when the 2/3 signature +//! or up to [`EARLY_AGGREGATION_WINDOW`] early when the 2/3 signature //! threshold is met — via //! [`run_aggregation_worker`]. The actor stays on its message loop; the worker //! runs the expensive XMSS proofs on a `spawn_blocking` thread and streams @@ -43,16 +43,16 @@ pub(crate) const PRIOR_WORKER_JOIN_TIMEOUT: Duration = Duration::from_secs(2); /// Width of the early-aggregation window: a session may start at most this /// long before the interval-2 boundary, provided the signature threshold is /// met (see the check in `maybe_start_early_aggregation`). -pub(crate) const EARLY_AGGREGATION_WINDOW_MS: u64 = 600; +pub(crate) const EARLY_AGGREGATION_WINDOW: Duration = Duration::from_millis(600); // The window must fit within one interval: `maybe_start_early_aggregation` // subtracts it from the interval-2 offset, and the interval-1 tick schedules -// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS`. Keep +// the check at `MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW`. Keep // this invariant self-enforcing so a future bump to the window can't silently // underflow either subtraction. const _: () = assert!( - EARLY_AGGREGATION_WINDOW_MS <= MILLISECONDS_PER_INTERVAL, - "EARLY_AGGREGATION_WINDOW_MS must not exceed one interval" + EARLY_AGGREGATION_WINDOW.as_millis() <= MILLISECONDS_PER_INTERVAL as u128, + "EARLY_AGGREGATION_WINDOW must not exceed one interval" ); /// A single pre-prepared aggregation group. @@ -139,7 +139,7 @@ impl Message for AggregationDeadline { } /// One-shot self-message scheduled at the interval-1 tick; fires when the -/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW_MS) to run +/// early-aggregation window opens (T2 - EARLY_AGGREGATION_WINDOW) to run /// the threshold check for signatures that all arrived before the window. /// Arrivals inside the window are checked per insert instead. pub(crate) struct EarlyAggregationCheck; diff --git a/crates/blockchain/src/lib.rs b/crates/blockchain/src/lib.rs index cbca5f39..b3b85e44 100644 --- a/crates/blockchain/src/lib.rs +++ b/crates/blockchain/src/lib.rs @@ -15,8 +15,8 @@ use ethlambda_types::{ use crate::aggregation::{ AGGREGATION_DEADLINE, AggregateProduced, AggregationDeadline, AggregationDone, - AggregationSession, EARLY_AGGREGATION_WINDOW_MS, EarlyAggregationCheck, - PRIOR_WORKER_JOIN_TIMEOUT, run_aggregation_worker, + AggregationSession, EARLY_AGGREGATION_WINDOW, EarlyAggregationCheck, PRIOR_WORKER_JOIN_TIMEOUT, + run_aggregation_worker, }; use crate::key_manager::ValidatorKeyPair; use crate::sync_status::SyncStatusTracker; @@ -313,12 +313,10 @@ impl BlockChainServer { // Schedule the early-aggregation window check. This tick is // one interval before T2, so the timer fires right as the - // window opens at T2 - EARLY_AGGREGATION_WINDOW_MS. + // window opens at T2 - EARLY_AGGREGATION_WINDOW. if is_aggregator { send_after( - Duration::from_millis( - MILLISECONDS_PER_INTERVAL - EARLY_AGGREGATION_WINDOW_MS, - ), + Duration::from_millis(MILLISECONDS_PER_INTERVAL) - EARLY_AGGREGATION_WINDOW, ctx.clone(), EarlyAggregationCheck, ); @@ -467,7 +465,7 @@ impl BlockChainServer { } /// Early-aggregation trigger: start the slot's session ahead of the - /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, + /// interval-2 tick when, inside the window `[T2 - EARLY_AGGREGATION_WINDOW, T2)`, /// a single attestation-data group already holds 2/3 of the signatures /// expected from this node's aggregation subnets. Called after every /// stored current-slot gossip signature and once at the window opening via @@ -482,7 +480,7 @@ impl BlockChainServer { return; } // Only fire inside the early-aggregation window - // `[T2 - EARLY_AGGREGATION_WINDOW_MS, T2)`, where T2 is the current + // `[T2 - EARLY_AGGREGATION_WINDOW, T2)`, where T2 is the current // slot's interval-2 boundary; the slot is derived from the wall clock. let genesis_time_ms = self.store.config().genesis_time * 1000; let Some(ms_since_genesis) = unix_now_ms().checked_sub(genesis_time_ms) else { @@ -490,7 +488,8 @@ impl BlockChainServer { }; let ms_into_slot = ms_since_genesis % MILLISECONDS_PER_SLOT; let t2_offset = 2 * MILLISECONDS_PER_INTERVAL; - if ms_into_slot < t2_offset - EARLY_AGGREGATION_WINDOW_MS || ms_into_slot >= t2_offset { + let window_ms = EARLY_AGGREGATION_WINDOW.as_millis() as u64; + if ms_into_slot < t2_offset - window_ms || ms_into_slot >= t2_offset { return; } let slot = ms_since_genesis / MILLISECONDS_PER_SLOT;