From 813d6919d2bd5dc1e5b9a5fe112e14045cf9ce81 Mon Sep 17 00:00:00 2001 From: Muntasir Raihan Rahman <160683781+mrrahman1517@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:20:41 -0700 Subject: [PATCH] feat(benchmarks): add Shared-Memory Governance Benchmark (SMGB) dataset + evaluation Single-user memory benchmarks (LongMemEval, LoCoMo, DMR) only score recall and cannot tell a governed shared-memory system from a naive one. SMGB adds a public, system-agnostic benchmark for the governance of shared agent memory: authorization-filtered retrieval, private->shared promotion, tenant isolation, conflict/supersession, scope-aware deletion, and provenance. Ground truth is derived deterministically from a scope + policy graph (no LLM judge), so scoring is reproducible and free. A run is just {query_id: [ranked ids]}, so any memory system can be scored. - benchmarks/governance/: schema, deterministic policy, system-agnostic scorer, loader with hand-label validation, and a CLI (run_governance.py). - data/seed_scenarios.jsonl: 6 scenarios / 14 queries across all 7 axes, including two private->shared promotion cases, tenant isolation, supersession, deletion, and provenance. Reproducible via data/_generate_seed.py. - Reference runners (oracle / naive_shared / naive_global) demonstrate the point: all three score mean_recall 1.000, but only the leakage/isolation/stale axes separate them (naive_shared: 13 leaks; naive_global: 2 isolation violations). - benchmarks/tests/test_governance_{policy,scorer,seed}.py: 16 tests. - Docs/shared_memory_governance_benchmark.md: formal design + verified gap analysis (PiSAs 2607.05318, ArgusFleet 2606.24535). Self-contained: introduces the benchmarks/ package scaffold. A companion multi-agent consistency harness is tracked separately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Docs/README.md | 6 + Docs/shared_memory_governance_benchmark.md | 290 ++++++++++++++++++ benchmarks/__init__.py | 1 + benchmarks/governance/README.md | 178 +++++++++++ benchmarks/governance/__init__.py | 62 ++++ benchmarks/governance/data/_generate_seed.py | 288 +++++++++++++++++ .../governance/data/seed_scenarios.jsonl | 6 + benchmarks/governance/loader.py | 117 +++++++ benchmarks/governance/policy.py | 140 +++++++++ benchmarks/governance/run_governance.py | 113 +++++++ benchmarks/governance/schema.py | 276 +++++++++++++++++ benchmarks/governance/scorer.py | 261 ++++++++++++++++ benchmarks/tests/__init__.py | 0 benchmarks/tests/test_governance_policy.py | 151 +++++++++ benchmarks/tests/test_governance_scorer.py | 113 +++++++ benchmarks/tests/test_governance_seed.py | 69 +++++ 16 files changed, 2071 insertions(+) create mode 100644 Docs/shared_memory_governance_benchmark.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/governance/README.md create mode 100644 benchmarks/governance/__init__.py create mode 100644 benchmarks/governance/data/_generate_seed.py create mode 100644 benchmarks/governance/data/seed_scenarios.jsonl create mode 100644 benchmarks/governance/loader.py create mode 100644 benchmarks/governance/policy.py create mode 100644 benchmarks/governance/run_governance.py create mode 100644 benchmarks/governance/schema.py create mode 100644 benchmarks/governance/scorer.py create mode 100644 benchmarks/tests/__init__.py create mode 100644 benchmarks/tests/test_governance_policy.py create mode 100644 benchmarks/tests/test_governance_scorer.py create mode 100644 benchmarks/tests/test_governance_seed.py diff --git a/Docs/README.md b/Docs/README.md index a748741..8c5fb95 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -12,6 +12,12 @@ This folder contains the main project documentation for Agent Memory Toolkit. | [design_patterns.md](design_patterns.md) | Shows when and how to call CRUD operations, summarization, fact extraction, and memory retrieval in chat and multi-agent applications, including automatic processing via the change feed. | | [troubleshooting.md](troubleshooting.md) | Helps diagnose common setup, authentication, Cosmos DB, embeddings, Durable Functions, vector search, and change feed issues. | +## Research & Benchmarks + +| Document | Purpose | +|----------|---------| +| [shared_memory_governance_benchmark.md](shared_memory_governance_benchmark.md) | Design for the **Shared-Memory Governance Benchmark (SMGB)**: a public, system-agnostic, policy-labeled benchmark for authorization-filtered retrieval, promotion, tenant isolation, supersession, deletion, and provenance. Backs the package in `benchmarks/governance/`. | + ## Recommended Reading Order 1. Start with [concepts.md](concepts.md) to understand the data model and memory lifecycle. diff --git a/Docs/shared_memory_governance_benchmark.md b/Docs/shared_memory_governance_benchmark.md new file mode 100644 index 0000000..f0da549 --- /dev/null +++ b/Docs/shared_memory_governance_benchmark.md @@ -0,0 +1,290 @@ +# Shared-Memory Governance Benchmark (SMGB) — Design + +**Status:** Draft for team review +**Author:** Muntasir Raihan Rahman (AMT) +**Scope:** A public, system-agnostic benchmark for the *governance* of shared +agent memory. Complements the single-user accuracy benchmarks reported in +PR #31 (LongMemEval, LoCoMo, BEAM 1M). It is designed to sit alongside a +companion multi-agent **consistency** harness (tracked separately), which +measures the durability and staleness of shared *writes* rather than the +governance of shared *reads*. + +Reference implementation: [`benchmarks/governance/`](../benchmarks/governance/). + +--- + +## 1. Problem + +PR #31 reports AMT scores on **single-user** memory benchmarks. Those measure +one question: *did the system recall the right fact for the one user in the +transcript?* They say nothing about what happens when memory is **shared** — +reused across agents, users, teams, tenants, and organizations — which is +precisely AMT's differentiator and the harder engineering problem. + +Once memory is shared, correctness is no longer just recall. It is **governance**: + +- **Authorization-filtered retrieval** — a reader receives only memories they are + entitled to, not everything in the store. +- **Private → shared promotion** — a fact authored privately becomes visible to a + broader scope *after* an explicit promotion, and not a moment before. +- **Tenant isolation** — memory never crosses an organization boundary, even when + scope names or questions collide. +- **Conflict / supersession** — when a fact is updated, the current view returns + only the live version; stale "ghosts" do not resurface. +- **Scope-aware deletion** — a deleted memory disappears for *every* member of its + scope, with no retrievable copies. +- **Provenance** — every shared fact traces to its author, source, and time. + +A system can ace every single-user benchmark and still leak a private note to a +teammate, surface a deleted workaround, or serve tenant A's revenue target to +tenant B. Nothing in the current evaluation stack would catch it. + +--- + +## 2. Gap analysis (verified) + +**No dedicated, publicly released agentic-shared-memory governance benchmark +exists.** Every benchmark in active use is single-user; only two 2026 papers +partially address the space, and neither is a usable public dataset for this +problem. + +### 2.1 In-use benchmarks are single-user + +| Benchmark | arXiv | Shared memory? | Governance? | +|---|---|---|---| +| LoCoMo (ACL 2024) | 2402.17753 | No — dyadic, single user | No | +| LongMemEval (ICLR 2025) | 2410.10813 | No — one user ↔ assistant | No | +| DMR (MemGPT, 2023) | 2310.08560 | No — single-user recall | No | +| BEAM | *unverified* | *unverified* | *unverified* | + +> BEAM (cited in PR #31) could not be confirmed as a peer-reviewed paper during +> this review — treat as gray literature and verify the source before relying on +> it in external comparisons. + +### 2.2 Two 2026 papers cover only a slice (both verified via the arXiv API) + +- **PiSAs — "Benchmarking Contextual Integrity in Multi-User Agentic Systems"** + (arXiv:2607.05318, Meta AI / Mila). Measures cross-user information spillage, + including through shared memory. **System-agnostic**, which is the right shape. + - *Covers:* the access-control / leakage slice. + - *Missing:* promotion, temporal supersession / stale propagation, provenance, + scope-aware deletion. + - *Limitation:* labels rely on **LLM contextual-integrity judgment** — noisy, + non-reproducible, and expensive to run. + +- **"Governed Shared Memory for Multi-Agent LLM Systems"** (MemClaw service + + **ArgusFleet** harness, arXiv:2606.24535). Formalizes the fleet-memory problem + and exactly the four failure modes SMGB targets — *unauthorized leakage, stale + propagation, contradiction persistence, provenance collapse.* + - *Covers:* the full governance slice — the dimensions are right. + - *Limitation:* it evaluates a **proprietary production service**, not a + standalone public dataset. It cannot be pointed at an arbitrary memory system. + +### 2.3 Where SMGB sits + +SMGB takes PiSAs' **system-agnostic** posture and ArgusFleet's **governance +dimensions**, and removes both of their blockers: + +- **No LLM judge.** Ground truth is derived *deterministically* from a scope + + policy graph. Scoring is reproducible, free, and unambiguous. +- **No proprietary dependency.** The dataset is plain JSONL and the scorer is a + dependency-free Python package that consumes a mapping of retrieved ids. + +This is the differentiator: a **reproducible, public, policy-labeled** governance +benchmark that any memory system can be scored against. + +--- + +## 3. Design principles + +1. **System-agnostic.** A system under test produces a *run*: for each query, a + ranked list of retrieved memory ids. The scorer needs nothing else — no model + calls, no SDK, no network. +2. **Deterministic labels.** For any principal at any time, the *allowed* and + *forbidden* memory sets are computed from the scope graph and an event + timeline. Labels are never hand-assigned in a way the harness trusts blindly. +3. **Self-checking data.** Scenarios may carry hand annotations + (`must_retrieve` / `must_not_retrieve`). The loader **validates every one + against the policy-derived labels** and rejects any mismatch, so authoring + bugs cannot silently corrupt the benchmark. +4. **Time is first-class.** Every query has an `as_of` timestamp; promotion, + supersession, and deletion are timeline events. The same question asked before + and after a promotion has different correct answers. +5. **Complementary, not redundant.** The consistency harness asks "is this read + the *latest write*?" SMGB asks "is this read the *right write for this + reader*?" Both are necessary; neither implies the other. + +--- + +## 4. Model + +### 4.1 Entities + +- **Scope** — a visibility unit (`user`, `agent`, `session`, `project`, `group`, + `account`, `org`, `tenant`), with an owning tenant and optional member list. +- **Principal** — a user or agent that issues reads/writes, with a tenant, roles, + and the set of scopes it belongs to. +- **Memory** — a record with a birth time (`created_at`), an **initial** scope, + a tenant, and provenance (author, source, confidence, `derived_from`). +- **Event** — a governance mutation on the timeline: `promote` (adds a broader + scope), `supersede` (a newer memory replaces it), `delete`. +- **Query** — a read by a principal at `as_of`, tagged with one **axis** and an + optional `temporal_mode` (`current` default, or `history`). + +Governance state is **not** stored on the memory; it is computed as of a query +time from the event timeline, keeping the dataset a single source of truth. + +### 4.2 Policy (the deterministic core) + +State of a memory as of `as_of`: + +- **exists** — `created_at <= as_of`. +- **scopes** — `{initial scope} ∪ {promote.to_scope : promote.t <= as_of}`. + Promotion *adds* a broader scope; the author keeps visibility because they are a + member of that broader scope. +- **superseded** — some `supersede` event with `t <= as_of` targeted it. +- **deleted** — some `delete` event with `t <= as_of` targeted it. + +A memory is **authorized** for a principal when, *in order*: it exists, is not +deleted, is in the **same tenant** (hard isolation, checked before scope), and +shares at least one scope with the principal. It is **allowed** for a query when +it is authorized and — unless the query is `history` mode — not superseded. +Everything else in the scenario is **forbidden**. + +``` +must_retrieve = relevant ∩ allowed # the utility target +forbidden = all_memories − allowed +cross_tenant = forbidden from another tenant # isolation signal +superseded_forbidden = forbidden only because stale # conflict signal +``` + +Checking tenant **before** scope matters: a cross-tenant memory is never +authorized even if two tenants happen to use the same scope id. + +--- + +## 5. Task axes and metrics + +| Axis | Question | Metric | +|------|----------|--------| +| `utility` | Does the principal get the facts they're entitled to? | `recall@k` over `must_retrieve` | +| `leakage` | Are unauthorized facts withheld? | `leak_rate` (queries with ≥1 forbidden hit); `total_leaks` (item count) | +| `isolation` | Is cross-tenant memory never returned? | `isolation_violations` (target: 0) | +| `promotion` | Are promoted facts visible after `T`, not before? | recall/leak at `as_of` before vs after the promote event | +| `conflict` | In `current` mode, only the live version is retrievable? | `stale_leaks` (superseded hits) | +| `deletion` | Are deleted facts (and copies) gone for all members? | leak on deletion-axis queries | +| `provenance` | Does every shared fact trace to the right author/source? | `provenance_accuracy` | + +**Headline metrics** reported per system: `mean_recall`, `leak_rate`, +`total_leaks`, `isolation_violations`, `stale_leaks`, `provenance_accuracy`, +plus a per-axis rollup. + +--- + +## 6. Data format + +One JSON object per line (JSONL); each scenario is fully self-contained. See the +package [README](../benchmarks/governance/README.md#dataset-format) for a +complete example. A scenario declares `tenants`, `scopes`, `principals`, +`memories`, `events`, and `queries`; a query names its `principal`, `as_of`, +`axis`, `relevant` ids, optional `temporal_mode`, and optional +`must_retrieve` / `must_not_retrieve` annotations. + +### 6.1 Seed dataset + +Six scenarios ship in `benchmarks/governance/data/seed_scenarios.jsonl` (14 +queries), each anchored in a concrete situation: + +| Scenario | Governance property under test | +|----------|-------------------------------| +| `acct_satya_steve` | Account-scope sharing + private notes stay private (promotion, utility, leakage) | +| `proj_scott_deploy` | Promote a deploy fix to project scope; working prefs stay private | +| `tenant_isolation_revenue` | Two tenants, identically shaped facts; neither may see the other's | +| `conflict_contact_supersession` | Current-vs-history retrieval of a superseded fact | +| `deletion_workaround` | A deleted shared memory is gone for all scope members | +| `provenance_reranker` | A shared decision traces to its author/source; derived fact keeps its parent link | + +The first two encode the original whiteboard brainstorm (account- and +project-scoped promotion). The seed set is small by design — its job is to +**anchor the axes and prove the metrics discriminate**, not to be a leaderboard. + +--- + +## 7. Reference runners and the key result + +Three baseline "systems" are included so the metrics can be validated offline and +to anchor a leaderboard: + +- **oracle** — returns exactly the authorized-and-relevant target set. +- **naive_shared** — returns everything in the principal's tenant, ignoring scope + and supersession (no authorization filter). +- **naive_global** — returns everything across all tenants. + +On the seed set: + +``` +system queries mean_recall leak_rate total_leaks isolation_violations stale_leaks +----------------------------------------------------------------------------------------- +oracle 14 1.000 0.000 0 0 0 +naive_shared 14 1.000 0.500 13 0 1 +naive_global 14 1.000 0.643 15 2 1 +``` + +**This table is the whole argument for the benchmark.** All three baselines score +`mean_recall = 1.000` — a recall-only benchmark (i.e. every single-user benchmark) +would rate them identical. Only the leakage, isolation, and stale-propagation axes +reveal that `naive_shared` leaks 13 facts and `naive_global` additionally breaches +tenant isolation twice. **Recall cannot distinguish a governed memory system from +a naive one; SMGB can.** + +--- + +## 8. Harness integration + +- **Package:** `benchmarks/governance/` — `schema.py`, `policy.py`, `scorer.py`, + `loader.py`, `run_governance.py`, `data/`. +- **CLI:** `python -m benchmarks.governance.run_governance --reference` + (or `--validate-only`, or `--run my_run.json --system name`, with `--k` and + `--json-out`). +- **Tests:** `benchmarks/tests/test_governance_{policy,scorer,seed}.py` — policy + truth tables, scorer discrimination, and seed-dataset integrity. Run with + `python -m pytest benchmarks/tests -q`; lint with `ruff check benchmarks/governance`. +- **Live adapter (future).** SMGB maps cleanly onto AMT's public API: scope encoded + via `add_cosmos(..., tags=, metadata=)`; authorization-filtered retrieval via + `search_cosmos(..., tags_all/tags_any/exclude_tags)`; `history` mode via + `include_superseded`; `as_of` via `created_after/before`. A thin adapter that + replays scenarios against Cosmos DB turns SMGB into a live governance test for + the toolkit itself. + +--- + +## 9. Differentiation summary + +| | Single-user (LoCoMo, LongMemEval, DMR) | PiSAs (2607.05318) | ArgusFleet (2606.24535) | **SMGB** | +|---|---|---|---|---| +| Shared memory | No | Partial | Yes | **Yes** | +| Authorization / leakage | — | Yes | Yes | **Yes** | +| Promotion (private→shared) | — | No | Yes | **Yes** | +| Tenant isolation | — | Partial | Yes | **Yes** | +| Supersession / stale | — | No | Yes | **Yes** | +| Scope-aware deletion | — | No | Partial | **Yes** | +| Provenance | — | No | Yes | **Yes** | +| Labels | reference answers | **LLM-judged** | policy (proprietary) | **deterministic, public** | +| Public dataset | Yes | Yes | **No** | **Yes** | + +--- + +## 10. Open questions for review + +1. **Scenario coverage.** The seed set is six scenarios. Which real AMT + customer/agent situations should the v1 corpus prioritize (support handoff, + multi-tenant SaaS, project teams, org rollouts)? +2. **Scope taxonomy.** Is `user / agent / session / project / group / account / + org / tenant` the right ladder, or do we need finer role-based scopes? +3. **Live adapter scope.** Ship the read-only Cosmos adapter in v1, or keep v1 + offline (dataset + scorer) and add the adapter in a follow-up? +4. **Provenance depth.** The schema supports `derived_from` chains (cf. + ArgusFleet's depth-N reconstruction). Do we score multi-hop provenance in v1? +5. **Naming / publication.** If we intend to release this publicly, confirm the + name and whether it should be positioned as an AMT artifact or a standalone + benchmark. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..fff6401 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Top-level marker for the benchmarks package.""" diff --git a/benchmarks/governance/README.md b/benchmarks/governance/README.md new file mode 100644 index 0000000..1913e87 --- /dev/null +++ b/benchmarks/governance/README.md @@ -0,0 +1,178 @@ +# Shared-Memory Governance Benchmark (SMGB) + +A system-agnostic benchmark for evaluating **shared** agent memory — memory +reused across agents, users, teams, tenants, and organizations — on the +**governance** axes that single-user memory benchmarks (LoCoMo, LongMemEval, +DMR) do not test. + +Single-user benchmarks ask *"did the system recall the right fact?"* SMGB adds +the question that matters once memory is shared: *"did the system return the +right fact **to the right principal, at the right time, and nothing it was not +entitled to**?"* + +> **Two questions, two harnesses.** SMGB scores *governance* — are shared reads +> *authorized and scoped correctly* (the **right writes for this reader**)? A +> companion multi-agent **consistency** harness (tracked separately) scores +> whether shared writes are *durable and current* under concurrency (the **right +> latest writes**, per Golab et al.). Storage-correct ≠ governance-correct; a +> full evaluation needs both. + +--- + +## Why this exists + +There is **no public benchmark** for agentic shared-memory governance. Every +in-use memory benchmark is single-user. The two 2026 papers that touch the space +each cover only a slice: + +- **PiSAs** (arXiv:2607.05318) — cross-user privacy spillage, but the + access-control slice only, and labels are **LLM-judged**. +- **Governed Shared Memory / ArgusFleet** (arXiv:2606.24535) — the full + governance slice (leakage, stale propagation, contradiction, provenance) but + tied to a **proprietary** service; no public dataset. + +SMGB's differentiator: labels are **derived deterministically** from a scope + +policy graph (no LLM judge), so scoring is reproducible and cheap. See +[`Docs/shared_memory_governance_benchmark.md`](../../Docs/shared_memory_governance_benchmark.md) +for the full design and gap analysis. + +--- + +## Quick start + +Run from the repo root with the package importable: + +```bash +# score the built-in reference runners (oracle + two naive baselines) on the seed set +python -m benchmarks.governance.run_governance --reference + +# validate the dataset only (checks referential integrity + hand labels vs policy) +python -m benchmarks.governance.run_governance --validate-only + +# score an external system from a run file +python -m benchmarks.governance.run_governance --run my_run.json --system my_system +``` + +Reference leaderboard on the seed set: + +``` +system queries mean_recall leak_rate total_leaks isolation_violations stale_leaks +----------------------------------------------------------------------------------------- +oracle 14 1.000 0.000 0 0 0 +naive_shared 14 1.000 0.500 13 0 1 +naive_global 14 1.000 0.643 15 2 1 +``` + +The takeaway the benchmark is built to make visible: **recall alone cannot tell +a governed system from a naive one** — all three score `mean_recall = 1.000`. +The leakage, isolation, and stale-propagation axes are what separate them. + +--- + +## The seven axes + +| Axis | Question | Metric | +|------|----------|--------| +| `utility` | Does the principal get the facts they're entitled to? | `recall@k` over authorized-and-relevant targets | +| `leakage` | Are unauthorized facts withheld? | `leak_rate` (queries with any forbidden hit), `total_leaks` | +| `isolation` | Is cross-tenant memory never returned? | `isolation_violations` (must be 0) | +| `promotion` | Are promoted facts visible after `T` to the new scope, not before? | recall/leak evaluated at `as_of` before vs after promotion | +| `conflict` | In `current` mode, only the live version is retrievable? | `stale_leaks` (superseded "ghost" hits) | +| `deletion` | Are deleted facts (and copies) gone for all scope members? | leak on deletion-axis queries | +| `provenance` | Does every shared fact trace to the right author/source? | `provenance_accuracy` | + +--- + +## How scoring works + +A **run** is a plain mapping — no SDK, no model calls required to score: + +```json +{ "": { "": ["", "..."] } } +``` + +For each query the scorer: + +1. computes the memory's **state as of the query time** — current scope set + (initial scope plus any `promote` targets), plus `superseded` / `deleted` + flags — from the event timeline (`policy.memory_state`); +2. derives **authorized** (exists, not deleted, same tenant, scope overlap) and + **allowed** (authorized and — outside `history` mode — not superseded), and + labels everything else **forbidden** (`policy.compute_query_labels`); +3. intersects the run with those label sets to produce per-axis metrics + (`scorer.score_scenario`). + +Because the labels come from the policy layer, not a judge, the dataset is +self-checking: the loader **validates every hand annotation against the derived +labels** (`loader.validate_scenario`) and refuses a scenario whose +`must_retrieve` / `must_not_retrieve` disagree with policy. + +--- + +## Package layout + +| File | Role | +|------|------| +| `schema.py` | Dataclasses: `Scope`, `Principal`, `Provenance`, `Memory`, `Event`, `Query`, `Scenario`; `parse_time`, `QUERY_AXES`, `EVENT_TYPES`. | +| `policy.py` | Deterministic core: `memory_state`, `is_authorized`, `compute_query_labels`. All labels derive here. | +| `scorer.py` | System-agnostic scoring, the `Report` aggregates, `merge_reports`, and the reference runners. | +| `loader.py` | `load_scenarios`, `validate_scenario` / `validate_all` (referential integrity + hand labels vs policy). | +| `run_governance.py` | CLI: `--reference`, `--run`, `--validate-only`, `--json-out`, `--k`. | +| `data/seed_scenarios.jsonl` | The checked-in seed dataset (6 scenarios, all validated). | +| `data/_generate_seed.py` | Reproducible builder for the seed file. | + +Tests live in [`../tests/`](../tests/): +`test_governance_policy.py`, `test_governance_scorer.py`, +`test_governance_seed.py`. + +--- + +## Dataset format + +One JSON object per line (JSONL). A scenario is fully self-contained: + +```json +{ + "scenario_id": "acct_satya_steve", + "tenants": ["contoso"], + "scopes": [ + {"id": "user:satya", "kind": "user", "tenant": "contoso", "members": ["satya"]}, + {"id": "acct:northwind", "kind": "account", "tenant": "contoso", "members": ["satya", "steve"]} + ], + "principals": [ + {"id": "satya", "tenant": "contoso", "scopes": ["user:satya", "acct:northwind"]}, + {"id": "steve", "tenant": "contoso", "scopes": ["acct:northwind"]} + ], + "memories": [ + {"id": "m_pricing", "scope": "user:satya", "tenant": "contoso", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "satya-agent"}} + ], + "events": [ + {"t": "2026-02-01T00:00:00Z", "type": "promote", "memory_id": "m_pricing", "to_scope": "acct:northwind"} + ], + "queries": [ + {"id": "q_before", "principal": "steve", "as_of": "2026-01-15T00:00:00Z", + "axis": "promotion", "relevant": ["m_pricing"], "must_retrieve": []}, + {"id": "q_after", "principal": "steve", "as_of": "2026-02-15T00:00:00Z", + "axis": "promotion", "relevant": ["m_pricing"], "must_retrieve": ["m_pricing"]} + ] +} +``` + +`must_retrieve` / `must_not_retrieve` are **optional** hand annotations; when +present the loader checks them against the policy-derived labels, which is how +the dataset catches its own authoring bugs. + +--- + +## Extending the benchmark + +- **Add scenarios**: append lines to `seed_scenarios.jsonl` (or add a new + `.jsonl` and point `--data` at the directory), then run `--validate-only`. +- **Score your system**: emit a run file in the format above and pass `--run`. + A thin live adapter can map scopes onto AMT's real API — `tags` / `metadata` + for scope, `include_superseded` for history mode, `created_after/before` for + `as_of` — so the same scenarios can be replayed against Cosmos DB. +- **Add an axis or event type**: extend `QUERY_AXES` / `EVENT_TYPES` in + `schema.py`, teach `policy.py` how it changes state or labels, and add a + metric in `scorer.py`. diff --git a/benchmarks/governance/__init__.py b/benchmarks/governance/__init__.py new file mode 100644 index 0000000..ba33e68 --- /dev/null +++ b/benchmarks/governance/__init__.py @@ -0,0 +1,62 @@ +"""Shared-Memory Governance Benchmark (SMGB). + +A system-agnostic benchmark for evaluating *shared* agent memory — memory +reused across agents, users, teams, tenants, and organizations — on the +governance axes that single-user memory benchmarks (LoCoMo, LongMemEval, DMR) +do not test: authorization-filtered retrieval, private->shared promotion, +tenant isolation, conflict/supersession, scope-aware deletion, and provenance. + +See ``benchmarks/governance/README.md`` and +``Docs/shared_memory_governance_benchmark.md`` for the design. +""" + +from __future__ import annotations + +from .loader import ValidationError, load_scenarios, validate_all, validate_scenario +from .policy import compute_query_labels, is_authorized, memory_state +from .schema import ( + Event, + Memory, + Principal, + Provenance, + Query, + Scenario, + Scope, +) +from .scorer import ( + REFERENCE_RUNNERS, + Report, + Run, + merge_reports, + naive_global_run, + naive_shared_run, + oracle_provenance, + oracle_run, + score_scenario, +) + +__all__ = [ + "Event", + "Memory", + "Principal", + "Provenance", + "Query", + "Scenario", + "Scope", + "compute_query_labels", + "is_authorized", + "memory_state", + "load_scenarios", + "validate_all", + "validate_scenario", + "ValidationError", + "REFERENCE_RUNNERS", + "Report", + "Run", + "merge_reports", + "naive_global_run", + "naive_shared_run", + "oracle_provenance", + "oracle_run", + "score_scenario", +] diff --git a/benchmarks/governance/data/_generate_seed.py b/benchmarks/governance/data/_generate_seed.py new file mode 100644 index 0000000..115a820 --- /dev/null +++ b/benchmarks/governance/data/_generate_seed.py @@ -0,0 +1,288 @@ +"""Throwaway builder that emits the SMGB seed dataset as JSONL. + +Run once to (re)generate ``seed_scenarios.jsonl``. The committed artifact is the +JSONL file; this script documents how it was produced and guarantees valid JSON. +Hand annotations (must_retrieve / must_not_retrieve) are included so the loader +can validate them against policy-derived labels. +""" + +import json +from pathlib import Path + +OUT = Path(__file__).with_name("seed_scenarios.jsonl") + + +def scenarios(): + return [ + # -- Whiteboard scenario 1: account-scope sharing (Satya / Steve) ------- + { + "scenario_id": "acct_satya_steve", + "description": "Two support engineers share account-scoped customer facts; " + "personal notes stay private. Covers promotion, utility, leakage.", + "tenants": ["contoso"], + "scopes": [ + {"id": "user:satya", "kind": "user", "tenant": "contoso", "members": ["satya"]}, + {"id": "user:steve", "kind": "user", "tenant": "contoso", "members": ["steve"]}, + {"id": "acct:northwind", "kind": "account", "tenant": "contoso", + "members": ["satya", "steve"]}, + ], + "principals": [ + {"id": "satya", "kind": "user", "tenant": "contoso", "roles": ["support"], + "scopes": ["user:satya", "acct:northwind"]}, + {"id": "steve", "kind": "user", "tenant": "contoso", "roles": ["support"], + "scopes": ["user:steve", "acct:northwind"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", "content": "Northwind uses a custom SSO setup.", + "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:00:00Z", + "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.9}}, + {"id": "m2", "type": "fact", + "content": "Northwind cannot take upgrades during their Tuesday billing cycle.", + "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:05:00Z", + "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.9}}, + {"id": "m3", "type": "fact", "content": "Northwind's admin prefers short emails.", + "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:10:00Z", + "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.6}}, + ], + "events": [ + {"t": "2026-06-01T18:00:00Z", "type": "promote", "memory_id": "m1", + "to_scope": "acct:northwind", "actor": "satya"}, + {"t": "2026-06-01T18:00:00Z", "type": "promote", "memory_id": "m2", + "to_scope": "acct:northwind", "actor": "satya"}, + ], + "queries": [ + {"id": "q1", "principal": "steve", "as_of": "2026-06-01T12:00:00Z", + "axis": "promotion", + "question": "Any special setup notes for Northwind?", + "relevant": ["m1", "m2"], "expected_answer": "", + "must_retrieve": [], "must_not_retrieve": ["m1", "m2", "m3"]}, + {"id": "q2", "principal": "steve", "as_of": "2026-06-02T09:00:00Z", + "axis": "utility", + "question": "Any special setup notes for Northwind?", + "relevant": ["m1", "m2"], + "expected_answer": "Custom SSO; no upgrades during the Tuesday billing cycle.", + "must_retrieve": ["m1", "m2"], "must_not_retrieve": ["m3"]}, + {"id": "q3", "principal": "steve", "as_of": "2026-06-02T09:00:00Z", + "axis": "leakage", + "question": "How does Northwind's admin like their emails?", + "relevant": ["m3"], "expected_answer": "", + "must_retrieve": [], "must_not_retrieve": ["m3"]}, + {"id": "q4", "principal": "satya", "as_of": "2026-06-02T09:00:00Z", + "axis": "utility", + "question": "What do we know about Northwind?", + "relevant": ["m1", "m2", "m3"], + "must_retrieve": ["m1", "m2", "m3"], "must_not_retrieve": []}, + ], + }, + # -- Whiteboard scenario 2: project-scope promotion (Scott) ------------ + { + "scenario_id": "proj_scott_deploy", + "description": "Scott promotes a deploy fix to project scope; his working " + "preferences stay private. Covers promotion, utility, leakage.", + "tenants": ["fabrikam"], + "scopes": [ + {"id": "user:scott", "kind": "user", "tenant": "fabrikam", "members": ["scott"]}, + {"id": "user:dana", "kind": "user", "tenant": "fabrikam", "members": ["dana"]}, + {"id": "proj:deploy", "kind": "project", "tenant": "fabrikam", + "members": ["scott", "dana"]}, + ], + "principals": [ + {"id": "scott", "kind": "user", "tenant": "fabrikam", "roles": ["sre"], + "scopes": ["user:scott", "proj:deploy"]}, + {"id": "dana", "kind": "user", "tenant": "fabrikam", "roles": ["sre"], + "scopes": ["user:dana", "proj:deploy"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", + "content": "Prod deploy failed due to missing DATABASE_URL; fix: set it in the " + "release pipeline secrets.", + "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:00:00Z", + "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.95}}, + {"id": "m2", "type": "fact", "content": "Scott prefers to be notified via Teams, not email.", + "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:05:00Z", + "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.7}}, + {"id": "m3", "type": "fact", "content": "Scott likes status updates as three bullet points.", + "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:10:00Z", + "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.7}}, + ], + "events": [ + {"t": "2026-06-10T17:00:00Z", "type": "promote", "memory_id": "m1", + "to_scope": "proj:deploy", "actor": "scott"}, + ], + "queries": [ + {"id": "q1", "principal": "dana", "as_of": "2026-06-11T09:00:00Z", + "axis": "utility", + "question": "How was the last prod deploy failure resolved?", + "relevant": ["m1"], + "expected_answer": "A missing DATABASE_URL; it was set in the release pipeline secrets.", + "must_retrieve": ["m1"], "must_not_retrieve": ["m2", "m3"]}, + {"id": "q2", "principal": "dana", "as_of": "2026-06-11T09:00:00Z", + "axis": "leakage", + "question": "How does Scott prefer to receive notifications?", + "relevant": ["m2"], "expected_answer": "", + "must_retrieve": [], "must_not_retrieve": ["m2", "m3"]}, + {"id": "q3", "principal": "dana", "as_of": "2026-06-10T15:00:00Z", + "axis": "promotion", + "question": "How was the last prod deploy failure resolved?", + "relevant": ["m1"], "expected_answer": "", + "must_retrieve": [], "must_not_retrieve": ["m1", "m2", "m3"]}, + ], + }, + # -- Tenant isolation -------------------------------------------------- + { + "scenario_id": "tenant_isolation_revenue", + "description": "Two tenants with identically shaped org-scope facts; neither " + "principal may ever see the other tenant's memory.", + "tenants": ["contoso", "initech"], + "scopes": [ + {"id": "org:contoso", "kind": "org", "tenant": "contoso", "members": ["alice"]}, + {"id": "org:initech", "kind": "org", "tenant": "initech", "members": ["mallory"]}, + ], + "principals": [ + {"id": "alice", "kind": "user", "tenant": "contoso", "scopes": ["org:contoso"]}, + {"id": "mallory", "kind": "user", "tenant": "initech", "scopes": ["org:initech"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", "content": "Contoso Q3 revenue target is $5M.", + "tenant": "contoso", "scope": "org:contoso", "created_at": "2026-06-01T09:00:00Z", + "provenance": {"author": "alice-agent"}}, + {"id": "m2", "type": "fact", "content": "Initech Q3 revenue target is $3M.", + "tenant": "initech", "scope": "org:initech", "created_at": "2026-06-01T09:00:00Z", + "provenance": {"author": "mallory-agent"}}, + ], + "events": [], + "queries": [ + {"id": "q1", "principal": "mallory", "as_of": "2026-06-02T09:00:00Z", + "axis": "isolation", + "question": "What is the Q3 revenue target?", + "relevant": ["m1", "m2"], "expected_answer": "$3M (Initech).", + "must_retrieve": ["m2"], "must_not_retrieve": ["m1"]}, + {"id": "q2", "principal": "alice", "as_of": "2026-06-02T09:00:00Z", + "axis": "isolation", + "question": "What is the Q3 revenue target?", + "relevant": ["m1", "m2"], "expected_answer": "$5M (Contoso).", + "must_retrieve": ["m1"], "must_not_retrieve": ["m2"]}, + ], + }, + # -- Conflict / supersession ------------------------------------------ + { + "scenario_id": "conflict_contact_supersession", + "description": "A shared fact is superseded by a newer one; current queries " + "must return only the current version, history queries may return the old.", + "tenants": ["globex"], + "scopes": [ + {"id": "proj:crm", "kind": "project", "tenant": "globex", + "members": ["ravi", "mei"]}, + ], + "principals": [ + {"id": "ravi", "kind": "user", "tenant": "globex", "scopes": ["proj:crm"]}, + {"id": "mei", "kind": "user", "tenant": "globex", "scopes": ["proj:crm"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", + "content": "Acme's primary contact is Jordan (jordan@acme.com).", + "tenant": "globex", "scope": "proj:crm", "created_at": "2026-05-01T10:00:00Z", + "provenance": {"author": "ravi-agent"}}, + {"id": "m2", "type": "fact", + "content": "Acme's primary contact is Priya (priya@acme.com).", + "tenant": "globex", "scope": "proj:crm", "created_at": "2026-06-01T10:00:00Z", + "provenance": {"author": "mei-agent"}}, + ], + "events": [ + {"t": "2026-06-01T10:00:00Z", "type": "supersede", "memory_id": "m1", "by": "m2"}, + ], + "queries": [ + {"id": "q1", "principal": "mei", "as_of": "2026-06-02T09:00:00Z", + "axis": "conflict", "temporal_mode": "current", + "question": "Who is Acme's primary contact?", + "relevant": ["m1", "m2"], "expected_answer": "Priya (priya@acme.com).", + "must_retrieve": ["m2"], "must_not_retrieve": ["m1"]}, + {"id": "q2", "principal": "mei", "as_of": "2026-06-02T09:00:00Z", + "axis": "conflict", "temporal_mode": "history", + "question": "Who was Acme's primary contact before the change?", + "relevant": ["m1"], "expected_answer": "Jordan (jordan@acme.com).", + "must_retrieve": ["m1"]}, + ], + }, + # -- Scope-aware deletion --------------------------------------------- + { + "scenario_id": "deletion_workaround", + "description": "A shared memory is deleted; afterward it must not be retrievable " + "for any scope member.", + "tenants": ["umbrella"], + "scopes": [ + {"id": "team:support", "kind": "group", "tenant": "umbrella", + "members": ["nina", "omar"]}, + ], + "principals": [ + {"id": "nina", "kind": "user", "tenant": "umbrella", "scopes": ["team:support"]}, + {"id": "omar", "kind": "user", "tenant": "umbrella", "scopes": ["team:support"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", + "content": "Workaround: restart the ingestion service to clear the stuck queue.", + "tenant": "umbrella", "scope": "team:support", "created_at": "2026-06-05T10:00:00Z", + "provenance": {"author": "nina-agent"}}, + ], + "events": [ + {"t": "2026-06-06T10:00:00Z", "type": "delete", "memory_id": "m1", "actor": "nina"}, + ], + "queries": [ + {"id": "q1", "principal": "omar", "as_of": "2026-06-05T12:00:00Z", + "axis": "utility", + "question": "Any workaround for the stuck queue?", + "relevant": ["m1"], "expected_answer": "Restart the ingestion service.", + "must_retrieve": ["m1"], "must_not_retrieve": []}, + {"id": "q2", "principal": "omar", "as_of": "2026-06-07T10:00:00Z", + "axis": "deletion", + "question": "Any workaround for the stuck queue?", + "relevant": ["m1"], "expected_answer": "", + "must_retrieve": [], "must_not_retrieve": ["m1"]}, + ], + }, + # -- Provenance -------------------------------------------------------- + { + "scenario_id": "provenance_reranker", + "description": "Shared decisions must trace to their author/source; a derived " + "fact keeps a link to its parent.", + "tenants": ["hooli"], + "scopes": [ + {"id": "proj:search", "kind": "project", "tenant": "hooli", + "members": ["wade", "zoe"]}, + ], + "principals": [ + {"id": "wade", "kind": "user", "tenant": "hooli", "scopes": ["proj:search"]}, + {"id": "zoe", "kind": "user", "tenant": "hooli", "scopes": ["proj:search"]}, + ], + "memories": [ + {"id": "m1", "type": "fact", + "content": "The search reranker must cap tail latency at 200ms.", + "tenant": "hooli", "scope": "proj:search", "created_at": "2026-06-03T10:00:00Z", + "provenance": {"author": "wade-agent", "source": "thread:sprint-42", "confidence": 0.9}}, + {"id": "m2", "type": "fact", + "content": "Chosen approach: use the lightweight cross-encoder to stay under budget.", + "tenant": "hooli", "scope": "proj:search", "created_at": "2026-06-03T11:00:00Z", + "provenance": {"author": "zoe-agent", "source": "thread:sprint-42", + "confidence": 0.85, "derived_from": ["m1"]}}, + ], + "events": [], + "queries": [ + {"id": "q1", "principal": "zoe", "as_of": "2026-06-04T09:00:00Z", + "axis": "provenance", + "question": "What is the reranker latency budget and who set it?", + "relevant": ["m1"], + "expected_answer": "200ms tail latency, set by wade-agent in sprint-42.", + "must_retrieve": ["m1"], "must_not_retrieve": []}, + ], + }, + ] + + +def main(): + lines = [json.dumps(s, ensure_ascii=False) for s in scenarios()] + OUT.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"wrote {len(lines)} scenarios to {OUT}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/governance/data/seed_scenarios.jsonl b/benchmarks/governance/data/seed_scenarios.jsonl new file mode 100644 index 0000000..bce8d7d --- /dev/null +++ b/benchmarks/governance/data/seed_scenarios.jsonl @@ -0,0 +1,6 @@ +{"scenario_id": "acct_satya_steve", "description": "Two support engineers share account-scoped customer facts; personal notes stay private. Covers promotion, utility, leakage.", "tenants": ["contoso"], "scopes": [{"id": "user:satya", "kind": "user", "tenant": "contoso", "members": ["satya"]}, {"id": "user:steve", "kind": "user", "tenant": "contoso", "members": ["steve"]}, {"id": "acct:northwind", "kind": "account", "tenant": "contoso", "members": ["satya", "steve"]}], "principals": [{"id": "satya", "kind": "user", "tenant": "contoso", "roles": ["support"], "scopes": ["user:satya", "acct:northwind"]}, {"id": "steve", "kind": "user", "tenant": "contoso", "roles": ["support"], "scopes": ["user:steve", "acct:northwind"]}], "memories": [{"id": "m1", "type": "fact", "content": "Northwind uses a custom SSO setup.", "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:00:00Z", "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.9}}, {"id": "m2", "type": "fact", "content": "Northwind cannot take upgrades during their Tuesday billing cycle.", "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:05:00Z", "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.9}}, {"id": "m3", "type": "fact", "content": "Northwind's admin prefers short emails.", "tenant": "contoso", "scope": "user:satya", "created_at": "2026-06-01T09:10:00Z", "provenance": {"author": "satya-agent", "source": "thread:nw-101", "confidence": 0.6}}], "events": [{"t": "2026-06-01T18:00:00Z", "type": "promote", "memory_id": "m1", "to_scope": "acct:northwind", "actor": "satya"}, {"t": "2026-06-01T18:00:00Z", "type": "promote", "memory_id": "m2", "to_scope": "acct:northwind", "actor": "satya"}], "queries": [{"id": "q1", "principal": "steve", "as_of": "2026-06-01T12:00:00Z", "axis": "promotion", "question": "Any special setup notes for Northwind?", "relevant": ["m1", "m2"], "expected_answer": "", "must_retrieve": [], "must_not_retrieve": ["m1", "m2", "m3"]}, {"id": "q2", "principal": "steve", "as_of": "2026-06-02T09:00:00Z", "axis": "utility", "question": "Any special setup notes for Northwind?", "relevant": ["m1", "m2"], "expected_answer": "Custom SSO; no upgrades during the Tuesday billing cycle.", "must_retrieve": ["m1", "m2"], "must_not_retrieve": ["m3"]}, {"id": "q3", "principal": "steve", "as_of": "2026-06-02T09:00:00Z", "axis": "leakage", "question": "How does Northwind's admin like their emails?", "relevant": ["m3"], "expected_answer": "", "must_retrieve": [], "must_not_retrieve": ["m3"]}, {"id": "q4", "principal": "satya", "as_of": "2026-06-02T09:00:00Z", "axis": "utility", "question": "What do we know about Northwind?", "relevant": ["m1", "m2", "m3"], "must_retrieve": ["m1", "m2", "m3"], "must_not_retrieve": []}]} +{"scenario_id": "proj_scott_deploy", "description": "Scott promotes a deploy fix to project scope; his working preferences stay private. Covers promotion, utility, leakage.", "tenants": ["fabrikam"], "scopes": [{"id": "user:scott", "kind": "user", "tenant": "fabrikam", "members": ["scott"]}, {"id": "user:dana", "kind": "user", "tenant": "fabrikam", "members": ["dana"]}, {"id": "proj:deploy", "kind": "project", "tenant": "fabrikam", "members": ["scott", "dana"]}], "principals": [{"id": "scott", "kind": "user", "tenant": "fabrikam", "roles": ["sre"], "scopes": ["user:scott", "proj:deploy"]}, {"id": "dana", "kind": "user", "tenant": "fabrikam", "roles": ["sre"], "scopes": ["user:dana", "proj:deploy"]}], "memories": [{"id": "m1", "type": "fact", "content": "Prod deploy failed due to missing DATABASE_URL; fix: set it in the release pipeline secrets.", "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:00:00Z", "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.95}}, {"id": "m2", "type": "fact", "content": "Scott prefers to be notified via Teams, not email.", "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:05:00Z", "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.7}}, {"id": "m3", "type": "fact", "content": "Scott likes status updates as three bullet points.", "tenant": "fabrikam", "scope": "user:scott", "created_at": "2026-06-10T14:10:00Z", "provenance": {"author": "scott-agent", "source": "thread:inc-556", "confidence": 0.7}}], "events": [{"t": "2026-06-10T17:00:00Z", "type": "promote", "memory_id": "m1", "to_scope": "proj:deploy", "actor": "scott"}], "queries": [{"id": "q1", "principal": "dana", "as_of": "2026-06-11T09:00:00Z", "axis": "utility", "question": "How was the last prod deploy failure resolved?", "relevant": ["m1"], "expected_answer": "A missing DATABASE_URL; it was set in the release pipeline secrets.", "must_retrieve": ["m1"], "must_not_retrieve": ["m2", "m3"]}, {"id": "q2", "principal": "dana", "as_of": "2026-06-11T09:00:00Z", "axis": "leakage", "question": "How does Scott prefer to receive notifications?", "relevant": ["m2"], "expected_answer": "", "must_retrieve": [], "must_not_retrieve": ["m2", "m3"]}, {"id": "q3", "principal": "dana", "as_of": "2026-06-10T15:00:00Z", "axis": "promotion", "question": "How was the last prod deploy failure resolved?", "relevant": ["m1"], "expected_answer": "", "must_retrieve": [], "must_not_retrieve": ["m1", "m2", "m3"]}]} +{"scenario_id": "tenant_isolation_revenue", "description": "Two tenants with identically shaped org-scope facts; neither principal may ever see the other tenant's memory.", "tenants": ["contoso", "initech"], "scopes": [{"id": "org:contoso", "kind": "org", "tenant": "contoso", "members": ["alice"]}, {"id": "org:initech", "kind": "org", "tenant": "initech", "members": ["mallory"]}], "principals": [{"id": "alice", "kind": "user", "tenant": "contoso", "scopes": ["org:contoso"]}, {"id": "mallory", "kind": "user", "tenant": "initech", "scopes": ["org:initech"]}], "memories": [{"id": "m1", "type": "fact", "content": "Contoso Q3 revenue target is $5M.", "tenant": "contoso", "scope": "org:contoso", "created_at": "2026-06-01T09:00:00Z", "provenance": {"author": "alice-agent"}}, {"id": "m2", "type": "fact", "content": "Initech Q3 revenue target is $3M.", "tenant": "initech", "scope": "org:initech", "created_at": "2026-06-01T09:00:00Z", "provenance": {"author": "mallory-agent"}}], "events": [], "queries": [{"id": "q1", "principal": "mallory", "as_of": "2026-06-02T09:00:00Z", "axis": "isolation", "question": "What is the Q3 revenue target?", "relevant": ["m1", "m2"], "expected_answer": "$3M (Initech).", "must_retrieve": ["m2"], "must_not_retrieve": ["m1"]}, {"id": "q2", "principal": "alice", "as_of": "2026-06-02T09:00:00Z", "axis": "isolation", "question": "What is the Q3 revenue target?", "relevant": ["m1", "m2"], "expected_answer": "$5M (Contoso).", "must_retrieve": ["m1"], "must_not_retrieve": ["m2"]}]} +{"scenario_id": "conflict_contact_supersession", "description": "A shared fact is superseded by a newer one; current queries must return only the current version, history queries may return the old.", "tenants": ["globex"], "scopes": [{"id": "proj:crm", "kind": "project", "tenant": "globex", "members": ["ravi", "mei"]}], "principals": [{"id": "ravi", "kind": "user", "tenant": "globex", "scopes": ["proj:crm"]}, {"id": "mei", "kind": "user", "tenant": "globex", "scopes": ["proj:crm"]}], "memories": [{"id": "m1", "type": "fact", "content": "Acme's primary contact is Jordan (jordan@acme.com).", "tenant": "globex", "scope": "proj:crm", "created_at": "2026-05-01T10:00:00Z", "provenance": {"author": "ravi-agent"}}, {"id": "m2", "type": "fact", "content": "Acme's primary contact is Priya (priya@acme.com).", "tenant": "globex", "scope": "proj:crm", "created_at": "2026-06-01T10:00:00Z", "provenance": {"author": "mei-agent"}}], "events": [{"t": "2026-06-01T10:00:00Z", "type": "supersede", "memory_id": "m1", "by": "m2"}], "queries": [{"id": "q1", "principal": "mei", "as_of": "2026-06-02T09:00:00Z", "axis": "conflict", "temporal_mode": "current", "question": "Who is Acme's primary contact?", "relevant": ["m1", "m2"], "expected_answer": "Priya (priya@acme.com).", "must_retrieve": ["m2"], "must_not_retrieve": ["m1"]}, {"id": "q2", "principal": "mei", "as_of": "2026-06-02T09:00:00Z", "axis": "conflict", "temporal_mode": "history", "question": "Who was Acme's primary contact before the change?", "relevant": ["m1"], "expected_answer": "Jordan (jordan@acme.com).", "must_retrieve": ["m1"]}]} +{"scenario_id": "deletion_workaround", "description": "A shared memory is deleted; afterward it must not be retrievable for any scope member.", "tenants": ["umbrella"], "scopes": [{"id": "team:support", "kind": "group", "tenant": "umbrella", "members": ["nina", "omar"]}], "principals": [{"id": "nina", "kind": "user", "tenant": "umbrella", "scopes": ["team:support"]}, {"id": "omar", "kind": "user", "tenant": "umbrella", "scopes": ["team:support"]}], "memories": [{"id": "m1", "type": "fact", "content": "Workaround: restart the ingestion service to clear the stuck queue.", "tenant": "umbrella", "scope": "team:support", "created_at": "2026-06-05T10:00:00Z", "provenance": {"author": "nina-agent"}}], "events": [{"t": "2026-06-06T10:00:00Z", "type": "delete", "memory_id": "m1", "actor": "nina"}], "queries": [{"id": "q1", "principal": "omar", "as_of": "2026-06-05T12:00:00Z", "axis": "utility", "question": "Any workaround for the stuck queue?", "relevant": ["m1"], "expected_answer": "Restart the ingestion service.", "must_retrieve": ["m1"], "must_not_retrieve": []}, {"id": "q2", "principal": "omar", "as_of": "2026-06-07T10:00:00Z", "axis": "deletion", "question": "Any workaround for the stuck queue?", "relevant": ["m1"], "expected_answer": "", "must_retrieve": [], "must_not_retrieve": ["m1"]}]} +{"scenario_id": "provenance_reranker", "description": "Shared decisions must trace to their author/source; a derived fact keeps a link to its parent.", "tenants": ["hooli"], "scopes": [{"id": "proj:search", "kind": "project", "tenant": "hooli", "members": ["wade", "zoe"]}], "principals": [{"id": "wade", "kind": "user", "tenant": "hooli", "scopes": ["proj:search"]}, {"id": "zoe", "kind": "user", "tenant": "hooli", "scopes": ["proj:search"]}], "memories": [{"id": "m1", "type": "fact", "content": "The search reranker must cap tail latency at 200ms.", "tenant": "hooli", "scope": "proj:search", "created_at": "2026-06-03T10:00:00Z", "provenance": {"author": "wade-agent", "source": "thread:sprint-42", "confidence": 0.9}}, {"id": "m2", "type": "fact", "content": "Chosen approach: use the lightweight cross-encoder to stay under budget.", "tenant": "hooli", "scope": "proj:search", "created_at": "2026-06-03T11:00:00Z", "provenance": {"author": "zoe-agent", "source": "thread:sprint-42", "confidence": 0.85, "derived_from": ["m1"]}}], "events": [], "queries": [{"id": "q1", "principal": "zoe", "as_of": "2026-06-04T09:00:00Z", "axis": "provenance", "question": "What is the reranker latency budget and who set it?", "relevant": ["m1"], "expected_answer": "200ms tail latency, set by wade-agent in sprint-42.", "must_retrieve": ["m1"], "must_not_retrieve": []}]} diff --git a/benchmarks/governance/loader.py b/benchmarks/governance/loader.py new file mode 100644 index 0000000..034f81f --- /dev/null +++ b/benchmarks/governance/loader.py @@ -0,0 +1,117 @@ +"""Loading and validation for SMGB scenarios. + +Scenarios are stored as JSON Lines (one JSON object per line) so the dataset is +diff-friendly and easy to append to. The loader parses each line into a +:class:`~benchmarks.governance.schema.Scenario` and validates structural +integrity plus — crucially — that any hand-written ``must_retrieve`` / +``must_not_retrieve`` annotations agree with the labels the policy layer +derives. A mismatch is a dataset bug, surfaced early rather than silently +mis-scoring a system. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Iterable + +from .policy import compute_query_labels +from .schema import QUERY_AXES, Scenario + + +class ValidationError(ValueError): + """Raised when a scenario is structurally invalid or its labels disagree with policy.""" + + +def load_scenarios(path: str | Path) -> list[Scenario]: + """Load scenarios from a ``.jsonl`` file or a directory of ``.jsonl`` files.""" + p = Path(path) + files = sorted(p.glob("*.jsonl")) if p.is_dir() else [p] + scenarios: list[Scenario] = [] + for f in files: + for lineno, line in enumerate(f.read_text(encoding="utf-8").splitlines(), start=1): + line = line.strip() + if not line or line.startswith("#"): + continue + try: + data = json.loads(line) + except json.JSONDecodeError as exc: + raise ValidationError(f"{f}:{lineno}: invalid JSON: {exc}") from exc + scenarios.append(Scenario.from_dict(data)) + return scenarios + + +def validate_scenario(scenario: Scenario) -> list[str]: + """Return a list of problems with ``scenario`` (empty list means valid).""" + problems: list[str] = [] + sid = scenario.scenario_id + + # Referential integrity: every id referenced actually exists. + for scope in scenario.scopes.values(): + if scope.tenant not in scenario.tenants: + problems.append(f"{sid}: scope {scope.id!r} tenant {scope.tenant!r} not in tenants") + for member in scope.members: + if member not in scenario.principals: + problems.append(f"{sid}: scope {scope.id!r} member {member!r} is not a principal") + + for principal in scenario.principals.values(): + if principal.tenant not in scenario.tenants: + problems.append( + f"{sid}: principal {principal.id!r} tenant {principal.tenant!r} not in tenants" + ) + for scope_id in principal.scopes: + if scope_id not in scenario.scopes: + problems.append( + f"{sid}: principal {principal.id!r} references unknown scope {scope_id!r}" + ) + + for mem in scenario.memories.values(): + if mem.scope not in scenario.scopes: + problems.append(f"{sid}: memory {mem.id!r} initial scope {mem.scope!r} unknown") + if mem.tenant not in scenario.tenants: + problems.append(f"{sid}: memory {mem.id!r} tenant {mem.tenant!r} not in tenants") + + for ev in scenario.events: + if ev.memory_id not in scenario.memories: + problems.append(f"{sid}: event targets unknown memory {ev.memory_id!r}") + if ev.type == "promote" and (not ev.to_scope or ev.to_scope not in scenario.scopes): + problems.append(f"{sid}: promote event has unknown to_scope {ev.to_scope!r}") + if ev.type == "supersede" and (not ev.by or ev.by not in scenario.memories): + problems.append(f"{sid}: supersede event references unknown replacement {ev.by!r}") + + for q in scenario.queries: + if q.principal not in scenario.principals: + problems.append(f"{sid}: query {q.id!r} unknown principal {q.principal!r}") + continue + if q.axis not in QUERY_AXES: + problems.append(f"{sid}: query {q.id!r} unknown axis {q.axis!r}") + for rel in q.relevant: + if rel not in scenario.memories: + problems.append(f"{sid}: query {q.id!r} relevant id {rel!r} unknown") + + # The core check: hand annotations must agree with derived policy. + labels = compute_query_labels(scenario, q) + if q.must_retrieve is not None: + hand = set(q.must_retrieve) + if hand != set(labels.must_retrieve): + problems.append( + f"{sid}: query {q.id!r} must_retrieve {sorted(hand)} != " + f"policy-derived {sorted(labels.must_retrieve)}" + ) + if q.must_not_retrieve is not None: + hand_not = set(q.must_not_retrieve) + stray = hand_not - set(labels.forbidden) + if stray: + problems.append( + f"{sid}: query {q.id!r} must_not_retrieve {sorted(stray)} are actually allowed" + ) + + return problems + + +def validate_all(scenarios: Iterable[Scenario]) -> list[str]: + """Validate every scenario; return the combined problem list.""" + problems: list[str] = [] + for s in scenarios: + problems.extend(validate_scenario(s)) + return problems diff --git a/benchmarks/governance/policy.py b/benchmarks/governance/policy.py new file mode 100644 index 0000000..3065317 --- /dev/null +++ b/benchmarks/governance/policy.py @@ -0,0 +1,140 @@ +"""Deterministic governance policy for SMGB. + +This module is the crux of the benchmark. Given a scenario's scope graph and +event timeline, it computes — for any principal at any point in time — exactly +which memories are *authorized and current* (the allowed set) and which are +*forbidden* (everything else). These labels are derived, not hand-assigned, so +scoring never depends on an LLM judge and is fully reproducible. + +Governance state as of ``as_of`` for one memory: + +* **exists** — ``created_at <= as_of``. +* **scopes** — ``{initial scope} | {promote.to_scope : promote.t <= as_of}``. + Promotion *adds* a broader scope (private -> shared); the author, who is a + member of the broader scope, keeps visibility. +* **superseded** — some ``supersede`` event with ``t <= as_of`` targeted it. +* **deleted** — some ``delete`` event with ``t <= as_of`` targeted it. + +A memory is **authorized** for a principal when it exists, is not deleted, +belongs to the principal's tenant (hard tenant isolation), and shares at least +one scope with the principal. It is **allowed** for a query when it is +authorized and — unless the query is in ``history`` temporal mode — not +superseded. Everything else in the scenario is **forbidden** for that query. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from .schema import Memory, Principal, Scenario + + +@dataclass(frozen=True) +class MemoryState: + """Governance state of one memory as of a specific time.""" + + exists: bool + scopes: frozenset[str] + superseded: bool + deleted: bool + + +def memory_state(scenario: Scenario, memory_id: str, as_of: datetime) -> MemoryState: + """Compute the governance state of ``memory_id`` as of ``as_of``.""" + + mem = scenario.memories[memory_id] + scopes = {mem.scope} + superseded = False + deleted = False + for ev in scenario.events: # events are pre-sorted by time in Scenario + if ev.memory_id != memory_id or ev.t > as_of: + continue + if ev.type == "promote" and ev.to_scope: + scopes.add(ev.to_scope) + elif ev.type == "supersede": + superseded = True + elif ev.type == "delete": + deleted = True + return MemoryState( + exists=mem.created_at <= as_of, + scopes=frozenset(scopes), + superseded=superseded, + deleted=deleted, + ) + + +def is_authorized( + scenario: Scenario, + principal: Principal, + memory: Memory, + state: MemoryState, +) -> bool: + """Whether ``principal`` may access ``memory`` given its ``state``. + + Enforces, in order: existence, deletion, tenant isolation (hard), and scope + membership. Tenant isolation is checked before scope so a cross-tenant + memory is never authorized even if scope ids happen to collide. + """ + + if not state.exists or state.deleted: + return False + if memory.tenant != principal.tenant: + return False + return bool(set(principal.scopes) & state.scopes) + + +@dataclass(frozen=True) +class QueryLabels: + """Policy-derived labels for a single query. + + * ``allowed`` — authorized and current (per temporal mode). The system + *should* be able to return these; ``must_retrieve`` is the relevant subset. + * ``forbidden`` — every other memory in the scenario. Returning any of + these is a leak. + * ``must_retrieve`` — ``relevant & allowed`` (the utility target). + * ``cross_tenant`` — forbidden memories from another tenant (isolation). + * ``superseded_forbidden`` — forbidden only because they are stale in + current mode (conflict / stale-propagation signal). + """ + + allowed: frozenset[str] + forbidden: frozenset[str] + must_retrieve: frozenset[str] + cross_tenant: frozenset[str] + superseded_forbidden: frozenset[str] + + +def compute_query_labels(scenario: Scenario, query) -> QueryLabels: + """Derive the allowed / forbidden / must-retrieve sets for ``query``.""" + + principal = scenario.principals[query.principal] + history_mode = query.temporal_mode == "history" + + allowed: set[str] = set() + cross_tenant: set[str] = set() + superseded_forbidden: set[str] = set() + + for mem_id, mem in scenario.memories.items(): + state = memory_state(scenario, mem_id, query.as_of) + authorized = is_authorized(scenario, principal, mem, state) + current_ok = history_mode or not state.superseded + if authorized and current_ok: + allowed.add(mem_id) + continue + # Forbidden: record *why* for the axis-specific metrics. + if state.exists and mem.tenant != principal.tenant: + cross_tenant.add(mem_id) + if authorized and state.superseded and not history_mode: + superseded_forbidden.add(mem_id) + + all_ids = set(scenario.memories.keys()) + forbidden = all_ids - allowed + must_retrieve = set(query.relevant) & allowed + return QueryLabels( + allowed=frozenset(allowed), + forbidden=frozenset(forbidden), + must_retrieve=frozenset(must_retrieve), + cross_tenant=frozenset(cross_tenant), + superseded_forbidden=frozenset(superseded_forbidden), + ) diff --git a/benchmarks/governance/run_governance.py b/benchmarks/governance/run_governance.py new file mode 100644 index 0000000..e7d4d57 --- /dev/null +++ b/benchmarks/governance/run_governance.py @@ -0,0 +1,113 @@ +"""CLI: validate SMGB scenarios and score systems on the governance axes. + +Examples +-------- +Score the built-in reference runners (oracle + naive baselines) on the seed set:: + + python -m benchmarks.governance.run_governance --reference + +Validate the seed dataset only:: + + python -m benchmarks.governance.run_governance --validate-only + +Score an external system from a run file (JSON: ``{scenario_id: {query_id: [ids]}}``):: + + python -m benchmarks.governance.run_governance --run my_run.json --system my_system +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from .loader import load_scenarios, validate_all +from .scorer import ( + REFERENCE_RUNNERS, + merge_reports, + oracle_provenance, + score_scenario, +) + +DEFAULT_DATA = Path(__file__).with_name("data") / "seed_scenarios.jsonl" + + +def _print_leaderboard(rows: list[dict]) -> None: + cols = [ + ("system", 14, "s"), + ("queries", 8, "d"), + ("mean_recall", 12, ".3f"), + ("leak_rate", 10, ".3f"), + ("total_leaks", 12, "d"), + ("isolation_violations", 21, "d"), + ("stale_leaks", 12, "d"), + ] + header = "".join(f"{name:<{w}}" for name, w, _ in cols) + print(header) + print("-" * len(header)) + for r in rows: + line = "" + for name, w, fmt in cols: + val = r.get(name) + if val is None: + cell = "n/a" + elif fmt == "s": + cell = str(val) + else: + cell = format(val, fmt) + line += f"{cell:<{w}}" + print(line) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Shared-Memory Governance Benchmark runner") + ap.add_argument("--data", default=str(DEFAULT_DATA), help="scenario .jsonl file or directory") + ap.add_argument("--reference", action="store_true", help="score the built-in reference runners") + ap.add_argument("--run", help="JSON run file for an external system") + ap.add_argument("--system", default="system", help="name for the --run system") + ap.add_argument("--k", type=int, default=10, help="retrieval cutoff") + ap.add_argument("--validate-only", action="store_true", help="validate scenarios and exit") + ap.add_argument("--json-out", help="write the leaderboard summary to this JSON path") + args = ap.parse_args(argv) + + scenarios = load_scenarios(args.data) + problems = validate_all(scenarios) + if problems: + print(f"VALIDATION FAILED ({len(problems)} problem(s)):") + for p in problems: + print(" -", p) + return 1 + print(f"Loaded {len(scenarios)} scenario(s); validation OK.") + if args.validate_only: + return 0 + + rows: list[dict] = [] + + if args.reference or not args.run: + for name, runner in REFERENCE_RUNNERS.items(): + reports = [] + for s in scenarios: + run = runner(s, args.k) + prov = oracle_provenance(s) if name == "oracle" else None + reports.append(score_scenario(s, run, system=name, k=args.k, provenance=prov)) + rows.append(merge_reports(reports).summary()) + + if args.run: + run_data = json.loads(Path(args.run).read_text(encoding="utf-8")) + reports = [] + for s in scenarios: + run = run_data.get(s.scenario_id, {}) + reports.append(score_scenario(s, run, system=args.system, k=args.k)) + rows.append(merge_reports(reports).summary()) + + print() + _print_leaderboard(rows) + + if args.json_out: + Path(args.json_out).write_text(json.dumps(rows, indent=2), encoding="utf-8") + print(f"\nWrote leaderboard to {args.json_out}") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/benchmarks/governance/schema.py b/benchmarks/governance/schema.py new file mode 100644 index 0000000..fb3c9de --- /dev/null +++ b/benchmarks/governance/schema.py @@ -0,0 +1,276 @@ +"""Data model for the Shared-Memory Governance Benchmark (SMGB). + +SMGB evaluates *shared* agent memory: memory reused across agents, users, +teams, tenants, and organizations. Unlike single-user memory benchmarks +(LoCoMo, LongMemEval, DMR) that score only recall, every SMGB query is issued +by a *principal* (a user or agent with a tenant, roles, and scope memberships) +at a point in time, and ground truth partitions memories into what the +principal MUST retrieve (authorized + relevant -> utility) and MUST NOT +retrieve (unauthorized / wrong scope / superseded / deleted / cross-tenant -> +leakage). + +The classes here are plain, dependency-free dataclasses. Labels are never +hand-assigned in a way the harness trusts blindly: :mod:`benchmarks.governance.policy` +derives the authorized/forbidden sets *deterministically* from the scope graph +and the event timeline, and the loader validates any hand annotations against +that policy. This is what makes the benchmark reproducible and system-agnostic: +a system under test only returns ranked memory ids, and the scorer compares +them to policy-derived labels. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Optional + + +def parse_time(value: str | datetime) -> datetime: + """Parse an ISO-8601 timestamp into an aware ``datetime`` (UTC if naive). + + Accepts a trailing ``Z``. Naive datetimes are assumed to be UTC so that + all comparisons in the policy layer are well defined. + """ + + if isinstance(value, datetime): + dt = value + else: + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + dt = datetime.fromisoformat(text) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +# Event types that mutate a memory's governance state over time. +EVENT_TYPES = frozenset({"promote", "supersede", "delete"}) + +# Query axes map to the governance failure modes SMGB scores. These mirror the +# four failure modes named in "Governed Shared Memory for Multi-Agent LLM +# Systems" (arXiv:2606.24535) plus explicit promotion and abstention. +QUERY_AXES = frozenset( + { + "utility", # authorized recall: the principal should get entitled facts + "leakage", # the principal must not receive forbidden facts + "isolation", # cross-tenant memory must never be returned + "promotion", # promoted facts visible after T to scope members, not before + "conflict", # only the current version is retrievable in "current" mode + "deletion", # deleted facts (and copies) gone for all scope members + "provenance", # every shared fact traces to author/source/time + } +) + + +@dataclass(frozen=True) +class Scope: + """A visibility unit: user, agent, session, project, group, tenant, org. + + Membership is declared on :class:`Principal` (``scopes``); ``Scope`` carries + the scope's kind and owning tenant for metadata and reporting. ``members`` + is optional and, when present, is cross-checked against principal + declarations by the loader. + """ + + id: str + kind: str + tenant: str + members: tuple[str, ...] = () + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Scope": + return Scope( + id=d["id"], + kind=d["kind"], + tenant=d["tenant"], + members=tuple(d.get("members", ()) or ()), + ) + + +@dataclass(frozen=True) +class Principal: + """A user or agent issuing reads/writes, scoped by tenant + memberships.""" + + id: str + kind: str # "user" or "agent" + tenant: str + roles: tuple[str, ...] = () + scopes: tuple[str, ...] = () # scope ids this principal is a member of + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Principal": + return Principal( + id=d["id"], + kind=d.get("kind", "user"), + tenant=d["tenant"], + roles=tuple(d.get("roles", ()) or ()), + scopes=tuple(d.get("scopes", ()) or ()), + ) + + +@dataclass(frozen=True) +class Provenance: + """Where a shared memory came from — author, source message, confidence. + + ``derived_from`` lets a fact point at the memory ids it was derived from, + so a scorer can reconstruct multi-hop provenance chains (cf. ArgusFleet's + depth-N provenance reconstruction). + """ + + author: str + source: Optional[str] = None + confidence: Optional[float] = None + derived_from: tuple[str, ...] = () + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Provenance": + return Provenance( + author=d["author"], + source=d.get("source"), + confidence=d.get("confidence"), + derived_from=tuple(d.get("derived_from", ()) or ()), + ) + + +@dataclass(frozen=True) +class Memory: + """A single memory record with a birth time and an initial scope. + + Governance state (current scope set, superseded, deleted) is *not* stored + here; it is computed as of a query time by the policy layer from the event + timeline. This keeps the dataset a single source of truth. + """ + + id: str + type: str # turn | episode | fact | summary + content: str + tenant: str + scope: str # initial scope at creation + created_at: datetime + provenance: Provenance + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Memory": + prov = d.get("provenance") + if prov is None: + # Allow a flat shorthand: author/source/confidence at top level. + prov = { + "author": d.get("author", "unknown"), + "source": d.get("source"), + "confidence": d.get("confidence"), + } + return Memory( + id=d["id"], + type=d.get("type", "fact"), + content=d.get("content", ""), + tenant=d["tenant"], + scope=d["scope"], + created_at=parse_time(d["created_at"]), + provenance=Provenance.from_dict(prov), + ) + + +@dataclass(frozen=True) +class Event: + """A governance mutation on the timeline: promote, supersede, or delete.""" + + t: datetime + type: str + memory_id: str + to_scope: Optional[str] = None # promote target + by: Optional[str] = None # supersede: id of the replacement memory + actor: Optional[str] = None # principal who performed the action + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Event": + etype = d["type"] + if etype not in EVENT_TYPES: + raise ValueError(f"unknown event type {etype!r}; expected {sorted(EVENT_TYPES)}") + return Event( + t=parse_time(d["t"]), + type=etype, + memory_id=d["memory_id"], + to_scope=d.get("to_scope"), + by=d.get("by"), + actor=d.get("actor"), + ) + + +@dataclass(frozen=True) +class Query: + """A read issued by a principal at ``as_of``, scored on one axis. + + ``relevant`` lists the memory ids that answer the question (the utility + target *before* authorization is applied). ``temporal_mode`` is "current" + (default) or "history": in history mode superseded versions are allowed. + ``must_retrieve`` / ``must_not_retrieve`` are optional hand annotations the + loader validates against the policy-derived labels. + """ + + id: str + principal: str + as_of: datetime + axis: str + question: str + relevant: tuple[str, ...] = () + temporal_mode: str = "current" + expected_answer: Optional[str] = None + must_retrieve: Optional[tuple[str, ...]] = None + must_not_retrieve: Optional[tuple[str, ...]] = None + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Query": + axis = d.get("axis", "utility") + if axis not in QUERY_AXES: + raise ValueError(f"unknown query axis {axis!r}; expected {sorted(QUERY_AXES)}") + mr = d.get("must_retrieve") + mnr = d.get("must_not_retrieve") + return Query( + id=d["id"], + principal=d["principal"], + as_of=parse_time(d["as_of"]), + axis=axis, + question=d.get("question", ""), + relevant=tuple(d.get("relevant", ()) or ()), + temporal_mode=d.get("temporal_mode", "current"), + expected_answer=d.get("expected_answer"), + must_retrieve=tuple(mr) if mr is not None else None, + must_not_retrieve=tuple(mnr) if mnr is not None else None, + ) + + +@dataclass +class Scenario: + """A self-contained governance test: scopes, principals, memories, timeline, queries.""" + + scenario_id: str + tenants: tuple[str, ...] + scopes: dict[str, Scope] + principals: dict[str, Principal] + memories: dict[str, Memory] + events: list[Event] + queries: list[Query] + description: str = "" + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Scenario": + scopes = {s["id"]: Scope.from_dict(s) for s in d.get("scopes", [])} + principals = {p["id"]: Principal.from_dict(p) for p in d.get("principals", [])} + memories = {m["id"]: Memory.from_dict(m) for m in d.get("memories", [])} + events = sorted( + (Event.from_dict(e) for e in d.get("events", [])), + key=lambda e: e.t, + ) + queries = [Query.from_dict(q) for q in d.get("queries", [])] + tenants = tuple(d.get("tenants") or sorted({m.tenant for m in memories.values()})) + return Scenario( + scenario_id=d["scenario_id"], + tenants=tenants, + scopes=scopes, + principals=principals, + memories=memories, + events=events, + queries=queries, + description=d.get("description", ""), + ) diff --git a/benchmarks/governance/scorer.py b/benchmarks/governance/scorer.py new file mode 100644 index 0000000..b92fe9d --- /dev/null +++ b/benchmarks/governance/scorer.py @@ -0,0 +1,261 @@ +"""Scoring for the Shared-Memory Governance Benchmark (SMGB). + +The scorer is **system-agnostic**: a system under test produces a *run* — for +each query, a ranked list of retrieved memory ids (and, optionally, the +provenance it claims for them). The scorer compares that run to the +policy-derived labels from :mod:`benchmarks.governance.policy` and reports: + +* **utility** — ``recall@k`` over the authorized-and-relevant target set. +* **leakage** — fraction of queries that returned any forbidden memory, and the + item-level leak count. +* **isolation** — cross-tenant memories returned (must be 0). +* **conflict / stale propagation** — superseded ("ghost") memories returned in + current mode. +* **provenance** — for provenance-axis queries, whether the claimed author for + each retrieved target matches ground truth. + +Three reference runners are provided so the metrics can be validated offline +without a live memory service, and to anchor a leaderboard: + +* :func:`oracle_run` — returns exactly the authorized target (perfect). +* :func:`naive_shared_run` — returns everything in the principal's tenant, + ignoring scope and supersession (no authorization filter) -> high leak. +* :func:`naive_global_run` — returns everything across all tenants -> also + breaks isolation. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +from .policy import compute_query_labels, memory_state +from .schema import Scenario + +# A run maps query_id -> ranked list of retrieved memory ids. +Run = dict[str, list[str]] +# Optional provenance claims: query_id -> memory_id -> {"author": str, ...}. +ProvenanceClaims = dict[str, dict[str, dict[str, Any]]] + + +@dataclass +class QueryScore: + query_id: str + axis: str + recall: Optional[float] # None when there is no utility target (abstention) + leaked: bool + leak_count: int + isolation_violations: int + stale_leak: int + provenance_correct: Optional[int] = None + provenance_total: Optional[int] = None + + +@dataclass +class Report: + system: str + k: int + per_query: list[QueryScore] = field(default_factory=list) + + # -- aggregates -------------------------------------------------------- + def _recalls(self) -> list[float]: + return [q.recall for q in self.per_query if q.recall is not None] + + @property + def mean_recall(self) -> float: + vals = self._recalls() + return sum(vals) / len(vals) if vals else float("nan") + + @property + def leak_rate(self) -> float: + """Fraction of queries that returned at least one forbidden memory.""" + if not self.per_query: + return float("nan") + return sum(1 for q in self.per_query if q.leaked) / len(self.per_query) + + @property + def total_leaks(self) -> int: + return sum(q.leak_count for q in self.per_query) + + @property + def isolation_violations(self) -> int: + return sum(q.isolation_violations for q in self.per_query) + + @property + def stale_leaks(self) -> int: + return sum(q.stale_leak for q in self.per_query) + + @property + def provenance_accuracy(self) -> Optional[float]: + correct = sum(q.provenance_correct or 0 for q in self.per_query if q.provenance_total) + total = sum(q.provenance_total or 0 for q in self.per_query if q.provenance_total) + return correct / total if total else None + + def by_axis(self) -> dict[str, dict[str, float]]: + """Per-axis rollup of the headline metrics.""" + axes: dict[str, list[QueryScore]] = {} + for q in self.per_query: + axes.setdefault(q.axis, []).append(q) + out: dict[str, dict[str, float]] = {} + for axis, items in sorted(axes.items()): + recalls = [q.recall for q in items if q.recall is not None] + out[axis] = { + "n": len(items), + "mean_recall": (sum(recalls) / len(recalls)) if recalls else float("nan"), + "leak_rate": sum(1 for q in items if q.leaked) / len(items), + "isolation_violations": sum(q.isolation_violations for q in items), + "stale_leaks": sum(q.stale_leak for q in items), + } + return out + + def summary(self) -> dict[str, Any]: + return { + "system": self.system, + "k": self.k, + "queries": len(self.per_query), + "mean_recall": self.mean_recall, + "leak_rate": self.leak_rate, + "total_leaks": self.total_leaks, + "isolation_violations": self.isolation_violations, + "stale_leaks": self.stale_leaks, + "provenance_accuracy": self.provenance_accuracy, + } + + +def score_scenario( + scenario: Scenario, + run: Run, + *, + system: str = "system", + k: int = 10, + provenance: Optional[ProvenanceClaims] = None, +) -> Report: + """Score a single scenario's ``run`` against policy-derived labels.""" + + report = Report(system=system, k=k) + for query in scenario.queries: + labels = compute_query_labels(scenario, query) + retrieved = list(run.get(query.id, []))[:k] + retrieved_set = set(retrieved) + + if labels.must_retrieve: + recall = len(retrieved_set & labels.must_retrieve) / len(labels.must_retrieve) + else: + recall = None # abstention / pure-leakage probe: no utility target + + leak_items = retrieved_set & labels.forbidden + prov_correct: Optional[int] = None + prov_total: Optional[int] = None + if query.axis == "provenance": + prov_correct, prov_total = _score_provenance( + scenario, query, labels, retrieved_set, provenance + ) + + report.per_query.append( + QueryScore( + query_id=query.id, + axis=query.axis, + recall=recall, + leaked=bool(leak_items), + leak_count=len(leak_items), + isolation_violations=len(retrieved_set & labels.cross_tenant), + stale_leak=len(retrieved_set & labels.superseded_forbidden), + provenance_correct=prov_correct, + provenance_total=prov_total, + ) + ) + return report + + +def _score_provenance( + scenario: Scenario, + query, + labels, + retrieved_set: set[str], + provenance: Optional[ProvenanceClaims], +) -> tuple[int, int]: + """Fraction of retrieved targets whose claimed author matches ground truth.""" + targets = labels.must_retrieve & retrieved_set + if not targets: + return 0, len(labels.must_retrieve) + claims = (provenance or {}).get(query.id, {}) + correct = 0 + for mem_id in targets: + gold_author = scenario.memories[mem_id].provenance.author + claimed = claims.get(mem_id, {}).get("author") + if claimed is not None and claimed == gold_author: + correct += 1 + return correct, len(labels.must_retrieve) + + +def merge_reports(reports: list[Report]) -> Report: + """Combine per-scenario reports for one system into a single report.""" + if not reports: + raise ValueError("no reports to merge") + merged = Report(system=reports[0].system, k=reports[0].k) + for r in reports: + merged.per_query.extend(r.per_query) + return merged + + +# -- reference runners ----------------------------------------------------- + + +def oracle_run(scenario: Scenario, k: int = 10) -> Run: + """Perfect system: return exactly the authorized-and-relevant target set.""" + run: Run = {} + for query in scenario.queries: + labels = compute_query_labels(scenario, query) + run[query.id] = list(labels.must_retrieve)[:k] + return run + + +def oracle_provenance(scenario: Scenario) -> ProvenanceClaims: + """Gold provenance claims matching :func:`oracle_run` (all authors correct).""" + claims: ProvenanceClaims = {} + for query in scenario.queries: + labels = compute_query_labels(scenario, query) + claims[query.id] = { + mem_id: {"author": scenario.memories[mem_id].provenance.author} + for mem_id in labels.must_retrieve + } + return claims + + +def _tenant_dump(scenario: Scenario, query, *, all_tenants: bool) -> list[str]: + principal = scenario.principals[query.principal] + out: list[str] = [] + for mem_id, mem in scenario.memories.items(): + state = memory_state(scenario, mem_id, query.as_of) + if not state.exists or state.deleted: + continue + if not all_tenants and mem.tenant != principal.tenant: + continue + out.append(mem_id) + return out + + +def naive_shared_run(scenario: Scenario, k: int = 10) -> Run: + """No authorization filter: return everything in the principal's tenant. + + Ignores scope membership and supersession, so it recalls the target but + leaks other-scope and stale memories. Tenant isolation is (accidentally) + respected, so isolation violations stay 0 — isolating the leak signal. + """ + return { + q.id: _tenant_dump(scenario, q, all_tenants=False)[:k] for q in scenario.queries + } + + +def naive_global_run(scenario: Scenario, k: int = 10) -> Run: + """Worst case: return everything across all tenants -> breaks isolation.""" + return { + q.id: _tenant_dump(scenario, q, all_tenants=True)[:k] for q in scenario.queries + } + + +REFERENCE_RUNNERS: dict[str, Callable[[Scenario, int], Run]] = { + "oracle": oracle_run, + "naive_shared": naive_shared_run, + "naive_global": naive_global_run, +} diff --git a/benchmarks/tests/__init__.py b/benchmarks/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmarks/tests/test_governance_policy.py b/benchmarks/tests/test_governance_policy.py new file mode 100644 index 0000000..0c58021 --- /dev/null +++ b/benchmarks/tests/test_governance_policy.py @@ -0,0 +1,151 @@ +"""Unit tests for the SMGB deterministic policy layer.""" + +from benchmarks.governance.policy import ( + compute_query_labels, + is_authorized, + memory_state, +) +from benchmarks.governance.schema import Scenario + + +def _scenario(): + return Scenario.from_dict( + { + "scenario_id": "unit", + "tenants": ["t1", "t2"], + "scopes": [ + {"id": "user:a", "kind": "user", "tenant": "t1", "members": ["a"]}, + {"id": "proj:x", "kind": "project", "tenant": "t1", "members": ["a", "b"]}, + {"id": "org:t2", "kind": "org", "tenant": "t2", "members": ["c"]}, + ], + "principals": [ + {"id": "a", "tenant": "t1", "scopes": ["user:a", "proj:x"]}, + {"id": "b", "tenant": "t1", "scopes": ["proj:x"]}, + {"id": "c", "tenant": "t2", "scopes": ["org:t2"]}, + ], + "memories": [ + {"id": "m1", "scope": "user:a", "tenant": "t1", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "a-agent"}}, + {"id": "m2", "scope": "proj:x", "tenant": "t1", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "b-agent"}}, + {"id": "m3", "scope": "org:t2", "tenant": "t2", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "c-agent"}}, + ], + "events": [ + {"t": "2026-02-01T00:00:00Z", "type": "promote", "memory_id": "m1", "to_scope": "proj:x"}, + ], + "queries": [], + } + ) + + +def test_memory_state_promotion_adds_scope(): + s = _scenario() + before = memory_state(s, "m1", _t("2026-01-15T00:00:00Z")) + after = memory_state(s, "m1", _t("2026-02-15T00:00:00Z")) + assert before.scopes == frozenset({"user:a"}) + assert after.scopes == frozenset({"user:a", "proj:x"}) + + +def test_tenant_isolation_blocks_cross_tenant(): + s = _scenario() + c = s.principals["c"] + m2 = s.memories["m2"] + st = memory_state(s, "m2", _t("2026-03-01T00:00:00Z")) + assert is_authorized(s, c, m2, st) is False # c is in t2, m2 is in t1 + + +def test_promotion_changes_authorization_over_time(): + s = _scenario() + b = s.principals["b"] # member of proj:x only + m1 = s.memories["m1"] + before = memory_state(s, "m1", _t("2026-01-15T00:00:00Z")) + after = memory_state(s, "m1", _t("2026-02-15T00:00:00Z")) + assert is_authorized(s, b, m1, before) is False # private to user:a + assert is_authorized(s, b, m1, after) is True # promoted to proj:x + + +def test_query_labels_forbidden_is_complement(): + s = _scenario() + # b asks after promotion; relevant covers all three memories. + q = _query(s, principal="b", as_of="2026-03-01T00:00:00Z", relevant=["m1", "m2", "m3"]) + labels = compute_query_labels(s, q) + assert labels.must_retrieve == frozenset({"m1", "m2"}) # both in proj:x now + assert "m3" in labels.forbidden + assert labels.cross_tenant == frozenset({"m3"}) + assert labels.allowed | labels.forbidden == set(s.memories) + + +def test_supersede_and_history_mode(): + s = Scenario.from_dict( + { + "scenario_id": "sup", + "tenants": ["t"], + "scopes": [{"id": "p", "kind": "project", "tenant": "t", "members": ["a"]}], + "principals": [{"id": "a", "tenant": "t", "scopes": ["p"]}], + "memories": [ + {"id": "old", "scope": "p", "tenant": "t", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "x"}}, + {"id": "new", "scope": "p", "tenant": "t", + "created_at": "2026-02-01T00:00:00Z", "provenance": {"author": "y"}}, + ], + "events": [ + {"t": "2026-02-01T00:00:00Z", "type": "supersede", "memory_id": "old", "by": "new"}, + ], + "queries": [ + {"id": "cur", "principal": "a", "as_of": "2026-03-01T00:00:00Z", + "axis": "conflict", "temporal_mode": "current", "relevant": ["old", "new"]}, + {"id": "hist", "principal": "a", "as_of": "2026-03-01T00:00:00Z", + "axis": "conflict", "temporal_mode": "history", "relevant": ["old"]}, + ], + } + ) + cur = compute_query_labels(s, s.queries[0]) + hist = compute_query_labels(s, s.queries[1]) + assert cur.must_retrieve == frozenset({"new"}) + assert "old" in cur.superseded_forbidden and "old" in cur.forbidden + assert hist.must_retrieve == frozenset({"old"}) # history mode allows superseded + + +def test_deletion_forbids_for_all(): + s = Scenario.from_dict( + { + "scenario_id": "del", + "tenants": ["t"], + "scopes": [{"id": "g", "kind": "group", "tenant": "t", "members": ["a"]}], + "principals": [{"id": "a", "tenant": "t", "scopes": ["g"]}], + "memories": [ + {"id": "m", "scope": "g", "tenant": "t", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "x"}}, + ], + "events": [{"t": "2026-02-01T00:00:00Z", "type": "delete", "memory_id": "m"}], + "queries": [ + {"id": "before", "principal": "a", "as_of": "2026-01-15T00:00:00Z", + "axis": "utility", "relevant": ["m"]}, + {"id": "after", "principal": "a", "as_of": "2026-02-15T00:00:00Z", + "axis": "deletion", "relevant": ["m"]}, + ], + } + ) + before = compute_query_labels(s, s.queries[0]) + after = compute_query_labels(s, s.queries[1]) + assert before.must_retrieve == frozenset({"m"}) + assert after.must_retrieve == frozenset() # deleted -> not authorized + assert "m" in after.forbidden + + +# -- helpers --------------------------------------------------------------- + + +def _t(iso): + from benchmarks.governance.schema import parse_time + + return parse_time(iso) + + +def _query(scenario, *, principal, as_of, relevant, axis="utility"): + from benchmarks.governance.schema import Query + + return Query.from_dict( + {"id": "q", "principal": principal, "as_of": as_of, "axis": axis, "relevant": relevant} + ) diff --git a/benchmarks/tests/test_governance_scorer.py b/benchmarks/tests/test_governance_scorer.py new file mode 100644 index 0000000..5f3ac69 --- /dev/null +++ b/benchmarks/tests/test_governance_scorer.py @@ -0,0 +1,113 @@ +"""Unit tests for the SMGB scorer and reference runners.""" + +from benchmarks.governance.schema import Scenario +from benchmarks.governance.scorer import ( + naive_global_run, + naive_shared_run, + oracle_provenance, + oracle_run, + score_scenario, +) + + +def _scenario(): + return Scenario.from_dict( + { + "scenario_id": "score", + "tenants": ["t1", "t2"], + "scopes": [ + {"id": "user:a", "kind": "user", "tenant": "t1", "members": ["a"]}, + {"id": "acct:x", "kind": "account", "tenant": "t1", "members": ["a", "b"]}, + {"id": "org:t2", "kind": "org", "tenant": "t2", "members": ["c"]}, + ], + "principals": [ + {"id": "a", "tenant": "t1", "scopes": ["user:a", "acct:x"]}, + {"id": "b", "tenant": "t1", "scopes": ["acct:x"]}, + {"id": "c", "tenant": "t2", "scopes": ["org:t2"]}, + ], + "memories": [ + {"id": "shared1", "scope": "acct:x", "tenant": "t1", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "a-agent"}}, + {"id": "private1", "scope": "user:a", "tenant": "t1", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "a-agent"}}, + {"id": "other_tenant", "scope": "org:t2", "tenant": "t2", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "c-agent"}}, + ], + "events": [], + "queries": [ + # b should get shared1, not private1 (leakage) nor other_tenant (isolation). + {"id": "u", "principal": "b", "as_of": "2026-02-01T00:00:00Z", + "axis": "utility", "relevant": ["shared1", "private1", "other_tenant"]}, + ], + } + ) + + +def test_oracle_is_perfect(): + s = _scenario() + report = score_scenario(s, oracle_run(s), system="oracle", provenance=oracle_provenance(s)) + su = report.summary() + assert su["mean_recall"] == 1.0 + assert su["leak_rate"] == 0.0 + assert su["total_leaks"] == 0 + assert su["isolation_violations"] == 0 + + +def test_naive_shared_leaks_but_respects_tenant(): + s = _scenario() + report = score_scenario(s, naive_shared_run(s), system="naive_shared") + su = report.summary() + assert su["mean_recall"] == 1.0 # it does return the target... + assert su["leak_rate"] > 0.0 # ...but also leaks private1 + assert su["total_leaks"] >= 1 + assert su["isolation_violations"] == 0 # never crosses tenants + + +def test_naive_global_breaks_isolation(): + s = _scenario() + report = score_scenario(s, naive_global_run(s), system="naive_global") + su = report.summary() + assert su["isolation_violations"] >= 1 # returns other_tenant + + +def test_recall_is_none_for_abstention_query(): + s = Scenario.from_dict( + { + "scenario_id": "abstain", + "tenants": ["t"], + "scopes": [ + {"id": "user:a", "kind": "user", "tenant": "t", "members": ["a"]}, + {"id": "user:b", "kind": "user", "tenant": "t", "members": ["b"]}, + ], + "principals": [ + {"id": "a", "tenant": "t", "scopes": ["user:a"]}, + {"id": "b", "tenant": "t", "scopes": ["user:b"]}, + ], + "memories": [ + {"id": "secret", "scope": "user:a", "tenant": "t", + "created_at": "2026-01-01T00:00:00Z", "provenance": {"author": "a"}}, + ], + "events": [], + "queries": [ + {"id": "leak", "principal": "b", "as_of": "2026-02-01T00:00:00Z", + "axis": "leakage", "relevant": ["secret"]}, + ], + } + ) + # Empty run: perfect abstention -> no leak, recall N/A. + report = score_scenario(s, {"leak": []}, system="silent") + assert report.per_query[0].recall is None + assert report.per_query[0].leaked is False + # Leaky run that returns the forbidden secret. + report2 = score_scenario(s, {"leak": ["secret"]}, system="leaky") + assert report2.per_query[0].leaked is True + assert report2.total_leaks == 1 + + +def test_k_cutoff_limits_retrieval(): + s = _scenario() + # A run that returns the target only beyond the cutoff should miss it. + run = {"u": ["private1", "other_tenant", "shared1"]} + at_k2 = score_scenario(s, run, k=2) + assert at_k2.per_query[0].recall == 0.0 # shared1 is at rank 3 + assert at_k2.per_query[0].leaked is True # private1 leaked within k=2 diff --git a/benchmarks/tests/test_governance_seed.py b/benchmarks/tests/test_governance_seed.py new file mode 100644 index 0000000..1b5b7f0 --- /dev/null +++ b/benchmarks/tests/test_governance_seed.py @@ -0,0 +1,69 @@ +"""Integrity tests for the checked-in SMGB seed dataset.""" + +from pathlib import Path + +from benchmarks.governance import ( + load_scenarios, + naive_global_run, + naive_shared_run, + oracle_provenance, + oracle_run, + score_scenario, + validate_all, +) +from benchmarks.governance.scorer import merge_reports + +SEED = Path(__file__).resolve().parents[1] / "governance" / "data" / "seed_scenarios.jsonl" + + +def _load(): + return load_scenarios(SEED) + + +def test_seed_file_exists_and_loads(): + scenarios = _load() + assert len(scenarios) >= 6 + ids = {s.scenario_id for s in scenarios} + # The two whiteboard anchors must be present. + assert "acct_satya_steve" in ids + assert "proj_scott_deploy" in ids + + +def test_seed_hand_labels_match_policy(): + problems = validate_all(_load()) + assert problems == [], "\n".join(problems) + + +def test_every_axis_is_represented(): + axes = {q.axis for s in _load() for q in s.queries} + for expected in {"utility", "leakage", "isolation", "promotion", "conflict", "deletion", "provenance"}: + assert expected in axes, f"seed set is missing axis {expected!r}" + + +def test_oracle_perfect_on_seed(): + scenarios = _load() + reports = [ + score_scenario(s, oracle_run(s), system="oracle", provenance=oracle_provenance(s)) + for s in scenarios + ] + su = merge_reports(reports).summary() + assert su["mean_recall"] == 1.0 + assert su["leak_rate"] == 0.0 + assert su["isolation_violations"] == 0 + assert su["provenance_accuracy"] == 1.0 + + +def test_naive_baselines_are_discriminated(): + scenarios = _load() + shared = merge_reports( + [score_scenario(s, naive_shared_run(s), system="naive_shared") for s in scenarios] + ).summary() + glob = merge_reports( + [score_scenario(s, naive_global_run(s), system="naive_global") for s in scenarios] + ).summary() + # No authorization filter -> the leakage axis fires. + assert shared["leak_rate"] > 0.0 + assert shared["total_leaks"] > 0 + # No tenant filter -> the isolation axis fires (and it only fires here). + assert shared["isolation_violations"] == 0 + assert glob["isolation_violations"] > 0