feat: add Evaluation + Optimization closed-loop pipeline (#91)

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
 
       - name: Run tests with coverage
         # run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term tests/
-        run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/
+        run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ examples/optimization/eval_optimize_loop/tests/
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

examples/optimization/eval_optimize_loop/README.md

@@ -0,0 +1,61 @@
+# Eval + Optimize Closed Loop
+
+A reproducible Evaluation + Optimization pipeline that builds a closed loop of "baseline evaluation → failure attribution → prompt optimization → candidate validation → acceptance gating → audit reporting".
+
+## Quickstart (trace mode, no API keys)
+
+```bash
+source .venv/bin/activate
+python run_pipeline.py
+```
+
+This runs against the 6 sample cases (3 train, 3 val) using pre-recorded traces. No API keys required.
+
+## Pipeline stages
+
+1. **Baseline Evaluation** — Runs AgentEvaluator on train and val evalsets against the baseline prompt, recording per-case metrics, pass/fail status, and key trajectories.
+2. **Failure Attribution** — Clusters failed cases by type: final_response_mismatch, format_violation, etc. Each failed case gets at least one attributed category.
+3. **Optimization** — In live mode, delegates to AgentOptimizer (GEPA reflective optimization). In trace mode, uses a pre-cooked optimized prompt.
+4. **Candidate Validation** — Re-evaluates the candidate prompt on both train and val sets, producing per-case results for delta comparison.
+5. **Delta Analysis** — Compares baseline vs candidate per case: newly passing, newly failing, per-metric score deltas.
+6. **Acceptance Gate** — Configurable rules: min_improvement, allow_new_fails, protected_case_ids, max_cost_usd, max_duration_seconds. Detects overfitting (train improves but val degrades).
+7. **Audit Reports** — Generates optimization_report.json (machine-readable) and optimization_report.md (human-readable).
+
+## Configuration
+
+See `pipeline.json` for trace mode or create your own. Key sections:
+
+- `mode`: `"trace"` (no API keys) or `"live"` (requires call_agent)
+- `evaluate`: Metric definitions and thresholds
+- `gate`: Acceptance rules
+
+## Sample data
+
+The 6 sample cases are designed to demonstrate three scenarios:
+- **Optimizable**: Case fails with baseline, passes with optimized prompt
+- **No improvement**: Case fails with both prompts
+- **Regression**: Case passes with baseline, fails with optimized prompt
+
+With `allow_new_fails: false`, the gate correctly REJECTS when the candidate introduces new failures (anti-overfitting protection).
+
+## Live mode
+
+To use live mode with your own agent:
+
+```python
+from pipeline import EvalOptimizePipeline
+from trpc_agent_sdk.evaluation import TargetPrompt
+
+target = TargetPrompt().add_path("system_prompt", "path/to/prompt.md")
+pipeline = EvalOptimizePipeline.from_config(
+    "pipeline.json",
+    call_agent=your_call_agent,
+    target_prompt=target,
+)
+result = await pipeline.run()
+```
+
+## Output
+
+- `outputs/optimization_report.json` — Full structured result
+- `outputs/optimization_report.md` — Human-readable summary with verdict, pass rates, per-case delta table, failure attribution, gate results, overfitting check, and audit trail

examples/optimization/eval_optimize_loop/delta.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from .models import PerCaseDelta, SplitDelta, SplitResult
+
+
+def _per_case_delta(baseline: SplitResult, candidate: SplitResult) -> PerCaseDelta:
+    newly_passing: list[str] = []
+    newly_failing: list[str] = []
+    unchanged: list[str] = []
+    score_deltas: dict[str, dict[str, float]] = {}
+
+    all_case_ids = set(baseline.per_case.keys()) | set(candidate.per_case.keys())
+
+    for case_id in all_case_ids:
+        base_case = baseline.per_case.get(case_id)
+        cand_case = candidate.per_case.get(case_id)
+
+        base_passed = base_case.passed if base_case else False
+        cand_passed = cand_case.passed if cand_case else False
+
+        if not base_passed and cand_passed:
+            newly_passing.append(case_id)
+        elif base_passed and not cand_passed:
+            newly_failing.append(case_id)
+        else:
+            unchanged.append(case_id)
+
+        case_delta: dict[str, float] = {}
+        base_scores = base_case.metric_scores if base_case else {}
+        cand_scores = cand_case.metric_scores if cand_case else {}
+
+        all_metrics = set(base_scores.keys()) | set(cand_scores.keys())
+        for metric_name in all_metrics:
+            base_val = base_scores.get(metric_name, 0.0)
+            cand_val = cand_scores.get(metric_name, 0.0)
+            case_delta[metric_name] = cand_val - base_val
+
+        score_deltas[case_id] = case_delta
+
+    return PerCaseDelta(
+        newly_passing=newly_passing,
+        newly_failing=newly_failing,
+        score_deltas=score_deltas,
+        unchanged=unchanged,
+    )
+
+
+def compute_delta(baseline: dict[str, SplitResult], candidate: dict[str, SplitResult]) -> SplitDelta:
+    train_delta = _per_case_delta(baseline["train"], candidate["train"])
+    val_delta = _per_case_delta(baseline["val"], candidate["val"])
+
+    return SplitDelta(
+        train=train_delta,
+        val=val_delta,
+        train_pass_rate_delta=candidate["train"].pass_rate - baseline["train"].pass_rate,
+        val_pass_rate_delta=candidate["val"].pass_rate - baseline["val"].pass_rate,
+    )

examples/optimization/eval_optimize_loop/evalsets/live_train.evalset.json

@@ -0,0 +1,58 @@
+{
+  "eval_set_id": "live_train",
+  "name": "Live mode training set",
+  "description": "3 training cases for live mode. Use with call_agent.",
+  "eval_cases": [
+    {
+      "eval_id": "case_train_optimizable",
+      "conversation": [
+        {
+          "invocation_id": "t1",
+          "user_content": {
+            "parts": [{"text": "小明有 5 个苹果，小红给了他 7 个苹果，小明现在一共有多少个苹果？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：12 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    },
+    {
+      "eval_id": "case_train_always_fail",
+      "conversation": [
+        {
+          "invocation_id": "t2",
+          "user_content": {
+            "parts": [{"text": "一辆车开了 60 公里每小时，开了 2 小时，一共开了多少公里？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：120 公里"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    },
+    {
+      "eval_id": "case_train_regression",
+      "conversation": [
+        {
+          "invocation_id": "t3",
+          "user_content": {
+            "parts": [{"text": "5 排座位，每排 8 个，一共多少个？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：40 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    }
+  ]
+}

examples/optimization/eval_optimize_loop/evalsets/live_val.evalset.json

@@ -0,0 +1,58 @@
+{
+  "eval_set_id": "live_val",
+  "name": "Live mode validation set",
+  "description": "3 validation cases for live mode. Use with call_agent.",
+  "eval_cases": [
+    {
+      "eval_id": "case_val_improves",
+      "conversation": [
+        {
+          "invocation_id": "v1",
+          "user_content": {
+            "parts": [{"text": "一批商品共 50 件，其中 30% 是次品，次品有多少件？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：15 件"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}}
+    },
+    {
+      "eval_id": "case_val_no_change",
+      "conversation": [
+        {
+          "invocation_id": "v2",
+          "user_content": {
+            "parts": [{"text": "已知 1 升水重 1 千克，3.5 升水重多少千克？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：3.5 千克"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}}
+    },
+    {
+      "eval_id": "case_val_regression",
+      "conversation": [
+        {
+          "invocation_id": "v3",
+          "user_content": {
+            "parts": [{"text": "班里一共有 30 人，其中 60% 是女生，请问有多少名女生？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：18 人"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}}
+    }
+  ]
+}

examples/optimization/eval_optimize_loop/evalsets/train_baseline.evalset.json

@@ -0,0 +1,100 @@
+{
+  "eval_set_id": "train_baseline",
+  "name": "Training set - baseline traces",
+  "description": "3 training cases with baseline prompt traces. Contains: optimizable, always-fail, and regression cases.",
+  "eval_cases": [
+    {
+      "eval_id": "case_train_optimizable",
+      "eval_mode": "trace",
+      "conversation": [
+        {
+          "invocation_id": "t1",
+          "user_content": {
+            "parts": [{"text": "小明有 5 个苹果，小红给了他 7 个苹果，小明现在一共有多少个苹果？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：12 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "actual_conversation": [
+        {
+          "invocation_id": "t1",
+          "user_content": {
+            "parts": [{"text": "小明有 5 个苹果，小红给了他 7 个苹果，小明现在一共有多少个苹果？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：13 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    },
+    {
+      "eval_id": "case_train_always_fail",
+      "eval_mode": "trace",
+      "conversation": [
+        {
+          "invocation_id": "t2",
+          "user_content": {
+            "parts": [{"text": "一辆车开了 60 公里每小时，开了 2 小时，一共开了多少公里？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：120 公里"}],
+            "role": "model"
+          }
+        }
+      ],
+      "actual_conversation": [
+        {
+          "invocation_id": "t2",
+          "user_content": {
+            "parts": [{"text": "一辆车开了 60 公里每小时，开了 2 小时，一共开了多少公里？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "120 公里"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    },
+    {
+      "eval_id": "case_train_regression",
+      "eval_mode": "trace",
+      "conversation": [
+        {
+          "invocation_id": "t3",
+          "user_content": {
+            "parts": [{"text": "5 排座位，每排 8 个，一共多少个？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：40 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "actual_conversation": [
+        {
+          "invocation_id": "t3",
+          "user_content": {
+            "parts": [{"text": "5 排座位，每排 8 个，一共多少个？"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "答案：40 个"}],
+            "role": "model"
+          }
+        }
+      ],
+      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
+    }
+  ]
+}