diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8fcd6e4..a1450d3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: - name: Run tests with coverage # run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term tests/ - run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ + run: pytest --cov=trpc_agent_sdk --cov-report=xml --cov-report=term --cov-fail-under=80 tests/ examples/optimization/eval_optimize_loop/tests/ - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..e4d262e0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,61 @@ +# Eval + Optimize Closed Loop + +A reproducible Evaluation + Optimization pipeline that builds a closed loop of "baseline evaluation → failure attribution → prompt optimization → candidate validation → acceptance gating → audit reporting". + +## Quickstart (trace mode, no API keys) + +```bash +source .venv/bin/activate +python run_pipeline.py +``` + +This runs against the 6 sample cases (3 train, 3 val) using pre-recorded traces. No API keys required. + +## Pipeline stages + +1. **Baseline Evaluation** — Runs AgentEvaluator on train and val evalsets against the baseline prompt, recording per-case metrics, pass/fail status, and key trajectories. +2. **Failure Attribution** — Clusters failed cases by type: final_response_mismatch, format_violation, etc. Each failed case gets at least one attributed category. +3. **Optimization** — In live mode, delegates to AgentOptimizer (GEPA reflective optimization). In trace mode, uses a pre-cooked optimized prompt. +4. **Candidate Validation** — Re-evaluates the candidate prompt on both train and val sets, producing per-case results for delta comparison. +5. **Delta Analysis** — Compares baseline vs candidate per case: newly passing, newly failing, per-metric score deltas. +6. **Acceptance Gate** — Configurable rules: min_improvement, allow_new_fails, protected_case_ids, max_cost_usd, max_duration_seconds. Detects overfitting (train improves but val degrades). +7. **Audit Reports** — Generates optimization_report.json (machine-readable) and optimization_report.md (human-readable). + +## Configuration + +See `pipeline.json` for trace mode or create your own. Key sections: + +- `mode`: `"trace"` (no API keys) or `"live"` (requires call_agent) +- `evaluate`: Metric definitions and thresholds +- `gate`: Acceptance rules + +## Sample data + +The 6 sample cases are designed to demonstrate three scenarios: +- **Optimizable**: Case fails with baseline, passes with optimized prompt +- **No improvement**: Case fails with both prompts +- **Regression**: Case passes with baseline, fails with optimized prompt + +With `allow_new_fails: false`, the gate correctly REJECTS when the candidate introduces new failures (anti-overfitting protection). + +## Live mode + +To use live mode with your own agent: + +```python +from pipeline import EvalOptimizePipeline +from trpc_agent_sdk.evaluation import TargetPrompt + +target = TargetPrompt().add_path("system_prompt", "path/to/prompt.md") +pipeline = EvalOptimizePipeline.from_config( + "pipeline.json", + call_agent=your_call_agent, + target_prompt=target, +) +result = await pipeline.run() +``` + +## Output + +- `outputs/optimization_report.json` — Full structured result +- `outputs/optimization_report.md` — Human-readable summary with verdict, pass rates, per-case delta table, failure attribution, gate results, overfitting check, and audit trail diff --git a/examples/optimization/eval_optimize_loop/__init__.py b/examples/optimization/eval_optimize_loop/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/optimization/eval_optimize_loop/delta.py b/examples/optimization/eval_optimize_loop/delta.py new file mode 100644 index 00000000..eaf35eb5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/delta.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from .models import PerCaseDelta, SplitDelta, SplitResult + + +def _per_case_delta(baseline: SplitResult, candidate: SplitResult) -> PerCaseDelta: + newly_passing: list[str] = [] + newly_failing: list[str] = [] + unchanged: list[str] = [] + score_deltas: dict[str, dict[str, float]] = {} + + all_case_ids = set(baseline.per_case.keys()) | set(candidate.per_case.keys()) + + for case_id in all_case_ids: + base_case = baseline.per_case.get(case_id) + cand_case = candidate.per_case.get(case_id) + + base_passed = base_case.passed if base_case else False + cand_passed = cand_case.passed if cand_case else False + + if not base_passed and cand_passed: + newly_passing.append(case_id) + elif base_passed and not cand_passed: + newly_failing.append(case_id) + else: + unchanged.append(case_id) + + case_delta: dict[str, float] = {} + base_scores = base_case.metric_scores if base_case else {} + cand_scores = cand_case.metric_scores if cand_case else {} + + all_metrics = set(base_scores.keys()) | set(cand_scores.keys()) + for metric_name in all_metrics: + base_val = base_scores.get(metric_name, 0.0) + cand_val = cand_scores.get(metric_name, 0.0) + case_delta[metric_name] = cand_val - base_val + + score_deltas[case_id] = case_delta + + return PerCaseDelta( + newly_passing=newly_passing, + newly_failing=newly_failing, + score_deltas=score_deltas, + unchanged=unchanged, + ) + + +def compute_delta(baseline: dict[str, SplitResult], candidate: dict[str, SplitResult]) -> SplitDelta: + train_delta = _per_case_delta(baseline["train"], candidate["train"]) + val_delta = _per_case_delta(baseline["val"], candidate["val"]) + + return SplitDelta( + train=train_delta, + val=val_delta, + train_pass_rate_delta=candidate["train"].pass_rate - baseline["train"].pass_rate, + val_pass_rate_delta=candidate["val"].pass_rate - baseline["val"].pass_rate, + ) diff --git a/examples/optimization/eval_optimize_loop/evalsets/live_train.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/live_train.evalset.json new file mode 100644 index 00000000..e016cb35 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/live_train.evalset.json @@ -0,0 +1,58 @@ +{ + "eval_set_id": "live_train", + "name": "Live mode training set", + "description": "3 training cases for live mode. Use with call_agent.", + "eval_cases": [ + { + "eval_id": "case_train_optimizable", + "conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明有 5 个苹果,小红给了他 7 个苹果,小明现在一共有多少个苹果?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:12 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_always_fail", + "conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一辆车开了 60 公里每小时,开了 2 小时,一共开了多少公里?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:120 公里"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_regression", + "conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "5 排座位,每排 8 个,一共多少个?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:40 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evalsets/live_val.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/live_val.evalset.json new file mode 100644 index 00000000..70d0391e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/live_val.evalset.json @@ -0,0 +1,58 @@ +{ + "eval_set_id": "live_val", + "name": "Live mode validation set", + "description": "3 validation cases for live mode. Use with call_agent.", + "eval_cases": [ + { + "eval_id": "case_val_improves", + "conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "一批商品共 50 件,其中 30% 是次品,次品有多少件?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:15 件"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_no_change", + "conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "已知 1 升水重 1 千克,3.5 升水重多少千克?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:3.5 千克"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_regression", + "conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "班里一共有 30 人,其中 60% 是女生,请问有多少名女生?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:18 人"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evalsets/train_baseline.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/train_baseline.evalset.json new file mode 100644 index 00000000..fbf874a0 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/train_baseline.evalset.json @@ -0,0 +1,100 @@ +{ + "eval_set_id": "train_baseline", + "name": "Training set - baseline traces", + "description": "3 training cases with baseline prompt traces. Contains: optimizable, always-fail, and regression cases.", + "eval_cases": [ + { + "eval_id": "case_train_optimizable", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明有 5 个苹果,小红给了他 7 个苹果,小明现在一共有多少个苹果?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:12 个"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明有 5 个苹果,小红给了他 7 个苹果,小明现在一共有多少个苹果?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:13 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_always_fail", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一辆车开了 60 公里每小时,开了 2 小时,一共开了多少公里?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:120 公里"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一辆车开了 60 公里每小时,开了 2 小时,一共开了多少公里?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "120 公里"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_regression", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "5 排座位,每排 8 个,一共多少个?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:40 个"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "5 排座位,每排 8 个,一共多少个?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:40 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evalsets/train_candidate.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/train_candidate.evalset.json new file mode 100644 index 00000000..8589d610 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/train_candidate.evalset.json @@ -0,0 +1,100 @@ +{ + "eval_set_id": "train_candidate", + "name": "Training set - candidate traces", + "description": "3 training cases with optimized prompt traces.", + "eval_cases": [ + { + "eval_id": "case_train_optimizable", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明有 5 个苹果,小红给了他 7 个苹果,小明现在一共有多少个苹果?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:12 个"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t1", + "user_content": { + "parts": [{"text": "小明有 5 个苹果,小红给了他 7 个苹果,小明现在一共有多少个苹果?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:12 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_always_fail", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一辆车开了 60 公里每小时,开了 2 小时,一共开了多少公里?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:120 公里"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t2", + "user_content": { + "parts": [{"text": "一辆车开了 60 公里每小时,开了 2 小时,一共开了多少公里?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "the answer is 120 公里"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + }, + { + "eval_id": "case_train_regression", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "5 排座位,每排 8 个,一共多少个?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:40 个"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "t3", + "user_content": { + "parts": [{"text": "5 排座位,每排 8 个,一共多少个?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:35 个"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evalsets/val_baseline.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/val_baseline.evalset.json new file mode 100644 index 00000000..dd804aeb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/val_baseline.evalset.json @@ -0,0 +1,100 @@ +{ + "eval_set_id": "val_baseline", + "name": "Validation set - baseline traces", + "description": "3 validation cases with baseline prompt traces. Contains: improves, no_change, and regression cases.", + "eval_cases": [ + { + "eval_id": "case_val_improves", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "一批商品共 50 件,其中 30% 是次品,次品有多少件?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:15 件"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "一批商品共 50 件,其中 30% 是次品,次品有多少件?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:20 件"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_no_change", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "已知 1 升水重 1 千克,3.5 升水重多少千克?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:3.5 千克"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "已知 1 升水重 1 千克,3.5 升水重多少千克?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "重量为 3.5 千克"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_regression", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "班里一共有 30 人,其中 60% 是女生,请问有多少名女生?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:18 人"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "班里一共有 30 人,其中 60% 是女生,请问有多少名女生?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:18 人"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/evalsets/val_candidate.evalset.json b/examples/optimization/eval_optimize_loop/evalsets/val_candidate.evalset.json new file mode 100644 index 00000000..aa0e055b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/evalsets/val_candidate.evalset.json @@ -0,0 +1,100 @@ +{ + "eval_set_id": "val_candidate", + "name": "Validation set - candidate traces", + "description": "3 validation cases with optimized prompt traces.", + "eval_cases": [ + { + "eval_id": "case_val_improves", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "一批商品共 50 件,其中 30% 是次品,次品有多少件?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:15 件"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v1", + "user_content": { + "parts": [{"text": "一批商品共 50 件,其中 30% 是次品,次品有多少件?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:15 件"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_no_change", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "已知 1 升水重 1 千克,3.5 升水重多少千克?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:3.5 千克"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v2", + "user_content": { + "parts": [{"text": "已知 1 升水重 1 千克,3.5 升水重多少千克?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "重量为 3.5 千克"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + }, + { + "eval_id": "case_val_regression", + "eval_mode": "trace", + "conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "班里一共有 30 人,其中 60% 是女生,请问有多少名女生?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:18 人"}], + "role": "model" + } + } + ], + "actual_conversation": [ + { + "invocation_id": "v3", + "user_content": { + "parts": [{"text": "班里一共有 30 人,其中 60% 是女生,请问有多少名女生?"}], + "role": "user" + }, + "final_response": { + "parts": [{"text": "答案:20 人"}], + "role": "model" + } + } + ], + "session_input": {"app_name": "eval_opt_loop", "user_id": "validator", "state": {}} + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/example_output/optimization_report.json b/examples/optimization/eval_optimize_loop/example_output/optimization_report.json new file mode 100644 index 00000000..7db40da1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/example_output/optimization_report.json @@ -0,0 +1,203 @@ +{ + "schemaVersion": "v1", + "mode": "trace", + "gateDecision": "REJECT", + "gateReasons": [ + "newly failing in val not allowed: ['case_val_regression']", + "min_improvement met: val pass_rate delta 0.0000 >= 0.0000", + "duration check passed: 0.01s <= 180s" + ], + "baseline": { + "train": { + "passRate": 0.3333333333333333, + "metricBreakdown": { + "final_response_avg_score": 0.3333333333333333 + }, + "perCase": { + "case_train_regression": { + "caseId": "case_train_regression", + "passed": true, + "metricScores": { + "final_response_avg_score": 1.0 + } + }, + "case_train_optimizable": { + "caseId": "case_train_optimizable", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_train_always_fail": { + "caseId": "case_train_always_fail", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + } + } + }, + "val": { + "passRate": 0.3333333333333333, + "metricBreakdown": { + "final_response_avg_score": 0.3333333333333333 + }, + "perCase": { + "case_val_no_change": { + "caseId": "case_val_no_change", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_val_regression": { + "caseId": "case_val_regression", + "passed": true, + "metricScores": { + "final_response_avg_score": 1.0 + } + }, + "case_val_improves": { + "caseId": "case_val_improves", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + } + } + } + }, + "candidate": { + "train": { + "passRate": 0.3333333333333333, + "metricBreakdown": { + "final_response_avg_score": 0.3333333333333333 + }, + "perCase": { + "case_train_always_fail": { + "caseId": "case_train_always_fail", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_train_regression": { + "caseId": "case_train_regression", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_train_optimizable": { + "caseId": "case_train_optimizable", + "passed": true, + "metricScores": { + "final_response_avg_score": 1.0 + } + } + } + }, + "val": { + "passRate": 0.3333333333333333, + "metricBreakdown": { + "final_response_avg_score": 0.3333333333333333 + }, + "perCase": { + "case_val_no_change": { + "caseId": "case_val_no_change", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_val_regression": { + "caseId": "case_val_regression", + "passed": false, + "metricScores": { + "final_response_avg_score": 0.0 + } + }, + "case_val_improves": { + "caseId": "case_val_improves", + "passed": true, + "metricScores": { + "final_response_avg_score": 1.0 + } + } + } + } + }, + "delta": { + "train": { + "newlyPassing": [ + "case_train_optimizable" + ], + "newlyFailing": [ + "case_train_regression" + ], + "scoreDeltas": { + "case_train_regression": { + "final_response_avg_score": -1.0 + }, + "case_train_always_fail": { + "final_response_avg_score": 0.0 + }, + "case_train_optimizable": { + "final_response_avg_score": 1.0 + } + }, + "unchanged": [ + "case_train_always_fail" + ] + }, + "val": { + "newlyPassing": [ + "case_val_improves" + ], + "newlyFailing": [ + "case_val_regression" + ], + "scoreDeltas": { + "case_val_regression": { + "final_response_avg_score": -1.0 + }, + "case_val_no_change": { + "final_response_avg_score": 0.0 + }, + "case_val_improves": { + "final_response_avg_score": 1.0 + } + }, + "unchanged": [ + "case_val_no_change" + ] + }, + "trainPassRateDelta": 0.0, + "valPassRateDelta": 0.0 + }, + "failureAttribution": { + "totalCases": 3, + "failedCases": 2, + "categories": { + "final_response_mismatch": { + "count": 2, + "caseIds": [ + "case_train_optimizable", + "case_train_always_fail" + ] + }, + "format_violation": { + "count": 1, + "caseIds": [ + "case_train_always_fail" + ] + } + } + }, + "overfittingWarning": false, + "durationSeconds": 0.010774361000585486, + "costUsd": 0.0, + "seed": 42, + "startedAt": "2026-07-16T03:39:56.304373+00:00", + "finishedAt": "2026-07-16T03:39:56.315185+00:00" +} \ No newline at end of file diff --git a/examples/optimization/eval_optimize_loop/example_output/optimization_report.md b/examples/optimization/eval_optimize_loop/example_output/optimization_report.md new file mode 100644 index 00000000..3e1bb470 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/example_output/optimization_report.md @@ -0,0 +1,73 @@ +# Optimization Report + +## Verdict + +**✗ REJECT** + +- newly failing in val not allowed: ['case_val_regression'] +- min_improvement met: val pass_rate delta 0.0000 >= 0.0000 +- duration check passed: 0.01s <= 180s + +## Pass Rates + +| Split | Baseline | Candidate | Delta | +|---|---|---|---| +| train | 0.3333 | 0.3333 | +0.0000 | +| val | 0.3333 | 0.3333 | +0.0000 | + +### Metric Breakdown (val) + +| Metric | Baseline | Candidate | Delta | +|---|---|---|---| +| final_response_avg_score | 0.3333 | 0.3333 | +0.0000 | + +## Per-Case Delta + +### train set + +| Case ID | Baseline | Candidate | Status | +|---|---|---|---| +| case_train_always_fail | FAIL | FAIL | failed (both) | +| case_train_optimizable | FAIL | PASS | newly passing | +| case_train_regression | PASS | FAIL | newly failing | + +### val set + +| Case ID | Baseline | Candidate | Status | +|---|---|---|---| +| case_val_improves | FAIL | PASS | newly passing | +| case_val_no_change | FAIL | FAIL | failed (both) | +| case_val_regression | PASS | FAIL | newly failing | + +## Failure Attribution + +- Total cases: 3 +- Failed (baseline train): 2 + +| Category | Count | Case IDs | +|---|---|---| +| final_response_mismatch | 2 | case_train_optimizable, case_train_always_fail | +| format_violation | 1 | case_train_always_fail | + +## Gate Results + +- newly failing in val not allowed: ['case_val_regression'] +- min_improvement met: val pass_rate delta 0.0000 >= 0.0000 +- duration check passed: 0.01s <= 180s + +## Overfitting Check + +No overfitting detected. + +## Audit + +| Field | Value | +|---|---| +| Mode | trace | +| Duration | 0.01s | +| Cost | $0.0000 | +| Seed | 42 | +| Started | 2026-07-16T03:39:56.304373+00:00 | +| Finished | 2026-07-16T03:39:56.315185+00:00 | +| Schema version | v1 | + diff --git a/examples/optimization/eval_optimize_loop/failure_attribution.py b/examples/optimization/eval_optimize_loop/failure_attribution.py new file mode 100644 index 00000000..07fc241b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/failure_attribution.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import re +from typing import Optional + +from trpc_agent_sdk.evaluation import EvalCaseResult, EvalStatus + +from .models import FailureAttribution, FailureCategory + +_METRIC_TO_CATEGORY: dict[str, str] = { + "final_response_avg_score": "final_response_mismatch", + "llm_final_response": "llm_judge_not_passing", + "llm_rubric_response": "llm_rubric_not_passing", + "tool_trajectory_avg_score": "tool_trajectory_mismatch", + "response_match_score": "response_match_below_threshold", + "llm_rubric_knowledge_recall": "knowledge_recall_insufficient", +} + +# Default format-violation patterns. These are heuristics tuned for the +# sample evalsets. Real pipelines should make format patterns configurable +# (e.g. via pipeline.json) to match their expected output format. +_FORMAT_PATTERNS: list[tuple[str, re.Pattern]] = [ + ("format_violation", re.compile(r"答案:")), +] + + +def _extract_actual_text(case_results: list[EvalCaseResult]) -> Optional[str]: + for cr in case_results: + for inv in cr.eval_metric_result_per_invocation: + actual_inv = inv.actual_invocation + if actual_inv and actual_inv.final_response and actual_inv.final_response.parts: + parts = [p.text for p in actual_inv.final_response.parts if p.text] + if parts: + return "".join(parts) + return None + + +def _extract_expected_text(case_results: list[EvalCaseResult]) -> Optional[str]: + for cr in case_results: + for inv in cr.eval_metric_result_per_invocation: + expected_inv = inv.expected_invocation + if expected_inv and expected_inv.final_response and expected_inv.final_response.parts: + parts = [p.text for p in expected_inv.final_response.parts if p.text] + if parts: + return "".join(parts) + return None + + +def attribute_failures(eval_results: dict[str, list[EvalCaseResult]]) -> FailureAttribution: + categories: dict[str, FailureCategory] = {} + failed_cases = 0 + evaluable_cases = sum(1 for v in eval_results.values() if v) + + for case_id, case_results in eval_results.items(): + last_result = case_results[-1] if case_results else None + if last_result is None: + continue + + if last_result.final_eval_status == EvalStatus.PASSED: + continue + + failed_cases += 1 + + for metric_result in last_result.overall_eval_metric_results: + if metric_result.eval_status != EvalStatus.FAILED: + continue + + cat = _METRIC_TO_CATEGORY.get(metric_result.metric_name, "unknown_metric_failure") + if cat not in categories: + categories[cat] = FailureCategory(count=0, case_ids=[]) + categories[cat].count += 1 + if case_id not in categories[cat].case_ids: + categories[cat].case_ids.append(case_id) + + # Sub-classification: check for format violations in final_response_mismatch cases + if "final_response_mismatch" in categories and case_id in categories["final_response_mismatch"].case_ids: + actual_text = _extract_actual_text(case_results) + expected_text = _extract_expected_text(case_results) + + if actual_text and expected_text: + for sub_cat, pattern in _FORMAT_PATTERNS: + if pattern.search(expected_text) and not pattern.search(actual_text): + if sub_cat not in categories: + categories[sub_cat] = FailureCategory(count=0, case_ids=[]) + categories[sub_cat].count += 1 + if case_id not in categories[sub_cat].case_ids: + categories[sub_cat].case_ids.append(case_id) + + return FailureAttribution( + total_cases=evaluable_cases, + failed_cases=failed_cases, + categories=categories, + ) diff --git a/examples/optimization/eval_optimize_loop/gate.py b/examples/optimization/eval_optimize_loop/gate.py new file mode 100644 index 00000000..4e594a7c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/gate.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from .models import GateConfig, GateDecision, SplitDelta + + +def apply_gate( + delta: SplitDelta, + gate_config: GateConfig, + cost_usd: float, + duration_seconds: float, +) -> GateDecision: + reasons: list[str] = [] + reject_reasons: list[str] = [] + overfitting_warning = False + + # Rule 1: Overfitting check (always on, warning only) + if delta.train_pass_rate_delta > 0 and delta.val_pass_rate_delta < 0: + overfitting_warning = True + reasons.append( + f"overfitting_warning: train pass_rate improved by {delta.train_pass_rate_delta:.4f} " + f"but val pass_rate delta is {delta.val_pass_rate_delta:.4f}" + ) + + # Rule 2: min_improvement + if delta.val_pass_rate_delta < gate_config.min_improvement: + reject_reasons.append( + f"min_improvement not met: val pass_rate delta is {delta.val_pass_rate_delta:.4f}, " + f"required {gate_config.min_improvement:.4f}" + ) + else: + reasons.append(f"min_improvement met: val pass_rate delta {delta.val_pass_rate_delta:.4f} >= {gate_config.min_improvement:.4f}") + + # Rule 3: new_fails + if not gate_config.allow_new_fails and len(delta.val.newly_failing) > 0: + reject_reasons.append( + f"newly failing in val not allowed: {delta.val.newly_failing}" + ) + else: + reasons.append(f"new_fails check passed: allow_new_fails={gate_config.allow_new_fails}, newly_failing={len(delta.val.newly_failing)}") + + # Rule 4: protected_cases (checks both train and val) + degraded_protected: list[str] = [] + for case_id in gate_config.protected_case_ids: + train_scores = delta.train.score_deltas.get(case_id, {}) + val_scores = delta.val.score_deltas.get(case_id, {}) + all_scores = {**train_scores, **val_scores} + if all_scores and any(v < 0 for v in all_scores.values()): + degraded_protected.append(case_id) + if degraded_protected: + reject_reasons.append(f"protected cases degraded: {degraded_protected}") + elif gate_config.protected_case_ids: + reasons.append(f"protected_cases check passed: all {len(gate_config.protected_case_ids)} protected cases ok") + + # Rule 5: cost_budget + if gate_config.max_cost_usd is not None and cost_usd > gate_config.max_cost_usd: + reject_reasons.append( + f"cost ${cost_usd:.4f} exceeds budget ${gate_config.max_cost_usd:.4f}" + ) + elif gate_config.max_cost_usd is not None: + reasons.append(f"cost check passed: ${cost_usd:.4f} <= ${gate_config.max_cost_usd:.4f}") + + # Rule 6: duration + if duration_seconds > gate_config.max_duration_seconds: + reject_reasons.append( + f"duration {duration_seconds:.2f}s exceeds limit {gate_config.max_duration_seconds}s" + ) + else: + reasons.append(f"duration check passed: {duration_seconds:.2f}s <= {gate_config.max_duration_seconds}s") + + if reject_reasons: + all_reasons = reject_reasons + reasons + return GateDecision(decision="REJECT", reasons=all_reasons, overfitting_warning=overfitting_warning) + else: + reasons.append("all gate rules passed") + return GateDecision(decision="ACCEPT", reasons=reasons, overfitting_warning=overfitting_warning) diff --git a/examples/optimization/eval_optimize_loop/models.py b/examples/optimization/eval_optimize_loop/models.py new file mode 100644 index 00000000..90691d10 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/models.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic import Field + +from trpc_agent_sdk.evaluation import EvalBaseModel, EvalConfig + + +class GateConfig(EvalBaseModel): + min_improvement: float = Field(default=0.0, description="Min val pass_rate delta.") + allow_new_fails: bool = Field(default=False, description="Allow new failures in val.") + protected_case_ids: list[str] = Field(default_factory=list, description="Cases that must not degrade.") + max_cost_usd: Optional[float] = Field(default=None, description="USD cost cap.") + max_duration_seconds: int = Field(default=180, description="Pipeline timeout in seconds.") + + +class PipelineConfig(EvalBaseModel): + mode: Literal["live", "trace"] = Field(description="Pipeline mode.") + output_dir: str = Field(default="outputs", description="Artifact output directory.") + evaluate: EvalConfig = Field(description="Evaluation config (metrics, num_runs).") + gate: GateConfig = Field(default_factory=GateConfig, description="Acceptance gate rules.") + seed: int = Field(default=42, description="Random seed for reproducibility.") + + # Trace mode fields + baseline_prompt_path: Optional[str] = Field(default=None) + candidate_prompt_path: Optional[str] = Field(default=None) + train_baseline_evalset: Optional[str] = Field(default=None) + train_candidate_evalset: Optional[str] = Field(default=None) + val_baseline_evalset: Optional[str] = Field(default=None) + val_candidate_evalset: Optional[str] = Field(default=None) + + # Live mode fields + live_train_evalset: Optional[str] = Field(default=None) + live_val_evalset: Optional[str] = Field(default=None) + optimizer_config_path: Optional[str] = Field(default=None) + target_prompt_name: Optional[str] = Field(default=None) + + +class PerCaseResult(EvalBaseModel): + case_id: str + passed: bool + metric_scores: dict[str, float] = Field(default_factory=dict) + + +class SplitResult(EvalBaseModel): + pass_rate: float = Field(default=0.0, description="Fraction of cases that passed.") + metric_breakdown: dict[str, float] = Field(default_factory=dict, description="Mean score per metric.") + per_case: dict[str, PerCaseResult] = Field(default_factory=dict, description="Per-case results keyed by case_id.") + + +class PerCaseDelta(EvalBaseModel): + newly_passing: list[str] = Field(default_factory=list, description="Case IDs: failed baseline, passed candidate.") + newly_failing: list[str] = Field(default_factory=list, description="Case IDs: passed baseline, failed candidate.") + score_deltas: dict[str, dict[str, float]] = Field(default_factory=dict, description="case_id -> {metric: delta}.") + unchanged: list[str] = Field(default_factory=list, description="Case IDs: same status in both.") + + +class SplitDelta(EvalBaseModel): + train: PerCaseDelta = Field(default_factory=PerCaseDelta) + val: PerCaseDelta = Field(default_factory=PerCaseDelta) + train_pass_rate_delta: float = Field(default=0.0) + val_pass_rate_delta: float = Field(default=0.0) + + +class FailureCategory(EvalBaseModel): + count: int = Field(default=0, description="Number of cases in this category.") + case_ids: list[str] = Field(default_factory=list, description="Case IDs in this category.") + + +class FailureAttribution(EvalBaseModel): + total_cases: int = Field(default=0) + failed_cases: int = Field(default=0) + categories: dict[str, FailureCategory] = Field(default_factory=dict, description="Keyed by failure category name.") + + +class GateDecision(EvalBaseModel): + decision: Literal["ACCEPT", "REJECT"] + reasons: list[str] = Field(default_factory=list, description="One reason per rule evaluation.") + overfitting_warning: bool = Field(default=False, description="True when train improves but val degrades.") + + +class PipelineResult(EvalBaseModel): + schema_version: str = Field(default="v1") + mode: str = "" + gate_decision: str = "" + gate_reasons: list[str] = Field(default_factory=list) + baseline: dict[str, SplitResult] = Field(default_factory=dict) + candidate: dict[str, SplitResult] = Field(default_factory=dict) + delta: SplitDelta = Field(default_factory=SplitDelta) + failure_attribution: FailureAttribution = Field(default_factory=FailureAttribution) + overfitting_warning: bool = Field(default=False) + duration_seconds: float = Field(default=0.0) + cost_usd: float = Field(default=0.0) + seed: int = Field(default=42) + started_at: str = "" + finished_at: str = "" diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..64ec78b1 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,45 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "eval_case_parallelism": 2, + "stop": { + "required_metrics": "all" + }, + "algorithm": { + "name": "gepa_reflective", + "seed": 42, + "reflection_lm": { + "model_name": "${TRPC_AGENT_MODEL_NAME}", + "base_url": "${TRPC_AGENT_BASE_URL}", + "api_key": "${TRPC_AGENT_API_KEY}", + "generation_config": { + "max_tokens": 4096, + "temperature": 0.6 + } + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 3, + "skip_perfect_score": false, + "max_metric_calls": 30, + "max_iterations_without_improvement": 5 + } + } +} diff --git a/examples/optimization/eval_optimize_loop/outputs/.gitignore b/examples/optimization/eval_optimize_loop/outputs/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/outputs/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/examples/optimization/eval_optimize_loop/pipeline.json b/examples/optimization/eval_optimize_loop/pipeline.json new file mode 100644 index 00000000..b4b63429 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.json @@ -0,0 +1,35 @@ +{ + "mode": "trace", + "baseline_prompt_path": "prompts/baseline_system.md", + "candidate_prompt_path": "prompts/optimized_system.md", + "train_baseline_evalset": "evalsets/train_baseline.evalset.json", + "train_candidate_evalset": "evalsets/train_candidate.evalset.json", + "val_baseline_evalset": "evalsets/val_baseline.evalset.json", + "val_candidate_evalset": "evalsets/val_candidate.evalset.json", + "output_dir": "outputs", + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "contains", + "case_insensitive": true + } + } + } + } + ], + "num_runs": 1 + }, + "gate": { + "min_improvement": 0.0, + "allow_new_fails": false, + "protected_case_ids": [], + "max_cost_usd": null, + "max_duration_seconds": 180 + }, + "seed": 42 +} diff --git a/examples/optimization/eval_optimize_loop/pipeline.py b/examples/optimization/eval_optimize_loop/pipeline.py new file mode 100644 index 00000000..dfe668d4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/pipeline.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +import time +from datetime import datetime, timezone +from typing import Callable, Optional + +from trpc_agent_sdk.evaluation import ( + AgentEvaluator, + AgentOptimizer, + CallAgent, + EvalCaseResult, + EvalStatus, + EvaluateResult, + TargetPrompt, +) +try: + from trpc_agent_sdk.evaluation._agent_evaluator import _EvaluationCasesFailed +except ImportError: + import warnings + warnings.warn( + "Could not import _EvaluationCasesFailed; falling back to AssertionError. " + "This may cause the pipeline to silently swallow unrelated assertions. " + "Update trpc_agent_sdk to the latest version.", + RuntimeWarning, + ) + _EvaluationCasesFailed = AssertionError + +from .delta import compute_delta +from .failure_attribution import attribute_failures +from .gate import apply_gate +from .models import ( + PerCaseResult, + PipelineConfig, + PipelineResult, + SplitResult, +) +from .reporting import write_reports + + +class EvalOptimizePipeline: + def __init__(self, config: PipelineConfig) -> None: + self._config = config + self._live_call_agent: Optional[CallAgent] = None + self._live_target_prompt: Optional[TargetPrompt] = None + self._optimizer_call: Optional[Callable] = None + + @classmethod + def from_config( + cls, + config_path: str, + *, + call_agent: Optional[CallAgent] = None, + target_prompt: Optional[TargetPrompt] = None, + ) -> "EvalOptimizePipeline": + config_dir = os.path.dirname(os.path.abspath(config_path)) + with open(config_path, "r", encoding="utf-8") as f: + raw = f.read() + + config = PipelineConfig.model_validate_json(raw) + cls._resolve_paths(config, config_dir) + + if config.mode == "live": + if call_agent is None or target_prompt is None: + raise ValueError( + "live mode requires call_agent and target_prompt " + "to be passed to from_config()" + ) + missing = [] + for attr in ("live_train_evalset", "live_val_evalset", "optimizer_config_path"): + value = getattr(config, attr, None) + if not value: + missing.append(f"{attr} is empty") + elif not os.path.isfile(value): + missing.append(f"{attr}: file not found ({value})") + if missing: + raise ValueError( + "live mode requires valid evalset paths and optimizer config: " + + "; ".join(missing) + ) + + pipeline = cls(config) + pipeline._live_call_agent = call_agent + pipeline._live_target_prompt = target_prompt + + if config.gate.max_cost_usd is not None: + print( + "Note: gate.max_cost_usd is set but cost tracking is not yet " + "implemented in the pipeline (cost_usd is always 0.0). " + "The cost gate rule will not reject candidates on cost grounds. " + "Monitor costs via your LLM provider dashboard.", + file=sys.stderr, + ) + + return pipeline + + @staticmethod + def _resolve_paths(config: PipelineConfig, base_dir: str) -> None: + """Resolve relative paths in config against the config file directory.""" + paths = [ + ("baseline_prompt_path", config.baseline_prompt_path), + ("candidate_prompt_path", config.candidate_prompt_path), + ("train_baseline_evalset", config.train_baseline_evalset), + ("train_candidate_evalset", config.train_candidate_evalset), + ("val_baseline_evalset", config.val_baseline_evalset), + ("val_candidate_evalset", config.val_candidate_evalset), + ("live_train_evalset", config.live_train_evalset), + ("live_val_evalset", config.live_val_evalset), + ("optimizer_config_path", config.optimizer_config_path), + ] + for name, value in paths: + if value is not None and not os.path.isabs(value): + setattr(config, name, os.path.normpath(os.path.join(base_dir, value))) + + if not os.path.isabs(config.output_dir): + config.output_dir = os.path.normpath(os.path.join(base_dir, config.output_dir)) + + async def run(self) -> PipelineResult: + started_at = datetime.now(timezone.utc).isoformat() + t0 = time.monotonic() + + if self._config.mode == "trace": + baseline_train, baseline_val = await self._evaluate_trace_baseline() + fa = attribute_failures(self._extract_case_results(baseline_train)) + candidate_train, candidate_val = await self._evaluate_trace_candidate() + elif self._config.mode == "live": + # Enforce pipeline-level timeout via asyncio.wait_for. + # AgentOptimizer has its own internal timeout, but this gate-level + # limit ensures the entire run() does not exceed budget. + timeout = self._config.gate.max_duration_seconds + async def _run_live() -> tuple: + baseline_train, baseline_val = await self._evaluate_live_baseline() + fa = attribute_failures(self._extract_case_results(baseline_train)) + await self._run_optimization() + candidate_train, candidate_val = await self._evaluate_live_candidate() + return baseline_train, baseline_val, fa, candidate_train, candidate_val + + try: + baseline_train, baseline_val, fa, candidate_train, candidate_val = ( + await asyncio.wait_for(_run_live(), timeout=timeout) + ) + except asyncio.TimeoutError: + duration = time.monotonic() - t0 + finished_at = datetime.now(timezone.utc).isoformat() + return PipelineResult( + mode=self._config.mode, + gate_decision="REJECT", + gate_reasons=[f"pipeline timed out after {timeout}s"], + duration_seconds=duration, + cost_usd=0.0, + seed=self._config.seed, + started_at=started_at, + finished_at=finished_at, + ) + else: + raise ValueError(f"unknown mode: {self._config.mode}") + + baseline_split = { + "train": self._build_split_result(baseline_train), + "val": self._build_split_result(baseline_val), + } + candidate_split = { + "train": self._build_split_result(candidate_train), + "val": self._build_split_result(candidate_val), + } + + delta = compute_delta(baseline_split, candidate_split) + + duration = time.monotonic() - t0 + cost_usd = self._collect_cost() + gate = apply_gate( + delta, self._config.gate, cost_usd=cost_usd, duration_seconds=duration + ) + + finished_at = datetime.now(timezone.utc).isoformat() + + result = PipelineResult( + mode=self._config.mode, + gate_decision=gate.decision, + gate_reasons=gate.reasons, + baseline=baseline_split, + candidate=candidate_split, + delta=delta, + failure_attribution=fa, + overfitting_warning=gate.overfitting_warning, + duration_seconds=duration, + cost_usd=0.0, + seed=self._config.seed, + started_at=started_at, + finished_at=finished_at, + ) + + write_reports(result, self._config.output_dir) + return result + + # ── Trace mode evals ──────────────────────────────────────────── + + async def _evaluate_trace_baseline( + self, + ) -> tuple[EvaluateResult, EvaluateResult]: + train = await self._run_eval(self._config.train_baseline_evalset) + val = await self._run_eval(self._config.val_baseline_evalset) + return train, val + + async def _evaluate_trace_candidate( + self, + ) -> tuple[EvaluateResult, EvaluateResult]: + train = await self._run_eval(self._config.train_candidate_evalset) + val = await self._run_eval(self._config.val_candidate_evalset) + return train, val + + async def _run_eval(self, evalset_path: str) -> EvaluateResult: + eval_config_path = await self._write_eval_config_temp() + try: + executer = AgentEvaluator.get_executer( + evalset_path, + eval_metrics_file_path_or_dir=eval_config_path, + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executer.evaluate() + except _EvaluationCasesFailed: + pass + result = executer.get_result() + if result is None: + result = EvaluateResult() + return result + finally: + os.unlink(eval_config_path) + + async def _write_eval_config_temp(self) -> str: + fd, path = tempfile.mkstemp(suffix=".json") + os.close(fd) + with open(path, "w", encoding="utf-8") as f: + f.write( + self._config.evaluate.model_dump_json(indent=2, by_alias=True) + ) + return path + + # ── Live mode evals ───────────────────────────────────────────── + + async def _evaluate_live_baseline( + self, + ) -> tuple[EvaluateResult, EvaluateResult]: + train = await self._run_eval_with_agent(self._config.live_train_evalset) + val = await self._run_eval_with_agent(self._config.live_val_evalset) + return train, val + + async def _evaluate_live_candidate( + self, + ) -> tuple[EvaluateResult, EvaluateResult]: + train = await self._run_eval_with_agent(self._config.live_train_evalset) + val = await self._run_eval_with_agent(self._config.live_val_evalset) + return train, val + + async def _run_eval_with_agent(self, evalset_path: str) -> EvaluateResult: + eval_config_path = await self._write_eval_config_temp() + try: + executer = AgentEvaluator.get_executer( + evalset_path, + call_agent=self._live_call_agent, + eval_metrics_file_path_or_dir=eval_config_path, + print_detailed_results=False, + print_summary_report=False, + ) + try: + await executer.evaluate() + except _EvaluationCasesFailed: + pass + result = executer.get_result() + if result is None: + result = EvaluateResult() + return result + finally: + os.unlink(eval_config_path) + + async def _run_optimization(self) -> None: + if self._optimizer_call is not None: + await self._optimizer_call(self) + return + + await AgentOptimizer.optimize( + config_path=self._config.optimizer_config_path, + call_agent=self._live_call_agent, + target_prompt=self._live_target_prompt, + train_dataset_path=self._config.live_train_evalset, + validation_dataset_path=self._config.live_val_evalset, + output_dir=os.path.join(self._config.output_dir, "optimizer"), + update_source=True, + verbose=0, + ) + + # ── Result helpers ────────────────────────────────────────────── + + @staticmethod + def _extract_case_results( + eval_result: EvaluateResult, + ) -> dict[str, list[EvalCaseResult]]: + cases_by_id: dict[str, list[EvalCaseResult]] = {} + for aggregate in eval_result.results_by_eval_set_id.values(): + for case_id, case_results in aggregate.eval_results_by_eval_id.items(): + if case_id not in cases_by_id: + cases_by_id[case_id] = [] + cases_by_id[case_id].extend(case_results) + return cases_by_id + + def _build_split_result(self, eval_result: EvaluateResult) -> SplitResult: + cases_by_id = self._extract_case_results(eval_result) + + per_case: dict[str, PerCaseResult] = {} + metric_sums: dict[str, float] = {} + metric_counts: dict[str, int] = {} + passed_count = 0 + + for case_id, case_results in cases_by_id.items(): + if not case_results: + continue + all_passed = all( + cr.final_eval_status == EvalStatus.PASSED and not cr.error_message + for cr in case_results + ) + if all_passed: + passed_count += 1 + + run_scores: dict[str, list[float]] = {} + for cr in case_results: + for mr in cr.overall_eval_metric_results: + score = mr.score if mr.score is not None else 0.0 + if mr.metric_name not in run_scores: + run_scores[mr.metric_name] = [] + run_scores[mr.metric_name].append(score) + + avg_scores: dict[str, float] = {} + for name, scores in run_scores.items(): + avg = sum(scores) / len(scores) + avg_scores[name] = avg + metric_sums[name] = metric_sums.get(name, 0.0) + avg + metric_counts[name] = metric_counts.get(name, 0) + 1 + + per_case[case_id] = PerCaseResult( + case_id=case_id, + passed=all_passed, + metric_scores=avg_scores, + ) + + total = len(per_case) + pass_rate = passed_count / total if total > 0 else 0.0 + + metric_breakdown: dict[str, float] = {} + for name in metric_sums: + if metric_counts[name] > 0: + metric_breakdown[name] = metric_sums[name] / metric_counts[name] + + return SplitResult( + pass_rate=pass_rate, + metric_breakdown=metric_breakdown, + per_case=per_case, + ) + + @property + def output_dir(self) -> str: + return self._config.output_dir + + def _collect_cost(self) -> float: + # Cost tracking from AgentOptimizer is not yet implemented. + # The gate's max_cost_usd rule is therefore dormant — users + # should monitor costs via their LLM provider dashboard. + return 0.0 diff --git a/examples/optimization/eval_optimize_loop/prompts/baseline_system.md b/examples/optimization/eval_optimize_loop/prompts/baseline_system.md new file mode 100644 index 00000000..1a626425 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/baseline_system.md @@ -0,0 +1 @@ +You are a math tutor that solves arithmetic word problems. Provide only the final numeric answer with units in the format: 答案:[number] [unit]. diff --git a/examples/optimization/eval_optimize_loop/prompts/optimized_system.md b/examples/optimization/eval_optimize_loop/prompts/optimized_system.md new file mode 100644 index 00000000..1d22db08 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/optimized_system.md @@ -0,0 +1 @@ +You are a math tutor. Your task is to solve arithmetic word problems accurately. Always answer in the exact format: 答案: . Do not include any explanation, just the answer. diff --git a/examples/optimization/eval_optimize_loop/reporting.py b/examples/optimization/eval_optimize_loop/reporting.py new file mode 100644 index 00000000..5d3f5de2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/reporting.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import json +import os + +from .models import PipelineResult + + +def _escape_md(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def write_reports(result: PipelineResult, output_dir: str) -> None: + os.makedirs(output_dir, exist_ok=True) + + # JSON report + json_path = os.path.join(output_dir, "optimization_report.json") + json_str = result.model_dump_json(indent=2, by_alias=True) + with open(json_path, "w", encoding="utf-8") as f: + f.write(json_str) + + # Markdown report + md_path = os.path.join(output_dir, "optimization_report.md") + lines = _build_markdown(result) + with open(md_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + + +def _build_markdown(result: PipelineResult) -> list[str]: + lines: list[str] = [] + + lines.append("# Optimization Report") + lines.append("") + + # Verdict + verdict_icon = "✓" if result.gate_decision == "ACCEPT" else "✗" + lines.append("## Verdict") + lines.append("") + lines.append(f"**{verdict_icon} {result.gate_decision}**") + lines.append("") + for reason in result.gate_reasons: + lines.append(f"- {reason}") + lines.append("") + + # Pass Rates + lines.append("## Pass Rates") + lines.append("") + lines.append("| Split | Baseline | Candidate | Delta |") + lines.append("|---|---|---|---|") + for split_name in ["train", "val"]: + base = result.baseline.get(split_name) + cand = result.candidate.get(split_name) + base_pr = base.pass_rate if base else 0.0 + cand_pr = cand.pass_rate if cand else 0.0 + delta_val = cand_pr - base_pr + lines.append(f"| {split_name} | {base_pr:.4f} | {cand_pr:.4f} | {delta_val:+.4f} |") + lines.append("") + + # Metric breakdown + if result.baseline: + base_ref = result.baseline.get("val") or result.baseline.get("train") + cand_ref = result.candidate.get("val") or result.candidate.get("train") + split_label = "val" if result.baseline.get("val") else "train" + if base_ref and base_ref.metric_breakdown: + lines.append(f"### Metric Breakdown ({split_label})") + lines.append("") + lines.append("| Metric | Baseline | Candidate | Delta |") + lines.append("|---|---|---|---|") + all_metrics = sorted(set(base_ref.metric_breakdown.keys()) | set(cand_ref.metric_breakdown.keys()) if cand_ref else set()) + for m in all_metrics: + b = base_ref.metric_breakdown.get(m, 0.0) + c = cand_ref.metric_breakdown.get(m, 0.0) if cand_ref else 0.0 + lines.append(f"| {_escape_md(m)} | {b:.4f} | {c:.4f} | {c-b:+.4f} |") + lines.append("") + + # Per-Case Delta + lines.append("## Per-Case Delta") + lines.append("") + for split_name in ["train", "val"]: + lines.append(f"### {split_name} set") + lines.append("") + lines.append("| Case ID | Baseline | Candidate | Status |") + lines.append("|---|---|---|---|") + base_sr = result.baseline.get(split_name) + cand_sr = result.candidate.get(split_name) + + if base_sr and cand_sr: + all_cases = sorted(set(base_sr.per_case.keys()) | set(cand_sr.per_case.keys())) + for case_id in all_cases: + base_passed = base_sr.per_case[case_id].passed if case_id in base_sr.per_case else False + cand_passed = cand_sr.per_case[case_id].passed if case_id in cand_sr.per_case else False + if not base_passed and cand_passed: + status = "newly passing" + elif base_passed and not cand_passed: + status = "newly failing" + elif base_passed and cand_passed: + status = "passed (both)" + else: + status = "failed (both)" + status_str = "PASS" if base_passed else "FAIL" + cand_status_str = "PASS" if cand_passed else "FAIL" + lines.append(f"| {_escape_md(case_id)} | {status_str} | {cand_status_str} | {status} |") + lines.append("") + + # Failure Attribution + lines.append("## Failure Attribution") + lines.append("") + fa = result.failure_attribution + lines.append(f"- Total cases: {fa.total_cases}") + lines.append(f"- Failed (baseline train): {fa.failed_cases}") + lines.append("") + if fa.categories: + lines.append("| Category | Count | Case IDs |") + lines.append("|---|---|---|") + for cat_name, cat in sorted(fa.categories.items()): + lines.append(f"| {_escape_md(cat_name)} | {cat.count} | {', '.join(cat.case_ids)} |") + else: + lines.append("No failures to attribute.") + lines.append("") + + # Gate Results + lines.append("## Gate Results") + lines.append("") + for reason in result.gate_reasons: + lines.append(f"- {reason}") + lines.append("") + + # Overfitting Check + lines.append("## Overfitting Check") + lines.append("") + if result.overfitting_warning: + lines.append("**WARNING: Possible overfitting detected.**") + lines.append("") + lines.append(f"- Train pass rate delta: {result.delta.train_pass_rate_delta:+.4f}") + lines.append(f"- Val pass rate delta: {result.delta.val_pass_rate_delta:+.4f}") + else: + lines.append("No overfitting detected.") + lines.append("") + + # Audit + lines.append("## Audit") + lines.append("") + lines.append("| Field | Value |") + lines.append("|---|---|") + lines.append(f"| Mode | {result.mode} |") + lines.append(f"| Duration | {result.duration_seconds:.2f}s |") + lines.append(f"| Cost | ${result.cost_usd:.4f} |") + lines.append(f"| Seed | {result.seed} |") + lines.append(f"| Started | {result.started_at} |") + lines.append(f"| Finished | {result.finished_at} |") + lines.append(f"| Schema version | {result.schema_version} |") + lines.append("") + + return lines diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..c3483321 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import asyncio +import sys +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +_REPO_ROOT = _HERE.parents[2] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +try: + from .pipeline import EvalOptimizePipeline # noqa: E402 +except ImportError: + from pipeline import EvalOptimizePipeline # type: ignore + + +async def main() -> None: + parser = argparse.ArgumentParser( + description="Eval + Optimize closed-loop pipeline", + ) + parser.add_argument( + "--pipeline-config", + type=str, + default=str(_HERE / "pipeline.json"), + help="Path to pipeline.json (default: pipeline.json in same directory)", + ) + parser.add_argument( + "--fail-on-reject", + action="store_true", + help="Exit with code 1 when the pipeline REJECTs the candidate (for CI gating). " + "Without this flag, REJECT is considered a valid pipeline outcome and exits 0.", + ) + args = parser.parse_args() + + pipeline = EvalOptimizePipeline.from_config(args.pipeline_config) + result = await pipeline.run() + + output_dir = Path(pipeline.output_dir).resolve() + print(f"\nPipeline complete: {result.gate_decision}") + print(f" Duration: {result.duration_seconds:.2f}s") + print(f" Reports: {output_dir}/") + print(f" optimization_report.json") + print(f" optimization_report.md") + + if args.fail_on_reject and result.gate_decision == "REJECT": + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/optimization/eval_optimize_loop/tests/__init__.py b/examples/optimization/eval_optimize_loop/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/optimization/eval_optimize_loop/tests/test_delta.py b/examples/optimization/eval_optimize_loop/tests/test_delta.py new file mode 100644 index 00000000..cba4749c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_delta.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import pytest + +from ..delta import compute_delta +from ..models import PerCaseResult, SplitResult, SplitDelta, PerCaseDelta + + +def test_compute_delta_all_passing(): + baseline = { + "train": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + "c2": PerCaseResult(case_id="c2", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + "c2": PerCaseResult(case_id="c2", passed=True, metric_scores={"m1": 1.0}), + }, + ), + } + candidate = { + "train": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + "c2": PerCaseResult(case_id="c2", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + "c2": PerCaseResult(case_id="c2", passed=True, metric_scores={"m1": 1.0}), + }, + ), + } + delta = compute_delta(baseline, candidate) + assert isinstance(delta, SplitDelta) + assert delta.train_pass_rate_delta == 0.0 + assert delta.val_pass_rate_delta == 0.0 + assert len(delta.train.newly_passing) == 0 + assert len(delta.train.newly_failing) == 0 + assert len(delta.val.unchanged) == 2 + + +def test_compute_delta_improvement(): + baseline = { + "train": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + "val": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + } + candidate = { + "train": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + } + delta = compute_delta(baseline, candidate) + assert delta.train_pass_rate_delta == 1.0 + assert delta.val_pass_rate_delta == 1.0 + assert delta.train.newly_passing == ["c1"] + assert delta.train.newly_failing == [] + assert delta.val.newly_passing == ["c1"] + + +def test_compute_delta_regression(): + baseline = { + "train": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + } + candidate = { + "train": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + "val": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + } + delta = compute_delta(baseline, candidate) + assert delta.train_pass_rate_delta == -1.0 + assert delta.train.newly_failing == ["c1"] + assert delta.val.newly_failing == ["c1"] + + +def test_compute_delta_mixed(): + """Train improves but val degrades (overfitting scenario).""" + baseline = { + "train": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + } + candidate = { + "train": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=0.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.0}), + }, + ), + } + delta = compute_delta(baseline, candidate) + assert delta.train_pass_rate_delta == 1.0 + assert delta.val_pass_rate_delta == -1.0 + assert delta.train.newly_passing == ["c1"] + assert delta.val.newly_failing == ["c1"] + + +def test_compute_delta_score_deltas(): + baseline = { + "train": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 0.8, "m2": 0.9}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 0.8, "m2": 0.9}), + }, + ), + } + candidate = { + "train": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 0.9, "m2": 0.7}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 0.9, "m2": 0.7}), + }, + ), + } + delta = compute_delta(baseline, candidate) + assert delta.train.score_deltas["c1"]["m1"] == pytest.approx(0.1) + assert delta.train.score_deltas["c1"]["m2"] == pytest.approx(-0.2) + assert delta.val.score_deltas["c1"]["m1"] == pytest.approx(0.1) + assert delta.val.score_deltas["c1"]["m2"] == pytest.approx(-0.2) diff --git a/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py new file mode 100644 index 00000000..67889bdb --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_failure_attribution.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +from trpc_agent_sdk.evaluation import ( + EvalCaseResult, + EvalMetricResult, + EvalMetricResultPerInvocation, + EvalStatus, + Invocation, +) +from trpc_agent_sdk.types import Content, Part + +from ..failure_attribution import attribute_failures +from ..models import FailureAttribution + + +def _make_invocation(text: str, invocation_id: str = "t1") -> Invocation: + return Invocation( + invocation_id=invocation_id, + user_content=Content(parts=[Part.from_text(text="query")], role="user"), + final_response=Content(parts=[Part.from_text(text=text)], role="model"), + ) + + +def _make_metric_result(metric_name: str, score: float, threshold: float) -> EvalMetricResult: + status = EvalStatus.PASSED if score >= threshold else EvalStatus.FAILED + return EvalMetricResult(metric_name=metric_name, score=score, threshold=threshold, eval_status=status) + + +def _make_case_result( + case_id: str, + metric_results: list[EvalMetricResult], + actual_text: str = "", + expected_text: str = "", +) -> EvalCaseResult: + actual_inv = _make_invocation(actual_text) + expected_inv = _make_invocation(expected_text) + return EvalCaseResult( + eval_set_id="test_set", + eval_id=case_id, + final_eval_status=EvalStatus.FAILED if any(m.eval_status == EvalStatus.FAILED for m in metric_results) else EvalStatus.PASSED, + overall_eval_metric_results=metric_results, + eval_metric_result_per_invocation=[ + EvalMetricResultPerInvocation( + actual_invocation=actual_inv, + expected_invocation=expected_inv, + eval_metric_results=metric_results, + ) + ], + session_id="session1", + ) + + +def test_attribute_failures_all_pass(): + """All cases pass -- empty attribution.""" + results = { + "case_a": [ + _make_case_result("case_a", [_make_metric_result("final_response_avg_score", 1.0, 1.0)]) + ], + "case_b": [ + _make_case_result("case_b", [_make_metric_result("final_response_avg_score", 1.0, 1.0)]) + ], + } + attr = attribute_failures(results) + assert isinstance(attr, FailureAttribution) + assert attr.total_cases == 2 + assert attr.failed_cases == 0 + assert len(attr.categories) == 0 + + +def test_attribute_failures_final_response_mismatch(): + """Final response mismatch detected.""" + results = { + "case_a": [ + _make_case_result( + "case_a", + [_make_metric_result("final_response_avg_score", 0.0, 1.0)], + actual_text="wrong answer", + expected_text="expected answer", + ) + ], + } + attr = attribute_failures(results) + assert attr.failed_cases == 1 + assert "final_response_mismatch" in attr.categories + assert attr.categories["final_response_mismatch"].count == 1 + assert "case_a" in attr.categories["final_response_mismatch"].case_ids + + +def test_attribute_failures_multiple_categories(): + """Case failing on multiple metrics gets attributed to all categories.""" + results = { + "case_x": [ + _make_case_result( + "case_x", + [ + _make_metric_result("final_response_avg_score", 0.0, 1.0), + _make_metric_result("tool_trajectory_avg_score", 0.0, 1.0), + ], + ) + ], + } + attr = attribute_failures(results) + assert attr.failed_cases == 1 + assert "final_response_mismatch" in attr.categories + assert "tool_trajectory_mismatch" in attr.categories + + +def test_attribute_failures_format_violation(): + """Detect format violation when response is missing required prefix.""" + results = { + "case_z": [ + _make_case_result( + "case_z", + [_make_metric_result("final_response_avg_score", 0.0, 1.0)], + actual_text="the result is 42", + expected_text="答案:42", + ) + ], + } + attr = attribute_failures(results) + assert "format_violation" in attr.categories + assert attr.categories["format_violation"].count == 1 + + +def test_attribute_failures_unknown_metric(): + """Unknown metric name falls into unknown_metric_failure.""" + results = { + "case_u": [ + _make_case_result("case_u", [_make_metric_result("custom_custom_metric", 0.0, 0.8)]) + ], + } + attr = attribute_failures(results) + assert attr.failed_cases == 1 + assert "unknown_metric_failure" in attr.categories + + +def test_attribute_failures_empty_case_results(): + """Case with empty list of results is skipped.""" + results = { + "case_empty": [], + "case_b": [ + _make_case_result("case_b", [_make_metric_result("final_response_avg_score", 0.0, 1.0)]) + ], + } + attr = attribute_failures(results) + assert attr.total_cases == 1 + assert attr.failed_cases == 1 + assert "final_response_mismatch" in attr.categories + assert "case_b" in attr.categories["final_response_mismatch"].case_ids + + +def test_attribute_failures_mixed_metrics(): + """Only failed metrics are attributed; passed ones are skipped.""" + results = { + "case_m": [ + _make_case_result( + "case_m", + [ + _make_metric_result("final_response_avg_score", 0.0, 1.0), + _make_metric_result("tool_trajectory_avg_score", 1.0, 1.0), + ], + ) + ], + } + attr = attribute_failures(results) + assert attr.failed_cases == 1 + assert "final_response_mismatch" in attr.categories + assert "tool_trajectory_mismatch" not in attr.categories diff --git a/examples/optimization/eval_optimize_loop/tests/test_gate.py b/examples/optimization/eval_optimize_loop/tests/test_gate.py new file mode 100644 index 00000000..0e042cc6 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_gate.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from ..gate import apply_gate +from ..models import ( + PerCaseDelta, + PerCaseResult, + SplitDelta, + SplitResult, + GateConfig, + GateDecision, +) + + +def _make_delta( + train_pass_rate_delta: float = 0.0, + val_pass_rate_delta: float = 0.0, + train_newly_passing: list[str] | None = None, + train_newly_failing: list[str] | None = None, + val_newly_passing: list[str] | None = None, + val_newly_failing: list[str] | None = None, +) -> SplitDelta: + return SplitDelta( + train_pass_rate_delta=train_pass_rate_delta, + val_pass_rate_delta=val_pass_rate_delta, + train=PerCaseDelta( + newly_passing=train_newly_passing or [], + newly_failing=train_newly_failing or [], + unchanged=[], + ), + val=PerCaseDelta( + newly_passing=val_newly_passing or [], + newly_failing=val_newly_failing or [], + unchanged=[], + ), + ) + + +def test_gate_accept_no_new_fails_improvement(): + """Val improved, no new fails → ACCEPT.""" + delta = _make_delta(val_pass_rate_delta=0.33) + gate = GateConfig(min_improvement=0.0, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "ACCEPT" + assert not decision.overfitting_warning + + +def test_gate_reject_below_min_improvement(): + """Val improvement below threshold → REJECT.""" + delta = _make_delta(val_pass_rate_delta=0.05) + gate = GateConfig(min_improvement=0.1, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "REJECT" + assert any("min_improvement" in r.lower() for r in decision.reasons) + + +def test_gate_reject_new_fails(): + """New failures in val, allow_new_fails=False → REJECT.""" + delta = _make_delta(val_pass_rate_delta=0.33, val_newly_failing=["case_1"]) + gate = GateConfig(min_improvement=0.0, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "REJECT" + assert any("newly failing" in r.lower() for r in decision.reasons) + + +def test_gate_accept_new_fails_allowed(): + """New failures in val, allow_new_fails=True → ACCEPT (if min_improvement met).""" + delta = _make_delta(val_pass_rate_delta=0.5, val_newly_failing=["case_1"]) + gate = GateConfig(min_improvement=0.0, allow_new_fails=True) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "ACCEPT" + + +def test_gate_reject_cost_over_budget(): + """Cost exceeds max_cost_usd → REJECT.""" + delta = _make_delta(val_pass_rate_delta=0.5) + gate = GateConfig(min_improvement=0.0, max_cost_usd=1.0) + decision = apply_gate(delta, gate, cost_usd=5.0, duration_seconds=10.0) + assert decision.decision == "REJECT" + assert any("cost" in r.lower() for r in decision.reasons) + + +def test_gate_reject_duration_over_budget(): + """Duration exceeds max_duration_seconds → REJECT.""" + delta = _make_delta(val_pass_rate_delta=0.5) + gate = GateConfig(min_improvement=0.0, max_duration_seconds=60) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=120.0) + assert decision.decision == "REJECT" + assert any("duration" in r.lower() for r in decision.reasons) + + +def test_gate_overfitting_warning(): + """Train improves but val degrades → overfitting_warning=True.""" + delta = _make_delta(train_pass_rate_delta=0.5, val_pass_rate_delta=-0.1) + gate = GateConfig(min_improvement=0.0, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.overfitting_warning is True + assert any("overfitting" in r.lower() for r in decision.reasons) + + +def test_gate_no_overfitting_when_val_flat(): + """Train improves but val is flat (0.0) -> no overfitting warning.""" + delta = _make_delta(train_pass_rate_delta=0.5, val_pass_rate_delta=0.0) + gate = GateConfig(min_improvement=0.0, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.overfitting_warning is False + + +def test_gate_reject_overfitting_with_degradation(): + """Train improves but val degrades → overfitting_warning + REJECT (below min_improvement).""" + delta = _make_delta(train_pass_rate_delta=0.5, val_pass_rate_delta=-0.2) + gate = GateConfig(min_improvement=0.0, allow_new_fails=False) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.overfitting_warning is True + assert decision.decision == "REJECT" + + +def test_gate_protected_case_degradation(): + """Protected case score drops → REJECT.""" + delta = _make_delta(val_pass_rate_delta=0.33) + delta.val.score_deltas = {"case_key": {"m1": -0.2}} + gate = GateConfig(min_improvement=0.0, protected_case_ids=["case_key"]) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "REJECT" + assert any("protected" in r.lower() for r in decision.reasons) + + +def test_gate_protected_case_no_degradation(): + """Protected case score same or better → ok.""" + delta = _make_delta(val_pass_rate_delta=0.33) + delta.val.score_deltas = {"case_key": {"m1": 0.1}} + gate = GateConfig(min_improvement=0.0, protected_case_ids=["case_key"]) + decision = apply_gate(delta, gate, cost_usd=0.0, duration_seconds=10.0) + assert decision.decision == "ACCEPT" + + +def test_gate_all_rules_pass(): + """All rules pass → ACCEPT with all reasons.""" + delta = _make_delta(val_pass_rate_delta=0.33) + delta.train_pass_rate_delta = 0.0 # No overfitting + gate = GateConfig(min_improvement=0.0, allow_new_fails=False, max_cost_usd=10.0, max_duration_seconds=300) + decision = apply_gate(delta, gate, cost_usd=1.0, duration_seconds=30.0) + assert decision.decision == "ACCEPT" + assert len(decision.reasons) >= 4 # one per rule that passed diff --git a/examples/optimization/eval_optimize_loop/tests/test_integration.py b/examples/optimization/eval_optimize_loop/tests/test_integration.py new file mode 100644 index 00000000..f7c1fc65 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_integration.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + +import pytest + +_HERE = Path(__file__).resolve().parent +_EXAMPLE_ROOT = _HERE.parent + +from ..pipeline import EvalOptimizePipeline # noqa: E402 + + +@pytest.mark.asyncio +async def test_trace_mode_accept(): + pipeline_json = { + "mode": "trace", + "baseline_prompt_path": str(_EXAMPLE_ROOT / "prompts" / "baseline_system.md"), + "candidate_prompt_path": str(_EXAMPLE_ROOT / "prompts" / "optimized_system.md"), + "train_baseline_evalset": str(_EXAMPLE_ROOT / "evalsets" / "train_baseline.evalset.json"), + "train_candidate_evalset": str(_EXAMPLE_ROOT / "evalsets" / "train_candidate.evalset.json"), + "val_baseline_evalset": str(_EXAMPLE_ROOT / "evalsets" / "val_baseline.evalset.json"), + "val_candidate_evalset": str(_EXAMPLE_ROOT / "evalsets" / "val_candidate.evalset.json"), + "output_dir": str(tempfile.mkdtemp()), + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains", "case_insensitive": True}}}, + } + ], + "num_runs": 1, + }, + "gate": { + "min_improvement": 0.0, + "allow_new_fails": True, + }, + "seed": 42, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(pipeline_json, f) + config_path = f.name + + try: + pipeline = EvalOptimizePipeline.from_config(config_path) + result = await pipeline.run() + + assert result.mode == "trace" + assert result.gate_decision == "ACCEPT" + + output_dir = pipeline_json["output_dir"] + assert os.path.isfile(os.path.join(output_dir, "optimization_report.json")) + assert os.path.isfile(os.path.join(output_dir, "optimization_report.md")) + + baseline_train = result.baseline["train"] + candidate_train = result.candidate["train"] + assert baseline_train.pass_rate == pytest.approx(0.333, abs=0.01) + assert candidate_train.pass_rate == pytest.approx(0.333, abs=0.01) + + baseline_val = result.baseline["val"] + candidate_val = result.candidate["val"] + assert baseline_val.pass_rate == pytest.approx(0.333, abs=0.01) + assert candidate_val.pass_rate == pytest.approx(0.333, abs=0.01) + + assert "case_train_optimizable" in result.delta.train.newly_passing + assert "case_train_regression" in result.delta.train.newly_failing + assert "case_val_improves" in result.delta.val.newly_passing + assert "case_val_regression" in result.delta.val.newly_failing + + assert result.failure_attribution.failed_cases >= 1 + assert len(result.failure_attribution.categories) >= 1 + + finally: + os.unlink(config_path) + + +@pytest.mark.asyncio +async def test_trace_mode_reject(): + pipeline_json = { + "mode": "trace", + "baseline_prompt_path": str(_EXAMPLE_ROOT / "prompts" / "baseline_system.md"), + "candidate_prompt_path": str(_EXAMPLE_ROOT / "prompts" / "optimized_system.md"), + "train_baseline_evalset": str(_EXAMPLE_ROOT / "evalsets" / "train_baseline.evalset.json"), + "train_candidate_evalset": str(_EXAMPLE_ROOT / "evalsets" / "train_candidate.evalset.json"), + "val_baseline_evalset": str(_EXAMPLE_ROOT / "evalsets" / "val_baseline.evalset.json"), + "val_candidate_evalset": str(_EXAMPLE_ROOT / "evalsets" / "val_candidate.evalset.json"), + "output_dir": str(tempfile.mkdtemp()), + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains", "case_insensitive": True}}}, + } + ], + "num_runs": 1, + }, + "gate": { + "min_improvement": 0.0, + "allow_new_fails": False, + }, + "seed": 42, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(pipeline_json, f) + config_path = f.name + + try: + pipeline = EvalOptimizePipeline.from_config(config_path) + result = await pipeline.run() + + assert result.mode == "trace" + assert result.gate_decision == "REJECT" + assert any("newly failing" in r.lower() for r in result.gate_reasons) + + output_dir = pipeline_json["output_dir"] + assert os.path.isfile(os.path.join(output_dir, "optimization_report.json")) + assert os.path.isfile(os.path.join(output_dir, "optimization_report.md")) + + finally: + os.unlink(config_path) + + +@pytest.mark.asyncio +async def test_trace_mode_overfitting(): + with tempfile.TemporaryDirectory() as tmpdir: + train_base_path = os.path.join(tmpdir, "train_base.json") + train_cand_path = os.path.join(tmpdir, "train_cand.json") + val_base_path = os.path.join(tmpdir, "val_base.json") + val_cand_path = os.path.join(tmpdir, "val_cand.json") + + base_evalset = lambda eid, actual: { + "eval_set_id": eid, + "eval_cases": [ + { + "eval_id": "case_1", + "eval_mode": "trace", + "conversation": [{ + "invocation_id": "t1", + "user_content": {"parts": [{"text": "q"}], "role": "user"}, + "final_response": {"parts": [{"text": "答案:42"}], "role": "model"}, + }], + "actual_conversation": [{ + "invocation_id": "t1", + "user_content": {"parts": [{"text": "q"}], "role": "user"}, + "final_response": {"parts": [{"text": actual}], "role": "model"}, + }], + "session_input": {"app_name": "test", "user_id": "u", "state": {}}, + } + ] + } + + with open(train_base_path, "w") as f: + json.dump(base_evalset("train_base", "答案:99"), f) + with open(train_cand_path, "w") as f: + json.dump(base_evalset("train_cand", "答案:42"), f) + with open(val_base_path, "w") as f: + json.dump(base_evalset("val_base", "答案:42"), f) + with open(val_cand_path, "w") as f: + json.dump(base_evalset("val_cand", "答案:99"), f) + + pipeline_json = { + "mode": "trace", + "train_baseline_evalset": train_base_path, + "train_candidate_evalset": train_cand_path, + "val_baseline_evalset": val_base_path, + "val_candidate_evalset": val_cand_path, + "output_dir": tmpdir, + "evaluate": { + "metrics": [{ + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains", "case_insensitive": True}}}, + }], + "num_runs": 1, + }, + "gate": {"min_improvement": -1.0, "allow_new_fails": True}, + "seed": 42, + } + + config_path = os.path.join(tmpdir, "config.json") + with open(config_path, "w") as f: + json.dump(pipeline_json, f) + + pipeline = EvalOptimizePipeline.from_config(config_path) + result = await pipeline.run() + + assert result.overfitting_warning is True + assert result.delta.train_pass_rate_delta > 0 + assert result.delta.val_pass_rate_delta < 0 + assert result.gate_decision == "ACCEPT" diff --git a/examples/optimization/eval_optimize_loop/tests/test_models.py b/examples/optimization/eval_optimize_loop/tests/test_models.py new file mode 100644 index 00000000..e029773d --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_models.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from ..models import GateConfig, GateDecision, PipelineConfig, PipelineResult + + +def test_gate_config_defaults(): + gc = GateConfig() + assert gc.min_improvement == 0.0 + assert gc.allow_new_fails is False + assert gc.protected_case_ids == [] + assert gc.max_cost_usd is None + assert gc.max_duration_seconds == 180 + + +def test_gate_decision_accept(): + gd = GateDecision(decision="ACCEPT", reasons=["val improved by 0.33"]) + assert gd.decision == "ACCEPT" + assert len(gd.reasons) == 1 + assert gd.overfitting_warning is False + + +def test_gate_decision_reject(): + gd = GateDecision(decision="REJECT", reasons=["new failures in val"], overfitting_warning=True) + assert gd.decision == "REJECT" + assert gd.overfitting_warning is True + + +def test_pipeline_config_trace_mode(): + data = { + "mode": "trace", + "baseline_prompt_path": "prompts/baseline.md", + "candidate_prompt_path": "prompts/optimized.md", + "train_baseline_evalset": "evalsets/train_base.json", + "train_candidate_evalset": "evalsets/train_cand.json", + "val_baseline_evalset": "evalsets/val_base.json", + "val_candidate_evalset": "evalsets/val_cand.json", + "output_dir": "outputs", + "evaluate": {"metrics": [{"metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {"final_response": {"text": {"match": "contains"}}}}], "num_runs": 1}, + "gate": {"min_improvement": 0.1, "allow_new_fails": False}, + "seed": 42, + } + config = PipelineConfig.model_validate(data) + assert config.mode == "trace" + assert config.baseline_prompt_path == "prompts/baseline.md" + assert config.gate.min_improvement == 0.1 + + +def test_pipeline_config_live_mode(): + data = { + "mode": "live", + "live_train_evalset": "evalsets/live_train.json", + "live_val_evalset": "evalsets/live_val.json", + "optimizer_config_path": "optimizer.json", + "target_prompt_name": "system_prompt", + "output_dir": "outputs", + "evaluate": {"metrics": [{"metric_name": "final_response_avg_score", "threshold": 1.0, "criterion": {"final_response": {"text": {"match": "contains"}}}}], "num_runs": 1}, + "seed": 42, + } + config = PipelineConfig.model_validate(data) + assert config.mode == "live" + assert config.optimizer_config_path == "optimizer.json" + assert config.target_prompt_name == "system_prompt" + + +def test_pipeline_result_roundtrip(): + result = PipelineResult( + mode="trace", + gate_decision="ACCEPT", + gate_reasons=["val pass rate improved from 0.33 to 1.00"], + duration_seconds=2.5, + seed=42, + started_at="2026-01-01T00:00:00", + finished_at="2026-01-01T00:00:05", + ) + # Verify direct read-back works + loaded = PipelineResult.model_validate_json(result.model_dump_json()) + assert loaded.mode == "trace" + assert loaded.gate_decision == "ACCEPT" + assert loaded.seed == 42 diff --git a/examples/optimization/eval_optimize_loop/tests/test_pipeline.py b/examples/optimization/eval_optimize_loop/tests/test_pipeline.py new file mode 100644 index 00000000..7135e256 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_pipeline.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +_HERE = Path(__file__).resolve().parent +_EXAMPLE_ROOT = _HERE.parent +if str(_EXAMPLE_ROOT.parents[1]) not in sys.path: + sys.path.insert(0, str(_EXAMPLE_ROOT.parents[1])) + +from ..models import PipelineConfig +from ..pipeline import EvalOptimizePipeline + + +def test_pipeline_from_config_trace_mode(): + """Pipeline loads trace mode config correctly.""" + config_data = { + "mode": "trace", + "baseline_prompt_path": "/tmp/baseline.md", + "candidate_prompt_path": "/tmp/candidate.md", + "train_baseline_evalset": "/tmp/train_base.json", + "train_candidate_evalset": "/tmp/train_cand.json", + "val_baseline_evalset": "/tmp/val_base.json", + "val_candidate_evalset": "/tmp/val_cand.json", + "output_dir": "/tmp/outputs", + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains"}}}, + } + ], + "num_runs": 1, + }, + "seed": 42, + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + import json + + json.dump(config_data, f) + config_path = f.name + + try: + pipeline = EvalOptimizePipeline.from_config(config_path) + assert pipeline._config.mode == "trace" + assert pipeline._config.baseline_prompt_path == "/tmp/baseline.md" + assert pipeline._config.seed == 42 + finally: + os.unlink(config_path) + + +def test_pipeline_from_config_live_mode(): + """Pipeline loads live mode config correctly.""" + import json + + train_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + val_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + opt_f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) + train_f.close() + val_f.close() + opt_f.close() + + config_data = { + "mode": "live", + "live_train_evalset": train_f.name, + "live_val_evalset": val_f.name, + "optimizer_config_path": opt_f.name, + "target_prompt_name": "system_prompt", + "output_dir": tempfile.mkdtemp(), + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains"}}}, + } + ], + "num_runs": 1, + }, + "seed": 42, + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump(config_data, f) + config_path = f.name + + try: + pipeline = EvalOptimizePipeline.from_config( + config_path, call_agent=AsyncMock(), target_prompt=AsyncMock() + ) + assert pipeline._config.mode == "live" + assert pipeline._live_call_agent is not None + finally: + os.unlink(config_path) + os.unlink(train_f.name) + os.unlink(val_f.name) + os.unlink(opt_f.name) + + +def test_pipeline_live_mode_missing_params(): + """Live mode requires call_agent and target_prompt.""" + config_data = { + "mode": "live", + "live_train_evalset": "/tmp/train.json", + "live_val_evalset": "/tmp/val.json", + "output_dir": "/tmp/outputs", + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains"}}}, + } + ], + "num_runs": 1, + }, + } + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + import json + + json.dump(config_data, f) + config_path = f.name + + try: + with pytest.raises(ValueError, match="call_agent"): + EvalOptimizePipeline.from_config(config_path) + finally: + os.unlink(config_path) + + +@pytest.mark.asyncio +async def test_pipeline_build_split_result(): + """_build_split_result extracts pass rates from EvaluateResult.""" + from trpc_agent_sdk.evaluation import ( + EvalCaseResult, + EvalMetricResult, + EvalStatus, + EvaluateResult, + EvalSetAggregateResult, + ) + + eval_results_by_eval_id = { + "case_a": [ + EvalCaseResult( + eval_set_id="test", + eval_id="case_a", + final_eval_status=EvalStatus.PASSED, + overall_eval_metric_results=[ + EvalMetricResult( + metric_name="m1", + score=1.0, + threshold=1.0, + eval_status=EvalStatus.PASSED, + ), + ], + eval_metric_result_per_invocation=[], + session_id="s1", + ) + ], + "case_b": [ + EvalCaseResult( + eval_set_id="test", + eval_id="case_b", + final_eval_status=EvalStatus.FAILED, + overall_eval_metric_results=[ + EvalMetricResult( + metric_name="m1", + score=0.0, + threshold=1.0, + eval_status=EvalStatus.FAILED, + ), + ], + eval_metric_result_per_invocation=[], + session_id="s1", + ) + ], + } + + result = EvaluateResult( + results_by_eval_set_id={ + "test": EvalSetAggregateResult( + eval_results_by_eval_id=eval_results_by_eval_id, + num_runs=1, + ) + } + ) + + pipeline = EvalOptimizePipeline.__new__(EvalOptimizePipeline) + pipeline._config = PipelineConfig.model_validate( + { + "mode": "trace", + "output_dir": "/tmp", + "evaluate": { + "metrics": [ + { + "metric_name": "m1", + "threshold": 1.0, + "criterion": { + "final_response": {"text": {"match": "contains"}} + }, + } + ] + }, + } + ) + + sr = pipeline._build_split_result(result) + assert sr.pass_rate == 0.5 + assert "case_a" in sr.per_case + assert sr.per_case["case_a"].passed is True + assert sr.per_case["case_b"].passed is False + assert "m1" in sr.metric_breakdown + + +# ── Helpers ────────────────────────────────────────────────────── + + +def _make_fake_eval_result( + eval_set_id: str, + case_ids: list[str], + passed: list[bool], + metric_name: str = "m1", +) -> "EvaluateResult": + from trpc_agent_sdk.evaluation import ( + EvalCaseResult, + EvalMetricResult, + EvalStatus, + EvaluateResult, + EvalSetAggregateResult, + ) + + eval_results_by_eval_id: dict[str, list[EvalCaseResult]] = {} + for case_id, is_pass in zip(case_ids, passed): + status = EvalStatus.PASSED if is_pass else EvalStatus.FAILED + score = 1.0 if is_pass else 0.0 + eval_results_by_eval_id[case_id] = [ + EvalCaseResult( + eval_set_id=eval_set_id, + eval_id=case_id, + final_eval_status=status, + overall_eval_metric_results=[ + EvalMetricResult( + metric_name=metric_name, + score=score, + threshold=1.0, + eval_status=status, + ), + ], + eval_metric_result_per_invocation=[], + session_id=f"s_{case_id}", + ) + ] + + return EvaluateResult( + results_by_eval_set_id={ + eval_set_id: EvalSetAggregateResult( + eval_results_by_eval_id=eval_results_by_eval_id, + num_runs=1, + ) + } + ) + + +def _make_pipeline(config_overrides: dict | None = None) -> EvalOptimizePipeline: + base = { + "mode": "trace", + "output_dir": "/tmp/fake_outputs", + "evaluate": { + "metrics": [ + { + "metric_name": "m1", + "threshold": 1.0, + "criterion": {"final_response": {"text": {"match": "contains"}}}, + } + ], + "num_runs": 1, + }, + "train_baseline_evalset": "/tmp/train_base.json", + "val_baseline_evalset": "/tmp/val_base.json", + "train_candidate_evalset": "/tmp/train_cand.json", + "val_candidate_evalset": "/tmp/val_cand.json", + "seed": 42, + } + if config_overrides: + base.update(config_overrides) + + pipeline = EvalOptimizePipeline.__new__(EvalOptimizePipeline) + pipeline._config = PipelineConfig.model_validate(base) + pipeline._live_call_agent = None + pipeline._live_target_prompt = None + pipeline._optimizer_call = None + return pipeline + + +# ── New high-priority tests ───────────────────────────────────── + + +@pytest.mark.asyncio +async def test_write_eval_config_temp_creates_and_returns_path(): + pipeline = _make_pipeline() + path = await pipeline._write_eval_config_temp() + try: + assert os.path.isfile(path) + import json + with open(path) as f: + data = json.load(f) + assert "metrics" in data + assert data["numRuns"] == 1 + finally: + os.unlink(path) + + +@pytest.mark.asyncio +async def test_run_optimization_calls_injected_hook(): + pipeline = _make_pipeline({"mode": "live"}) + hook = AsyncMock() + pipeline._optimizer_call = hook + await pipeline._run_optimization() + hook.assert_called_once_with(pipeline) + + +@pytest.mark.asyncio +async def test_run_trace_mode_orchestration(): + pipeline = _make_pipeline() + + fake_train_base = _make_fake_eval_result("train_base", ["a", "b"], [False, False]) + fake_train_cand = _make_fake_eval_result("train_cand", ["a", "b"], [True, False]) + fake_val_base = _make_fake_eval_result("val_base", ["c", "d"], [True, True]) + fake_val_cand = _make_fake_eval_result("val_cand", ["c", "d"], [True, False]) + + async def _fake_run_eval(_path: str): + if "train_base" in _path: + return fake_train_base + if "train_cand" in _path: + return fake_train_cand + if "val_base" in _path: + return fake_val_base + if "val_cand" in _path: + return fake_val_cand + raise RuntimeError(f"unexpected path: {_path}") + + with patch.object(pipeline, "_run_eval", side_effect=_fake_run_eval): + with patch("examples.optimization.eval_optimize_loop.pipeline.write_reports"): + result = await pipeline.run() + + assert result.mode == "trace" + assert result.seed == 42 + assert "train" in result.baseline + assert "val" in result.baseline + assert result.baseline["train"].pass_rate == 0.0 # both fail + assert result.baseline["val"].pass_rate == 1.0 # both pass + assert result.candidate["train"].pass_rate == 0.5 # one passes now + assert result.candidate["val"].pass_rate == 0.5 # one regressed + # Delta: train has newly_passing, val has newly_failing + assert "a" in result.delta.train.newly_passing + assert "d" in result.delta.val.newly_failing + + +@pytest.mark.asyncio +async def test_run_optimization_calls_agent_optimizer_when_no_hook(): + pipeline = _make_pipeline( + { + "mode": "live", + "optimizer_config_path": "/tmp/opt.json", + "live_train_evalset": "/tmp/train.json", + "live_val_evalset": "/tmp/val.json", + } + ) + with patch( + "examples.optimization.eval_optimize_loop.pipeline.AgentOptimizer.optimize", + new_callable=AsyncMock, + ) as mock_optimize: + await pipeline._run_optimization() + + mock_optimize.assert_called_once() + call_kwargs = mock_optimize.call_args.kwargs + assert call_kwargs["config_path"] == "/tmp/opt.json" + assert call_kwargs["train_dataset_path"] == "/tmp/train.json" diff --git a/examples/optimization/eval_optimize_loop/tests/test_reporting.py b/examples/optimization/eval_optimize_loop/tests/test_reporting.py new file mode 100644 index 00000000..6d586f12 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/tests/test_reporting.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import json +import os +import tempfile + +from ..reporting import write_reports +from ..models import ( + PipelineResult, + SplitResult, + SplitDelta, + PerCaseDelta, + PerCaseResult, + FailureAttribution, + FailureCategory, +) + + +def test_write_reports_json(): + result = PipelineResult( + mode="trace", + gate_decision="ACCEPT", + gate_reasons=["val improved"], + baseline={ + "train": SplitResult( + pass_rate=0.0, + metric_breakdown={"m1": 0.5}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=False, metric_scores={"m1": 0.5}), + }, + ), + "val": SplitResult( + pass_rate=0.0, + metric_breakdown={"m1": 0.5}, + per_case={ + "c2": PerCaseResult(case_id="c2", passed=False, metric_scores={"m1": 0.5}), + }, + ), + }, + candidate={ + "train": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c1": PerCaseResult(case_id="c1", passed=True, metric_scores={"m1": 1.0}), + }, + ), + "val": SplitResult( + pass_rate=1.0, + metric_breakdown={"m1": 1.0}, + per_case={ + "c2": PerCaseResult(case_id="c2", passed=True, metric_scores={"m1": 1.0}), + }, + ), + }, + delta=SplitDelta( + train=PerCaseDelta(newly_passing=["c1"], newly_failing=[], score_deltas={"c1": {"m1": 0.5}}, unchanged=[]), + val=PerCaseDelta(newly_passing=["c2"], newly_failing=[], score_deltas={"c2": {"m1": 0.5}}, unchanged=[]), + train_pass_rate_delta=1.0, + val_pass_rate_delta=1.0, + ), + failure_attribution=FailureAttribution( + total_cases=3, + failed_cases=3, + categories={ + "final_response_mismatch": FailureCategory(count=3, case_ids=["c1", "c2", "c3"]), + }, + ), + duration_seconds=1.5, + seed=42, + started_at="2026-01-01T00:00:00", + finished_at="2026-01-01T00:00:02", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + write_reports(result, tmpdir) + + json_path = os.path.join(tmpdir, "optimization_report.json") + md_path = os.path.join(tmpdir, "optimization_report.md") + + assert os.path.isfile(json_path), "optimization_report.json not created" + assert os.path.isfile(md_path), "optimization_report.md not created" + + # Verify JSON structure + with open(json_path) as f: + loaded = json.load(f) + assert loaded["schemaVersion"] == "v1" + assert loaded["gateDecision"] == "ACCEPT" + assert loaded["mode"] == "trace" + assert loaded["overfittingWarning"] is False + + # Verify MD contains key sections + with open(md_path) as f: + md_content = f.read() + assert "# Optimization Report" in md_content + assert "## Verdict" in md_content + assert "ACCEPT" in md_content + assert "## Pass Rates" in md_content + assert "## Per-Case Delta" in md_content + assert "## Failure Attribution" in md_content + assert "## Gate Results" in md_content + assert "## Overfitting Check" in md_content + assert "## Audit" in md_content + + +def test_write_reports_reject(): + result = PipelineResult( + mode="trace", + gate_decision="REJECT", + gate_reasons=["new_fails: val has newly failing cases"], + overfitting_warning=True, + duration_seconds=1.0, + seed=42, + started_at="2026-01-01T00:00:00", + finished_at="2026-01-01T00:00:01", + ) + with tempfile.TemporaryDirectory() as tmpdir: + write_reports(result, tmpdir) + md_path = os.path.join(tmpdir, "optimization_report.md") + with open(md_path) as f: + md_content = f.read() + assert "REJECT" in md_content + assert "overfitting" in md_content.lower() + assert "newly failing" in md_content.lower() + + +def test_write_reports_per_case_delta_all_transitions(): + result = PipelineResult( + mode="trace", + gate_decision="MIXED", + gate_reasons=["all 4 transition types covered"], + baseline={ + "train": SplitResult( + pass_rate=0.5, + metric_breakdown={"m1": 0.5}, + per_case={ + "c_newp": PerCaseResult(case_id="c_newp", passed=False, metric_scores={"m1": 0.0}), + "c_newf": PerCaseResult(case_id="c_newf", passed=True, metric_scores={"m1": 1.0}), + "c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}), + "c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}), + }, + ), + }, + candidate={ + "train": SplitResult( + pass_rate=0.5, + metric_breakdown={"m1": 0.5}, + per_case={ + "c_newp": PerCaseResult(case_id="c_newp", passed=True, metric_scores={"m1": 1.0}), + "c_newf": PerCaseResult(case_id="c_newf", passed=False, metric_scores={"m1": 0.0}), + "c_bothp": PerCaseResult(case_id="c_bothp", passed=True, metric_scores={"m1": 1.0}), + "c_bothf": PerCaseResult(case_id="c_bothf", passed=False, metric_scores={"m1": 0.0}), + }, + ), + }, + delta=SplitDelta( + train=PerCaseDelta( + newly_passing=["c_newp"], + newly_failing=["c_newf"], + score_deltas={"c_newp": {"m1": 1.0}, "c_newf": {"m1": -1.0}}, + unchanged=["c_bothp", "c_bothf"], + ), + val=PerCaseDelta( + newly_passing=[], + newly_failing=[], + score_deltas={}, + unchanged=[], + ), + train_pass_rate_delta=0.0, + val_pass_rate_delta=0.0, + ), + failure_attribution=FailureAttribution( + total_cases=4, + failed_cases=2, + categories={}, + ), + duration_seconds=0.5, + seed=42, + started_at="2026-01-01T00:00:00", + finished_at="2026-01-01T00:00:01", + ) + + with tempfile.TemporaryDirectory() as tmpdir: + write_reports(result, tmpdir) + + md_path = os.path.join(tmpdir, "optimization_report.md") + assert os.path.isfile(md_path) + + with open(md_path) as f: + md_content = f.read() + + assert "## Per-Case Delta" in md_content + assert "newly passing" in md_content + assert "newly failing" in md_content + assert "passed (both)" in md_content + assert "failed (both)" in md_content + assert "MIXED" in md_content diff --git a/pyproject.toml b/pyproject.toml index 6c47d4d5..146e2281 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -204,5 +204,5 @@ split_before_logical_operator = true [tool.pytest.ini_options] minversion = "6.0" addopts = "-ra -q" -testpaths = ["tests"] +testpaths = ["tests", "examples/optimization/eval_optimize_loop/tests"] asyncio_mode = "auto"