From 9c8226c85f7da14e78e99d7df5b813a457025ce0 Mon Sep 17 00:00:00 2001 From: Dan Baciu Date: Tue, 10 Nov 2026 17:28:09 +0200 Subject: [PATCH 1/2] feat(sleep): reject candidates that regress on tonight's training tasks (gate_train_check) The nightly gate accepts a candidate when it scores higher than the current documents on the validation slice. That slice is a fraction of one night's tasks and can be a handful, so a rule that damages the agent can still come out ahead on it and be accepted. gate_train_check (default false) adds a condition to an accept. A candidate the validation gate would accept is also replayed on tonight's real training tasks (recalled tasks included, dream variants not). It is rejected when its mean gate score there is lower than the current documents' by more than gate_train_margin (default 0.1) and at least two more of those tasks got worse than got better, so one task flipping can never decide a night. These tasks are not held out: reflection drew on tonight's failures, so a higher score on them cannot rule out overfitting and is never used to accept anything. A clearly lower score, in particular breaking tasks the current documents already handle, is not explained by overfitting, so the comparison is used only to reject. Each candidate is compared with the documents it would replace: a skill candidate with the night's training replay, a memory candidate with the replay that follows an accepted skill. A task with a missing or malformed score counts as worse and a non-finite mean rejects. The check is skipped when the gate is off and when the validation slice is not disjoint from training. The added cost is one replay of the training tasks per candidate that passes validation. Both arms and every task-level change are kept in the gate records and a gate/train_check evidence event, and report.md shows the two means and the counts. These records exist only when the check is enabled, so a night without it writes what it always wrote. The flag must be a boolean and an invalid margin raises before consolidation makes any model call. The settings are forwarded at both nightly call sites, including per-skill fan-out groups. --- CHANGELOG.md | 10 + docs/sleep/README.md | 41 +- docs/sleep/multi-skill-staging.md | 4 +- plugins/README.md | 9 +- .../skills/skillopt-sleep/SKILL.md | 1 + plugins/codex/README.md | 3 +- plugins/codex/skills/skillopt-sleep/SKILL.md | 1 + .../copilot/copilot-instructions.snippet.md | 2 + plugins/cursor/skills/skillopt-sleep/SKILL.md | 3 + plugins/dsh/README.md | 2 +- plugins/dsh/README.zh.md | 1 + plugins/dsh/skills/skillopt-sleep/SKILL.md | 2 +- skillopt_sleep/config.py | 2 + skillopt_sleep/consolidate.py | 155 +++- skillopt_sleep/cycle.py | 53 +- skillopt_sleep/dream.py | 6 +- tests/test_sleep_engine.py | 4 + tests/test_sleep_gate_train_check.py | 733 ++++++++++++++++++ 18 files changed, 1011 insertions(+), 21 deletions(-) create mode 100644 tests/test_sleep_gate_train_check.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ef83812d9..57315ac04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,16 @@ All notable changes to SkillOpt are documented here. This project adheres to search setting. An API key echoed in an error body is redacted before the text reaches `fail_reason`, and the round deadline is checked again after each response. +- **SkillOpt-Sleep `gate_train_check`** config key (default `false`) with + `gate_train_margin` (default `0.1`): a candidate the validation gate would + accept is also replayed on tonight's real training tasks and is rejected + when its mean gate score there is lower than the current documents' by more + than the margin and at least two more of those tasks got worse than got + better. The check only rejects; it never accepts what validation rejected. + It is skipped with the gate off or a non-disjoint validation slice. Both + arms and every task-level change are kept in the gate records (`train_check`) + and a `gate`/`train_check` evidence event, and `report.md` shows the two + means and the counts. With the key unset, records and reports are unchanged. - **SkillOpt-Sleep paired A/B evalkit** (`python -m skillopt_sleep.evalkit`): McNemar plus percentile-bootstrap CIs on a fixed task manifest, with task-cluster multi-seed inference and seeded null calibration for exact-test diff --git a/docs/sleep/README.md b/docs/sleep/README.md index eec3e6cd8..8e4bbc132 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -35,6 +35,38 @@ candidate; an absent numeric score aborts evaluation. Task-level changes are included in `report.md`, `report.json`, `diagnostics.json`, the CLI's `--json` output, and the evidence log. +The optional `gate_train_check` config key adds a condition to an accept. It +defaults to `false`. A night's validation slice is a fraction of that night's +tasks and can be a handful, so a rule that damages the agent can still come out +ahead on it. When the check is on, a candidate the validation gate would accept +is also replayed on tonight's real training tasks (recalled tasks included, +dream variants not). It is rejected when two things hold: its mean score there, +under the configured `gate_metric`, is lower than the current documents' by +more than `gate_train_margin` (default `0.1`, a number between 0 and 1), and at +least two more of those tasks got worse than got better, so one task flipping +can never decide a night. + +These tasks are not held out, since reflection drew on tonight's failures, so +scoring higher on them proves nothing and the check never accepts anything the +validation gate rejected. Scoring clearly lower on them, in particular breaking +tasks the current documents already handle, is not explained by overfitting, +which is why the comparison is used only to reject. Each candidate is compared +with the documents it would replace: a skill candidate with the night's +training replay, a memory candidate with the replay that follows an accepted +skill. The margin therefore applies per candidate, not per night. The added +cost is one replay of the training tasks for each candidate that passes +validation. + +A task with a missing or malformed score counts as worse, and a non-finite +mean rejects the candidate. The check is skipped when `gate_mode` is `off` and +when the validation slice is not disjoint from training. The gate records +(`train_check`, in `report.json`, `diagnostics.json`, and the CLI's `--json` +output) and the evidence log (`gate` / `train_check`) keep both arms and every +task-level change; `report.md` shows the two means and the counts. These +records are written only when the check is enabled. `gate_train_check` must be +a boolean and, when it is `true`, an invalid margin raises before consolidation +makes any model call. + Rule-based heading checks are intentionally additive. `section_present` keeps its legacy strict behavior for existing task sets: an ATX heading cannot append text after the requested name. Use `section_contains` when a numbered, @@ -466,8 +498,9 @@ This is not a hard cap. A step that has started always finishes, so a night can end above the budget by the cost of one step: the first replay of the validation and training tasks, one edit step (a training replay, a reflect call and a validation replay of the candidate; with `dream_rollouts` set to K, -K attempts per training task), the final validation replay, or the held-out -test replay. Each fan-out group that starts runs its own first replay. +K attempts per training task; with `gate_train_check`, a replay of the +training tasks for a candidate that passes validation), the final validation +replay, or the held-out test replay. Each fan-out group that starts runs its own first replay. Spend comes from the backend's token counter, the one the report header prints as `tokens used`. The header shows the backend's running total and the note @@ -504,8 +537,8 @@ This runs one additional consolidation per group (including a catch-all group wh hinted and unhinted evidence are mixed), so it multiplies backend calls and token use; configured dream rollouts and synthetic variants multiply the per-group work too. Each group inherits the configured edit budget, gate mode/metric, -`gate_no_regression`, `dream_rollouts`, `dream_factor`, `recall_k`, and -`evolve_skill`. Recalled archive tasks are restricted to that same skill hint; +`gate_no_regression`, `gate_train_check`, `gate_train_margin`, +`dream_rollouts`, `dream_factor`, `recall_k`, and `evolve_skill`. Recalled archive tasks are restricted to that same skill hint; shared memory is read-only in fan-out runs. Setting `evolve_skill` to `false` therefore disables per-skill proposals as well as the managed skill proposal. diff --git a/docs/sleep/multi-skill-staging.md b/docs/sleep/multi-skill-staging.md index bb86b8169..198180f97 100644 --- a/docs/sleep/multi-skill-staging.md +++ b/docs/sleep/multi-skill-staging.md @@ -39,8 +39,8 @@ document. The managed catch-all remains on `proposed_SKILL.md` and is not duplicated as a per-skill row. Each group inherits `recall_k`, `dream_rollouts`, `dream_factor`, `edit_budget`, -`gate_mode`, `gate_metric`, `gate_mixed_weight`, `gate_no_regression`, and -`evolve_skill`. Recalled archive tasks are restricted to the same skill hint, +`gate_mode`, `gate_metric`, `gate_mixed_weight`, `gate_no_regression`, +`gate_train_check`, `gate_train_margin`, and `evolve_skill`. Recalled archive tasks are restricted to the same skill hint, and shared memory is read-only during group runs. Consequently, `evolve_skill=false` disables managed and fan-out skill proposals. diff --git a/plugins/README.md b/plugins/README.md index b76d7e0f7..6d69c1b75 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -249,10 +249,11 @@ python -m skillopt_sleep run --project "$(pwd)" \ ### Advanced config The JSON/YAML config under `~/.skillopt-sleep/` supports additional engine keys, -including `gate_mode`, `gate_metric`, `gate_no_regression`, `dream_rollouts`, -`dream_factor`, `recall_k`, `evolve_memory`, and `evolve_skill`. These are config -keys, not aliases for the unsupported CLI flags listed above. Shipping defaults -are conservative: `gate_mode="on"`, `gate_no_regression=false`, +including `gate_mode`, `gate_metric`, `gate_no_regression`, `gate_train_check`, +`gate_train_margin`, `dream_rollouts`, `dream_factor`, `recall_k`, +`evolve_memory`, and `evolve_skill`. These are config keys, not aliases for the +unsupported CLI flags listed above. Shipping defaults are conservative: +`gate_mode="on"`, `gate_no_regression=false`, `gate_train_check=false`, `dream_rollouts=1`, `dream_factor=0`, and `recall_k=0`. The managed `schedule` command stores only the project, backend, time, and diff --git a/plugins/claude-code/skills/skillopt-sleep/SKILL.md b/plugins/claude-code/skills/skillopt-sleep/SKILL.md index 0c45aa8f6..b3ed0abc6 100644 --- a/plugins/claude-code/skills/skillopt-sleep/SKILL.md +++ b/plugins/claude-code/skills/skillopt-sleep/SKILL.md @@ -110,6 +110,7 @@ Beyond the CLI flags, advanced behavior is controlled via config: - **`gate_mode`** — `on` (default, validation-gated) or `off` (greedy, accept all edits). - **`gate_metric`** — `hard`, `soft`, or `mixed` (default). Controls how the held-out gate scores. - **`gate_no_regression`** — `false` by default. Set to `true` to reject a candidate when any validation task's configured gate score decreases. +- **`gate_train_check`** — `false` by default. Set to `true` to also reject a candidate that scores clearly lower than the current skill and memory on tonight's training tasks (mean lower by more than `gate_train_margin`, default `0.1`, on at least two more tasks than improved). - **`dream_rollouts`** — >1 enables multi-rollout contrastive reflection per task. - **`recall_k`** — >0 recalls K similar past tasks into the dream (long-term memory). - **`evolve_memory`** / **`evolve_skill`** — independently toggle CLAUDE.md vs SKILL.md consolidation. diff --git a/plugins/codex/README.md b/plugins/codex/README.md index 7d35e8af2..c8e4aa41b 100644 --- a/plugins/codex/README.md +++ b/plugins/codex/README.md @@ -80,7 +80,8 @@ mining, replay, and reflection can be slow and otherwise quiet. Use prefer tasks that match the target skill's path, headings, and content. The implemented main-CLI flags work the same across the shared integrations, and `--preferences "..."` is available for house rules. Advanced keys such as -`gate_mode`, `gate_no_regression`, `dream_rollouts`, and `recall_k` belong in the Sleep config; the +`gate_mode`, `gate_no_regression`, `gate_train_check`, `dream_rollouts`, and `recall_k` belong in +the Sleep config; the nightly CLI does not expose `--gate`, `--rollouts-k`, token/time-budget, or optimizer/target-split flags. See the [shared CLI reference](../README.md#supported-cli-surface). diff --git a/plugins/codex/skills/skillopt-sleep/SKILL.md b/plugins/codex/skills/skillopt-sleep/SKILL.md index ac4a2bccf..8562b1fa5 100644 --- a/plugins/codex/skills/skillopt-sleep/SKILL.md +++ b/plugins/codex/skills/skillopt-sleep/SKILL.md @@ -134,6 +134,7 @@ without `crontab`, `schedule` prints a line for manual installation. - **`gate_mode`** — `on` (validation-gated, default) or `off` (greedy) - **`gate_metric`** — `hard` | `soft` | `mixed` (default) - **`gate_no_regression`** — `false` by default; set it to `true` to reject a candidate when any validation task's gate score decreases +- **`gate_train_check`** — `false` by default; set it to `true` to also reject a candidate that scores clearly lower on tonight's training tasks (mean lower by more than `gate_train_margin`, default `0.1`, on at least two more tasks than improved) - **`dream_rollouts`** — >1 for multi-rollout contrastive reflection - **`recall_k`** — >0 recalls similar past tasks from the archive diff --git a/plugins/copilot/copilot-instructions.snippet.md b/plugins/copilot/copilot-instructions.snippet.md index 3abc5a0df..69a6f60fb 100644 --- a/plugins/copilot/copilot-instructions.snippet.md +++ b/plugins/copilot/copilot-instructions.snippet.md @@ -47,6 +47,8 @@ my preferences", or "make the agent improve from past usage", use the MCP tools: - `gate_mode` — `on` (default) or `off` - `gate_no_regression` — `false` by default; set it to `true` to reject a candidate when any validation task's configured gate score decreases +- `gate_train_check` — `false` by default; set it to `true` to also reject a + candidate that scores clearly lower on tonight's training tasks - `dream_rollouts` — >1 for more signal - `evolve_memory` / `evolve_skill` — toggle which docs consolidate diff --git a/plugins/cursor/skills/skillopt-sleep/SKILL.md b/plugins/cursor/skills/skillopt-sleep/SKILL.md index 71cc664d5..4c330e028 100644 --- a/plugins/cursor/skills/skillopt-sleep/SKILL.md +++ b/plugins/cursor/skills/skillopt-sleep/SKILL.md @@ -173,6 +173,9 @@ commits, or generated summaries. `gate_no_regression` is a config-only safeguard in `~/.skillopt-sleep/config.json`. It defaults to `false`; set it to `true` to reject a candidate when any validation task's configured gate score decreases. +`gate_train_check` is config-only as well. It defaults to `false`; set it to +`true` to also reject a candidate that scores clearly lower on tonight's +training tasks. ## Scheduling diff --git a/plugins/dsh/README.md b/plugins/dsh/README.md index 92f2921e8..7759d1161 100644 --- a/plugins/dsh/README.md +++ b/plugins/dsh/README.md @@ -107,7 +107,7 @@ ask the agent: "Use skillopt_status to check the sleep cycle state." | `timeoutMs` | `600000` | per-call engine timeout in milliseconds | Advanced engine keys (`gate_mode`, `gate_metric`, `gate_no_regression`, -`dream_rollouts`, `recall_k`, `evolve_memory`/`evolve_skill`) go in +`gate_train_check`, `dream_rollouts`, `recall_k`, `evolve_memory`/`evolve_skill`) go in `~/.skillopt-sleep/config.json` — the same file shared by all integrations. ## Data boundary diff --git a/plugins/dsh/README.zh.md b/plugins/dsh/README.zh.md index f80dab047..04b927906 100644 --- a/plugins/dsh/README.zh.md +++ b/plugins/dsh/README.zh.md @@ -25,6 +25,7 @@ "gate_mode": "on", "gate_metric": "mixed", "gate_no_regression": false, + "gate_train_check": false, "dream_rollouts": 1, "recall_k": 0, "evolve_memory": true, diff --git a/plugins/dsh/skills/skillopt-sleep/SKILL.md b/plugins/dsh/skills/skillopt-sleep/SKILL.md index c12ccdd65..095f3499c 100644 --- a/plugins/dsh/skills/skillopt-sleep/SKILL.md +++ b/plugins/dsh/skills/skillopt-sleep/SKILL.md @@ -91,7 +91,7 @@ skillopt_schedule project= hour=3 minute=17 backend= Advanced engine keys go in `~/.skillopt-sleep/config.json`: `gate_mode` (on/off), `gate_metric` (hard/soft/mixed), `gate_no_regression`, -`dream_rollouts`, `recall_k`, `evolve_memory` / `evolve_skill`. +`gate_train_check`, `dream_rollouts`, `recall_k`, `evolve_memory` / `evolve_skill`. ## Hard rules diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 64814979d..43377b960 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -68,6 +68,8 @@ "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) "gate_mixed_weight": 0.5, "gate_no_regression": False, # reject any candidate that lowers a val-task score + "gate_train_check": False, # reject a candidate that is worse on tonight's training tasks + "gate_train_margin": 0.1, # drop in mean training-task score the check tolerates (0..1) "replay_mode": "mock", # report label; fresh-worktree replay is not implemented # ── dream + recall (opt-in; defaults reproduce the prior single-shot loop) ─ "dream_rollouts": 1, # >1 => multi-rollout contrastive reflection per task diff --git a/skillopt_sleep/consolidate.py b/skillopt_sleep/consolidate.py index 0963350a4..45e9d0a2e 100644 --- a/skillopt_sleep/consolidate.py +++ b/skillopt_sleep/consolidate.py @@ -32,6 +32,11 @@ # Its scores were not measured, so callers must not report them as results. BUDGET_EXHAUSTED = "budget_exhausted" +# The training-side check rejects only when at least this many more training +# tasks got worse than got better. One task flipping, in either direction, can +# therefore never decide a night on its own. +TRAIN_CHECK_MIN_NET_REGRESSED = 2 + def _finite_score(value: float) -> float | None: """Return a JSON-safe score, preserving finite values only.""" @@ -205,6 +210,82 @@ def _task_deltas( return out +def train_check_enabled(value: object) -> bool: + """Validate ``gate_train_check``: ``True``, ``False`` or ``None`` (off).""" + if value is None: + return False + if not isinstance(value, bool): + raise ValueError(f"gate_train_check must be true or false, got {value!r}") + return value + + +def train_check_margin(value: object) -> float: + """Validate ``gate_train_margin``: a finite number in ``[0, 1]``.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError( + f"gate_train_margin must be a number between 0 and 1, got {value!r}" + ) + margin = float(value) + if not math.isfinite(margin) or not 0.0 <= margin <= 1.0: + raise ValueError( + f"gate_train_margin must be a number between 0 and 1, got {value!r}" + ) + return margin + + +def _training_check( + tasks: List[TaskRecord], + baseline_pairs: List[Tuple[TaskRecord, ReplayResult]], + candidate_pairs: List[Tuple[TaskRecord, ReplayResult]], + metric: str, + mixed_weight: float, + margin: float, +) -> dict: + """Compare a candidate with the current documents on tonight's training tasks. + + These tasks are not held out: reflection drew on tonight's failures, so a + higher score here cannot rule out overfitting and is never used to accept + anything. A lower score is different. The tasks include the ones the + current documents already handle, and a candidate that breaks those, or + does clearly worse on the failures it was written to fix, is not explained + by overfitting. So the comparison is used only to reject. + + It is baseline-relative and aggregate, and it takes two things to reject: + the candidate's mean gate score on these tasks is lower than the current + documents' by more than ``margin``, and at least + ``TRAIN_CHECK_MIN_NET_REGRESSED`` more tasks got worse than got better. + Both arms and every task-level change are returned so the decision can be + audited. A task with a missing or malformed score counts as regressed, and + a non-finite aggregate rejects. + """ + base_hard, base_soft = aggregate_scores(baseline_pairs) + cand_hard, cand_soft = aggregate_scores(candidate_pairs) + baseline = select_gate_score(base_hard, base_soft, metric, mixed_weight) + candidate = select_gate_score(cand_hard, cand_soft, metric, mixed_weight) + task_deltas = _task_deltas( + tasks, baseline_pairs, candidate_pairs, metric, mixed_weight + ) + n_regressed = sum(1 for row in task_deltas if row["status"] == "regressed") + n_improved = sum(1 for row in task_deltas if row["status"] == "improved") + finite = math.isfinite(baseline) and math.isfinite(candidate) + # The tolerance keeps a drop of exactly the margin on the tolerated side: + # 0.8 - 0.1 is 0.7000000000000001 in floating point. + lower = (not finite) or (baseline - candidate) > margin + 1e-9 + broad = n_regressed - n_improved >= TRAIN_CHECK_MIN_NET_REGRESSED + return { + "n_tasks": len(tasks), + "n_regressed": n_regressed, + "n_improved": n_improved, + "min_net_regressed": TRAIN_CHECK_MIN_NET_REGRESSED, + "margin": margin, + "baseline_score": _finite_score(baseline), + "candidate_score": _finite_score(candidate), + "regressed": lower, + "blocked": bool((not finite) or (lower and broad)), + "task_deltas": task_deltas, + } + + def consolidate( backend: Backend, tasks: List[TaskRecord], @@ -215,6 +296,8 @@ def consolidate( gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, gate_no_regression: bool = False, + gate_train_check: bool = False, # also reject a candidate that is worse on tonight's training tasks + gate_train_margin: float = 0.1, # drop in mean training-task score that is tolerated gate_mode: str = "on", # "on" (hard/soft per gate_metric) | "off" (greedy) rollouts_k: int = 1, # >1 => multi-rollout contrastive reflection evolve_skill: bool = True, @@ -231,6 +314,17 @@ def consolidate( Skill and memory are evolved in sequence (skill first if both enabled). + ``gate_train_check`` adds a condition to an accept. A candidate the + validation gate would accept is also replayed on tonight's real training + tasks (recalled tasks included, dream variants not), and is rejected when + its mean gate score there is lower than the current documents' by more + than ``gate_train_margin`` and at least two more of those tasks got worse + than got better. It never accepts anything the validation gate rejected. + Each candidate is compared with the documents it would replace, so a + memory candidate is compared with the skill accepted earlier that night. + It is skipped when the gate is off and when the validation slice is not + disjoint from training. + ``budget_exhausted`` is checked before the epoch starts, before each evolve target and before the final validation replay. Once it returns True, the steps that have not started are skipped and listed in @@ -243,6 +337,8 @@ def consolidate( ev = evlog.get(backend) train_tasks, val_tasks, holdout_leaked = _split(tasks) gate_off = str(gate_mode).strip().lower() in {"off", "none", "false", "greedy"} + train_check_on = train_check_enabled(gate_train_check) + train_margin = train_check_margin(gate_train_margin) if train_check_on else 0.0 holdout_detail: List[dict] = [] budget_skipped: List[str] = [] @@ -301,6 +397,14 @@ def _budget_spent(step: str) -> bool: all_unmatched: List[EditRecord] = [] gate_trials: List[dict] = [] current_pairs = base_pairs + # Training-side check: tonight's real training tasks (recalled ones + # included). Dream variants restate them and are left out. The current + # documents' scores on these tasks are the replay that just ran. + check_tasks: List[TaskRecord] = [] + current_train_pairs: List[Tuple[TaskRecord, ReplayResult]] = [] + if train_check_on and not gate_off and not holdout_leaked: + check_tasks = [t for t in train_tasks if t.origin != "dream"] + current_train_pairs = [(t, r) for (t, r) in train_pairs if t.origin != "dream"] def _edits_payload(edits: List[EditRecord]) -> List[dict]: return [{"op": e.op, "content": e.content, "anchor": e.anchor, @@ -308,7 +412,7 @@ def _edits_payload(edits: List[EditRecord]) -> List[dict]: def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: nonlocal cand_skill, cand_memory, base_score, current_pairs - nonlocal all_applied, all_rejected + nonlocal all_applied, all_rejected, current_train_pairs if ev is not None: ev.log("reflect", "edits_returned", target=which, n_edits=len(edits), edits=_edits_payload(edits)) @@ -345,24 +449,52 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: ) trial_base_score = base_score improved = cand_score > base_score and not blocked_by_regression - gate_trials.append({ + # Only a candidate the validation gate would accept is worth the extra + # replay, and the check can only turn that accept into a reject. + train_check: Optional[dict] = None + train_cand_pairs: List[Tuple[TaskRecord, ReplayResult]] = [] + if improved and check_tasks: + evlog.set_phase(backend, f"gate_train_check:{which}") + train_cand_pairs = replay_batch( + backend, check_tasks, trial_skill, trial_memory + ) + train_check = _training_check( + check_tasks, current_train_pairs, train_cand_pairs, + gate_metric, gate_mixed_weight, train_margin, + ) + if ev is not None: + ev.log("gate", "train_check", target=which, **train_check) + if train_check["blocked"]: + improved = False + blocked_by_train_check = bool(train_check and train_check["blocked"]) + trial_record = { "target": which, "baseline_score": _finite_score(trial_base_score), "candidate_score": _finite_score(cand_score), "accepted": improved, "blocked_by_regression": blocked_by_regression, "task_deltas": task_deltas, - }) + } + trial_event = {} + if train_check_on: + # Present only when the check is enabled, so a night without it + # writes the records it always wrote. + trial_record["blocked_by_train_check"] = blocked_by_train_check + trial_record["train_check"] = train_check + trial_event["blocked_by_train_check"] = blocked_by_train_check + gate_trials.append(trial_record) if ev is not None: ev.log("gate", "trial", target=which, mode="gated", baseline_score=trial_base_score, cand_hard=h, cand_soft=s, cand_score=cand_score, accepted=improved, blocked_by_regression=blocked_by_regression, task_deltas=task_deltas, - n_edits=len(applied)) + n_edits=len(applied), **trial_event) if improved: base_score = cand_score current_pairs = pairs + if train_check is not None: + current_train_pairs = train_cand_pairs all_applied.extend(applied) return new_doc all_rejected.extend(applied) @@ -425,6 +557,12 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: # re-evaluate failures under the (possibly improved) skill evlog.set_phase(backend, "train_post_skill") train_pairs2 = replay_batch(backend, train_tasks, cand_skill, cand_memory) + if check_tasks: + # The memory candidate is compared with this replay of the current + # documents, not with the draw that passed the skill's own check. + current_train_pairs = [ + (t, r) for (t, r) in train_pairs2 if t.origin != "dream" + ] failures2 = [(t, r) for (t, r) in train_pairs2 if r.hard < 1.0] successes2 = [(t, r) for (t, r) in train_pairs2 if r.hard >= 1.0] edits_m = backend.reflect( @@ -519,14 +657,19 @@ def _gate_apply(doc: str, edits: List[EditRecord], which: str) -> str: if edit not in all_rejected: all_rejected.append(edit) all_applied = [] - gate_trials.append({ + final_record = { "target": "final", "baseline_score": _finite_score(base_gate_score), "candidate_score": _finite_score(final_score), "accepted": accepted, "blocked_by_regression": final_blocked_by_regression, "task_deltas": final_task_deltas, - }) + } + if train_check_on: + # The training-side check belongs to the candidate trials above. + final_record["blocked_by_train_check"] = False + final_record["train_check"] = None + gate_trials.append(final_record) if ev is not None: w = max(0.0, min(1.0, float(gate_mixed_weight))) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index b3711f7d8..2a014d274 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -24,7 +24,11 @@ from skillopt_sleep.backend import Backend, CursorBackendError, build_backend from skillopt_sleep.budget import Budget from skillopt_sleep.config import DEFAULTS, SleepConfig, load_config -from skillopt_sleep.consolidate import BUDGET_EXHAUSTED +from skillopt_sleep.consolidate import ( + BUDGET_EXHAUSTED, + train_check_enabled, + train_check_margin, +) from skillopt_sleep.dream import dream_consolidate from skillopt_sleep.evidence import EvidenceLog from skillopt_sleep.handoff_backend import PendingCalls @@ -118,6 +122,20 @@ def _night_token_budget(cfg: SleepConfig) -> Budget: return Budget(max_tokens=limit or None) +def _gate_train_check(cfg: SleepConfig) -> tuple[bool, float]: + """Return ``(gate_train_check, gate_train_margin)``. + + Raises ``ValueError`` when ``gate_train_check`` is not a boolean, or when + it is enabled and the margin is not a finite number between 0 and 1. A + null margin means the default. + """ + enabled = train_check_enabled(cfg.get("gate_train_check", False)) + raw = cfg.get("gate_train_margin", 0.1) + if not enabled: + return False, 0.1 + return True, train_check_margin(0.1 if raw is None else raw) + + # ── Model-swap detection (F16) ─────────────────────────────── def _make_model_key(cfg: SleepConfig) -> str: """Stable string identifying the backend object(s) actually used. @@ -384,6 +402,11 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: f"- tokens used: {report.tokens_used}", "", ] + train_check, train_margin = _gate_train_check(cfg) + if train_check and str(cfg.get("gate_mode", "on")).strip().lower() not in { + "off", "none", "false", "greedy", + }: + lines.insert(-1, f"- training-side check: enabled (margin {train_margin})") gate_on = str(cfg.get("gate_mode", "on")).strip().lower() not in { "off", "none", "false", "greedy", } @@ -411,6 +434,8 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: decision = "accepted" if accepted else "rejected" if blocked: decision += " (task regression)" + if trial.get("blocked_by_train_check", False): + decision += " (training-task regression)" baseline = _report_score(trial.get("baseline_score")) candidate = _report_score(trial.get("candidate_score")) lines.append( @@ -433,6 +458,24 @@ def _render_report_md(report: SleepReport, cfg: SleepConfig) -> str: f"| {candidate_score} | {status} |" ) lines.append("") + check = trial.get("train_check") + if isinstance(check, dict): + if check.get("blocked"): + verdict = "regressed, candidate rejected" + elif check.get("regressed"): + verdict = "lower, but not on enough tasks to reject" + else: + verdict = "no regression" + lines.append( + "Training-side check (these tasks are not held out) over " + f"{check.get('n_tasks')} training tasks: " + f"{_report_score(check.get('baseline_score'))} → " + f"{_report_score(check.get('candidate_score'))} " + f"(margin {check.get('margin')}; " + f"{check.get('n_regressed')} worse, " + f"{check.get('n_improved')} better): {verdict}." + ) + lines.append("") if any((report.edits, report.rejected_edits, report.unmatched_edits)): lines.append( "_Edit source IDs list the task evidence supplied to reflection; " @@ -723,6 +766,7 @@ def run_sleep_cycle( # Checked before the night writes anything (state, staging dir, evidence). harvest_source = normalize_transcript_source(cfg.get("transcript_source")) budget = _night_token_budget(cfg) + train_check, train_margin = _gate_train_check(cfg) state = SleepState.load(cfg.state_path) project = _project_paths(cfg) @@ -805,6 +849,9 @@ def budget_exhausted() -> bool: cycle_config["opencode_tool_replay"] = ( cfg.get("opencode_tool_replay", False) is True ) + if train_check: + cycle_config["gate_train_check"] = True + cycle_config["gate_train_margin"] = train_margin ev.log("cycle", "start", night=night, project=project, backend=backend.name, model=cfg.get("model", ""), config=cycle_config) @@ -976,6 +1023,8 @@ def budget_exhausted() -> bool: gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), gate_no_regression=cfg.get("gate_no_regression", False), + gate_train_check=train_check, + gate_train_margin=train_margin, gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), evolve_memory=cfg.get("evolve_memory", True), @@ -1082,6 +1131,8 @@ def budget_exhausted() -> bool: gate_metric=cfg.get("gate_metric", "mixed"), gate_mixed_weight=cfg.get("gate_mixed_weight", 0.5), gate_no_regression=cfg.get("gate_no_regression", False), + gate_train_check=train_check, + gate_train_margin=train_margin, gate_mode=cfg.get("gate_mode", "on"), evolve_skill=cfg.get("evolve_skill", True), night=night, diff --git a/skillopt_sleep/dream.py b/skillopt_sleep/dream.py index 5621cc3be..dbc7ac29b 100644 --- a/skillopt_sleep/dream.py +++ b/skillopt_sleep/dream.py @@ -141,6 +141,8 @@ def dream_consolidate( gate_metric: str = "mixed", gate_mixed_weight: float = 0.5, gate_no_regression: bool = False, + gate_train_check: bool = False, + gate_train_margin: float = 0.1, gate_mode: str = "on", evolve_skill: bool = True, evolve_memory: bool = True, @@ -180,7 +182,9 @@ def dream_consolidate( backend, enlarged, skill, memory, edit_budget=edit_budget, gate_metric=gate_metric, gate_mixed_weight=gate_mixed_weight, - gate_no_regression=gate_no_regression, gate_mode=gate_mode, + gate_no_regression=gate_no_regression, + gate_train_check=gate_train_check, gate_train_margin=gate_train_margin, + gate_mode=gate_mode, rollouts_k=dream_rollouts, evolve_skill=evolve_skill, evolve_memory=evolve_memory, night=night, budget_exhausted=budget_exhausted, diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index 8aaded740..7f7e62cac 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -3724,6 +3724,8 @@ def test_group_runs_inherit_dream_gate_budget_and_scoped_recall_config(self): gate_metric="mixed", gate_mixed_weight=0.37, gate_no_regression=True, + gate_train_check=True, + gate_train_margin=0.25, gate_mode="off", evolve_skill=True, evolve_memory=True, @@ -3799,6 +3801,8 @@ def _spy(backend, tasks, skill, memory, **kwargs): "gate_metric": "mixed", "gate_mixed_weight": 0.37, "gate_no_regression": True, + "gate_train_check": True, + "gate_train_margin": 0.25, "gate_mode": "off", "evolve_skill": True, } diff --git a/tests/test_sleep_gate_train_check.py b/tests/test_sleep_gate_train_check.py new file mode 100644 index 000000000..196adc5cb --- /dev/null +++ b/tests/test_sleep_gate_train_check.py @@ -0,0 +1,733 @@ +"""Training-side check: a candidate may not be clearly worse on tonight's training tasks.""" + +from __future__ import annotations + +import importlib +import json +import math +import os + +import pytest + +import skillopt_sleep.cycle as cycle_module +from skillopt_sleep.backend import Backend, DualBackend +from skillopt_sleep.config import DEFAULTS, load_config +from skillopt_sleep.consolidate import ( + TRAIN_CHECK_MIN_NET_REGRESSED, + consolidate, + train_check_enabled, + train_check_margin, +) +from skillopt_sleep.cycle import _render_report_md, run_sleep_cycle +from skillopt_sleep.dream import dream_consolidate +from skillopt_sleep.evidence import EvidenceLog, read_events +from skillopt_sleep.experiments.personas import researcher_persona +from skillopt_sleep.memory import set_learned +from skillopt_sleep.mine import assign_splits +from skillopt_sleep.types import EditRecord, SleepReport, TaskRecord + +SKILL_RULE = "Always add the category of the answer in parentheses." +MEMORY_RULE = "Remember to restate the question first." +SKILL = set_learned("# Skill\n", []) +MEMORY = set_learned("# Memory\n", []) + + +def _task(task_id: str, split: str, **extra) -> TaskRecord: + return TaskRecord(id=task_id, project="test", intent=f"question {task_id}", + split=split, **extra) + + +class ScriptedBackend(Backend): + """Each task scores a fixed value per document state. + + ``script`` maps a task's source id to its scores for + ``(no rule, skill rule, skill and memory rule)``; shorter tuples repeat + their last value. The memory rule never appears without the skill rule in + these tests. + """ + + name = "scripted-stub" + + def __init__(self, script: dict[str, tuple[float, ...]], propose_memory: bool = False): + self.script = script + self.propose_memory = propose_memory + self.calls: list[tuple[str, str]] = [] # (task id, phase) + + def attempt(self, task, skill, memory, sample_id: int = 0) -> str: + self.calls.append((task.id, self.evidence_phase)) + state = 2 if MEMORY_RULE in memory else (1 if SKILL_RULE in skill else 0) + source = (task.derived_from or task.id).removeprefix("recall:") + scores = self.script[source] + return repr(scores[min(state, len(scores) - 1)]) + + def judge(self, task, response): + score = float(response) + return score, score, "matched" if score >= 1.0 else "did not match" + + def reflect(self, failures, successes, skill, memory, *, edit_budget, + evolve_skill, evolve_memory): + if evolve_skill: + return [EditRecord(target="skill", op="add", content=SKILL_RULE, rationale="r")] + if evolve_memory and self.propose_memory: + return [EditRecord(target="memory", op="add", content=MEMORY_RULE, rationale="r")] + return [] + + +def _run(script, *, train=5, val=("v1",), **kwargs): + backend = ScriptedBackend(script) + tasks = [_task(f"t{i}", "train") for i in range(train)] + [_task(v, "val") for v in val] + kwargs.setdefault("evolve_memory", False) + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", **kwargs) + return backend, result + + +def _phases(backend: ScriptedBackend) -> set[str]: + return {phase for _task_id, phase in backend.calls} + + +def _check_ran(backend: ScriptedBackend) -> bool: + return any(phase.startswith("gate_train_check") for phase in _phases(backend)) + + +# A rule that fixes the one validation task and breaks every training task. +HARMFUL = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(5)}} +# A rule that fixes the validation task and the training tasks. +HELPFUL = {"v1": (0.0, 1.0), **{f"t{i}": (0.0, 1.0) for i in range(5)}} + + +# ── off by default, and then nothing changes ───────────────────────────────── + + +def test_default_makes_no_extra_replay_and_writes_the_same_records(tmp_path) -> None: + backend, result = _run(HARMFUL) + explicit_backend, explicit = _run(HARMFUL, gate_train_check=False) + none_backend, none = _run(HARMFUL, gate_train_check=None) + + assert DEFAULTS["gate_train_check"] is False + assert DEFAULTS["gate_train_margin"] == 0.1 + assert backend.calls == explicit_backend.calls == none_backend.calls + assert not _check_ran(backend) + # the validation slice alone accepts the harmful rule + assert result.accepted is explicit.accepted is none.accepted is True + for trial in result.gate_trials: + assert set(trial) == { + "target", "baseline_score", "candidate_score", "accepted", + "blocked_by_regression", "task_deltas", + } + + logged = ScriptedBackend(HARMFUL) + path = os.path.join(tmp_path, "evidence.jsonl") + logged.evidence = EvidenceLog(path) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + consolidate(logged, tasks, SKILL, MEMORY, gate_metric="hard", evolve_memory=False) + events = read_events(path) + assert not [e for e in events if e.get("event") == "train_check"] + (trial,) = [e for e in events if e.get("stage") == "gate" and e.get("event") == "trial"] + assert "blocked_by_train_check" not in trial + + +# ── what the check changes ─────────────────────────────────────────────────── + + +def test_a_candidate_that_breaks_its_own_training_tasks_is_rejected() -> None: + backend, result = _run(HARMFUL, gate_train_check=True) + + assert result.accepted is False + assert SKILL_RULE not in result.new_skill + assert [edit.content for edit in result.rejected_edits] == [SKILL_RULE] + trial = result.gate_trials[0] + assert trial["candidate_score"] > trial["baseline_score"] # validation liked it + assert trial["accepted"] is False + assert trial["blocked_by_train_check"] is True + check = trial["train_check"] + assert (check["n_tasks"], check["margin"]) == (5, 0.1) + assert (check["n_regressed"], check["n_improved"]) == (5, 0) + assert check["min_net_regressed"] == TRAIN_CHECK_MIN_NET_REGRESSED == 2 + assert (check["baseline_score"], check["candidate_score"]) == (1.0, 0.0) + assert check["regressed"] is True and check["blocked"] is True + # both arms of every training task are kept + assert [(row["task_id"], row["baseline_score"], row["candidate_score"], row["status"]) + for row in check["task_deltas"]] == [ + (f"t{i}", 1.0, 0.0, "regressed") for i in range(5) + ] + assert ("t0", "gate_train_check:skill") in backend.calls + final = result.gate_trials[-1] + assert final["target"] == "final" + assert final["blocked_by_train_check"] is False and final["train_check"] is None + json.dumps(result.gate_trials, allow_nan=False) + + +def test_a_candidate_that_helps_its_training_tasks_is_still_accepted() -> None: + _, result = _run(HELPFUL, gate_train_check=True) + + assert result.accepted is True + assert SKILL_RULE in result.new_skill + check = result.gate_trials[0]["train_check"] + assert (check["baseline_score"], check["candidate_score"]) == (0.0, 1.0) + assert (check["n_regressed"], check["n_improved"]) == (0, 5) + assert check["regressed"] is False and check["blocked"] is False + assert result.gate_trials[0]["blocked_by_train_check"] is False + + +def test_the_check_never_accepts_what_validation_rejected() -> None: + # Validation ties, so nothing is accepted and nothing is replayed for the check. + script = {"v1": (1.0, 1.0), **{f"t{i}": (0.0, 1.0) for i in range(5)}} + + backend, result = _run(script, gate_train_check=True) + + assert result.accepted is False + assert not _check_ran(backend) + assert result.gate_trials[0]["train_check"] is None + + +def test_a_candidate_the_no_regression_rule_blocked_is_not_replayed() -> None: + script = {"v1": (0.0, 1.0), "v2": (1.0, 0.5), **{f"t{i}": (1.0, 0.0) for i in range(5)}} + + backend, result = _run(script, val=("v1", "v2"), gate_train_check=True, + gate_no_regression=True) + + trial = result.gate_trials[0] + assert trial["blocked_by_regression"] is True + assert trial["blocked_by_train_check"] is False and trial["train_check"] is None + assert not _check_ran(backend) + + +# ── the rejection rule ─────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("margin", "accepted"), + [(0.0, False), (0.1, False), (0.39, False), (0.4, True), (0.5, True)], +) +def test_margin_is_the_tolerated_drop_in_mean_training_score(margin, accepted) -> None: + # Two of five training tasks go from right to wrong: the mean drops by 0.4. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), + **{f"t{i}": (1.0, 1.0) for i in range(2, 5)}} + + _, result = _run(script, gate_train_check=True, gate_train_margin=margin) + + check = result.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(1.0) + assert check["candidate_score"] == pytest.approx(0.6) + assert (check["n_regressed"], check["n_improved"]) == (2, 0) + assert result.accepted is accepted + assert check["blocked"] is (not accepted) + + +@pytest.mark.parametrize("passing_before", [3, 4, 5, 6, 7, 8, 9, 10]) +def test_a_drop_of_exactly_the_margin_is_tolerated_at_every_baseline(passing_before) -> None: + # Ten training tasks; two go from right to wrong, so the mean drops by + # exactly 0.2. With margin 0.2 that is tolerated whatever the baseline is + # (0.8 - 0.2 is not 0.6 in floating point). + script = {"v1": (0.0, 1.0)} + for i in range(10): + before = 1.0 if i < passing_before else 0.0 + script[f"t{i}"] = (before, 0.0 if i < 2 else before) + + _, at_margin = _run(script, train=10, gate_train_check=True, gate_train_margin=0.2) + _, below_margin = _run(script, train=10, gate_train_check=True, gate_train_margin=0.19) + + check = at_margin.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(passing_before / 10) + assert check["candidate_score"] == pytest.approx((passing_before - 2) / 10) + assert check["regressed"] is False and check["blocked"] is False + assert at_margin.accepted is True + assert below_margin.gate_trials[0]["train_check"]["blocked"] is True + + +def test_the_margin_is_absolute_not_relative() -> None: + # Baseline 0.4, candidate 0.2: an absolute drop of 0.2 and a relative drop of half. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), + "t2": (0.0, 0.0), "t3": (0.0, 0.0), "t4": (0.0, 1.0)} + _, tolerated = _run(script, gate_train_check=True, gate_train_margin=0.3) + + check = tolerated.gate_trials[0]["train_check"] + assert check["baseline_score"] == pytest.approx(0.4) + assert check["candidate_score"] == pytest.approx(0.2) + assert check["regressed"] is False + assert tolerated.accepted is True + + +def test_margin_zero_rejects_any_broad_drop() -> None: + # Two tasks lose a little: mean drop 0.04, far below the default margin. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.9), "t1": (1.0, 0.9), + **{f"t{i}": (1.0, 1.0) for i in range(2, 5)}} + + _, default = _run(script, gate_train_check=True) + _, zero = _run(script, gate_train_check=True, gate_train_margin=0) + + assert default.accepted is True + assert default.gate_trials[0]["train_check"]["regressed"] is False + assert zero.accepted is False + assert zero.gate_trials[0]["train_check"]["blocked"] is True + + +def test_one_task_flipping_can_never_reject() -> None: + # One of five training tasks goes from right to wrong. The mean drops by + # 0.2, more than the margin, but a single task is not enough to reject. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), + **{f"t{i}": (1.0, 1.0) for i in range(1, 5)}} + + _, result = _run(script, gate_train_check=True, gate_train_margin=0.0) + + check = result.gate_trials[0]["train_check"] + assert check["candidate_score"] == pytest.approx(0.8) + assert (check["n_regressed"], check["n_improved"]) == (1, 0) + assert check["regressed"] is True and check["blocked"] is False + assert result.gate_trials[0]["blocked_by_train_check"] is False + assert result.accepted is True + + +@pytest.mark.parametrize(("better", "blocked"), [(0, True), (1, True), (2, False)]) +def test_net_count_of_regressed_tasks_decides(better, blocked) -> None: + # Three tasks get much worse; ``better`` tasks get slightly better. The + # mean always drops well past the margin; only the net count changes. + script = {"v1": (0.0, 1.0), "t0": (1.0, 0.0), "t1": (1.0, 0.0), "t2": (1.0, 0.0)} + for i in (3, 4): + script[f"t{i}"] = (0.0, 0.2) if i - 3 < better else (0.0, 0.0) + + _, result = _run(script, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert (check["n_regressed"], check["n_improved"]) == (3, better) + assert check["regressed"] is True and check["blocked"] is blocked + assert result.accepted is (not blocked) + + +@pytest.mark.parametrize(("train", "blocked"), [(1, False), (2, True), (3, True)]) +def test_how_many_training_tasks_it_takes(train, blocked) -> None: + script = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(train)}} + + _, result = _run(script, train=train, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == train + assert check["regressed"] is True and check["blocked"] is blocked + assert result.accepted is (not blocked) + + +def test_a_malformed_training_score_fails_closed() -> None: + script = {"v1": (0.0, 1.0), "t0": (1.0, math.nan), + **{f"t{i}": (1.0, 1.0) for i in range(1, 5)}} + + _, result = _run(script, gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + assert check["candidate_score"] is None + assert check["regressed"] is True and check["blocked"] is True + assert result.accepted is False + json.dumps(result.gate_trials, allow_nan=False) + + +def test_a_training_task_missing_from_the_candidate_replay_counts_as_worse(monkeypatch) -> None: + module = importlib.import_module("skillopt_sleep.consolidate") + real = module.replay_batch + + def dropping(backend, tasks, skill, memory): + pairs = real(backend, tasks, skill, memory) + if backend.evidence_phase.startswith("gate_train_check"): + pairs = [pair for pair in pairs if pair[0].id not in {"t0", "t1"}] + return pairs + + monkeypatch.setattr(module, "replay_batch", dropping) + + _, result = _run(HELPFUL | {"t2": (1.0, 0.0), "t3": (1.0, 0.0), "t4": (1.0, 0.0)}, + gate_train_check=True) + + check = result.gate_trials[0]["train_check"] + statuses = {row["task_id"]: row["status"] for row in check["task_deltas"]} + assert statuses["t0"] == statuses["t1"] == "regressed" + assert check["n_regressed"] == 5 + assert check["blocked"] is True + assert result.accepted is False + + +# ── scope ──────────────────────────────────────────────────────────────────── + + +def test_recalled_tasks_are_checked_and_dream_variants_are_not() -> None: + backend = ScriptedBackend({ + "v1": (0.0, 1.0), + **{f"t{i}": (1.0, 1.0) for i in range(2)}, + **{f"old{i}": (1.0, 0.0) for i in range(3)}, + }) + tonight = [_task("t0", "train"), _task("t1", "train"), _task("v1", "val")] + history = [ + TaskRecord(id=f"old{i}", project="test", intent=f"question t{i % 2} again {i}", + split="train") + for i in range(3) + ] + + result = dream_consolidate( + backend, tonight, SKILL, MEMORY, history_tasks=history, recall_k=3, + dream_factor=2, gate_metric="hard", evolve_memory=False, + gate_train_check=True, + ) + + checked = sorted(task_id for task_id, phase in backend.calls + if phase == "gate_train_check:skill") + assert checked == ["recall:old0", "recall:old1", "recall:old2", "t0", "t1"] + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == 5 + assert check["baseline_score"] == pytest.approx(1.0) + assert check["candidate_score"] == pytest.approx(0.4) + assert result.accepted is False + + +def test_dream_variants_are_left_out_of_both_arms() -> None: + # The real training tasks improve; only their dream variants would look + # worse (a variant's score follows its source task here, so give the + # variants' sources different scripts through distinct real tasks). + backend = ScriptedBackend({ + "v1": (0.0, 1.0), "t0": (0.0, 1.0), "t1": (0.0, 1.0), + }) + tonight = [_task("t0", "train"), _task("t1", "train"), _task("v1", "val")] + + result = dream_consolidate( + backend, tonight, SKILL, MEMORY, dream_factor=3, gate_metric="hard", + evolve_memory=False, gate_train_check=True, + ) + + check = result.gate_trials[0]["train_check"] + assert check["n_tasks"] == 2 # not 2 real + 6 variants + assert [row["task_id"] for row in check["task_deltas"]] == ["t0", "t1"] + assert (check["baseline_score"], check["candidate_score"]) == (0.0, 1.0) + assert result.accepted is True + + +def test_gate_off_and_non_disjoint_validation_skip_the_check() -> None: + off_backend, off = _run(HARMFUL, gate_train_check=True, gate_mode="off") + leaked_backend, leaked = _run( + {f"t{i}": (0.0, 1.0) for i in range(5)}, val=(), gate_train_check=True, + ) + + assert not _check_ran(off_backend) + assert off.gate_action == "greedy_applied" + assert leaked.holdout_leaked is True + assert not _check_ran(leaked_backend) + + +def test_the_check_runs_with_dream_rollouts_and_parallel_workers(monkeypatch) -> None: + monkeypatch.setenv("SKILLOPT_SLEEP_WORKERS", "4") + + class Contrastive(ScriptedBackend): + def _call(self, prompt, *, max_tokens=1024): + return json.dumps([{"op": "add", "content": SKILL_RULE, "rationale": "r"}]) + + backend = Contrastive(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=False, rollouts_k=3, gate_train_check=True) + + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert result.accepted is False + checked = sorted(t for t, phase in backend.calls if phase == "gate_train_check:skill") + assert checked == [f"t{i}" for i in range(5)] + + +# ── skill then memory ──────────────────────────────────────────────────────── + + +def test_with_a_dual_backend_the_check_replays_on_the_target_only() -> None: + target, optimizer = ScriptedBackend(HARMFUL), ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + result = consolidate(DualBackend(target, optimizer), tasks, SKILL, MEMORY, + gate_metric="hard", evolve_memory=False, gate_train_check=True) + + assert result.accepted is False + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert sorted(task_id for task_id, _phase in target.calls).count("t0") == 2 # baseline, candidate + assert optimizer.calls == [] + + +def _skill_then_memory(script): + backend = ScriptedBackend(script, propose_memory=True) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True) + return backend, result + + +def test_memory_candidate_is_compared_with_the_accepted_skill() -> None: + # The skill rule helps everywhere and is accepted. The memory rule then + # lifts the validation task further but breaks the training tasks that the + # accepted skill had just fixed. + script = {"v1": (0.0, 0.5, 1.0), **{f"t{i}": (0.0, 1.0, 0.0) for i in range(5)}} + + backend, result = _skill_then_memory(script) + + skill_trial, memory_trial, _final = result.gate_trials + assert skill_trial["accepted"] is True + assert skill_trial["train_check"]["candidate_score"] == 1.0 + assert memory_trial["target"] == "memory" + assert memory_trial["blocked_by_train_check"] is True + assert memory_trial["train_check"]["baseline_score"] == 1.0 # post-skill scores + assert memory_trial["train_check"]["candidate_score"] == 0.0 + assert SKILL_RULE in result.new_skill + assert MEMORY_RULE not in result.new_memory + assert result.accepted is True + # the baseline arm is the post-skill training replay the night already runs + assert ("t0", "train_post_skill") in backend.calls + assert ("t0", "gate_train_check:memory") in backend.calls + + +def test_a_blocked_skill_does_not_become_the_memory_baseline() -> None: + # The skill rule breaks training and is blocked. The memory rule (which the + # stub can only reach with the skill rule absent) is then judged against + # the original documents' scores, not the blocked candidate's. + script = {"v1": (0.0, 1.0), **{f"t{i}": (1.0, 0.0) for i in range(5)}} + + class MemoryOnly(ScriptedBackend): + def attempt(self, task, skill, memory, sample_id: int = 0) -> str: + self.calls.append((task.id, self.evidence_phase)) + if MEMORY_RULE in memory: + return repr(1.0) # memory rule fixes everything + return repr(self.script[task.id][1 if SKILL_RULE in skill else 0]) + + backend = MemoryOnly(script, propose_memory=True) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True) + + skill_trial, memory_trial, _final = result.gate_trials + assert skill_trial["blocked_by_train_check"] is True + assert memory_trial["target"] == "memory" and memory_trial["accepted"] is True + assert memory_trial["train_check"]["baseline_score"] == 1.0 + assert memory_trial["train_check"]["candidate_score"] == 1.0 + assert SKILL_RULE not in result.new_skill and MEMORY_RULE in result.new_memory + + +# ── budget ─────────────────────────────────────────────────────────────────── + + +def test_the_check_is_part_of_the_edit_step_once_the_step_has_started() -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + checks = iter([False, False, True, True]) # epoch, skill edits, then spent + + result = consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=True, gate_train_check=True, + budget_exhausted=lambda: next(checks, True)) + + assert result.budget_skipped == ["memory edits", "final validation replay"] + assert _check_ran(backend) + assert result.gate_trials[0]["blocked_by_train_check"] is True + assert result.accepted is False + + +# ── validation of the settings ─────────────────────────────────────────────── + + +@pytest.mark.parametrize("bad", [-0.01, 1.01, math.nan, math.inf, "0.1", None, True, [0.1]]) +def test_an_invalid_margin_stops_before_any_model_call(bad) -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + with pytest.raises(ValueError, match="gate_train_margin"): + consolidate(backend, tasks, SKILL, MEMORY, gate_train_check=True, + gate_train_margin=bad) + assert backend.calls == [] + + +@pytest.mark.parametrize("bad", ["true", "false", 1, 0, 0.5, [True]]) +def test_a_non_boolean_flag_stops_before_any_model_call(bad) -> None: + backend = ScriptedBackend(HARMFUL) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + with pytest.raises(ValueError, match="gate_train_check must be true or false"): + consolidate(backend, tasks, SKILL, MEMORY, gate_train_check=bad) + assert backend.calls == [] + + +def test_setting_helpers() -> None: + assert train_check_enabled(None) is False + assert train_check_enabled(False) is False + assert train_check_enabled(True) is True + assert train_check_margin(0) == 0.0 and train_check_margin(1) == 1.0 + _, result = _run(HELPFUL, gate_train_margin="not a number") # ignored while off + assert result.accepted is True + + +# ── records ────────────────────────────────────────────────────────────────── + + +def test_evidence_log_keeps_both_arms_of_the_check(tmp_path) -> None: + backend = ScriptedBackend(HARMFUL) + path = os.path.join(tmp_path, "evidence.jsonl") + backend.evidence = EvidenceLog(path) + tasks = [_task(f"t{i}", "train") for i in range(5)] + [_task("v1", "val")] + + consolidate(backend, tasks, SKILL, MEMORY, gate_metric="hard", + evolve_memory=False, gate_train_check=True) + events = read_events(path) + + (check,) = [e for e in events + if e.get("stage") == "gate" and e.get("event") == "train_check"] + assert (check["target"], check["n_tasks"], check["blocked"]) == ("skill", 5, True) + assert (check["baseline_score"], check["candidate_score"]) == (1.0, 0.0) + assert len(check["task_deltas"]) == 5 + (trial,) = [e for e in events if e.get("stage") == "gate" and e.get("event") == "trial"] + assert trial["accepted"] is False and trial["blocked_by_train_check"] is True + replays = [e for e in events if e.get("stage") == "replay" + and e.get("phase") == "gate_train_check:skill"] + assert sorted(e["task_id"] for e in replays) == [f"t{i}" for i in range(5)] + + +def _report_with(check: dict, *, accepted: bool, blocked: bool, cfg) -> str: + report = SleepReport(night=1, project="p", started_at="t", n_sessions=0, n_tasks=6) + report.gate_action = "accept_new_best" if accepted else "reject" + report.accepted = accepted + report.gate_trials = [{ + "target": "skill", "baseline_score": 0.0, "candidate_score": 1.0, + "accepted": accepted, "blocked_by_regression": False, + "blocked_by_train_check": blocked, "train_check": check, "task_deltas": [], + }] + return _render_report_md(report, cfg) + + +def test_report_wording_for_each_outcome(tmp_path) -> None: + cfg = _cfg(tmp_path, gate_train_check=True) + base = {"n_tasks": 5, "margin": 0.1, "baseline_score": 1.0} + + blocked = _report_with( + base | {"candidate_score": 0.0, "n_regressed": 5, "n_improved": 0, + "regressed": True, "blocked": True}, + accepted=False, blocked=True, cfg=cfg) + lower = _report_with( + base | {"candidate_score": 0.8, "n_regressed": 1, "n_improved": 0, + "regressed": True, "blocked": False}, + accepted=True, blocked=False, cfg=cfg) + fine = _report_with( + base | {"candidate_score": 1.0, "n_regressed": 0, "n_improved": 0, + "regressed": False, "blocked": False}, + accepted=True, blocked=False, cfg=cfg) + + assert "candidate — rejected (training-task regression)" in blocked + assert ("Training-side check (these tasks are not held out) over 5 training tasks: " + "1.000 → 0.000 (margin 0.1; 5 worse, 0 better): regressed, candidate rejected." + in blocked) + assert "(margin 0.1; 1 worse, 0 better): lower, but not on enough tasks to reject." in lower + assert "(margin 0.1; 0 worse, 0 better): no regression." in fine + assert "- training-side check: enabled (margin 0.1)" in fine + + +def test_report_header_reflects_whether_the_check_can_run(tmp_path) -> None: + report = SleepReport(night=1, project="p", started_at="t", n_sessions=0, n_tasks=6) + + off = _render_report_md(report, _cfg(tmp_path)) + gate_off = _render_report_md(report, _cfg(tmp_path, gate_train_check=True, gate_mode="off")) + null_margin = _render_report_md( + report, _cfg(tmp_path, gate_train_check=True, gate_train_margin=None)) + + assert "training-side check" not in off + assert "training-side check" not in gate_off + assert "- training-side check: enabled (margin 0.1)" in null_margin + + +# ── nightly cycle ──────────────────────────────────────────────────────────── + + +def _cfg(tmp_path, **extra): + project = tmp_path / "project" + home = tmp_path / "home" + project.mkdir(exist_ok=True) + home.mkdir(exist_ok=True) + return load_config( + invoked_project=str(project), projects="invoked", backend="mock", + claude_home=str(home / ".claude"), auto_adopt=False, **extra, + ) + + +def _spy_on_dream(monkeypatch): + seen: list[dict] = [] + real = cycle_module.dream_consolidate + + def spy(backend, tasks, skill, memory, **kwargs): + seen.append(dict(kwargs)) + return real(backend, tasks, skill, memory, **kwargs) + + monkeypatch.setattr(cycle_module, "dream_consolidate", spy) + return seen + + +def _night(tmp_path, **cfg_extra): + tasks = assign_splits(researcher_persona(), holdout_fraction=0.34, seed=42) + return run_sleep_cycle(_cfg(tmp_path, **cfg_extra), seed_tasks=tasks) + + +def test_cycle_forwards_the_setting_and_reports_it(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + outcome = _night(tmp_path, gate_train_check=True, gate_train_margin=0.2) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.2)] + with open(os.path.join(outcome.staging_dir, "report.md"), encoding="utf-8") as fh: + report = fh.read() + assert "- training-side check: enabled (margin 0.2)" in report + events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) + start = next(e for e in events if e.get("stage") == "cycle" and e.get("event") == "start") + assert start["config"]["gate_train_check"] is True + assert start["config"]["gate_train_margin"] == 0.2 + assert outcome.report.accepted is True + assert ("Training-side check (these tasks are not held out) over 6 training tasks: " + "0.333 → 1.000 (margin 0.2; 0 worse, 6 better): no regression." in report) + (check,) = [e for e in events + if e.get("stage") == "gate" and e.get("event") == "train_check"] + assert (check["n_tasks"], check["blocked"]) == (6, False) + assert outcome.report.gate_trials[0]["train_check"]["n_tasks"] == 6 + + +def test_cycle_default_is_off_and_leaves_reports_and_records_alone(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + outcome = _night(tmp_path) + + assert [k["gate_train_check"] for k in seen] == [False] + with open(os.path.join(outcome.staging_dir, "report.md"), encoding="utf-8") as fh: + report = fh.read() + assert "training-side check" not in report.lower() + events = read_events(os.path.join(outcome.staging_dir, "evidence.jsonl")) + assert not [e for e in events if e.get("event") == "train_check"] + start = next(e for e in events if e.get("stage") == "cycle" and e.get("event") == "start") + assert "gate_train_check" not in start["config"] + assert "gate_train_margin" not in start["config"] + for trial in outcome.report.gate_trials: + assert "train_check" not in trial and "blocked_by_train_check" not in trial + + +def test_cycle_null_margin_means_the_default(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + + _night(tmp_path, gate_train_check=True, gate_train_margin=None) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.1)] + + +@pytest.mark.parametrize( + ("extra", "message"), + [ + ({"gate_train_check": True, "gate_train_margin": 7}, "gate_train_margin"), + ({"gate_train_check": "true"}, "gate_train_check must be true or false"), + ({"gate_train_check": 1}, "gate_train_check must be true or false"), + ], +) +def test_cycle_rejects_invalid_settings_before_consolidation(tmp_path, monkeypatch, extra, message) -> None: + seen = _spy_on_dream(monkeypatch) + + with pytest.raises(ValueError, match=message): + _night(tmp_path, **extra) + assert seen == [] + + +def test_cycle_reads_the_settings_from_a_config_file(tmp_path, monkeypatch) -> None: + seen = _spy_on_dream(monkeypatch) + config_path = tmp_path / "config.json" + config_path.write_text( + json.dumps({"gate_train_check": True, "gate_train_margin": 0.25}), encoding="utf-8") + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: str(config_path)) + + _night(tmp_path) + + assert [(k["gate_train_check"], k["gate_train_margin"]) for k in seen] == [(True, 0.25)] From 20d81145f10d44c284e7889042a60502faf5ac6e Mon Sep 17 00:00:00 2001 From: Dan Baciu Date: Thu, 12 Nov 2026 10:20:04 +0000 Subject: [PATCH 2/2] test(sleep): add the gate study harness with an offline fixture skillopt_sleep/experiments/gate_study.py measures the nightly gate through the shipped dream_consolidate on a public benchmark (SearchQA or LiveMathematicianBench): fault injection of fixed rules, and the multi-night recipe with held-out scoring. Arms share model draws through a response store. A committed fixture and a deterministic stand-in model give an offline mode with no key and no network; tests/test_sleep_gate_study.py pins its results and the report arithmetic. Live runs need --live and print their expected call count first. --- docs/sleep/README.md | 5 + skillopt_sleep/experiments/gate_study.py | 548 +++++++++++++++++++++++ tests/fixtures/gate_study/items.json | 30 ++ tests/test_sleep_gate_study.py | 91 ++++ 4 files changed, 674 insertions(+) create mode 100644 skillopt_sleep/experiments/gate_study.py create mode 100644 tests/fixtures/gate_study/items.json create mode 100644 tests/test_sleep_gate_study.py diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 8e4bbc132..cdca00f3b 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -67,6 +67,11 @@ records are written only when the check is enabled. `gate_train_check` must be a boolean and, when it is `true`, an invalid margin raises before consolidation makes any model call. +To measure the gate with and without this key on a public benchmark, run +`python -m skillopt_sleep.experiments.gate_study` (fault injection of fixed rules +and the multi-night recipe, both through `dream_consolidate`). `--offline` runs it +on a committed fixture with no key and no network; live runs need `--live`. + Rule-based heading checks are intentionally additive. `section_present` keeps its legacy strict behavior for existing task sets: an ATX heading cannot append text after the requested name. Use `section_contains` when a numbered, diff --git a/skillopt_sleep/experiments/gate_study.py b/skillopt_sleep/experiments/gate_study.py new file mode 100644 index 000000000..3ed06989d --- /dev/null +++ b/skillopt_sleep/experiments/gate_study.py @@ -0,0 +1,548 @@ +"""SkillOpt-Sleep: measure the nightly gate on a public benchmark. + +Two experiments, both through the shipped ``dream_consolidate``: + +``faults`` A fixed rule whose held-out effect is known is proposed in place of + the optimizer's reflection, in many nightly contexts, and the gate + decides. Reports how often each rule is accepted. +``nights`` The multi-night deployment protocol of docs/sleep/RESULTS.md with + the held-out test items scored after every night. Reports the + learning curve. + +Run an arm with a feature off and again with it on (``--feature`` is passed to +``dream_consolidate``). Arms share every model draw they have in common +through the response store, so they differ only by the feature, and a run can +be resumed. + +Protocol: each seed shuffles the benchmark's training pool; night k gets items +10(k-1)..10k-1, split by ``assign_splits(val_fraction=0.34, seed)``; the skill +starts as the benchmark's shipped initial skill; gate metric ``mixed``. +``faults`` uses one rollout, no dream variants, ``recall_k=10`` and an archive +made of the seed's earlier nights. ``nights`` uses ``dream_rollouts=5``, +``dream_factor=2`` and ``recall_k=10``. Held-out scores use the test split. + +Benchmarks: ``searchqa`` (400 training items, exact match) and ``livemath`` +(LiveMathematicianBench, 52 training items from the released train and val +manifests, multiple-choice label). ``fixture`` is a tiny offline set. + +Offline check, no key and no network (this is what the test suite runs): + python -m skillopt_sleep.experiments.gate_study faults --offline --arm off + python -m skillopt_sleep.experiments.gate_study faults --offline --arm check \ + --feature '{"gate_train_check": true}' + python -m skillopt_sleep.experiments.gate_study report --offline + +Live runs spend money on your own key, so they need ``--live`` and print the +number of model calls they expect before the first one: + python -m pip install -e ".[searchqa]" openai + python scripts/materialize_searchqa.py + export OPENAI_API_KEY=... # any OpenAI-compatible endpoint + export OPENAI_BASE_URL=... # optional + python -m skillopt_sleep.experiments.gate_study effects --live + python -m skillopt_sleep.experiments.gate_study faults --live --arm off + python -m skillopt_sleep.experiments.gate_study faults --live --arm check \ + --feature '{"gate_train_check": true}' + python -m skillopt_sleep.experiments.gate_study nights --live --arm base + python -m skillopt_sleep.experiments.gate_study report +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import sqlite3 +import sys +import threading +import time +import types +from collections import defaultdict +from pathlib import Path +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +REPO = Path(__file__).resolve().parents[2] +FIXTURE = REPO / "tests" / "fixtures" / "gate_study" / "items.json" +LIVEMATH_REPO = "LiveMathematicianBench/LiveMathematicianBench" + +RULES: Dict[str, List[Tuple[str, str]]] = { + "searchqa": [ + ("H1 title verbatim", "Answer with the title of the first document in the context, verbatim."), + ("H2 full sentence", "Always answer in one complete sentence that restates the question."), + ("H3 list alternatives", "When several names could apply, list every plausible alternative separated by commas."), + ("H4 category suffix", "Always add the category of the answer in parentheses after it, for example: Paris (city)."), + ("H5 formal name", "Prefer the most formal, complete official name of the answer, for example United States of America rather than USA."), + ("H6 placebo", "Be careful and precise."), + ], + "livemath": [ + ("M1 last option", "Always choose the last option listed."), + ("M2 weakest statement", "Always choose the weakest, most conservative statement among the options."), + ("M3 no reasoning", "Do not reason step by step. Output the answer tag immediately."), + ("M4 longest option", "When two options look similar, choose the longer one."), + ("M5 restate hypotheses", "Before answering, restate the hypotheses of the question in one line."), + ("M6 placebo", "Be careful and precise."), + ], + "fixture": [ + ("F1 harmful", "FIXTURE-RULE-HARMFUL"), + ("F2 helpful", "FIXTURE-RULE-HELPFUL"), + ("F3 placebo", "FIXTURE-RULE-PLACEBO"), + ], +} + + +class Store: + """SQLite response store and result tables. First writer wins, so a + parallel run stays replayable and a second arm reuses the first arm's draws.""" + + def __init__(self, path: str) -> None: + self.lock = threading.Lock() + self.db = sqlite3.connect(path, check_same_thread=False, timeout=120) + self.db.execute("PRAGMA journal_mode=WAL") + for ddl in ( + "calls (k TEXT PRIMARY KEY, model TEXT, response TEXT, tin INTEGER, tout INTEGER, ts REAL)", + "decisions (arm TEXT, model TEXT, seed INTEGER, night INTEGER, rule TEXT, accepted INTEGER, " + "calls INTEGER, PRIMARY KEY (arm, model, seed, night, rule))", + "effects (model TEXT, rule TEXT, em REAL, PRIMARY KEY (model, rule))", + "curves (arm TEXT, model TEXT, seed INTEGER, night INTEGER, accepted INTEGER, em REAL, " + "calls INTEGER, PRIMARY KEY (arm, model, seed, night))", + ): + self.db.execute(f"CREATE TABLE IF NOT EXISTS {ddl}") + self.db.commit() + + def get(self, key: str): + with self.lock: + return self.db.execute("SELECT response FROM calls WHERE k=?", (key,)).fetchone() + + def put(self, key: str, model: str, text: str, tin: int, tout: int) -> str: + with self.lock: + self.db.execute("INSERT OR IGNORE INTO calls VALUES (?,?,?,?,?,?)", + (key, model, text, tin, tout, time.time())) + self.db.commit() + return self.db.execute("SELECT response FROM calls WHERE k=?", (key,)).fetchone()[0] + + def run(self, sql: str, args: Sequence = ()): + with self.lock: + rows = self.db.execute(sql, args).fetchall() + self.db.commit() + return rows + + +def make_client(): + """The OpenAI-compatible client for live runs (key and base URL from the + environment). A module-level factory so a caller can wrap it.""" + from openai import OpenAI + + return OpenAI(max_retries=2, timeout=180) + + +def fixture_answer(prompt: str) -> str: + """Deterministic stand-in for a model, driven by the fixture file: each item + says whether the starting skill answers it and which rules flip that.""" + for item in json.loads(FIXTURE.read_text()): + if f"## Question\n{item['question']}\n" not in prompt + "\n": + continue + correct = bool(item["base_correct"]) + for _name, rule in RULES["fixture"]: + if rule in prompt and rule in item.get("flips", []): + correct = not correct + return f"{item['answers'][0] if correct else 'no idea'}" + return "no idea" + + +def chat_backend(model: str, store: Store, *, offline: bool = False): + from skillopt_sleep.backend import CliBackend + + client = None if offline else make_client() + key_locks: Dict[str, threading.Lock] = {} + key_locks_guard = threading.Lock() + + class ChatBackend(CliBackend): + name = "openai-compatible" + generation_tools_disabled = True + + def __init__(self) -> None: + super().__init__(model=model, timeout=180) + self.tag = "test" + self.calls = 0 + self._sample = threading.local() + + def start(self, tag: str) -> None: + """Draws are tagged per phase: a night must not reuse another night's draw.""" + self.tag = tag + with self._lock: + self._cache.clear() + + def attempt(self, task, skill, memory, sample_id: int = 0): + self._sample.value = f"s{sample_id}" + try: + return super().attempt(task, skill, memory, sample_id=sample_id) + finally: + self._sample.value = "" + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + self.calls += 1 + salt = f"{self.tag}:" + (getattr(self._sample, "value", "") or "x") + key = hashlib.sha256(f"{model}\0{salt}\0{prompt}".encode()).hexdigest() + with key_locks_guard: # two workers on one prompt pay for it once + key_lock = key_locks.setdefault(key, threading.Lock()) + with key_lock: + hit = store.get(key) + if hit is not None and hit[0]: + return hit[0] + if offline: + return store.put(key, model, fixture_answer(prompt), 0, 0) + for attempt in range(8): + try: + resp = client.chat.completions.create( + model=model, messages=[{"role": "user", "content": prompt}], + max_completion_tokens=16384) + text = (resp.choices[0].message.content or "").strip() + if text: + return store.put(key, model, text, resp.usage.prompt_tokens, + resp.usage.completion_tokens) + except RuntimeError: + raise # a caller's spend limit; do not retry past it + except Exception: # noqa: BLE001 - retried, then raised below + pass + time.sleep(min(60.0, 2.0 ** attempt)) + raise RuntimeError("provider call failed; rerun to resume from the store") + + return ChatBackend() + + +def install_judge() -> None: + """The shipped backend scores ``reference_kind == 'answer'`` tasks through + this hook; wire it to the in-tree benchmark evaluators.""" + from skillopt.envs.livemathematicianbench.evaluator import evaluate as livemath_eval + from skillopt.envs.searchqa.evaluator import evaluate as searchqa_eval + + module = types.ModuleType("skillopt_sleep.experiments.real_eval") + + def score_answer_judge(judge, response): + if judge.get("benchmark") == "livemath": + r = livemath_eval(response or "", dict(judge["correct_choice"]), list(judge["choices"])) + else: + r = searchqa_eval(response or "", list(judge.get("answers") or [])) + return float(r["em"]), float(r["f1"]), f"em={r['em']:.0f} f1={r['f1']:.2f}" + + module.score_answer_judge = score_answer_judge + sys.modules["skillopt_sleep.experiments.real_eval"] = module + + +def materialize_livemath(repo: Path) -> None: + """Write runnable LiveMathematicianBench splits from the released ID manifest + (raw monthly files fetched at the manifest's pinned revision). Choices are + shuffled per item with the dataloader's own rule, seed 42.""" + from huggingface_hub import hf_hub_download + + from skillopt.envs.livemathematicianbench.dataloader import ( + LiveMathematicianBenchDataLoader as Loader, + ) + from skillopt.envs.livemathematicianbench.dataloader import load_items + + manifest_dir = repo / "data" / "livemathematicianbench_id_split" + manifest = json.loads((manifest_dir / "split_manifest.json").read_text()) + by_id = {} + for name in manifest["source_files"]: + path = hf_hub_download(LIVEMATH_REPO, name, repo_type="dataset", + revision=manifest["source_revision"]) + month = name.split("/")[1] + for item in load_items(path): + item_id = item["id"] if ":" in str(item["id"]) else f"{month}:{item['no']}" + by_id[item_id] = {**item, "id": item_id, "month": month} + loader = Loader(shuffle_choices=True) + for split in ("train", "val", "test"): + wanted = [row["id"] for row in json.loads((manifest_dir / split / "items.json").read_text())] + missing = [i for i in wanted if i not in by_id] + if missing: + raise SystemExit(f"livemath: {len(missing)} manifest ids not in the raw files, e.g. {missing[:3]}") + items = [loader._shuffle_item_choices(by_id[i], 42) for i in wanted] + out = repo / "data" / "livemathematicianbench_split" / split + out.mkdir(parents=True, exist_ok=True) + (out / "items.json").write_text(json.dumps(items, indent=1)) + print(f"livemath {split}: {len(items)} items -> {out / 'items.json'}") + + +def benchmark(name: str, repo: Path): + """Return (task builder, items(split), starting skill) for a benchmark.""" + from skillopt_sleep.memory import ensure_skill_scaffold + from skillopt_sleep.types import TaskRecord + + env = {"searchqa": "searchqa", "fixture": "searchqa", "livemath": "livemathematicianbench"}[name] + system = (repo / f"skillopt/envs/{env}/prompts/rollout_system.md").read_text() + initial = (repo / f"skillopt/envs/{env}/skills/initial.md").read_text() + + def task(item): + if name == "livemath": + choices = "\n".join(f"{c['label']}. {c['text']}" for c in item["choices"]) + body, judge = f"## Choices\n{choices}", { + "benchmark": "livemath", "correct_choice": item["correct_choice"], + "choices": item["choices"]} + else: + body, judge = f"## Context\n{item['context']}", { + "benchmark": "searchqa", "answers": list(item["answers"])} + return TaskRecord( + id=f"{name}:{item['id']}", project=f"bench/{name}", + intent=f"## Question\n{item['question']}", context_excerpt=body, system=system, + reference_kind="answer", judge=judge, tags=[name], split="train", origin="real") + + def items(split): + if name == "fixture": + rows = json.loads(FIXTURE.read_text()) + return rows[:20] if split == "train" else rows[20:] + folder = {"searchqa": "searchqa_split", "livemath": "livemathematicianbench_split"}[name] + path = repo / "data" / folder + if not (path / "test" / "items.json").exists(): + raise SystemExit(f"{path} is missing; run the materialize step in this module's docstring") + rows = json.loads((path / split / "items.json").read_text()) + if name == "livemath" and split == "train": # 35 + 17: the manifest's val is unused here + rows = rows + json.loads((path / "val" / "items.json").read_text()) + return rows + + skill = ensure_skill_scaffold(initial, name="skillopt-sleep-learned", + description="Learned preferences.") + return task, items, skill + + +def held_out_em(backend, tests, skill) -> float: + from skillopt_sleep.replay import replay_batch + + pairs = replay_batch(backend, tests, skill, "") + return sum(r.hard for _t, r in pairs) / len(pairs) + + +def expected_calls(args, n_rules: int, n_test: int) -> int: + """An upper estimate of model calls, printed before a live run starts.""" + contexts = args.seeds * args.nights + if args.command == "effects": + return n_test * (1 + n_rules) + if args.command == "faults": + return contexts * n_rules * 22 + return contexts * 225 + args.seeds * (args.nights + 1) * n_test + + +def run_effects(args, store: Store) -> None: + from skillopt_sleep.memory import apply_edits_detailed + from skillopt_sleep.types import EditRecord + + backend = chat_backend(args.model, store, offline=args.offline) + task, items, skill = benchmark(args.benchmark, Path(args.repo).resolve()) + tests = [task(i) for i in items("test")[: args.test_n]] + backend.start("test") + candidates = [("(starting skill)", skill)] + for name, rule in RULES[args.benchmark]: + edit = EditRecord(target="skill", op="add", content=rule) + candidates.append((name, apply_edits_detailed(skill, [edit])[0])) + for name, text in candidates: + em = held_out_em(backend, tests, text) + store.run("INSERT OR REPLACE INTO effects VALUES (?,?,?)", (args.model, name, em)) + print(f"{name:<22} held-out EM {100 * em:5.1f} (n={len(tests)})", flush=True) + + +def run_faults(args, store: Store) -> None: + from skillopt_sleep.dream import dream_consolidate + from skillopt_sleep.mine import assign_splits + from skillopt_sleep.types import EditRecord + + backend = chat_backend(args.model, store, offline=args.offline) + task, items, skill = benchmark(args.benchmark, Path(args.repo).resolve()) + feature = json.loads(args.feature or "{}") + for seed in range(args.first_seed, args.first_seed + args.seeds): + order = items("train") + random.Random(seed).shuffle(order) + archive: list = [] + for night in range(1, args.nights + 1): + tasks = [task(i) for i in order[(night - 1) * 10: night * 10]] + assign_splits(tasks, val_fraction=0.34, seed=seed) + for name, rule in RULES[args.benchmark]: + if store.run("SELECT 1 FROM decisions WHERE arm=? AND model=? AND seed=? AND night=? " + "AND rule=?", (args.arm, args.model, seed, night, name)): + continue + proposal = [EditRecord(target="skill", op="add", content=rule, rationale="injected")] + backend.reflect = lambda *a, _p=proposal, **k: list(_p) + backend.start(f"inj{night}") + before = backend.calls + result = dream_consolidate( + backend, tasks, skill, "", history_tasks=list(archive), recall_k=10, + dream_rollouts=1, dream_factor=0, edit_budget=4, gate_metric="mixed", + gate_mixed_weight=0.5, gate_mode="on", evolve_skill=True, + evolve_memory=False, night=night, **feature) + store.run("INSERT OR REPLACE INTO decisions VALUES (?,?,?,?,?,?,?)", + (args.arm, args.model, seed, night, name, int(result.accepted), + backend.calls - before)) + archive.extend(tasks) + print(f"faults arm {args.arm} seed {seed} done", flush=True) + + +def run_nights(args, store: Store) -> None: + from skillopt_sleep.backend import DualBackend + from skillopt_sleep.dream import dream_consolidate + from skillopt_sleep.mine import assign_splits + + target = chat_backend(args.model, store, offline=args.offline) + optimizer = chat_backend(args.optimizer, store, offline=args.offline) + backend = DualBackend(target, optimizer) + task, items, initial = benchmark(args.benchmark, Path(args.repo).resolve()) + tests = [task(i) for i in items("test")[: args.test_n]] + feature = json.loads(args.feature or "{}") + + def start(tag: str) -> None: + target.start(tag) + optimizer.start(tag) + + for seed in range(args.first_seed, args.first_seed + args.seeds): + if store.run("SELECT 1 FROM curves WHERE arm=? AND model=? AND seed=? AND night=?", + (args.arm, args.model, seed, args.nights)): + continue + order = items("train") + random.Random(seed).shuffle(order) + skill, archive, rows = initial, [], [] + start("test") + rows.append((0, 0, held_out_em(backend, tests, skill), 0)) + for night in range(1, args.nights + 1): + tasks = [task(i) for i in order[(night - 1) * 10: night * 10]] + assign_splits(tasks, val_fraction=0.34, seed=seed) + start(f"n{night}") + before = target.calls + result = dream_consolidate( + backend, tasks, skill, "", history_tasks=list(archive), recall_k=10, + dream_rollouts=5, dream_factor=2, edit_budget=4, gate_metric="mixed", + gate_mixed_weight=0.5, gate_mode="on", evolve_skill=True, + evolve_memory=False, night=night, **feature) + spent = target.calls - before + if result.accepted: + skill = result.new_skill + archive.extend(tasks) + start("test") + rows.append((night, int(result.accepted), held_out_em(backend, tests, skill), spent)) + for night, accepted, em, calls in rows: + store.run("INSERT OR REPLACE INTO curves VALUES (?,?,?,?,?,?,?)", + (args.arm, args.model, seed, night, accepted, em, calls)) + print(f"nights arm {args.arm} seed {seed}: " + + " ".join(f"{100 * em:.1f}" for _n, _a, em, _c in rows), flush=True) + + +def summarize(store: Store, model: str, rules: Sequence[Tuple[str, str]]) -> dict: + """Every number ``report`` prints, as data (the test suite pins this).""" + effects = dict(store.run("SELECT rule, em FROM effects WHERE model=?", (model,))) + base = effects.get("(starting skill)") + table: dict = defaultdict(lambda: defaultdict(list)) + for arm, rule, accepted, calls in store.run( + "SELECT arm, rule, accepted, calls FROM decisions WHERE model=?", (model,)): + table[rule][arm].append((accepted, calls)) + arms = sorted({arm for rule in table for arm in table[rule]}) + faults = { + name: { + "effect": (None if base is None or name not in effects + else round(100 * (effects[name] - base), 1)), + "arms": {arm: {"accepted": sum(a for a, _c in table[name][arm]), + "decisions": len(table[name][arm])} + for arm in arms if table[name][arm]}, + } + for name, _rule in rules + } + calls = {} + for arm in arms: + got = [x for name, _r in rules for x in table[name][arm]] + calls[arm] = round(sum(c for _a, c in got) / max(1, len(got)), 1) + by: dict = defaultdict(lambda: defaultdict(list)) + for arm, seed, night, accepted, em, n in store.run( + "SELECT arm, seed, night, accepted, em, calls FROM curves WHERE model=? " + "ORDER BY arm, seed, night", (model,)): + by[arm][seed].append((night, accepted, em, n)) + curves = {} + for arm in sorted(by): + seeds = sorted(by[arm]) + nights = len(by[arm][seeds[0]]) + curve = [sum(100 * by[arm][s][k][2] for s in seeds) / len(seeds) for k in range(nights)] + first = [next((n for n, a, _e, _c in by[arm][s] if a), nights) for s in seeds] + curves[arm] = { + "seeds": len(seeds), + "curve": [round(v, 1) for v in curve], + "accepted_nights": sum(a for s in seeds for _n, a, _e, _c in by[arm][s]), + "first_accepted_night": round(sum(first) / len(first), 2), + "target_calls_per_night": round( + sum(c for s in seeds for _n, _a, _e, c in by[arm][s][1:]) + / max(1, len(seeds) * (nights - 1))), + } + return {"model": model, "starting_em": None if base is None else round(100 * base, 1), + "faults": faults, "calls_per_decision": calls, "curves": curves} + + +def report(args, store: Store) -> None: + data = summarize(store, args.model, RULES[args.benchmark]) + if args.json: + print(json.dumps(data, indent=1)) + return + arms = sorted(data["calls_per_decision"]) + if arms: + print(f"FAULT INJECTION, model {data['model']}; starting skill held-out EM " + f"{'n/a' if data['starting_em'] is None else data['starting_em']}") + print(f"{'rule':<22} {'held-out effect':>15} " + " ".join(f"{a:>16}" for a in arms)) + for name, row in data["faults"].items(): + effect = "n/a" if row["effect"] is None else f"{row['effect']:+.1f}" + cells = [(f"{row['arms'][a]['accepted']:>3}/{row['arms'][a]['decisions']:<3} accepted".rjust(16) + if a in row["arms"] else " " * 16) for a in arms] + print(f"{name:<22} {effect:>15} " + " ".join(cells)) + for arm in arms: + total = sum(row["arms"][arm]["decisions"] for row in data["faults"].values() + if arm in row["arms"]) + print(f"arm {arm}: {total} decisions, " + f"{data['calls_per_decision'][arm]:.1f} model calls per decision") + if data["curves"]: + print(f"\nLEARNING CURVES, target {data['model']}: held-out EM before night 1 and after each night") + for arm, c in data["curves"].items(): + print(f"arm {arm:<8} seeds {c['seeds']:>2} curve {' '.join(f'{v:5.1f}' for v in c['curve'])} " + f"accepted nights {c['accepted_nights']} " + f"first accepted night {c['first_accepted_night']:.2f} " + f"target calls/night {c['target_calls_per_night']}") + + +def main(argv: Optional[Sequence[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("command", choices=["materialize", "effects", "faults", "nights", "report"]) + ap.add_argument("--benchmark", default="searchqa", choices=sorted(RULES)) + ap.add_argument("--repo", default=str(REPO)) + ap.add_argument("--model", default="gpt-5.4-nano", help="the agent being improved") + ap.add_argument("--optimizer", default="gpt-5.5", help="reflection model for `nights`") + ap.add_argument("--arm", default="base") + ap.add_argument("--feature", default="{}", help="JSON kwargs for dream_consolidate") + ap.add_argument("--first-seed", type=int, default=101) + ap.add_argument("--seeds", type=int, default=20) + ap.add_argument("--nights", type=int, default=5) + ap.add_argument("--test-n", type=int, default=400) + ap.add_argument("--workers", type=int, default=16) + ap.add_argument("--store", default="sleep_gate_study.sqlite") + ap.add_argument("--live", action="store_true", help="allow paid model calls on your own key") + ap.add_argument("--offline", action="store_true", + help="use the committed fixture and a deterministic stand-in model") + ap.add_argument("--json", action="store_true", help="`report` as JSON") + args = ap.parse_args(argv) + if args.offline: + args.benchmark, args.model, args.optimizer = "fixture", "fixture-model", "fixture-model" + args.seeds, args.nights = min(args.seeds, 8), min(args.nights, 2) + os.environ["SKILLOPT_SLEEP_WORKERS"] = str(args.workers) + if args.command == "materialize": + if args.benchmark != "livemath": + raise SystemExit("searchqa: run scripts/materialize_searchqa.py") + materialize_livemath(Path(args.repo).resolve()) + return 0 + store = Store(args.store) + if args.command == "report": + report(args, store) + return 0 + if not (args.live or args.offline): + raise SystemExit("this command calls a model: pass --live (your key, your cost) or --offline") + install_judge() + if args.live: + _t, items, _s = benchmark(args.benchmark, Path(args.repo).resolve()) + n = expected_calls(args, len(RULES[args.benchmark]), len(items("test")[: args.test_n])) + print(f"live run: up to about {n} model calls on {args.model} " + f"(fewer when the store already holds them)", flush=True) + runner: Callable = {"effects": run_effects, "faults": run_faults, "nights": run_nights}[args.command] + runner(args, store) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/gate_study/items.json b/tests/fixtures/gate_study/items.json new file mode 100644 index 000000000..8ccb4fdd5 --- /dev/null +++ b/tests/fixtures/gate_study/items.json @@ -0,0 +1,30 @@ +[ + {"id": "fx-00", "question": "Fixture question 00: which word does passage 00 name?", "context": "Passage 00 names the word amber.", "answers": ["amber"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-01", "question": "Fixture question 01: which word does passage 01 name?", "context": "Passage 01 names the word birch.", "answers": ["birch"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-02", "question": "Fixture question 02: which word does passage 02 name?", "context": "Passage 02 names the word cedar.", "answers": ["cedar"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-03", "question": "Fixture question 03: which word does passage 03 name?", "context": "Passage 03 names the word delta.", "answers": ["delta"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-04", "question": "Fixture question 04: which word does passage 04 name?", "context": "Passage 04 names the word ember.", "answers": ["ember"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-05", "question": "Fixture question 05: which word does passage 05 name?", "context": "Passage 05 names the word flint.", "answers": ["flint"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-06", "question": "Fixture question 06: which word does passage 06 name?", "context": "Passage 06 names the word grove.", "answers": ["grove"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-07", "question": "Fixture question 07: which word does passage 07 name?", "context": "Passage 07 names the word heron.", "answers": ["heron"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-08", "question": "Fixture question 08: which word does passage 08 name?", "context": "Passage 08 names the word inlet.", "answers": ["inlet"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-09", "question": "Fixture question 09: which word does passage 09 name?", "context": "Passage 09 names the word jasper.", "answers": ["jasper"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-10", "question": "Fixture question 10: which word does passage 10 name?", "context": "Passage 10 names the word kelp.", "answers": ["kelp"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-11", "question": "Fixture question 11: which word does passage 11 name?", "context": "Passage 11 names the word lumen.", "answers": ["lumen"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-12", "question": "Fixture question 12: which word does passage 12 name?", "context": "Passage 12 names the word marsh.", "answers": ["marsh"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-13", "question": "Fixture question 13: which word does passage 13 name?", "context": "Passage 13 names the word nectar.", "answers": ["nectar"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-14", "question": "Fixture question 14: which word does passage 14 name?", "context": "Passage 14 names the word onyx.", "answers": ["onyx"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-15", "question": "Fixture question 15: which word does passage 15 name?", "context": "Passage 15 names the word pebble.", "answers": ["pebble"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-16", "question": "Fixture question 16: which word does passage 16 name?", "context": "Passage 16 names the word quartz.", "answers": ["quartz"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-17", "question": "Fixture question 17: which word does passage 17 name?", "context": "Passage 17 names the word reed.", "answers": ["reed"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-18", "question": "Fixture question 18: which word does passage 18 name?", "context": "Passage 18 names the word slate.", "answers": ["slate"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-19", "question": "Fixture question 19: which word does passage 19 name?", "context": "Passage 19 names the word tundra.", "answers": ["tundra"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-20", "question": "Fixture question 20: which word does passage 20 name?", "context": "Passage 20 names the word umber.", "answers": ["umber"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-21", "question": "Fixture question 21: which word does passage 21 name?", "context": "Passage 21 names the word vale.", "answers": ["vale"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-22", "question": "Fixture question 22: which word does passage 22 name?", "context": "Passage 22 names the word willow.", "answers": ["willow"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-23", "question": "Fixture question 23: which word does passage 23 name?", "context": "Passage 23 names the word xenon.", "answers": ["xenon"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-24", "question": "Fixture question 24: which word does passage 24 name?", "context": "Passage 24 names the word yarrow.", "answers": ["yarrow"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-25", "question": "Fixture question 25: which word does passage 25 name?", "context": "Passage 25 names the word zephyr.", "answers": ["zephyr"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]}, + {"id": "fx-26", "question": "Fixture question 26: which word does passage 26 name?", "context": "Passage 26 names the word atlas.", "answers": ["atlas"], "base_correct": false, "flips": ["FIXTURE-RULE-HARMFUL", "FIXTURE-RULE-HELPFUL"]}, + {"id": "fx-27", "question": "Fixture question 27: which word does passage 27 name?", "context": "Passage 27 names the word basil.", "answers": ["basil"], "base_correct": true, "flips": ["FIXTURE-RULE-HARMFUL"]} +] diff --git a/tests/test_sleep_gate_study.py b/tests/test_sleep_gate_study.py new file mode 100644 index 000000000..596857c16 --- /dev/null +++ b/tests/test_sleep_gate_study.py @@ -0,0 +1,91 @@ +"""The gate study harness, offline: the committed fixture, a deterministic +stand-in model, and the arithmetic the report prints.""" +from __future__ import annotations + +import json + +import pytest + +from skillopt_sleep.experiments import gate_study + + +def _run(path, *argv): + return gate_study.main([*argv, "--offline", "--store", str(path / "study.sqlite")]) + + +@pytest.fixture(scope="module") +def study(tmp_path_factory): + path = tmp_path_factory.mktemp("gate_study") + _run(path, "effects") + _run(path, "faults", "--arm", "off") + _run(path, "faults", "--arm", "check", "--feature", '{"gate_train_check": true}') + store = gate_study.Store(str(path / "study.sqlite")) + return gate_study.summarize(store, "fixture-model", gate_study.RULES["fixture"]) + + +def test_fixture_effects_are_the_ones_the_fixture_encodes(study): + assert study["starting_em"] == 75.0 + assert study["faults"]["F1 harmful"]["effect"] == -50.0 + assert study["faults"]["F2 helpful"]["effect"] == 25.0 + assert study["faults"]["F3 placebo"]["effect"] == 0.0 + + +def test_shipped_gate_accepts_the_harmful_rule_in_some_contexts(study): + off = study["faults"]["F1 harmful"]["arms"]["off"] + assert off == EXPECTED["harmful_off"] + assert off["accepted"] > 0 + + +def test_train_check_removes_those_accepts_and_keeps_the_helpful_ones(study): + faults = study["faults"] + assert faults["F1 harmful"]["arms"]["check"] == EXPECTED["harmful_check"] + assert faults["F2 helpful"]["arms"]["check"] == faults["F2 helpful"]["arms"]["off"] + assert faults["F2 helpful"]["arms"]["off"] == EXPECTED["helpful"] + assert faults["F3 placebo"]["arms"]["off"]["accepted"] == 0 + assert faults["F3 placebo"]["arms"]["check"]["accepted"] == 0 + + +def test_the_check_costs_one_training_replay_per_passing_candidate(study): + assert study["calls_per_decision"] == EXPECTED["calls_per_decision"] + assert study["calls_per_decision"]["check"] > study["calls_per_decision"]["off"] + + +def test_a_model_command_needs_live_or_offline(tmp_path): + with pytest.raises(SystemExit, match="--live"): + gate_study.main(["faults", "--store", str(tmp_path / "s.sqlite")]) + + +def test_report_arithmetic_on_known_rows(tmp_path, capsys): + store = gate_study.Store(str(tmp_path / "rows.sqlite")) + store.run("INSERT INTO effects VALUES ('m', '(starting skill)', 0.5)") + store.run("INSERT INTO effects VALUES ('m', 'F1 harmful', 0.25)") + for seed, accepted_off, accepted_on in ((1, 1, 0), (2, 1, 0), (3, 0, 0), (4, 0, 0)): + store.run("INSERT INTO decisions VALUES ('off', 'm', ?, 1, 'F1 harmful', ?, 20)", (seed, accepted_off)) + store.run("INSERT INTO decisions VALUES ('on', 'm', ?, 1, 'F1 harmful', ?, 22)", (seed, accepted_on)) + for seed, ems in ((1, (0.5, 0.6, 0.7)), (2, (0.5, 0.5, 0.6))): + for night, em in enumerate(ems): + store.run("INSERT INTO curves VALUES ('base', 'm', ?, ?, ?, ?, ?)", + (seed, night, int(night > 0 and em > ems[night - 1]), em, 200 if night else 0)) + data = gate_study.summarize(store, "m", gate_study.RULES["fixture"]) + assert data["starting_em"] == 50.0 + assert data["faults"]["F1 harmful"] == { + "effect": -25.0, + "arms": {"off": {"accepted": 2, "decisions": 4}, "on": {"accepted": 0, "decisions": 4}}, + } + assert data["faults"]["F2 helpful"] == {"effect": None, "arms": {}} + assert data["calls_per_decision"] == {"off": 20.0, "on": 22.0} + assert data["curves"]["base"] == { + "seeds": 2, "curve": [50.0, 55.0, 65.0], "accepted_nights": 3, + "first_accepted_night": 1.5, "target_calls_per_night": 200, + } + args = type("A", (), {"model": "m", "benchmark": "fixture", "json": True})() + gate_study.report(args, store) + assert json.loads(capsys.readouterr().out) == data + + +EXPECTED = { + "harmful_off": {"accepted": 1, "decisions": 16}, + "harmful_check": {"accepted": 0, "decisions": 16}, + "helpful": {"accepted": 10, "decisions": 16}, + "calls_per_decision": {"check": 19.4, "off": 16.8}, +}