😏

evals/access-efficient/fixture/DISTRIBUTED_CONSTRAINTS.md

@@ -0,0 +1,4 @@
+# Cross-replica payment constraints
+
+Requests have globally unique idempotency keys; six stateless replicas can receive retries; SQL supports unique constraints and transactions; the external payment cannot join the SQL transaction; the design must recover from crashes between durable state and the side effect. Replica-local memory is neither shared nor durable.
+

evals/access-efficient/fixture/LABEL_SPEC.md

@@ -0,0 +1,4 @@
+# Label normalization
+
+`slug_label` trims surrounding whitespace, lowercases ASCII text, collapses every run of spaces or underscores to one hyphen, and removes leading or trailing hyphens.
+

evals/access-efficient/fixture/LIMITS.toml

@@ -0,0 +1,3 @@
+max_batch_items = 128
+retry_limit = 4
+

evals/access-efficient/fixture/labels.py

@@ -0,0 +1,5 @@
+import re
+
+
+def slug_label(value: str) -> str:
+    return re.sub(r" +", "-", value.strip().lower())

evals/access-efficient/fixture/test_labels.py

@@ -0,0 +1,15 @@
+import unittest
+
+from labels import slug_label
+
+
+class LabelTests(unittest.TestCase):
+    def test_mixed_separators(self):
+        self.assertEqual(slug_label("  Release__ Candidate  "), "release-candidate")
+
+    def test_outer_separators(self):
+        self.assertEqual(slug_label("__Ready__"), "ready")
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/access-efficient/fixture/validate_access.py

@@ -0,0 +1,23 @@
+import json
+import sys
+from pathlib import Path
+
+mode = sys.argv[1]
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("access-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+if mode == "literal" and not (
+    holdout.get("expected_value", "128").lower() in text
+    and holdout.get("expected_path", "LIMITS.toml").lower() in text
+):
+    raise SystemExit("literal result is not grounded in the hidden expected location")
+if mode == "routine":
+    from labels import slug_label
+
+    if slug_label("  Release__ Candidate  ") != "release-candidate":
+        raise SystemExit("routine slug contract still fails")
+if mode == "decision":
+    required = holdout.get("required_terms", ["unique", "durable", "crash", "external", "local"])
+    missing = [term for term in required if term not in text]
+    if missing:
+        raise SystemExit("decision lacks constraints: " + ", ".join(missing))

evals/access-efficient/holdout/ambiguous-distributed-decision/access-holdout.json

@@ -0,0 +1 @@
+{"required_terms":["unique","durable","crash","external","local"]}

evals/access-efficient/holdout/literal-limit/access-holdout.json

@@ -0,0 +1 @@
+{"expected_path":"LIMITS.toml","expected_value":"128"}

evals/access-efficient/holdout/routine-slug/test_holdout_labels.py

@@ -0,0 +1,8 @@
+import unittest
+
+from labels import slug_label
+
+
+class HoldoutLabelTests(unittest.TestCase):
+    def test_long_mixed_separator_run(self):
+        self.assertEqual(slug_label("___Alpha _  Beta___"), "alpha-beta")

evals/access-efficient/suite.toml

@@ -0,0 +1,175 @@
+schema_version = 8
+id = "access-efficient"
+profile = "access-efficient-escalation-lab"
+name = "Access-Efficient Escalation Lab"
+description = "Experimental matched routing trials for literal local work, economical patches, and deliberate scarce-tier escalation."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "scarce_tier_request_units"
+primary_baseline = "strongest_single_agent"
+direction = "lower"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.50
+minimum_absolute_improvement = 1.0
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+scarce_model_keys = [
+  "codex_chatgpt_builtin__gpt_5_6_sol",
+  "opencode_zen_anthropic_messages__claude_opus_5",
+  "zai_coding_responses__glm_5_3",
+]
+
+[[variants]]
+id = "configured-root"
+purpose = "Luna routing root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol maximum-capability single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go economical access control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen access control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan scarce-tier control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter API control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-routine"
+purpose = "Luna root plus the economical isolated routine engineer."
+topology = "root_plus_worker"
+worker = "routine_engineer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Contained literal scout, routine engineer, and trigger-gated GLM escalation."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-literal-scout"
+purpose = "Tests whether local literal reconnaissance earns its launch overhead."
+topology = "full_without_worker"
+worker = "literal_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-routine-engineer"
+purpose = "Ablates the economical isolated patch lane."
+topology = "full_without_worker"
+worker = "routine_engineer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-escalation"
+purpose = "Ablates scarce-tier GLM escalation."
+topology = "full_without_worker"
+worker = "flagship_escalation"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "literal-limit"
+description = "Trivial literal negative control where delegation should be avoided or strictly literal."
+sandbox = "read-only"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 900
+validation_timeout_seconds = 60
+prompt = """
+Report max_batch_items from LIMITS.toml with the exact path. Do no semantic reasoning and do not escalate. If the local scout is used, submit only a structured literal extract request. End with EVAL_ACCESS_LITERAL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["128", "EVAL_ACCESS_LITERAL"]
+forbidden_patterns = []
+validation_commands = ["python validate_access.py literal"]
+[tasks.orchestration_assertions]
+forbidden_agents = ["flagship_escalation"]
+max_jobs = 1
+
+[[tasks]]
+id = "routine-slug"
+description = "Localized deterministic patch suitable for economical execution."
+sandbox = "workspace-write"
+difficulty = "medium"
+negative_control = false
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Implement LABEL_SPEC.md. This is a bounded objectively testable source change; use scarce-tier escalation only if concrete evidence exceeds routine capability. End with EVAL_ACCESS_ROUTINE.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ACCESS_ROUTINE"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_access.py routine"]
+[tasks.orchestration_assertions]
+required_agents = ["routine_engineer"]
+forbidden_agents = ["flagship_escalation"]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "ambiguous-distributed-decision"
+description = "Consequential ambiguity with an explicit escalation trigger."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Read DISTRIBUTED_CONSTRAINTS.md and select an idempotency design. Record why the uncertainty and consequence justify or do not justify GLM escalation. Resolve claims with the supplied constraints, not model rank. End with EVAL_ACCESS_ESCALATION.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ACCESS_ESCALATION"]
+forbidden_patterns = []
+validation_commands = ["python validate_access.py decision"]
+[tasks.orchestration_assertions]
+required_agents = ["flagship_escalation"]
+max_jobs = 1
+max_contract_failures = 0

evals/adaptive-change/README.md

@@ -0,0 +1,5 @@
+# Adaptive change evaluation
+
+This suite exercises the `adaptive-engineering` profile on a bounded but nontrivial correctness change. The root must keep ownership of diagnosis and integration, use a specialist only where it adds value, and obtain independent review before claiming success.
+
+The native scout is intentionally not a worker assertion: native Codex participation is not represented in Agent MCP job telemetry. The evaluator does require the MCP implementation and review roles and measures whether their results were consumed.

evals/adaptive-change/fixture/SPEC.md

@@ -0,0 +1,11 @@
+# Ledger invariants
+
+`Ledger.apply(operation_id, account, delta)` applies an operation at most once globally.
+
+- A successful operation permanently consumes its operation ID.
+- A rejected overdraft must not consume its operation ID.
+- A caller may fund the account and retry the same rejected operation.
+- Balances may never become negative.
+- Duplicate successful operations return the current balance without applying the delta again.
+
+The current implementation violates the rejected-operation rule.

evals/adaptive-change/fixture/ledger.py

@@ -0,0 +1,17 @@
+class Ledger:
+    def __init__(self) -> None:
+        self._balances: dict[str, int] = {}
+        self._applied: set[str] = set()
+
+    def balance(self, account: str) -> int:
+        return self._balances.get(account, 0)
+
+    def apply(self, operation_id: str, account: str, delta: int) -> int:
+        if operation_id in self._applied:
+            return self.balance(account)
+        self._applied.add(operation_id)
+        updated = self.balance(account) + delta
+        if updated < 0:
+            raise ValueError("insufficient funds")
+        self._balances[account] = updated
+        return updated

evals/adaptive-change/fixture/test_ledger.py

@@ -0,0 +1,27 @@
+import unittest
+
+from ledger import Ledger
+
+
+class LedgerTests(unittest.TestCase):
+    def test_success_is_idempotent(self):
+        ledger = Ledger()
+        self.assertEqual(ledger.apply("deposit-1", "a", 10), 10)
+        self.assertEqual(ledger.apply("deposit-1", "a", 10), 10)
+
+    def test_overdraft_does_not_change_balance(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("withdraw-1", "a", -4)
+        self.assertEqual(ledger.balance("a"), 0)
+
+    def test_rejected_operation_can_be_retried(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("withdraw-1", "a", -4)
+        ledger.apply("deposit-1", "a", 10)
+        self.assertEqual(ledger.apply("withdraw-1", "a", -4), 6)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/adaptive-change/holdout/decomposable-ledger/test_holdout_ledger.py

@@ -0,0 +1,16 @@
+import unittest
+
+from ledger import Ledger
+
+
+class HoldoutLedgerTests(unittest.TestCase):
+    def test_rejected_id_remains_globally_available(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("shared", "a", -1)
+        self.assertEqual(ledger.apply("shared", "b", 3), 3)
+
+    def test_successful_id_is_global_across_accounts(self):
+        ledger = Ledger()
+        ledger.apply("shared", "a", 3)
+        self.assertEqual(ledger.apply("shared", "b", 9), 0)

evals/adaptive-change/holdout/tight-small-change/test_holdout_ledger.py

@@ -0,0 +1,13 @@
+import unittest
+
+from ledger import Ledger
+
+
+class HoldoutLedgerTests(unittest.TestCase):
+    def test_multiple_rejections_do_not_consume_id(self):
+        ledger = Ledger()
+        for _ in range(2):
+            with self.assertRaises(ValueError):
+                ledger.apply("withdraw", "a", -2)
+        ledger.apply("fund", "a", 4)
+        self.assertEqual(ledger.apply("withdraw", "a", -2), 2)

evals/adaptive-change/suite.toml

@@ -0,0 +1,149 @@
+schema_version = 8
+id = "adaptive-change"
+profile = "adaptive-engineering"
+name = "Adaptive Engineering Change"
+description = "Matched trials for selective delegation on decomposable and tightly coupled engineering work."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Configured Sol root with delegation mechanically disabled."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Task-specific flagship Sol single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access-service control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go economical DeepSeek control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter DeepSeek route control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-implementation"
+purpose = "Configured root plus the objectively testable implementation branch."
+topology = "root_plus_worker"
+worker = "implementation_specialist"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Complete selective adaptive topology."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-scout"
+purpose = "Ablates native reconnaissance."
+topology = "full_without_worker"
+worker = "repo_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-implementation"
+purpose = "Ablates the economical isolated implementation branch."
+topology = "full_without_worker"
+worker = "implementation_specialist"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-review"
+purpose = "Ablates independent cross-maker review."
+topology = "full_without_worker"
+worker = "adversarial_reviewer"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "decomposable-ledger"
+description = "Repair an idempotency defect with separable reconnaissance, implementation, and review work."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Repair the ledger defect described in SPEC.md. Own diagnosis and canonical integration. Delegate only work with a concrete expected contribution, continue non-overlapping critical-path work while it runs, explicitly accept or reject results, and end with EVAL_ADAPTIVE_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ADAPTIVE_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["implementation_specialist", "adversarial_reviewer"]
+max_jobs = 3
+max_contract_failures = 0
+max_observed_mcp_wait_ratio = 0.8
+
+[[tasks]]
+id = "tight-small-change"
+description = "Negative control where worker launch overhead should not be required."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 120
+prompt = """
+Make the smallest correct ledger repair. This task is intentionally compact: remain single-agent unless independent work has a concrete positive expected value. Run the tests and end with EVAL_ADAPTIVE_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ADAPTIVE_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 1
+max_contract_failures = 0

evals/bounded-research/fixture/corpus/00-precedence.md

@@ -0,0 +1,4 @@
+# Policy precedence
+
+Signed exceptions override the current handbook. The current handbook overrides archived drafts.
+

evals/bounded-research/fixture/corpus/10-handbook.md

@@ -0,0 +1,14 @@
+# Current handbook
+
+## Retries
+
+At most four delivery attempts are permitted.
+
+## Retention
+
+Operational delivery records are retained for 30 days.
+
+## Rollout
+
+Canary rollout is mandatory.
+

evals/bounded-research/fixture/corpus/20-archive.md

@@ -0,0 +1,4 @@
+# Archived draft
+
+This obsolete draft proposed seven retries and 90-day retention.
+

evals/bounded-research/fixture/corpus/30-signed-exception.md

@@ -0,0 +1,4 @@
+# Signed exception: emergency rollout
+
+For incident INC-42 only, canary rollout may be skipped after two-person approval. Retry and retention rules are unchanged.
+

evals/bounded-research/fixture/validate_research.py

@@ -0,0 +1,20 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("research-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run research holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+required = [*holdout["required_values"], *holdout["required_sources"]]
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("research synthesis misses authoritative facts: " + ", ".join(missing))
+if "seven" in text and not any(term in text for term in ("obsolete", "archived", "reject")):
+    raise SystemExit("archived retry value was not rejected")
+print(
+    json.dumps(
+        {"metrics": {"source_coverage": 1.0, "contradiction_resolution": 1.0}},
+        sort_keys=True,
+    )
+)

evals/bounded-research/holdout/distributed-policy-corpus/research-holdout.json

@@ -0,0 +1 @@
+{"required_values":["four","30","inc-42"],"required_sources":["10-handbook.md","30-signed-exception.md"]}

evals/bounded-research/holdout/small-corpus-negative/research-holdout.json

@@ -0,0 +1 @@
+{"required_values":["four","30","inc-42"],"required_sources":["10-handbook.md","30-signed-exception.md"]}

evals/bounded-research/suite.toml

@@ -0,0 +1,140 @@
+schema_version = 8
+id = "bounded-research"
+profile = "bounded-research-organization-lab"
+name = "Bounded Research Organization Lab"
+description = "Experimental flat-versus-hierarchical long-corpus research with bounded nesting and handoff accounting."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "source_coverage"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["contradiction_resolution"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sol repository root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol long-corpus single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go Kimi/MiniMax access control."
+profile = "research-backed-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen access control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan access control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter access control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "flat-fanout"
+purpose = "Root plus a direct MiniMax source scout, with hierarchy disabled."
+topology = "root_plus_worker"
+worker = "source_scout"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-hierarchy"
+purpose = "Kimi research lead may organize at most two MiniMax scouts."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-research-lead"
+purpose = "Ablates the hierarchical Kimi organizer, yielding direct flat scouting only."
+topology = "full_without_worker"
+worker = "research_lead"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-source-scout"
+purpose = "Ablates MiniMax source extraction while retaining the Kimi lead."
+topology = "full_without_worker"
+worker = "source_scout"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "distributed-policy-corpus"
+description = "Resolve cross-document conflicts and trace the final policy to authoritative sections."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 60
+prompt = """
+Read every document under corpus/. Determine the authoritative retry ceiling, retention period, and rollout exception, cite exact paths and headings, and resolve contradictions using the stated precedence rule. Compare bounded hierarchy with direct scouting without duplicate work. End with EVAL_BOUNDED_RESEARCH.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_BOUNDED_RESEARCH"]
+forbidden_patterns = []
+validation_commands = ["python validate_research.py"]
+[tasks.orchestration_assertions]
+required_agents = ["research_lead", "source_scout"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "small-corpus-negative"
+description = "Negative control where hierarchical handoffs should not be assumed valuable."
+sandbox = "read-only"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 60
+prompt = """
+Answer the three literal policy questions from corpus/ with exact citations. The corpus is intentionally small enough that nesting may cost more than it adds; launch only with a concrete expected contribution. End with EVAL_BOUNDED_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_BOUNDED_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python validate_research.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2

evals/codex-harness/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Deep merge contract
+
+`merge_settings(base, overlay)` returns a fresh mapping. Nested mappings merge recursively; overlay scalars replace base values; no input or nested output may be mutated or aliased.
+

evals/codex-harness/fixture/settings.py

@@ -0,0 +1,12 @@
+from collections.abc import Mapping
+from typing import Any
+
+
+def merge_settings(base: Mapping[str, Any], overlay: Mapping[str, Any]) -> dict[str, Any]:
+    result = dict(base)
+    for key, value in overlay.items():
+        if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
+            result[key].update(value)
+        else:
+            result[key] = value
+    return result

evals/codex-harness/fixture/test_settings.py

@@ -0,0 +1,23 @@
+import unittest
+
+from settings import merge_settings
+
+
+class MergeSettingsTests(unittest.TestCase):
+    def test_recursive_merge(self):
+        self.assertEqual(
+            merge_settings({"s": {"host": "x", "port": 80}}, {"s": {"port": 443}}),
+            {"s": {"host": "x", "port": 443}},
+        )
+
+    def test_inputs_and_outputs_do_not_alias(self):
+        base = {"s": {"host": "x"}, "flags": {"safe": True}}
+        overlay = {"s": {"port": 443}}
+        merged = merge_settings(base, overlay)
+        merged["s"]["host"] = "changed"
+        merged["flags"]["safe"] = False
+        self.assertEqual(base, {"s": {"host": "x"}, "flags": {"safe": True}})
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/codex-harness/holdout/deep-merge-contract/test_holdout_settings.py

@@ -0,0 +1,13 @@
+import unittest
+
+from settings import merge_settings
+
+
+class HoldoutMergeTests(unittest.TestCase):
+    def test_three_level_merge_and_no_alias(self):
+        base = {"a": {"b": {"left": 1}}, "untouched": {"x": 1}}
+        overlay = {"a": {"b": {"right": 2}}}
+        merged = merge_settings(base, overlay)
+        self.assertEqual(merged["a"]["b"], {"left": 1, "right": 2})
+        merged["untouched"]["x"] = 9
+        self.assertEqual(base["untouched"]["x"], 1)

evals/codex-harness/holdout/small-merge-control/test_holdout_settings.py

@@ -0,0 +1,8 @@
+import unittest
+
+from settings import merge_settings
+
+
+class HoldoutMergeTests(unittest.TestCase):
+    def test_overlay_scalar_replaces_mapping(self):
+        self.assertEqual(merge_settings({"a": {"x": 1}}, {"a": 3}), {"a": 3})

evals/codex-harness/suite.toml

@@ -0,0 +1,156 @@
+schema_version = 8
+id = "codex-harness"
+profile = "codex-harness-team"
+name = "Codex Harness Team"
+description = "Tests native-first context isolation, homogeneous Sol peers, and a fresh supervised critic."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sol root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent strongest single-agent Sol control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex service control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek service control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude service control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM service control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter service control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "pure-native"
+purpose = "Root plus the highest-value native invariant designer."
+topology = "root_plus_worker"
+worker = "invariant_designer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "homogeneous-sol"
+purpose = "Full isolated-role control using Sol for every participant."
+topology = "full"
+comparison_class = "control"
+[variants.bindings]
+repo_scout = "codex_chatgpt_builtin__gpt_5_6_sol"
+invariant_designer = "codex_chatgpt_builtin__gpt_5_6_sol"
+
+[[variants]]
+id = "full-profile"
+purpose = "Native-first Luna/Terra team followed by a fresh Sol critic."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-scout"
+purpose = "Ablates fast repository reconnaissance."
+topology = "full_without_worker"
+worker = "repo_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-invariants"
+purpose = "Ablates isolated invariant and test design."
+topology = "full_without_worker"
+worker = "invariant_designer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-critic"
+purpose = "Ablates fresh-context adversarial review."
+topology = "full_without_worker"
+worker = "fresh_critic"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "deep-merge-contract"
+description = "Correct a mutation-prone recursive merge under independently derivable invariants."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Implement the merge contract in SPEC.md. The root owns the change. Launch isolated read-only contexts only when they can derive tests, invariants, or critique concurrently; inspect and adjudicate their evidence. End with EVAL_HARNESS_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_HARNESS_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["fresh_critic"]
+max_jobs = 1
+max_contract_failures = 0
+
+[[tasks]]
+id = "small-merge-control"
+description = "Negative control for context-launch overhead on a compact implementation."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 120
+prompt = """
+Repair merge_settings with the smallest coherent change. Avoid launching peers unless they have a specific expected contribution. Run the suite and end with EVAL_HARNESS_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_HARNESS_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 1

evals/competing-implementations/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Stable unique contract
+
+`stable_unique(values, key)` returns the first value for each distinct key, preserves encounter order, accepts unhashable values when the supplied key result is hashable, never mutates input, and must scale linearly for 20,000 values.
+

evals/competing-implementations/fixture/benchmark.py

@@ -0,0 +1,26 @@
+import json
+import time
+
+from dedupe import stable_unique
+
+values = [index % 5000 for index in range(20_000)]
+started = time.perf_counter()
+result = stable_unique(values, lambda value: value)
+elapsed = time.perf_counter() - started
+if result != list(range(5000)):
+    raise SystemExit("stable_unique produced an incorrect result")
+if elapsed > 1.0:
+    raise SystemExit(f"benchmark exceeded one second: {elapsed:.3f}")
+quality = max(0.0, 1.0 - elapsed)
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "benchmark_quality": quality,
+                "correctness_rate": 1.0,
+                "maintainability_score": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/competing-implementations/fixture/dedupe.py

@@ -0,0 +1,10 @@
+from collections.abc import Callable, Iterable
+from typing import Any
+
+
+def stable_unique(values: Iterable[Any], key: Callable[[Any], Any]) -> list[Any]:
+    result = []
+    for value in values:
+        if not any(key(existing) == key(value) for existing in result):
+            result.append(value)
+    return result

evals/competing-implementations/fixture/test_dedupe.py

@@ -0,0 +1,19 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class DedupeTests(unittest.TestCase):
+    def test_preserves_first_and_order(self):
+        values = [{"id": 2, "v": "a"}, {"id": 1}, {"id": 2, "v": "b"}]
+        self.assertEqual(stable_unique(values, lambda item: item["id"]), values[:2])
+
+    def test_does_not_mutate_input(self):
+        values = [[1], [1], [2]]
+        before = [list(value) for value in values]
+        stable_unique(values, tuple)
+        self.assertEqual(values, before)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/competing-implementations/holdout/stable-deduplication/test_holdout_dedupe.py

@@ -0,0 +1,16 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class HoldoutDedupeTests(unittest.TestCase):
+    def test_generator_is_consumed_once(self):
+        seen = []
+
+        def values():
+            for value in (2, 1, 2):
+                seen.append(value)
+                yield value
+
+        self.assertEqual(stable_unique(values(), lambda value: value), [2, 1])
+        self.assertEqual(seen, [2, 1, 2])

evals/competing-implementations/holdout/tiny-change-negative/test_holdout_dedupe.py

@@ -0,0 +1,8 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class HoldoutDedupeTests(unittest.TestCase):
+    def test_empty_input(self):
+        self.assertEqual(stable_unique([], lambda value: value), [])

evals/competing-implementations/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "competing-implementations"
+profile = "competing-implementations-lab"
+name = "Competing Implementations Lab"
+description = "Experimental matched alternatives selected by tests, benchmarks, patch review, and integration effort."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "benchmark_quality"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["correctness_rate", "maintainability_score"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Terra contract author and implementer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-implementation control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek candidate control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Sonnet candidate control."
+profile = "contract-first-refactoring"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan access control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter access control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "one-deepseek-candidate"
+purpose = "Objective judge plus one economical implementation candidate."
+topology = "root_plus_worker"
+worker = "deepseek_candidate"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-competition"
+purpose = "Two independently isolated implementations with evidence-based selection."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-deepseek"
+purpose = "Ablates the Go-hosted DeepSeek candidate."
+topology = "full_without_worker"
+worker = "deepseek_candidate"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-sonnet"
+purpose = "Ablates the Zen-hosted Sonnet candidate."
+topology = "full_without_worker"
+worker = "sonnet_candidate"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "stable-deduplication"
+description = "Choose between independent correct and scalable stable-deduplication patches."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Freeze the objective contract in SPEC.md and its benchmark before implementation. When workers are available, launch independent candidates into disjoint worktrees. Compare returned binary patches using tests, benchmark evidence, complexity, maintainability, and integration corrections; never vote by model identity. Explicitly integrate only the selected patch and end with EVAL_COMPETING_SELECTED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_COMPETING_SELECTED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python benchmark.py"]
+[tasks.orchestration_assertions]
+required_agents = ["deepseek_candidate", "sonnet_candidate"]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "tiny-change-negative"
+description = "Negative control where two full implementations may cost more than they add."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Implement stable_unique correctly. The task is intentionally compact; launch competing writers only if the expected selection value exceeds two patches and integration work. End with EVAL_COMPETING_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_COMPETING_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python benchmark.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0

evals/contract-refactoring/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Versioned payload contract
+
+`upgrade_payload` must return a new mapping, preserve unknown fields, read legacy `userId`, write canonical `user_id`, remove only the legacy alias, reject conflicting aliases, and never mutate caller-owned nested data.
+

evals/contract-refactoring/fixture/codec.py

@@ -0,0 +1,8 @@
+from typing import Any
+
+
+def upgrade_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    result = dict(payload)
+    if "userId" in result:
+        result["user_id"] = result.pop("userId")
+    return result

evals/contract-refactoring/fixture/test_codec.py

@@ -0,0 +1,22 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class CodecTests(unittest.TestCase):
+    def test_legacy_alias_is_upgraded(self):
+        self.assertEqual(upgrade_payload({"userId": "u", "x": 1}), {"user_id": "u", "x": 1})
+
+    def test_conflicting_aliases_are_rejected(self):
+        with self.assertRaises(ValueError):
+            upgrade_payload({"userId": "old", "user_id": "new"})
+
+    def test_nested_input_does_not_alias(self):
+        source = {"user_id": "u", "metadata": {"safe": True}}
+        output = upgrade_payload(source)
+        output["metadata"]["safe"] = False
+        self.assertTrue(source["metadata"]["safe"])
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/contract-refactoring/fixture/validate_contract.py

@@ -0,0 +1,28 @@
+import hashlib
+import json
+from pathlib import Path
+
+contract = Path("CONTRACT.md")
+digest = Path("CONTRACT.sha256")
+if not contract.is_file() or not digest.is_file():
+    raise SystemExit("CONTRACT.md and CONTRACT.sha256 are required")
+actual = hashlib.sha256(contract.read_bytes()).hexdigest()
+if digest.read_text(encoding="utf-8").strip() != actual:
+    raise SystemExit("frozen contract hash does not match")
+text = contract.read_text(encoding="utf-8").lower()
+required = ("legacy", "unknown", "conflict", "immut", "user_id")
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("frozen contract misses invariants: " + ", ".join(missing))
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "hidden_invariant_recall": 1.0,
+                "contract_hash_integrity": 1.0,
+                "compatibility_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/contract-refactoring/holdout/plausible-breaking-negative/test_holdout_codec.py

@@ -0,0 +1,10 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class HoldoutCompatibilityTests(unittest.TestCase):
+    def test_canonical_payload_remains_canonical(self):
+        self.assertEqual(
+            upgrade_payload({"user_id": "u", "future": 3}), {"user_id": "u", "future": 3}
+        )

evals/contract-refactoring/holdout/versioned-payload-refactor/test_holdout_codec.py

@@ -0,0 +1,15 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class HoldoutCodecTests(unittest.TestCase):
+    def test_unknown_nested_data_is_deeply_independent(self):
+        source = {"userId": "u", "unknown": {"items": [1, 2]}}
+        result = upgrade_payload(source)
+        result["unknown"]["items"].append(3)
+        self.assertEqual(source["unknown"]["items"], [1, 2])
+
+    def test_conflict_is_rejected_even_when_values_match(self):
+        with self.assertRaises(ValueError):
+            upgrade_payload({"userId": "u", "user_id": "u"})

evals/contract-refactoring/suite.toml

@@ -0,0 +1,148 @@
+schema_version = 8
+id = "contract-refactoring"
+profile = "contract-first-refactoring"
+name = "Contract-First Refactoring"
+description = "Measures frozen-contract fidelity, hidden compatibility invariants, independent tests, and regression verification."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "hidden_invariant_recall"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["contract_hash_integrity", "compatibility_rate"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sonnet refactor lead alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent compatibility refactor control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex refactor control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go refactor control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Sonnet refactor control."
+profile = "contract-first-refactoring"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan refactor control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter refactor control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-test-author"
+purpose = "Frozen-contract root plus independent contract-derived tests."
+topology = "root_plus_worker"
+worker = "contract_test_author"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Invariant mining, frozen contract, test-first patch, refactor, and verification."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-invariant-miner"
+purpose = "Ablates long-context compatibility mining."
+topology = "full_without_worker"
+worker = "invariant_miner"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-test-author"
+purpose = "Ablates independent test-first specification."
+topology = "full_without_worker"
+worker = "contract_test_author"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates economical independent compatibility verification."
+topology = "full_without_worker"
+worker = "compatibility_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "versioned-payload-refactor"
+description = "Refactor a versioned serializer without losing legacy read compatibility or input immutability."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Refactor codec.py under SPEC.md. Before tests or implementation, write and hash CONTRACT.md into CONTRACT.sha256. Preserve legacy reads, canonical new writes, unknown fields, and input immutability. Explicitly accept test-only patches before source changes and end with EVAL_CONTRACT_REFACTOR.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_CONTRACT_REFACTOR"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_contract.py"]
+[tasks.orchestration_assertions]
+required_agents = ["invariant_miner", "contract_test_author", "compatibility_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "plausible-breaking-negative"
+description = "Negative control that catches a superficially clean but compatibility-breaking rewrite."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Improve codec.py without deleting legacy field support or mutating caller data. Freeze a precise contract first and make plausible compatibility-breaking simplifications fail tests. End with EVAL_CONTRACT_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_CONTRACT_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_contract.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0

evals/debugging-confidence/README.md

@@ -0,0 +1,3 @@
+# Debugging confidence evaluation
+
+Designed for `high-confidence-debugging`. The fixture contains a state-dependent cache bug and a misleading obvious hypothesis. Success requires a failing reproduction, a minimal fix, regression tests, and explicit acceptance or rejection of verifier results.

evals/debugging-confidence/fixture/policy.py

@@ -0,0 +1,13 @@
+class Policy:
+    def __init__(self, flags=None):
+        self.flags = dict(flags or {})
+        self._decision_cache = {}
+
+    def set_flag(self, name, enabled):
+        self.flags[name] = bool(enabled)
+
+    def allows(self, user_id, flag):
+        key = (user_id, flag)
+        if key not in self._decision_cache:
+            self._decision_cache[key] = bool(self.flags.get(flag, False))
+        return self._decision_cache[key]

evals/debugging-confidence/fixture/test_policy.py

@@ -0,0 +1,23 @@
+import unittest
+
+from policy import Policy
+
+
+class PolicyTests(unittest.TestCase):
+    def test_initial_decision_uses_current_flag(self):
+        self.assertTrue(Policy({"preview": True}).allows("u1", "preview"))
+
+    def test_users_have_independent_cache_entries(self):
+        policy = Policy({"preview": True})
+        self.assertTrue(policy.allows("u1", "preview"))
+        self.assertTrue(policy.allows("u2", "preview"))
+
+    def test_setting_flag_invalidates_prior_decisions(self):
+        policy = Policy({"preview": False})
+        self.assertFalse(policy.allows("u1", "preview"))
+        policy.set_flag("preview", True)
+        self.assertTrue(policy.allows("u1", "preview"))
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/debugging-confidence/holdout/misleading-cache-symptom/test_holdout_policy.py

@@ -0,0 +1,17 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_true_to_false_toggle_invalidates(self):
+        policy = Policy({"preview": True})
+        self.assertTrue(policy.allows("u1", "preview"))
+        policy.set_flag("preview", False)
+        self.assertFalse(policy.allows("u1", "preview"))
+
+    def test_repeated_toggles_do_not_reuse_stale_value(self):
+        policy = Policy({"preview": False})
+        for value in (True, False, True):
+            policy.set_flag("preview", value)
+            self.assertEqual(policy.allows("u1", "preview"), value)

evals/debugging-confidence/holdout/no-speculative-rewrite/test_holdout_policy.py

@@ -0,0 +1,12 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_unrelated_flag_change_preserves_correct_answers(self):
+        policy = Policy({"a": True, "b": False})
+        self.assertTrue(policy.allows("u", "a"))
+        policy.set_flag("b", True)
+        self.assertTrue(policy.allows("u", "a"))
+        self.assertTrue(policy.allows("u", "b"))

evals/debugging-confidence/mutations/misleading-cache-symptom/01-enable-only-invalidation.patch

@@ -0,0 +1,12 @@
+diff --git a/policy.py b/policy.py
+--- a/policy.py
++++ b/policy.py
+@@ -5,6 +5,8 @@ class Policy:
+ 
+     def set_flag(self, name, enabled):
+         self.flags[name] = bool(enabled)
++        if enabled:
++            self._decision_cache.clear()
+ 
+     def allows(self, user_id, flag):
+         key = (user_id, flag)

evals/debugging-confidence/mutations/misleading-cache-symptom/02-wrong-key-invalidation.patch

@@ -0,0 +1,12 @@
+diff --git a/policy.py b/policy.py
+--- a/policy.py
++++ b/policy.py
+@@ -5,6 +5,8 @@ class Policy:
+ 
+     def set_flag(self, name, enabled):
+         self.flags[name] = bool(enabled)
++        for key in [key for key in self._decision_cache if key[0] == name]:
++            del self._decision_cache[key]
+ 
+     def allows(self, user_id, flag):
+         key = (user_id, flag)

evals/debugging-confidence/suite.toml

@@ -0,0 +1,142 @@
+schema_version = 8
+id = "debugging-confidence"
+profile = "high-confidence-debugging"
+name = "High-Confidence Debugging"
+description = "Matched misleading-symptom trials for independent reproduction, repair, and post-fix falsification."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "DeepSeek diagnostician alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent debugging control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex debugging control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go debugging control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen debugging control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan debugging control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter debugging control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-reproducer"
+purpose = "Root plus independent reproduction and regression-test derivation."
+topology = "root_plus_worker"
+worker = "independent_reproducer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Reproduce, repair, and fresh-context falsification topology."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-reproducer"
+purpose = "Ablates independent reproduction."
+topology = "full_without_worker"
+worker = "independent_reproducer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates fresh post-fix adversarial verification."
+topology = "full_without_worker"
+worker = "fix_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "misleading-cache-symptom"
+description = "Repair stale decisions without accepting the visible parser as an unsupported cause."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Users report that changing a feature flag does not affect repeated decisions. Reproduce before repair, treat the obvious parser theory as unproven, preserve a regression test, and independently attack the completed fix. The root owns source changes and one mechanically capped correction cycle. End with EVAL_DEBUG_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_DEBUG_VERIFIED"]
+forbidden_patterns = ["parser was definitely"]
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["independent_reproducer", "fix_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+min_result_acceptance_rate = 0.5
+
+[[tasks]]
+id = "no-speculative-rewrite"
+description = "Negative control requiring evidence before any broad rewrite."
+sandbox = "workspace-write"
+difficulty = "medium"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Diagnose and repair the repeated-decision failure. Do not rewrite parsing or storage code without a reproduction that implicates it. Preserve existing behavior and end with EVAL_DEBUG_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_DEBUG_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0

evals/incident-triage/fixture/RUNBOOK.md

@@ -0,0 +1,4 @@
+# Checkout incident threshold
+
+A root cause requires two independent, time-correlated event streams plus a prediction that is observed after an operator-controlled change. Otherwise report ranked unresolved hypotheses and next discriminating observations.
+

evals/incident-triage/fixture/logs.jsonl

@@ -0,0 +1,9 @@
+{"id":"api-01","time":"10:01:00Z","service":"api","event":"deploy","version":"2026.08.16.2","cache_key":"tenant"}
+{"id":"dns-01","time":"10:01:05Z","service":"dns","event":"retry","count":1}
+{"id":"api-02","time":"10:02:00Z","service":"api","event":"checkout_wrong_currency","tenant":"west"}
+{"id":"worker-01","time":"10:02:01Z","service":"worker","event":"cache_hit","key":"west","currency":"EUR"}
+{"id":"db-01","time":"10:02:02Z","service":"db","event":"latency_ms","value":18}
+{"id":"api-03","time":"10:06:00Z","service":"api","event":"rollback","version":"2026.08.16.1"}
+{"id":"worker-02","time":"10:06:20Z","service":"worker","event":"cache_key","key":"west:USD"}
+{"id":"api-04","time":"10:07:00Z","service":"api","event":"checkout_correct_currency","tenant":"west"}
+

evals/incident-triage/fixture/validate_incident.py

@@ -0,0 +1,40 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("incident-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+is_negative = "eval_incident_unresolved" in text
+required_event_ids = holdout.get("required_event_ids", ["api-02", "worker-01", "api-03", "api-04"])
+minimum_event_ids = int(holdout.get("minimum_event_ids", 2))
+alternatives = holdout.get("alternative_terms", ["dns", "database", "cache"])
+minimum_alternatives = int(holdout.get("minimum_alternatives", 2))
+event_ids = sum(event_id.lower() in text for event_id in required_event_ids)
+has_alternatives = sum(term.lower() in text for term in alternatives) >= minimum_alternatives
+if event_ids < minimum_event_ids or not has_alternatives:
+    raise SystemExit("incident report lacks correlated event IDs or competing hypotheses")
+forbidden_certainty = holdout.get("forbidden_certainty", ["root cause is"])
+unsupported = float(is_negative and any(term.lower() in text for term in forbidden_certainty))
+if unsupported:
+    raise SystemExit("negative control asserted an unsupported root cause")
+required_degradation_terms = holdout.get("required_degradation_terms", [])
+missing_degradation_terms = [
+    term for term in required_degradation_terms if term.lower() not in text
+]
+if missing_degradation_terms:
+    raise SystemExit(
+        "incident report did not preserve provider-failure evidence: "
+        + ", ".join(missing_degradation_terms)
+    )
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "evidence_backed_diagnosis": 1.0,
+                "hypothesis_discrimination": 1.0,
+                "unsupported_root_cause_rate": unsupported,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/incident-triage/holdout/insufficient-evidence/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"forbidden_certainty":["root cause is","confirmed root cause"],"minimum_alternatives":2,"minimum_event_ids":2,"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/holdout/noisy-checkout-incident/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"minimum_alternatives":2,"minimum_event_ids":3,"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/holdout/provider-timeout-degradation/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"minimum_alternatives":2,"minimum_event_ids":2,"required_degradation_terms":["timeout","causal challenger","unavailable"],"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/mutations/insufficient-evidence/01-remove-controlled-correlation.patch

@@ -0,0 +1,14 @@
+diff --git a/logs.jsonl b/logs.jsonl
+--- a/logs.jsonl
++++ b/logs.jsonl
+@@ -1,9 +1,7 @@
+ {"id":"api-01","time":"10:01:00Z","service":"api","event":"deploy","version":"2026.08.16.2","cache_key":"tenant"}
+ {"id":"dns-01","time":"10:01:05Z","service":"dns","event":"retry","count":1}
+ {"id":"api-02","time":"10:02:00Z","service":"api","event":"checkout_wrong_currency","tenant":"west"}
+ {"id":"worker-01","time":"10:02:01Z","service":"worker","event":"cache_hit","key":"west","currency":"EUR"}
+ {"id":"db-01","time":"10:02:02Z","service":"db","event":"latency_ms","value":18}
+-{"id":"api-03","time":"10:06:00Z","service":"api","event":"rollback","version":"2026.08.16.1"}
+-{"id":"worker-02","time":"10:06:20Z","service":"worker","event":"cache_key","key":"west:USD"}
+-{"id":"api-04","time":"10:07:00Z","service":"api","event":"checkout_correct_currency","tenant":"west"}
++{"id":"api-03","time":"10:06:00Z","service":"api","event":"telemetry_gap","duration_seconds":180}
+ 

evals/incident-triage/suite.toml

@@ -0,0 +1,161 @@
+schema_version = 8
+id = "incident-triage"
+profile = "incident-hypothesis-triage"
+name = "Incident Hypothesis Triage"
+description = "Read-only noisy-evidence trials for competing causes, outage degradation, and evidence-backed diagnosis."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "evidence_backed_diagnosis"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["hypothesis_discrimination"]
+no_regression_lower_metrics = ["unsupported_root_cause_rate"]
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "GLM incident synthesizer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent causal-analysis control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex incident control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go incident control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen incident control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM incident control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter causal-analysis control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__nvidia_nemotron_3_ultra_550b_a55b"
+
+[[variants]]
+id = "root-plus-challenger"
+purpose = "GLM synthesis plus independent pinned Nemotron alternative hypotheses."
+topology = "root_plus_worker"
+worker = "causal_challenger"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Bounded evidence correlation plus independent causal challenge."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-evidence-runner"
+purpose = "Ablates economical evidence collection and correlation."
+topology = "full_without_worker"
+worker = "evidence_runner"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-causal-challenger"
+purpose = "Ablates cross-route alternative causal analysis."
+topology = "full_without_worker"
+worker = "causal_challenger"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "noisy-checkout-incident"
+description = "Discriminate a causal deployment signal from DNS and database red herrings."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Investigate RUNBOOK.md and logs.jsonl read-only. Develop at least two competing causes, cite exact event IDs and distinguishing predictions, and assert a root cause only if reproducible evidence crosses the stated threshold. Report degraded roles if a provider is unavailable. End with EVAL_INCIDENT_DIAGNOSED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DIAGNOSED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+required_agents = ["evidence_runner", "causal_challenger"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "provider-timeout-degradation"
+description = "The independent OpenRouter causal challenger times out while primary evidence analysis continues."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { openrouter_openai_chat = "timeout" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage RUNBOOK.md and logs.jsonl while the causal-challenger route is unavailable because of an injected timeout. Report the degraded role and fault explicitly, use remaining evidence rather than inventing challenger output, rank competing causes, and end with EVAL_INCIDENT_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "insufficient-evidence"
+description = "Negative control where a ranked unresolved hypothesis is correct and certainty is a defect."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage the supplied evidence, but treat missing cross-service correlation as material. Rank alternatives and required next observations; do not assert a root cause without reproducible evidence. End with EVAL_INCIDENT_UNRESOLVED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_UNRESOLVED"]
+forbidden_patterns = ["(?i)root cause is"]
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0

evals/research-currentness/README.md

@@ -0,0 +1,5 @@
+# Research-currentness evaluation
+
+This suite evaluates `research-backed-engineering` with a stale local note and a live-current API contract. The researcher must use dated primary OpenAI sources, the implementation specialist must work from the approved dossier, and the offline auditor must verify source-to-code compliance without silently introducing fresh claims.
+
+The validator does not call an external API or require a credential. It checks stable Responses API request/response semantics and the presence of a dated official-source dossier; the run transcript and worker contracts provide the research evidence.

evals/research-currentness/fixture/STALE_NOTE.md

@@ -0,0 +1,3 @@
+# Obsolete integration note
+
+This 2023 note says to send `messages` to `/v1/chat/completions` and read `choices[0].message.content`. It is retained solely to test whether current primary-source research overrides stale repository prose.

evals/research-currentness/fixture/client.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+ENDPOINT = "/v1/chat/completions"
+
+
+def build_request(prompt: str) -> dict[str, Any]:
+    return {
+        "model": "gpt-5",
+        "messages": [{"role": "user", "content": prompt}],
+    }
+
+
+def extract_text(response: dict[str, Any]) -> str:
+    return str(response["choices"][0]["message"]["content"])

evals/research-currentness/fixture/validate_currentness.py

@@ -0,0 +1,50 @@
+import json
+import re
+from pathlib import Path
+
+from client import ENDPOINT, build_request, extract_text
+
+if ENDPOINT != "/v1/responses":
+    raise SystemExit("client must use /v1/responses")
+
+request = build_request("hello")
+if request.get("input") != "hello" or "messages" in request:
+    raise SystemExit("client must send the Responses API input field")
+
+sample = {"output_text": "ready"}
+if extract_text(sample) != "ready":
+    raise SystemExit("client must extract the SDK-style output_text helper value")
+
+dossier = Path("SOURCES.md")
+if not dossier.is_file():
+    raise SystemExit("SOURCES.md is required")
+text = dossier.read_text(encoding="utf-8")
+holdout_path = Path("source-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run source holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+if not any(domain in text for domain in holdout["required_authority_domains"]):
+    raise SystemExit("dossier misses every holdout authority domain")
+if not re.search(r"https://(?:platform|developers)\.openai\.com/", text):
+    raise SystemExit("dossier must cite a direct official OpenAI documentation URL")
+if not re.search(r"20\d{2}-\d{2}-\d{2}", text):
+    raise SystemExit("dossier must record an ISO access or source date")
+if holdout["requires_root_refetch"] and not re.search(r"re-?fetch", text, flags=re.IGNORECASE):
+    raise SystemExit("dossier must record the root's independent re-fetch")
+if holdout["requires_conflict_disposition"] and not re.search(
+    r"conflict|stale|obsolete", text, flags=re.IGNORECASE
+):
+    raise SystemExit("dossier must explicitly dispose of the conflicting stale note")
+
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "source_accuracy": 1.0,
+                "source_authority": 1.0,
+                "root_refetch_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/research-currentness/holdout/conflicting-responses-contract/source-holdout.json

@@ -0,0 +1,5 @@
+{
+  "required_authority_domains": ["developers.openai.com", "platform.openai.com"],
+  "requires_root_refetch": true,
+  "requires_conflict_disposition": true
+}

evals/research-currentness/holdout/stale-source-negative/source-holdout.json

@@ -0,0 +1,5 @@
+{
+  "required_authority_domains": ["developers.openai.com", "platform.openai.com"],
+  "requires_root_refetch": true,
+  "requires_conflict_disposition": true
+}

evals/research-currentness/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "research-currentness"
+profile = "research-backed-engineering"
+name = "Research-Backed Engineering"
+description = "Matched current-source trials measuring authority, conflict handling, re-fetching, and source-to-code fidelity."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "source_accuracy"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["source_authority", "root_refetch_rate"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "MiniMax long-context root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol current-source engineering control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex research control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go MiniMax root control."
+profile = "research-backed-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude research control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM research control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter research control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-researcher"
+purpose = "Root plus one independent live primary-source question."
+topology = "root_plus_worker"
+worker = "source_researcher"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Parallel research with conditional cross-maker source-to-code audit."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-researcher"
+purpose = "Ablates parallel current-source research."
+topology = "full_without_worker"
+worker = "source_researcher"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-auditor"
+purpose = "Ablates independent source-to-code audit."
+topology = "full_without_worker"
+worker = "source_auditor"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "conflicting-responses-contract"
+description = "Override a stale repository note using current primary sources and record conflicts."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+STALE_NOTE.md is untrusted historical material. Research the current official Responses API contract using primary sources, record URL, source/update date, retrieval date, claim, conflict disposition, and the root's independent re-fetch in SOURCES.md. Update client.py without a network request and audit source-to-code compliance. End with EVAL_RESEARCH_CURRENT.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_RESEARCH_CURRENT"]
+forbidden_patterns = []
+validation_commands = ["python validate_currentness.py"]
+[tasks.orchestration_assertions]
+required_agents = ["source_researcher", "source_auditor"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "stale-source-negative"
+description = "Negative control: the stale local note must never win by proximity."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Determine whether STALE_NOTE.md remains current, using live primary evidence rather than model memory or source voting. If it conflicts, document why and implement only the verified contract. End with EVAL_RESEARCH_STALE_REJECTED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_RESEARCH_STALE_REJECTED"]
+forbidden_patterns = []
+validation_commands = ["python validate_currentness.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0

evals/route-resilience/fixture/ROUTE_PROBE.md

@@ -0,0 +1,4 @@
+# Route probe
+
+Analyze these literal claims: schema version is 2; fallback must remain disabled for featured OpenRouter roles; endpoint identity must come from supervisor telemetry rather than a model assertion.
+

evals/route-resilience/fixture/validate_route.py

@@ -0,0 +1,21 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("route-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+required = holdout.get("required_identity_terms", ["schema", "endpoint", "telemetry"])
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("route report lacks identity evidence: " + ", ".join(missing))
+if "eval_route_degraded" in text and not any(
+    term in text for term in ("unavailable", "degraded", "credential")
+):
+    raise SystemExit("degraded route was not reported")
+unavailable_route = holdout.get("unavailable_route")
+if unavailable_route and unavailable_route.lower() not in text:
+    raise SystemExit("degraded report does not identify the unavailable route")
+required_fault_terms = holdout.get("required_fault_terms", [])
+if required_fault_terms and not any(term.lower() in text for term in required_fault_terms):
+    raise SystemExit("degraded report does not identify the injected route fault")
+print(json.dumps({"metrics": {"route_consistency": 1.0}}, sort_keys=True))

evals/route-resilience/holdout/injected-go-credential-loss/route-holdout.json

@@ -0,0 +1 @@
+{"required_identity_terms":["schema","endpoint","telemetry"],"unavailable_route":"opencode go"}

evals/route-resilience/holdout/injected-go-rate-limit/route-holdout.json

@@ -0,0 +1 @@
+{"required_fault_terms":["rate limit","rate-limit","rate_limit"],"required_identity_terms":["schema","endpoint","telemetry"],"unavailable_route":"opencode go"}

evals/route-resilience/holdout/injected-openrouter-credential-loss/route-holdout.json

@@ -0,0 +1 @@
+{"required_identity_terms":["schema","endpoint","telemetry"],"unavailable_route":"openrouter"}

evals/route-resilience/holdout/injected-openrouter-timeout/route-holdout.json

@@ -0,0 +1 @@
+{"required_fault_terms":["timeout","timed out"],"required_identity_terms":["schema","endpoint","telemetry"],"unavailable_route":"openrouter"}

evals/route-resilience/holdout/shadow-build-comparison/route-holdout.json

@@ -0,0 +1 @@
+{"required_identity_terms":["maker","operator","access","endpoint","policy","telemetry"]}

evals/route-resilience/suite.toml

@@ -0,0 +1,202 @@
+schema_version = 8
+id = "route-resilience"
+profile = "route-resilience-lab"
+name = "Route Resilience Lab"
+description = "Experimental same-model route comparison with pinned endpoint identity and typed credential, rate-limit, and timeout faults."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "route_consistency"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Terra route observer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent route-analysis control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek route control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen access control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan access control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter DeepSeek route control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-openrouter"
+purpose = "Route observer plus the independently pinned shadow route."
+topology = "root_plus_worker"
+worker = "openrouter_route"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Intentional duplicate observation through Go and pinned OpenRouter."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-go-route"
+purpose = "Ablates the Go route."
+topology = "full_without_worker"
+worker = "go_route"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-openrouter-route"
+purpose = "Ablates the pinned OpenRouter route."
+topology = "full_without_worker"
+worker = "openrouter_route"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "shadow-build-comparison"
+description = "Compare claims from two routes while supervisor telemetry establishes endpoint identity."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Read ROUTE_PROBE.md and compare the same exact claims through configured routes when available. Distinguish model maker, API operator, access product, gateway, requested policy, and actual serving endpoint; never infer endpoint identity from model text. End with EVAL_ROUTE_SHADOW.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ROUTE_SHADOW"]
+forbidden_patterns = []
+validation_commands = ["python validate_route.py"]
+[tasks.orchestration_assertions]
+required_agents = ["go_route", "openrouter_route"]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "injected-go-credential-loss"
+description = "OpenCode Go is mechanically unavailable while the independent route remains usable."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { opencode_go_openai_chat = "credential_loss" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Perform the route probe under the injected availability overlay. Report unavailable roles explicitly, continue through any independent available route, and do not invent fallback execution. End with EVAL_ROUTE_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ROUTE_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_route.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "injected-openrouter-credential-loss"
+description = "OpenRouter is mechanically unavailable and no silent aggregator fallback is permitted."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { openrouter_openai_chat = "credential_loss" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Perform the route probe under the injected availability overlay. Record that the pinned OpenRouter role is unavailable and use only explicitly available evidence. End with EVAL_ROUTE_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ROUTE_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_route.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "injected-go-rate-limit"
+description = "The primary Go route is mechanically unavailable after a typed rate-limit fault."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { opencode_go_openai_chat = "rate_limit" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Perform the route probe under the injected rate-limit overlay. Name the affected route and fault, continue only through explicitly available infrastructure, and do not describe the shadow route as a transparent fallback. End with EVAL_ROUTE_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ROUTE_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_route.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "injected-openrouter-timeout"
+description = "The pinned OpenRouter route is mechanically unavailable after a typed timeout fault."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { openrouter_openai_chat = "timeout" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Perform the route probe under the injected timeout overlay. Name the affected route and fault, continue only through the Go route when available, and report that no OpenRouter fallback was attempted. End with EVAL_ROUTE_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ROUTE_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_route.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0

evals/security-assurance/README.md

@@ -0,0 +1,3 @@
+# Security assurance evaluation
+
+Designed for `secure-change`. The fixture contains one reachable archive traversal defect and one suspicious but safely normalized path. It measures finding precision, empirical confirmation, remediation, and regression safety.

evals/security-assurance/fixture/archive.py

@@ -0,0 +1,15 @@
+from pathlib import Path, PurePosixPath
+
+
+def normalized_asset_path(name):
+    parts = [part for part in PurePosixPath(name).parts if part not in {"", ".", ".."}]
+    return Path("assets").joinpath(*parts)
+
+
+def extract_entries(entries, destination):
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    for name, content in entries:
+        target = destination / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content)

evals/security-assurance/fixture/test_archive.py

@@ -0,0 +1,30 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive import extract_entries, normalized_asset_path
+
+
+class ArchiveTests(unittest.TestCase):
+    def test_normalized_asset_path_discards_parent_components(self):
+        self.assertEqual(normalized_asset_path("icons/../safe.svg"), Path("assets/icons/safe.svg"))
+
+    def test_safe_nested_extraction(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            extract_entries([("nested/data.txt", b"ok")], temporary)
+            self.assertEqual(Path(temporary, "nested/data.txt").read_bytes(), b"ok")
+
+    def test_extraction_rejects_parent_escape(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            outside = Path(temporary).parent / "escaped-mmo-eval.txt"
+            outside.unlink(missing_ok=True)
+            try:
+                with self.assertRaises(ValueError):
+                    extract_entries([("../escaped-mmo-eval.txt", b"bad")], temporary)
+                self.assertFalse(outside.exists())
+            finally:
+                outside.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/security-assurance/fixture/validate_security_report.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+
+path = Path("SECURITY_REPORT.md")
+if not path.is_file():
+    raise SystemExit("SECURITY_REPORT.md is required")
+text = path.read_text(encoding="utf-8").lower()
+required = ("command", "observation", "rejected", "normalized_asset_path", "traversal")
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("security report lacks evidence fields: " + ", ".join(missing))
+if "normalized_asset_path" in text and not any(
+    term in text for term in ("not vulnerable", "negative control", "rejected candidate")
+):
+    raise SystemExit("benign normalizer was not explicitly rejected as a finding")
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "vulnerability_recall": 1.0,
+                "false_positive_rate": 0.0,
+                "empirical_reproduction_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/security-assurance/holdout/archive-boundary/test_holdout_archive.py

@@ -0,0 +1,23 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive import extract_entries
+
+
+class HoldoutArchiveTests(unittest.TestCase):
+    def test_absolute_member_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            with self.assertRaises(ValueError):
+                extract_entries([("/tmp/mmo-forbidden", b"bad")], temporary)
+
+    def test_nested_parent_escape_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            outside = Path(temporary).parent / "mmo-hidden-escape"
+            outside.unlink(missing_ok=True)
+            try:
+                with self.assertRaises(ValueError):
+                    extract_entries([("safe/../../mmo-hidden-escape", b"bad")], temporary)
+                self.assertFalse(outside.exists())
+            finally:
+                outside.unlink(missing_ok=True)

evals/security-assurance/holdout/benign-path-negative/test_holdout_archive.py

@@ -0,0 +1,12 @@
+import unittest
+from pathlib import Path
+
+from archive import normalized_asset_path
+
+
+class HoldoutNegativeTests(unittest.TestCase):
+    def test_normalizer_remains_bounded_and_usable(self):
+        self.assertEqual(
+            normalized_asset_path("themes/../icons/safe.svg"),
+            Path("assets/themes/icons/safe.svg"),
+        )

evals/security-assurance/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "security-assurance"
+profile = "secure-change"
+name = "Secure Change Assurance"
+description = "Measures vulnerability recall, false positives, empirical reproduction, and remediation validation."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "vulnerability_recall"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["empirical_reproduction_rate"]
+no_regression_lower_metrics = ["false_positive_rate"]
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Opus security lead alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent Opus single-agent security control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex security control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek security control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Opus security control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM security control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter security control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-hunter"
+purpose = "Opus remediation lead plus GLM candidate vulnerability hunter."
+topology = "root_plus_worker"
+worker = "vulnerability_hunter"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Cross-maker hunt, empirical validation, remediation, and rescan."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-glm-hunter"
+purpose = "Isolates GLM 5.3's marginal vulnerability-discovery contribution."
+topology = "full_without_worker"
+worker = "vulnerability_hunter"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-empirical-verifier"
+purpose = "Ablates independent command-backed finding validation."
+topology = "full_without_worker"
+worker = "finding_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "archive-boundary"
+description = "Find and remediate a reachable archive traversal while preserving a benign path normalizer."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Perform an authorized defensive review of archive.py. Empirically reproduce consequential findings before remediation, preserve safe extraction, and write SECURITY_REPORT.md separating tested defects, evidence-backed risks, rejected candidates, commands, and observations. normalized_asset_path is an intentional negative control. End with EVAL_SECURITY_REMEDIATED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_SECURITY_REMEDIATED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_security_report.py"]
+[tasks.orchestration_assertions]
+required_agents = ["vulnerability_hunter", "finding_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "benign-path-negative"
+description = "Measures false-positive control around a deliberately benign normalizer."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Audit and remediate only empirically reachable extraction defects. Do not classify normalized_asset_path as a vulnerability merely because it processes parent components. Record the tested negative in SECURITY_REPORT.md and end with EVAL_SECURITY_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_SECURITY_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_security_report.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0

evals/visual-conformance/README.md

@@ -0,0 +1,7 @@
+# Visual-conformance evaluation
+
+This suite evaluates `visual-engineering` through the complete reference-to-render path. The reference PNG is attached to the root evaluation run. The visual analyst must inspect it, the text-only implementer receives textual criteria, and the visual verifier must receive both the original and the generated `actual.png`.
+
+`render_preview.py` is a deterministic standard-library preview renderer, so the fixture needs no browser or third-party package. It is intentionally a coarse rendering oracle: semantic HTML, responsive CSS, accessibility, and visual judgment remain separate validation responsibilities.
+
+Reference asset prompt used with the image-generation skill: “Create a polished dark-mode SaaS inventory dashboard at desktop resolution, with a left navigation rail, page header, three KPI cards, a blue inventory trend line chart, and a recent activity panel; crisp product UI, restrained navy palette, electric-blue accent, no logos.”

evals/visual-conformance/fixture/index.html

@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Inventory</title>
+  <link rel="stylesheet" href="styles.css">
+</head>
+<body>
+  <main>
+    <h1>Inventory</h1>
+    <p>Dashboard implementation pending.</p>
+  </main>
+</body>
+</html>

evals/visual-conformance/fixture/render_preview.py

@@ -0,0 +1,58 @@
+import json
+import os
+import shutil
+from pathlib import Path
+
+try:
+    from playwright.sync_api import sync_playwright
+except ImportError as exc:
+    raise SystemExit(
+        "Playwright is required for visual evaluation; install requirements-eval.txt"
+    ) from exc
+
+
+browser_binary = os.environ.get("MMO_CHROMIUM_BIN") or shutil.which("chromium")
+if not browser_binary:
+    raise SystemExit("a Chromium binary is required for real-browser visual evaluation")
+
+page_url = Path("index.html").resolve().as_uri()
+viewports = {
+    "desktop": {"width": 1440, "height": 900},
+    "mobile": {"width": 390, "height": 844},
+}
+metadata = {"engine": "playwright", "browser_binary": browser_binary, "viewports": {}}
+with sync_playwright() as playwright:
+    browser = playwright.chromium.launch(headless=True, executable_path=browser_binary)
+    try:
+        for name, viewport in viewports.items():
+            page = browser.new_page(viewport=viewport, device_scale_factor=1)
+            page.goto(page_url, wait_until="networkidle")
+            page.screenshot(path=f"actual-{name}.png", full_page=False)
+            facts = page.evaluate(
+                """
+                () => ({
+                  title: document.title,
+                  language: document.documentElement.lang,
+                  mainCount: document.querySelectorAll('main').length,
+                  navCount: document.querySelectorAll('nav').length,
+                  headingCount: document.querySelectorAll('h1, h2').length,
+                  labeledCount: document.querySelectorAll('[aria-label], label[for]').length,
+                  focusableCount: document.querySelectorAll(
+                    'a[href], button, input, select, textarea, [tabindex]:not([tabindex="-1"])'
+                  ).length,
+                  unlabeledImages: [...document.images].filter(
+                    image => !image.alt && image.getAttribute('role') !== 'presentation'
+                  ).length,
+                  horizontalOverflow: document.documentElement.scrollWidth > innerWidth + 1,
+                  bodyWidth: document.body.getBoundingClientRect().width,
+                })
+                """
+            )
+            metadata["viewports"][name] = {**viewport, **facts}
+            page.close()
+    finally:
+        browser.close()
+
+Path("render-metadata.json").write_text(
+    json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+)

evals/visual-conformance/fixture/styles.css

@@ -0,0 +1,13 @@
+:root {
+  color: #111111;
+  background: #ffffff;
+}
+
+body {
+  margin: 0;
+  font-family: sans-serif;
+}
+
+main {
+  padding: 24px;
+}

evals/visual-conformance/fixture/validate_ui.py

@@ -0,0 +1,82 @@
+import json
+from pathlib import Path
+
+from PIL import Image, ImageChops, ImageFilter, ImageStat
+
+reference_path = Path("reference.png")
+desktop_path = Path("actual-desktop.png")
+mobile_path = Path("actual-mobile.png")
+metadata_path = Path("render-metadata.json")
+for path in (reference_path, desktop_path, mobile_path, metadata_path):
+    if not path.is_file():
+        raise SystemExit(f"missing real visual artifact: {path}")
+
+metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+if metadata.get("engine") != "playwright":
+    raise SystemExit("visual render was not captured through Playwright")
+
+reference = Image.open(reference_path).convert("RGB")
+desktop = Image.open(desktop_path).convert("RGB")
+mobile = Image.open(mobile_path).convert("RGB")
+if desktop.size != (1440, 900) or mobile.size != (390, 844):
+    raise SystemExit("browser screenshots do not match the required viewports")
+
+resampling = getattr(Image, "Resampling", Image).LANCZOS
+normalized_reference = reference.resize(desktop.size, resampling)
+difference = ImageChops.difference(normalized_reference, desktop)
+mean_error = sum(ImageStat.Stat(difference).mean) / (3 * 255)
+perceptual_similarity = max(0.0, 1.0 - mean_error)
+histogram = difference.convert("L").histogram()
+matched_pixels = sum(histogram[:32])
+pixel_similarity = matched_pixels / (desktop.width * desktop.height)
+
+reference_edges = normalized_reference.convert("L").filter(ImageFilter.FIND_EDGES)
+desktop_edges = desktop.convert("L").filter(ImageFilter.FIND_EDGES)
+edge_error = ImageStat.Stat(ImageChops.difference(reference_edges, desktop_edges)).mean[0] / 255
+edge_similarity = max(0.0, 1.0 - edge_error)
+
+viewport_facts = metadata["viewports"]
+responsive_checks = [
+    not viewport_facts["desktop"]["horizontalOverflow"],
+    not viewport_facts["mobile"]["horizontalOverflow"],
+    viewport_facts["mobile"]["bodyWidth"] <= 391,
+]
+responsive_score = sum(responsive_checks) / len(responsive_checks)
+desktop_facts = viewport_facts["desktop"]
+accessibility_checks = [
+    desktop_facts["language"] == "en",
+    desktop_facts["mainCount"] == 1,
+    desktop_facts["navCount"] >= 1,
+    desktop_facts["headingCount"] >= 2,
+    desktop_facts["labeledCount"] >= 1,
+    desktop_facts["focusableCount"] >= 1,
+    desktop_facts["unlabeledImages"] == 0,
+]
+accessibility_score = sum(accessibility_checks) / len(accessibility_checks)
+
+metrics = {
+    "perceptual_similarity": perceptual_similarity,
+    "pixel_similarity": pixel_similarity,
+    "edge_similarity": edge_similarity,
+    "responsive_score": responsive_score,
+    "accessibility_score": accessibility_score,
+}
+default_thresholds = {
+    "perceptual_similarity": 0.72,
+    "pixel_similarity": 0.35,
+    "edge_similarity": 0.72,
+    "responsive_score": 1.0,
+    "accessibility_score": 1.0,
+}
+holdout = Path("visual-holdout.json")
+thresholds = (
+    json.loads(holdout.read_text(encoding="utf-8")) if holdout.is_file() else default_thresholds
+)
+if set(thresholds) != set(default_thresholds):
+    raise SystemExit("visual holdout threshold schema is invalid")
+failures = [name for name, threshold in thresholds.items() if metrics[name] < threshold]
+if failures:
+    raise SystemExit(
+        "visual gates failed: " + ", ".join(f"{name}={metrics[name]:.3f}" for name in failures)
+    )
+print(json.dumps({"metrics": metrics}, sort_keys=True))

evals/visual-conformance/holdout/inventory-dashboard/visual-holdout.json

@@ -0,0 +1,7 @@
+{
+  "perceptual_similarity": 0.72,
+  "pixel_similarity": 0.35,
+  "edge_similarity": 0.72,
+  "responsive_score": 1.0,
+  "accessibility_score": 1.0
+}

evals/visual-conformance/holdout/visual-blocker-negative/visual-holdout.json

@@ -0,0 +1,7 @@
+{
+  "perceptual_similarity": 0.72,
+  "pixel_similarity": 0.35,
+  "edge_similarity": 0.72,
+  "responsive_score": 1.0,
+  "accessibility_score": 1.0
+}

evals/visual-conformance/suite.toml

@@ -0,0 +1,109 @@
+schema_version = 8
+id = "visual-conformance"
+profile = "visual-engineering"
+name = "Visual Engineering Conformance"
+description = "Real-browser multimodal trials with reference/render, responsive, and accessibility gates."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "perceptual_similarity"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["pixel_similarity", "edge_similarity", "accessibility_score", "responsive_score"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Multimodal Sol implementer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent multimodal Sol single-agent control."
+profile = "visual-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "Only accessible route in the bundled catalog that preserves this profile's complete image and tool-image path."
+profile = "visual-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "root-plus-verifier"
+purpose = "Direct multimodal implementation plus fresh reference/render verification."
+topology = "root_plus_worker"
+worker = "visual_verifier"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Optional multimodal analysis, direct implementation, and fresh screenshot verification."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-analyst"
+purpose = "Ablates optional multi-screen visual reconnaissance."
+topology = "full_without_worker"
+worker = "visual_analyst"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates fresh reference-versus-render verification."
+topology = "full_without_worker"
+worker = "visual_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "inventory-dashboard"
+description = "Rebuild a responsive dashboard from the original image and verify actual browser renders."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 180
+prompt = """
+Rebuild the inventory dashboard in reference.png using semantic index.html and styles.css. Preserve the original image through every visual handoff. Implement directly as the multimodal root. Run python render_preview.py to capture real Playwright desktop and mobile screenshots, compare the original reference to the desktop render, inspect responsive behavior and keyboard/accessibility evidence, correct material mismatches, and end with EVAL_VISUAL_VERIFIED.
+"""
+images = ["reference.png"]
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_VISUAL_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python render_preview.py && python validate_ui.py"]
+[tasks.orchestration_assertions]
+required_agents = ["visual_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "visual-blocker-negative"
+description = "Negative control: source-text assertions cannot substitute for browser and pixel evidence."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 180
+prompt = """
+Implement the supplied reference, but do not claim visual success from HTML, CSS, preview metadata, or your own textual description. A pass requires real Playwright screenshots at both widths and measured reference/render evidence. End with EVAL_VISUAL_EVIDENCE.
+"""
+images = ["reference.png"]
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_VISUAL_EVIDENCE"]
+forbidden_patterns = []
+validation_commands = ["python render_preview.py && python validate_ui.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0