😏

evals/adaptive-change/README.md

@@ -0,0 +1,5 @@
+# Adaptive change evaluation
+
+This suite exercises the `adaptive-engineering` profile on a bounded but nontrivial correctness change. The root must keep ownership of diagnosis and integration, use a specialist only where it adds value, and obtain independent review before claiming success.
+
+The native scout is intentionally not a worker assertion: native Codex participation is not represented in Agent MCP job telemetry. The evaluator does require the MCP implementation and review roles and measures whether their results were consumed.

evals/adaptive-change/fixture/SPEC.md

@@ -0,0 +1,11 @@
+# Ledger invariants
+
+`Ledger.apply(operation_id, account, delta)` applies an operation at most once globally.
+
+- A successful operation permanently consumes its operation ID.
+- A rejected overdraft must not consume its operation ID.
+- A caller may fund the account and retry the same rejected operation.
+- Balances may never become negative.
+- Duplicate successful operations return the current balance without applying the delta again.
+
+The current implementation violates the rejected-operation rule.

evals/adaptive-change/fixture/ledger.py

@@ -0,0 +1,17 @@
+class Ledger:
+    def __init__(self) -> None:
+        self._balances: dict[str, int] = {}
+        self._applied: set[str] = set()
+
+    def balance(self, account: str) -> int:
+        return self._balances.get(account, 0)
+
+    def apply(self, operation_id: str, account: str, delta: int) -> int:
+        if operation_id in self._applied:
+            return self.balance(account)
+        self._applied.add(operation_id)
+        updated = self.balance(account) + delta
+        if updated < 0:
+            raise ValueError("insufficient funds")
+        self._balances[account] = updated
+        return updated

evals/adaptive-change/fixture/test_ledger.py

@@ -0,0 +1,27 @@
+import unittest
+
+from ledger import Ledger
+
+
+class LedgerTests(unittest.TestCase):
+    def test_success_is_idempotent(self):
+        ledger = Ledger()
+        self.assertEqual(ledger.apply("deposit-1", "a", 10), 10)
+        self.assertEqual(ledger.apply("deposit-1", "a", 10), 10)
+
+    def test_overdraft_does_not_change_balance(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("withdraw-1", "a", -4)
+        self.assertEqual(ledger.balance("a"), 0)
+
+    def test_rejected_operation_can_be_retried(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("withdraw-1", "a", -4)
+        ledger.apply("deposit-1", "a", 10)
+        self.assertEqual(ledger.apply("withdraw-1", "a", -4), 6)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/adaptive-change/holdout/decomposable-ledger/test_holdout_ledger.py

@@ -0,0 +1,16 @@
+import unittest
+
+from ledger import Ledger
+
+
+class HoldoutLedgerTests(unittest.TestCase):
+    def test_rejected_id_remains_globally_available(self):
+        ledger = Ledger()
+        with self.assertRaises(ValueError):
+            ledger.apply("shared", "a", -1)
+        self.assertEqual(ledger.apply("shared", "b", 3), 3)
+
+    def test_successful_id_is_global_across_accounts(self):
+        ledger = Ledger()
+        ledger.apply("shared", "a", 3)
+        self.assertEqual(ledger.apply("shared", "b", 9), 0)

evals/adaptive-change/holdout/tight-small-change/test_holdout_ledger.py

@@ -0,0 +1,13 @@
+import unittest
+
+from ledger import Ledger
+
+
+class HoldoutLedgerTests(unittest.TestCase):
+    def test_multiple_rejections_do_not_consume_id(self):
+        ledger = Ledger()
+        for _ in range(2):
+            with self.assertRaises(ValueError):
+                ledger.apply("withdraw", "a", -2)
+        ledger.apply("fund", "a", 4)
+        self.assertEqual(ledger.apply("withdraw", "a", -2), 2)

evals/adaptive-change/suite.toml

@@ -0,0 +1,149 @@
+schema_version = 8
+id = "adaptive-change"
+profile = "adaptive-engineering"
+name = "Adaptive Engineering Change"
+description = "Matched trials for selective delegation on decomposable and tightly coupled engineering work."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Configured Sol root with delegation mechanically disabled."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Task-specific flagship Sol single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access-service control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go economical DeepSeek control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter DeepSeek route control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-implementation"
+purpose = "Configured root plus the objectively testable implementation branch."
+topology = "root_plus_worker"
+worker = "implementation_specialist"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Complete selective adaptive topology."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-scout"
+purpose = "Ablates native reconnaissance."
+topology = "full_without_worker"
+worker = "repo_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-implementation"
+purpose = "Ablates the economical isolated implementation branch."
+topology = "full_without_worker"
+worker = "implementation_specialist"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-review"
+purpose = "Ablates independent cross-maker review."
+topology = "full_without_worker"
+worker = "adversarial_reviewer"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "decomposable-ledger"
+description = "Repair an idempotency defect with separable reconnaissance, implementation, and review work."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Repair the ledger defect described in SPEC.md. Own diagnosis and canonical integration. Delegate only work with a concrete expected contribution, continue non-overlapping critical-path work while it runs, explicitly accept or reject results, and end with EVAL_ADAPTIVE_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ADAPTIVE_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["implementation_specialist", "adversarial_reviewer"]
+max_jobs = 3
+max_contract_failures = 0
+max_observed_mcp_wait_ratio = 0.8
+
+[[tasks]]
+id = "tight-small-change"
+description = "Negative control where worker launch overhead should not be required."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 120
+prompt = """
+Make the smallest correct ledger repair. This task is intentionally compact: remain single-agent unless independent work has a concrete positive expected value. Run the tests and end with EVAL_ADAPTIVE_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ADAPTIVE_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 1
+max_contract_failures = 0