😏

evals/debugging-confidence/README.md

@@ -0,0 +1,3 @@
+# Debugging confidence evaluation
+
+Designed for `high-confidence-debugging`. The fixture contains a state-dependent cache bug and a misleading obvious hypothesis. Success requires a failing reproduction, a minimal fix, regression tests, and explicit acceptance or rejection of verifier results.

evals/debugging-confidence/fixture/policy.py

@@ -0,0 +1,13 @@
+class Policy:
+    def __init__(self, flags=None):
+        self.flags = dict(flags or {})
+        self._decision_cache = {}
+
+    def set_flag(self, name, enabled):
+        self.flags[name] = bool(enabled)
+
+    def allows(self, user_id, flag):
+        key = (user_id, flag)
+        if key not in self._decision_cache:
+            self._decision_cache[key] = bool(self.flags.get(flag, False))
+        return self._decision_cache[key]

evals/debugging-confidence/fixture/test_policy.py

@@ -0,0 +1,23 @@
+import unittest
+
+from policy import Policy
+
+
+class PolicyTests(unittest.TestCase):
+    def test_initial_decision_uses_current_flag(self):
+        self.assertTrue(Policy({"preview": True}).allows("u1", "preview"))
+
+    def test_users_have_independent_cache_entries(self):
+        policy = Policy({"preview": True})
+        self.assertTrue(policy.allows("u1", "preview"))
+        self.assertTrue(policy.allows("u2", "preview"))
+
+    def test_setting_flag_invalidates_prior_decisions(self):
+        policy = Policy({"preview": False})
+        self.assertFalse(policy.allows("u1", "preview"))
+        policy.set_flag("preview", True)
+        self.assertTrue(policy.allows("u1", "preview"))
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/debugging-confidence/holdout/misleading-cache-symptom/test_holdout_policy.py

@@ -0,0 +1,17 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_true_to_false_toggle_invalidates(self):
+        policy = Policy({"preview": True})
+        self.assertTrue(policy.allows("u1", "preview"))
+        policy.set_flag("preview", False)
+        self.assertFalse(policy.allows("u1", "preview"))
+
+    def test_repeated_toggles_do_not_reuse_stale_value(self):
+        policy = Policy({"preview": False})
+        for value in (True, False, True):
+            policy.set_flag("preview", value)
+            self.assertEqual(policy.allows("u1", "preview"), value)

evals/debugging-confidence/holdout/no-speculative-rewrite/test_holdout_policy.py

@@ -0,0 +1,12 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_unrelated_flag_change_preserves_correct_answers(self):
+        policy = Policy({"a": True, "b": False})
+        self.assertTrue(policy.allows("u", "a"))
+        policy.set_flag("b", True)
+        self.assertTrue(policy.allows("u", "a"))
+        self.assertTrue(policy.allows("u", "b"))

evals/debugging-confidence/mutations/misleading-cache-symptom/01-enable-only-invalidation.patch

@@ -0,0 +1,12 @@
+diff --git a/policy.py b/policy.py
+--- a/policy.py
++++ b/policy.py
+@@ -5,6 +5,8 @@ class Policy:
+ 
+     def set_flag(self, name, enabled):
+         self.flags[name] = bool(enabled)
++        if enabled:
++            self._decision_cache.clear()
+ 
+     def allows(self, user_id, flag):
+         key = (user_id, flag)

evals/debugging-confidence/mutations/misleading-cache-symptom/02-wrong-key-invalidation.patch

@@ -0,0 +1,12 @@
+diff --git a/policy.py b/policy.py
+--- a/policy.py
++++ b/policy.py
+@@ -5,6 +5,8 @@ class Policy:
+ 
+     def set_flag(self, name, enabled):
+         self.flags[name] = bool(enabled)
++        for key in [key for key in self._decision_cache if key[0] == name]:
++            del self._decision_cache[key]
+ 
+     def allows(self, user_id, flag):
+         key = (user_id, flag)

evals/debugging-confidence/suite.toml

@@ -0,0 +1,142 @@
+schema_version = 8
+id = "debugging-confidence"
+profile = "high-confidence-debugging"
+name = "High-Confidence Debugging"
+description = "Matched misleading-symptom trials for independent reproduction, repair, and post-fix falsification."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "DeepSeek diagnostician alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent debugging control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex debugging control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go debugging control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen debugging control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan debugging control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter debugging control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-reproducer"
+purpose = "Root plus independent reproduction and regression-test derivation."
+topology = "root_plus_worker"
+worker = "independent_reproducer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Reproduce, repair, and fresh-context falsification topology."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-reproducer"
+purpose = "Ablates independent reproduction."
+topology = "full_without_worker"
+worker = "independent_reproducer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates fresh post-fix adversarial verification."
+topology = "full_without_worker"
+worker = "fix_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "misleading-cache-symptom"
+description = "Repair stale decisions without accepting the visible parser as an unsupported cause."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Users report that changing a feature flag does not affect repeated decisions. Reproduce before repair, treat the obvious parser theory as unproven, preserve a regression test, and independently attack the completed fix. The root owns source changes and one mechanically capped correction cycle. End with EVAL_DEBUG_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_DEBUG_VERIFIED"]
+forbidden_patterns = ["parser was definitely"]
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["independent_reproducer", "fix_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+min_result_acceptance_rate = 0.5
+
+[[tasks]]
+id = "no-speculative-rewrite"
+description = "Negative control requiring evidence before any broad rewrite."
+sandbox = "workspace-write"
+difficulty = "medium"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Diagnose and repair the repeated-decision failure. Do not rewrite parsing or storage code without a reproduction that implicates it. Preserve existing behavior and end with EVAL_DEBUG_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_DEBUG_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0