😏

evals/codex-harness/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Deep merge contract
+
+`merge_settings(base, overlay)` returns a fresh mapping. Nested mappings merge recursively; overlay scalars replace base values; no input or nested output may be mutated or aliased.
+

evals/codex-harness/fixture/settings.py

@@ -0,0 +1,12 @@
+from collections.abc import Mapping
+from typing import Any
+
+
+def merge_settings(base: Mapping[str, Any], overlay: Mapping[str, Any]) -> dict[str, Any]:
+    result = dict(base)
+    for key, value in overlay.items():
+        if isinstance(value, Mapping) and isinstance(result.get(key), Mapping):
+            result[key].update(value)
+        else:
+            result[key] = value
+    return result

evals/codex-harness/fixture/test_settings.py

@@ -0,0 +1,23 @@
+import unittest
+
+from settings import merge_settings
+
+
+class MergeSettingsTests(unittest.TestCase):
+    def test_recursive_merge(self):
+        self.assertEqual(
+            merge_settings({"s": {"host": "x", "port": 80}}, {"s": {"port": 443}}),
+            {"s": {"host": "x", "port": 443}},
+        )
+
+    def test_inputs_and_outputs_do_not_alias(self):
+        base = {"s": {"host": "x"}, "flags": {"safe": True}}
+        overlay = {"s": {"port": 443}}
+        merged = merge_settings(base, overlay)
+        merged["s"]["host"] = "changed"
+        merged["flags"]["safe"] = False
+        self.assertEqual(base, {"s": {"host": "x"}, "flags": {"safe": True}})
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/codex-harness/holdout/deep-merge-contract/test_holdout_settings.py

@@ -0,0 +1,13 @@
+import unittest
+
+from settings import merge_settings
+
+
+class HoldoutMergeTests(unittest.TestCase):
+    def test_three_level_merge_and_no_alias(self):
+        base = {"a": {"b": {"left": 1}}, "untouched": {"x": 1}}
+        overlay = {"a": {"b": {"right": 2}}}
+        merged = merge_settings(base, overlay)
+        self.assertEqual(merged["a"]["b"], {"left": 1, "right": 2})
+        merged["untouched"]["x"] = 9
+        self.assertEqual(base["untouched"]["x"], 1)

evals/codex-harness/holdout/small-merge-control/test_holdout_settings.py

@@ -0,0 +1,8 @@
+import unittest
+
+from settings import merge_settings
+
+
+class HoldoutMergeTests(unittest.TestCase):
+    def test_overlay_scalar_replaces_mapping(self):
+        self.assertEqual(merge_settings({"a": {"x": 1}}, {"a": 3}), {"a": 3})

evals/codex-harness/suite.toml

@@ -0,0 +1,156 @@
+schema_version = 8
+id = "codex-harness"
+profile = "codex-harness-team"
+name = "Codex Harness Team"
+description = "Tests native-first context isolation, homogeneous Sol peers, and a fresh supervised critic."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "success_rate"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sol root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent strongest single-agent Sol control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex service control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek service control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude service control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM service control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter service control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "pure-native"
+purpose = "Root plus the highest-value native invariant designer."
+topology = "root_plus_worker"
+worker = "invariant_designer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "homogeneous-sol"
+purpose = "Full isolated-role control using Sol for every participant."
+topology = "full"
+comparison_class = "control"
+[variants.bindings]
+repo_scout = "codex_chatgpt_builtin__gpt_5_6_sol"
+invariant_designer = "codex_chatgpt_builtin__gpt_5_6_sol"
+
+[[variants]]
+id = "full-profile"
+purpose = "Native-first Luna/Terra team followed by a fresh Sol critic."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-scout"
+purpose = "Ablates fast repository reconnaissance."
+topology = "full_without_worker"
+worker = "repo_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-invariants"
+purpose = "Ablates isolated invariant and test design."
+topology = "full_without_worker"
+worker = "invariant_designer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-critic"
+purpose = "Ablates fresh-context adversarial review."
+topology = "full_without_worker"
+worker = "fresh_critic"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "deep-merge-contract"
+description = "Correct a mutation-prone recursive merge under independently derivable invariants."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Implement the merge contract in SPEC.md. The root owns the change. Launch isolated read-only contexts only when they can derive tests, invariants, or critique concurrently; inspect and adjudicate their evidence. End with EVAL_HARNESS_VERIFIED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_HARNESS_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+required_agents = ["fresh_critic"]
+max_jobs = 1
+max_contract_failures = 0
+
+[[tasks]]
+id = "small-merge-control"
+description = "Negative control for context-launch overhead on a compact implementation."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 120
+prompt = """
+Repair merge_settings with the smallest coherent change. Avoid launching peers unless they have a specific expected contribution. Run the suite and end with EVAL_HARNESS_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_HARNESS_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v"]
+[tasks.orchestration_assertions]
+max_jobs = 1