😏

evals/contract-refactoring/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Versioned payload contract
+
+`upgrade_payload` must return a new mapping, preserve unknown fields, read legacy `userId`, write canonical `user_id`, remove only the legacy alias, reject conflicting aliases, and never mutate caller-owned nested data.
+

evals/contract-refactoring/fixture/codec.py

@@ -0,0 +1,8 @@
+from typing import Any
+
+
+def upgrade_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    result = dict(payload)
+    if "userId" in result:
+        result["user_id"] = result.pop("userId")
+    return result

evals/contract-refactoring/fixture/test_codec.py

@@ -0,0 +1,22 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class CodecTests(unittest.TestCase):
+    def test_legacy_alias_is_upgraded(self):
+        self.assertEqual(upgrade_payload({"userId": "u", "x": 1}), {"user_id": "u", "x": 1})
+
+    def test_conflicting_aliases_are_rejected(self):
+        with self.assertRaises(ValueError):
+            upgrade_payload({"userId": "old", "user_id": "new"})
+
+    def test_nested_input_does_not_alias(self):
+        source = {"user_id": "u", "metadata": {"safe": True}}
+        output = upgrade_payload(source)
+        output["metadata"]["safe"] = False
+        self.assertTrue(source["metadata"]["safe"])
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/contract-refactoring/fixture/validate_contract.py

@@ -0,0 +1,28 @@
+import hashlib
+import json
+from pathlib import Path
+
+contract = Path("CONTRACT.md")
+digest = Path("CONTRACT.sha256")
+if not contract.is_file() or not digest.is_file():
+    raise SystemExit("CONTRACT.md and CONTRACT.sha256 are required")
+actual = hashlib.sha256(contract.read_bytes()).hexdigest()
+if digest.read_text(encoding="utf-8").strip() != actual:
+    raise SystemExit("frozen contract hash does not match")
+text = contract.read_text(encoding="utf-8").lower()
+required = ("legacy", "unknown", "conflict", "immut", "user_id")
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("frozen contract misses invariants: " + ", ".join(missing))
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "hidden_invariant_recall": 1.0,
+                "contract_hash_integrity": 1.0,
+                "compatibility_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/contract-refactoring/holdout/plausible-breaking-negative/test_holdout_codec.py

@@ -0,0 +1,10 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class HoldoutCompatibilityTests(unittest.TestCase):
+    def test_canonical_payload_remains_canonical(self):
+        self.assertEqual(
+            upgrade_payload({"user_id": "u", "future": 3}), {"user_id": "u", "future": 3}
+        )

evals/contract-refactoring/holdout/versioned-payload-refactor/test_holdout_codec.py

@@ -0,0 +1,15 @@
+import unittest
+
+from codec import upgrade_payload
+
+
+class HoldoutCodecTests(unittest.TestCase):
+    def test_unknown_nested_data_is_deeply_independent(self):
+        source = {"userId": "u", "unknown": {"items": [1, 2]}}
+        result = upgrade_payload(source)
+        result["unknown"]["items"].append(3)
+        self.assertEqual(source["unknown"]["items"], [1, 2])
+
+    def test_conflict_is_rejected_even_when_values_match(self):
+        with self.assertRaises(ValueError):
+            upgrade_payload({"userId": "u", "user_id": "u"})

evals/contract-refactoring/suite.toml

@@ -0,0 +1,148 @@
+schema_version = 8
+id = "contract-refactoring"
+profile = "contract-first-refactoring"
+name = "Contract-First Refactoring"
+description = "Measures frozen-contract fidelity, hidden compatibility invariants, independent tests, and regression verification."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "hidden_invariant_recall"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["contract_hash_integrity", "compatibility_rate"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sonnet refactor lead alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent compatibility refactor control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex refactor control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go refactor control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Sonnet refactor control."
+profile = "contract-first-refactoring"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan refactor control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter refactor control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-test-author"
+purpose = "Frozen-contract root plus independent contract-derived tests."
+topology = "root_plus_worker"
+worker = "contract_test_author"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Invariant mining, frozen contract, test-first patch, refactor, and verification."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-invariant-miner"
+purpose = "Ablates long-context compatibility mining."
+topology = "full_without_worker"
+worker = "invariant_miner"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-test-author"
+purpose = "Ablates independent test-first specification."
+topology = "full_without_worker"
+worker = "contract_test_author"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates economical independent compatibility verification."
+topology = "full_without_worker"
+worker = "compatibility_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "versioned-payload-refactor"
+description = "Refactor a versioned serializer without losing legacy read compatibility or input immutability."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Refactor codec.py under SPEC.md. Before tests or implementation, write and hash CONTRACT.md into CONTRACT.sha256. Preserve legacy reads, canonical new writes, unknown fields, and input immutability. Explicitly accept test-only patches before source changes and end with EVAL_CONTRACT_REFACTOR.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_CONTRACT_REFACTOR"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_contract.py"]
+[tasks.orchestration_assertions]
+required_agents = ["invariant_miner", "contract_test_author", "compatibility_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "plausible-breaking-negative"
+description = "Negative control that catches a superficially clean but compatibility-breaking rewrite."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Improve codec.py without deleting legacy field support or mutating caller data. Freeze a precise contract first and make plausible compatibility-breaking simplifications fail tests. End with EVAL_CONTRACT_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_CONTRACT_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_contract.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0