😏

evals/competing-implementations/fixture/SPEC.md

@@ -0,0 +1,4 @@
+# Stable unique contract
+
+`stable_unique(values, key)` returns the first value for each distinct key, preserves encounter order, accepts unhashable values when the supplied key result is hashable, never mutates input, and must scale linearly for 20,000 values.
+

evals/competing-implementations/fixture/benchmark.py

@@ -0,0 +1,26 @@
+import json
+import time
+
+from dedupe import stable_unique
+
+values = [index % 5000 for index in range(20_000)]
+started = time.perf_counter()
+result = stable_unique(values, lambda value: value)
+elapsed = time.perf_counter() - started
+if result != list(range(5000)):
+    raise SystemExit("stable_unique produced an incorrect result")
+if elapsed > 1.0:
+    raise SystemExit(f"benchmark exceeded one second: {elapsed:.3f}")
+quality = max(0.0, 1.0 - elapsed)
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "benchmark_quality": quality,
+                "correctness_rate": 1.0,
+                "maintainability_score": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/competing-implementations/fixture/dedupe.py

@@ -0,0 +1,10 @@
+from collections.abc import Callable, Iterable
+from typing import Any
+
+
+def stable_unique(values: Iterable[Any], key: Callable[[Any], Any]) -> list[Any]:
+    result = []
+    for value in values:
+        if not any(key(existing) == key(value) for existing in result):
+            result.append(value)
+    return result

evals/competing-implementations/fixture/test_dedupe.py

@@ -0,0 +1,19 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class DedupeTests(unittest.TestCase):
+    def test_preserves_first_and_order(self):
+        values = [{"id": 2, "v": "a"}, {"id": 1}, {"id": 2, "v": "b"}]
+        self.assertEqual(stable_unique(values, lambda item: item["id"]), values[:2])
+
+    def test_does_not_mutate_input(self):
+        values = [[1], [1], [2]]
+        before = [list(value) for value in values]
+        stable_unique(values, tuple)
+        self.assertEqual(values, before)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/competing-implementations/holdout/stable-deduplication/test_holdout_dedupe.py

@@ -0,0 +1,16 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class HoldoutDedupeTests(unittest.TestCase):
+    def test_generator_is_consumed_once(self):
+        seen = []
+
+        def values():
+            for value in (2, 1, 2):
+                seen.append(value)
+                yield value
+
+        self.assertEqual(stable_unique(values(), lambda value: value), [2, 1])
+        self.assertEqual(seen, [2, 1, 2])

evals/competing-implementations/holdout/tiny-change-negative/test_holdout_dedupe.py

@@ -0,0 +1,8 @@
+import unittest
+
+from dedupe import stable_unique
+
+
+class HoldoutDedupeTests(unittest.TestCase):
+    def test_empty_input(self):
+        self.assertEqual(stable_unique([], lambda value: value), [])

evals/competing-implementations/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "competing-implementations"
+profile = "competing-implementations-lab"
+name = "Competing Implementations Lab"
+description = "Experimental matched alternatives selected by tests, benchmarks, patch review, and integration effort."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "benchmark_quality"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["correctness_rate", "maintainability_score"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Terra contract author and implementer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-implementation control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek candidate control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Sonnet candidate control."
+profile = "contract-first-refactoring"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan access control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter access control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "one-deepseek-candidate"
+purpose = "Objective judge plus one economical implementation candidate."
+topology = "root_plus_worker"
+worker = "deepseek_candidate"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-competition"
+purpose = "Two independently isolated implementations with evidence-based selection."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-deepseek"
+purpose = "Ablates the Go-hosted DeepSeek candidate."
+topology = "full_without_worker"
+worker = "deepseek_candidate"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-sonnet"
+purpose = "Ablates the Zen-hosted Sonnet candidate."
+topology = "full_without_worker"
+worker = "sonnet_candidate"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "stable-deduplication"
+description = "Choose between independent correct and scalable stable-deduplication patches."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Freeze the objective contract in SPEC.md and its benchmark before implementation. When workers are available, launch independent candidates into disjoint worktrees. Compare returned binary patches using tests, benchmark evidence, complexity, maintainability, and integration corrections; never vote by model identity. Explicitly integrate only the selected patch and end with EVAL_COMPETING_SELECTED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_COMPETING_SELECTED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python benchmark.py"]
+[tasks.orchestration_assertions]
+required_agents = ["deepseek_candidate", "sonnet_candidate"]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "tiny-change-negative"
+description = "Negative control where two full implementations may cost more than they add."
+sandbox = "workspace-write"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Implement stable_unique correctly. The task is intentionally compact; launch competing writers only if the expected selection value exceeds two patches and integration work. End with EVAL_COMPETING_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_COMPETING_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python benchmark.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2
+max_contract_failures = 0