😏

evals/bounded-research/fixture/validate_research.py

@@ -0,0 +1,20 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("research-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run research holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+required = [*holdout["required_values"], *holdout["required_sources"]]
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("research synthesis misses authoritative facts: " + ", ".join(missing))
+if "seven" in text and not any(term in text for term in ("obsolete", "archived", "reject")):
+    raise SystemExit("archived retry value was not rejected")
+print(
+    json.dumps(
+        {"metrics": {"source_coverage": 1.0, "contradiction_resolution": 1.0}},
+        sort_keys=True,
+    )
+)