21 lines
834 B
Python
21 lines
834 B
Python
import json
|
|
from pathlib import Path
|
|
|
|
text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
|
|
holdout_path = Path("research-holdout.json")
|
|
if not holdout_path.is_file():
|
|
raise SystemExit("post-run research holdout is required")
|
|
holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
|
|
required = [*holdout["required_values"], *holdout["required_sources"]]
|
|
missing = [term for term in required if term not in text]
|
|
if missing:
|
|
raise SystemExit("research synthesis misses authoritative facts: " + ", ".join(missing))
|
|
if "seven" in text and not any(term in text for term in ("obsolete", "archived", "reject")):
|
|
raise SystemExit("archived retry value was not rejected")
|
|
print(
|
|
json.dumps(
|
|
{"metrics": {"source_coverage": 1.0, "contradiction_resolution": 1.0}},
|
|
sort_keys=True,
|
|
)
|
|
)
|