😏

evals/access-efficient/fixture/validate_access.py

@@ -0,0 +1,23 @@
+import json
+import sys
+from pathlib import Path
+
+mode = sys.argv[1]
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("access-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+if mode == "literal" and not (
+    holdout.get("expected_value", "128").lower() in text
+    and holdout.get("expected_path", "LIMITS.toml").lower() in text
+):
+    raise SystemExit("literal result is not grounded in the hidden expected location")
+if mode == "routine":
+    from labels import slug_label
+
+    if slug_label("  Release__ Candidate  ") != "release-candidate":
+        raise SystemExit("routine slug contract still fails")
+if mode == "decision":
+    required = holdout.get("required_terms", ["unique", "durable", "crash", "external", "local"])
+    missing = [term for term in required if term not in text]
+    if missing:
+        raise SystemExit("decision lacks constraints: " + ", ".join(missing))