😏

evals/route-resilience/fixture/validate_route.py

@@ -0,0 +1,21 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("route-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+required = holdout.get("required_identity_terms", ["schema", "endpoint", "telemetry"])
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("route report lacks identity evidence: " + ", ".join(missing))
+if "eval_route_degraded" in text and not any(
+    term in text for term in ("unavailable", "degraded", "credential")
+):
+    raise SystemExit("degraded route was not reported")
+unavailable_route = holdout.get("unavailable_route")
+if unavailable_route and unavailable_route.lower() not in text:
+    raise SystemExit("degraded report does not identify the unavailable route")
+required_fault_terms = holdout.get("required_fault_terms", [])
+if required_fault_terms and not any(term.lower() in text for term in required_fault_terms):
+    raise SystemExit("degraded report does not identify the injected route fault")
+print(json.dumps({"metrics": {"route_consistency": 1.0}}, sort_keys=True))