22 lines
1.1 KiB
Python
22 lines
1.1 KiB
Python
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
|
||
|
|
holdout_path = Path("route-holdout.json")
|
||
|
|
holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
|
||
|
|
required = holdout.get("required_identity_terms", ["schema", "endpoint", "telemetry"])
|
||
|
|
missing = [term for term in required if term not in text]
|
||
|
|
if missing:
|
||
|
|
raise SystemExit("route report lacks identity evidence: " + ", ".join(missing))
|
||
|
|
if "eval_route_degraded" in text and not any(
|
||
|
|
term in text for term in ("unavailable", "degraded", "credential")
|
||
|
|
):
|
||
|
|
raise SystemExit("degraded route was not reported")
|
||
|
|
unavailable_route = holdout.get("unavailable_route")
|
||
|
|
if unavailable_route and unavailable_route.lower() not in text:
|
||
|
|
raise SystemExit("degraded report does not identify the unavailable route")
|
||
|
|
required_fault_terms = holdout.get("required_fault_terms", [])
|
||
|
|
if required_fault_terms and not any(term.lower() in text for term in required_fault_terms):
|
||
|
|
raise SystemExit("degraded report does not identify the injected route fault")
|
||
|
|
print(json.dumps({"metrics": {"route_consistency": 1.0}}, sort_keys=True))
|