41 lines
1.8 KiB
Python
41 lines
1.8 KiB
Python
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
|
||
|
|
holdout_path = Path("incident-holdout.json")
|
||
|
|
holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
|
||
|
|
is_negative = "eval_incident_unresolved" in text
|
||
|
|
required_event_ids = holdout.get("required_event_ids", ["api-02", "worker-01", "api-03", "api-04"])
|
||
|
|
minimum_event_ids = int(holdout.get("minimum_event_ids", 2))
|
||
|
|
alternatives = holdout.get("alternative_terms", ["dns", "database", "cache"])
|
||
|
|
minimum_alternatives = int(holdout.get("minimum_alternatives", 2))
|
||
|
|
event_ids = sum(event_id.lower() in text for event_id in required_event_ids)
|
||
|
|
has_alternatives = sum(term.lower() in text for term in alternatives) >= minimum_alternatives
|
||
|
|
if event_ids < minimum_event_ids or not has_alternatives:
|
||
|
|
raise SystemExit("incident report lacks correlated event IDs or competing hypotheses")
|
||
|
|
forbidden_certainty = holdout.get("forbidden_certainty", ["root cause is"])
|
||
|
|
unsupported = float(is_negative and any(term.lower() in text for term in forbidden_certainty))
|
||
|
|
if unsupported:
|
||
|
|
raise SystemExit("negative control asserted an unsupported root cause")
|
||
|
|
required_degradation_terms = holdout.get("required_degradation_terms", [])
|
||
|
|
missing_degradation_terms = [
|
||
|
|
term for term in required_degradation_terms if term.lower() not in text
|
||
|
|
]
|
||
|
|
if missing_degradation_terms:
|
||
|
|
raise SystemExit(
|
||
|
|
"incident report did not preserve provider-failure evidence: "
|
||
|
|
+ ", ".join(missing_degradation_terms)
|
||
|
|
)
|
||
|
|
print(
|
||
|
|
json.dumps(
|
||
|
|
{
|
||
|
|
"metrics": {
|
||
|
|
"evidence_backed_diagnosis": 1.0,
|
||
|
|
"hypothesis_discrimination": 1.0,
|
||
|
|
"unsupported_root_cause_rate": unsupported,
|
||
|
|
}
|
||
|
|
},
|
||
|
|
sort_keys=True,
|
||
|
|
)
|
||
|
|
)
|