😏

evals/incident-triage/fixture/RUNBOOK.md

@@ -0,0 +1,4 @@
+# Checkout incident threshold
+
+A root cause requires two independent, time-correlated event streams plus a prediction that is observed after an operator-controlled change. Otherwise report ranked unresolved hypotheses and next discriminating observations.
+

evals/incident-triage/fixture/logs.jsonl

@@ -0,0 +1,9 @@
+{"id":"api-01","time":"10:01:00Z","service":"api","event":"deploy","version":"2026.08.16.2","cache_key":"tenant"}
+{"id":"dns-01","time":"10:01:05Z","service":"dns","event":"retry","count":1}
+{"id":"api-02","time":"10:02:00Z","service":"api","event":"checkout_wrong_currency","tenant":"west"}
+{"id":"worker-01","time":"10:02:01Z","service":"worker","event":"cache_hit","key":"west","currency":"EUR"}
+{"id":"db-01","time":"10:02:02Z","service":"db","event":"latency_ms","value":18}
+{"id":"api-03","time":"10:06:00Z","service":"api","event":"rollback","version":"2026.08.16.1"}
+{"id":"worker-02","time":"10:06:20Z","service":"worker","event":"cache_key","key":"west:USD"}
+{"id":"api-04","time":"10:07:00Z","service":"api","event":"checkout_correct_currency","tenant":"west"}
+

evals/incident-triage/fixture/validate_incident.py

@@ -0,0 +1,40 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("incident-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+is_negative = "eval_incident_unresolved" in text
+required_event_ids = holdout.get("required_event_ids", ["api-02", "worker-01", "api-03", "api-04"])
+minimum_event_ids = int(holdout.get("minimum_event_ids", 2))
+alternatives = holdout.get("alternative_terms", ["dns", "database", "cache"])
+minimum_alternatives = int(holdout.get("minimum_alternatives", 2))
+event_ids = sum(event_id.lower() in text for event_id in required_event_ids)
+has_alternatives = sum(term.lower() in text for term in alternatives) >= minimum_alternatives
+if event_ids < minimum_event_ids or not has_alternatives:
+    raise SystemExit("incident report lacks correlated event IDs or competing hypotheses")
+forbidden_certainty = holdout.get("forbidden_certainty", ["root cause is"])
+unsupported = float(is_negative and any(term.lower() in text for term in forbidden_certainty))
+if unsupported:
+    raise SystemExit("negative control asserted an unsupported root cause")
+required_degradation_terms = holdout.get("required_degradation_terms", [])
+missing_degradation_terms = [
+    term for term in required_degradation_terms if term.lower() not in text
+]
+if missing_degradation_terms:
+    raise SystemExit(
+        "incident report did not preserve provider-failure evidence: "
+        + ", ".join(missing_degradation_terms)
+    )
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "evidence_backed_diagnosis": 1.0,
+                "hypothesis_discrimination": 1.0,
+                "unsupported_root_cause_rate": unsupported,
+            }
+        },
+        sort_keys=True,
+    )
+)