😏

evals/incident-triage/fixture/RUNBOOK.md

@@ -0,0 +1,4 @@
+# Checkout incident threshold
+
+A root cause requires two independent, time-correlated event streams plus a prediction that is observed after an operator-controlled change. Otherwise report ranked unresolved hypotheses and next discriminating observations.
+

evals/incident-triage/fixture/logs.jsonl

@@ -0,0 +1,9 @@
+{"id":"api-01","time":"10:01:00Z","service":"api","event":"deploy","version":"2026.08.16.2","cache_key":"tenant"}
+{"id":"dns-01","time":"10:01:05Z","service":"dns","event":"retry","count":1}
+{"id":"api-02","time":"10:02:00Z","service":"api","event":"checkout_wrong_currency","tenant":"west"}
+{"id":"worker-01","time":"10:02:01Z","service":"worker","event":"cache_hit","key":"west","currency":"EUR"}
+{"id":"db-01","time":"10:02:02Z","service":"db","event":"latency_ms","value":18}
+{"id":"api-03","time":"10:06:00Z","service":"api","event":"rollback","version":"2026.08.16.1"}
+{"id":"worker-02","time":"10:06:20Z","service":"worker","event":"cache_key","key":"west:USD"}
+{"id":"api-04","time":"10:07:00Z","service":"api","event":"checkout_correct_currency","tenant":"west"}
+

evals/incident-triage/fixture/validate_incident.py

@@ -0,0 +1,40 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("incident-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+is_negative = "eval_incident_unresolved" in text
+required_event_ids = holdout.get("required_event_ids", ["api-02", "worker-01", "api-03", "api-04"])
+minimum_event_ids = int(holdout.get("minimum_event_ids", 2))
+alternatives = holdout.get("alternative_terms", ["dns", "database", "cache"])
+minimum_alternatives = int(holdout.get("minimum_alternatives", 2))
+event_ids = sum(event_id.lower() in text for event_id in required_event_ids)
+has_alternatives = sum(term.lower() in text for term in alternatives) >= minimum_alternatives
+if event_ids < minimum_event_ids or not has_alternatives:
+    raise SystemExit("incident report lacks correlated event IDs or competing hypotheses")
+forbidden_certainty = holdout.get("forbidden_certainty", ["root cause is"])
+unsupported = float(is_negative and any(term.lower() in text for term in forbidden_certainty))
+if unsupported:
+    raise SystemExit("negative control asserted an unsupported root cause")
+required_degradation_terms = holdout.get("required_degradation_terms", [])
+missing_degradation_terms = [
+    term for term in required_degradation_terms if term.lower() not in text
+]
+if missing_degradation_terms:
+    raise SystemExit(
+        "incident report did not preserve provider-failure evidence: "
+        + ", ".join(missing_degradation_terms)
+    )
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "evidence_backed_diagnosis": 1.0,
+                "hypothesis_discrimination": 1.0,
+                "unsupported_root_cause_rate": unsupported,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/incident-triage/holdout/insufficient-evidence/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"forbidden_certainty":["root cause is","confirmed root cause"],"minimum_alternatives":2,"minimum_event_ids":2,"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/holdout/noisy-checkout-incident/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"minimum_alternatives":2,"minimum_event_ids":3,"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/holdout/provider-timeout-degradation/incident-holdout.json

@@ -0,0 +1 @@
+{"alternative_terms":["dns","database","cache"],"minimum_alternatives":2,"minimum_event_ids":2,"required_degradation_terms":["timeout","causal challenger","unavailable"],"required_event_ids":["api-02","worker-01","api-03","api-04"]}

evals/incident-triage/mutations/insufficient-evidence/01-remove-controlled-correlation.patch

@@ -0,0 +1,14 @@
+diff --git a/logs.jsonl b/logs.jsonl
+--- a/logs.jsonl
++++ b/logs.jsonl
+@@ -1,9 +1,7 @@
+ {"id":"api-01","time":"10:01:00Z","service":"api","event":"deploy","version":"2026.08.16.2","cache_key":"tenant"}
+ {"id":"dns-01","time":"10:01:05Z","service":"dns","event":"retry","count":1}
+ {"id":"api-02","time":"10:02:00Z","service":"api","event":"checkout_wrong_currency","tenant":"west"}
+ {"id":"worker-01","time":"10:02:01Z","service":"worker","event":"cache_hit","key":"west","currency":"EUR"}
+ {"id":"db-01","time":"10:02:02Z","service":"db","event":"latency_ms","value":18}
+-{"id":"api-03","time":"10:06:00Z","service":"api","event":"rollback","version":"2026.08.16.1"}
+-{"id":"worker-02","time":"10:06:20Z","service":"worker","event":"cache_key","key":"west:USD"}
+-{"id":"api-04","time":"10:07:00Z","service":"api","event":"checkout_correct_currency","tenant":"west"}
++{"id":"api-03","time":"10:06:00Z","service":"api","event":"telemetry_gap","duration_seconds":180}
+ 

evals/incident-triage/suite.toml

@@ -0,0 +1,161 @@
+schema_version = 8
+id = "incident-triage"
+profile = "incident-hypothesis-triage"
+name = "Incident Hypothesis Triage"
+description = "Read-only noisy-evidence trials for competing causes, outage degradation, and evidence-backed diagnosis."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "evidence_backed_diagnosis"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["hypothesis_discrimination"]
+no_regression_lower_metrics = ["unsupported_root_cause_rate"]
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "GLM incident synthesizer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent causal-analysis control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex incident control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go incident control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen incident control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM incident control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter causal-analysis control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__nvidia_nemotron_3_ultra_550b_a55b"
+
+[[variants]]
+id = "root-plus-challenger"
+purpose = "GLM synthesis plus independent pinned Nemotron alternative hypotheses."
+topology = "root_plus_worker"
+worker = "causal_challenger"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Bounded evidence correlation plus independent causal challenge."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-evidence-runner"
+purpose = "Ablates economical evidence collection and correlation."
+topology = "full_without_worker"
+worker = "evidence_runner"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-causal-challenger"
+purpose = "Ablates cross-route alternative causal analysis."
+topology = "full_without_worker"
+worker = "causal_challenger"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "noisy-checkout-incident"
+description = "Discriminate a causal deployment signal from DNS and database red herrings."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Investigate RUNBOOK.md and logs.jsonl read-only. Develop at least two competing causes, cite exact event IDs and distinguishing predictions, and assert a root cause only if reproducible evidence crosses the stated threshold. Report degraded roles if a provider is unavailable. End with EVAL_INCIDENT_DIAGNOSED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DIAGNOSED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+required_agents = ["evidence_runner", "causal_challenger"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "provider-timeout-degradation"
+description = "The independent OpenRouter causal challenger times out while primary evidence analysis continues."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { openrouter_openai_chat = "timeout" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage RUNBOOK.md and logs.jsonl while the causal-challenger route is unavailable because of an injected timeout. Report the degraded role and fault explicitly, use remaining evidence rather than inventing challenger output, rank competing causes, and end with EVAL_INCIDENT_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "insufficient-evidence"
+description = "Negative control where a ranked unresolved hypothesis is correct and certainty is a defect."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage the supplied evidence, but treat missing cross-service correlation as material. Rank alternatives and required next observations; do not assert a root cause without reproducible evidence. End with EVAL_INCIDENT_UNRESOLVED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_UNRESOLVED"]
+forbidden_patterns = ["(?i)root cause is"]
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0