😏

evals/incident-triage/suite.toml

@@ -0,0 +1,161 @@
+schema_version = 8
+id = "incident-triage"
+profile = "incident-hypothesis-triage"
+name = "Incident Hypothesis Triage"
+description = "Read-only noisy-evidence trials for competing causes, outage degradation, and evidence-backed diagnosis."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "evidence_backed_diagnosis"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["hypothesis_discrimination"]
+no_regression_lower_metrics = ["unsupported_root_cause_rate"]
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "GLM incident synthesizer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol single-agent causal-analysis control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex incident control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go incident control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen incident control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM incident control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter causal-analysis control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__nvidia_nemotron_3_ultra_550b_a55b"
+
+[[variants]]
+id = "root-plus-challenger"
+purpose = "GLM synthesis plus independent pinned Nemotron alternative hypotheses."
+topology = "root_plus_worker"
+worker = "causal_challenger"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Bounded evidence correlation plus independent causal challenge."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-evidence-runner"
+purpose = "Ablates economical evidence collection and correlation."
+topology = "full_without_worker"
+worker = "evidence_runner"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-causal-challenger"
+purpose = "Ablates cross-route alternative causal analysis."
+topology = "full_without_worker"
+worker = "causal_challenger"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "noisy-checkout-incident"
+description = "Discriminate a causal deployment signal from DNS and database red herrings."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Investigate RUNBOOK.md and logs.jsonl read-only. Develop at least two competing causes, cite exact event IDs and distinguishing predictions, and assert a root cause only if reproducible evidence crosses the stated threshold. Report degraded roles if a provider is unavailable. End with EVAL_INCIDENT_DIAGNOSED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DIAGNOSED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+required_agents = ["evidence_runner", "causal_challenger"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "provider-timeout-degradation"
+description = "The independent OpenRouter causal challenger times out while primary evidence analysis continues."
+sandbox = "read-only"
+difficulty = "adversarial"
+negative_control = true
+route_faults = { openrouter_openai_chat = "timeout" }
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage RUNBOOK.md and logs.jsonl while the causal-challenger route is unavailable because of an injected timeout. Report the degraded role and fault explicitly, use remaining evidence rather than inventing challenger output, rank competing causes, and end with EVAL_INCIDENT_DEGRADED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_DEGRADED"]
+forbidden_patterns = []
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "insufficient-evidence"
+description = "Negative control where a ranked unresolved hypothesis is correct and certainty is a defect."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Triage the supplied evidence, but treat missing cross-service correlation as material. Rank alternatives and required next observations; do not assert a root cause without reproducible evidence. End with EVAL_INCIDENT_UNRESOLVED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_INCIDENT_UNRESOLVED"]
+forbidden_patterns = ["(?i)root cause is"]
+validation_commands = ["python validate_incident.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0