codex-mmo/evals/route-resilience/suite.toml

schema_version = 8
id = "route-resilience"
profile = "route-resilience-lab"
name = "Route Resilience Lab"
description = "Experimental same-model route comparison with pinned endpoint identity and typed credential, rate-limit, and timeout faults."
fixture = "fixture"
development_trials = 3
release_trials = 5

[promotion]
primary_metric = "route_consistency"
direction = "higher"
strongest_success_tolerance = 0.02
minimum_relative_improvement = 0.10
minimum_absolute_improvement = 0.05
worker_minimum_success_contribution = 0.02
worker_minimum_metric_contribution = 0.10
no_regression_higher_metrics = []
no_regression_lower_metrics = []
require_complete_api_cost = true

[[variants]]
id = "configured-root"
purpose = "Terra route observer alone."
topology = "root_only"
comparison_class = "configured_root_alone"

[[variants]]
id = "strongest-task-single"
purpose = "Sol single-agent route-analysis control."
profile = "adaptive-engineering"
topology = "root_only"
comparison_class = "strongest_single_agent"

[[variants]]
id = "codex-access-single"
purpose = "ChatGPT Codex access control."
profile = "adaptive-engineering"
topology = "root_only"
comparison_class = "access_service_single_agent"
access_product = "chatgpt_codex"

[[variants]]
id = "go-access-single"
purpose = "OpenCode Go DeepSeek route control."
profile = "high-confidence-debugging"
topology = "root_only"
comparison_class = "access_service_single_agent"
access_product = "opencode_go"

[[variants]]
id = "zen-access-single"
purpose = "OpenCode Zen access control."
profile = "secure-change"
topology = "root_only"
comparison_class = "access_service_single_agent"
access_product = "opencode_zen"

[[variants]]
id = "zai-access-single"
purpose = "Z.AI Coding Plan access control."
profile = "incident-hypothesis-triage"
topology = "root_only"
comparison_class = "access_service_single_agent"
access_product = "zai_coding_plan"

[[variants]]
id = "openrouter-access-single"
purpose = "Pinned OpenRouter DeepSeek route control."
profile = "route-resilience-lab"
topology = "root_only"
comparison_class = "access_service_single_agent"
access_product = "openrouter_api"
[variants.bindings]
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"

[[variants]]
id = "root-plus-openrouter"
purpose = "Route observer plus the independently pinned shadow route."
topology = "root_plus_worker"
worker = "openrouter_route"
comparison_class = "root_plus_highest_value"

[[variants]]
id = "full-profile"
purpose = "Intentional duplicate observation through Go and pinned OpenRouter."
topology = "full"
comparison_class = "full_profile"

[[variants]]
id = "without-go-route"
purpose = "Ablates the Go route."
topology = "full_without_worker"
worker = "go_route"
comparison_class = "ablation"

[[variants]]
id = "without-openrouter-route"
purpose = "Ablates the pinned OpenRouter route."
topology = "full_without_worker"
worker = "openrouter_route"
comparison_class = "ablation"

[[tasks]]
id = "shadow-build-comparison"
description = "Compare claims from two routes while supervisor telemetry establishes endpoint identity."
sandbox = "read-only"
difficulty = "hard"
negative_control = false
wall_timeout_seconds = 1800
validation_timeout_seconds = 60
prompt = """
Read ROUTE_PROBE.md and compare the same exact claims through configured routes when available. Distinguish model maker, API operator, access product, gateway, requested policy, and actual serving endpoint; never infer endpoint identity from model text. End with EVAL_ROUTE_SHADOW.
"""
[tasks.outcome_assertions]
expected_patterns = ["EVAL_ROUTE_SHADOW"]
forbidden_patterns = []
validation_commands = ["python validate_route.py"]
[tasks.orchestration_assertions]
required_agents = ["go_route", "openrouter_route"]
max_jobs = 2
max_contract_failures = 0

[[tasks]]
id = "injected-go-credential-loss"
description = "OpenCode Go is mechanically unavailable while the independent route remains usable."
sandbox = "read-only"
difficulty = "adversarial"
negative_control = true
route_faults = { opencode_go_openai_chat = "credential_loss" }
wall_timeout_seconds = 1800
validation_timeout_seconds = 60
prompt = """
Perform the route probe under the injected availability overlay. Report unavailable roles explicitly, continue through any independent available route, and do not invent fallback execution. End with EVAL_ROUTE_DEGRADED.
"""
[tasks.outcome_assertions]
expected_patterns = ["EVAL_ROUTE_DEGRADED"]
forbidden_patterns = []
validation_commands = ["python validate_route.py"]
[tasks.orchestration_assertions]
max_jobs = 2
max_contract_failures = 0

[[tasks]]
id = "injected-openrouter-credential-loss"
description = "OpenRouter is mechanically unavailable and no silent aggregator fallback is permitted."
sandbox = "read-only"
difficulty = "adversarial"
negative_control = true
route_faults = { openrouter_openai_chat = "credential_loss" }
wall_timeout_seconds = 1800
validation_timeout_seconds = 60
prompt = """
Perform the route probe under the injected availability overlay. Record that the pinned OpenRouter role is unavailable and use only explicitly available evidence. End with EVAL_ROUTE_DEGRADED.
"""
[tasks.outcome_assertions]
expected_patterns = ["EVAL_ROUTE_DEGRADED"]
forbidden_patterns = []
validation_commands = ["python validate_route.py"]
[tasks.orchestration_assertions]
max_jobs = 2
max_contract_failures = 0

[[tasks]]
id = "injected-go-rate-limit"
description = "The primary Go route is mechanically unavailable after a typed rate-limit fault."
sandbox = "read-only"
difficulty = "adversarial"
negative_control = true
route_faults = { opencode_go_openai_chat = "rate_limit" }
wall_timeout_seconds = 1800
validation_timeout_seconds = 60
prompt = """
Perform the route probe under the injected rate-limit overlay. Name the affected route and fault, continue only through explicitly available infrastructure, and do not describe the shadow route as a transparent fallback. End with EVAL_ROUTE_DEGRADED.
"""
[tasks.outcome_assertions]
expected_patterns = ["EVAL_ROUTE_DEGRADED"]
forbidden_patterns = []
validation_commands = ["python validate_route.py"]
[tasks.orchestration_assertions]
max_jobs = 2
max_contract_failures = 0

[[tasks]]
id = "injected-openrouter-timeout"
description = "The pinned OpenRouter route is mechanically unavailable after a typed timeout fault."
sandbox = "read-only"
difficulty = "adversarial"
negative_control = true
route_faults = { openrouter_openai_chat = "timeout" }
wall_timeout_seconds = 1800
validation_timeout_seconds = 60
prompt = """
Perform the route probe under the injected timeout overlay. Name the affected route and fault, continue only through the Go route when available, and report that no OpenRouter fallback was attempted. End with EVAL_ROUTE_DEGRADED.
"""
[tasks.outcome_assertions]
expected_patterns = ["EVAL_ROUTE_DEGRADED"]
forbidden_patterns = []
validation_commands = ["python validate_route.py"]
[tasks.orchestration_assertions]
max_jobs = 2
max_contract_failures = 0