176 lines
5.6 KiB
TOML
176 lines
5.6 KiB
TOML
schema_version = 8
|
|
id = "access-efficient"
|
|
profile = "access-efficient-escalation-lab"
|
|
name = "Access-Efficient Escalation Lab"
|
|
description = "Experimental matched routing trials for literal local work, economical patches, and deliberate scarce-tier escalation."
|
|
fixture = "fixture"
|
|
development_trials = 3
|
|
release_trials = 5
|
|
|
|
[promotion]
|
|
primary_metric = "scarce_tier_request_units"
|
|
primary_baseline = "strongest_single_agent"
|
|
direction = "lower"
|
|
strongest_success_tolerance = 0.02
|
|
minimum_relative_improvement = 0.50
|
|
minimum_absolute_improvement = 1.0
|
|
worker_minimum_success_contribution = 0.02
|
|
worker_minimum_metric_contribution = 0.10
|
|
no_regression_higher_metrics = []
|
|
no_regression_lower_metrics = []
|
|
require_complete_api_cost = true
|
|
scarce_model_keys = [
|
|
"codex_chatgpt_builtin__gpt_5_6_sol",
|
|
"opencode_zen_anthropic_messages__claude_opus_5",
|
|
"zai_coding_responses__glm_5_3",
|
|
]
|
|
|
|
[[variants]]
|
|
id = "configured-root"
|
|
purpose = "Luna routing root alone."
|
|
topology = "root_only"
|
|
comparison_class = "configured_root_alone"
|
|
|
|
[[variants]]
|
|
id = "strongest-task-single"
|
|
purpose = "Sol maximum-capability single-agent control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "strongest_single_agent"
|
|
|
|
[[variants]]
|
|
id = "codex-access-single"
|
|
purpose = "ChatGPT Codex access control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "chatgpt_codex"
|
|
|
|
[[variants]]
|
|
id = "go-access-single"
|
|
purpose = "OpenCode Go economical access control."
|
|
profile = "high-confidence-debugging"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_go"
|
|
|
|
[[variants]]
|
|
id = "zen-access-single"
|
|
purpose = "OpenCode Zen access control."
|
|
profile = "secure-change"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_zen"
|
|
|
|
[[variants]]
|
|
id = "zai-access-single"
|
|
purpose = "Z.AI Coding Plan scarce-tier control."
|
|
profile = "incident-hypothesis-triage"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "zai_coding_plan"
|
|
|
|
[[variants]]
|
|
id = "openrouter-access-single"
|
|
purpose = "Pinned OpenRouter API control."
|
|
profile = "route-resilience-lab"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "openrouter_api"
|
|
[variants.bindings]
|
|
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
|
|
|
|
[[variants]]
|
|
id = "root-plus-routine"
|
|
purpose = "Luna root plus the economical isolated routine engineer."
|
|
topology = "root_plus_worker"
|
|
worker = "routine_engineer"
|
|
comparison_class = "root_plus_highest_value"
|
|
|
|
[[variants]]
|
|
id = "full-profile"
|
|
purpose = "Contained literal scout, routine engineer, and trigger-gated GLM escalation."
|
|
topology = "full"
|
|
comparison_class = "full_profile"
|
|
|
|
[[variants]]
|
|
id = "without-literal-scout"
|
|
purpose = "Tests whether local literal reconnaissance earns its launch overhead."
|
|
topology = "full_without_worker"
|
|
worker = "literal_scout"
|
|
comparison_class = "ablation"
|
|
|
|
[[variants]]
|
|
id = "without-routine-engineer"
|
|
purpose = "Ablates the economical isolated patch lane."
|
|
topology = "full_without_worker"
|
|
worker = "routine_engineer"
|
|
comparison_class = "ablation"
|
|
|
|
[[variants]]
|
|
id = "without-escalation"
|
|
purpose = "Ablates scarce-tier GLM escalation."
|
|
topology = "full_without_worker"
|
|
worker = "flagship_escalation"
|
|
comparison_class = "ablation"
|
|
|
|
[[tasks]]
|
|
id = "literal-limit"
|
|
description = "Trivial literal negative control where delegation should be avoided or strictly literal."
|
|
sandbox = "read-only"
|
|
difficulty = "easy"
|
|
negative_control = true
|
|
wall_timeout_seconds = 900
|
|
validation_timeout_seconds = 60
|
|
prompt = """
|
|
Report max_batch_items from LIMITS.toml with the exact path. Do no semantic reasoning and do not escalate. If the local scout is used, submit only a structured literal extract request. End with EVAL_ACCESS_LITERAL.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["128", "EVAL_ACCESS_LITERAL"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python validate_access.py literal"]
|
|
[tasks.orchestration_assertions]
|
|
forbidden_agents = ["flagship_escalation"]
|
|
max_jobs = 1
|
|
|
|
[[tasks]]
|
|
id = "routine-slug"
|
|
description = "Localized deterministic patch suitable for economical execution."
|
|
sandbox = "workspace-write"
|
|
difficulty = "medium"
|
|
negative_control = false
|
|
wall_timeout_seconds = 1800
|
|
validation_timeout_seconds = 120
|
|
prompt = """
|
|
Implement LABEL_SPEC.md. This is a bounded objectively testable source change; use scarce-tier escalation only if concrete evidence exceeds routine capability. End with EVAL_ACCESS_ROUTINE.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_ACCESS_ROUTINE"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python -m unittest -v && python validate_access.py routine"]
|
|
[tasks.orchestration_assertions]
|
|
required_agents = ["routine_engineer"]
|
|
forbidden_agents = ["flagship_escalation"]
|
|
max_jobs = 2
|
|
max_contract_failures = 0
|
|
|
|
[[tasks]]
|
|
id = "ambiguous-distributed-decision"
|
|
description = "Consequential ambiguity with an explicit escalation trigger."
|
|
sandbox = "read-only"
|
|
difficulty = "hard"
|
|
negative_control = false
|
|
wall_timeout_seconds = 1800
|
|
validation_timeout_seconds = 60
|
|
prompt = """
|
|
Read DISTRIBUTED_CONSTRAINTS.md and select an idempotency design. Record why the uncertainty and consequence justify or do not justify GLM escalation. Resolve claims with the supplied constraints, not model rank. End with EVAL_ACCESS_ESCALATION.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_ACCESS_ESCALATION"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python validate_access.py decision"]
|
|
[tasks.orchestration_assertions]
|
|
required_agents = ["flagship_escalation"]
|
|
max_jobs = 1
|
|
max_contract_failures = 0
|