141 lines
4.5 KiB
TOML
141 lines
4.5 KiB
TOML
schema_version = 8
|
|
id = "bounded-research"
|
|
profile = "bounded-research-organization-lab"
|
|
name = "Bounded Research Organization Lab"
|
|
description = "Experimental flat-versus-hierarchical long-corpus research with bounded nesting and handoff accounting."
|
|
fixture = "fixture"
|
|
development_trials = 3
|
|
release_trials = 5
|
|
|
|
[promotion]
|
|
primary_metric = "source_coverage"
|
|
direction = "higher"
|
|
strongest_success_tolerance = 0.02
|
|
minimum_relative_improvement = 0.10
|
|
minimum_absolute_improvement = 0.05
|
|
worker_minimum_success_contribution = 0.02
|
|
worker_minimum_metric_contribution = 0.10
|
|
no_regression_higher_metrics = ["contradiction_resolution"]
|
|
no_regression_lower_metrics = []
|
|
require_complete_api_cost = true
|
|
|
|
[[variants]]
|
|
id = "configured-root"
|
|
purpose = "Sol repository root alone."
|
|
topology = "root_only"
|
|
comparison_class = "configured_root_alone"
|
|
|
|
[[variants]]
|
|
id = "strongest-task-single"
|
|
purpose = "Sol long-corpus single-agent control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "strongest_single_agent"
|
|
|
|
[[variants]]
|
|
id = "codex-access-single"
|
|
purpose = "ChatGPT Codex access control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "chatgpt_codex"
|
|
|
|
[[variants]]
|
|
id = "go-access-single"
|
|
purpose = "OpenCode Go Kimi/MiniMax access control."
|
|
profile = "research-backed-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_go"
|
|
|
|
[[variants]]
|
|
id = "zen-access-single"
|
|
purpose = "OpenCode Zen access control."
|
|
profile = "secure-change"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_zen"
|
|
|
|
[[variants]]
|
|
id = "zai-access-single"
|
|
purpose = "Z.AI Coding Plan access control."
|
|
profile = "incident-hypothesis-triage"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "zai_coding_plan"
|
|
|
|
[[variants]]
|
|
id = "openrouter-access-single"
|
|
purpose = "Pinned OpenRouter access control."
|
|
profile = "route-resilience-lab"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "openrouter_api"
|
|
[variants.bindings]
|
|
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
|
|
|
|
[[variants]]
|
|
id = "flat-fanout"
|
|
purpose = "Root plus a direct MiniMax source scout, with hierarchy disabled."
|
|
topology = "root_plus_worker"
|
|
worker = "source_scout"
|
|
comparison_class = "root_plus_highest_value"
|
|
|
|
[[variants]]
|
|
id = "full-hierarchy"
|
|
purpose = "Kimi research lead may organize at most two MiniMax scouts."
|
|
topology = "full"
|
|
comparison_class = "full_profile"
|
|
|
|
[[variants]]
|
|
id = "without-research-lead"
|
|
purpose = "Ablates the hierarchical Kimi organizer, yielding direct flat scouting only."
|
|
topology = "full_without_worker"
|
|
worker = "research_lead"
|
|
comparison_class = "ablation"
|
|
|
|
[[variants]]
|
|
id = "without-source-scout"
|
|
purpose = "Ablates MiniMax source extraction while retaining the Kimi lead."
|
|
topology = "full_without_worker"
|
|
worker = "source_scout"
|
|
comparison_class = "ablation"
|
|
|
|
[[tasks]]
|
|
id = "distributed-policy-corpus"
|
|
description = "Resolve cross-document conflicts and trace the final policy to authoritative sections."
|
|
sandbox = "read-only"
|
|
difficulty = "hard"
|
|
negative_control = false
|
|
wall_timeout_seconds = 2400
|
|
validation_timeout_seconds = 60
|
|
prompt = """
|
|
Read every document under corpus/. Determine the authoritative retry ceiling, retention period, and rollout exception, cite exact paths and headings, and resolve contradictions using the stated precedence rule. Compare bounded hierarchy with direct scouting without duplicate work. End with EVAL_BOUNDED_RESEARCH.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_BOUNDED_RESEARCH"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python validate_research.py"]
|
|
[tasks.orchestration_assertions]
|
|
required_agents = ["research_lead", "source_scout"]
|
|
max_jobs = 3
|
|
max_contract_failures = 0
|
|
|
|
[[tasks]]
|
|
id = "small-corpus-negative"
|
|
description = "Negative control where hierarchical handoffs should not be assumed valuable."
|
|
sandbox = "read-only"
|
|
difficulty = "easy"
|
|
negative_control = true
|
|
wall_timeout_seconds = 1200
|
|
validation_timeout_seconds = 60
|
|
prompt = """
|
|
Answer the three literal policy questions from corpus/ with exact citations. The corpus is intentionally small enough that nesting may cost more than it adds; launch only with a concrete expected contribution. End with EVAL_BOUNDED_SMALL.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_BOUNDED_SMALL"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python validate_research.py"]
|
|
[tasks.orchestration_assertions]
|
|
max_jobs = 2
|