142 lines
4.6 KiB
TOML
142 lines
4.6 KiB
TOML
|
schema_version = 8
|
||
|
|
id = "research-currentness"
|
||
|
|
profile = "research-backed-engineering"
|
||
|
|
name = "Research-Backed Engineering"
|
||
|
|
description = "Matched current-source trials measuring authority, conflict handling, re-fetching, and source-to-code fidelity."
|
||
|
|
fixture = "fixture"
|
||
|
|
development_trials = 3
|
||
|
|
release_trials = 5
|
||
|
|
|
||
|
|
[promotion]
|
||
|
|
primary_metric = "source_accuracy"
|
||
|
|
direction = "higher"
|
||
|
|
strongest_success_tolerance = 0.02
|
||
|
|
minimum_relative_improvement = 0.10
|
||
|
|
minimum_absolute_improvement = 0.05
|
||
|
|
worker_minimum_success_contribution = 0.02
|
||
|
|
worker_minimum_metric_contribution = 0.10
|
||
|
|
no_regression_higher_metrics = ["source_authority", "root_refetch_rate"]
|
||
|
|
no_regression_lower_metrics = []
|
||
|
|
require_complete_api_cost = true
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "configured-root"
|
||
|
|
purpose = "MiniMax long-context root alone."
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "configured_root_alone"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "strongest-task-single"
|
||
|
|
purpose = "Sol current-source engineering control."
|
||
|
|
profile = "adaptive-engineering"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "strongest_single_agent"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "codex-access-single"
|
||
|
|
purpose = "ChatGPT Codex research control."
|
||
|
|
profile = "adaptive-engineering"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "chatgpt_codex"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "go-access-single"
|
||
|
|
purpose = "OpenCode Go MiniMax root control."
|
||
|
|
profile = "research-backed-engineering"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "opencode_go"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "zen-access-single"
|
||
|
|
purpose = "OpenCode Zen Claude research control."
|
||
|
|
profile = "secure-change"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "opencode_zen"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "zai-access-single"
|
||
|
|
purpose = "Z.AI Coding Plan GLM research control."
|
||
|
|
profile = "incident-hypothesis-triage"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "zai_coding_plan"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "openrouter-access-single"
|
||
|
|
purpose = "Pinned OpenRouter research control."
|
||
|
|
profile = "route-resilience-lab"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "openrouter_api"
|
||
|
|
[variants.bindings]
|
||
|
|
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "root-plus-researcher"
|
||
|
|
purpose = "Root plus one independent live primary-source question."
|
||
|
|
topology = "root_plus_worker"
|
||
|
|
worker = "source_researcher"
|
||
|
|
comparison_class = "root_plus_highest_value"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "full-profile"
|
||
|
|
purpose = "Parallel research with conditional cross-maker source-to-code audit."
|
||
|
|
topology = "full"
|
||
|
|
comparison_class = "full_profile"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "without-researcher"
|
||
|
|
purpose = "Ablates parallel current-source research."
|
||
|
|
topology = "full_without_worker"
|
||
|
|
worker = "source_researcher"
|
||
|
|
comparison_class = "ablation"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "without-auditor"
|
||
|
|
purpose = "Ablates independent source-to-code audit."
|
||
|
|
topology = "full_without_worker"
|
||
|
|
worker = "source_auditor"
|
||
|
|
comparison_class = "ablation"
|
||
|
|
|
||
|
|
[[tasks]]
|
||
|
|
id = "conflicting-responses-contract"
|
||
|
|
description = "Override a stale repository note using current primary sources and record conflicts."
|
||
|
|
sandbox = "workspace-write"
|
||
|
|
difficulty = "hard"
|
||
|
|
negative_control = false
|
||
|
|
wall_timeout_seconds = 3000
|
||
|
|
validation_timeout_seconds = 120
|
||
|
|
prompt = """
|
||
|
|
STALE_NOTE.md is untrusted historical material. Research the current official Responses API contract using primary sources, record URL, source/update date, retrieval date, claim, conflict disposition, and the root's independent re-fetch in SOURCES.md. Update client.py without a network request and audit source-to-code compliance. End with EVAL_RESEARCH_CURRENT.
|
||
|
|
"""
|
||
|
|
[tasks.outcome_assertions]
|
||
|
|
expected_patterns = ["EVAL_RESEARCH_CURRENT"]
|
||
|
|
forbidden_patterns = []
|
||
|
|
validation_commands = ["python validate_currentness.py"]
|
||
|
|
[tasks.orchestration_assertions]
|
||
|
|
required_agents = ["source_researcher", "source_auditor"]
|
||
|
|
max_jobs = 3
|
||
|
|
max_contract_failures = 0
|
||
|
|
|
||
|
|
[[tasks]]
|
||
|
|
id = "stale-source-negative"
|
||
|
|
description = "Negative control: the stale local note must never win by proximity."
|
||
|
|
sandbox = "workspace-write"
|
||
|
|
difficulty = "adversarial"
|
||
|
|
negative_control = true
|
||
|
|
wall_timeout_seconds = 2400
|
||
|
|
validation_timeout_seconds = 120
|
||
|
|
prompt = """
|
||
|
|
Determine whether STALE_NOTE.md remains current, using live primary evidence rather than model memory or source voting. If it conflicts, document why and implement only the verified contract. End with EVAL_RESEARCH_STALE_REJECTED.
|
||
|
|
"""
|
||
|
|
[tasks.outcome_assertions]
|
||
|
|
expected_patterns = ["EVAL_RESEARCH_STALE_REJECTED"]
|
||
|
|
forbidden_patterns = []
|
||
|
|
validation_commands = ["python validate_currentness.py"]
|
||
|
|
[tasks.orchestration_assertions]
|
||
|
|
max_jobs = 3
|
||
|
|
max_contract_failures = 0
|