142 lines
4.7 KiB
TOML
142 lines
4.7 KiB
TOML
schema_version = 8
|
|
id = "competing-implementations"
|
|
profile = "competing-implementations-lab"
|
|
name = "Competing Implementations Lab"
|
|
description = "Experimental matched alternatives selected by tests, benchmarks, patch review, and integration effort."
|
|
fixture = "fixture"
|
|
development_trials = 3
|
|
release_trials = 5
|
|
|
|
[promotion]
|
|
primary_metric = "benchmark_quality"
|
|
direction = "higher"
|
|
strongest_success_tolerance = 0.02
|
|
minimum_relative_improvement = 0.10
|
|
minimum_absolute_improvement = 0.05
|
|
worker_minimum_success_contribution = 0.02
|
|
worker_minimum_metric_contribution = 0.10
|
|
no_regression_higher_metrics = ["correctness_rate", "maintainability_score"]
|
|
no_regression_lower_metrics = []
|
|
require_complete_api_cost = true
|
|
|
|
[[variants]]
|
|
id = "configured-root"
|
|
purpose = "Terra contract author and implementer alone."
|
|
topology = "root_only"
|
|
comparison_class = "configured_root_alone"
|
|
|
|
[[variants]]
|
|
id = "strongest-task-single"
|
|
purpose = "Sol single-implementation control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "strongest_single_agent"
|
|
|
|
[[variants]]
|
|
id = "codex-access-single"
|
|
purpose = "ChatGPT Codex access control."
|
|
profile = "adaptive-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "chatgpt_codex"
|
|
|
|
[[variants]]
|
|
id = "go-access-single"
|
|
purpose = "OpenCode Go DeepSeek candidate control."
|
|
profile = "high-confidence-debugging"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_go"
|
|
|
|
[[variants]]
|
|
id = "zen-access-single"
|
|
purpose = "OpenCode Zen Sonnet candidate control."
|
|
profile = "contract-first-refactoring"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "opencode_zen"
|
|
|
|
[[variants]]
|
|
id = "zai-access-single"
|
|
purpose = "Z.AI Coding Plan access control."
|
|
profile = "incident-hypothesis-triage"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "zai_coding_plan"
|
|
|
|
[[variants]]
|
|
id = "openrouter-access-single"
|
|
purpose = "Pinned OpenRouter access control."
|
|
profile = "route-resilience-lab"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "openrouter_api"
|
|
[variants.bindings]
|
|
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
|
|
|
|
[[variants]]
|
|
id = "one-deepseek-candidate"
|
|
purpose = "Objective judge plus one economical implementation candidate."
|
|
topology = "root_plus_worker"
|
|
worker = "deepseek_candidate"
|
|
comparison_class = "root_plus_highest_value"
|
|
|
|
[[variants]]
|
|
id = "full-competition"
|
|
purpose = "Two independently isolated implementations with evidence-based selection."
|
|
topology = "full"
|
|
comparison_class = "full_profile"
|
|
|
|
[[variants]]
|
|
id = "without-deepseek"
|
|
purpose = "Ablates the Go-hosted DeepSeek candidate."
|
|
topology = "full_without_worker"
|
|
worker = "deepseek_candidate"
|
|
comparison_class = "ablation"
|
|
|
|
[[variants]]
|
|
id = "without-sonnet"
|
|
purpose = "Ablates the Zen-hosted Sonnet candidate."
|
|
topology = "full_without_worker"
|
|
worker = "sonnet_candidate"
|
|
comparison_class = "ablation"
|
|
|
|
[[tasks]]
|
|
id = "stable-deduplication"
|
|
description = "Choose between independent correct and scalable stable-deduplication patches."
|
|
sandbox = "workspace-write"
|
|
difficulty = "hard"
|
|
negative_control = false
|
|
wall_timeout_seconds = 3000
|
|
validation_timeout_seconds = 120
|
|
prompt = """
|
|
Freeze the objective contract in SPEC.md and its benchmark before implementation. When workers are available, launch independent candidates into disjoint worktrees. Compare returned binary patches using tests, benchmark evidence, complexity, maintainability, and integration corrections; never vote by model identity. Explicitly integrate only the selected patch and end with EVAL_COMPETING_SELECTED.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_COMPETING_SELECTED"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python -m unittest -v && python benchmark.py"]
|
|
[tasks.orchestration_assertions]
|
|
required_agents = ["deepseek_candidate", "sonnet_candidate"]
|
|
max_jobs = 2
|
|
max_contract_failures = 0
|
|
|
|
[[tasks]]
|
|
id = "tiny-change-negative"
|
|
description = "Negative control where two full implementations may cost more than they add."
|
|
sandbox = "workspace-write"
|
|
difficulty = "easy"
|
|
negative_control = true
|
|
wall_timeout_seconds = 1800
|
|
validation_timeout_seconds = 120
|
|
prompt = """
|
|
Implement stable_unique correctly. The task is intentionally compact; launch competing writers only if the expected selection value exceeds two patches and integration work. End with EVAL_COMPETING_SMALL.
|
|
"""
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_COMPETING_SMALL"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python -m unittest -v && python benchmark.py"]
|
|
[tasks.orchestration_assertions]
|
|
max_jobs = 2
|
|
max_contract_failures = 0
|