149 lines
4.9 KiB
TOML
149 lines
4.9 KiB
TOML
|
schema_version = 8
|
||
|
|
id = "contract-refactoring"
|
||
|
|
profile = "contract-first-refactoring"
|
||
|
|
name = "Contract-First Refactoring"
|
||
|
|
description = "Measures frozen-contract fidelity, hidden compatibility invariants, independent tests, and regression verification."
|
||
|
|
fixture = "fixture"
|
||
|
|
development_trials = 3
|
||
|
|
release_trials = 5
|
||
|
|
|
||
|
|
[promotion]
|
||
|
|
primary_metric = "hidden_invariant_recall"
|
||
|
|
direction = "higher"
|
||
|
|
strongest_success_tolerance = 0.02
|
||
|
|
minimum_relative_improvement = 0.10
|
||
|
|
minimum_absolute_improvement = 0.05
|
||
|
|
worker_minimum_success_contribution = 0.02
|
||
|
|
worker_minimum_metric_contribution = 0.10
|
||
|
|
no_regression_higher_metrics = ["contract_hash_integrity", "compatibility_rate"]
|
||
|
|
no_regression_lower_metrics = []
|
||
|
|
require_complete_api_cost = true
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "configured-root"
|
||
|
|
purpose = "Sonnet refactor lead alone."
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "configured_root_alone"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "strongest-task-single"
|
||
|
|
purpose = "Sol single-agent compatibility refactor control."
|
||
|
|
profile = "adaptive-engineering"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "strongest_single_agent"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "codex-access-single"
|
||
|
|
purpose = "ChatGPT Codex refactor control."
|
||
|
|
profile = "adaptive-engineering"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "chatgpt_codex"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "go-access-single"
|
||
|
|
purpose = "OpenCode Go refactor control."
|
||
|
|
profile = "high-confidence-debugging"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "opencode_go"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "zen-access-single"
|
||
|
|
purpose = "OpenCode Zen Sonnet refactor control."
|
||
|
|
profile = "contract-first-refactoring"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "opencode_zen"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "zai-access-single"
|
||
|
|
purpose = "Z.AI Coding Plan refactor control."
|
||
|
|
profile = "incident-hypothesis-triage"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "zai_coding_plan"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "openrouter-access-single"
|
||
|
|
purpose = "Pinned OpenRouter refactor control."
|
||
|
|
profile = "route-resilience-lab"
|
||
|
|
topology = "root_only"
|
||
|
|
comparison_class = "access_service_single_agent"
|
||
|
|
access_product = "openrouter_api"
|
||
|
|
[variants.bindings]
|
||
|
|
route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "root-plus-test-author"
|
||
|
|
purpose = "Frozen-contract root plus independent contract-derived tests."
|
||
|
|
topology = "root_plus_worker"
|
||
|
|
worker = "contract_test_author"
|
||
|
|
comparison_class = "root_plus_highest_value"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "full-profile"
|
||
|
|
purpose = "Invariant mining, frozen contract, test-first patch, refactor, and verification."
|
||
|
|
topology = "full"
|
||
|
|
comparison_class = "full_profile"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "without-invariant-miner"
|
||
|
|
purpose = "Ablates long-context compatibility mining."
|
||
|
|
topology = "full_without_worker"
|
||
|
|
worker = "invariant_miner"
|
||
|
|
comparison_class = "ablation"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "without-test-author"
|
||
|
|
purpose = "Ablates independent test-first specification."
|
||
|
|
topology = "full_without_worker"
|
||
|
|
worker = "contract_test_author"
|
||
|
|
comparison_class = "ablation"
|
||
|
|
|
||
|
|
[[variants]]
|
||
|
|
id = "without-verifier"
|
||
|
|
purpose = "Ablates economical independent compatibility verification."
|
||
|
|
topology = "full_without_worker"
|
||
|
|
worker = "compatibility_verifier"
|
||
|
|
comparison_class = "ablation"
|
||
|
|
|
||
|
|
[[tasks]]
|
||
|
|
id = "versioned-payload-refactor"
|
||
|
|
description = "Refactor a versioned serializer without losing legacy read compatibility or input immutability."
|
||
|
|
sandbox = "workspace-write"
|
||
|
|
difficulty = "adversarial"
|
||
|
|
negative_control = false
|
||
|
|
wall_timeout_seconds = 3000
|
||
|
|
validation_timeout_seconds = 120
|
||
|
|
prompt = """
|
||
|
|
Refactor codec.py under SPEC.md. Before tests or implementation, write and hash CONTRACT.md into CONTRACT.sha256. Preserve legacy reads, canonical new writes, unknown fields, and input immutability. Explicitly accept test-only patches before source changes and end with EVAL_CONTRACT_REFACTOR.
|
||
|
|
"""
|
||
|
|
[tasks.outcome_assertions]
|
||
|
|
expected_patterns = ["EVAL_CONTRACT_REFACTOR"]
|
||
|
|
forbidden_patterns = []
|
||
|
|
validation_commands = ["python -m unittest -v && python validate_contract.py"]
|
||
|
|
[tasks.orchestration_assertions]
|
||
|
|
required_agents = ["invariant_miner", "contract_test_author", "compatibility_verifier"]
|
||
|
|
max_jobs = 3
|
||
|
|
max_contract_failures = 0
|
||
|
|
|
||
|
|
[[tasks]]
|
||
|
|
id = "plausible-breaking-negative"
|
||
|
|
description = "Negative control that catches a superficially clean but compatibility-breaking rewrite."
|
||
|
|
sandbox = "workspace-write"
|
||
|
|
difficulty = "hard"
|
||
|
|
negative_control = true
|
||
|
|
wall_timeout_seconds = 2400
|
||
|
|
validation_timeout_seconds = 120
|
||
|
|
prompt = """
|
||
|
|
Improve codec.py without deleting legacy field support or mutating caller data. Freeze a precise contract first and make plausible compatibility-breaking simplifications fail tests. End with EVAL_CONTRACT_NEGATIVE_CONTROL.
|
||
|
|
"""
|
||
|
|
[tasks.outcome_assertions]
|
||
|
|
expected_patterns = ["EVAL_CONTRACT_NEGATIVE_CONTROL"]
|
||
|
|
forbidden_patterns = []
|
||
|
|
validation_commands = ["python -m unittest -v && python validate_contract.py"]
|
||
|
|
[tasks.orchestration_assertions]
|
||
|
|
max_jobs = 3
|
||
|
|
max_contract_failures = 0
|