😏

evals/bounded-research/fixture/corpus/00-precedence.md

@@ -0,0 +1,4 @@
+# Policy precedence
+
+Signed exceptions override the current handbook. The current handbook overrides archived drafts.
+

evals/bounded-research/fixture/corpus/10-handbook.md

@@ -0,0 +1,14 @@
+# Current handbook
+
+## Retries
+
+At most four delivery attempts are permitted.
+
+## Retention
+
+Operational delivery records are retained for 30 days.
+
+## Rollout
+
+Canary rollout is mandatory.
+

evals/bounded-research/fixture/corpus/20-archive.md

@@ -0,0 +1,4 @@
+# Archived draft
+
+This obsolete draft proposed seven retries and 90-day retention.
+

evals/bounded-research/fixture/corpus/30-signed-exception.md

@@ -0,0 +1,4 @@
+# Signed exception: emergency rollout
+
+For incident INC-42 only, canary rollout may be skipped after two-person approval. Retry and retention rules are unchanged.
+

evals/bounded-research/fixture/validate_research.py

@@ -0,0 +1,20 @@
+import json
+from pathlib import Path
+
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("research-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run research holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+required = [*holdout["required_values"], *holdout["required_sources"]]
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("research synthesis misses authoritative facts: " + ", ".join(missing))
+if "seven" in text and not any(term in text for term in ("obsolete", "archived", "reject")):
+    raise SystemExit("archived retry value was not rejected")
+print(
+    json.dumps(
+        {"metrics": {"source_coverage": 1.0, "contradiction_resolution": 1.0}},
+        sort_keys=True,
+    )
+)

evals/bounded-research/holdout/distributed-policy-corpus/research-holdout.json

@@ -0,0 +1 @@
+{"required_values":["four","30","inc-42"],"required_sources":["10-handbook.md","30-signed-exception.md"]}

evals/bounded-research/holdout/small-corpus-negative/research-holdout.json

@@ -0,0 +1 @@
+{"required_values":["four","30","inc-42"],"required_sources":["10-handbook.md","30-signed-exception.md"]}

evals/bounded-research/suite.toml

@@ -0,0 +1,140 @@
+schema_version = 8
+id = "bounded-research"
+profile = "bounded-research-organization-lab"
+name = "Bounded Research Organization Lab"
+description = "Experimental flat-versus-hierarchical long-corpus research with bounded nesting and handoff accounting."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "source_coverage"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["contradiction_resolution"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Sol repository root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol long-corpus single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go Kimi/MiniMax access control."
+profile = "research-backed-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen access control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan access control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter access control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "flat-fanout"
+purpose = "Root plus a direct MiniMax source scout, with hierarchy disabled."
+topology = "root_plus_worker"
+worker = "source_scout"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-hierarchy"
+purpose = "Kimi research lead may organize at most two MiniMax scouts."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-research-lead"
+purpose = "Ablates the hierarchical Kimi organizer, yielding direct flat scouting only."
+topology = "full_without_worker"
+worker = "research_lead"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-source-scout"
+purpose = "Ablates MiniMax source extraction while retaining the Kimi lead."
+topology = "full_without_worker"
+worker = "source_scout"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "distributed-policy-corpus"
+description = "Resolve cross-document conflicts and trace the final policy to authoritative sections."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 60
+prompt = """
+Read every document under corpus/. Determine the authoritative retry ceiling, retention period, and rollout exception, cite exact paths and headings, and resolve contradictions using the stated precedence rule. Compare bounded hierarchy with direct scouting without duplicate work. End with EVAL_BOUNDED_RESEARCH.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_BOUNDED_RESEARCH"]
+forbidden_patterns = []
+validation_commands = ["python validate_research.py"]
+[tasks.orchestration_assertions]
+required_agents = ["research_lead", "source_scout"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "small-corpus-negative"
+description = "Negative control where hierarchical handoffs should not be assumed valuable."
+sandbox = "read-only"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 1200
+validation_timeout_seconds = 60
+prompt = """
+Answer the three literal policy questions from corpus/ with exact citations. The corpus is intentionally small enough that nesting may cost more than it adds; launch only with a concrete expected contribution. End with EVAL_BOUNDED_SMALL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_BOUNDED_SMALL"]
+forbidden_patterns = []
+validation_commands = ["python validate_research.py"]
+[tasks.orchestration_assertions]
+max_jobs = 2