😏

evals/research-currentness/README.md

@@ -0,0 +1,5 @@
+# Research-currentness evaluation
+
+This suite evaluates `research-backed-engineering` with a stale local note and a live-current API contract. The researcher must use dated primary OpenAI sources, the implementation specialist must work from the approved dossier, and the offline auditor must verify source-to-code compliance without silently introducing fresh claims.
+
+The validator does not call an external API or require a credential. It checks stable Responses API request/response semantics and the presence of a dated official-source dossier; the run transcript and worker contracts provide the research evidence.

evals/research-currentness/fixture/STALE_NOTE.md

@@ -0,0 +1,3 @@
+# Obsolete integration note
+
+This 2023 note says to send `messages` to `/v1/chat/completions` and read `choices[0].message.content`. It is retained solely to test whether current primary-source research overrides stale repository prose.

evals/research-currentness/fixture/client.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+ENDPOINT = "/v1/chat/completions"
+
+
+def build_request(prompt: str) -> dict[str, Any]:
+    return {
+        "model": "gpt-5",
+        "messages": [{"role": "user", "content": prompt}],
+    }
+
+
+def extract_text(response: dict[str, Any]) -> str:
+    return str(response["choices"][0]["message"]["content"])

evals/research-currentness/fixture/validate_currentness.py

@@ -0,0 +1,50 @@
+import json
+import re
+from pathlib import Path
+
+from client import ENDPOINT, build_request, extract_text
+
+if ENDPOINT != "/v1/responses":
+    raise SystemExit("client must use /v1/responses")
+
+request = build_request("hello")
+if request.get("input") != "hello" or "messages" in request:
+    raise SystemExit("client must send the Responses API input field")
+
+sample = {"output_text": "ready"}
+if extract_text(sample) != "ready":
+    raise SystemExit("client must extract the SDK-style output_text helper value")
+
+dossier = Path("SOURCES.md")
+if not dossier.is_file():
+    raise SystemExit("SOURCES.md is required")
+text = dossier.read_text(encoding="utf-8")
+holdout_path = Path("source-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run source holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+if not any(domain in text for domain in holdout["required_authority_domains"]):
+    raise SystemExit("dossier misses every holdout authority domain")
+if not re.search(r"https://(?:platform|developers)\.openai\.com/", text):
+    raise SystemExit("dossier must cite a direct official OpenAI documentation URL")
+if not re.search(r"20\d{2}-\d{2}-\d{2}", text):
+    raise SystemExit("dossier must record an ISO access or source date")
+if holdout["requires_root_refetch"] and not re.search(r"re-?fetch", text, flags=re.IGNORECASE):
+    raise SystemExit("dossier must record the root's independent re-fetch")
+if holdout["requires_conflict_disposition"] and not re.search(
+    r"conflict|stale|obsolete", text, flags=re.IGNORECASE
+):
+    raise SystemExit("dossier must explicitly dispose of the conflicting stale note")
+
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "source_accuracy": 1.0,
+                "source_authority": 1.0,
+                "root_refetch_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/research-currentness/holdout/conflicting-responses-contract/source-holdout.json

@@ -0,0 +1,5 @@
+{
+  "required_authority_domains": ["developers.openai.com", "platform.openai.com"],
+  "requires_root_refetch": true,
+  "requires_conflict_disposition": true
+}

evals/research-currentness/holdout/stale-source-negative/source-holdout.json

@@ -0,0 +1,5 @@
+{
+  "required_authority_domains": ["developers.openai.com", "platform.openai.com"],
+  "requires_root_refetch": true,
+  "requires_conflict_disposition": true
+}

evals/research-currentness/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "research-currentness"
+profile = "research-backed-engineering"
+name = "Research-Backed Engineering"
+description = "Matched current-source trials measuring authority, conflict handling, re-fetching, and source-to-code fidelity."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "source_accuracy"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["source_authority", "root_refetch_rate"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "MiniMax long-context root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol current-source engineering control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex research control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go MiniMax root control."
+profile = "research-backed-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Claude research control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM research control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter research control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-researcher"
+purpose = "Root plus one independent live primary-source question."
+topology = "root_plus_worker"
+worker = "source_researcher"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Parallel research with conditional cross-maker source-to-code audit."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-researcher"
+purpose = "Ablates parallel current-source research."
+topology = "full_without_worker"
+worker = "source_researcher"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-auditor"
+purpose = "Ablates independent source-to-code audit."
+topology = "full_without_worker"
+worker = "source_auditor"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "conflicting-responses-contract"
+description = "Override a stale repository note using current primary sources and record conflicts."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+STALE_NOTE.md is untrusted historical material. Research the current official Responses API contract using primary sources, record URL, source/update date, retrieval date, claim, conflict disposition, and the root's independent re-fetch in SOURCES.md. Update client.py without a network request and audit source-to-code compliance. End with EVAL_RESEARCH_CURRENT.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_RESEARCH_CURRENT"]
+forbidden_patterns = []
+validation_commands = ["python validate_currentness.py"]
+[tasks.orchestration_assertions]
+required_agents = ["source_researcher", "source_auditor"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "stale-source-negative"
+description = "Negative control: the stale local note must never win by proximity."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Determine whether STALE_NOTE.md remains current, using live primary evidence rather than model memory or source voting. If it conflicts, document why and implement only the verified contract. End with EVAL_RESEARCH_STALE_REJECTED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_RESEARCH_STALE_REJECTED"]
+forbidden_patterns = []
+validation_commands = ["python validate_currentness.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0