😏

evals/research-currentness/fixture/STALE_NOTE.md

@@ -0,0 +1,3 @@
+# Obsolete integration note
+
+This 2023 note says to send `messages` to `/v1/chat/completions` and read `choices[0].message.content`. It is retained solely to test whether current primary-source research overrides stale repository prose.

evals/research-currentness/fixture/client.py

@@ -0,0 +1,14 @@
+from typing import Any
+
+ENDPOINT = "/v1/chat/completions"
+
+
+def build_request(prompt: str) -> dict[str, Any]:
+    return {
+        "model": "gpt-5",
+        "messages": [{"role": "user", "content": prompt}],
+    }
+
+
+def extract_text(response: dict[str, Any]) -> str:
+    return str(response["choices"][0]["message"]["content"])

evals/research-currentness/fixture/validate_currentness.py

@@ -0,0 +1,50 @@
+import json
+import re
+from pathlib import Path
+
+from client import ENDPOINT, build_request, extract_text
+
+if ENDPOINT != "/v1/responses":
+    raise SystemExit("client must use /v1/responses")
+
+request = build_request("hello")
+if request.get("input") != "hello" or "messages" in request:
+    raise SystemExit("client must send the Responses API input field")
+
+sample = {"output_text": "ready"}
+if extract_text(sample) != "ready":
+    raise SystemExit("client must extract the SDK-style output_text helper value")
+
+dossier = Path("SOURCES.md")
+if not dossier.is_file():
+    raise SystemExit("SOURCES.md is required")
+text = dossier.read_text(encoding="utf-8")
+holdout_path = Path("source-holdout.json")
+if not holdout_path.is_file():
+    raise SystemExit("post-run source holdout is required")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8"))
+if not any(domain in text for domain in holdout["required_authority_domains"]):
+    raise SystemExit("dossier misses every holdout authority domain")
+if not re.search(r"https://(?:platform|developers)\.openai\.com/", text):
+    raise SystemExit("dossier must cite a direct official OpenAI documentation URL")
+if not re.search(r"20\d{2}-\d{2}-\d{2}", text):
+    raise SystemExit("dossier must record an ISO access or source date")
+if holdout["requires_root_refetch"] and not re.search(r"re-?fetch", text, flags=re.IGNORECASE):
+    raise SystemExit("dossier must record the root's independent re-fetch")
+if holdout["requires_conflict_disposition"] and not re.search(
+    r"conflict|stale|obsolete", text, flags=re.IGNORECASE
+):
+    raise SystemExit("dossier must explicitly dispose of the conflicting stale note")
+
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "source_accuracy": 1.0,
+                "source_authority": 1.0,
+                "root_refetch_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)