😏

evals/access-efficient/fixture/DISTRIBUTED_CONSTRAINTS.md

@@ -0,0 +1,4 @@
+# Cross-replica payment constraints
+
+Requests have globally unique idempotency keys; six stateless replicas can receive retries; SQL supports unique constraints and transactions; the external payment cannot join the SQL transaction; the design must recover from crashes between durable state and the side effect. Replica-local memory is neither shared nor durable.
+

evals/access-efficient/fixture/LABEL_SPEC.md

@@ -0,0 +1,4 @@
+# Label normalization
+
+`slug_label` trims surrounding whitespace, lowercases ASCII text, collapses every run of spaces or underscores to one hyphen, and removes leading or trailing hyphens.
+

evals/access-efficient/fixture/LIMITS.toml

@@ -0,0 +1,3 @@
+max_batch_items = 128
+retry_limit = 4
+

evals/access-efficient/fixture/labels.py

@@ -0,0 +1,5 @@
+import re
+
+
+def slug_label(value: str) -> str:
+    return re.sub(r" +", "-", value.strip().lower())

evals/access-efficient/fixture/test_labels.py

@@ -0,0 +1,15 @@
+import unittest
+
+from labels import slug_label
+
+
+class LabelTests(unittest.TestCase):
+    def test_mixed_separators(self):
+        self.assertEqual(slug_label("  Release__ Candidate  "), "release-candidate")
+
+    def test_outer_separators(self):
+        self.assertEqual(slug_label("__Ready__"), "ready")
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/access-efficient/fixture/validate_access.py

@@ -0,0 +1,23 @@
+import json
+import sys
+from pathlib import Path
+
+mode = sys.argv[1]
+text = Path(".mmo-eval-final.txt").read_text(encoding="utf-8").lower()
+holdout_path = Path("access-holdout.json")
+holdout = json.loads(holdout_path.read_text(encoding="utf-8")) if holdout_path.is_file() else {}
+if mode == "literal" and not (
+    holdout.get("expected_value", "128").lower() in text
+    and holdout.get("expected_path", "LIMITS.toml").lower() in text
+):
+    raise SystemExit("literal result is not grounded in the hidden expected location")
+if mode == "routine":
+    from labels import slug_label
+
+    if slug_label("  Release__ Candidate  ") != "release-candidate":
+        raise SystemExit("routine slug contract still fails")
+if mode == "decision":
+    required = holdout.get("required_terms", ["unique", "durable", "crash", "external", "local"])
+    missing = [term for term in required if term not in text]
+    if missing:
+        raise SystemExit("decision lacks constraints: " + ", ".join(missing))

evals/access-efficient/holdout/ambiguous-distributed-decision/access-holdout.json

@@ -0,0 +1 @@
+{"required_terms":["unique","durable","crash","external","local"]}

evals/access-efficient/holdout/literal-limit/access-holdout.json

@@ -0,0 +1 @@
+{"expected_path":"LIMITS.toml","expected_value":"128"}

evals/access-efficient/holdout/routine-slug/test_holdout_labels.py

@@ -0,0 +1,8 @@
+import unittest
+
+from labels import slug_label
+
+
+class HoldoutLabelTests(unittest.TestCase):
+    def test_long_mixed_separator_run(self):
+        self.assertEqual(slug_label("___Alpha _  Beta___"), "alpha-beta")

evals/access-efficient/suite.toml

@@ -0,0 +1,175 @@
+schema_version = 8
+id = "access-efficient"
+profile = "access-efficient-escalation-lab"
+name = "Access-Efficient Escalation Lab"
+description = "Experimental matched routing trials for literal local work, economical patches, and deliberate scarce-tier escalation."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "scarce_tier_request_units"
+primary_baseline = "strongest_single_agent"
+direction = "lower"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.50
+minimum_absolute_improvement = 1.0
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = []
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+scarce_model_keys = [
+  "codex_chatgpt_builtin__gpt_5_6_sol",
+  "opencode_zen_anthropic_messages__claude_opus_5",
+  "zai_coding_responses__glm_5_3",
+]
+
+[[variants]]
+id = "configured-root"
+purpose = "Luna routing root alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Sol maximum-capability single-agent control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex access control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go economical access control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen access control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan scarce-tier control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter API control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-routine"
+purpose = "Luna root plus the economical isolated routine engineer."
+topology = "root_plus_worker"
+worker = "routine_engineer"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Contained literal scout, routine engineer, and trigger-gated GLM escalation."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-literal-scout"
+purpose = "Tests whether local literal reconnaissance earns its launch overhead."
+topology = "full_without_worker"
+worker = "literal_scout"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-routine-engineer"
+purpose = "Ablates the economical isolated patch lane."
+topology = "full_without_worker"
+worker = "routine_engineer"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-escalation"
+purpose = "Ablates scarce-tier GLM escalation."
+topology = "full_without_worker"
+worker = "flagship_escalation"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "literal-limit"
+description = "Trivial literal negative control where delegation should be avoided or strictly literal."
+sandbox = "read-only"
+difficulty = "easy"
+negative_control = true
+wall_timeout_seconds = 900
+validation_timeout_seconds = 60
+prompt = """
+Report max_batch_items from LIMITS.toml with the exact path. Do no semantic reasoning and do not escalate. If the local scout is used, submit only a structured literal extract request. End with EVAL_ACCESS_LITERAL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["128", "EVAL_ACCESS_LITERAL"]
+forbidden_patterns = []
+validation_commands = ["python validate_access.py literal"]
+[tasks.orchestration_assertions]
+forbidden_agents = ["flagship_escalation"]
+max_jobs = 1
+
+[[tasks]]
+id = "routine-slug"
+description = "Localized deterministic patch suitable for economical execution."
+sandbox = "workspace-write"
+difficulty = "medium"
+negative_control = false
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 120
+prompt = """
+Implement LABEL_SPEC.md. This is a bounded objectively testable source change; use scarce-tier escalation only if concrete evidence exceeds routine capability. End with EVAL_ACCESS_ROUTINE.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ACCESS_ROUTINE"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_access.py routine"]
+[tasks.orchestration_assertions]
+required_agents = ["routine_engineer"]
+forbidden_agents = ["flagship_escalation"]
+max_jobs = 2
+max_contract_failures = 0
+
+[[tasks]]
+id = "ambiguous-distributed-decision"
+description = "Consequential ambiguity with an explicit escalation trigger."
+sandbox = "read-only"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 1800
+validation_timeout_seconds = 60
+prompt = """
+Read DISTRIBUTED_CONSTRAINTS.md and select an idempotency design. Record why the uncertainty and consequence justify or do not justify GLM escalation. Resolve claims with the supplied constraints, not model rank. End with EVAL_ACCESS_ESCALATION.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_ACCESS_ESCALATION"]
+forbidden_patterns = []
+validation_commands = ["python validate_access.py decision"]
+[tasks.orchestration_assertions]
+required_agents = ["flagship_escalation"]
+max_jobs = 1
+max_contract_failures = 0