vance/codex-mmo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

😏

Browse Source
This commit is contained in:
vance
2026-08-24 08:11:59 -07:00
commit 53df0eed10
275 changed files with 133056 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libexec/mmo_app_server.py
+2339
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_catalog.py
Executable
+1177
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_catalog_data.py
+863
View File
@@ -0,0 +1,863 @@
#!/usr/bin/env python3
"""Provider/model catalog loading, overlays, and semantic validation."""
from __future__ import annotations
import math
import re
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from mmo_util import (
config_root,
deep_merge,
install_root,
read_toml,
valid_absolute_uri,
valid_http_header_value,
validate_id,
)
from mmo_version import MMO_SCHEMA_VERSION
ALLOWED_DRIVERS = {
"switchyard",
"codex_builtin",
"codex_custom",
"codex_oss",
"catalog_only",
}
ALLOWED_WIRE_PROTOCOLS = {
"openai_chat",
"openai_responses",
"anthropic_messages",
"codex_builtin",
"codex_oss",
"catalog_only",
}
ALLOWED_BILLING_MODES = {"api", "subscription", "chatgpt_subscription", "local", "catalog_only"}
ALLOWED_MODEL_KINDS = {
"chat",
"vision_chat",
"ocr",
"image_generation",
"video_generation",
"audio_transcription",
"agent_service",
}
ALLOWED_MODALITIES = {"text", "image", "audio", "video", "file"}
ALLOWED_REASONING = {"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}
def user_catalog_root() -> Path:
return config_root() / "catalog.d"
ROUTE_FIELDS = {
"driver",
"name",
"api_operator",
"access_product",
"wire_protocol",
"billing_mode",
"base_url",
"provider_id",
"auth",
"auth_link_mode",
"wire_api",
"credential_envs",
"http_headers",
"env_http_headers",
"extra_headers",
"request_max_retries",
"stream_max_retries",
"stream_idle_timeout_ms",
"max_retries",
"transport_modalities",
"transport_output_modalities",
"tool_calling",
"parallel_tool_calls",
"preserves_tool_media",
"tool_result_modalities",
"supports_documents",
"resource_group",
"inventory",
"openrouter_policy",
}
ROUTE_COMMON_FIELDS = {
"driver",
"name",
"api_operator",
"access_product",
"wire_protocol",
"billing_mode",
"transport_modalities",
"transport_output_modalities",
"tool_calling",
"parallel_tool_calls",
"preserves_tool_media",
"tool_result_modalities",
"supports_documents",
"resource_group",
"inventory",
"openrouter_policy",
}
ROUTE_DRIVER_FIELDS = {
"switchyard": {
"base_url",
"credential_envs",
"extra_headers",
"max_retries",
},
"codex_custom": {
"base_url",
"wire_api",
"credential_envs",
"http_headers",
"env_http_headers",
"request_max_retries",
"stream_max_retries",
"stream_idle_timeout_ms",
},
"codex_builtin": {"provider_id", "auth", "auth_link_mode"},
"codex_oss": {"provider_id"},
# Catalog-only providers retain endpoint/auth facts for inventory and
# operator diagnostics, but none of these fields is emitted for execution.
"catalog_only": {
"base_url",
"wire_api",
"credential_envs",
"auth",
},
}
MODEL_FIELDS = {
"maker",
"route",
"upstream_id",
"display_name",
"description",
"kind",
"agent_compatible",
"context_window",
"max_output_tokens",
"reasoning_levels",
"default_reasoning",
"modalities",
"output_modalities",
"supports_tool_images",
"supports_documents",
"tool_calling",
"supports_custom_tools",
"parallel_tool_calls",
"supports_reasoning_summaries",
"structured_output",
"availability",
"capability_confidence",
"source",
"availability_source",
"capability_source",
"pricing_source",
"inventory",
"resource_group",
"input_cost_per_million",
"cached_input_cost_per_million",
"cache_write_input_cost_per_million",
"output_cost_per_million",
"unit_cost_usd",
"extra_body",
"route_policy",
}
OPENROUTER_POLICY_FIELDS = {
"only",
"order",
"allow_fallbacks",
"require_parameters",
"data_collection",
"zdr",
"quantizations",
"sort",
"max_price",
}
RESOURCE_FIELDS = {"description", "lock_key", "max_active"}
def _reject_unknown_fields(value: Mapping[str, Any], allowed: set[str], label: str) -> None:
unknown = sorted(set(value) - allowed)
if unknown:
raise ValueError(f"{label} has unknown fields: {', '.join(unknown)}")
def _schema_version(value: Any, expected: int, label: str) -> int:
if not isinstance(value, int) or isinstance(value, bool) or value != expected:
raise ValueError(f"unsupported {label} schema_version")
return value
def _nonempty_string(value: Any, label: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ValueError(f"{label} must be a non-empty string")
return value
def _optional_nonempty_string(value: Any, label: str) -> str | None:
if value is None:
return None
return _nonempty_string(value, label)
def _enum_string(value: Any, allowed: set[str], label: str) -> str:
if not isinstance(value, str) or value not in allowed:
raise ValueError(f"{label} must be one of {sorted(allowed)}")
return value
def _load_catalog_fragment(path: Path) -> dict[str, Any]:
data = read_toml(path)
_reject_unknown_fields(
data, {"schema_version", "routes", "models", "resources"}, f"catalog {path}"
)
try:
_schema_version(data.get("schema_version"), MMO_SCHEMA_VERSION, "catalog")
except ValueError as exc:
raise ValueError(f"unsupported catalog schema_version in {path}") from exc
result: dict[str, Any] = {}
for section in ("routes", "models", "resources"):
value = data.get(section, {})
if not isinstance(value, Mapping):
raise ValueError(f"catalog {section!r} must be a table in {path}")
if not all(isinstance(item, Mapping) for item in value.values()):
raise ValueError(f"catalog {section!r} entries must be tables in {path}")
result[section] = value
return result
def load_global_catalog() -> dict[str, Any]:
"""Load the bundled catalog plus deterministic user overlays.
User fragments are merged lexicographically so an operator can override or
extend a provider/model without editing installed files. Profile-local
fragments are applied separately by :func:`load_catalog`.
"""
catalog = _load_catalog_fragment(install_root() / "config" / "catalog.toml")
root = user_catalog_root()
if root.is_dir():
for path in sorted(root.glob("*.toml")):
catalog = deep_merge(catalog, _load_catalog_fragment(path))
return catalog
def validate_catalog_data(catalog: Mapping[str, Any], *, label: str = "catalog") -> dict[str, Any]:
"""Validate one fully merged catalog and all cross-section references."""
if not isinstance(catalog, Mapping):
raise ValueError(f"{label} must be a table")
_reject_unknown_fields(catalog, {"schema_version", "routes", "models", "resources"}, label)
if "schema_version" in catalog:
_schema_version(catalog["schema_version"], MMO_SCHEMA_VERSION, label)
sections: dict[str, Mapping[str, Any]] = {}
for section in ("routes", "models", "resources"):
value = catalog.get(section, {})
if not isinstance(value, Mapping):
raise ValueError(f"{label}.{section} must be a table")
if not all(isinstance(item, Mapping) for item in value.values()):
raise ValueError(f"{label}.{section} entries must be tables")
sections[section] = value
routes = {key: _validate_route(key, value) for key, value in sections["routes"].items()}
resources = _validate_resources(sections["resources"])
models = _validate_models(sections["models"], routes)
for key, route in routes.items():
resource = route.get("resource_group")
if resource is not None and resource not in resources:
raise ValueError(f"route {key}: unknown resource group {resource!r}")
for key, model in models.items():
resource = model.get("resource_group")
if resource is not None and resource not in resources:
raise ValueError(f"model {key}: unknown resource group {resource!r}")
return {"routes": routes, "models": models, "resources": resources}
def validated_global_catalog() -> dict[str, Any]:
"""Return the merged global catalog after full semantic validation."""
return validate_catalog_data(load_global_catalog(), label="global catalog")
def load_catalog(profile_dir: Path, profile_data: Mapping[str, Any]) -> dict[str, Any]:
catalog = load_global_catalog()
profile_catalog = profile_data.get("catalog", "catalog.toml")
if not isinstance(profile_catalog, str) or not profile_catalog:
raise ValueError("profile.catalog must be a non-empty relative path string")
if Path(profile_catalog).is_absolute():
raise ValueError("profile.catalog must be a non-empty relative path string")
path = (profile_dir / profile_catalog).resolve()
if profile_dir not in path.parents:
raise ValueError("profile catalog path escapes the profile pack")
if path.is_file():
catalog = deep_merge(catalog, _load_catalog_fragment(path))
elif "catalog" in profile_data:
raise ValueError(f"explicit profile catalog file is missing: {profile_catalog}")
return catalog
def _string_list(value: Any, label: str, *, allow_empty: bool = True) -> list[str]:
if value is None and allow_empty:
return []
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise ValueError(f"{label} must be an array of strings")
if any(not item.strip() for item in value):
raise ValueError(f"{label} cannot contain empty strings")
if not allow_empty and not value:
raise ValueError(f"{label} cannot be empty")
if len(value) != len(set(value)):
raise ValueError(f"{label} cannot contain duplicates")
return list(value)
def _positive_int(value: Any, label: str, minimum: int = 1, maximum: int = 1_000_000) -> int:
if not isinstance(value, int) or isinstance(value, bool) or not minimum <= value <= maximum:
raise ValueError(f"{label} must be an integer between {minimum} and {maximum}")
return value
_HEADER_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+")
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
def _validate_http_base_url(value: Any, label: str) -> str:
"""Validate an HTTP API root before downstream endpoint concatenation."""
if not isinstance(value, str) or not valid_absolute_uri(value):
raise ValueError(f"{label}: valid HTTP(S) base_url is required")
try:
parsed = urlsplit(value)
# Accessing ``port`` performs the range and syntax checks that
# ``urlsplit`` intentionally defers.
_parsed_port = parsed.port
except ValueError as exc:
raise ValueError(f"{label}: valid HTTP(S) base_url is required") from exc
if (
parsed.scheme.lower() not in {"http", "https"}
or parsed.hostname is None
or parsed.username is not None
or parsed.password is not None
# Codex 0.149 trims slashes before appending the endpoint path, while
# Switchyard 0.2.0 also normalizes known endpoint suffixes. Neither
# treats a query or fragment embedded in the base as a URI join input.
or "?" in value
or "#" in value
):
raise ValueError(f"{label}: valid HTTP(S) base_url is required")
return value
def _header_map(value: Any, label: str, *, environment_values: bool = False) -> dict[str, str]:
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a string-to-string table")
result: dict[str, str] = {}
seen: set[str] = set()
for raw_name, raw_value in value.items():
if not isinstance(raw_name, str) or not _HEADER_NAME.fullmatch(raw_name):
raise ValueError(f"{label}: invalid HTTP header name {raw_name!r}")
normalized_name = raw_name.lower()
if normalized_name in seen:
raise ValueError(f"{label}: duplicate case-insensitive HTTP header name {raw_name!r}")
seen.add(normalized_name)
if not isinstance(raw_value, str):
raise ValueError(f"{label}.{raw_name} must be a string")
if environment_values:
if not _ENVIRONMENT_NAME.fullmatch(raw_value):
raise ValueError(
f"{label}.{raw_name}: invalid environment variable name {raw_value!r}"
)
elif not valid_http_header_value(raw_value):
raise ValueError(f"{label}.{raw_name} contains a prohibited control character")
result[raw_name] = raw_value
return result
def _json_compatible_value(value: Any, label: str) -> Any:
"""Validate a value destined for Switchyard's serde_json::Value map."""
if isinstance(value, (str, bool)):
return value
if isinstance(value, int) and not isinstance(value, bool):
if -(2**63) <= value <= 2**63 - 1:
return value
raise ValueError(f"{label} integer is outside the TOML/JSON target range")
if isinstance(value, float):
if math.isfinite(value):
return value
raise ValueError(f"{label} must not contain a non-finite number")
if isinstance(value, list):
return [
_json_compatible_value(item, f"{label}[{index}]") for index, item in enumerate(value)
]
if isinstance(value, Mapping):
result: dict[str, Any] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ValueError(f"{label} object keys must be strings")
result[key] = _json_compatible_value(item, f"{label}.{key}")
return result
raise ValueError(f"{label} must contain only JSON-compatible TOML values")
def _credential_envs(route: Mapping[str, Any], label: str) -> list[str]:
"""Validate a route's ordered credential environment names."""
raw = route.get("credential_envs", [])
values = _string_list(raw, f"{label}.credential_envs")
for value in values:
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value):
raise ValueError(f"{label}: invalid credential environment variable {value!r}")
return list(dict.fromkeys(values))
def _validate_openrouter_policy(value: Any, label: str) -> dict[str, Any] | None:
if value is None:
return None
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a table")
_reject_unknown_fields(value, OPENROUTER_POLICY_FIELDS, label)
result = dict(value)
for field in ("only", "order", "quantizations"):
if field in result:
result[field] = _string_list(result[field], f"{label}.{field}", allow_empty=False)
for field in ("allow_fallbacks", "require_parameters", "zdr"):
if field in result and not isinstance(result[field], bool):
raise ValueError(f"{label}.{field} must be boolean")
if "data_collection" in result:
result["data_collection"] = _enum_string(
result["data_collection"], {"allow", "deny"}, f"{label}.data_collection"
)
if "sort" in result:
result["sort"] = _enum_string(
result["sort"], {"price", "throughput", "latency"}, f"{label}.sort"
)
if "max_price" in result:
price = result["max_price"]
if not isinstance(price, Mapping):
raise ValueError(f"{label}.max_price must be a table")
unknown = sorted(set(price) - {"prompt", "completion", "request", "image"})
if unknown:
raise ValueError(f"{label}.max_price has unknown fields: {', '.join(unknown)}")
normalized: dict[str, float] = {}
for field, raw in price.items():
if not isinstance(raw, (int, float)) or isinstance(raw, bool) or raw < 0:
raise ValueError(f"{label}.max_price.{field} must be non-negative and finite")
try:
numeric = float(raw)
except OverflowError:
raise ValueError(
f"{label}.max_price.{field} must be non-negative and finite"
) from None
if not math.isfinite(numeric):
raise ValueError(f"{label}.max_price.{field} must be non-negative and finite")
normalized[str(field)] = numeric
result["max_price"] = normalized
return result
def _validate_route(key: str, route: Mapping[str, Any]) -> dict[str, Any]:
validate_id(key, "route id")
_reject_unknown_fields(route, ROUTE_FIELDS, f"route {key}")
driver = _enum_string(route.get("driver"), ALLOWED_DRIVERS, f"route {key}.driver")
unsupported = sorted(set(route) - ROUTE_COMMON_FIELDS - ROUTE_DRIVER_FIELDS[driver])
if unsupported:
raise ValueError(f"route {key}: fields unsupported by driver {driver!r}: {unsupported}")
result = dict(route)
result.setdefault("name", key)
result["name"] = _nonempty_string(result["name"], f"route {key}.name")
for field in ("api_operator", "access_product"):
result[field] = _nonempty_string(result.get(field), f"route {key}.{field}")
result["wire_protocol"] = _enum_string(
result.get("wire_protocol"), ALLOWED_WIRE_PROTOCOLS, f"route {key}.wire_protocol"
)
result["billing_mode"] = _enum_string(
result.get("billing_mode"), ALLOWED_BILLING_MODES, f"route {key}.billing_mode"
)
expected_protocols = {
"switchyard": {"openai_chat", "openai_responses", "anthropic_messages"},
"codex_custom": {"openai_responses"},
"codex_builtin": {"codex_builtin"},
"codex_oss": {"codex_oss"},
"catalog_only": {"catalog_only", "openai_chat", "openai_responses", "anthropic_messages"},
}[driver]
if result["wire_protocol"] not in expected_protocols:
raise ValueError(f"route {key}.wire_protocol is incompatible with driver {driver!r}")
result["openrouter_policy"] = _validate_openrouter_policy(
result.get("openrouter_policy"), f"route {key}.openrouter_policy"
)
if result["openrouter_policy"] is not None and result["api_operator"] != "openrouter":
raise ValueError(f"route {key}: openrouter_policy requires api_operator='openrouter'")
result.setdefault("transport_modalities", ["text"])
result["transport_modalities"] = _string_list(
result["transport_modalities"], f"route {key}.transport_modalities", allow_empty=False
)
unknown_modalities = sorted(set(result["transport_modalities"]) - ALLOWED_MODALITIES)
if unknown_modalities:
raise ValueError(f"route {key}: invalid transport modalities {unknown_modalities}")
result.setdefault(
"transport_output_modalities",
["text", "image", "video", "audio", "file"] if driver == "catalog_only" else ["text"],
)
result["transport_output_modalities"] = _string_list(
result["transport_output_modalities"],
f"route {key}.transport_output_modalities",
allow_empty=False,
)
unknown_output_modalities = sorted(
set(result["transport_output_modalities"]) - ALLOWED_MODALITIES
)
if unknown_output_modalities:
raise ValueError(f"route {key}: invalid output modalities {unknown_output_modalities}")
result.setdefault("tool_calling", driver != "catalog_only")
result.setdefault("parallel_tool_calls", driver != "catalog_only")
result.setdefault(
"preserves_tool_media",
driver in {"codex_builtin", "codex_custom"},
)
result.setdefault(
"tool_result_modalities",
list(result["transport_modalities"]) if result["preserves_tool_media"] else ["text"],
)
result["tool_result_modalities"] = _string_list(
result["tool_result_modalities"],
f"route {key}.tool_result_modalities",
allow_empty=False,
)
unknown_tool_modalities = sorted(set(result["tool_result_modalities"]) - ALLOWED_MODALITIES)
if unknown_tool_modalities:
raise ValueError(f"route {key}: invalid tool-result modalities {unknown_tool_modalities}")
result.setdefault("supports_documents", "file" in result["transport_modalities"])
for boolean_field in (
"tool_calling",
"parallel_tool_calls",
"preserves_tool_media",
"supports_documents",
):
if not isinstance(result[boolean_field], bool):
raise ValueError(f"route {key}.{boolean_field} must be boolean")
if result["parallel_tool_calls"] and not result["tool_calling"]:
raise ValueError(f"route {key}: parallel_tool_calls requires tool_calling")
if not result["preserves_tool_media"] and set(result["tool_result_modalities"]) - {"text"}:
raise ValueError(
f"route {key}: non-text tool_result_modalities require preserves_tool_media=true"
)
result.setdefault("resource_group", None)
result.setdefault("inventory", None)
for field in ("resource_group", "inventory"):
result[field] = _optional_nonempty_string(result[field], f"route {key}.{field}")
credentials = _credential_envs(result, f"route {key}")
result["credential_envs"] = credentials
if driver == "switchyard":
if "extra_headers" in result:
result["extra_headers"] = _header_map(
result["extra_headers"], f"route {key}.extra_headers"
)
result["base_url"] = _validate_http_base_url(result.get("base_url"), f"route {key}")
result.setdefault("max_retries", 1)
result["max_retries"] = _positive_int(
result["max_retries"], f"route {key}.max_retries", 0, 10
)
elif driver == "codex_custom":
if "http_headers" in result:
result["http_headers"] = _header_map(
result["http_headers"], f"route {key}.http_headers"
)
if "env_http_headers" in result:
result["env_http_headers"] = _header_map(
result["env_http_headers"],
f"route {key}.env_http_headers",
environment_values=True,
)
static_names = {name.lower() for name in result.get("http_headers", {})}
environment_names = {name.lower() for name in result.get("env_http_headers", {})}
overlap = sorted(static_names & environment_names)
if overlap:
raise ValueError(
f"route {key} defines headers in both http_headers and env_http_headers: "
+ ", ".join(overlap)
)
result["base_url"] = _validate_http_base_url(result.get("base_url"), f"route {key}")
result.setdefault("wire_api", "responses")
if not isinstance(result["wire_api"], str) or result["wire_api"] != "responses":
raise ValueError(f"route {key}: Codex custom routes require wire_api='responses'")
for field, default, minimum, maximum in (
("request_max_retries", 1, 0, 2**63 - 1),
("stream_max_retries", 1, 0, 2**63 - 1),
("stream_idle_timeout_ms", 600_000, 1_000, 2**63 - 1),
):
result.setdefault(field, default)
result[field] = _positive_int(result[field], f"route {key}.{field}", minimum, maximum)
elif driver == "codex_builtin":
provider_id = result.get("provider_id")
provider_id = _nonempty_string(provider_id, f"route {key}.provider_id")
if provider_id not in {"openai", "amazon-bedrock"}:
raise ValueError(
f"route {key}: Codex built-in provider_id must be openai or amazon-bedrock"
)
result["provider_id"] = provider_id
result.setdefault("auth", "chatgpt" if provider_id == "openai" else "builtin")
expected_auth = "chatgpt" if provider_id == "openai" else "builtin"
if result.get("auth") != expected_auth:
raise ValueError(
f"route {key}: auth must be {expected_auth!r} for built-in {provider_id!r}"
)
if result.get("auth_link_mode") is not None and not isinstance(
result.get("auth_link_mode"), str
):
raise ValueError(f"route {key}: invalid auth_link_mode")
if result.get("auth_link_mode") not in {None, "shared", "copy", "none"}:
raise ValueError(f"route {key}: invalid auth_link_mode")
elif driver == "codex_oss":
provider_id = result.get("provider_id")
if not isinstance(provider_id, str) or provider_id not in {"ollama", "lmstudio"}:
raise ValueError(f"route {key}: codex_oss provider_id must be ollama or lmstudio")
elif driver == "catalog_only":
if "base_url" in result:
result["base_url"] = _validate_http_base_url(result["base_url"], f"route {key}")
for field in ("wire_api", "auth"):
if field in result:
result[field] = _nonempty_string(result[field], f"route {key}.{field}")
header_fields = {"http_headers", "env_http_headers", "extra_headers"}
allowed_header_fields = {
"switchyard": {"extra_headers"},
"codex_custom": {"http_headers", "env_http_headers"},
}.get(driver, set())
unsupported_headers = sorted((header_fields & set(result)) - allowed_header_fields)
if unsupported_headers:
detail = f"header fields are unsupported by driver {driver}"
raise ValueError(f"route {key}: {detail}: {unsupported_headers}")
return result
def validate_model_entry(
key: str, model: Mapping[str, Any], routes: Mapping[str, Any]
) -> dict[str, Any]:
validate_id(key, "model id")
_reject_unknown_fields(model, MODEL_FIELDS, f"model {key}")
route_key = model.get("route")
if not isinstance(route_key, str) or route_key not in routes:
raise ValueError(f"model {key}: unknown route {route_key!r}")
if not key.startswith(f"{route_key}__"):
raise ValueError(
f"model {key}: catalog key must start with exact route namespace "
f"{route_key!r} followed by '__'"
)
if not isinstance(model.get("upstream_id"), str) or not model["upstream_id"].strip():
raise ValueError(f"model {key}: upstream_id is required")
if not isinstance(model.get("maker"), str) or not model["maker"].strip():
raise ValueError(f"model {key}: maker is required")
result = dict(model)
result.setdefault("display_name", key)
result.setdefault("description", "")
result.setdefault("kind", "chat")
result["display_name"] = _nonempty_string(result["display_name"], f"model {key}.display_name")
if not isinstance(result["description"], str):
raise ValueError(f"model {key}.description must be a string")
result["kind"] = _enum_string(result["kind"], ALLOWED_MODEL_KINDS, f"model {key}.kind")
result.setdefault(
"agent_compatible",
result["kind"] in {"chat", "vision_chat"} and routes[route_key]["driver"] != "catalog_only",
)
if not isinstance(result["agent_compatible"], bool):
raise ValueError(f"model {key}.agent_compatible must be boolean")
result.setdefault("context_window", 131072 if result["agent_compatible"] else 0)
result["context_window"] = _positive_int(
result["context_window"],
f"model {key}.context_window",
1024 if result["agent_compatible"] else 0,
20_000_000,
)
result.setdefault("reasoning_levels", ["none", "low", "medium", "high"])
result["reasoning_levels"] = _string_list(
result["reasoning_levels"], f"model {key}.reasoning_levels", allow_empty=False
)
unknown_reasoning = sorted(set(result["reasoning_levels"]) - ALLOWED_REASONING)
if unknown_reasoning:
raise ValueError(f"model {key}: invalid reasoning levels {unknown_reasoning}")
result.setdefault("default_reasoning", result["reasoning_levels"][-1])
if (
not isinstance(result["default_reasoning"], str)
or result["default_reasoning"] not in result["reasoning_levels"]
):
raise ValueError(f"model {key}: default_reasoning is not supported")
route = routes[route_key]
result["route_policy"] = _validate_openrouter_policy(
result.get("route_policy"), f"model {key}.route_policy"
)
if result["route_policy"] is not None and route["api_operator"] != "openrouter":
raise ValueError(f"model {key}: route_policy requires an OpenRouter route")
if "extra_body" in result:
if route["driver"] != "switchyard":
raise ValueError(f"model {key}: extra_body is supported only by Switchyard targets")
if not isinstance(result["extra_body"], Mapping):
raise ValueError(f"model {key}.extra_body must be a table")
result["extra_body"] = _json_compatible_value(
result["extra_body"], f"model {key}.extra_body"
)
result.setdefault("modalities", route.get("transport_modalities", ["text"]))
result["modalities"] = _string_list(
result["modalities"], f"model {key}.modalities", allow_empty=False
)
unknown_modalities = sorted(set(result["modalities"]) - ALLOWED_MODALITIES)
if unknown_modalities:
raise ValueError(f"model {key}: invalid modalities {unknown_modalities}")
missing_input_transport = sorted(set(result["modalities"]) - set(route["transport_modalities"]))
if missing_input_transport:
raise ValueError(
f"model {key}: route transport cannot carry input modalities {missing_input_transport}"
)
result.setdefault("output_modalities", ["text"])
result["output_modalities"] = _string_list(
result["output_modalities"], f"model {key}.output_modalities", allow_empty=False
)
unknown_output_modalities = sorted(set(result["output_modalities"]) - ALLOWED_MODALITIES)
if unknown_output_modalities:
raise ValueError(f"model {key}: invalid output modalities {unknown_output_modalities}")
missing_output_transport = sorted(
set(result["output_modalities"]) - set(route["transport_output_modalities"])
)
if missing_output_transport:
raise ValueError(
f"model {key}: route transport cannot carry output modalities "
f"{missing_output_transport}"
)
result.setdefault(
"supports_tool_images",
"image" in result["modalities"]
and route.get("preserves_tool_media", False)
and "image" in route.get("tool_result_modalities", []),
)
result.setdefault(
"supports_documents",
"file" in result["modalities"] and route.get("supports_documents", False),
)
for boolean_field in ("supports_tool_images", "supports_documents"):
if not isinstance(result[boolean_field], bool):
raise ValueError(f"model {key}.{boolean_field} must be boolean")
if result["supports_tool_images"] and "image" not in result["modalities"]:
raise ValueError(f"model {key}: supports_tool_images requires image input")
if result["supports_documents"] and "file" not in result["modalities"]:
raise ValueError(f"model {key}: supports_documents requires file input")
result.setdefault("tool_calling", route.get("tool_calling", True))
# OpenAI Responses distinguishes schema-defined function tools from
# free-form custom tools. Some compatible endpoints implement the former
# but reject the latter. Keep that narrower capability independent so a
# model can still use shell and MCP function tools without receiving
# Codex's free-form apply_patch tool.
result.setdefault("supports_custom_tools", result["tool_calling"])
result.setdefault("parallel_tool_calls", route.get("parallel_tool_calls", True))
result.setdefault("supports_reasoning_summaries", False)
result.setdefault("structured_output", False)
for boolean_field in (
"tool_calling",
"supports_custom_tools",
"parallel_tool_calls",
"supports_reasoning_summaries",
"structured_output",
):
if not isinstance(result[boolean_field], bool):
raise ValueError(f"model {key}.{boolean_field} must be boolean")
if result["tool_calling"] and not route["tool_calling"]:
raise ValueError(f"model {key}: tool_calling exceeds route transport capability")
if result["supports_custom_tools"] and not result["tool_calling"]:
raise ValueError(f"model {key}: supports_custom_tools requires tool_calling")
if result["parallel_tool_calls"] and not result["tool_calling"]:
raise ValueError(f"model {key}: parallel_tool_calls requires tool_calling")
if result["parallel_tool_calls"] and not route["parallel_tool_calls"]:
raise ValueError(f"model {key}: parallel_tool_calls exceeds route transport capability")
result.setdefault("max_output_tokens", None)
if result["max_output_tokens"] is not None:
result["max_output_tokens"] = _positive_int(
result["max_output_tokens"], f"model {key}.max_output_tokens", 1, 20_000_000
)
result.setdefault("availability", "current")
result.setdefault("capability_confidence", "documented")
result.setdefault("source", "bundled-catalog")
for field in ("availability", "capability_confidence", "source"):
result[field] = _nonempty_string(result[field], f"model {key}.{field}")
for field in (
"availability_source",
"capability_source",
"pricing_source",
"inventory",
"resource_group",
):
if field in result:
result[field] = _optional_nonempty_string(result[field], f"model {key}.{field}")
result.setdefault("resource_group", route.get("resource_group"))
for field in (
"input_cost_per_million",
"cached_input_cost_per_million",
"cache_write_input_cost_per_million",
"output_cost_per_million",
"unit_cost_usd",
):
if field not in result:
continue
value = result[field]
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or (isinstance(value, float) and not math.isfinite(value))
or value < 0
):
raise ValueError(f"model {key}.{field} must be a non-negative finite number")
return result
def _validate_models(
values: Mapping[str, Any], routes: Mapping[str, Any]
) -> dict[str, dict[str, Any]]:
"""Validate model rows and reject duplicate route/upstream identities."""
result: dict[str, dict[str, Any]] = {}
identities: dict[tuple[str, str], str] = {}
for key, value in values.items():
model = validate_model_entry(key, value, routes)
identity = (str(model["route"]), str(model["upstream_id"]))
previous = identities.get(identity)
if previous is not None:
raise ValueError(
f"models {previous!r} and {key!r} duplicate route/upstream binding {identity!r}"
)
identities[identity] = key
result[key] = model
return result
def _validate_resource(key: str, resource: Mapping[str, Any]) -> dict[str, Any]:
validate_id(key, "resource group id")
_reject_unknown_fields(resource, RESOURCE_FIELDS, f"resource {key}")
result = dict(resource)
result.setdefault("description", "")
if not isinstance(result["description"], str):
raise ValueError(f"resource {key}.description must be a string")
result.setdefault("lock_key", key)
if not isinstance(result["lock_key"], str) or not result["lock_key"]:
raise ValueError(f"resource {key}: lock_key is required")
result.setdefault("max_active", 4)
result["max_active"] = _positive_int(
result["max_active"], f"resource {key}.max_active", 1, 1024
)
return result
def _validate_resources(values: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
result = {key: _validate_resource(key, value) for key, value in values.items()}
capacities: dict[str, tuple[str, int]] = {}
for key, resource in result.items():
lock_key = str(resource["lock_key"])
maximum = int(resource["max_active"])
previous = capacities.get(lock_key)
if previous is not None and previous[1] != maximum:
raise ValueError(
f"resources {previous[0]!r} and {key!r} share lock_key {lock_key!r} "
f"with conflicting max_active values {previous[1]} and {maximum}"
)
capacities[lock_key] = (key, maximum)
return result
libexec/mmo_cli_output.py
+462
View File
@@ -0,0 +1,462 @@
"""Terminal-aware presentation helpers for the Codex MMO command line."""
from __future__ import annotations
import json
import shutil
import sys
from collections.abc import Mapping, Sequence
from typing import Any, TextIO
def json_text(value: Any) -> str:
"""Serialize one strict, deterministic JSON value."""
return json.dumps(
value,
ensure_ascii=False,
indent=2,
sort_keys=True,
allow_nan=False,
)
def emit_json(value: Any, *, stream: TextIO | None = None) -> None:
print(json_text(value), file=stream or sys.stdout)
def stdout_is_tty() -> bool:
return bool(getattr(sys.stdout, "isatty", lambda: False)())
def stderr_is_tty() -> bool:
return bool(getattr(sys.stderr, "isatty", lambda: False)())
def progress(message: str, *, quiet: bool = False) -> None:
"""Emit stable, non-animated progress only to an interactive diagnostic stream."""
if not quiet and stderr_is_tty():
print(message, file=sys.stderr, flush=True)
def emit_error(
*,
category: str,
message: str,
hint: str | None = None,
as_json: bool = False,
exception_type: str | None = None,
traceback_text: str | None = None,
) -> None:
"""Write one user-facing error without contaminating stdout."""
if as_json:
value: dict[str, Any] = {"error_type": category, "error": message}
if hint:
value["hint"] = hint
if exception_type:
value["exception_type"] = exception_type
if traceback_text:
value["traceback"] = traceback_text
emit_json(value, stream=sys.stderr)
return
print(f"error: {message}", file=sys.stderr)
if hint:
print(f"hint: {hint}", file=sys.stderr)
if exception_type:
print(f"exception: {exception_type}", file=sys.stderr)
if traceback_text:
print(traceback_text.rstrip(), file=sys.stderr)
def emit_usage_error(
usage: str,
message: str,
*,
hint: str | None = None,
as_json: bool = False,
) -> None:
if not as_json:
print(usage.rstrip(), file=sys.stderr)
emit_error(
category="usage",
message=message,
hint=hint,
as_json=as_json,
)
def _display(value: Any) -> str:
if value is None:
return "-"
if isinstance(value, bool):
return "yes" if value else "no"
if isinstance(value, float):
return f"{value:g}"
if isinstance(value, (list, tuple)):
return ", ".join(_display(item) for item in value) if value else "-"
return str(value)
def _clip(value: str, width: int) -> str:
if width < 4 or len(value) <= width:
return value
return value[: width - 3].rstrip() + "..."
def _terminal_width() -> int:
return max(40, shutil.get_terminal_size(fallback=(100, 24)).columns)
# (heading, mapping key, required, optional maximum width)
Column = tuple[str, str, bool, int | None]
def _table(
rows: Sequence[Mapping[str, Any]],
columns: Sequence[Column],
*,
empty: str,
) -> None:
if not rows:
print(empty)
return
values = [
{key: _display(row.get(key)) for _heading, key, _required, _maximum in columns}
for row in rows
]
widths: dict[str, int] = {}
for heading, key, _required, maximum in columns:
natural = max(len(heading), *(len(row[key]) for row in values))
widths[key] = min(natural, maximum) if maximum is not None else natural
selected = list(columns)
def table_width(items: Sequence[Column]) -> int:
return sum(widths[key] for _heading, key, _required, _maximum in items) + 2 * (
len(items) - 1
)
available = _terminal_width()
for column in reversed(columns):
if table_width(selected) <= available:
break
if not column[2] and column in selected:
selected.remove(column)
if table_width(selected) > available:
# Identifiers and other required values remain lossless in narrow terminals.
for index, row in enumerate(values):
if index:
print()
for heading, key, _required, _maximum in columns:
print(f"{heading.title()}: {row[key]}")
return
print(" ".join(f"{heading:<{widths[key]}}" for heading, key, _r, _m in selected))
for row in values:
print(
" ".join(
f"{_clip(row[key], widths[key]):<{widths[key]}}"
for _heading, key, _required, _maximum in selected
).rstrip()
)
def _mapping_rows(value: Mapping[str, Any], *, key_name: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for key, item in value.items():
if isinstance(item, Mapping):
rows.append({key_name: key, **item})
else:
rows.append({key_name: key, "value": item})
return rows
def _render_nested(value: Any, *, indent: int = 0) -> None:
prefix = " " * indent
if isinstance(value, Mapping):
if not value:
print(prefix + "(none)")
return
for key, item in value.items():
label = str(key).replace("_", " ").capitalize()
if isinstance(item, (Mapping, list, tuple)):
print(f"{prefix}{label}:")
_render_nested(item, indent=indent + 2)
else:
print(f"{prefix}{label}: {_display(item)}")
return
if isinstance(value, (list, tuple)):
if not value:
print(prefix + "(none)")
return
for item in value:
if isinstance(item, (Mapping, list, tuple)):
print(prefix + "-")
_render_nested(item, indent=indent + 2)
else:
print(f"{prefix}- {_display(item)}")
return
print(prefix + _display(value))
def _render_pass_report(value: Mapping[str, Any]) -> None:
passed = value.get("passed", value.get("valid"))
if passed is not None:
print(f"Status: {'passed' if passed else 'failed'}")
for key in ("profile", "profile_id", "suite", "suite_id", "run_id", "report_path"):
if value.get(key) is not None:
print(f"{key.replace('_', ' ').title()}: {_display(value[key])}")
checks = value.get("checks")
if isinstance(checks, Mapping):
print("Checks:")
for name, check in checks.items():
check_passed = check.get("passed") if isinstance(check, Mapping) else bool(check)
print(f" {'pass' if check_passed else 'FAIL'} {name}")
for key in ("errors", "warnings", "failure", "error"):
item = value.get(key)
if item:
print(f"{key.capitalize()}:")
_render_nested(item, indent=2)
remaining = {
key: item
for key, item in value.items()
if key
not in {
"passed",
"valid",
"profile",
"profile_id",
"suite",
"suite_id",
"run_id",
"report_path",
"checks",
"errors",
"warnings",
"failure",
"error",
}
}
if remaining:
print("Details:")
_render_nested(remaining, indent=2)
def render_human(command: str, value: Any) -> None:
"""Render one complete structured result for an interactive terminal."""
if command == "profile.list" and isinstance(value, Mapping):
_table(
_mapping_rows(value, key_name="id"),
(
("ACTIVE", "active", False, 6),
("PROFILE", "id", True, None),
("MATURITY", "maturity", False, 10),
("ROOT", "root", False, 24),
("SOURCE", "source", False, 8),
("DESCRIPTION", "description", False, 60),
),
empty="No profiles found.",
)
return
if command == "tool-mcp.list" and isinstance(value, Mapping):
servers = value.get("servers")
rows = _mapping_rows(servers, key_name="server") if isinstance(servers, Mapping) else []
for row in rows:
row["ready"] = bool(row.get("ready"))
row["tools"] = len(row.get("enabled_tools") or [])
_table(
rows,
(
("SERVER", "server", True, None),
("TRANSPORT", "transport", False, 16),
("READY", "ready", False, 5),
("TOOLS", "tools", False, 5),
("SOURCE", "source", False, 40),
),
empty=f"No Tool MCP servers are defined under {value.get('registry_root', '-')}",
)
return
if command == "catalog.models" and isinstance(value, Mapping):
_table(
_mapping_rows(value, key_name="key"),
(
("MODEL", "key", True, None),
("MAKER", "maker", False, 16),
("ROUTE", "route", False, 28),
("INVENTORY", "inventory", False, 18),
("AVAILABLE", "availability", False, 12),
),
empty="No matching catalog models found.",
)
return
if command == "catalog.routes" and isinstance(value, Mapping):
_table(
_mapping_rows(value, key_name="route"),
(
("ROUTE", "route", True, None),
("DRIVER", "driver", False, 20),
("PROTOCOL", "wire_protocol", False, 20),
("ACCESS", "access_product", False, 22),
("ENDPOINT", "base_url", False, 50),
),
empty="No catalog routes found.",
)
return
if command == "catalog.resources" and isinstance(value, Mapping):
_table(
_mapping_rows(value, key_name="resource"),
(
("RESOURCE", "resource", True, None),
("CAPACITY", "max_active", False, 8),
("LOCK", "lock_key", False, 32),
("DESCRIPTION", "description", False, 60),
),
empty="No catalog resources found.",
)
return
if command == "gateway.list" and isinstance(value, Sequence):
_table(
[item for item in value if isinstance(item, Mapping)],
(
("SNAPSHOT", "snapshot_hash", True, None),
("STATUS", "status", True, 12),
("PROFILE", "profile_id", False, 30),
("PID", "pid", False, 8),
("ENDPOINT", "base_url", False, 36),
),
empty="No gateways found.",
)
return
if command == "session.list" and isinstance(value, Sequence):
_table(
[item for item in value if isinstance(item, Mapping)],
(
("SESSION", "session_id", True, None),
("STATUS", "status", True, 12),
("PROFILE", "profile_id", False, 30),
("KIND", "session_kind", False, 14),
("LAST ACTIVE (UTC)", "last_active_at", False, 26),
("RESUMABLE", "resumable", False, 9),
),
empty="No sessions found.",
)
return
if command == "session.runs" and isinstance(value, Sequence):
_table(
[item for item in value if isinstance(item, Mapping)],
(
("RUN", "run_id", True, None),
("STATUS", "status", True, 12),
("KIND", "kind", False, 12),
("CREATED (UTC)", "created_at", False, 26),
("EXIT", "exit_code", False, 6),
),
empty="No runs found for this session.",
)
return
if command in {"jobs.list", "jobs.status"} and isinstance(value, Sequence):
_table(
[item for item in value if isinstance(item, Mapping)],
(
("JOB", "job_id", True, None),
("STATUS", "status", True, 12),
("AGENT", "agent", False, 28),
("BACKEND", "backend", False, 12),
("MODEL", "model", False, 42),
("LAST PROGRESS (UTC)", "last_progress_at", False, 26),
),
empty="No jobs found.",
)
return
if command == "jobs.result" and isinstance(value, Mapping):
text = value.get("text", value.get("result", ""))
if text:
print(str(text), end="" if str(text).endswith("\n") else "\n")
else:
print("No result text is available.")
cursor = value.get("next_cursor")
if cursor is not None:
print(
f"More result data is available; rerun with --cursor {cursor}.",
file=sys.stderr,
)
return
if command == "eval.suites" and isinstance(value, Mapping):
_table(
_mapping_rows(value, key_name="id"),
(
("SUITE", "id", True, None),
("TASKS", "task_count", False, 7),
("SOURCE", "source", False, 8),
("NAME", "name", False, 36),
("DESCRIPTION", "description", False, 60),
),
empty="No evaluation suites found.",
)
return
if command == "eval.list" and isinstance(value, Sequence):
evaluation_rows: list[Mapping[str, Any]] = [
item for item in value if isinstance(item, Mapping)
]
_table(
evaluation_rows,
(
("RUN", "run_id", True, None),
("STATUS", "status", True, 12),
("PROFILE", "profile_id", False, 30),
("SUITE", "suite_id", False, 28),
("CREATED (UTC)", "created_at", False, 26),
),
empty="No evaluation runs found.",
)
return
if command in {
"doctor",
"profile.doctor",
"profile.validate",
"profile.smoke",
"validate",
"tool-mcp.validate",
"catalog.inventory",
"catalog.verify",
"catalog.discover",
"catalog.refresh",
"eval.validate",
"eval.run",
} and isinstance(value, Mapping):
_render_pass_report(value)
return
if isinstance(value, Mapping):
_render_nested(value)
return
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
_render_nested(value)
return
print(_display(value))
def emit_structured(command: str, value: Any, *, force_json: bool = False) -> None:
if force_json or not stdout_is_tty():
emit_json(value)
else:
render_human(command, value)
def emit_scalar(value: str, *, json_value: Any | None = None, force_json: bool = False) -> None:
if force_json:
emit_json(json_value if json_value is not None else {"value": value})
else:
print(value)
def emit_tty_success(
message: str, *, json_value: Any | None = None, force_json: bool = False
) -> None:
if force_json:
emit_json(json_value if json_value is not None else {"status": "ok"})
elif stdout_is_tty():
print(message)
libexec/mmo_codex_home.py
+1212
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_diagnostics.py
+977
View File
@@ -0,0 +1,977 @@
#!/usr/bin/env python3
"""Profile validation, environment diagnostics, and live smoke workflows."""
from __future__ import annotations
import contextlib
import json
import os
import select
import shutil
import subprocess
import sys
import textwrap
import time
from collections.abc import Callable
from pathlib import Path
from typing import Any
from mmo_app_server import _flat_mcp_dynamic_tool_name, app_server_protocol_status
from mmo_gateway import dry_run_gateway, ensure_gateway, gateway_models
from mmo_profiles import (
builtin_auth_link_mode,
load_settings,
profile_summary,
resolve_profile,
)
from mmo_runtime import (
cancel_job,
cancel_session,
create_session,
finish_session,
list_jobs,
mark_session_running,
run_root_exec,
spawn_job,
stop_session,
wait_for_jobs,
)
from mmo_snapshot import compile_profile
from mmo_state import TERMINAL_SESSION_STATUSES, root_mcp_capability_environment
from mmo_tool_mcp import (
load_tool_mcp_registry_with_sources,
tool_mcp_readiness,
tool_mcp_registry_root,
)
from mmo_util import (
config_root,
filtered_environment,
install_root,
package_version,
parse_env_file,
read_toml,
state_root,
strict_json_loads,
)
def _root_harness_prompt(prompt: str, execution_mode: str) -> str:
"""Make a bounded harness task terminal under the root's real lifecycle."""
prompt = prompt.strip()
if execution_mode != "goal":
return prompt
return (
prompt + "\n\nHarness lifecycle requirement: this root uses a durable Codex goal. "
"After every requested action and validation is complete, call `update_goal` "
'with `status="complete"` exactly once before the final assistant message. '
"Do not mark the goal complete while required work or descendant integration "
"remains; a final message alone does not terminate an active goal."
)
def tool_mcp_status(resolved: dict[str, Any] | None = None) -> dict[str, Any]:
registry, sources = load_tool_mcp_registry_with_sources()
selected = set(registry)
agents: dict[str, Any] = {}
required_by: dict[str, list[str]] = {server_id: [] for server_id in registry}
if resolved is not None:
selected = set(resolved.get("tool_mcp_servers", {}))
registry = {
server_id: server for server_id, server in resolved.get("tool_mcp_servers", {}).items()
}
for agent_id, agent in resolved["agents"].items():
grants = agent.get("tool_mcp_servers", {})
agents[agent_id] = grants
for server_id, grant in grants.items():
if grant["required"]:
required_by.setdefault(server_id, []).append(agent_id)
servers: dict[str, Any] = {}
for server_id in sorted(selected):
status = tool_mcp_readiness(server_id, registry[server_id], sources=sources)
status["required_by"] = sorted(required_by.get(server_id, []))
servers[server_id] = status
required = (
list(servers.values())
if resolved is None
else [item for item in servers.values() if item["required_by"]]
)
return {
"registry_root": str(tool_mcp_registry_root()),
"servers": servers,
"agents": agents,
"transport_ok": all(bool(item["transport_ready"]) for item in required),
"credentials_ok": all(bool(item["environment_ready"]) for item in required),
"passed": all(bool(item["ready"]) for item in required),
}
def profile_validation_report(profile: str, bindings: dict[str, str]) -> dict[str, Any]:
resolved = resolve_profile(profile, bindings=bindings)
snapshot = compile_profile(profile, bindings=bindings)
routes_valid = True
routes_error = None
routes_path = Path(snapshot["directory"]) / "routes.toml"
if routes_path.is_file():
try:
read_toml(routes_path)
except Exception as exc:
routes_valid = False
routes_error = str(exc)
return {
"valid": routes_valid,
"profile": profile_summary(profile, bindings=bindings),
"snapshot": snapshot["manifest"],
"generated_routes_toml_valid": routes_valid,
"generated_routes_error": routes_error,
"tool_mcp": tool_mcp_status(resolved),
"modalities": {
key: {
"requires": agent["requires_modalities"],
"model": resolved["models"][agent["model"]]["modalities"],
"transport": resolved["routes"][agent["route"]]["transport_modalities"],
}
for key, agent in resolved["agents"].items()
},
}
def _credentials_status(resolved: dict[str, Any]) -> dict[str, Any]:
values = parse_env_file(config_root() / "credentials.env")
values.update({key: value for key, value in os.environ.items() if value})
required: dict[str, bool] = {}
builtin_auth: dict[str, Any] = {}
settings = load_settings()
base_home = Path(str(settings.get("base_codex_home", "~/.codex"))).expanduser()
for key, route in resolved["routes"].items():
credential_envs = route.get("credential_envs", [])
if credential_envs:
label = "/".join(str(item) for item in credential_envs)
required[label] = any(values.get(str(item)) for item in credential_envs)
if route["driver"] == "codex_builtin" and route.get("auth") == "chatgpt":
mode = builtin_auth_link_mode(route, settings)
builtin_auth[key] = {
"base_codex_home": str(base_home),
"auth_json": (base_home / "auth.json").is_file(),
"auth_link_mode": mode,
"file_auth_transferable": mode in {"shared", "copy"}
and (base_home / "auth.json").is_file(),
"note": (
"Codex 0.149 keyring entries are scoped to canonical CODEX_HOME; "
"an isolated generated home requires file-backed auth.json."
),
}
return {"environment_credentials": required, "builtin_auth": builtin_auth}
def _codex_auth_status(binary: str, home: Path) -> dict[str, Any]:
command = [binary, "login", "status"]
try:
result = subprocess.run(
command,
env=filtered_environment(extra={"CODEX_HOME": str(home)}),
text=True,
capture_output=True,
timeout=30,
check=False,
)
except (OSError, subprocess.SubprocessError) as exc:
return {
"passed": False,
"command": command,
"error": f"{type(exc).__name__}: {exc}",
}
return {
"passed": result.returncode == 0,
"command": command,
"exit_code": result.returncode,
"stdout": result.stdout[-2000:],
"stderr": result.stderr[-2000:],
}
def _mcp_handshake(session: dict[str, Any]) -> dict[str, Any]:
env = filtered_environment(
extra={
"MMO_INSTALL_ROOT": str(install_root()),
"MMO_CONFIG_ROOT": str(config_root()),
"MMO_STATE_ROOT": str(state_root()),
**root_mcp_capability_environment(session),
"MMO_PROFILE_SNAPSHOT": session["snapshot_hash"],
"MMO_ALLOWED_ROOT": session["allowed_root"],
}
)
initialize = {
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2025-06-18",
"capabilities": {},
"clientInfo": {"name": "doctor", "version": package_version()},
},
}
process = subprocess.Popen(
[sys.executable, str(install_root() / "libexec" / "mmo_mcp.py")],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
env=env,
start_new_session=True,
)
responses: list[Any] = []
stderr = ""
handshake_error = None
try:
if process.stdin is None or process.stdout is None:
raise RuntimeError("MCP doctor probe did not receive its requested pipes")
process.stdin.write(json.dumps(initialize, allow_nan=False) + "\n")
process.stdin.flush()
readable, _writable, _exceptional = select.select([process.stdout], [], [], 15)
if not readable:
raise subprocess.TimeoutExpired(process.args, 15)
initialize_line = process.stdout.readline()
if not initialize_line:
raise RuntimeError("MCP server closed stdout before initialize response")
initialize_response = strict_json_loads(initialize_line)
responses.append(initialize_response)
initialize_result = (
initialize_response.get("result")
if isinstance(initialize_response, dict)
and initialize_response.get("jsonrpc") == "2.0"
and initialize_response.get("id") == 1
and "error" not in initialize_response
else None
)
if (
not isinstance(initialize_result, dict)
or initialize_result.get("protocolVersion") != "2025-06-18"
):
raise RuntimeError("MCP initialize response is invalid")
for message in (
{"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}},
{"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}},
):
process.stdin.write(json.dumps(message, allow_nan=False) + "\n")
process.stdin.flush()
process.stdin.close()
process.stdin = None
stdout, stderr = process.communicate(timeout=15)
for line in stdout.splitlines():
with contextlib.suppress(json.JSONDecodeError, ValueError):
responses.append(strict_json_loads(line))
except Exception as exc:
handshake_error = f"{type(exc).__name__}: {exc}"
with contextlib.suppress(OSError):
if process.stdin is not None:
process.stdin.close()
process.stdin = None
if process.poll() is None:
process.kill()
with contextlib.suppress(subprocess.SubprocessError, OSError):
_stdout, stderr = process.communicate(timeout=2)
tools = []
for response in responses:
if not isinstance(response, dict):
continue
if response.get("id") == 2:
result_value = response.get("result", {})
if isinstance(result_value, dict):
tools = [
item["name"]
for item in result_value.get("tools", [])
if isinstance(item, dict) and isinstance(item.get("name"), str)
]
return {
"passed": handshake_error is None and process.returncode == 0 and "agent_status" in tools,
"exit_code": process.returncode,
"tools": tools,
"initialize": responses[0] if responses else None,
"error": handshake_error,
"stderr": stderr[-4000:],
}
def doctor(
profile: str,
*,
live: bool,
probe: bool,
bindings: dict[str, str],
progress: Callable[[str], None] | None = None,
) -> dict[str, Any]:
if probe and not live:
raise ValueError("--probe requires --live because it performs a live model request")
if progress:
progress(f"Resolving and validating profile {profile}...")
resolved = resolve_profile(profile, bindings=bindings)
validation = profile_validation_report(profile, bindings)
checks: dict[str, Any] = {
"package_version": package_version(),
"python": {"version": sys.version.split()[0], "supported": sys.version_info >= (3, 11)},
"profile": validation,
"paths": {
"install_root": str(install_root()),
"config_root": str(config_root()),
"state_root": str(state_root()),
},
"binaries": {
"codex": shutil.which(
os.environ.get("MMO_CODEX_BIN") or str(load_settings().get("codex_bin", "codex"))
),
"switchyard-server": shutil.which(
str(load_settings().get("switchyard_bin", "switchyard-server"))
),
"git": shutil.which("git"),
},
"credentials": _credentials_status(resolved),
"tool_mcp": tool_mcp_status(resolved),
"live": None,
}
checks["app_server_protocol"] = app_server_protocol_status(checks["binaries"]["codex"])
required_env = checks["credentials"]["environment_credentials"]
builtin_auth_required = bool(checks["credentials"]["builtin_auth"])
if builtin_auth_required:
codex_binary = checks["binaries"]["codex"]
base_home = Path(str(load_settings().get("base_codex_home", "~/.codex"))).expanduser()
checks["credentials"]["codex_login_status"] = (
_codex_auth_status(str(codex_binary), base_home)
if codex_binary
else {"passed": False, "error": "Codex binary not found"}
)
login_ok = bool(checks["credentials"]["codex_login_status"].get("passed"))
for auth in checks["credentials"]["builtin_auth"].values():
transferable = bool(auth["file_auth_transferable"])
auth["usable_by_generated_home"] = login_ok and transferable
if not login_ok:
auth["reason"] = "the configured base Codex home is not logged in"
elif auth["auth_link_mode"] == "none":
auth["reason"] = "settings disable authentication propagation"
elif not auth["auth_json"]:
auth["reason"] = (
"login is keyring-only; configure file-backed Codex auth for isolated homes"
)
else:
auth["reason"] = None
if progress:
progress(f"Compiling diagnostic snapshot for {profile}...")
snapshot = compile_profile(profile, bindings=bindings)
gateway_required = bool(snapshot["manifest"]["gateway_required"])
offline_ok = (
checks["python"]["supported"]
and validation["valid"]
and bool(checks["binaries"]["git"])
and bool(checks["binaries"]["codex"])
and bool(checks["app_server_protocol"]["passed"])
and (not gateway_required or bool(checks["binaries"]["switchyard-server"]))
and bool(checks["tool_mcp"]["transport_ok"])
)
checks["offline_ok"] = offline_ok
checks["credentials_ok"] = (
all(required_env.values())
and (
not builtin_auth_required
or all(
bool(auth.get("usable_by_generated_home"))
for auth in checks["credentials"]["builtin_auth"].values()
)
)
and bool(checks["tool_mcp"]["credentials_ok"])
)
if live:
live_result: dict[str, Any] = {}
if snapshot["manifest"]["gateway_required"]:
if progress:
progress("Checking the live Switchyard gateway and advertised routes...")
try:
dry = dry_run_gateway(snapshot["manifest"]["snapshot_hash"])
live_result["switchyard_dry_run"] = {
"passed": dry is not None and dry.returncode == 0,
"exit_code": dry.returncode if dry else None,
"stdout": dry.stdout[-4000:] if dry else "",
"stderr": dry.stderr[-4000:] if dry else "",
}
gateway = ensure_gateway(snapshot["manifest"]["snapshot_hash"])
live_result["gateway"] = gateway
models = gateway_models(snapshot["manifest"]["snapshot_hash"])
live_result["gateway_models"] = models
advertised = (
{
item["id"]
for item in models.get("data", [])
if isinstance(item, dict) and isinstance(item.get("id"), str)
}
if isinstance(models, dict)
else set()
)
expected = set(snapshot["manifest"]["route_ids"].values())
live_result["routes_advertised"] = {
"passed": expected == advertised,
"expected": sorted(expected),
"advertised": sorted(advertised),
"missing": sorted(expected - advertised),
"unexpected": sorted(advertised - expected),
}
except Exception as exc:
live_result["gateway_error"] = f"{type(exc).__name__}: {exc}"
if resolved["capabilities"]["mcp_agents"]:
if progress:
progress("Checking the internal Agent MCP handshake...")
session = None
try:
session = create_session(profile=profile, cwd=os.getcwd(), bindings=bindings)
mark_session_running(
session["session_id"],
os.getpid(),
expected_run_id=str(session["current_run_id"]),
)
live_result["mcp"] = _mcp_handshake(session)
except Exception as exc:
live_result["mcp_error"] = f"{type(exc).__name__}: {exc}"
finally:
if session is not None:
with contextlib.suppress(Exception):
finish_session(
session["session_id"],
exit_code=0 if "mcp_error" not in live_result else 1,
error=live_result.get("mcp_error"),
expected_run_id=str(session["current_run_id"]),
)
else:
live_result["mcp"] = {
"passed": True,
"skipped": True,
"reason": "profile has no MCP participants",
}
if probe:
if progress:
progress("Sending the live root-model probe...")
try:
root = resolved["agents"][resolved["profile"]["root"]]
result = run_root_exec(
profile=profile,
cwd=os.getcwd(),
prompt=_root_harness_prompt(
"Return exactly MMO_ROOT_OK and nothing else as the final assistant "
"message. Do not spawn agents or call tools other than the required "
"goal-lifecycle update.",
root["execution_mode"],
),
bindings=bindings,
wall_timeout_seconds=300,
sandbox_mode="read-only",
label="doctor-root-probe",
)
live_result["root_model_probe"] = {
"passed": result.get("exit_code") == 0
and result.get("result", "").strip() == "MMO_ROOT_OK",
"result": result,
}
if result.get("status") not in TERMINAL_SESSION_STATUSES:
live_result["root_model_probe"]["cleanup"] = _cleanup_detached_harness_session(
result["session"]["session_id"]
)
except Exception as exc:
live_result["root_model_probe"] = {
"passed": False,
"error": f"{type(exc).__name__}: {exc}",
}
failed_session_id = getattr(exc, "mmo_session_id", None)
if (
isinstance(failed_session_id, str)
and getattr(exc, "mmo_session_status", None) not in TERMINAL_SESSION_STATUSES
):
try:
live_result["root_model_probe"]["cleanup"] = (
_cleanup_detached_harness_session(failed_session_id)
)
except Exception as cleanup_exc:
live_result["root_model_probe"]["cleanup_error"] = (
f"{type(cleanup_exc).__name__}: {cleanup_exc}"
)
checks["live"] = live_result
live_checks = [
value.get("passed")
for value in live_result.values()
if isinstance(value, dict) and "passed" in value
]
checks["live_ok"] = not any(key.endswith("_error") for key in live_result) and all(
live_checks
)
checks["passed"] = (
bool(checks["offline_ok"])
and bool(checks["credentials_ok"])
and (not live or bool(checks.get("live_ok")))
)
return checks
def _smoke_backend(task: dict[str, Any], agent: dict[str, Any]) -> str:
"""Select the declared execution backend for a smoke task.
Ambiguous hybrid tasks default to MCP because that path has enforceable
scope, resource, lineage, and result-contract semantics. Profiles that need
to exercise Codex native subagents must say ``backend = "native"``.
"""
explicit = task.get("backend")
if explicit:
return str(explicit)
backends = list(agent.get("backends", []))
if len(backends) == 1:
return str(backends[0])
if "mcp" in backends:
return "mcp"
if "native" in backends:
return "native"
raise ValueError(f"agent {task.get('agent')!r} has no executable smoke backend")
def _native_smoke_prompt(
*,
agent_id: str,
native_name: str,
task_kind: str,
task: str,
) -> str:
return textwrap.dedent(
f"""
This is an automated Codex native-subagent acceptance test.
You MUST delegate the work below through Codex's native subagent tool to
the configured custom role named `{native_name}` (profile agent
`{agent_id}`). Do not call the `mmo_mesh` Agent MCP server for this task,
and do not perform the delegated investigation yourself.
Task kind: {task_kind}
Delegated task:
{task}
Wait for that native subagent to complete, inspect its returned result,
and then summarize it. If and only if the named native subagent was
actually used and returned successfully, include this exact line at the
end of your answer:
MMO_NATIVE_SMOKE_OK
"""
).strip()
def _dynamic_tool_aliases(required_tools: list[str]) -> dict[str, str]:
"""Map every valid server/tool split to one required smoke-tool name.
Tool MCP server IDs may contain dots, as may tool names. Profile validation
has already proved that exactly one split is granted to the smoke role. The
compatibility bridge exposes the selected split as a flat dynamic function;
accepting every syntactic split here lets the evidence reader recognize that
function without weakening the earlier grant/ambiguity validation.
"""
aliases: dict[str, str] = {}
for qualified in required_tools:
for index, character in enumerate(qualified):
if character != ".":
continue
server = qualified[:index]
tool = qualified[index + 1 :]
if server and tool:
aliases[_flat_mcp_dynamic_tool_name(server, tool)] = qualified
return aliases
def _successful_mcp_tools(
events_path: str | Path | None,
*,
dynamic_tool_aliases: dict[str, str] | None = None,
) -> set[str]:
"""Return MCP tools whose Codex event reached a successful terminal state."""
if not events_path:
return set()
path = Path(events_path)
if not path.is_file():
return set()
successful: set[str] = set()
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
pending: list[Any] = [event]
while pending:
value = pending.pop()
if isinstance(value, dict):
item_type = value.get("type")
if (
isinstance(item_type, str)
and item_type in {"mcp_tool_call", "mcpToolCall"}
and value.get("status") == "completed"
and value.get("error") is None
):
server = value.get("server") or value.get("serverName")
tool = value.get("tool") or value.get("toolName")
if isinstance(server, str) and server and isinstance(tool, str) and tool:
successful.add(f"{server}.{tool}")
elif (
item_type == "dynamicToolCall"
and value.get("status") == "completed"
and value.get("success") is True
):
tool = value.get("tool")
if isinstance(tool, str) and dynamic_tool_aliases:
qualified = dynamic_tool_aliases.get(tool)
if qualified is not None:
successful.add(qualified)
pending.extend(value.values())
elif isinstance(value, list):
pending.extend(value)
return successful
def _smoke_tool_evidence(
task: dict[str, Any], events_path: str | Path | None
) -> tuple[list[str], list[str], list[str]]:
required = list(task.get("required_mcp_tools", []))
observed = sorted(
_successful_mcp_tools(
events_path,
dynamic_tool_aliases=_dynamic_tool_aliases(required),
)
)
missing = sorted(set(required) - set(observed))
return required, observed, missing
def _wait_for_smoke_job(
job_id: str,
*,
session_id: str,
wall_timeout_seconds: int,
wait_seconds: int,
) -> dict[str, Any]:
"""Wait in MCP-sized slices without turning the harness wall into a role timeout."""
started = time.monotonic()
deadline = started + wall_timeout_seconds
waited: dict[str, Any] | None = None
while True:
remaining = deadline - time.monotonic()
timeout = (
0
if wait_seconds == 0 or remaining <= 0
else min(120, wait_seconds, max(1, int(remaining + 0.999)))
)
waited = wait_for_jobs(
[job_id],
session_id=session_id,
timeout_seconds=timeout,
include_results=True,
)
if not waited["unfinished"] or wait_seconds == 0 or time.monotonic() >= deadline:
break
waited["harness_wall_timeout_seconds"] = wall_timeout_seconds
waited["harness_wall_exhausted"] = bool(waited["unfinished"])
waited["harness_elapsed_seconds"] = time.monotonic() - started
return waited
def _cleanup_detached_harness_session(session_id: str) -> dict[str, Any]:
"""Guarantee that a smoke harness leaves no recoverable run mutating its fixture."""
try:
return {"mode": "graceful_stop", "result": stop_session(session_id, grace_seconds=0)}
except Exception as stop_error:
try:
return {"mode": "immediate_cancel", "result": cancel_session(session_id)}
except Exception as cancel_error:
raise RuntimeError(
"smoke harness could not retire detached session: "
f"{type(stop_error).__name__}: {stop_error}; "
f"{type(cancel_error).__name__}: {cancel_error}"
) from cancel_error
def smoke_profile(
profile: str,
*,
cwd: str,
bindings: dict[str, str],
root_only: bool,
workers_only: bool,
progress: Callable[[str], None] | None = None,
) -> dict[str, Any]:
if progress:
progress(f"Resolving smoke tasks for profile {profile}...")
resolved = resolve_profile(profile, bindings=bindings)
smoke = resolved.get("smoke") or {"tasks": []}
root_id = resolved["profile"]["root"]
tasks = smoke.get("tasks", [])
results: list[dict[str, Any]] = []
root_tasks = [task for task in tasks if task["agent"] == root_id]
worker_tasks = [task for task in tasks if task["agent"] != root_id]
root_execution_mode = resolved["agents"][root_id]["execution_mode"]
if not workers_only:
for task in root_tasks:
if progress:
progress(f"Running root smoke task for {task['agent']}...")
try:
result = run_root_exec(
profile=profile,
cwd=cwd,
prompt=_root_harness_prompt(task["task"], root_execution_mode),
bindings=bindings,
wall_timeout_seconds=int(task.get("wall_timeout_seconds", 600)),
sandbox_mode=task.get("mode", "read-only"),
label=f"smoke-{task['agent']}",
)
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(
task, result.get("events_path")
)
results.append(
{
"agent": task["agent"],
"backend": "root",
"passed": result["exit_code"] == 0 and not missing_tools,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"result": result,
}
)
if result.get("status") not in TERMINAL_SESSION_STATUSES:
try:
results[-1]["cleanup"] = _cleanup_detached_harness_session(
result["session"]["session_id"]
)
except Exception as cleanup_exc:
results[-1]["passed"] = False
results[-1]["cleanup_error"] = (
f"{type(cleanup_exc).__name__}: {cleanup_exc}"
)
except Exception as exc:
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(task, None)
failure = {
"agent": task["agent"],
"backend": "root",
"passed": False,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"error": f"{type(exc).__name__}: {exc}",
}
failed_session_id = getattr(exc, "mmo_session_id", None)
if (
isinstance(failed_session_id, str)
and getattr(exc, "mmo_session_status", None) not in TERMINAL_SESSION_STATUSES
):
try:
failure["cleanup"] = _cleanup_detached_harness_session(failed_session_id)
except Exception as cleanup_exc:
failure["cleanup_error"] = f"{type(cleanup_exc).__name__}: {cleanup_exc}"
results.append(failure)
if not root_only:
native_tasks: list[dict[str, Any]] = []
mcp_tasks: list[dict[str, Any]] = []
for task in worker_tasks:
backend = _smoke_backend(task, resolved["agents"][task["agent"]])
if backend == "native":
native_tasks.append(task)
elif backend == "mcp":
mcp_tasks.append(task)
else: # profile validation should make this unreachable
results.append(
{
"agent": task["agent"],
"backend": backend,
"passed": False,
"error": f"unsupported smoke backend: {backend}",
}
)
# Native subagents are owned by a Codex root thread, so the live smoke
# test must exercise the actual root -> native-agent path. A successful
# direct model call would not prove that the custom role is discoverable
# or that Codex can spawn it.
for task in native_tasks:
if progress:
progress(f"Running native-agent smoke task for {task['agent']}...")
agent = resolved["agents"][task["agent"]]
try:
result = run_root_exec(
profile=profile,
cwd=cwd,
prompt=_root_harness_prompt(
_native_smoke_prompt(
agent_id=task["agent"],
native_name=agent["native_name"],
task_kind=task["task_kind"],
task=task["task"],
),
root_execution_mode,
),
bindings=bindings,
wall_timeout_seconds=int(task.get("wall_timeout_seconds", 900)),
sandbox_mode=task.get("mode", "read-only"),
label=f"smoke-native-{task['agent']}",
)
marker_present = "MMO_NATIVE_SMOKE_OK" in result.get("result", "")
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(
task, result.get("events_path")
)
results.append(
{
"agent": task["agent"],
"native_name": agent["native_name"],
"backend": "native",
"passed": (
result["exit_code"] == 0 and marker_present and not missing_tools
),
"marker_present": marker_present,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"result": result,
}
)
if result.get("status") not in TERMINAL_SESSION_STATUSES:
try:
results[-1]["cleanup"] = _cleanup_detached_harness_session(
result["session"]["session_id"]
)
except Exception as cleanup_exc:
results[-1]["passed"] = False
results[-1]["cleanup_error"] = (
f"{type(cleanup_exc).__name__}: {cleanup_exc}"
)
except Exception as exc:
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(task, None)
failure = {
"agent": task["agent"],
"native_name": agent["native_name"],
"backend": "native",
"passed": False,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"error": f"{type(exc).__name__}: {exc}",
}
failed_session_id = getattr(exc, "mmo_session_id", None)
if (
isinstance(failed_session_id, str)
and getattr(exc, "mmo_session_status", None) not in TERMINAL_SESSION_STATUSES
):
try:
failure["cleanup"] = _cleanup_detached_harness_session(failed_session_id)
except Exception as cleanup_exc:
failure["cleanup_error"] = f"{type(cleanup_exc).__name__}: {cleanup_exc}"
results.append(failure)
# Agent MCP tasks share one execution run so active capacity, scope
# leases, resource groups, and result contracts are tested against
# the same durable supervisor state.
if mcp_tasks:
session = None
try:
session = create_session(profile=profile, cwd=cwd, bindings=bindings)
mark_session_running(
session["session_id"],
os.getpid(),
expected_run_id=str(session["current_run_id"]),
)
for task in mcp_tasks:
if progress:
progress(f"Running Agent MCP smoke task for {task['agent']}...")
try:
job = spawn_job(
session_id=session["session_id"],
caller_agent=root_id,
caller_job_id=None,
agent_id=task["agent"],
task_kind=task.get("task_kind"),
task=task.get("task"),
literal_task=task.get("literal_task"),
mode=task.get("mode", "read-only"),
write_scope_values=task.get("write_scope", []),
attachments=task.get("attachments", []),
label=f"smoke-{task['agent']}",
)
waited = _wait_for_smoke_job(
job["job_id"],
session_id=session["session_id"],
wall_timeout_seconds=int(task.get("wall_timeout_seconds", 900)),
wait_seconds=int(task.get("wait_seconds", 120)),
)
if waited["unfinished"]:
waited["cleanup"] = cancel_job(
job["job_id"],
session_id=session["session_id"],
cascade=True,
reason="smoke harness wall limit expired",
)
waited["cleanup_wait"] = wait_for_jobs(
[job["job_id"]],
session_id=session["session_id"],
timeout_seconds=30,
include_results=True,
)
final = load_job_public(job["job_id"])
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(
task, final.get("events_path")
)
passed = (
final["status"] in {"completed", "completed_with_warnings"}
and not missing_tools
)
results.append(
{
"agent": task["agent"],
"backend": "mcp",
"passed": passed,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"job": final,
"wait": waited,
}
)
except Exception as exc:
required_tools, observed_tools, missing_tools = _smoke_tool_evidence(
task, None
)
results.append(
{
"agent": task["agent"],
"backend": "mcp",
"passed": False,
"required_mcp_tools": required_tools,
"observed_mcp_tools": observed_tools,
"missing_mcp_tools": missing_tools,
"error": f"{type(exc).__name__}: {exc}",
}
)
finally:
if session:
finished = finish_session(
session["session_id"],
exit_code=0 if all(item["passed"] for item in results) else 1,
expected_run_id=str(session["current_run_id"]),
)
if finished.get("status") not in TERMINAL_SESSION_STATUSES:
cleanup = _cleanup_detached_harness_session(session["session_id"])
for item in results:
if item.get("backend") == "mcp":
item.setdefault("session_cleanup", cleanup)
return {
"profile": resolved["profile"]["id"],
"passed": bool(results) and all(item["passed"] for item in results),
"results": results,
}
def load_job_public(job_id: str) -> dict[str, Any]:
for item in list_jobs(job_ids=[job_id], limit=1):
return item
raise FileNotFoundError(job_id)
libexec/mmo_eval.py
+2238
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_gateway.py
Executable
+838
View File
@@ -0,0 +1,838 @@
#!/usr/bin/env python3
"""Lifecycle manager for route-set-addressed Switchyard gateway processes."""
from __future__ import annotations
import contextlib
import datetime as dt
import json
import os
import re
import shutil
import socket
import subprocess
import time
from collections.abc import Mapping
from ipaddress import ip_address
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from mmo_profiles import load_settings
from mmo_snapshot import load_snapshot
from mmo_state import (
ACTIVE_SESSION_STATUSES,
RECOVERABLE_JOB_STATUSES,
iter_job_records,
iter_session_records,
)
from mmo_util import (
atomic_write_json,
config_root,
file_lock,
filtered_environment,
http_json,
http_ready,
parse_env_file,
port_available,
process_matches,
process_start_token,
read_json,
state_root,
strict_json_loads,
terminate_process_group,
utc_now,
)
from mmo_version import MMO_SCHEMA_VERSION
_GATEWAY_PROCESSES: dict[int, subprocess.Popen[bytes]] = {}
_GATEWAY_ADMISSION_GRACE_SECONDS = 60.0
def _reap_gateway(pid: int | None) -> None:
"""Reap terminal gateway children without dropping live handles."""
if not pid:
return
process = _GATEWAY_PROCESSES.get(int(pid))
if process is None:
return
try:
process.wait(timeout=1.0)
except subprocess.TimeoutExpired:
return
except OSError:
pass
_GATEWAY_PROCESSES.pop(int(pid), None)
def gateways_root() -> Path:
root = state_root() / "gateways"
root.mkdir(parents=True, exist_ok=True, mode=0o700)
return root
def _validate_hash(value: str, label: str) -> str:
if len(value) != 64 or any(char not in "0123456789abcdef" for char in value):
raise ValueError(f"invalid {label}")
return value
def _gateway_key(snapshot_hash: str) -> str:
snapshot = load_snapshot(_validate_hash(snapshot_hash, "snapshot hash"))
return _validate_hash(
str(snapshot["manifest"].get("gateway_hash") or snapshot_hash),
"gateway hash",
)
def _gateway_dir_for_key(gateway_hash: str) -> Path:
path = gateways_root() / _validate_hash(gateway_hash, "gateway hash")
path.mkdir(parents=True, exist_ok=True, mode=0o700)
return path
def gateway_dir(snapshot_hash: str) -> Path:
"""Return the route-set gateway directory for a profile snapshot."""
return _gateway_dir_for_key(_gateway_key(snapshot_hash))
def _gateway_state_path_for_key(gateway_hash: str) -> Path:
return _gateway_dir_for_key(gateway_hash) / "gateway.json"
def gateway_state_path(snapshot_hash: str) -> Path:
return _gateway_state_path_for_key(_gateway_key(snapshot_hash))
def _binary(settings: Mapping[str, Any]) -> str:
configured = str(settings.get("switchyard_bin", "switchyard-server"))
resolved = shutil.which(configured)
if resolved:
return resolved
path = Path(configured).expanduser()
if path.is_file() and os.access(path, os.X_OK):
return str(path.resolve())
raise FileNotFoundError(f"Switchyard binary not found: {configured}")
def switchyard_version(binary: str | None = None) -> str:
"""Return the exact release reported by the configured gateway binary."""
resolved = binary or _binary(load_settings())
try:
result = subprocess.run(
[resolved, "--version"],
capture_output=True,
text=True,
check=False,
timeout=10,
env=filtered_environment(),
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise RuntimeError(f"unable to inspect Switchyard version at {resolved}: {exc}") from exc
output = (result.stdout + "\n" + result.stderr).strip()
match = re.fullmatch(r"switchyard-server ([0-9]+\.[0-9]+\.[0-9]+)", output)
if result.returncode != 0 or match is None:
raise RuntimeError(
"Switchyard did not report a supported semantic version: "
f"status={result.returncode}, output={output[-1000:]!r}"
)
return match.group(1)
def route_availability(snapshot: Mapping[str, Any]) -> dict[str, dict[str, Any]]:
"""Resolve current per-route availability without making workers startup-critical."""
credentials_path = config_root() / "credentials.env"
file_values = parse_env_file(credentials_path)
values = dict(file_values)
values.update({key: value for key, value in os.environ.items() if value})
result: dict[str, dict[str, Any]] = {}
for route_key, route in sorted(snapshot["resolved"]["routes"].items()):
credentials = list(route.get("credential_envs", []))
selected = next((name for name in credentials if values.get(name)), None)
available = selected is not None or not credentials
reason = None if available else "missing credential: " + "/".join(credentials)
if available and route.get("billing_mode") == "local" and route.get("base_url"):
parsed = urlsplit(str(route["base_url"]))
port = parsed.port or (443 if parsed.scheme == "https" else 80)
try:
with socket.create_connection((str(parsed.hostname), port), timeout=0.2):
pass
except OSError:
available = False
reason = f"local endpoint unavailable: {parsed.hostname}:{port}"
result[route_key] = {
"available": available,
"selected_credential_env": selected,
"reason": reason,
}
return result
def _credentials(snapshot: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]:
credentials_path = config_root() / "credentials.env"
file_values = parse_env_file(credentials_path)
values = dict(file_values)
values.update({key: value for key, value in os.environ.items() if value})
root_agent = snapshot["resolved"]["agents"][snapshot["manifest"]["root_agent"]]
root_route = str(root_agent["route"])
groups = snapshot["manifest"].get("credential_groups", [])
resolved: dict[str, str] = {}
missing: list[str] = []
for group in groups:
alternatives = list(group.get("alternatives") or [group.get("target_env")])
source = next((name for name in alternatives if name and values.get(name)), None)
target = str(group.get("target_env") or alternatives[0])
if source is None:
route_key = str(group.get("route"))
if route_key == root_route:
missing.append("/".join(name for name in alternatives if name))
else:
# Switchyard resolves client environment keys when parsing its
# immutable route set. The supervisor prevents this sentinel
# client from receiving work through the availability overlay.
resolved[target] = f"mmo-unavailable-{route_key}"
continue
resolved[target] = values[source]
return resolved, missing
def _select_port(snapshot_hash: str, settings: Mapping[str, Any]) -> int:
minimum = int(settings.get("gateway_port_min", 42000))
maximum = int(settings.get("gateway_port_max", 51999))
if not 1024 <= minimum <= maximum <= 65535:
raise ValueError("invalid gateway port range in settings")
span = maximum - minimum + 1
start = int(snapshot_hash[:16], 16) % span
host = str(settings.get("gateway_host", "127.0.0.1"))
for offset in range(span):
port = minimum + ((start + offset) % span)
if port_available(host, port):
return port
raise RuntimeError("no free port is available in the configured gateway range")
def _health_url(host: str, port: int) -> str:
address = f"[{host}]" if ":" in host and not host.startswith("[") else host
return f"http://{address}:{port}/health"
def _base_url(host: str, port: int) -> str:
address = f"[{host}]" if ":" in host and not host.startswith("[") else host
return f"http://{address}:{port}/v1"
def _timestamp(value: Any, label: str) -> float:
if not isinstance(value, str) or not value:
raise ValueError(f"invalid {label}")
try:
parsed = dt.datetime.fromisoformat(value)
except ValueError as exc:
raise ValueError(f"invalid {label}") from exc
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=dt.UTC)
return parsed.timestamp()
def _read_gateway_state_by_key(gateway_hash: str) -> dict[str, Any] | None:
path = _gateway_state_path_for_key(gateway_hash)
if not path.is_file():
return None
try:
state = read_json(path)
except (OSError, ValueError) as exc:
return {
"gateway_hash": gateway_hash,
"status": "invalid",
"error": f"invalid gateway state: {type(exc).__name__}: {exc}",
}
if not isinstance(state, dict):
return {
"gateway_hash": gateway_hash,
"status": "invalid",
"error": "invalid gateway state: root must be an object",
}
schema_version = state.get("schema_version")
observed_switchyard_version = state.get("switchyard_version")
snapshot_hashes = state.get("snapshot_hashes", [])
profile_ids = state.get("profile_ids", [])
host = state.get("host")
port = state.get("port")
endpoint_identity_valid = False
if (
isinstance(host, str)
and isinstance(port, int)
and not isinstance(port, bool)
and 1 <= port <= 65535
):
with contextlib.suppress(ValueError):
endpoint_identity_valid = (
ip_address(host).is_loopback
and state.get("base_url") == _base_url(host, port)
and state.get("health_url") == _health_url(host, port)
)
if (
not isinstance(schema_version, int)
or isinstance(schema_version, bool)
or schema_version != MMO_SCHEMA_VERSION
or state.get("gateway_hash") != gateway_hash
or not isinstance(observed_switchyard_version, str)
or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", observed_switchyard_version) is None
or not isinstance(state.get("pid"), int)
or isinstance(state.get("pid"), bool)
or not isinstance(state.get("process_start_token"), str)
or not isinstance(snapshot_hashes, list)
or not all(isinstance(value, str) for value in snapshot_hashes)
or not isinstance(profile_ids, list)
or not all(isinstance(value, str) for value in profile_ids)
or not endpoint_identity_valid
):
return {
"gateway_hash": gateway_hash,
"status": "invalid",
"error": "invalid gateway state identity or process metadata",
}
try:
_timestamp(state.get("last_used_at"), "gateway last-used timestamp")
except ValueError:
return {
"gateway_hash": gateway_hash,
"status": "invalid",
"error": "invalid gateway state last-used timestamp",
}
pid = state.get("pid")
if not process_matches(pid, state.get("process_start_token")):
state["status"] = "stopped"
return state
health = state.get("health_url")
state["status"] = "running" if isinstance(health, str) and http_ready(health) else "starting"
return state
def read_gateway_state(snapshot_hash: str) -> dict[str, Any] | None:
return _read_gateway_state_by_key(_gateway_key(snapshot_hash))
def _record_gateway_lease(
state: dict[str, Any], snapshot: Mapping[str, Any], gateway_hash: str
) -> dict[str, Any]:
snapshot_hash = str(snapshot["manifest"]["snapshot_hash"])
profile_id = str(snapshot["manifest"]["profile_id"])
state["gateway_hash"] = gateway_hash
state["snapshot_hashes"] = sorted(set(state.get("snapshot_hashes", [])) | {snapshot_hash})
state["profile_ids"] = sorted(set(state.get("profile_ids", [])) | {profile_id})
state["last_used_at"] = utc_now()
atomic_write_json(_gateway_state_path_for_key(gateway_hash), state)
return state
def ensure_gateway(snapshot_hash: str) -> dict[str, Any] | None:
snapshot = load_snapshot(snapshot_hash)
if not snapshot["manifest"].get("gateway_required"):
return None
gateway_hash = _validate_hash(
str(snapshot["manifest"].get("gateway_hash") or snapshot_hash),
"gateway hash",
)
settings = load_settings()
binary = _binary(settings)
observed_switchyard_version = switchyard_version(binary)
resolved_availability = route_availability(snapshot)
lock = gateways_root() / ".gateway.lock"
with file_lock(lock):
current = _read_gateway_state_by_key(gateway_hash)
if current and current.get("status") == "invalid":
raise RuntimeError(str(current.get("error")))
if (
current
and current.get("status") == "running"
and current.get("switchyard_version") == observed_switchyard_version
and current.get("route_availability") == resolved_availability
):
return _record_gateway_lease(current, snapshot, gateway_hash)
if current and process_matches(current.get("pid"), current.get("process_start_token")):
terminate_process_group(int(current["pid"]))
_reap_gateway(current.get("pid"))
credentials, missing = _credentials(snapshot)
if missing:
raise RuntimeError(
"missing root-route credentials for this profile: " + ", ".join(sorted(missing))
)
host = str(settings.get("gateway_host", "127.0.0.1"))
port = _select_port(gateway_hash, settings)
directory = _gateway_dir_for_key(gateway_hash)
log_path = directory / "gateway.log"
routing_log_path = directory / "routing.jsonl"
routes_path = Path(snapshot["directory"]) / "routes.toml"
command = [
binary,
"--config",
str(routes_path),
"--host",
host,
"--port",
str(port),
"--routing-log-file",
str(routing_log_path),
]
environment = filtered_environment(
allow_sensitive=credentials,
extra={
**credentials,
"RUST_LOG": os.environ.get("RUST_LOG", "switchyard_server=info,libsy=info"),
},
)
log_handle = log_path.open("ab", buffering=0)
try:
process = subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
stdout=log_handle,
stderr=subprocess.STDOUT,
close_fds=True,
start_new_session=True,
env=environment,
cwd=directory,
)
finally:
log_handle.close()
_GATEWAY_PROCESSES[process.pid] = process
ready = False
try:
start_token = process_start_token(process.pid)
if start_token is None:
initial_returncode = process.poll()
tail = ""
with contextlib.suppress(OSError):
tail = log_path.read_text(encoding="utf-8", errors="replace")[-4000:]
error = (
f"Switchyard exited during startup with status {initial_returncode}"
if initial_returncode is not None
else "unable to fingerprint the Switchyard process"
)
raise RuntimeError(error + (f"\n{tail}" if tail else ""))
state = {
"schema_version": MMO_SCHEMA_VERSION,
"gateway_hash": gateway_hash,
"snapshot_hash": snapshot_hash,
"snapshot_hashes": [snapshot_hash],
"profile_id": snapshot["manifest"]["profile_id"],
"profile_ids": [snapshot["manifest"]["profile_id"]],
"switchyard_version": observed_switchyard_version,
"pid": process.pid,
"process_start_token": start_token,
"host": host,
"port": port,
"base_url": _base_url(host, port),
"health_url": _health_url(host, port),
"routes_path": str(routes_path),
"log_path": str(log_path),
"routing_log_path": str(routing_log_path),
"command": command,
"route_availability": resolved_availability,
"started_at": utc_now(),
"last_used_at": utc_now(),
"status": "starting",
}
atomic_write_json(_gateway_state_path_for_key(gateway_hash), state)
timeout = float(settings.get("gateway_start_timeout_seconds", 15))
deadline = time.monotonic() + max(1.0, timeout)
while time.monotonic() < deadline:
if process.poll() is not None:
break
if http_ready(state["health_url"], timeout=0.4):
state["status"] = "running"
state["ready_at"] = utc_now()
atomic_write_json(_gateway_state_path_for_key(gateway_hash), state)
ready = True
return state
time.sleep(0.1)
tail = ""
with contextlib.suppress(OSError):
tail = log_path.read_text(encoding="utf-8", errors="replace")[-4000:]
state["status"] = "failed"
state["finished_at"] = utc_now()
state["error"] = "Switchyard did not become healthy before the startup deadline"
atomic_write_json(_gateway_state_path_for_key(gateway_hash), state)
raise RuntimeError(state["error"] + (f"\n{tail}" if tail else ""))
finally:
if not ready:
terminate_process_group(process.pid)
_reap_gateway(process.pid)
def stop_gateway(snapshot_hash: str) -> dict[str, Any]:
gateway_hash = _gateway_key(snapshot_hash)
lock = gateways_root() / ".gateway.lock"
with file_lock(lock):
state = _read_gateway_state_by_key(gateway_hash)
if not state:
return {"snapshot_hash": snapshot_hash, "status": "not_found"}
if state.get("status") == "invalid":
raise RuntimeError(str(state.get("error")))
pid = state.get("pid")
if process_matches(pid, state.get("process_start_token")):
if not isinstance(pid, int) or isinstance(pid, bool):
raise RuntimeError("gateway state contains an invalid process id")
terminate_process_group(int(pid))
_reap_gateway(pid)
state["status"] = "stopped"
state["stopped_at"] = utc_now()
atomic_write_json(_gateway_state_path_for_key(gateway_hash), state)
return state
def gateway_status(snapshot_hash: str) -> dict[str, Any]:
state = read_gateway_state(snapshot_hash)
if state is None:
return {"snapshot_hash": snapshot_hash, "status": "not_started"}
return state
def list_gateways() -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
for directory in sorted(gateways_root().iterdir()):
if (
not directory.is_dir()
or len(directory.name) != 64
or any(char not in "0123456789abcdef" for char in directory.name)
):
continue
state = _read_gateway_state_by_key(directory.name)
if state:
results.append(state)
return results
def gateway_models(snapshot_hash: str) -> Any:
state = ensure_gateway(snapshot_hash)
if state is None:
return {"object": "list", "data": []}
return http_json(state["base_url"].rstrip("/") + "/models")
def dry_run_gateway(snapshot_hash: str) -> subprocess.CompletedProcess[str] | None:
snapshot = load_snapshot(snapshot_hash)
if not snapshot["manifest"].get("gateway_required"):
return None
settings = load_settings()
credentials, missing = _credentials(snapshot)
if missing:
raise RuntimeError("missing credentials: " + ", ".join(missing))
binary = _binary(settings)
environment = filtered_environment(
allow_sensitive=credentials,
extra={**credentials, "RUST_LOG": "error"},
)
return subprocess.run(
[binary, "--config", str(Path(snapshot["directory"]) / "routes.toml"), "--dry-run"],
env=environment,
capture_output=True,
text=True,
check=False,
timeout=max(1.0, float(settings.get("gateway_start_timeout_seconds", 15))),
)
def route_telemetry(
metadata: Mapping[str, Any],
session: Mapping[str, Any],
events_path: Path,
) -> dict[str, Any]:
"""Extract observed serving-route identity from Codex and Switchyard evidence."""
provider_slugs: list[str] = []
endpoint_tags: list[str] = []
routing_attempts: list[int] = []
retry_counts: list[int] = []
fallback_indices: list[int] = []
def add_unique(target: list[str], value: Any) -> None:
if isinstance(value, str) and value.strip() and value not in target:
target.append(value)
def visit(value: Any, *, scope: str | None = None) -> None:
if isinstance(value, dict):
selected = value.get("selected")
status = value.get("status")
successful_attempt_record = (
isinstance(status, int) and not isinstance(status, bool) and 200 <= status < 300
)
for key, child in value.items():
normalized = str(key).casefold()
if normalized in {"openrouter_metadata", "routing"}:
visit(child, scope=normalized)
continue
if normalized in {
"provider_slug",
"serving_provider_slug",
"actual_provider_slug",
}:
add_unique(provider_slugs, child)
elif normalized == "provider" and scope in {
"openrouter_metadata",
"routing",
}:
if (
selected is True
or successful_attempt_record
or (scope == "routing" and selected is None)
):
add_unique(provider_slugs, child)
elif normalized == "provider_name" and (
scope in {"openrouter_metadata", "routing"}
or "upstream_id" in value
or "total_cost" in value
):
add_unique(provider_slugs, child)
elif normalized in {"provider_tag", "endpoint_tag", "serving_endpoint_tag"}:
add_unique(endpoint_tags, child)
elif normalized == "tag" and isinstance(value.get("selected"), bool):
if value.get("selected"):
add_unique(endpoint_tags, child)
elif (
normalized == "attempt"
and isinstance(child, int)
and not isinstance(child, bool)
and scope in {"openrouter_metadata", "routing"}
):
routing_attempts.append(child)
elif (
normalized in {"retry_count", "retries"}
and isinstance(child, int)
and not isinstance(child, bool)
):
retry_counts.append(max(0, child))
elif (
normalized == "fallback_index"
and isinstance(child, int)
and not isinstance(child, bool)
):
fallback_indices.append(max(0, child))
visit(child, scope=scope)
elif isinstance(value, list):
for child in value:
visit(child, scope=scope)
sources: list[str] = []
if events_path.is_file():
for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines():
with contextlib.suppress(json.JSONDecodeError, ValueError):
visit(strict_json_loads(line))
sources.append("codex_events")
routing_path = session.get("gateway_routing_log_path")
if isinstance(routing_path, str) and Path(routing_path).is_file():
needles = {
str(metadata.get("model_key") or ""),
str(metadata.get("model") or ""),
str(metadata.get("route") or ""),
}
for line in Path(routing_path).read_text(encoding="utf-8", errors="replace").splitlines():
if not any(needle and needle in line for needle in needles):
continue
with contextlib.suppress(json.JSONDecodeError, ValueError):
visit(strict_json_loads(line))
sources.append("switchyard_routing_log")
successful_attempt = max(routing_attempts) if routing_attempts else None
observed_fallback_index = (
max(fallback_indices)
if fallback_indices
else max((attempt - 1 for attempt in routing_attempts if attempt >= 1), default=None)
)
retries = max(retry_counts) if retry_counts else None
requested_policy = metadata.get("requested_route_policy")
return {
"requested_policy": requested_policy,
"actual_serving_provider_slugs": provider_slugs,
"actual_serving_endpoint_tags": endpoint_tags,
"successful_attempt": successful_attempt,
"fallback_index": observed_fallback_index,
"retries": retries,
"retry_telemetry_complete": retries is not None,
"sources": sources,
"complete": requested_policy is None or bool(provider_slugs or endpoint_tags),
}
def route_telemetry_warnings(
metadata: Mapping[str, Any], observation: Mapping[str, Any]
) -> list[str]:
"""Interpret provider-route evidence without leaking that policy into workers."""
if metadata.get("requested_route_policy") is not None and not observation.get("complete"):
return [
"OpenRouter serving-provider telemetry is incomplete; requested policy is recorded "
"but the actual endpoint was not present in captured events"
]
return []
def _latest_record_timestamp(
record: Mapping[str, Any], fields: tuple[str, ...], label: str
) -> float:
observed: list[float] = []
for field in fields:
value = record.get(field)
if value is not None:
observed.append(_timestamp(value, f"{label} {field}"))
if not observed:
raise ValueError(f"{label} has no lifecycle timestamp")
return max(observed)
def _session_gateway_hash(session: Mapping[str, Any], *, verify_snapshot: bool) -> str | None:
recorded = session.get("gateway_hash")
if recorded is None:
if session.get("gateway_base_url") is not None:
raise ValueError("session has a gateway endpoint without a gateway identity")
return None
gateway_hash = _validate_hash(str(recorded), "session gateway hash")
if verify_snapshot:
expected = _gateway_key(str(session["snapshot_hash"]))
if gateway_hash != expected:
raise ValueError("session gateway identity does not match its snapshot")
return gateway_hash
def _session_admission_timestamp(session: Mapping[str, Any]) -> float:
return _timestamp(
session.get("last_active_at") or session.get("run_created_at") or session.get("created_at"),
"session admission timestamp",
)
def _job_admission_timestamp(job: Mapping[str, Any]) -> float:
return _timestamp(
job.get("recovery_requested_at") or job.get("created_at"),
"worker admission timestamp",
)
def _session_retains_gateway(session: Mapping[str, Any], now: float) -> bool:
status = session.get("status")
if status not in ACTIVE_SESSION_STATUSES:
return False
if process_matches(session.get("root_pid"), session.get("root_start_token")) or process_matches(
session.get("root_app_server_pid"), session.get("root_app_server_start_token")
):
return True
if status == "starting":
admitted_at = _session_admission_timestamp(session)
return now <= admitted_at + _GATEWAY_ADMISSION_GRACE_SECONDS
return False
def _job_retains_gateway(job: Mapping[str, Any], now: float) -> bool:
status = job.get("status")
if status not in RECOVERABLE_JOB_STATUSES:
return False
if process_matches(job.get("runner_pid"), job.get("runner_start_token")) or process_matches(
job.get("app_server_pid"), job.get("app_server_start_token")
):
return True
if status in {"queued", "starting", "recovering"}:
admitted_at = _job_admission_timestamp(job)
return now <= admitted_at + _GATEWAY_ADMISSION_GRACE_SECONDS
return False
def stop_idle_gateways() -> list[str]:
"""Stop route-set gateways only after every durable execution host releases them."""
settings = load_settings()
idle_seconds = int(settings.get("gateway_idle_timeout_seconds", 3600))
now = time.time()
active_gateway_hashes: set[str] = set()
last_consumer_at: dict[str, float] = {}
session_gateway_hashes: dict[str, str | None] = {}
try:
sessions = iter_session_records(strict=True)
jobs = iter_job_records(strict=True)
for session in sessions:
active = session.get("status") in ACTIVE_SESSION_STATUSES
gateway_hash = _session_gateway_hash(session, verify_snapshot=active)
session_id = str(session["session_id"])
session_gateway_hashes[session_id] = gateway_hash
if gateway_hash is None:
continue
release_at = _latest_record_timestamp(
session,
(
"finished_at",
"paused_at",
"suspended_at",
"detached_at",
"last_active_at",
"run_created_at",
"created_at",
),
f"session {session_id}",
)
if session.get("status") == "starting":
release_at = max(
release_at,
_session_admission_timestamp(session) + _GATEWAY_ADMISSION_GRACE_SECONDS,
)
last_consumer_at[gateway_hash] = max(
last_consumer_at.get(gateway_hash, 0.0), release_at
)
if _session_retains_gateway(session, now):
active_gateway_hashes.add(gateway_hash)
for job in jobs:
session_id = str(job["session_id"])
if session_id not in session_gateway_hashes:
if job.get("status") in RECOVERABLE_JOB_STATUSES:
raise ValueError(f"active worker {job['job_id']} refers to a missing session")
continue
gateway_hash = session_gateway_hashes[session_id]
if gateway_hash is None:
continue
release_at = _latest_record_timestamp(
job,
(
"finished_at",
"paused_at",
"suspended_at",
"last_progress_at",
"created_at",
),
f"worker {job['job_id']}",
)
if job.get("status") in {"queued", "starting", "recovering"}:
release_at = max(
release_at,
_job_admission_timestamp(job) + _GATEWAY_ADMISSION_GRACE_SECONDS,
)
last_consumer_at[gateway_hash] = max(
last_consumer_at.get(gateway_hash, 0.0), release_at
)
if _job_retains_gateway(job, now):
active_gateway_hashes.add(gateway_hash)
except (KeyError, OSError, RuntimeError, ValueError) as exc:
raise RuntimeError(
"cannot determine gateway idleness from invalid durable execution state"
) from exc
stopped: list[str] = []
for state in list_gateways():
gateway_hash = state.get("gateway_hash") or state.get("snapshot_hash")
if not gateway_hash or gateway_hash in active_gateway_hashes:
continue
if state.get("status") == "invalid":
raise RuntimeError(
"cannot determine gateway idleness from invalid gateway state: "
+ str(state.get("error"))
)
last_used_at = _timestamp(state.get("last_used_at"), "gateway last-used timestamp")
idle_since = max(last_used_at, last_consumer_at.get(str(gateway_hash), 0.0))
if now - idle_since >= idle_seconds and state.get("status") in {"running", "starting"}:
representative = str(state.get("snapshot_hash") or "")
if representative:
stop_gateway(representative)
stopped.append(representative)
return stopped
libexec/mmo_guidance.py
+478
View File
@@ -0,0 +1,478 @@
#!/usr/bin/env python3
"""Deterministic Codex guidance compiled from one resolved MMO profile."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
PROFILE_SKILL_NAME = "mmo-profile-orchestration"
PROFILE_SKILL_RELATIVE_PATH = f"guidance/skills/{PROFILE_SKILL_NAME}/SKILL.md"
def agent_guidance_relative_path(agent_id: str) -> str:
return f"guidance/agents/{agent_id}.md"
def coordination_capable_agents(resolved: Mapping[str, Any]) -> list[str]:
return sorted(
agent_id
for agent_id, agent in resolved["agents"].items()
if agent.get("can_spawn") or agent.get("controls")
)
def _mechanical_limit_lines(resolved: Mapping[str, Any]) -> list[str]:
coordination = resolved["coordination"]
orchestration = coordination["orchestration"]
lines = [f"- Orchestration: `{orchestration}`."]
if orchestration in {"mcp", "hybrid"}:
lines.extend(
[
"- Maximum simultaneously admitted Agent MCP jobs plus the root: "
f"{coordination['max_active_agents']}.",
"- Terminal, suspended, and cold-paused MCP jobs release admission capacity; "
"sequential delegation is not limited by a lifetime spawn counter.",
f"- Maximum MCP delegation depth: {coordination['max_depth']}.",
f"- Maximum active MCP writers: {coordination['max_active_writers']}.",
"- MCP write-scope conflicts are rejected mechanically.",
f"- MCP result visibility: {coordination['result_visibility']}.",
]
)
if orchestration in {"native", "hybrid"}:
lines.append(
"- Maximum concurrent native spawned threads, excluding the root: "
f"{coordination['native_max_concurrent_threads']}."
)
lines.append(f"- Contradictions are resolved by `{coordination['contradiction_policy']}`.")
return lines
def _profile_coordination_text(resolved: Mapping[str, Any]) -> str:
coordination = resolved["coordination"]
root = resolved["profile"]["root"]
orchestration = coordination["orchestration"]
if orchestration == "mcp":
backend_text = """Delegation is handled by the `mmo_mesh` Agent MCP supervisor. Use its
`agent_spawn` or `agents_spawn` tools. Each child is an isolated asynchronous
Codex app-server worker with a durable thread, event trace, partial evidence,
bounded lineage, resource admission, write-scope leasing, lifecycle control,
and output-contract validation."""
elif orchestration == "native":
backend_text = """Delegation is handled by Codex native subagents. Use the native agent tools
and the generated custom roles. Native threads integrate with `/agent` and have
lower launch overhead, but MMO cannot mechanically enforce per-child write
scopes, output contracts, or every spawn-graph edge. Keep writes disjoint and
validate all material results."""
else:
backend_text = """Both delegation paths are available.
- Use Codex native subagents for low-latency, read-heavy, tightly coupled work
where the `/agent` UI and shared live context are valuable.
- Use the `mmo_mesh` Agent MCP supervisor for durable asynchronous jobs, strict
role/model pinning, nested bounded delegation, output contracts, explicit
write scopes, live steering, detach-safe continuation, auditability, or
low-trust participants.
Do not launch the same assignment through both paths unless independent
redundancy is intentional. Native and MCP results are both evidence that the
caller must reconcile."""
limits = "\n".join(_mechanical_limit_lines(resolved))
return f"""## Codex MMO coordination contract
This session is pinned to immutable profile `{resolved["profile"]["id"]}` and logical profile
`{resolved["logical_hash"][:16]}`. The root role is `{root}`.
Orchestration backend: `{orchestration}`.
{backend_text}
Runtime policy:
{limits}
Keep the immediate critical path. Launch independent side work early, then
continue non-overlapping work. Do not wait merely because a child exists. Wait
only when the next required action depends on unfinished output. Treat every
child result as evidence, not authority. Resolve disagreement from source,
commands, tests, specifications, or other reproducible primary evidence; never
decide by model vote.
"""
def agent_instructions_text(resolved: Mapping[str, Any], agent_id: str, *, is_root: bool) -> str:
agent = resolved["agents"][agent_id]
model = resolved["models"][agent["model"]]
route = resolved["routes"][model["route"]]
role = "root integration authority" if is_root else "delegated participant"
native_children = [
child for child in agent["can_spawn"] if "native" in resolved["agents"][child]["backends"]
]
mcp_children = [
child for child in agent["can_spawn"] if "mcp" in resolved["agents"][child]["backends"]
]
child_lines: list[str] = []
for child in agent["can_spawn"]:
child_agent = resolved["agents"][child]
backends = "/".join(child_agent["backends"])
child_lines.append(
f"- `{child}` via {backends}: {child_agent.get('description') or 'profile participant'}"
)
children_text = "\n".join(child_lines) or "- none"
if agent["execution_mode"] == "goal":
execution_text = f"""Execution mode: durable Codex `goal`. The initial token budget is
{agent["goal_token_budget"]} and the profile ceiling is {agent["max_goal_token_budget"]}.
Silence for {agent["stall_warning_seconds"]} seconds produces an operator warning only; it does
not interrupt the model. A terminal schema turn may use up to
{agent["finalization_grace_seconds"]} seconds after investigation is complete. The host owns
token accounting and lifecycle state. Never estimate elapsed time or emit checkpoint prose merely
to prove liveness. A final assistant message does not finish an active goal. When the objective is
actually achieved and no required work remains, call `update_goal` with `status="complete"` in the
terminal turn, then provide the final result. Use `blocked` only under the tool-defined repeated-
impasse rule; do not use it for ordinary uncertainty, slow work, or a nearly exhausted budget."""
else:
execution_text = f"""Execution mode: one durable Codex `turn`, with no profile wall-clock
task deadline. Silence for {agent["stall_warning_seconds"]} seconds produces an operator warning
only. A strict terminal repair may use up to {agent["finalization_grace_seconds"]} seconds. Never
estimate elapsed time or emit checkpoint prose merely to prove liveness."""
text = f"""# Codex MMO agent: {agent_id}
You are the `{agent_id}` {role} in profile `{resolved["profile"]["id"]}`.
Model binding: `{model["upstream_id"]}` through route `{model["route"]}` ({route["name"]}).
Trust policy: `{agent["trust"]}`.
Verification policy: `{agent["verification"]}`.
Maximum permissions: `{agent["permissions"]}`.
{execution_text}
Permitted child roles and execution paths:
{children_text}
{_profile_coordination_text(resolved)}
"""
if not is_root:
text += """
The parent and root retain ownership of integration and the final user-facing
answer. Stay inside the delegated objective. Do not broaden scope or make an
unstated architectural or product decision. Preserve unrelated changes and
return precise evidence, validation, risks, and blockers.
"""
if agent["can_spawn"]:
text += """
## Required delegation checkpoint
Use `$mmo-profile-orchestration` before prolonged work. Within at most three
substantive task calls, or before starting a second independent workstream,
identify work retained here and eligible independent work for a direct child.
A substantive call performs repository discovery, shell execution, external
lookup, analysis, or implementation; loading guidance and managing an existing
child do not count.
For a nontrivial task, launch at least one eligible independent branch early.
When two independent branches are eligible and capacity permits, batch-launch
them. If no branch is launched, state the concrete reason before continuing:
the task is atomic, no direct child is independently useful, a required route
is unavailable, a graph or resource limit is exhausted, the user prohibited
delegation, or an authority/tool boundary prevents a safe handoff. Do not use a
generic claim that solo work is easier.
"""
if native_children:
text += """
## Native delegation
Use the generated Codex native roles for fast read-heavy parallelism and
closely coupled work. Native agents share the Codex process and workspace; do
not assume MMO write-scope or output-contract enforcement applies to them.
"""
if mcp_children or agent.get("controls"):
text += """
## Agent MCP delegation
Use `mmo_mesh` for durable asynchronous jobs, strict model pinning, disjoint
write scopes, contract validation, nested bounded delegation, cancellation, or
low-trust roles. Continue separate work after spawning when this role has spawn
authority. Start with `agent_list`: controls use opaque `agent_run_ref` values,
never guessed job IDs or model-supplied identities. Inspect or trace an
authorized long-running agent when its state matters; silence produces a
warning, not an automatic failure. Use only the actions granted for the target:
steer, answer pending input, change an allowed effort, interrupt the current
turn, pause continued work, continue the same thread (or extend a recoverable
goal within its compiled token ceiling), detach without stopping work, compact,
fork, request terminal serialization, or fully stop while retaining evidence.
Use the revision from the latest inspection for every mutating control.
Use the `progress_revision` values from `agent_status` or a prior
`agents_wait` call as the exact `after_revision` map for every requested job;
this returns compact state on the first
durable change instead of repeatedly injecting unchanged job metadata. Result
previews are opt-in and bounded. Read each material terminal result through
`agent_result`, beginning at cursor 0 and following every returned
`next_cursor` until it is null. Never read MMO supervisor state, job result
files, event logs, stderr files, or sockets directly from the state directory;
use `agent_result`, `agent_inspect`, `agent_trace`, and `agent_trace_record`.
Page a truncated trace summary through its `record_cursor`. When this role
has lineage disposition authority, explicitly accept or reject a successfully
completed result before relying on it and integrate a writable result only
after acceptance and review.
A lost client or app-server transport does not imply lost work. After recovery,
use `agent_list`, `agent_status`, and `agent_inspect` to find retained runs before
spawning any replacement. Continue the same suspended run when its original
objective remains useful. A replacement turn-mode host first settles an
orphaned active turn and starts at most one continuation; a terminal result
that completed during that race remains authoritative.
A provider limit, transport failure, malformed tool call, or failed terminal
turn remains typed on the same run with raw error and partial evidence. Treat
provider reset text without a timezone as provider-local/unspecified; do not
invent a timezone or replace the affected role with an undeclared route.
A detached job retains its live host, thread, and evidence. A cold-paused MCP
job retains its persisted thread and evidence while retiring its host and
releasing execution capacity; continuation starts one replacement host for
that same thread after fresh admission. Native pause remains logical because
native threads share the root host. A suspended job also retains its persisted
thread and evidence. Failed, stopped, or cancelled jobs retain inspectable
evidence but cannot be dispositioned as successful results.
"""
if agent["can_spawn"]:
text += """
You remain responsible for consuming and reconciling every material descendant
result before reporting upward.
"""
elif agent.get("controls"):
text += """
You have control authority but no spawn authority. Do not create agents. Use
the exact control graph only to unblock, correct, preserve, or conclude work
that another authorized role already admitted.
"""
else:
text += "\nYou are a true leaf participant with no spawn or control authority.\n"
if agent["trust"] == "low":
text += """
## Low-trust evidence boundary
Perform only bounded, literal, directly verifiable work. Do not infer intent,
architecture, correctness, causality, or recommended action unless the task and
contract explicitly permit it. Report conflicts without choosing a winner. The
parent must independently verify every material claim.
"""
if agent["verification"] == "root_adjudication":
text += (
"\nYour conclusions are adversarial input for root adjudication, not final decisions.\n"
)
profile_text = agent.get("instructions_text", "").strip()
if profile_text:
text += "\n## Profile-specific role instructions\n\n" + profile_text + "\n"
return text
def _cell(value: Any) -> str:
return str(value).replace("|", "\\|").replace("\n", " ").strip()
def profile_skill_text(resolved: Mapping[str, Any]) -> str:
profile_id = str(resolved["profile"]["id"])
coordination = resolved["coordination"]
role_rows: list[str] = []
for agent_id in sorted(resolved["agents"]):
agent = resolved["agents"][agent_id]
backends = "/".join(agent.get("backends", [])) or "root"
task_kinds = ", ".join(agent.get("allowed_task_kinds", [])) or "root-owned"
children = ", ".join(agent.get("can_spawn", [])) or "none"
role_rows.append(
"| "
+ " | ".join(
_cell(value)
for value in (
f"`{agent_id}`",
backends,
task_kinds,
agent["permissions"],
f"{agent['trust']}/{agent['verification']}",
children,
)
)
+ " |"
)
rows = "\n".join(role_rows)
control_rows = "\n".join(
f"- `{agent_id}` -> `{target}`: " + ", ".join(f"`{action}`" for action in grant["actions"])
for agent_id, agent in sorted(resolved["agents"].items())
for target, grant in sorted(agent.get("controls", {}).items())
)
orchestration = coordination["orchestration"]
if orchestration == "native":
path_description = "Codex native agents"
execution_paths = """- Use the generated roles through Codex native agent tools. They are
appropriate for fast, read-heavy, tightly coupled work. Keep writes disjoint
because MMO does not enforce native write scopes or result contracts
mechanically.
- `mmo_mesh` Agent MCP delegation is not available in this profile."""
elif orchestration == "mcp":
path_description = "mmo_mesh"
execution_paths = """- Use MCP roles through `mmo_mesh`. Tool MCP servers such as IDA or
Firecrawl provide capabilities; they do not launch agents. `mmo_mesh` owns
agent lineage, admission, job state, cancellation, contracts, and result
decisions.
- Codex native agent delegation is not available in this profile."""
else:
path_description = "native agents or mmo_mesh"
execution_paths = """- Use native roles through Codex native agent tools. They are appropriate for
fast, read-heavy, tightly coupled work. Keep writes disjoint because MMO does
not enforce native write scopes or result contracts mechanically.
- Use MCP roles through `mmo_mesh`. Tool MCP servers such as IDA or Firecrawl
provide capabilities; they do not launch agents. `mmo_mesh` owns agent
lineage, admission, job state, cancellation, contracts, and result decisions.
- Do not send the same assignment through both paths unless independent
reproduction or adversarial diversity is the explicit objective."""
has_mcp_lifecycle = orchestration in {"mcp", "hybrid"} and any(
agent.get("controls")
or any(
"mcp" in resolved["agents"][child].get("backends", [])
for child in agent.get("can_spawn", [])
)
for agent in resolved["agents"].values()
)
mcp_lifecycle = ""
if has_mcp_lifecycle:
mcp_lifecycle = """
## Use the MCP lifecycle
For each MCP task, provide an objective, necessary context and paths, non-goals,
the required deliverable or result contract, and validation evidence. Use
`agents_spawn` for independent batches and `agent_spawn` for one branch.
Remain productive while jobs run. Check status only when useful. Call
`agents_wait` only at a genuine dependency barrier. Pass the latest exact
per-job `progress_revision` map for every requested job as `after_revision` so unchanged work does not bloat
the caller context; the call returns when durable state changes or its bounded
wait expires. Result previews are opt-in and never the complete result. Read
every terminal result with
`agent_result`: begin at cursor 0 and keep calling it with each `next_cursor`
until `next_cursor` is null. Concatenate text pages in cursor order without
overlap; a complete strict structured result may instead arrive once as JSON.
Never bypass this lifecycle by opening MMO job result files, event logs, stderr
files, sockets, or other supervisor state directly. Use `agent_result`,
`agent_inspect`, and `agent_trace`; when a trace summary is truncated, use its
`record_cursor` with `agent_trace_record` and page through every `next_cursor`.
Same-user filesystem access is not an authorization boundary.
Use `agent_list` to discover root, native, and MCP runs plus their opaque refs;
use `agent_status` for a known supervised job, then use `agent_inspect` and
`agent_trace` instead of polling blindly. After any client or transport
recovery, discover and inspect retained work before spawning replacements.
Continue the same suspended run when its objective remains useful. Mutating
controls are compare-and-swap operations: inspect first and pass the returned
revision. Use `agent_steer` to add direction to an active turn without replacing
its existing task. `agent_interrupt` stops only the current turn and an active
goal may continue; `agent_pause` first pauses the goal, interrupts it, retains
partial evidence, and cold-retires a supervised MCP host.
`agent_detach` removes the client while work continues, `agent_continue`
reactivates recoverable work (and may raise its token budget only within the
compiled ceiling), and `agent_stop` pauses, interrupts, retains evidence, and
retires a supervised MCP host or terminates a native run without retiring its
shared root host. Use `agent_respond` for pending requests with the exact
method-specific response shape, `agent_finalize` for strict terminal
serialization, `agent_compact` for thread compaction, `agent_set_effort` within
the role grant. `agent_fork` normally admits an independent MCP job; a native
fork instead inherits its role, cwd, and sandbox in the shared root host, obeys
the native-thread limit, and cannot accept MCP write-scope or attachment overrides.
For a successfully completed job, use `agent_result_accept` or
`agent_result_reject` with a concrete reason when those lineage-authority tools
are exposed to the current role. A detached job keeps its live host. A
cold-paused supervised MCP job keeps its thread, trace, partial evidence, and
artifacts while releasing host capacity; a paused native thread remains in its
shared root host. A suspended job keeps the same durable evidence;
continuation starts or reattaches exactly one host for that thread after fresh
admission. Failed, stopped, and
cancelled jobs cannot be dispositioned; preserve their status and uncertainty.
Integrate an accepted writable patch only after reviewing its scope and tests.
Use `agent_cancel` only for work that is stale, superseded, unsafe, or no longer
worth its cost; cancellation is immediate and distinct from evidence-preserving
finalization.
Every root and supervised MCP worker owns one Unix app-server host and durable
thread; native agents are durable child threads inside their parent root host.
Interactive clients attach to the current root generation of the immutable MMO
session and run. An intentional stock-TUI fresh-context action may create a new
top-level Codex root generation only while the verified attached root client is
idle; it does not create another MMO session or run. Resume by the MMO session
ID or any predecessor root-thread ID attaches to the current generation. This
is host-owned lifecycle: do not simulate it by spawning a replacement root or
starting another MMO session. A replacement turn-mode worker host first settles
an orphaned active turn before starting at most one same-thread continuation;
a terminal result completed during that race is preserved. Goal roles are
bounded by Codex token accounting; turn roles have no task wall clock. Stall
intervals are warning-only and provider/model slowness does not erase work.
Do not tell a model to watch a clock or emit periodic checkpoint prose. Strict
contracts apply to the
explicit terminal serialization turn, followed by at most one same-thread
repair.
Resolve conflicts with primary evidence, not voting or model reputation. A
worker failure, budget suspension, unavailable route, or malformed contract is explicit
uncertainty; it is not permission to silently substitute another route.
"""
if control_rows:
mcp_lifecycle += f"""
## Control graph
{control_rows}
"""
limits = "\n".join(_mechanical_limit_lines(resolved))
return f"""---
name: {PROFILE_SKILL_NAME}
description: "Coordinate the resolved Codex MMO profile {profile_id}. Use for nontrivial work when the current MMO role can delegate or control durable work through {path_description}, especially for parallel investigation, specialist work, independent verification, or bounded nested delegation."
---
# Orchestrate the profile
Identify the current role from `AGENTS.md`. Spawn only its direct children and
obey the exact profile graph and limits below. Keep ownership of the critical
path and final integration.
## Delegation checkpoint
This checkpoint applies only when the current role's `Direct children` cell is
not `none`. A control-only role must not spawn; it should use the MCP lifecycle
and exact control graph below only at a real dependency, correction, or risk
boundary.
Within at most three substantive task calls, or before entering a second
independent workstream:
1. State the critical-path work retained in the current role.
2. Identify independent work matched to a permitted child.
3. Launch at least one useful branch for a nontrivial decomposable task.
4. Batch-launch two or more independent branches when capacity permits.
5. Continue non-overlapping work immediately after launch.
If delegation is skipped, state one concrete reason before continuing: atomic
task, no independently useful direct child, unavailable route, exhausted graph
or resource limit, explicit user prohibition, or an authority/tool boundary.
Do not treat unfamiliarity with the orchestration tools as a skip reason.
## Choose the execution path
{execution_paths}
{mcp_lifecycle}
## Profile graph
| Role | Backend | Task kinds | Permissions | Trust/verification | Direct children |
|---|---|---|---|---|---|
{rows}
## Mechanical limits
{limits}
"""
def compiled_guidance(resolved: Mapping[str, Any]) -> dict[str, str]:
root = resolved["profile"]["root"]
payload = {
agent_guidance_relative_path(agent_id): agent_instructions_text(
resolved, agent_id, is_root=agent_id == root
)
for agent_id in sorted(resolved["agents"])
}
if coordination_capable_agents(resolved):
payload[PROFILE_SKILL_RELATIVE_PATH] = profile_skill_text(resolved)
return dict(sorted(payload.items()))
libexec/mmo_inventory_snapshot.py
+2260
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_mcp.py
Executable
+1167
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_profiles.py
Executable
+1775
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_runtime.py
Executable
+6019
View File
File diff suppressed because it is too large Load Diff
libexec/mmo_schema.py
Executable
+447
View File
@@ -0,0 +1,447 @@
#!/usr/bin/env python3
"""Small JSON-schema subset used for worker output contracts.
The project intentionally avoids a runtime dependency on jsonschema. Profiles
may use the supported, auditable subset documented in PROFILE_SCHEMA.md.
"""
from __future__ import annotations
import json
import math
import re
from collections.abc import Mapping
from typing import Any
from mmo_util import (
strict_json_decoder,
strict_json_loads,
valid_absolute_uri,
validate_json_unicode,
)
SUPPORTED_TYPES = {"object", "array", "string", "integer", "number", "boolean", "null"}
SUPPORTED_FORMATS = {"uri", "date", "date-time"}
_RFC3339_DATE = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})", re.ASCII)
_RFC3339_DATE_TIME = re.compile(
r"([0-9]{4})-([0-9]{2})-([0-9]{2})"
r"[Tt]([0-9]{2}):([0-9]{2}):([0-9]{2})"
r"(?:\.([0-9]+))?"
r"(?:([Zz])|([+-])([0-9]{2}):([0-9]{2}))",
re.ASCII,
)
def _is_finite_number(value: Any) -> bool:
if isinstance(value, bool) or not isinstance(value, (int, float)):
return False
# Converting an arbitrarily large JSON integer to float can overflow even
# though the integer itself is finite.
return isinstance(value, int) or math.isfinite(value)
def _is_integer_number(value: Any) -> bool:
"""Return whether a JSON number has a zero fractional part."""
return _is_finite_number(value) and (
isinstance(value, int) or (isinstance(value, float) and value.is_integer())
)
def _json_value_key(value: Any) -> tuple[Any, ...] | None:
"""Return a strict JSON value key using JSON Schema equality rules."""
if value is None:
return ("null",)
if isinstance(value, bool):
return ("boolean", value)
if isinstance(value, int):
return ("number", value)
if isinstance(value, float):
return ("number", value) if math.isfinite(value) else None
if isinstance(value, str):
return ("string", value)
if isinstance(value, list):
items = [_json_value_key(item) for item in value]
if any(item is None for item in items):
return None
return ("array", tuple(items))
if isinstance(value, Mapping):
if not all(isinstance(key, str) for key in value):
return None
members = [(key, _json_value_key(value[key])) for key in sorted(value)]
if any(item is None for _key, item in members):
return None
return ("object", tuple(members))
return None
def validate_schema_definition(schema: Any, path: str = "$") -> list[str]:
errors: list[str] = []
if isinstance(schema, bool):
return errors
if not isinstance(schema, Mapping):
return [f"{path}: schema must be an object or boolean"]
if not all(isinstance(key, str) for key in schema):
errors.append(f"{path}: schema keyword names must be strings")
schema_type = schema.get("type")
if schema_type is not None:
if isinstance(schema_type, str):
if schema_type not in SUPPORTED_TYPES:
errors.append(f"{path}.type: unsupported type {schema_type!r}")
elif isinstance(schema_type, list):
if not schema_type:
errors.append(f"{path}.type: array cannot be empty")
invalid = [
item
for item in schema_type
if not isinstance(item, str) or item not in SUPPORTED_TYPES
]
if invalid:
errors.append(f"{path}.type: unsupported types {invalid!r}")
elif len(schema_type) != len(set(schema_type)):
errors.append(f"{path}.type: array contains duplicate types")
else:
errors.append(f"{path}.type: must be a string or array")
for keyword in ("oneOf", "anyOf", "allOf"):
if keyword in schema:
value = schema[keyword]
if not isinstance(value, list) or not value:
errors.append(f"{path}.{keyword}: must be a non-empty array")
else:
for index, item in enumerate(value):
errors.extend(validate_schema_definition(item, f"{path}.{keyword}[{index}]"))
for keyword in ("not", "if", "then", "else"):
if keyword in schema:
errors.extend(validate_schema_definition(schema[keyword], f"{path}.{keyword}"))
if ("then" in schema or "else" in schema) and "if" not in schema:
errors.append(f"{path}: then/else requires if")
if "properties" in schema:
properties = schema["properties"]
if not isinstance(properties, Mapping):
errors.append(f"{path}.properties: must be an object")
else:
for key, value in properties.items():
if not isinstance(key, str):
errors.append(f"{path}.properties: property names must be strings")
continue
errors.extend(validate_schema_definition(value, f"{path}.properties.{key}"))
if "items" in schema:
errors.extend(validate_schema_definition(schema["items"], f"{path}.items"))
if "required" in schema:
required = schema["required"]
if not (isinstance(required, list) and all(isinstance(item, str) for item in required)):
errors.append(f"{path}.required: must be an array of strings")
elif len(required) != len(set(required)):
errors.append(f"{path}.required: contains duplicate property names")
if "enum" in schema:
enum = schema["enum"]
if not isinstance(enum, list) or not enum:
errors.append(f"{path}.enum: must be a non-empty array")
else:
keys = [_json_value_key(item) for item in enum]
if any(key is None for key in keys):
errors.append(f"{path}.enum: values must be valid finite JSON values")
elif len(keys) != len(set(keys)):
errors.append(f"{path}.enum: values must be unique")
if "const" in schema and _json_value_key(schema["const"]) is None:
errors.append(f"{path}.const: must be a valid finite JSON value")
for keyword in ("$schema", "$id", "title", "description"):
if keyword in schema and not isinstance(schema[keyword], str):
errors.append(f"{path}.{keyword}: must be a string")
for keyword in ("minLength", "maxLength", "minItems", "maxItems"):
if keyword in schema:
value = schema[keyword]
if not _is_integer_number(value) or value < 0:
errors.append(f"{path}.{keyword}: must be a non-negative integer")
for minimum, maximum in (("minLength", "maxLength"), ("minItems", "maxItems")):
if (
_is_integer_number(schema.get(minimum))
and _is_integer_number(schema.get(maximum))
and schema[minimum] > schema[maximum]
):
errors.append(f"{path}: {minimum} exceeds {maximum}")
for keyword in ("minimum", "maximum"):
if keyword in schema:
value = schema[keyword]
if not _is_finite_number(value):
errors.append(f"{path}.{keyword}: must be a finite number")
if (
isinstance(schema.get("minimum"), (int, float))
and not isinstance(schema.get("minimum"), bool)
and isinstance(schema.get("maximum"), (int, float))
and not isinstance(schema.get("maximum"), bool)
and schema["minimum"] > schema["maximum"]
):
errors.append(f"{path}: minimum exceeds maximum")
if "uniqueItems" in schema and not isinstance(schema["uniqueItems"], bool):
errors.append(f"{path}.uniqueItems: must be boolean")
if "additionalProperties" in schema:
additional = schema["additionalProperties"]
if isinstance(additional, Mapping):
errors.extend(validate_schema_definition(additional, f"{path}.additionalProperties"))
elif not isinstance(additional, bool):
errors.append(f"{path}.additionalProperties: must be boolean or a schema")
if "pattern" in schema:
pattern = schema["pattern"]
if not isinstance(pattern, str):
errors.append(f"{path}.pattern: must be a string")
else:
try:
re.compile(pattern)
except re.error as exc:
errors.append(f"{path}.pattern: invalid regular expression: {exc}")
if "format" in schema:
schema_format = schema["format"]
if not isinstance(schema_format, str) or schema_format not in SUPPORTED_FORMATS:
errors.append(f"{path}.format: must be one of {sorted(SUPPORTED_FORMATS)}")
known = {
"$schema",
"$id",
"title",
"description",
"type",
"properties",
"required",
"additionalProperties",
"items",
"enum",
"const",
"minLength",
"maxLength",
"minimum",
"maximum",
"minItems",
"maxItems",
"uniqueItems",
"pattern",
"oneOf",
"anyOf",
"allOf",
"not",
"if",
"then",
"else",
"format",
}
unknown = sorted(key for key in schema if isinstance(key, str) and key not in known)
if unknown:
errors.append(f"{path}: unsupported schema keywords: {', '.join(unknown)}")
return errors
def _types(schema: Mapping[str, Any]) -> set[str] | None:
value = schema.get("type")
if value is None:
return None
return {value} if isinstance(value, str) else set(value)
def _matches_type(value: Any, expected: str) -> bool:
if expected == "null":
return value is None
if expected == "boolean":
return isinstance(value, bool)
if expected == "integer":
return _is_integer_number(value)
if expected == "number":
return _is_finite_number(value)
if expected == "string":
return isinstance(value, str)
if expected == "array":
return isinstance(value, list)
if expected == "object":
return isinstance(value, Mapping)
return False
def _rfc3339_month_days(year: int, month: int) -> int:
if not 1 <= month <= 12:
return 0
leap = year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
month_days = (31, 29 if leap else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)
return month_days[month - 1]
def _valid_rfc3339_date(year: int, month: int, day: int) -> bool:
return 1 <= day <= _rfc3339_month_days(year, month)
def _shift_rfc3339_date(year: int, month: int, day: int, day_delta: int) -> tuple[int, int, int]:
"""Shift a valid RFC 3339 date by the at-most-one-day offset boundary."""
if day_delta == -1:
if day > 1:
return year, month, day - 1
if month > 1:
month -= 1
else:
year -= 1
month = 12
return year, month, _rfc3339_month_days(year, month)
if day_delta == 1:
if day < _rfc3339_month_days(year, month):
return year, month, day + 1
if month < 12:
return year, month + 1, 1
return year + 1, 1, 1
return year, month, day
def _matches_rfc3339_date(value: str) -> bool:
matched = _RFC3339_DATE.fullmatch(value)
return matched is not None and _valid_rfc3339_date(
int(matched.group(1)), int(matched.group(2)), int(matched.group(3))
)
def _matches_rfc3339_date_time(value: str) -> bool:
matched = _RFC3339_DATE_TIME.fullmatch(value)
if matched is None:
return False
year, month, day, hour, minute, second = map(int, matched.groups()[:6])
if not _valid_rfc3339_date(year, month, day) or hour > 23 or minute > 59 or second > 60:
return False
if matched.group(8) is not None:
offset_minutes = 0
else:
offset_hour = int(matched.group(10))
offset_minute = int(matched.group(11))
if offset_hour > 23 or offset_minute > 59:
return False
offset_minutes = offset_hour * 60 + offset_minute
if matched.group(9) == "-":
offset_minutes = -offset_minutes
if second == 60:
day_delta, utc_minute = divmod(hour * 60 + minute - offset_minutes, 24 * 60)
utc_year, utc_month, utc_day = _shift_rfc3339_date(year, month, day, day_delta)
# RFC 3339 section 5.7 allows :60 only at the end of a month in UTC.
# The local spelling may fall on the adjacent date after applying its
# numeric offset, so validate the shifted UTC calendar date.
return utc_minute == 23 * 60 + 59 and utc_day == _rfc3339_month_days(utc_year, utc_month)
return True
def _matches_format(value: str, schema_format: str) -> bool:
if schema_format == "uri":
return valid_absolute_uri(value)
if schema_format == "date":
return _matches_rfc3339_date(value)
if schema_format == "date-time":
return _matches_rfc3339_date_time(value)
return False
def validate_instance(
value: Any,
schema: Mapping[str, Any] | bool,
path: str = "$",
) -> list[str]:
if schema is True:
return []
if schema is False:
return [f"{path}: value is rejected by false schema"]
errors: list[str] = []
allowed_types = _types(schema)
if allowed_types and not any(_matches_type(value, item) for item in allowed_types):
return [
f"{path}: expected {' or '.join(sorted(allowed_types))}, got {type(value).__name__}"
]
if "const" in schema:
value_key = _json_value_key(value)
const_key = _json_value_key(schema["const"])
if value_key is None or const_key is None or value_key != const_key:
errors.append(f"{path}: expected constant {schema['const']!r}")
if "enum" in schema:
value_key = _json_value_key(value)
enum_keys = {_json_value_key(item) for item in schema["enum"]}
if value_key is None or value_key not in enum_keys:
errors.append(f"{path}: value {value!r} is not in the allowed enum")
if "oneOf" in schema:
matches = [not validate_instance(value, item, path) for item in schema["oneOf"]]
if sum(matches) != 1:
errors.append(f"{path}: value must match exactly one oneOf schema")
if "anyOf" in schema:
if not any(not validate_instance(value, item, path) for item in schema["anyOf"]):
errors.append(f"{path}: value does not match any anyOf schema")
if "allOf" in schema:
for item in schema["allOf"]:
errors.extend(validate_instance(value, item, path))
if "not" in schema and not validate_instance(value, schema["not"], path):
errors.append(f"{path}: value matches prohibited not schema")
if "if" in schema:
branch = "then" if not validate_instance(value, schema["if"], path) else "else"
if branch in schema:
errors.extend(validate_instance(value, schema[branch], path))
if isinstance(value, str):
if len(value) < int(schema.get("minLength", 0)):
errors.append(f"{path}: string is shorter than minLength")
if "maxLength" in schema and len(value) > int(schema["maxLength"]):
errors.append(f"{path}: string is longer than maxLength")
if "pattern" in schema and not re.search(str(schema["pattern"]), value):
errors.append(f"{path}: string does not match required pattern")
if "format" in schema and not _matches_format(value, str(schema["format"])):
errors.append(f"{path}: string does not match {schema['format']} format")
if isinstance(value, (int, float)) and not isinstance(value, bool):
if "minimum" in schema and value < schema["minimum"]:
errors.append(f"{path}: value is below minimum")
if "maximum" in schema and value > schema["maximum"]:
errors.append(f"{path}: value is above maximum")
if isinstance(value, list):
if len(value) < int(schema.get("minItems", 0)):
errors.append(f"{path}: array has fewer than minItems")
if "maxItems" in schema and len(value) > int(schema["maxItems"]):
errors.append(f"{path}: array has more than maxItems")
if schema.get("uniqueItems"):
serialized = [_json_value_key(item) for item in value]
if any(item is None for item in serialized):
errors.append(f"{path}: array items must be valid finite JSON values")
elif len(serialized) != len(set(serialized)):
errors.append(f"{path}: array items must be unique")
item_schema = schema.get("items")
if isinstance(item_schema, (Mapping, bool)):
for index, item in enumerate(value):
errors.extend(validate_instance(item, item_schema, f"{path}[{index}]"))
if isinstance(value, Mapping):
required = schema.get("required", [])
for key in required:
if key not in value:
errors.append(f"{path}: required property {key!r} is missing")
properties = schema.get("properties", {})
additional = schema.get("additionalProperties", True)
for key, item in value.items():
if key in properties:
errors.extend(validate_instance(item, properties[key], f"{path}.{key}"))
elif additional is False:
errors.append(f"{path}: additional property {key!r} is not allowed")
elif isinstance(additional, Mapping):
errors.extend(validate_instance(item, additional, f"{path}.{key}"))
return errors
def extract_json_document(text: str) -> tuple[Any | None, str | None]:
stripped = text.strip()
candidates = [stripped]
fenced = re.findall(r"```(?:json)?\s*(.*?)```", stripped, flags=re.IGNORECASE | re.DOTALL)
candidates.extend(item.strip() for item in fenced)
for candidate in candidates:
if not candidate:
continue
try:
return strict_json_loads(candidate), None
except (json.JSONDecodeError, ValueError):
pass
decoder = strict_json_decoder()
for match in re.finditer(r"[\[{]", stripped):
try:
value, end = decoder.raw_decode(stripped[match.start() :])
except (json.JSONDecodeError, ValueError):
continue
try:
validate_json_unicode(value)
except ValueError:
continue
remaining = stripped[match.start() + end :].strip()
if not remaining or remaining.startswith("```"):
return value, None
return None, "no valid JSON document was found in the final worker response"
libexec/mmo_snapshot.py
Executable
+525
View File
@@ -0,0 +1,525 @@
#!/usr/bin/env python3
"""Compile validated composition profiles into immutable logical snapshots."""
from __future__ import annotations
import contextlib
import json
import os
import shutil
import stat
import tempfile
from collections.abc import Mapping
from pathlib import Path, PurePosixPath
from typing import Any
from mmo_guidance import compiled_guidance, coordination_capable_agents
from mmo_profiles import resolve_profile
from mmo_util import (
atomic_write_json,
atomic_write_text,
file_lock,
make_tree_read_only,
manifest_for_tree,
read_json,
sha256_bytes,
stable_hash,
state_root,
toml_dumps,
)
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION
SWITCHYARD_SCHEMA_VERSION = 1
_VERIFIED_SNAPSHOT_STATS: dict[str, tuple[tuple[str, int, int, int, int, int], ...]] = {}
def _snapshot_fingerprint(
resolved: Mapping[str, Any],
*,
guidance: Mapping[str, str] | None = None,
guidance_schema_version: int = MMO_SCHEMA_VERSION,
) -> dict[str, Any]:
"""Return every semantic input that determines compiled snapshot bytes."""
return {
"snapshot_schema_version": MMO_SCHEMA_VERSION,
"guidance_schema_version": guidance_schema_version,
"package_version": resolved["package_version"],
"resolved": resolved,
"guidance": dict(guidance) if guidance is not None else compiled_guidance(resolved),
}
def snapshots_root() -> Path:
root = state_root() / "snapshots"
root.mkdir(parents=True, exist_ok=True, mode=0o700)
return root
def snapshot_dir(snapshot_hash: str) -> Path:
if len(snapshot_hash) != 64 or any(char not in "0123456789abcdef" for char in snapshot_hash):
raise ValueError("invalid snapshot hash")
path = snapshots_root() / snapshot_hash
try:
mode = path.lstat().st_mode
except FileNotFoundError as exc:
raise FileNotFoundError(f"unknown profile snapshot: {snapshot_hash}") from exc
if not stat.S_ISDIR(mode):
raise RuntimeError(f"snapshot path is not a regular directory: {path}")
return path
def _remove_snapshot_tree(path: Path) -> None:
"""Remove a snapshot tree without following attacker-controlled links."""
try:
mode = path.lstat().st_mode
except FileNotFoundError:
return
if stat.S_ISLNK(mode):
path.unlink()
return
if not stat.S_ISDIR(mode):
raise RuntimeError(f"snapshot path is not a directory: {path}")
for item in path.rglob("*"):
with contextlib.suppress(OSError):
item_mode = item.lstat().st_mode
if not stat.S_ISLNK(item_mode):
os.chmod(item, 0o700 if stat.S_ISDIR(item_mode) else 0o600)
with contextlib.suppress(OSError):
os.chmod(path, 0o700)
shutil.rmtree(path)
def _route_id(snapshot_hash: str, model_key: str) -> str:
return f"mmo-{snapshot_hash[:12]}-{model_key}"
def _switchyard_routes(
resolved: Mapping[str, Any],
) -> tuple[dict[str, Any], dict[str, str], str | None]:
clients: dict[str, Any] = {}
targets: dict[str, Any] = {}
routes: dict[str, Any] = {}
route_ids: dict[str, str] = {}
for route_key, route in resolved["routes"].items():
if route["driver"] != "switchyard":
continue
client: dict[str, Any] = {
"format": route["wire_protocol"],
"base_url": route["base_url"],
"max_retries": int(route.get("max_retries", 1)),
}
credentials = list(route.get("credential_envs", []))
if credentials:
client["api_key_env"] = credentials[0]
if route.get("extra_headers"):
client["extra_headers"] = route["extra_headers"]
clients[route_key] = client
for model_key, model in resolved["models"].items():
route = resolved["routes"][model["route"]]
if route["driver"] != "switchyard":
continue
targets[model_key] = {
"id": model["upstream_id"],
"llm_client": model["route"],
}
extra_body = dict(model.get("extra_body") or {})
route_policy = model.get("route_policy") or route.get("openrouter_policy")
if route_policy is not None:
extra_body["provider"] = route_policy
if extra_body:
targets[model_key]["extra_body"] = extra_body
if not targets:
return (
{
"schema_version": SWITCHYARD_SCHEMA_VERSION,
"llm_clients": clients,
"targets": targets,
"routes": routes,
},
route_ids,
None,
)
route_semantics: dict[str, dict[str, Any]] = {}
for model_key, model in resolved["models"].items():
route = resolved["routes"][model["route"]]
if route["driver"] != "switchyard":
continue
route_semantics[model_key] = {
"type": "passthrough",
"target": model_key,
"context_window": int(model["context_window"]),
"tool_calling": bool(model.get("tool_calling", True)),
"reasoning": model.get("default_reasoning") != "none",
}
# Gateway identity is derived only from transport/model semantics, not from
# the surrounding profile. Profiles with an identical route set can share
# one Switchyard process while retaining independent immutable snapshots.
gateway_hash = stable_hash(
{
"schema_version": SWITCHYARD_SCHEMA_VERSION,
"llm_clients": clients,
"targets": targets,
"routes": route_semantics,
}
)
for model_key, semantics in route_semantics.items():
route_id = _route_id(gateway_hash, model_key)
route_ids[model_key] = route_id
routes[model_key] = {
"id": route_id,
**semantics,
}
return (
{
"schema_version": SWITCHYARD_SCHEMA_VERSION,
"llm_clients": clients,
"targets": targets,
"routes": routes,
},
route_ids,
gateway_hash,
)
def _credential_groups(resolved: Mapping[str, Any], *, drivers: set[str]) -> list[dict[str, Any]]:
groups: list[dict[str, Any]] = []
for route_key, route in sorted(resolved["routes"].items()):
if route["driver"] not in drivers:
continue
alternatives = list(route.get("credential_envs", []))
if not alternatives:
continue
groups.append(
{
"route": route_key,
"driver": route["driver"],
"target_env": alternatives[0],
"alternatives": alternatives,
}
)
return groups
def _json_text(value: Any) -> str:
return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n"
def _snapshot_material(
resolved: Mapping[str, Any],
snapshot_hash: str,
*,
package: str,
) -> tuple[dict[str, Any], dict[str, str]]:
"""Derive every manifest field and payload byte from hashed profile data."""
routes, route_ids, gateway_hash = _switchyard_routes(resolved)
gateway_credential_groups = _credential_groups(resolved, drivers={"switchyard"})
direct_credential_groups = _credential_groups(resolved, drivers={"codex_custom"})
credentials = sorted({group["target_env"] for group in gateway_credential_groups})
direct_credentials = sorted({group["target_env"] for group in direct_credential_groups})
manifest = {
"schema_version": MMO_SCHEMA_VERSION,
"guidance_schema_version": MMO_SCHEMA_VERSION,
"snapshot_hash": snapshot_hash,
"logical_hash": resolved["logical_hash"],
"package_version": package,
"profile_id": resolved["profile"]["id"],
"profile_version": resolved["profile"]["version"],
"root_agent": resolved["profile"]["root"],
"orchestration": resolved["coordination"]["orchestration"],
"native_agents": resolved["capabilities"]["native_agents"],
"mcp_agents": resolved["capabilities"]["mcp_agents"],
"warnings": resolved.get("warnings", []),
"gateway_required": bool(routes["routes"]),
"gateway_hash": gateway_hash,
"route_ids": route_ids,
"credential_envs": credentials,
"credential_groups": gateway_credential_groups,
"direct_credential_envs": direct_credentials,
"direct_credential_groups": direct_credential_groups,
"agents": sorted(resolved["agents"]),
"models": sorted(resolved["models"]),
"routes": sorted(resolved["routes"]),
"coordination_capable_agents": coordination_capable_agents(resolved),
}
tool_mcp_servers = resolved.get("tool_mcp_servers", {})
manifest["tool_mcp_servers"] = sorted(tool_mcp_servers)
payload = {
"resolved-profile.json": _json_text(resolved),
"resolved-profile.toml": toml_dumps(
{key: value for key, value in resolved.items() if key != "logical_hash"}
),
}
if routes["routes"]:
payload["routes.toml"] = toml_dumps(routes)
for agent_key, agent in resolved["agents"].items():
payload[f"instructions/{agent_key}.md"] = agent.get("instructions_text", "").rstrip() + "\n"
contract = agent.get("output_contract_schema")
if contract is not None:
payload[f"contracts/{agent_key}.json"] = _json_text(contract)
if resolved.get("smoke") is not None:
payload["smoke.json"] = _json_text(resolved["smoke"])
payload.update(compiled_guidance(resolved))
return manifest, payload
def compile_profile(
profile: str | Path,
*,
bindings: Mapping[str, str] | None = None,
force: bool = False,
) -> dict[str, Any]:
resolved = resolve_profile(profile, bindings=bindings)
return compile_resolved_profile(resolved, force=force)
def compile_resolved_profile(resolved: Mapping[str, Any], *, force: bool = False) -> dict[str, Any]:
"""Compile already validated profile semantics into a content-addressed snapshot.
This is intentionally narrow: evaluation uses it to reduce a validated
profile's delegation graph for matched root-only and ablation controls.
Callers may only pass a complete current-schema document and must recompute its
logical hash after any restriction.
"""
if resolved.get("schema_version") != MMO_SCHEMA_VERSION or isinstance(
resolved.get("schema_version"), bool
):
raise ValueError(f"compile_resolved_profile requires resolved schema v{MMO_SCHEMA_VERSION}")
if not isinstance(resolved.get("logical_hash"), str):
raise ValueError("resolved profile logical_hash is required")
package = resolved.get("package_version")
if not isinstance(package, str) or not package:
raise ValueError("resolved profile package_version is required")
fingerprint = _snapshot_fingerprint(resolved)
snapshot_hash = stable_hash(fingerprint)
destination = snapshots_root() / snapshot_hash
lock = state_root() / ".snapshot-compiler.lock"
with file_lock(lock):
if destination.is_dir() and not force:
return load_snapshot(snapshot_hash)
if destination.exists() or destination.is_symlink():
_remove_snapshot_tree(destination)
temporary = Path(tempfile.mkdtemp(prefix=f".{snapshot_hash[:12]}-", dir=snapshots_root()))
published = False
try:
manifest, payload = _snapshot_material(resolved, snapshot_hash, package=package)
instructions_dir = temporary / "instructions"
contracts_dir = temporary / "contracts"
instructions_dir.mkdir(mode=0o755)
contracts_dir.mkdir(mode=0o755)
for relative, text in payload.items():
atomic_write_text(temporary / relative, text, 0o644)
manifest["payload_files"] = {
relative: sha256_bytes(text.encode("utf-8"))
for relative, text in sorted(payload.items())
}
atomic_write_json(temporary / "manifest.json", manifest, 0o644)
os.replace(temporary, destination)
published = True
make_tree_read_only(destination)
except Exception:
with contextlib.suppress(OSError):
_remove_snapshot_tree(temporary)
if published and (destination.exists() or destination.is_symlink()):
with contextlib.suppress(OSError):
_remove_snapshot_tree(destination)
raise
return load_snapshot(snapshot_hash)
def _snapshot_stat_fingerprint(
directory: Path, expected_files: Mapping[str, str]
) -> tuple[tuple[str, int, int, int, int, int], ...]:
rows: list[tuple[str, int, int, int, int, int]] = []
names = {".", *expected_files}
for relative in expected_files:
parent = Path(relative).parent
while parent != Path("."):
names.add(parent.as_posix())
parent = parent.parent
for relative in sorted(names):
path = directory if relative == "." else directory / relative
try:
status = path.lstat()
except OSError:
rows.append((relative, -1, -1, -1, -1, -1))
continue
rows.append(
(
relative,
int(status.st_size),
int(status.st_mtime_ns),
int(status.st_ctime_ns),
int(status.st_mode),
int(status.st_ino),
)
)
return tuple(rows)
def _stored_snapshot_guidance(
directory: Path,
manifest: Mapping[str, Any],
) -> dict[str, str]:
"""Load guidance bytes whose identity is already bound by the snapshot hash.
Guidance is generated documentation, so its prose may legitimately evolve
between MMO releases without changing an existing immutable snapshot. A
historical snapshot must therefore authenticate its stored guidance rather
than regenerate that prose with the currently installed compiler.
"""
payload_files = manifest.get("payload_files")
if not isinstance(payload_files, Mapping):
raise RuntimeError(f"snapshot payload manifest is missing in {directory}")
guidance: dict[str, str] = {}
for relative in sorted(payload_files):
if not isinstance(relative, str) or not relative.startswith("guidance/"):
continue
parsed = PurePosixPath(relative)
if (
parsed.is_absolute()
or parsed.as_posix() != relative
or any(part in {"", ".", ".."} for part in parsed.parts)
):
raise RuntimeError(f"snapshot guidance path is unsafe in {directory}: {relative!r}")
path = directory.joinpath(*parsed.parts)
try:
status = path.lstat()
except OSError as exc:
raise RuntimeError(f"snapshot guidance is unreadable: {path}") from exc
if not stat.S_ISREG(status.st_mode):
raise RuntimeError(f"snapshot guidance is not a regular file: {path}")
try:
guidance[relative] = path.read_text(encoding="utf-8")
except (OSError, UnicodeError) as exc:
raise RuntimeError(f"snapshot guidance is not valid UTF-8: {path}") from exc
if not guidance:
raise RuntimeError(f"snapshot guidance payload is missing in {directory}")
return guidance
def load_snapshot(snapshot_hash: str) -> dict[str, Any]:
directory = snapshot_dir(snapshot_hash)
for name in ("manifest.json", "resolved-profile.json"):
status = (directory / name).lstat()
if not stat.S_ISREG(status.st_mode):
raise RuntimeError(f"snapshot required file is not regular: {directory / name}")
manifest = read_json(directory / "manifest.json")
if not isinstance(manifest, Mapping):
raise RuntimeError(f"snapshot manifest is not an object in {directory}")
schema_version = manifest.get("schema_version")
if (
not isinstance(schema_version, int)
or isinstance(schema_version, bool)
or schema_version != MMO_SCHEMA_VERSION
):
raise RuntimeError(f"unsupported snapshot schema in {directory}")
if manifest.get("snapshot_hash") != snapshot_hash:
raise RuntimeError(f"snapshot manifest identity mismatch in {directory}")
resolved = read_json(directory / "resolved-profile.json")
if not isinstance(resolved, Mapping):
raise RuntimeError(f"snapshot resolved profile is not an object in {directory}")
resolved_schema = resolved.get("schema_version")
if (
not isinstance(resolved_schema, int)
or isinstance(resolved_schema, bool)
or resolved_schema != MMO_SCHEMA_VERSION
):
raise RuntimeError(f"unsupported resolved profile schema in {directory}")
guidance_schema_version = manifest.get("guidance_schema_version")
if not isinstance(guidance_schema_version, int) or isinstance(guidance_schema_version, bool):
raise RuntimeError(f"snapshot guidance schema is invalid in {directory}")
stored_guidance = _stored_snapshot_guidance(directory, manifest)
fingerprint = _snapshot_fingerprint(
resolved,
guidance=stored_guidance,
guidance_schema_version=guidance_schema_version,
)
expected_hash = stable_hash(fingerprint)
if expected_hash != snapshot_hash:
raise RuntimeError(
f"snapshot content-address mismatch in {directory}: expected {expected_hash}"
)
package_version = resolved.get("package_version")
profile = resolved.get("profile")
if not isinstance(package_version, str) or package_version != PACKAGE_VERSION:
raise RuntimeError(f"snapshot package version must be {PACKAGE_VERSION} in {directory}")
if not isinstance(profile, Mapping) or profile.get("version") != PACKAGE_VERSION:
raise RuntimeError(f"snapshot profile version must be {PACKAGE_VERSION} in {directory}")
expected_manifest, expected_payload = _snapshot_material(
resolved,
snapshot_hash,
package=package_version,
)
expected_manifest["guidance_schema_version"] = guidance_schema_version
expected_payload = {
relative: text
for relative, text in expected_payload.items()
if not relative.startswith("guidance/")
}
expected_payload.update(stored_guidance)
expected_files = {
relative: sha256_bytes(text.encode("utf-8"))
for relative, text in sorted(expected_payload.items())
}
complete_expected_manifest = {
**expected_manifest,
"payload_files": expected_files,
}
if manifest != complete_expected_manifest:
raise RuntimeError(f"snapshot manifest integrity failure in {directory}")
if not expected_files:
raise RuntimeError(f"snapshot payload manifest is missing in {directory}")
fingerprint_files = {"manifest.json": "", **expected_files}
stat_fingerprint = _snapshot_stat_fingerprint(directory, fingerprint_files)
if _VERIFIED_SNAPSHOT_STATS.get(snapshot_hash) != stat_fingerprint:
try:
actual_files = manifest_for_tree(directory, exclude=("manifest.json",))
except ValueError as exc:
raise RuntimeError(f"snapshot payload integrity failure in {directory}: {exc}") from exc
if actual_files != expected_files:
missing = sorted(set(expected_files) - set(actual_files))
unexpected = sorted(set(actual_files) - set(expected_files))
changed = sorted(
key
for key in set(actual_files) & set(expected_files)
if actual_files[key] != expected_files[key]
)
raise RuntimeError(
"snapshot payload integrity failure in "
f"{directory}: missing={missing}, unexpected={unexpected}, changed={changed}"
)
_VERIFIED_SNAPSHOT_STATS[snapshot_hash] = stat_fingerprint
return {
"directory": str(directory),
"manifest": manifest,
"resolved": resolved,
}
def find_snapshot_for_profile(profile_id: str) -> list[dict[str, Any]]:
results: list[dict[str, Any]] = []
for directory in snapshots_root().iterdir():
if not directory.is_dir() or len(directory.name) != 64:
continue
try:
snapshot = load_snapshot(directory.name)
manifest = snapshot["manifest"]
except (OSError, RuntimeError, ValueError, json.JSONDecodeError):
continue
if manifest.get("profile_id") == profile_id:
results.append(
{
"snapshot_hash": directory.name,
"profile_id": profile_id,
"profile_version": manifest.get("profile_version"),
"logical_hash": manifest.get("logical_hash"),
}
)
return sorted(results, key=lambda item: item["snapshot_hash"])
libexec/mmo_state.py
+659
View File
@@ -0,0 +1,659 @@
#!/usr/bin/env python3
"""Durable local session/job state paths, validation, and publication."""
from __future__ import annotations
import contextlib
import copy
import json
import os
import re
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from mmo_util import (
SAFE_JOB_ID,
append_jsonl,
atomic_write_json,
is_within,
process_alive,
process_group_alive,
process_matches,
read_json_object,
state_root,
terminate_process,
terminate_process_group,
utc_now,
)
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION
ACTIVE_JOB_STATUSES = {
"queued",
"starting",
"running",
"waiting",
"paused",
"detached",
"recovering",
"finalizing",
"cancelling",
}
RECOVERABLE_JOB_STATUSES = ACTIVE_JOB_STATUSES | {"suspended"}
ADMITTING_JOB_STATUSES = {"queued", "starting", "running", "waiting", "paused", "detached"}
TERMINAL_JOB_STATUSES = {
"completed",
"completed_with_warnings",
"failed",
"stopped",
"cancelled",
}
ADMITTING_SESSION_STATUSES = {"starting", "running", "detached", "paused", "suspended"}
ACTIVE_SESSION_STATUSES = {
"starting",
"running",
"detached",
"paused",
"suspended",
"finishing",
"stopping",
"cancelling",
}
TERMINAL_SESSION_STATUSES = {"completed", "stopped", "failed", "cancelled"}
ROOT_EXECUTION_HOSTS = frozenset({"app_server"})
RETIRED_SESSION_FIELDS = frozenset(
{
"root_execution_policy_enforced",
"root_rollout_path",
}
)
RETIRED_JOB_FIELDS = frozenset(
{
"execution_policy",
"renewal_quantum_seconds",
"max_active_work_seconds",
"active_work_cap_seconds",
"active_work_seconds",
"automatic_renewal_enabled",
"app_server_rollout_path",
}
)
def _validate_switchyard_identity(data: Mapping[str, Any], label: str) -> None:
if "switchyard_version" not in data:
raise ValueError(f"{label} has no Switchyard version identity")
version = data.get("switchyard_version")
if data.get("gateway_base_url") is None:
if version is not None:
raise ValueError(f"{label} has a Switchyard version without a gateway")
return
if not isinstance(version, str) or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) is None:
raise ValueError(f"{label} Switchyard version is invalid")
def root_mcp_token(session_id: str) -> str:
"""Read the canonical persisted root MCP capability for an active session."""
try:
data = read_json_object(
session_dir(session_id) / "capabilities.json",
label="session capabilities",
)
except FileNotFoundError as exc:
raise RuntimeError("root MCP caller capability is unavailable for this session") from exc
token = data.get("root")
if not isinstance(token, str) or not token:
raise RuntimeError("root MCP caller capability is invalid")
return token
def store_session_capabilities(
directory: Path,
*,
root_token: str,
native_tokens: Mapping[str, str],
) -> None:
"""Persist host-scoped MCP identities needed by detached app-server processes."""
if not root_token or not all(
isinstance(value, str) and value for value in native_tokens.values()
):
raise ValueError("session capabilities must be non-empty strings")
atomic_write_json(
directory / "capabilities.json",
{
"schema_version": MMO_SCHEMA_VERSION,
"root": root_token,
"native": dict(native_tokens),
},
0o600,
)
def load_session_capabilities(directory: Path) -> tuple[str, dict[str, str]]:
data = read_json_object(directory / "capabilities.json", label="session capabilities")
root_token = data.get("root")
native_tokens = data.get("native")
if (
data.get("schema_version") != MMO_SCHEMA_VERSION
or not isinstance(root_token, str)
or not root_token
or not isinstance(native_tokens, Mapping)
or not all(
isinstance(key, str) and isinstance(value, str) and value
for key, value in native_tokens.items()
)
):
raise ValueError("session capabilities are invalid")
return root_token, dict(native_tokens)
def root_mcp_capability_environment(session: Mapping[str, Any]) -> dict[str, str]:
"""Return the authenticated Agent-MCP identity for a root-owned process."""
session_id = str(session["session_id"])
return {
"MMO_ROOT_SESSION_ID": session_id,
"MMO_RUN_ID": str(session["current_run_id"]),
"MMO_CALLER_AGENT": str(session["root_agent"]),
"MMO_CALLER_TOKEN": root_mcp_token(session_id),
}
def revoke_session_capabilities(session_id: str) -> None:
"""Revoke the bearer identities for a terminal session."""
# A terminal session retains its immutable transcript and evidence, but its
# bearer credentials no longer authorize any control-plane operation.
(session_dir(session_id) / "capabilities.json").unlink(missing_ok=True)
def sessions_root() -> Path:
root = state_root() / "sessions"
root.mkdir(parents=True, exist_ok=True, mode=0o700)
return root
def jobs_root() -> Path:
root = state_root() / "jobs"
root.mkdir(parents=True, exist_ok=True, mode=0o700)
return root
def runtime_lock_path() -> Path:
return state_root() / ".runtime.lock"
def session_dir(session_id: str) -> Path:
if not SAFE_JOB_ID.fullmatch(session_id):
raise ValueError("invalid session id")
path = sessions_root() / session_id
if not path.is_dir():
raise FileNotFoundError(f"unknown session: {session_id}")
return path
def job_dir(job_id: str) -> Path:
if not SAFE_JOB_ID.fullmatch(job_id):
raise ValueError("invalid job id")
path = jobs_root() / job_id
if not path.is_dir():
raise FileNotFoundError(f"unknown agent job: {job_id}")
return path
def session_state_path(directory: Path) -> Path:
return directory / "session.json"
def job_state_path(directory: Path) -> Path:
return directory / "metadata.json"
def ensure_runs_root(directory: Path) -> Path:
root = directory / "runs"
if root.is_symlink():
raise RuntimeError("session runs directory cannot be a symlink")
root.mkdir(parents=True, exist_ok=True, mode=0o700)
if not is_within(root.resolve(), directory.resolve()):
raise RuntimeError("session runs directory escapes its logical session")
return root
def _run_path(directory: Path, run_id: str) -> Path:
if not SAFE_JOB_ID.fullmatch(run_id):
raise ValueError("invalid run id")
root = directory / "runs"
if root.is_symlink():
raise RuntimeError("session runs directory cannot be a symlink")
path = root / run_id / "run.json"
if not path.is_file():
raise FileNotFoundError(f"unknown session run: {run_id}")
return path
def session_lifecycle_lock_path(directory: Path) -> Path:
path = directory / "lifecycle.lock"
if directory.is_symlink() or path.is_symlink():
raise RuntimeError("persistent session lifecycle lock cannot traverse a symlink")
if not is_within(path.resolve(), directory.resolve()):
raise RuntimeError("persistent session lifecycle lock escapes its logical session")
return path
def job_control_lock_path(directory: Path) -> Path:
path = directory / "control.lock"
if directory.is_symlink() or path.is_symlink():
raise RuntimeError("persistent job control lock cannot traverse a symlink")
if not is_within(path.resolve(), directory.resolve()):
raise RuntimeError("persistent job control lock escapes its logical job")
return path
def _read_run_state(path: Path) -> dict[str, Any]:
session_directory = path.parent.parent.parent
if path.parent.parent.is_symlink() or path.parent.is_symlink() or path.is_symlink():
raise ValueError("session run state cannot traverse a symlink")
if not is_within(path.resolve(), session_directory.resolve()):
raise ValueError("session run state escapes its logical session")
data = read_json_object(path, label="session run state")
if data.get("schema_version") != MMO_SCHEMA_VERSION or isinstance(
data.get("schema_version"), bool
):
raise ValueError(f"unsupported session run schema; expected {MMO_SCHEMA_VERSION}")
if data.get("package_version") != PACKAGE_VERSION:
raise ValueError(f"persistent session run package must be {PACKAGE_VERSION}")
sequence = data.get("sequence")
if not isinstance(sequence, int) or isinstance(sequence, bool) or sequence != 1:
raise ValueError("immutable session run sequence must be exactly 1")
if data.get("kind") != "initial":
raise ValueError("immutable session run kind must be initial")
if data.get("status") not in ACTIVE_SESSION_STATUSES | TERMINAL_SESSION_STATUSES:
raise ValueError("session run status is invalid")
_validate_switchyard_identity(data, "persistent session run")
expected_id = path.parent.name
if data.get("run_id") != expected_id:
raise ValueError(
"session run identity mismatch: "
f"directory is {expected_id!r}, record is {data.get('run_id')!r}"
)
if data.get("session_id") != path.parent.parent.parent.name:
raise ValueError("session run belongs to another logical session")
return data
def load_session_run(session_id: str, run_id: str) -> dict[str, Any]:
return _read_run_state(_run_path(session_dir(session_id), run_id))
def iter_session_runs(session_id: str) -> list[dict[str, Any]]:
directory = session_dir(session_id)
root = directory / "runs"
if root.is_symlink():
raise RuntimeError("session runs directory cannot be a symlink")
if not root.is_dir():
return []
results: list[dict[str, Any]] = []
for candidate in sorted(root.iterdir(), reverse=True):
path = candidate / "run.json"
if not candidate.is_dir() or not path.is_file():
continue
try:
results.append(_read_run_state(path))
except (OSError, ValueError, json.JSONDecodeError) as exc:
raise RuntimeError(f"invalid session run state: {path}: {exc}") from exc
return results
def public_run(data: Mapping[str, Any]) -> dict[str, Any]:
keys = (
"run_id",
"package_version",
"session_id",
"sequence",
"kind",
"status",
"created_at",
"started_at",
"finished_at",
"root_pid",
"exit_code",
"gateway_base_url",
"switchyard_version",
"root_execution_host",
"root_goal_status",
"root_goal_tokens_used",
"root_goal_token_budget",
"error",
)
return {key: data.get(key) for key in keys if data.get(key) is not None}
_RUN_MIRROR_FIELDS = (
"status",
"started_at",
"finished_at",
"root_pid",
"root_pgid",
"root_start_token",
"root_process_group_isolated",
"exit_code",
"error",
"cancel_requested_at",
"transition_started_at",
"requested_terminal_status",
"gateway_base_url",
"gateway_pid",
"gateway_hash",
"gateway_routing_log_path",
"switchyard_version",
"route_availability",
"root_mcp_token_hash",
"native_token_hashes",
"root_execution_host",
"root_execution_mode",
"root_thread_id",
"root_thread_generation",
"root_thread_lineage",
"root_thread_transition",
"root_goal_status",
"root_goal_objective",
"root_goal_bootstrap_pending",
"root_goal_token_budget",
"root_max_goal_token_budget",
"root_goal_tokens_used",
"root_goal_time_used_seconds",
"root_stall_warning_seconds",
"root_last_progress_at",
"root_finalization_grace_seconds",
"root_finalization_started_at",
"root_app_server_lifecycle_timeout_seconds",
"root_pending_request_count",
"root_last_turn_id",
"root_turn_start_pending",
"root_finalizing",
"root_completion_deferred",
"root_completion_deferred_jobs",
)
def _validate_root_thread_lineage(data: Mapping[str, Any]) -> None:
generation = data.get("root_thread_generation")
lineage = data.get("root_thread_lineage")
transition = data.get("root_thread_transition")
if not isinstance(generation, int) or isinstance(generation, bool) or generation < 0:
raise ValueError("persistent session root thread generation is invalid")
if not isinstance(lineage, list):
raise ValueError("persistent session root thread lineage must be a list")
if len(lineage) != generation:
raise ValueError("persistent session root thread lineage disagrees with its generation")
seen: set[str] = set()
for index, raw in enumerate(lineage, 1):
if not isinstance(raw, Mapping):
raise ValueError("persistent session root thread lineage entry is invalid")
thread_id = raw.get("thread_id")
if (
raw.get("generation") != index
or isinstance(raw.get("generation"), bool)
or not isinstance(thread_id, str)
or not thread_id
or thread_id in seen
or not isinstance(raw.get("codex_session_id"), str)
or not raw.get("codex_session_id")
or not isinstance(raw.get("adopted_at"), str)
or not raw.get("adopted_at")
or not isinstance(raw.get("reason"), str)
or not raw.get("reason")
):
raise ValueError("persistent session root thread lineage entry is invalid")
seen.add(thread_id)
superseded_at = raw.get("superseded_at")
successor_id = raw.get("successor_thread_id")
terminal_entry = index == generation
if terminal_entry:
if superseded_at is not None or successor_id is not None:
raise ValueError("current root thread lineage entry cannot be superseded")
else:
successor = lineage[index]
if (
not isinstance(successor, Mapping)
or not isinstance(superseded_at, str)
or not superseded_at
or not isinstance(successor_id, str)
or successor_id != successor.get("thread_id")
):
raise ValueError("superseded root thread lineage entry is incomplete")
root_thread_id = data.get("root_thread_id")
if generation == 0:
if root_thread_id is not None:
raise ValueError("uninitialized root thread lineage has a current thread")
elif root_thread_id != lineage[-1].get("thread_id"):
raise ValueError("persistent session current root thread disagrees with its lineage")
if transition is not None:
if not isinstance(transition, Mapping):
raise ValueError("persistent session root thread transition is invalid")
if (
transition.get("from_thread_id") != root_thread_id
or not isinstance(transition.get("to_thread_id"), str)
or not transition.get("to_thread_id")
or transition.get("to_thread_id") in seen
or transition.get("generation") != generation + 1
or isinstance(transition.get("generation"), bool)
or not isinstance(transition.get("observed_at"), str)
or not transition.get("observed_at")
or not isinstance(transition.get("reason"), str)
or not transition.get("reason")
):
raise ValueError("persistent session root thread transition is inconsistent")
def mirror_active_run(directory: Path, session: Mapping[str, Any]) -> None:
run_id = session.get("current_run_id")
if not isinstance(run_id, str):
return
path = _run_path(directory, run_id)
run = _read_run_state(path)
for key in _RUN_MIRROR_FIELDS:
if key in session:
run[key] = copy.deepcopy(session[key])
else:
run.pop(key, None)
atomic_write_json(path, run)
def _validate_session_record(path: Path) -> dict[str, Any]:
data = read_json_object(path, label="session state")
retired = sorted(RETIRED_SESSION_FIELDS & set(data))
if retired:
raise ValueError("persistent session contains retired fields: " + ", ".join(retired))
if data.get("schema_version") != MMO_SCHEMA_VERSION or isinstance(
data.get("schema_version"), bool
):
raise ValueError(f"unsupported session state schema; expected {MMO_SCHEMA_VERSION}")
if data.get("package_version") != PACKAGE_VERSION:
raise ValueError(f"persistent session package must be {PACKAGE_VERSION}")
if data.get("profile_version") != PACKAGE_VERSION:
raise ValueError(f"persistent session profile must be {PACKAGE_VERSION}")
if data.get("session_kind") not in {"interactive", "noninteractive"}:
raise ValueError("persistent session kind must be interactive or noninteractive")
if data.get("root_execution_host") not in ROOT_EXECUTION_HOSTS:
raise ValueError("persistent session root execution host is invalid")
_validate_switchyard_identity(data, "persistent session")
_validate_root_thread_lineage(data)
socket_path = data.get("root_app_server_socket")
if not isinstance(socket_path, str) or not Path(socket_path).is_absolute():
raise ValueError("persistent session app-server socket path is invalid")
run_sequence = data.get("run_sequence")
if not isinstance(run_sequence, int) or isinstance(run_sequence, bool) or run_sequence != 1:
raise ValueError("immutable session run sequence must be exactly 1")
for key in ("current_run_id", "last_run_id"):
run_id = data.get(key)
if run_id is not None and (
not isinstance(run_id, str) or not SAFE_JOB_ID.fullmatch(run_id)
):
raise ValueError(f"persistent session {key} is invalid")
expected_id = path.parent.name
if data.get("session_id") != expected_id:
raise ValueError(
"session state identity mismatch: "
f"directory is {expected_id!r}, record is {data.get('session_id')!r}"
)
return data
def _validate_job_record(path: Path) -> dict[str, Any]:
data = read_json_object(path, label="job state")
retired = sorted(RETIRED_JOB_FIELDS & set(data))
if retired:
raise ValueError("persistent job contains retired fields: " + ", ".join(retired))
if data.get("schema_version") != MMO_SCHEMA_VERSION or isinstance(
data.get("schema_version"), bool
):
raise ValueError(f"unsupported job state schema; expected {MMO_SCHEMA_VERSION}")
if data.get("package_version") != PACKAGE_VERSION:
raise ValueError(f"persistent job package must be {PACKAGE_VERSION}")
expected_id = path.parent.name
if data.get("job_id") != expected_id:
raise ValueError(
"job state identity mismatch: "
f"directory is {expected_id!r}, record is {data.get('job_id')!r}"
)
return data
def read_session_record(directory: Path) -> dict[str, Any]:
"""Read and validate the session record owned by ``directory``."""
return _validate_session_record(session_state_path(directory))
def read_job_record(directory: Path) -> dict[str, Any]:
"""Read and validate the worker-job record owned by ``directory``."""
return _validate_job_record(job_state_path(directory))
def publish_session_record(
directory: Path,
session: Mapping[str, Any],
*,
mirror_run: bool,
) -> None:
"""Atomically publish session state and optionally refresh its active-run mirror."""
atomic_write_json(session_state_path(directory), dict(session))
if mirror_run:
mirror_active_run(directory, session)
def publish_initial_session_records(
directory: Path,
session: Mapping[str, Any],
run: Mapping[str, Any],
) -> None:
"""Publish a new run before making its canonical session record visible."""
run_id = run.get("run_id")
if not isinstance(run_id, str) or not SAFE_JOB_ID.fullmatch(run_id):
raise ValueError("initial session run id is invalid")
if run.get("session_id") != session.get("session_id"):
raise ValueError("initial session and run identities disagree")
run_directory = ensure_runs_root(directory) / run_id
if run_directory.is_symlink():
raise RuntimeError("initial session run directory cannot be a symlink")
run_directory.mkdir(mode=0o700)
atomic_write_json(run_directory / "run.json", dict(run))
publish_session_record(directory, session, mirror_run=False)
def publish_job_record(directory: Path, job: Mapping[str, Any]) -> None:
"""Atomically publish the canonical worker-job record."""
atomic_write_json(job_state_path(directory), dict(job))
def terminate_recorded_process_group(
data: Mapping[str, Any],
*,
prefix: str,
grace_seconds: float,
) -> None:
pid = data.get(f"{prefix}_pid")
start_token = data.get(f"{prefix}_start_token")
if (
not isinstance(pid, int)
or isinstance(pid, bool)
or pid <= 1
or pid == os.getpid()
or not isinstance(start_token, str)
):
return
if bool(data.get(f"{prefix}_process_group_isolated", True)):
pgid = data.get(f"{prefix}_pgid", pid)
if not isinstance(pgid, int) or isinstance(pgid, bool) or pgid <= 1 or pgid != pid:
return
if process_matches(pid, start_token) or (
not process_alive(pid) and process_group_alive(pgid)
):
terminate_process_group(pgid, grace_seconds=grace_seconds)
if process_group_alive(pgid):
raise RuntimeError(f"{prefix} process group {pgid} did not terminate")
return
if process_matches(pid, start_token):
terminate_process(pid, grace_seconds=grace_seconds)
if process_matches(pid, start_token):
raise RuntimeError(f"{prefix} process {pid} did not terminate")
def iter_session_records(*, strict: bool = True) -> list[dict[str, Any]]:
"""Enumerate validated session records without lifecycle side effects."""
results: list[dict[str, Any]] = []
for directory in sorted(sessions_root().iterdir(), reverse=True):
path = session_state_path(directory)
if not directory.is_dir() or not path.is_file():
continue
try:
results.append(_validate_session_record(path))
except (OSError, ValueError, json.JSONDecodeError) as exc:
if strict:
raise RuntimeError(
f"invalid session state blocks safe accounting: {path}: {exc}"
) from exc
continue
return results
def iter_job_records(*, strict: bool = True) -> list[dict[str, Any]]:
"""Enumerate validated worker-job records without lifecycle side effects."""
results: list[dict[str, Any]] = []
for directory in sorted(jobs_root().iterdir(), reverse=True):
path = job_state_path(directory)
if not directory.is_dir() or not path.is_file():
continue
try:
results.append(_validate_job_record(path))
except (OSError, ValueError, json.JSONDecodeError) as exc:
if strict:
raise RuntimeError(
f"invalid job state blocks safe accounting: {path}: {exc}"
) from exc
continue
return results
def append_audit(session_id: str, event: str, **data: Any) -> None:
directory = session_dir(session_id)
if not isinstance(data.get("run_id"), str):
with contextlib.suppress(OSError, ValueError, json.JSONDecodeError):
session = read_session_record(directory)
run_id = session.get("current_run_id") or session.get("last_run_id")
if isinstance(run_id, str):
data["run_id"] = run_id
append_jsonl(
directory / "audit.jsonl",
{"timestamp": utc_now(), "event": event, "session_id": session_id, **data},
)
libexec/mmo_tool_mcp.py
+524
View File
@@ -0,0 +1,524 @@
#!/usr/bin/env python3
"""Operator-owned tool MCP registry, validation, and Codex rendering.
Tool MCP servers expose third-party tools to Codex. They are deliberately
separate from the MMO Agent MCP server (``mmo_mesh``), which launches and
supervises profile participants.
"""
from __future__ import annotations
import math
import os
import re
import shutil
from collections.abc import Mapping
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit
from mmo_util import (
config_root,
parse_env_file,
read_toml,
valid_absolute_uri,
valid_http_header_value,
validate_id,
)
from mmo_version import MMO_SCHEMA_VERSION
TOOL_MCP_RESERVED_IDS = {"mmo_mesh"}
TOOL_MCP_TRANSPORTS = {"stdio", "streamable_http"}
TOOL_MCP_APPROVAL_MODES = {"auto", "prompt", "writes", "approve"}
_DOCUMENT_FIELDS = {"schema_version", "tool_mcp_servers"}
_COMMON_SERVER_FIELDS = {
"transport",
"enabled_tools",
"default_tools_approval_mode",
"startup_timeout_sec",
"tool_timeout_sec",
"supports_parallel_tool_calls",
"tools",
}
_STDIO_FIELDS = {"command", "args", "env", "env_vars", "cwd"}
_HTTP_FIELDS = {
"url",
"bearer_token_env_var",
"http_headers",
"env_http_headers",
}
_OAUTH_FIELDS = {"auth", "scopes", "oauth", "oauth_resource", "environment_id"}
_REGISTRY_OWNED_POLICY_FIELDS = {"enabled", "required", "disabled_tools"}
_TOOL_FIELDS = {"approval_mode"}
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_HEADER_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+")
_SENSITIVE_ENVIRONMENT_NAME = re.compile(
r"(?:^|_)(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|COOKIE|AUTH|PRIVATE)(?:_|$)",
re.IGNORECASE,
)
_SENSITIVE_HEADER_NAME = re.compile(
r"(?:^|-)(?:authorization|cookie|api-?key|key|token|secret|password|passwd|"
r"credential|auth|private)(?:-|$)",
re.IGNORECASE,
)
def tool_mcp_registry_root() -> Path:
"""Return the operator-owned directory containing registry fragments."""
return config_root() / "tool-mcp.d"
def _reject_unknown_fields(value: Mapping[str, Any], allowed: set[str], label: str) -> None:
unknown = sorted(set(value) - allowed)
if unknown:
raise ValueError(f"{label} has unknown fields: {', '.join(unknown)}")
def _nonempty_string(value: Any, label: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ValueError(f"{label} must be a non-empty string")
if any(ord(character) < 0x20 or ord(character) == 0x7F for character in value):
raise ValueError(f"{label} cannot contain control characters")
return value
def _string_list(value: Any, label: str, *, allow_empty: bool) -> list[str]:
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
raise ValueError(f"{label} must be an array of strings")
if not allow_empty and not value:
raise ValueError(f"{label} cannot be empty")
if any(not item.strip() for item in value):
raise ValueError(f"{label} cannot contain empty strings")
if len(value) != len(set(value)):
raise ValueError(f"{label} cannot contain duplicates")
return list(value)
def _environment_name(value: Any, label: str) -> str:
if not isinstance(value, str) or not _ENVIRONMENT_NAME.fullmatch(value):
raise ValueError(f"{label} must be a valid environment variable name")
return value
def _positive_seconds(value: Any, label: str) -> float:
if (
not isinstance(value, (int, float))
or isinstance(value, bool)
or not math.isfinite(float(value))
or not 0 < float(value) <= 172_800
):
raise ValueError(f"{label} must be a finite number between 0 and 172800")
return float(value)
def _string_map(value: Any, label: str) -> dict[str, str]:
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a table of strings")
result: dict[str, str] = {}
for raw_key, raw_value in value.items():
if not isinstance(raw_key, str) or not isinstance(raw_value, str):
raise ValueError(f"{label} must be a table of strings")
result[raw_key] = raw_value
return result
def _validate_stdio_environment(value: Any, label: str) -> dict[str, str]:
environment = _string_map(value, label)
result: dict[str, str] = {}
for name, raw_value in environment.items():
_environment_name(name, f"{label} key")
if _SENSITIVE_ENVIRONMENT_NAME.search(name):
raise ValueError(
f"{label}.{name} looks credential-bearing; use env_vars and credentials.env"
)
if "\x00" in raw_value:
raise ValueError(f"{label}.{name} cannot contain NUL")
result[name] = raw_value
return dict(sorted(result.items()))
def _validate_headers(value: Any, label: str, *, environment_backed: bool) -> dict[str, str]:
headers = _string_map(value, label)
result: dict[str, str] = {}
seen: set[str] = set()
for raw_name, raw_value in headers.items():
if not _HEADER_NAME.fullmatch(raw_name):
raise ValueError(f"{label} contains an invalid HTTP header name: {raw_name!r}")
normalized = raw_name.lower()
if normalized in seen:
raise ValueError(f"{label} contains duplicate case-insensitive header {raw_name!r}")
seen.add(normalized)
if environment_backed:
_environment_name(raw_value, f"{label}.{raw_name}")
else:
if _SENSITIVE_HEADER_NAME.search(raw_name):
raise ValueError(
f"{label}.{raw_name} is credential-bearing; use bearer_token_env_var "
"or env_http_headers"
)
if not valid_http_header_value(raw_value):
raise ValueError(f"{label}.{raw_name} contains an invalid HTTP header value")
result[raw_name] = raw_value
return dict(sorted(result.items(), key=lambda item: item[0].lower()))
def _validate_http_url(value: Any, label: str) -> str:
url = _nonempty_string(value, label)
if not valid_absolute_uri(url):
raise ValueError(f"{label} must be a valid HTTP(S) URL")
try:
parsed = urlsplit(url)
_ = parsed.port
except ValueError as exc:
raise ValueError(f"{label} must be a valid HTTP(S) URL") from exc
if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
raise ValueError(f"{label} must be a valid HTTP(S) URL")
if parsed.username is not None or parsed.password is not None:
raise ValueError(f"{label} cannot contain embedded credentials")
if parsed.fragment:
raise ValueError(f"{label} cannot contain a URL fragment")
return url
def _validate_stdio_command(value: Any, label: str) -> str:
command = _nonempty_string(value, label)
if "/" not in command and not command.startswith("~"):
return command
try:
path = Path(command).expanduser()
except RuntimeError as exc:
raise ValueError(f"{label} has an unknown home-directory user") from exc
if not path.is_absolute():
raise ValueError(f"{label} must be a PATH executable name or an absolute path")
return str(path.resolve())
def _validate_tool_policy(
value: Any,
*,
label: str,
enabled_tools: set[str],
) -> dict[str, dict[str, str]]:
if value is None:
return {}
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a table")
result: dict[str, dict[str, str]] = {}
for raw_name, raw_policy in value.items():
tool_name = _nonempty_string(raw_name, f"{label} tool name")
if tool_name not in enabled_tools:
raise ValueError(f"{label}.{tool_name} is not present in enabled_tools")
if not isinstance(raw_policy, Mapping):
raise ValueError(f"{label}.{tool_name} must be a table")
_reject_unknown_fields(raw_policy, _TOOL_FIELDS, f"{label}.{tool_name}")
approval = raw_policy.get("approval_mode")
if not isinstance(approval, str) or approval not in TOOL_MCP_APPROVAL_MODES:
raise ValueError(
f"{label}.{tool_name}.approval_mode must be one of "
f"{sorted(TOOL_MCP_APPROVAL_MODES)}"
)
result[tool_name] = {"approval_mode": approval}
return dict(sorted(result.items()))
def validate_tool_mcp_server(server_id: str, value: Any, *, label: str) -> dict[str, Any]:
"""Validate and normalize one operator-defined tool MCP server."""
server_id = validate_id(server_id, "tool MCP server id")
if server_id in TOOL_MCP_RESERVED_IDS:
raise ValueError(f"tool MCP server id {server_id!r} is reserved for Agent MCP")
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a table")
if "bearer_token" in value:
raise ValueError(f"{label}.bearer_token is forbidden; use bearer_token_env_var")
oauth_fields = sorted(set(value) & _OAUTH_FIELDS)
if oauth_fields:
raise ValueError(
f"{label} uses unsupported OAuth fields: {', '.join(oauth_fields)}; "
"the Tool MCP registry supports environment-backed authentication only"
)
policy_fields = sorted(set(value) & _REGISTRY_OWNED_POLICY_FIELDS)
if policy_fields:
raise ValueError(f"{label} cannot set agent-owned fields: {', '.join(policy_fields)}")
transport = value.get("transport")
if not isinstance(transport, str) or transport not in TOOL_MCP_TRANSPORTS:
raise ValueError(f"{label}.transport must be one of {sorted(TOOL_MCP_TRANSPORTS)}")
allowed = _COMMON_SERVER_FIELDS | (_STDIO_FIELDS if transport == "stdio" else _HTTP_FIELDS)
_reject_unknown_fields(value, allowed, label)
enabled_tools = _string_list(
value.get("enabled_tools"), f"{label}.enabled_tools", allow_empty=False
)
approval = value.get("default_tools_approval_mode")
if not isinstance(approval, str) or approval not in TOOL_MCP_APPROVAL_MODES:
raise ValueError(
f"{label}.default_tools_approval_mode must be one of {sorted(TOOL_MCP_APPROVAL_MODES)}"
)
normalized: dict[str, Any] = {
"transport": transport,
"enabled_tools": enabled_tools,
"default_tools_approval_mode": approval,
"supports_parallel_tool_calls": False,
}
if "supports_parallel_tool_calls" in value:
if not isinstance(value["supports_parallel_tool_calls"], bool):
raise ValueError(f"{label}.supports_parallel_tool_calls must be boolean")
normalized["supports_parallel_tool_calls"] = value["supports_parallel_tool_calls"]
for timeout_field in ("startup_timeout_sec", "tool_timeout_sec"):
if timeout_field in value:
normalized[timeout_field] = _positive_seconds(
value[timeout_field], f"{label}.{timeout_field}"
)
normalized["tools"] = _validate_tool_policy(
value.get("tools"),
label=f"{label}.tools",
enabled_tools=set(enabled_tools),
)
if transport == "stdio":
normalized["command"] = _validate_stdio_command(value.get("command"), f"{label}.command")
normalized["args"] = _string_list(value.get("args", []), f"{label}.args", allow_empty=True)
if any("\x00" in argument for argument in normalized["args"]):
raise ValueError(f"{label}.args cannot contain NUL")
normalized["env"] = _validate_stdio_environment(value.get("env", {}), f"{label}.env")
env_vars = _string_list(value.get("env_vars", []), f"{label}.env_vars", allow_empty=True)
for name in env_vars:
_environment_name(name, f"{label}.env_vars")
overlap = sorted(set(normalized["env"]) & set(env_vars))
if overlap:
raise ValueError(
f"{label} defines the same variables in env and env_vars: {', '.join(overlap)}"
)
normalized["env_vars"] = env_vars
if "cwd" in value:
raw_cwd = _nonempty_string(value["cwd"], f"{label}.cwd")
try:
cwd = Path(raw_cwd).expanduser()
except RuntimeError as exc:
raise ValueError(f"{label}.cwd has an unknown home-directory user") from exc
if not cwd.is_absolute():
raise ValueError(f"{label}.cwd must expand to an absolute path")
normalized["cwd"] = str(cwd.resolve())
else:
normalized["url"] = _validate_http_url(value.get("url"), f"{label}.url")
if "bearer_token_env_var" in value:
normalized["bearer_token_env_var"] = _environment_name(
value["bearer_token_env_var"], f"{label}.bearer_token_env_var"
)
normalized["http_headers"] = _validate_headers(
value.get("http_headers", {}),
f"{label}.http_headers",
environment_backed=False,
)
normalized["env_http_headers"] = _validate_headers(
value.get("env_http_headers", {}),
f"{label}.env_http_headers",
environment_backed=True,
)
static_names = {name.lower() for name in normalized["http_headers"]}
environment_names = {name.lower() for name in normalized["env_http_headers"]}
overlap = sorted(static_names & environment_names)
if overlap:
raise ValueError(
f"{label} defines headers in both http_headers and env_http_headers: "
+ ", ".join(overlap)
)
return normalized
def _load_tool_mcp_fragment(path: Path) -> dict[str, dict[str, Any]]:
data = read_toml(path)
_reject_unknown_fields(data, _DOCUMENT_FIELDS, f"tool MCP registry {path}")
schema = data.get("schema_version")
if not isinstance(schema, int) or isinstance(schema, bool) or schema != MMO_SCHEMA_VERSION:
raise ValueError(f"unsupported tool MCP registry schema_version in {path}")
raw_servers = data.get("tool_mcp_servers", {})
if not isinstance(raw_servers, Mapping):
raise ValueError(f"tool_mcp_servers must be a table in {path}")
return {
server_id: validate_tool_mcp_server(
server_id,
server,
label=f"tool MCP server {server_id} in {path}",
)
for server_id, server in raw_servers.items()
}
def load_tool_mcp_registry_with_sources() -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
"""Load deterministic operator fragments, replacing duplicate entries whole."""
registry: dict[str, dict[str, Any]] = {}
sources: dict[str, str] = {}
root = tool_mcp_registry_root()
if root.is_dir():
for path in sorted(root.glob("*.toml")):
for server_id, server in _load_tool_mcp_fragment(path).items():
registry[server_id] = server
sources[server_id] = str(path)
return dict(sorted(registry.items())), dict(sorted(sources.items()))
def load_tool_mcp_registry() -> dict[str, dict[str, Any]]:
return load_tool_mcp_registry_with_sources()[0]
def validate_tool_mcp_grants(
value: Any,
registry: Mapping[str, Mapping[str, Any]],
*,
label: str,
) -> dict[str, dict[str, Any]]:
"""Validate an agent's required/optional grants against operator maxima."""
if value is None:
return {}
if not isinstance(value, Mapping):
raise ValueError(f"{label} must be a table")
result: dict[str, dict[str, Any]] = {}
for raw_server_id, raw_grant in value.items():
server_id = validate_id(raw_server_id, "tool MCP server id")
if server_id not in registry:
raise ValueError(
f"{label}.{server_id} references an undefined operator tool MCP server; "
f"define it under {tool_mcp_registry_root()}"
)
if not isinstance(raw_grant, Mapping):
raise ValueError(f"{label}.{server_id} must be a table")
_reject_unknown_fields(
raw_grant,
{"required", "enabled_tools"},
f"{label}.{server_id}",
)
required = raw_grant.get("required", True)
if not isinstance(required, bool):
raise ValueError(f"{label}.{server_id}.required must be boolean")
maximum = list(registry[server_id]["enabled_tools"])
selected = (
maximum
if "enabled_tools" not in raw_grant
else _string_list(
raw_grant["enabled_tools"],
f"{label}.{server_id}.enabled_tools",
allow_empty=False,
)
)
outside_maximum = sorted(set(selected) - set(maximum))
if outside_maximum:
raise ValueError(
f"{label}.{server_id}.enabled_tools exceeds the operator allowlist: "
+ ", ".join(outside_maximum)
)
result[server_id] = {"required": required, "enabled_tools": selected}
return dict(sorted(result.items()))
def tool_mcp_environment_names(server: Mapping[str, Any]) -> list[str]:
"""Return process environment names referenced by a normalized definition."""
names: set[str] = set()
if server["transport"] == "stdio":
names.update(str(name) for name in server.get("env_vars", []))
else:
names.update(tool_mcp_http_environment_names(server))
return sorted(names)
def tool_mcp_http_environment_names(server: Mapping[str, Any]) -> list[str]:
"""Return environment names whose values become HTTP header material."""
if server["transport"] != "streamable_http":
return []
names = {str(name) for name in server.get("env_http_headers", {}).values()}
bearer = server.get("bearer_token_env_var")
if bearer:
names.add(str(bearer))
return sorted(names)
def codex_tool_mcp_server_config(
server: Mapping[str, Any],
grant: Mapping[str, Any] | None,
) -> dict[str, Any]:
"""Compile one normalized definition/grant to Codex's mcp_servers shape."""
transport = str(server["transport"])
fields = _STDIO_FIELDS if transport == "stdio" else _HTTP_FIELDS
result = {
key: server[key] for key in sorted(fields) if key in server and server[key] not in ({}, [])
}
maximum = list(server["enabled_tools"])
selected = set(grant["enabled_tools"] if grant is not None else ())
result.update(
{
"enabled": grant is not None,
"required": bool(grant["required"]) if grant is not None else False,
"supports_parallel_tool_calls": bool(server["supports_parallel_tool_calls"]),
"enabled_tools": maximum,
# Always emit this array so a native role can clear a narrower
# deny-list inherited from its parent config layer.
"disabled_tools": [tool for tool in maximum if tool not in selected],
"default_tools_approval_mode": server["default_tools_approval_mode"],
}
)
for timeout_field in ("startup_timeout_sec", "tool_timeout_sec"):
if timeout_field in server:
result[timeout_field] = server[timeout_field]
if server.get("tools"):
result["tools"] = server["tools"]
return result
def tool_mcp_readiness(
server_id: str,
server: Mapping[str, Any],
*,
sources: Mapping[str, str] | None = None,
) -> dict[str, Any]:
"""Return non-launching command, cwd, and environment readiness."""
values = parse_env_file(config_root() / "credentials.env")
values.update({key: value for key, value in os.environ.items() if value})
http_environment = set(tool_mcp_http_environment_names(server))
environment: dict[str, bool] = {}
invalid_environment: list[str] = []
for name in tool_mcp_environment_names(server):
value = values.get(name)
valid = bool(value) and "\x00" not in str(value)
if valid and name in http_environment:
valid = valid_http_header_value(str(value))
environment[name] = valid
if value and not valid:
invalid_environment.append(name)
result: dict[str, Any] = {
"id": server_id,
"source": (sources or {}).get(server_id),
"transport": server["transport"],
"enabled_tools": list(server["enabled_tools"]),
"default_tools_approval_mode": server["default_tools_approval_mode"],
"environment": environment,
"invalid_environment": invalid_environment,
}
if server["transport"] == "stdio":
command = str(server["command"])
command_path = Path(command).expanduser()
resolved = (
str(command_path.resolve())
if "/" in command and command_path.is_file() and os.access(command_path, os.X_OK)
else shutil.which(command)
)
result["command"] = command
result["resolved_command"] = resolved
cwd = server.get("cwd")
result["cwd"] = cwd
result["cwd_ready"] = cwd is None or Path(str(cwd)).is_dir()
result["transport_ready"] = bool(resolved) and bool(result["cwd_ready"])
else:
result["url"] = server["url"]
result["transport_ready"] = True
result["environment_ready"] = all(environment.values())
result["ready"] = bool(result["transport_ready"]) and bool(result["environment_ready"])
return result
libexec/mmo_util.py
Executable
+899
View File
@@ -0,0 +1,899 @@
#!/usr/bin/env python3
"""Low-level utilities shared by Codex MMO.
The Python control plane intentionally has no third-party package dependency;
documented external executables still apply to the selected execution path.
"""
from __future__ import annotations
import contextlib
import datetime as dt
import fcntl
import hashlib
import ipaddress
import json
import math
import os
import re
import shutil
import signal
import socket
import tempfile
import time
import tomllib
import urllib.error
import urllib.request
from collections.abc import Iterable, Iterator, Mapping, Sequence
from pathlib import Path
from typing import Any
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION
ID_PATTERN = re.compile(r"[a-z][a-z0-9_.-]{1,63}")
SAFE_JOB_ID = re.compile(r"[A-Za-z0-9_.-]{8,128}")
_URI_SCHEME = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*", re.ASCII)
_URI_UNRESERVED = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")
_URI_SUB_DELIMS = frozenset("!$&'()*+,;=")
_URI_HEXDIGITS = frozenset("0123456789ABCDEFabcdef")
_URI_PCHAR = _URI_UNRESERVED | _URI_SUB_DELIMS | frozenset(":@")
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
"""Keep readiness checks scoped to the configured endpoint response."""
def redirect_request(self, *args: Any, **kwargs: Any) -> None:
return None
def utc_now() -> str:
return dt.datetime.now(dt.UTC).isoformat(timespec="milliseconds")
def shell_exit_status(returncode: int) -> int:
"""Translate a Python child return code to the shell's signal convention."""
return 128 - returncode if returncode < 0 else returncode
def install_root() -> Path:
override = os.environ.get("MMO_INSTALL_ROOT")
if override:
return Path(override).expanduser().resolve()
return Path(__file__).resolve().parents[1]
def package_version() -> str:
return PACKAGE_VERSION
def install_runtime_path() -> Path:
return install_root() / "config" / "runtime.json"
def load_install_runtime() -> dict[str, Any]:
path = install_runtime_path()
try:
data = strict_json_loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise RuntimeError(f"installed runtime configuration is missing: {path}") from exc
if not isinstance(data, dict):
raise RuntimeError(f"installed runtime configuration root must be an object: {path}")
schema_version = data.get("schema_version")
if (
not isinstance(schema_version, int)
or isinstance(schema_version, bool)
or schema_version != MMO_SCHEMA_VERSION
):
raise RuntimeError(f"unsupported installed runtime schema in {path}")
for key in ("install_root", "config_root", "state_root", "bin_dir", "python_bin"):
value = data.get(key)
if not isinstance(value, str) or not value:
raise RuntimeError(f"installed runtime field {key!r} is invalid in {path}")
for key in ("install_root", "config_root", "state_root", "bin_dir"):
if not Path(data[key]).expanduser().is_absolute():
raise RuntimeError(f"installed runtime path {key!r} is not absolute in {path}")
return data
def valid_http_header_value(value: str) -> bool:
"""Match the byte validity rule used by Rust's ``http::HeaderValue``.
RFC 9110 permits HTAB plus visible/obsolete field-value bytes. The Rust
HTTP stack used by both Codex and Switchyard rejects every other C0 control
byte and DEL. Encoding first also rejects lone Python surrogates.
"""
try:
encoded = value.encode("utf-8")
except UnicodeEncodeError:
return False
return all(byte == 0x09 or (byte >= 0x20 and byte != 0x7F) for byte in encoded)
def _valid_uri_component(value: str, allowed: frozenset[str]) -> bool:
"""Validate one RFC 3986 component, including percent triplets."""
index = 0
while index < len(value):
character = value[index]
if character == "%":
if (
index + 2 >= len(value)
or value[index + 1] not in _URI_HEXDIGITS
or value[index + 2] not in _URI_HEXDIGITS
):
return False
index += 3
continue
if character not in allowed:
return False
index += 1
return True
def _valid_uri_authority(authority: str) -> bool:
"""Validate RFC 3986 authority syntax without performing DNS resolution."""
if authority.count("@") > 1:
return False
if "@" in authority:
userinfo, host_port = authority.split("@", 1)
if not _valid_uri_component(userinfo, _URI_UNRESERVED | _URI_SUB_DELIMS | frozenset(":")):
return False
else:
host_port = authority
if host_port.startswith("["):
closing = host_port.find("]")
if closing < 0:
return False
literal = host_port[1:closing]
remainder = host_port[closing + 1 :]
if remainder and not remainder.startswith(":"):
return False
if remainder[1:] and (
not remainder[1:].isascii()
or not all("0" <= character <= "9" for character in remainder[1:])
):
return False
ipv_future = re.fullmatch(
r"[Vv][0-9A-Fa-f]+\.[A-Za-z0-9._~!$&'()*+,;=:-]+",
literal,
re.ASCII,
)
if ipv_future is None:
try:
ipaddress.IPv6Address(literal)
except ValueError:
return False
return True
if "[" in host_port or "]" in host_port or host_port.count(":") > 1:
return False
if ":" in host_port:
host, port = host_port.rsplit(":", 1)
if port and (not port.isascii() or not all("0" <= character <= "9" for character in port)):
return False
else:
host = host_port
# A digit-and-dot string that is not an IPv4 address remains a valid
# reg-name. RFC 3986 deliberately gives a valid IPv4 address first-match
# precedence; it does not otherwise reserve that spelling.
return _valid_uri_component(host, _URI_UNRESERVED | _URI_SUB_DELIMS)
def valid_absolute_uri(value: str) -> bool:
"""Return whether ``value`` is an absolute URI under RFC 3986.
This is syntax validation only: it neither resolves hostnames nor imposes
scheme-specific semantics. URI references are intentionally rejected.
"""
if not isinstance(value, str):
return False
colon = value.find(":")
if colon <= 0 or _URI_SCHEME.fullmatch(value[:colon]) is None:
return False
remainder = value[colon + 1 :]
if "#" in remainder:
hierarchical, fragment = remainder.split("#", 1)
if not _valid_uri_component(fragment, _URI_PCHAR | frozenset("/?")):
return False
else:
hierarchical = remainder
if "?" in hierarchical:
hierarchical, query = hierarchical.split("?", 1)
if not _valid_uri_component(query, _URI_PCHAR | frozenset("/?")):
return False
if hierarchical.startswith("//"):
authority_and_path = hierarchical[2:]
slash = authority_and_path.find("/")
if slash < 0:
authority, path = authority_and_path, ""
else:
authority, path = authority_and_path[:slash], authority_and_path[slash:]
return _valid_uri_authority(authority) and _valid_uri_component(
path, _URI_PCHAR | frozenset("/")
)
# The remaining hier-part alternatives are path-absolute, path-rootless,
# and path-empty. A leading "//" was handled above; every non-empty
# rootless path therefore has the required non-empty first segment.
return _valid_uri_component(hierarchical, _URI_PCHAR | frozenset("/"))
def config_root(runtime: Mapping[str, Any] | None = None) -> Path:
override = os.environ.get("MMO_CONFIG_ROOT")
if override:
return Path(override).expanduser().resolve()
data = runtime or load_install_runtime()
return Path(str(data["config_root"])).expanduser().resolve()
def state_root(runtime: Mapping[str, Any] | None = None) -> Path:
override = os.environ.get("MMO_STATE_ROOT")
if override:
return Path(override).expanduser().resolve()
data = runtime or load_install_runtime()
return Path(str(data["state_root"])).expanduser().resolve()
def atomic_write_bytes(path: Path, data: bytes, mode: int = 0o600) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
with os.fdopen(fd, "wb") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
os.chmod(temporary, mode)
os.replace(temporary, path)
with contextlib.suppress(OSError):
directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
with contextlib.suppress(FileNotFoundError):
os.unlink(temporary)
def atomic_write_text(path: Path, text: str, mode: int = 0o600) -> None:
atomic_write_bytes(path, text.encode("utf-8"), mode)
def atomic_write_json(path: Path, data: Any, mode: int = 0o600) -> None:
atomic_write_text(
path,
json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False) + "\n",
mode,
)
def _reject_json_constant(value: str) -> Any:
raise ValueError(f"non-standard JSON constant: {value}")
def _finite_json_float(value: str) -> float:
parsed = float(value)
if not math.isfinite(parsed):
raise ValueError(f"JSON number is outside the supported finite range: {value}")
return parsed
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, item in pairs:
if key in result:
raise ValueError(f"duplicate JSON object member: {key!r}")
result[key] = item
return result
def validate_json_unicode(value: Any) -> None:
"""Reject strings that contain lone UTF-16 surrogate code points."""
if isinstance(value, str):
try:
value.encode("utf-8")
except UnicodeEncodeError as exc:
raise ValueError("JSON strings must contain valid Unicode scalar values") from exc
elif isinstance(value, list):
for item in value:
validate_json_unicode(item)
elif isinstance(value, Mapping):
for key, item in value.items():
validate_json_unicode(key)
validate_json_unicode(item)
def strict_json_decoder() -> json.JSONDecoder:
"""Build the decoder shared by full-document and embedded JSON parsing."""
return json.JSONDecoder(
parse_constant=_reject_json_constant,
parse_float=_finite_json_float,
object_pairs_hook=_unique_json_object,
)
def strict_json_loads(value: str | bytes | bytearray) -> Any:
"""Parse interoperable RFC 8259 JSON without ambiguous extensions."""
parsed = json.loads(
value,
parse_constant=_reject_json_constant,
parse_float=_finite_json_float,
object_pairs_hook=_unique_json_object,
)
validate_json_unicode(parsed)
return parsed
def read_json(path: Path) -> Any:
return strict_json_loads(path.read_text(encoding="utf-8"))
def read_json_object(path: Path, *, label: str = "JSON document") -> dict[str, Any]:
value = read_json(path)
if not isinstance(value, dict):
raise ValueError(f"{label} root must be an object: {path}")
return value
def read_toml(path: Path) -> dict[str, Any]:
try:
with path.open("rb") as handle:
value = tomllib.load(handle)
except tomllib.TOMLDecodeError as exc:
raise ValueError(f"invalid TOML in {path}: {exc}") from exc
if not isinstance(value, dict):
raise ValueError(f"TOML root must be a table: {path}")
return value
def canonical_json_bytes(value: Any) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def stable_hash(value: Any) -> str:
return sha256_bytes(canonical_json_bytes(value))
def validate_id(value: str, label: str = "identifier") -> str:
if not isinstance(value, str) or not ID_PATTERN.fullmatch(value):
raise ValueError(
f"invalid {label} {value!r}; use 2-64 lowercase letters, digits, '.', '_' or '-'"
)
return value
def safe_name(value: str) -> str:
cleaned = re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-.")
return cleaned[:80] or "item"
def toml_quote(value: str) -> str:
try:
value.encode("utf-8")
except UnicodeEncodeError as exc:
raise ValueError("generated TOML strings must contain valid Unicode scalar values") from exc
# JSON and TOML basic-string escapes overlap for the values emitted here,
# except that JSON permits a raw DEL character while TOML forbids it.
return json.dumps(value, ensure_ascii=False, allow_nan=False).replace("\x7f", "\\u007F")
def _toml_scalar(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, int):
return str(value)
if isinstance(value, float):
if value != value or value in (float("inf"), float("-inf")):
raise ValueError("non-finite floats are not supported in generated TOML")
return repr(value)
if isinstance(value, str):
return toml_quote(value)
if isinstance(value, list):
if any(isinstance(item, dict) for item in value):
raise TypeError("array-of-table values are emitted separately")
return "[" + ", ".join(_toml_scalar(item) for item in value) + "]"
if value is None:
raise TypeError("TOML has no null value")
raise TypeError(f"unsupported TOML scalar: {type(value).__name__}")
def toml_dumps(data: Mapping[str, Any]) -> str:
"""Serialize the subset of TOML used by this project deterministically."""
lines: list[str] = []
def emit_table(table: Mapping[str, Any], path: tuple[str, ...], header: bool) -> None:
scalar_items: list[tuple[str, Any]] = []
child_tables: list[tuple[str, Mapping[str, Any]]] = []
arrays_of_tables: list[tuple[str, list[Mapping[str, Any]]]] = []
for key in sorted(table):
value = table[key]
if value is None:
continue
if isinstance(value, Mapping):
child_tables.append((key, value))
elif (
isinstance(value, list)
and value
and all(isinstance(item, Mapping) for item in value)
):
arrays_of_tables.append((key, value))
else:
scalar_items.append((key, value))
if header:
if lines and lines[-1] != "":
lines.append("")
lines.append("[" + ".".join(toml_quote(part) for part in path) + "]")
for key, value in scalar_items:
lines.append(f"{toml_quote(key)} = {_toml_scalar(value)}")
for key, child in child_tables:
emit_table(child, path + (key,), True)
for key, values in arrays_of_tables:
for item in values:
if lines and lines[-1] != "":
lines.append("")
item_path = path + (key,)
lines.append("[[" + ".".join(toml_quote(part) for part in item_path) + "]]")
item_scalars = {
item_key: item_value
for item_key, item_value in item.items()
if not isinstance(item_value, Mapping)
}
nested = {
item_key: item_value
for item_key, item_value in item.items()
if isinstance(item_value, Mapping)
}
for item_key in sorted(item_scalars):
lines.append(f"{toml_quote(item_key)} = {_toml_scalar(item_scalars[item_key])}")
for nested_key, nested_value in sorted(nested.items()):
emit_table(nested_value, item_path + (nested_key,), True)
emit_table(data, (), False)
return "\n".join(lines).rstrip() + "\n"
def deep_merge(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
result: dict[str, Any] = dict(base)
for key, value in override.items():
existing = result.get(key)
if isinstance(existing, Mapping) and isinstance(value, Mapping):
result[key] = deep_merge(existing, value)
else:
result[key] = value
return result
def parse_env_file(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
if not path.is_file():
return values
for number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
line = raw.strip()
if not line or line.startswith("#"):
continue
if line.startswith("export "):
line = line[7:].lstrip()
if "=" not in line:
raise ValueError(f"invalid environment assignment at {path}:{number}")
key, value = line.split("=", 1)
key = key.strip()
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
raise ValueError(f"invalid environment variable at {path}:{number}: {key!r}")
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
quote = value[0]
value = value[1:-1]
if quote == '"':
decoded: list[str] = []
index = 0
escapes = {"n": "\n", "r": "\r", "t": "\t", '"': '"', "\\": "\\"}
while index < len(value):
if value[index] != "\\" or index + 1 >= len(value):
decoded.append(value[index])
index += 1
continue
escaped = value[index + 1]
if escaped in escapes:
decoded.append(escapes[escaped])
else:
# Preserve unknown escapes literally. This avoids the
# lossy/non-ASCII behavior of ``unicode_escape`` while
# remaining compatible with ordinary dotenv values.
decoded.extend(("\\", escaped))
index += 2
value = "".join(decoded)
values[key] = value
return values
def filtered_environment(
*,
allow_sensitive: Iterable[str] = (),
extra: Mapping[str, str] | None = None,
) -> dict[str, str]:
sensitive = re.compile(
r"(?:^|_)(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|COOKIE|AUTH|PRIVATE)(?:_|$)",
re.IGNORECASE,
)
allowed = set(allow_sensitive)
result: dict[str, str] = {}
for key, value in os.environ.items():
if key.startswith("MMO_"):
continue
if sensitive.search(key) and key not in allowed:
continue
result[key] = value
if extra:
result.update(extra)
return result
@contextlib.contextmanager
def file_lock(path: Path) -> Iterator[None]:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
with path.open("a+", encoding="utf-8") as handle:
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def _proc_stat_fields(value: str) -> list[str]:
"""Return Linux ``/proc/<pid>/stat`` fields beginning with process state.
The parenthesized command name may contain spaces and closing parentheses,
so tokenizing the complete record is not safe. The kernel's fields after
the final closing parenthesis have a stable whitespace-delimited layout.
"""
closing = value.rfind(")")
if closing < 0:
return []
return value[closing + 1 :].split()
def process_alive(pid: int | None) -> bool:
if not pid or pid <= 0:
return False
# kill(pid, 0) reports unreaped zombies as alive. On Linux this caused
# cancellation and gateway shutdown to consume the full grace period even
# though the process had already exited. Treat a /proc zombie as terminal.
stat = Path(f"/proc/{pid}/stat")
with contextlib.suppress(OSError):
fields = _proc_stat_fields(stat.read_text(encoding="ascii", errors="replace"))
if fields and fields[0] == "Z":
return False
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def process_start_token(pid: int | None) -> str | None:
"""Return the Linux process start-time token used to detect PID reuse."""
if not pid or pid <= 0:
return None
try:
value = Path(f"/proc/{pid}/stat").read_text(encoding="ascii", errors="replace")
fields = _proc_stat_fields(value)
return fields[19]
except (OSError, IndexError):
return None
def process_matches(pid: int | None, start_token: object = None) -> bool:
"""Return whether *pid* is alive and still denotes the recorded process."""
if start_token is None or not process_alive(pid):
return False
return process_start_token(pid) == str(start_token)
def process_group_alive(pgid: int | None) -> bool:
if not pgid or pgid <= 1:
return False
proc = Path("/proc")
if proc.is_dir():
inspected = False
for stat_path in proc.glob("[0-9]*/stat"):
try:
value = stat_path.read_text(encoding="ascii", errors="replace")
fields = _proc_stat_fields(value)
inspected = True
if int(fields[2]) == pgid and fields[0] != "Z":
return True
except (OSError, ValueError, IndexError):
continue
if inspected:
return False
try:
os.killpg(pgid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def terminate_process_group(pgid: int, grace_seconds: float = 8.0) -> None:
"""Terminate an isolated process group, including after its leader exits."""
if not process_group_alive(pgid):
return
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(pgid, signal.SIGTERM)
deadline = time.monotonic() + max(0.1, grace_seconds)
while time.monotonic() < deadline:
if not process_group_alive(pgid):
return
time.sleep(0.05)
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(pgid, signal.SIGKILL)
# Give the kernel a bounded interval to retire the process group. This is
# especially important for archive/release tests, which must not leave
# detached descendants behind after a forced cancellation.
kill_deadline = time.monotonic() + 2.0
while time.monotonic() < kill_deadline:
if not process_group_alive(pgid):
return
time.sleep(0.02)
def terminate_process(pid: int, grace_seconds: float = 8.0) -> None:
"""Terminate one process when it cannot safely own an isolated group."""
if not process_alive(pid):
return
with contextlib.suppress(ProcessLookupError, PermissionError):
os.kill(pid, signal.SIGTERM)
deadline = time.monotonic() + max(0.1, grace_seconds)
while time.monotonic() < deadline:
if not process_alive(pid):
return
time.sleep(0.05)
with contextlib.suppress(ProcessLookupError, PermissionError):
os.kill(pid, signal.SIGKILL)
kill_deadline = time.monotonic() + 2.0
while time.monotonic() < kill_deadline:
if not process_alive(pid):
return
time.sleep(0.02)
def is_within(path: Path, root: Path) -> bool:
try:
path.relative_to(root)
except ValueError:
return False
return True
def resolve_inside(value: str | Path, root: Path, *, must_exist: bool = False) -> Path:
candidate = Path(value).expanduser()
if not candidate.is_absolute():
candidate = root / candidate
resolved = candidate.resolve(strict=must_exist)
if not is_within(resolved, root):
raise ValueError(f"path escapes allowed root {root}: {value}")
return resolved
def copy_tree_static(source: Path, destination: Path) -> None:
"""Copy a static profile tree while rejecting links and special files."""
for path in sorted(source.rglob("*")):
relative = path.relative_to(source)
target = destination / relative
if path.is_symlink():
raise ValueError(f"symbolic links are not allowed in profile packs: {relative}")
if path.is_dir():
target.mkdir(parents=True, exist_ok=True, mode=0o755)
continue
if not path.is_file():
raise ValueError(f"special files are not allowed in profile packs: {relative}")
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(path, target)
os.chmod(target, 0o644)
def make_tree_read_only(root: Path) -> None:
for path in sorted(root.rglob("*"), reverse=True):
if path.is_dir():
os.chmod(path, 0o555)
elif path.is_file():
os.chmod(path, 0o444)
os.chmod(root, 0o555)
def http_json(url: str, *, timeout: float = 10.0) -> Any:
request = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(request, timeout=timeout) as response:
return strict_json_loads(response.read().decode("utf-8"))
def http_ready(url: str, *, timeout: float = 0.5) -> bool:
try:
request = urllib.request.Request(url, headers={"Accept": "application/json"})
opener = urllib.request.build_opener(_NoRedirectHandler())
with opener.open(request, timeout=timeout) as response:
return 200 <= response.status < 300
except urllib.error.HTTPError as exc:
exc.close()
return False
except (OSError, urllib.error.URLError, ValueError):
return False
def port_available(host: str, port: int) -> bool:
family = socket.AF_INET6 if ":" in host else socket.AF_INET
with socket.socket(family, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((host, port))
except OSError:
return False
return True
def append_jsonl(path: Path, value: Mapping[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
line = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + "\n"
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
try:
# O_APPEND makes each individual write atomic, but a short write would
# otherwise let another writer splice a record between retry chunks.
fcntl.flock(fd, fcntl.LOCK_EX)
data = line.encode("utf-8")
offset = 0
while offset < len(data):
written = os.write(fd, data[offset:])
if written <= 0:
raise OSError("unable to append complete JSONL record")
offset += written
os.fsync(fd)
finally:
with contextlib.suppress(OSError):
fcntl.flock(fd, fcntl.LOCK_UN)
os.close(fd)
def event_usage(path: Path) -> dict[str, int]:
"""Extract canonical token usage, preferring cumulative app-server totals."""
maxima: dict[str, int] = {}
app_server_totals: dict[str, int] = {}
event_count = 0
interesting = {
"input_tokens",
"output_tokens",
"cached_input_tokens",
"cache_write_input_tokens",
"reasoning_output_tokens",
"reasoning_tokens",
"total_tokens",
}
app_server_names = {
"inputTokens": "input_tokens",
"outputTokens": "output_tokens",
"cachedInputTokens": "cached_input_tokens",
"cacheWriteInputTokens": "cache_write_input_tokens",
"reasoningOutputTokens": "reasoning_output_tokens",
"totalTokens": "total_tokens",
}
def visit(value: Any) -> None:
if isinstance(value, Mapping):
for key, child in value.items():
if key in interesting and isinstance(child, int) and not isinstance(child, bool):
maxima[key] = max(maxima.get(key, 0), child)
visit(child)
elif isinstance(value, list):
for child in value:
visit(child)
if path.is_file():
with path.open("r", encoding="utf-8", errors="replace") as handle:
for line in handle:
try:
value = strict_json_loads(line)
except (json.JSONDecodeError, ValueError):
continue
if not isinstance(value, Mapping):
continue
event_count += 1
message = value.get("message")
if (
isinstance(message, Mapping)
and message.get("method") == "thread/tokenUsage/updated"
):
params = message.get("params")
token_usage = params.get("tokenUsage") if isinstance(params, Mapping) else None
total = token_usage.get("total") if isinstance(token_usage, Mapping) else None
if isinstance(total, Mapping):
# App-server reports a cumulative thread total. Notifications
# may be repeated during recovery, so summing them would
# double-count usage. Keep the greatest observed cumulative
# value for each canonical category.
for source, target in app_server_names.items():
count = total.get(source)
if isinstance(count, int) and not isinstance(count, bool):
app_server_totals[target] = max(
app_server_totals.get(target, 0), count
)
visit(value)
maxima.update(app_server_totals)
maxima["event_count"] = event_count
return maxima
def bounded_text(text: str, limit: int) -> tuple[str, bool]:
limit = max(0, limit)
if len(text) <= limit:
return text, False
marker = "\n\n...[truncated]...\n\n"
if limit <= len(marker):
return marker[:limit], True
payload_limit = limit - len(marker)
head = max(1, payload_limit * 2 // 3)
tail = payload_limit - head
return text[:head] + marker + (text[-tail:] if tail else ""), True
def walk_files(root: Path) -> list[Path]:
files: list[Path] = []
for path in root.rglob("*"):
if path.is_symlink():
raise ValueError(f"symbolic link is not allowed in manifest tree: {path}")
if path.is_file():
files.append(path)
elif not path.is_dir():
raise ValueError(f"special file is not allowed in manifest tree: {path}")
return sorted(files)
def manifest_for_tree(root: Path, *, exclude: Sequence[str] = ()) -> dict[str, str]:
excluded = set(exclude)
result: dict[str, str] = {}
for path in walk_files(root):
relative = path.relative_to(root).as_posix()
if relative in excluded:
continue
result[relative] = sha256_file(path)
return result
libexec/mmo_version.py
+37
View File
@@ -0,0 +1,37 @@
#!/usr/bin/env python3
"""Codex MMO package and development-schema identity."""
from __future__ import annotations
import re
from pathlib import Path
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
PACKAGE_VERSION = (PACKAGE_ROOT / "VERSION").read_text(encoding="utf-8").strip()
_VERSION_MATCH = re.fullmatch(
r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)",
PACKAGE_VERSION,
)
if _VERSION_MATCH is None:
raise RuntimeError("VERSION must contain MAJOR.MINOR.PATCH")
# During active development every MMO-owned serialized format advances as one
# deliberately breaking generation. Independently versioned upstream formats
# (Codex app-server, MCP, and Switchyard) do not use this value.
MMO_SCHEMA_VERSION = int(_VERSION_MATCH.group(1))
# Codex app-server schemas are generated artifacts tied to one exact CLI
# release. Runtime admission and the optional installer deliberately share
# this single pin so an installer upgrade cannot silently outrun the reviewed
# wire contract.
APP_SERVER_PROTOCOL_CODEX_VERSION = "0.149.0"
# Switchyard's generated route schema and optional installer are reviewed
# against one baseline release. The MCP namespace bridge is a separate,
# deliberately literal compatibility pin. Upstream fix NVIDIA-NeMo/Switchyard
# #384 (commit c7beccd4891fa5cfe3a3b94fdd376f5765864507) is merged but is not in
# v0.2.0. The first published switchyard-server release containing that commit
# is the removal trigger. When the baseline advances, the guard test requires
# this shim to be removed instead of silently carrying it into that release.
SWITCHYARD_BASELINE_VERSION = "0.2.0"
SWITCHYARD_MCP_NAMESPACE_BRIDGE_VERSION = "0.2.0"
libexec/mmo_workspace.py
+335
View File
@@ -0,0 +1,335 @@
#!/usr/bin/env python3
"""Git workspace isolation, scope fingerprinting, and patch transport."""
from __future__ import annotations
import contextlib
import hashlib
import mimetypes
import os
import subprocess
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Any
from mmo_util import atomic_write_bytes
class WorkspaceTargetNotGit(RuntimeError):
"""A writable worker target has no Git repository to isolate."""
def _git(
cwd: Path, *args: str, env: Mapping[str, str] | None = None
) -> subprocess.CompletedProcess[bytes]:
process_env = os.environ.copy()
if env:
process_env.update(env)
return subprocess.run(
["git", "-C", str(cwd), *args],
stdin=subprocess.DEVNULL,
capture_output=True,
check=False,
env=process_env,
)
def _git_root(cwd: Path) -> Path | None:
result = _git(cwd, "rev-parse", "--show-toplevel")
if result.returncode != 0:
return None
root = Path(os.fsdecode(result.stdout.removesuffix(b"\n"))).resolve()
return root if root.is_dir() else None
def _content_fingerprint(path: Path) -> str:
digest = hashlib.sha256()
try:
status = path.lstat()
except FileNotFoundError:
return "missing"
digest.update(f"{status.st_mode:o}\0{status.st_size}\0".encode())
if path.is_symlink():
digest.update(os.fsencode(os.readlink(path)))
elif path.is_file():
with path.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _scope_fingerprints(repo_root: Path, cwd: Path, scopes: Sequence[str]) -> dict[str, Any]:
repo_scopes: list[str] = []
for scope in scopes:
absolute = (cwd / scope).resolve(strict=False)
try:
repo_scopes.append(absolute.relative_to(repo_root).as_posix())
except ValueError as exc:
raise ValueError(f"write scope escapes Git repository: {scope}") from exc
listed = _git(
repo_root,
"ls-files",
"-co",
"--exclude-standard",
"-z",
"--",
*repo_scopes,
)
if listed.returncode != 0:
raise RuntimeError(
"unable to fingerprint write scope: "
+ listed.stderr.decode("utf-8", errors="replace")[-2000:]
)
paths = sorted(
{os.fsdecode(value) for value in listed.stdout.split(b"\0") if value} | set(repo_scopes)
)
return {
"repo_root": str(repo_root),
"scope_roots": repo_scopes,
"paths": {relative: _content_fingerprint(repo_root / relative) for relative in paths},
}
def create_isolated_worktree(
canonical_cwd: Path,
directory: Path,
scopes: Sequence[str],
attachments: Sequence[str],
) -> dict[str, Any]:
repo_root = _git_root(canonical_cwd)
if repo_root is None:
raise WorkspaceTargetNotGit("writable MCP workers require a Git worktree target")
try:
relative_cwd = canonical_cwd.relative_to(repo_root)
except ValueError as exc:
raise RuntimeError("job cwd is outside its reported Git root") from exc
index_path = directory / "synthetic.index"
index_env = {"GIT_INDEX_FILE": str(index_path)}
head = _git(repo_root, "rev-parse", "--verify", "HEAD")
if head.returncode == 0:
read_tree = _git(repo_root, "read-tree", "HEAD", env=index_env)
else:
read_tree = _git(repo_root, "read-tree", "--empty", env=index_env)
if read_tree.returncode != 0:
raise RuntimeError("unable to initialize synthetic Git index")
add = _git(repo_root, "add", "-A", "--", ".", env=index_env)
if add.returncode != 0:
raise RuntimeError(
"unable to snapshot Git workspace: "
+ add.stderr.decode("utf-8", errors="replace")[-2000:]
)
tree = _git(repo_root, "write-tree", env=index_env)
if tree.returncode != 0:
raise RuntimeError("unable to write synthetic Git tree")
# ``git commit-tree`` reads its log message from stdin unless one is
# supplied. MMO normally runs beneath an interactive Codex TUI, so
# inheriting stdin here can block a writable worker forever waiting for
# operator input. Keep the synthetic snapshot non-interactive and
# deterministic.
commit_args = [
"commit-tree",
tree.stdout.decode("ascii").strip(),
"-m",
"Codex MMO isolated workspace snapshot",
]
if head.returncode == 0:
commit_args.extend(["-p", head.stdout.decode("ascii").strip()])
commit = _git(
repo_root,
*commit_args,
env={
"GIT_AUTHOR_NAME": "Codex MMO",
"GIT_AUTHOR_EMAIL": "codex-mmo@localhost",
"GIT_COMMITTER_NAME": "Codex MMO",
"GIT_COMMITTER_EMAIL": "codex-mmo@localhost",
},
)
with contextlib.suppress(OSError):
index_path.unlink()
if commit.returncode != 0:
raise RuntimeError(
"unable to create synthetic Git commit: "
+ commit.stderr.decode("utf-8", errors="replace")[-2000:]
)
base_commit = commit.stdout.decode("ascii").strip()
worktree_root = directory / "worktree"
added = _git(repo_root, "worktree", "add", "--detach", str(worktree_root), base_commit)
if added.returncode != 0:
raise RuntimeError(
"unable to create isolated Git worktree: "
+ added.stderr.decode("utf-8", errors="replace")[-2000:]
)
execution_cwd = (worktree_root / relative_cwd).resolve()
mapped_attachments: list[str] = []
for raw in attachments:
canonical = Path(raw).resolve()
mapped_attachments.append(str(worktree_root / canonical.relative_to(repo_root)))
return {
"canonical_cwd": str(canonical_cwd),
"canonical_repo_root": str(repo_root),
"worktree_root": str(worktree_root),
"cwd": execution_cwd,
"attachments": mapped_attachments,
"base_commit": base_commit,
"base_fingerprints": _scope_fingerprints(repo_root, canonical_cwd, scopes),
}
def remove_isolated_worktree(metadata: Mapping[str, Any]) -> None:
root_value = metadata.get("canonical_repo_root")
worktree_value = metadata.get("worktree_root")
if not isinstance(root_value, str) or not isinstance(worktree_value, str):
return
_git(Path(root_value), "worktree", "remove", "--force", worktree_value)
def _nul_paths(result: subprocess.CompletedProcess[bytes]) -> set[str]:
if result.returncode != 0:
return set()
return {os.fsdecode(value) for value in result.stdout.split(b"\0") if value}
def _relative_to_job_cwd(repo_root: Path, job_cwd: Path, repo_relative: str) -> str | None:
absolute = (repo_root / repo_relative).resolve(strict=False)
try:
return absolute.relative_to(job_cwd).as_posix()
except ValueError:
return None
def path_within_scope(relative: str, scopes: list[str]) -> bool:
path = Path(relative)
for raw in scopes:
scope = Path(raw)
if raw == "." or path == scope:
return True
with contextlib.suppress(ValueError):
path.relative_to(scope)
return True
return False
def capture_isolated_patch(
metadata: dict[str, Any], directory: Path, cwd: Path
) -> tuple[list[str], list[dict[str, Any]], dict[str, Any] | None]:
"""Capture one scope-checked binary patch from a disposable worker worktree."""
worktree_value = metadata.get("worktree_root")
base_commit = metadata.get("base_commit")
if not isinstance(worktree_value, str) or not isinstance(base_commit, str):
return ["writable worker lacks isolated worktree metadata"], [], None
worktree_root = Path(worktree_value)
staged = _git(worktree_root, "add", "-A", "--", ".")
if staged.returncode != 0:
return ["unable to stage isolated worker changes"], [], None
names = _git(
worktree_root,
"diff",
"--cached",
"--no-renames",
"--name-only",
"-z",
base_commit,
"--",
)
if names.returncode != 0:
return ["unable to enumerate isolated worker changes"], [], None
changed = sorted(_nul_paths(names))
violations: list[str] = []
relative_changes: list[tuple[str, str]] = []
scopes = list(metadata.get("write_scope", []))
for repo_relative in changed:
relative = _relative_to_job_cwd(worktree_root, cwd, repo_relative)
if relative is None or not path_within_scope(relative, scopes):
violations.append(repo_relative)
else:
relative_changes.append((repo_relative, relative))
if violations:
return ["out-of-scope mutation: " + ", ".join(violations[:40])], [], None
patch_result = _git(
worktree_root,
"diff",
"--cached",
"--binary",
"--full-index",
"--no-renames",
base_commit,
"--",
)
if patch_result.returncode != 0:
return ["unable to produce isolated binary patch"], [], None
patch_path = directory / "changes.patch"
atomic_write_bytes(patch_path, patch_result.stdout, 0o600)
artifacts: list[dict[str, Any]] = []
for repo_relative, relative in relative_changes:
path = worktree_root / repo_relative
if not path.is_file():
artifacts.append(
{
"relative_path": relative,
"sha256": hashlib.sha256(b"").hexdigest(),
"size": 0,
"media_type": "application/x-deleted",
"state": "deleted",
}
)
continue
content = path.read_bytes()
media_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
artifacts.append(
{
"relative_path": relative,
"sha256": hashlib.sha256(content).hexdigest(),
"size": len(content),
"media_type": media_type,
"state": "present",
}
)
patch = {
"path": str(patch_path),
"sha256": hashlib.sha256(patch_result.stdout).hexdigest(),
"size": len(patch_result.stdout),
"base_commit": base_commit,
"base_fingerprints": metadata.get("base_fingerprints"),
"changed_paths": [relative for _repo, relative in relative_changes],
}
return [], artifacts, patch
def apply_validated_patch(
*,
canonical_cwd: Path,
repo_root: Path,
scopes: Sequence[str],
base_fingerprints: Any,
patch_path: Path,
) -> None:
"""Verify a worker snapshot boundary and apply its already-authenticated patch."""
if _git_root(canonical_cwd) != repo_root:
raise RuntimeError("canonical Git repository identity changed before integration")
current = _scope_fingerprints(repo_root, canonical_cwd, scopes)
if current != base_fingerprints:
raise RuntimeError(
"canonical write scope changed after worker snapshot; integration refused"
)
check = _git(repo_root, "apply", "--check", "--binary", str(patch_path))
if check.returncode != 0:
raise RuntimeError(
"git apply --check rejected worker patch: "
+ check.stderr.decode("utf-8", errors="replace")[-2000:]
)
applied = _git(repo_root, "apply", "--binary", str(patch_path))
if applied.returncode != 0:
raise RuntimeError(
"worker patch integration failed: "
+ applied.stderr.decode("utf-8", errors="replace")[-2000:]
)
def reverse_applied_patch(repo_root: Path, patch_path: Path) -> None:
"""Reverse one patch after its lifecycle publication fails."""
rollback = _git(repo_root, "apply", "--reverse", "--binary", str(patch_path))
if rollback.returncode != 0:
raise RuntimeError(rollback.stderr.decode("utf-8", errors="replace")[-2000:])
libexec/mmoctl.py
Executable
+1820
View File
File diff suppressed because it is too large Load Diff
libexec/root_runner.py
+1848
View File
File diff suppressed because it is too large Load Diff
libexec/worker_runner.py
Executable
+2421
View File
File diff suppressed because it is too large Load Diff