vance/codex-mmo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
53df0eed107973a483cfadf2b62355f5d6e926e0
codex-mmo/libexec/worker_runner.py
T
vance 53df0eed10 😏
2026-08-24 08:11:59 -07:00

2422 lines
97 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
"""Detached runner for one generic, profile-pinned Codex MMO participant."""
from __future__ import annotations
import atexit
import contextlib
import hashlib
import json
import mimetypes
import os
import signal
import socket
import sys
import threading
import time
from collections.abc import Callable, Mapping
from pathlib import Path
from typing import Any
from mmo_app_server import (
APP_SERVER_INITIALIZE_TIMEOUT_SECONDS,
APP_SERVER_LIFECYCLE_TIMEOUT_SECONDS,
APP_SERVER_RECOVERY_DELAYS_SECONDS,
AppServerClient,
AppServerError,
PersistentThreadHost,
app_server_listen_command,
app_server_socket_path,
bounded_goal_objective,
normalize_turn_failure,
receive_control_request,
require_app_server_codex_version,
retain_partial_evidence,
send_control_response,
validate_server_request_response,
)
from mmo_app_server import (
last_agent_message as _last_agent_message,
)
from mmo_app_server import (
turn_input as _turn_input,
)
from mmo_codex_home import session_environment
from mmo_gateway import route_telemetry, route_telemetry_warnings
from mmo_runtime import taint_session
from mmo_schema import extract_json_document, validate_instance
from mmo_state import (
append_audit,
publish_job_record,
read_job_record,
read_session_record,
runtime_lock_path,
session_dir,
terminate_recorded_process_group,
)
from mmo_util import (
atomic_write_json,
event_usage,
file_lock,
package_version,
process_matches,
process_start_token,
shell_exit_status,
strict_json_loads,
utc_now,
)
from mmo_version import SWITCHYARD_MCP_NAMESPACE_BRIDGE_VERSION
from mmo_workspace import capture_isolated_patch, path_within_scope, remove_isolated_worktree
_TERMINAL_OR_CANCELLING_STATUSES = {
"cancelling",
"completed",
"completed_with_warnings",
"failed",
"stopped",
"cancelled",
}
_CODEX_UNSUPPORTED_OUTPUT_SCHEMA_KEYWORDS = frozenset(
{
"$id",
"$schema",
"allOf",
"else",
"if",
"not",
"oneOf",
"then",
"uniqueItems",
}
)
_CODEX_OUTPUT_SCHEMA_FORMATS = frozenset(
{
"date",
"date-time",
"duration",
"email",
"hostname",
"ipv4",
"ipv6",
"time",
"uuid",
}
)
class WorkerRunner:
"""Own the process-scoped stop signal and one durable worker app-server client."""
def __init__(self, directory: Path) -> None:
self.directory = directory
self.stop_requested = False
self.app_server: AppServerClient | None = None
def install_signal_handlers(self) -> None:
signal.signal(signal.SIGTERM, self.handle_signal)
signal.signal(signal.SIGINT, self.handle_signal)
def handle_signal(self, _signum: int, _frame: object) -> None:
self.stop_requested = True
# Do not acquire AppServerClient locks or wait from a Python signal
# handler. Retire the separately isolated app-server group so any blocked
# request wakes; the normal runner path retains evidence and publishes the
# cancelled terminal state.
client = self.app_server
if client is not None and client.pid is not None:
with contextlib.suppress(ProcessLookupError, PermissionError):
os.killpg(client.pid, signal.SIGTERM)
def terminate_child(self) -> None:
client = self.app_server
if client is not None:
if client.process is not None:
client.stop_host(grace_seconds=8.0)
else:
client.close()
with contextlib.suppress(OSError, RuntimeError, ValueError):
terminate_recorded_process_group(
_read_metadata(self.directory),
prefix="app_server",
grace_seconds=8.0,
)
def run(self) -> int:
return _run_worker(self)
def _read_metadata(directory: Path) -> dict[str, Any]:
return read_job_record(directory)
def _codex_output_schema_errors(schema: Any) -> list[str]:
"""Return reasons an MMO contract cannot safely be sent to Codex as an output schema."""
errors: list[str] = []
if not isinstance(schema, dict) or schema.get("type") != "object":
return ["$: Codex output schemas must have an object root"]
def visit(value: Any, path: str) -> None:
if not isinstance(value, dict):
errors.append(f"{path}: Codex output schemas require object subschemas")
return
for keyword in sorted(_CODEX_UNSUPPORTED_OUTPUT_SCHEMA_KEYWORDS & value.keys()):
errors.append(f"{path}: Codex output schemas do not support {keyword}")
schema_format = value.get("format")
if schema_format is not None and schema_format not in _CODEX_OUTPUT_SCHEMA_FORMATS:
errors.append(f"{path}.format: Codex output schemas do not support {schema_format!r}")
schema_type = value.get("type")
schema_types = set(schema_type) if isinstance(schema_type, list) else {schema_type}
object_schema = "object" in schema_types or "properties" in value
if object_schema:
properties = value.get("properties")
required = value.get("required")
if not isinstance(properties, dict):
errors.append(f"{path}.properties: Codex object schemas require properties")
elif not isinstance(required, list) or set(required) != set(properties):
errors.append(f"{path}: every Codex object-schema property must be required")
if value.get("additionalProperties") is not False:
errors.append(f"{path}: Codex object schemas require additionalProperties=false")
if isinstance(properties, dict):
for key, child in properties.items():
visit(child, f"{path}.properties.{key}")
if "items" in value:
visit(value["items"], f"{path}.items")
variants = value.get("anyOf")
if isinstance(variants, list):
for index, child in enumerate(variants):
visit(child, f"{path}.anyOf[{index}]")
visit(schema, "$")
return errors
def _codex_output_schema(schema: dict[str, Any]) -> dict[str, Any] | None:
"""Project a full MMO contract onto Codex's strict-output schema subset.
The complete contract remains in the prompt and is always applied by MMO's
final validator. This projection removes validation-only keywords that the
app server cannot accept, while preserving the document shape, scalar
types, bounds, and enums that make malformed JSON much less likely.
"""
dropped = {
"$id",
"$schema",
"allOf",
"else",
"if",
"not",
"then",
"uniqueItems",
}
def project(value: Any) -> Any:
if isinstance(value, list):
return [project(item) for item in value]
if not isinstance(value, dict):
return value
result: dict[str, Any] = {}
for key, child in value.items():
if key in dropped:
continue
if key == "format" and child not in _CODEX_OUTPUT_SCHEMA_FORMATS:
continue
target_key = "anyOf" if key == "oneOf" else key
result[target_key] = project(child)
properties = result.get("properties")
schema_type = result.get("type")
schema_types = set(schema_type) if isinstance(schema_type, list) else {schema_type}
if isinstance(properties, dict) or "object" in schema_types:
if not isinstance(properties, dict):
return result
required = value.get("required")
required_names = set(required) if isinstance(required, list) else set()
# Codex requires every advertised property to be required. Omitting
# optional properties from the transport projection preserves the
# original contract's omission semantics instead of coercing a
# model into inventing a value merely to satisfy transport shape.
properties = {key: child for key, child in properties.items() if key in required_names}
result["properties"] = properties
result["additionalProperties"] = False
result["required"] = list(properties)
return result
projected = project(schema)
if not isinstance(projected, dict) or _codex_output_schema_errors(projected):
return None
return projected
def update(directory: Path, **changes: Any) -> dict[str, Any]:
# The supervisor and detached runner may both publish metadata during the
# launch/cancellation boundary. Serialize updates through the same lock as
# admission so neither side can revert a newer state with stale data.
with file_lock(runtime_lock_path()):
data = _read_metadata(directory)
# Reader notifications and control replies are asynchronous. They may
# arrive after an operator has closed admission or after final result
# publication; never let a late "running"/"waiting" update resurrect
# a cancelling or terminal worker.
if (
data.get("status") in _TERMINAL_OR_CANCELLING_STATUSES
and changes.get("status") != "cancelled"
):
changes.pop("status", None)
data.update(changes)
publish_job_record(directory, data)
return data
def _settle_recovery_control(
directory: Path,
state: Mapping[str, Any],
*,
status: str,
error: str | None = None,
) -> bool:
"""CAS one suspended-host relaunch acknowledgement against its exact revision."""
if status not in {"applied", "failed", "delivery_unknown"}:
raise ValueError(f"invalid recovery-control status: {status}")
revision = state.get("recovery_control_revision")
action = state.get("recovery_action")
if (
not isinstance(revision, int)
or isinstance(revision, bool)
or action not in {"continue", "finalize"}
):
return False
with file_lock(runtime_lock_path()):
current = _read_metadata(directory)
if (
current.get("control_revision") != revision
or current.get("last_control_action") != action
or current.get("last_control_status") != "pending"
):
return False
current["last_control_status"] = status
if error is None:
current.pop("last_control_error", None)
else:
current["last_control_error"] = error
publish_job_record(directory, current)
return True
def finalize(
directory: Path,
*,
runner: WorkerRunner | None = None,
**changes: Any,
) -> tuple[dict[str, Any], bool]:
"""Publish one terminal state atomically; cancellation is authoritative."""
with file_lock(runtime_lock_path()):
data = _read_metadata(directory)
cancelled = data.get("status") in {"cancelling", "cancelled"} or bool(
runner and runner.stop_requested
)
if (
data.get("status")
in {
"completed",
"completed_with_warnings",
"failed",
"stopped",
"cancelled",
}
and not cancelled
):
return data, False
requested_status = str(changes.pop("status"))
data.update(changes)
if cancelled:
data["status"] = "cancelled"
data["finished_at"] = data.get("finished_at") or utc_now()
data.pop("error", None)
else:
data["status"] = requested_status
data["finished_at"] = data.get("finished_at") or utc_now()
publish_job_record(directory, data)
return data, cancelled
def begin_running(
directory: Path,
*,
runner: WorkerRunner | None = None,
) -> dict[str, Any] | None:
"""Publish running state without resurrecting a cancelling job."""
with file_lock(runtime_lock_path()):
data = _read_metadata(directory)
if data.get("status") in {"cancelling", "cancelled"} or bool(
runner and runner.stop_requested
):
data.update(
status="cancelled",
finished_at=data.get("finished_at") or utc_now(),
warning="worker launch was cancelled before Codex started",
)
publish_job_record(directory, data)
return None
if data.get("status") not in {"queued", "recovering"}:
data.update(
status="failed",
finished_at=utc_now(),
error=f"worker runner expected queued state, found {data.get('status')!r}",
)
publish_job_record(directory, data)
return None
start_token = process_start_token(os.getpid())
if start_token is None:
data.update(
status="failed",
finished_at=utc_now(),
error="unable to fingerprint worker runner process",
)
publish_job_record(directory, data)
return None
data.update(
worker_runtime_package_version=package_version(),
worker_runtime_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
status="running",
started_at=data.get("started_at") or utc_now(),
runner_pid=os.getpid(),
runner_start_token=start_token,
)
publish_job_record(directory, data)
return data
def _cleanup_worktree_on_exit(directory: Path) -> None:
with contextlib.suppress(Exception):
current = _read_metadata(directory)
if current.get("status") in {"suspended", "recovering", "paused"}:
return
remove_isolated_worktree(current)
def _walk_objects(value: Any) -> list[dict[str, Any]]:
objects: list[dict[str, Any]] = []
if isinstance(value, dict):
objects.append(value)
for child in value.values():
objects.extend(_walk_objects(child))
elif isinstance(value, list):
for child in value:
objects.extend(_walk_objects(child))
return objects
def _verify_literal_result(
structured: Any, literal_task: dict[str, Any], cwd: Path
) -> tuple[list[str], list[dict[str, Any]]]:
errors: list[str] = []
artifacts: list[dict[str, Any]] = []
operation = literal_task["operation"]
objects = _walk_objects(structured)
if operation == "summarize_supplied":
expected = literal_task["input_sha256"]
if not any(item.get("input_sha256") == expected for item in objects):
errors.append("literal summary does not correlate to supplied input_sha256")
return errors, artifacts
evidence = [
item for item in objects if {"path", "sha256", "start_line", "end_line"}.issubset(item)
]
if not evidence:
return ["literal result contains no path/hash/line evidence"], artifacts
allowed_paths = literal_task.get("paths", [literal_task.get("path")])
query = literal_task.get("needle") or literal_task.get("symbol")
for index, item in enumerate(evidence):
raw_path = item.get("path")
if not isinstance(raw_path, str) or Path(raw_path).is_absolute():
errors.append(f"literal evidence {index} has invalid relative path")
continue
path = (cwd / raw_path).resolve(strict=False)
try:
relative = path.relative_to(cwd).as_posix()
except ValueError:
errors.append(f"literal evidence {index} escapes cwd")
continue
if not any(path_within_scope(relative, [str(scope)]) for scope in allowed_paths if scope):
errors.append(f"literal evidence {index} is outside requested paths")
continue
if not path.is_file() or path.is_symlink():
errors.append(f"literal evidence {index} does not identify a regular file")
continue
content = path.read_bytes()
observed_hash = hashlib.sha256(content).hexdigest()
if item.get("sha256") != observed_hash:
errors.append(f"literal evidence {index} file hash does not match")
start = item.get("start_line")
end = item.get("end_line")
lines = content.decode("utf-8", errors="replace").splitlines()
if (
not isinstance(start, int)
or isinstance(start, bool)
or not isinstance(end, int)
or isinstance(end, bool)
or start < 1
or end < start
or end > len(lines)
):
errors.append(f"literal evidence {index} has invalid line range")
continue
excerpt = "\n".join(lines[start - 1 : end])
if "excerpt" in item and item["excerpt"] != excerpt:
errors.append(f"literal evidence {index} excerpt does not match file lines")
if query is not None and query not in excerpt:
errors.append(f"literal evidence {index} does not contain requested literal")
artifacts.append(
{
"relative_path": relative,
"sha256": observed_hash,
"size": len(content),
"media_type": mimetypes.guess_type(path.name)[0] or "application/octet-stream",
"start_line": start,
"end_line": end,
}
)
if operation == "extract" and not any(
item.get("path") == literal_task["path"]
and item.get("sha256") == literal_task["sha256"]
and item.get("start_line") == literal_task["start_line"]
and item.get("end_line") == literal_task["end_line"]
for item in evidence
):
errors.append("literal extraction does not correlate to the requested file/range/hash")
return errors, artifacts
def _correlate_command_evidence(structured: Any, events_path: Path) -> list[str]:
declarations = [
item
for item in _walk_objects(structured)
if isinstance(item.get("command"), str) and "exit_code" in item
]
if not declarations:
return []
observations: list[tuple[str, int]] = []
if events_path.is_file():
for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines():
with contextlib.suppress(json.JSONDecodeError, ValueError):
row = strict_json_loads(line)
for item in _walk_objects(row):
raw_command = item.get("command", item.get("cmd"))
raw_exit = item.get("exit_code", item.get("exitCode"))
if isinstance(raw_command, list) and all(
isinstance(part, str) for part in raw_command
):
raw_command = " ".join(raw_command)
if (
isinstance(raw_command, str)
and isinstance(raw_exit, int)
and not isinstance(raw_exit, bool)
):
observations.append((raw_command, raw_exit))
errors: list[str] = []
for index, declaration in enumerate(declarations):
command = str(declaration["command"])
exit_code = declaration.get("exit_code")
matching_commands = [
observed_exit
for observed_command, observed_exit in observations
if command in observed_command or observed_command in command
]
if not matching_commands:
errors.append(
f"declared command evidence {index} is absent from captured command events"
)
elif exit_code not in matching_commands:
errors.append(
f"declared command evidence {index} exit_code does not match captured events"
)
return errors
def _correlate_artifact_evidence(
structured: Any, cwd: Path
) -> tuple[list[str], list[dict[str, Any]]]:
"""Verify every declared relative-path/hash artifact against the delegated tree."""
declarations = [
item for item in _walk_objects(structured) if "relative_path" in item and "sha256" in item
]
errors: list[str] = []
artifacts: list[dict[str, Any]] = []
cwd = cwd.resolve()
for index, declaration in enumerate(declarations):
raw_path = declaration.get("relative_path")
expected_hash = declaration.get("sha256")
if not isinstance(raw_path, str) or not raw_path or Path(raw_path).is_absolute():
errors.append(f"declared artifact {index} has invalid relative_path")
continue
path = (cwd / raw_path).resolve(strict=False)
try:
relative = path.relative_to(cwd).as_posix()
except ValueError:
errors.append(f"declared artifact {index} escapes delegated cwd")
continue
if not path.is_file() or path.is_symlink():
errors.append(f"declared artifact {index} does not identify a regular file")
continue
content = path.read_bytes()
observed_hash = hashlib.sha256(content).hexdigest()
if expected_hash != observed_hash:
errors.append(f"declared artifact {index} hash does not match")
continue
artifacts.append(
{
"relative_path": relative,
"sha256": observed_hash,
"size": len(content),
"media_type": mimetypes.guess_type(path.name)[0] or "application/octet-stream",
"state": "observed",
}
)
return errors, artifacts
def _validate_result_contract(
result_text: str,
*,
metadata: Mapping[str, Any],
cwd: Path,
events_path: Path,
) -> tuple[Any, bool | None, list[str], list[dict[str, Any]]]:
"""Validate one candidate through the complete mechanically owned contract path."""
contract = metadata.get("output_contract")
if contract is None:
return None, None, [], []
structured, extraction_error = extract_json_document(result_text)
errors = [extraction_error] if extraction_error else validate_instance(structured, contract)
artifacts: list[dict[str, Any]] = []
if not extraction_error:
errors.extend(_correlate_command_evidence(structured, events_path))
artifact_errors, artifacts = _correlate_artifact_evidence(structured, cwd)
errors.extend(artifact_errors)
literal_task = metadata.get("literal_task")
if isinstance(literal_task, dict):
literal_errors, literal_artifacts = _verify_literal_result(
structured, literal_task, cwd
)
errors.extend(literal_errors)
artifacts.extend(literal_artifacts)
return structured, not errors, errors, artifacts
def _successful_current_turn_result(turn: Any, expected_turn_id: Any) -> str:
"""Return only the successful terminal message for the expected current turn."""
if (
not isinstance(turn, Mapping)
or not isinstance(expected_turn_id, str)
or turn.get("id") != expected_turn_id
or turn.get("status") != "completed"
):
return ""
return _last_agent_message(turn)
def _strict_result_candidate_is_valid(
result_text: str,
*,
metadata: Mapping[str, Any],
cwd: Path,
events_path: Path,
) -> bool:
"""Accept budget-boundary recovery only when strict validation is conclusive."""
if metadata.get("contract_enforcement") != "strict" or metadata.get("output_contract") is None:
return False
_structured, valid, _errors, _artifacts = _validate_result_contract(
result_text,
metadata=metadata,
cwd=cwd,
events_path=events_path,
)
return valid is True
def _extract_usage(events_path: Path) -> dict[str, int]:
return event_usage(events_path)
def _partial_result(directory: Path, *, reason: str) -> dict[str, Any]:
"""Retain deterministic readable evidence even without a valid final contract."""
metadata = _read_metadata(directory)
changes = retain_partial_evidence(metadata, directory, reason=reason)
if metadata.get("sandbox_mode") == "workspace-write":
cwd = Path(str(metadata["cwd"])).resolve()
boundary_errors, artifacts, patch = capture_isolated_patch(metadata, directory, cwd)
changes.update(artifacts=artifacts, patch=patch)
if boundary_errors:
changes["partial_boundary_errors"] = boundary_errors
update(directory, **changes)
return changes
def _retire_control_socket(
directory: Path,
socket_path: Path,
bound_identity: tuple[int, int],
) -> None:
"""Unpublish only the control-socket generation owned by this runner."""
with contextlib.suppress(Exception), file_lock(runtime_lock_path()):
owned_socket = False
path_absent = not socket_path.exists() and not socket_path.is_symlink()
if not path_absent and not socket_path.is_symlink():
current_stat = socket_path.stat(follow_symlinks=False)
owned_socket = (current_stat.st_dev, current_stat.st_ino) == bound_identity
if owned_socket:
socket_path.unlink()
if owned_socket or path_absent:
current = _read_metadata(directory)
if current.get("control_socket_path") == str(socket_path):
current["control_socket_ready"] = False
publish_job_record(directory, current)
def _serve_control(
directory: Path,
state: dict[str, Any],
state_lock: threading.RLock,
stop_event: threading.Event,
ready_event: threading.Event,
) -> None:
metadata = _read_metadata(directory)
socket_path = Path(str(metadata.get("control_socket_path", "")))
expected_socket = app_server_socket_path(f"control:job:{metadata['job_id']}")
if socket_path != expected_socket:
raise RuntimeError("worker control socket does not match its canonical job identity")
with contextlib.ExitStack() as stack:
server = stack.enter_context(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM))
# Serialize path replacement and publication with controller relaunches
# and retiring generations. The stable path may name different socket
# inodes over a job's lifetime; only its owning generation may unlink it.
with file_lock(runtime_lock_path()):
if socket_path.exists() or socket_path.is_symlink():
if socket_path.is_symlink() or not socket_path.is_socket():
raise RuntimeError("worker control socket path is unsafe")
socket_path.unlink()
server.bind(str(socket_path))
os.chmod(socket_path, 0o600)
server.listen(8)
server.settimeout(0.5)
bound_stat = socket_path.stat(follow_symlinks=False)
bound_identity = (bound_stat.st_dev, bound_stat.st_ino)
stack.callback(_retire_control_socket, directory, socket_path, bound_identity)
current = _read_metadata(directory)
current.update(control_socket_path=str(socket_path), control_socket_ready=True)
publish_job_record(directory, current)
ready_event.set()
while not stop_event.is_set():
try:
connection, _ = server.accept()
except TimeoutError:
continue
with connection:
connection.settimeout(30.0)
try:
request = receive_control_request(connection)
action = str(request.get("action", ""))
with state_lock:
candidate_client = state.get("client")
if not isinstance(candidate_client, AppServerClient):
raise RuntimeError("worker app-server client is unavailable")
client = candidate_client
thread_value = state.get("thread_id")
if not isinstance(thread_value, str) or not thread_value:
raise RuntimeError("worker app-server thread is unavailable")
thread_id = thread_value
turn_id = state.get("active_turn_id")
inspection = {
"agent_run_ref": state.get("agent_run_ref"),
"thread_id": thread_id,
"active_turn_id": turn_id,
"paused": bool(state.get("paused")),
"current_effort": state.get("current_effort"),
"goal": state.get("goal"),
"execution_mode": state.get("execution_mode"),
}
# Never hold state_lock across an app-server round trip.
# Codex may publish a notification before the response, and
# the reader callback needs this lock to consume it.
if action == "inspect":
result = {
**inspection,
"pending_requests": client.pending_server_requests_for_thread(
thread_id
),
}
elif action == "steer":
if not turn_id:
raise RuntimeError("worker has no active turn to steer")
result = client.request(
"turn/steer",
{
"threadId": thread_id,
"expectedTurnId": str(request["expected_turn_id"]),
"input": _turn_input(str(request["input"]), []),
},
)
elif action in {"interrupt", "pause"}:
if action == "pause" and state["execution_mode"] == "goal":
host = state.get("host")
if not isinstance(host, PersistentThreadHost):
raise RuntimeError("worker host is unavailable")
host.set_goal(status="paused")
if not turn_id:
result = {"interrupted": False}
else:
result = client.request(
"turn/interrupt",
{
"threadId": thread_id,
"turnId": str(request["expected_turn_id"]),
},
)
if action == "pause":
with state_lock:
state["paused"] = True
state["retire_after_pause"] = bool(
request.get("retire_host", False)
)
state["finalize_requested"] = False
state["finalization_deadline_monotonic"] = None
state["finalization_turn_id"] = None
state["finalization_prompt"] = None
update(
directory,
status="paused",
goal_status=(
"paused" if state["execution_mode"] == "goal" else None
),
finalization_started_at=None,
)
elif state["execution_mode"] == "turn":
with state_lock:
state["paused"] = True
update(directory, status="paused")
elif action in {"continue", "finalize"}:
if action == "finalize":
host = state.get("host")
if state["execution_mode"] == "goal" and isinstance(
host, PersistentThreadHost
):
host.set_goal(status="paused")
finalization_deadline = time.monotonic() + max(
0.0, float(state["finalization_grace_seconds"])
)
with state_lock:
state["finalize_requested"] = True
state["finalization_deadline_monotonic"] = finalization_deadline
state["finalization_turn_id"] = None
state["finalization_prompt"] = str(request["input"])
state["output_schema"] = state.get("terminal_output_schema")
update(
directory,
status="finalizing",
finalization_started_at=utc_now(),
)
action_timeout = (
min(
30.0,
max(0.1, float(state["finalization_grace_seconds"])),
)
if action == "finalize"
else 60.0
)
if action == "continue" and state["execution_mode"] == "goal":
host = state.get("host")
if not isinstance(host, PersistentThreadHost):
raise RuntimeError("worker host is unavailable")
token_budget = request.get("goal_token_budget")
result = {
"goal": host.set_goal(
status="active",
token_budget=(
int(token_budget)
if isinstance(token_budget, int)
and not isinstance(token_budget, bool)
else None
),
)
}
with state_lock:
state["paused"] = False
state.pop("terminal_limit_status", None)
state["finalize_requested"] = False
state["finalization_deadline_monotonic"] = None
state["finalization_turn_id"] = None
state["finalization_prompt"] = None
update(
directory,
status="running",
goal_status="active",
finalization_started_at=None,
)
elif action == "finalize" and state["execution_mode"] == "goal":
if turn_id:
result = client.request(
"turn/interrupt",
{
"threadId": thread_id,
"turnId": str(turn_id),
},
timeout=action_timeout,
)
else:
result = {"finalizing": True}
elif turn_id:
if action != "finalize":
raise RuntimeError("worker already has an active turn")
with state_lock:
state["finalization_turn_id"] = str(turn_id)
result = client.request(
"turn/steer",
{
"threadId": thread_id,
"expectedTurnId": str(turn_id),
"input": _turn_input(str(request["input"]), []),
},
timeout=action_timeout,
)
else:
_start_turn(
client,
state,
str(request["input"]),
[],
state_lock=state_lock,
timeout=min(60.0, action_timeout),
)
with state_lock:
state["paused"] = False
if action == "finalize":
state["finalization_turn_id"] = state.get("active_turn_id")
if action == "continue":
update(
directory,
status="running",
finalization_started_at=None,
)
result = {"continued": True}
elif action == "detach":
update(directory, status="detached", detached_at=utc_now())
result = {"detached": True}
elif action == "stop":
host = state.get("host")
if state["execution_mode"] == "goal" and isinstance(
host, PersistentThreadHost
):
host.set_goal(status="paused")
if turn_id:
with contextlib.suppress(AppServerError):
client.request(
"turn/interrupt",
{"threadId": thread_id, "turnId": str(turn_id)},
)
with state_lock:
state["stop_requested"] = True
result = {"stopping": True}
elif action == "compact":
result = client.request("thread/compact/start", {"threadId": thread_id})
elif action == "respond":
request_id = request["request_id"]
pending_request = next(
(
item
for item in client.pending_server_requests_for_thread(thread_id)
if type(item.get("id")) is type(request_id)
and item.get("id") == request_id
),
None,
)
if pending_request is None:
raise RuntimeError("app-server request is not pending")
method = pending_request.get("method")
if not isinstance(method, str):
raise RuntimeError("pending app-server request has no method")
response = request["response"]
if not isinstance(response, Mapping):
raise ValueError("app-server response must be an object")
validate_server_request_response(method, response)
client.respond(request_id, response)
pending_count = len(client.pending_server_requests_for_thread(thread_id))
result = {"responded": True, "request_id": request_id}
update(
directory,
status="waiting" if pending_count else "running",
pending_request_count=pending_count,
)
elif action == "set_effort":
effort = str(request["effort"])
result = client.request(
"thread/settings/update",
{"threadId": thread_id, "effort": effort},
)
with state_lock:
state["current_effort"] = effort
update(directory, reasoning_effort=effort)
else:
raise ValueError(f"unsupported worker control action: {action}")
send_control_response(connection, {"ok": True, "result": result})
except Exception as exc:
# The controller may abandon a bounded request while the
# app server is still replying. A broken response socket
# must not terminate this worker's control thread.
with contextlib.suppress(OSError):
send_control_response(
connection,
{"ok": False, "error": f"{type(exc).__name__}: {exc}"},
)
def _start_turn(
client: AppServerClient,
state: dict[str, Any],
prompt: str,
attachments: list[str],
*,
state_lock: threading.RLock,
timeout: float = 60.0,
) -> str:
"""Bridge existing worker control paths onto the shared persistent host."""
host = state.get("host")
if not isinstance(host, PersistentThreadHost):
raise AppServerError("worker persistent thread host is unavailable")
if host.client is not client or host.state_lock is not state_lock:
raise AppServerError("worker persistent thread host identity changed")
return host.start_turn(
_turn_input(prompt, attachments),
effort=state.get("current_effort"),
output_schema=state.get("output_schema"),
timeout=timeout,
)
def _advance_finalization(
*,
host: PersistentThreadHost,
state: dict[str, Any],
state_lock: threading.RLock,
completed_turn: Mapping[str, Any] | None,
execution_mode: str,
terminal_output_schema: Mapping[str, Any] | None,
lifecycle_timeout: float,
) -> tuple[str, str, int, float | None]:
"""Advance one host-timed finalization step without imposing a task clock."""
with state_lock:
requested = bool(state.get("finalize_requested"))
raw_deadline = state.get("finalization_deadline_monotonic")
finalization_turn_id = state.get("finalization_turn_id")
active_turn_id = state.get("active_turn_id")
finalization_prompt = state.get("finalization_prompt")
if not requested:
return "inactive", "", 0, None
deadline = (
float(raw_deadline)
if isinstance(raw_deadline, (int, float)) and not isinstance(raw_deadline, bool)
else time.monotonic()
)
if isinstance(completed_turn, Mapping) and (
completed_turn.get("id") == finalization_turn_id
or (execution_mode == "turn" and finalization_turn_id is None)
):
return (
"completed",
_last_agent_message(completed_turn),
1 if completed_turn.get("status") == "failed" else 0,
deadline,
)
remaining = deadline - time.monotonic()
if remaining <= 0:
return "expired", "", 0, deadline
if execution_mode == "goal" and finalization_turn_id is None and active_turn_id is None:
new_turn_id = host.start_turn(
_turn_input(
str(finalization_prompt)
if isinstance(finalization_prompt, str) and finalization_prompt.strip()
else "Terminal serialization turn. Do not investigate or call tools. Return the "
"best supported final result using only evidence already present in this thread.",
[],
),
effort=state.get("current_effort"),
output_schema=terminal_output_schema,
timeout=min(lifecycle_timeout, max(0.1, remaining)),
)
with state_lock:
state["finalization_turn_id"] = new_turn_id
state["paused"] = False
return "waiting", "", 0, deadline
def _fail(
directory: Path,
exit_code: int,
message: str,
*,
runner: WorkerRunner | None = None,
warning: str | None = None,
usage: dict[str, int] | None = None,
) -> int:
partial = _partial_result(directory, reason=message)
changes: dict[str, Any] = {
"status": "failed",
"exit_code": exit_code,
"error": message,
"warning": warning or message,
}
if usage is not None:
changes["usage"] = usage
changes.update(partial)
_current, cancelled = finalize(
directory,
runner=runner,
**changes,
)
return 130 if cancelled else shell_exit_status(exit_code or 1)
def _cancelled_worker_exit(runner: WorkerRunner, started: float) -> int:
directory = runner.directory
runner.terminate_child()
partial = _partial_result(directory, reason="worker cancelled")
finalize(
directory,
runner=runner,
status="cancelled",
elapsed_seconds=time.monotonic() - started,
**partial,
)
return 130
def _file_not_found_exit(
runner: WorkerRunner,
state: Mapping[str, Any],
codex_bin: str,
error: FileNotFoundError,
) -> int:
directory = runner.directory
runner.terminate_child()
_settle_recovery_control(
directory,
state,
status=("delivery_unknown" if state.get("recovery_control_dispatched") else "failed"),
error=f"{type(error).__name__}: {error}",
)
if _has_durable_thread_work(state):
partial = _partial_result(
directory,
reason=f"Codex executable unavailable during durable-thread hosting: {error}",
)
update(
directory,
status="suspended",
error=f"unable to execute Codex binary {codex_bin!r}: {error}",
**partial,
)
return 75
return _fail(
directory,
127,
f"unable to execute Codex binary {codex_bin!r}: {error}",
runner=runner,
)
def _app_server_error_exit(
runner: WorkerRunner,
state: Mapping[str, Any],
started: float,
error: AppServerError,
) -> int:
directory = runner.directory
if runner.stop_requested:
return _cancelled_worker_exit(runner, started)
runner.terminate_child()
recovery_status = "delivery_unknown" if state.get("recovery_control_dispatched") else "failed"
_settle_recovery_control(
directory,
state,
status=recovery_status,
error=f"{type(error).__name__}: {error}",
)
if _has_durable_thread_work(state):
partial = _partial_result(directory, reason=f"app-server host suspended: {error}")
update(
directory,
status="suspended",
error=f"app-server host suspended: {error}",
**partial,
)
return 75
return _fail(directory, 1, f"app-server startup failed: {error}", runner=runner)
def _has_durable_thread_work(state: Mapping[str, Any]) -> bool:
"""Distinguish useful persisted work from a newly allocated empty thread."""
return bool(
isinstance(state.get("active_turn_id"), str)
or isinstance(state.get("last_turn_id"), str)
or state.get("turn_start_pending") is True
or isinstance(state.get("completed_turn"), Mapping)
or isinstance(state.get("goal"), Mapping)
)
def _repair_strict_contract(
*,
runner: WorkerRunner | None = None,
directory: Path,
state: dict[str, Any],
state_lock: threading.RLock,
metadata: Mapping[str, Any],
output_contract: Any,
result_text: str,
result_path: Path,
deadline: float | None = None,
) -> str:
"""Attempt one same-thread schema repair and retain the original on timeout."""
if output_contract is None or metadata.get("contract_enforcement") != "strict":
return result_text
structured_preview, extraction_error = extract_json_document(result_text)
preview_errors = (
[extraction_error]
if extraction_error
else validate_instance(structured_preview, output_contract)
)
if not preview_errors:
return result_text
repair_deadline = deadline or (time.monotonic() + int(metadata["finalization_grace_seconds"]))
remaining = repair_deadline - time.monotonic()
if remaining <= 0:
return result_text
update(directory, status="finalizing", contract_repair_attempted=True)
repair_prompt = (
"Repair only the final JSON result. Do not call tools or add prose. "
"Return exactly one document satisfying this schema:\n"
+ json.dumps(
output_contract,
ensure_ascii=False,
indent=2,
sort_keys=True,
allow_nan=False,
)
+ "\n\nValidation errors:\n- "
+ "\n- ".join(preview_errors[:20])
+ "\n\nInvalid result:\n"
+ result_text[:100000]
)
client = state.get("client")
if not isinstance(client, AppServerClient):
raise AppServerError("app-server client disappeared before contract repair")
try:
_start_turn(
client,
state,
repair_prompt,
[],
state_lock=state_lock,
timeout=min(60.0, max(0.1, remaining)),
)
except AppServerError:
if time.monotonic() >= repair_deadline:
return result_text
raise
while time.monotonic() < repair_deadline:
if runner is not None and runner.stop_requested:
raise KeyboardInterrupt
if state["turn_event"].wait(timeout=0.25):
with state_lock:
repaired_turn = state.get("completed_turn")
state["turn_event"].clear()
repaired = _last_agent_message(repaired_turn)
if repaired.strip():
result_path.write_text(repaired, encoding="utf-8")
return repaired
return result_text
if not client.alive:
raise AppServerError("app-server transport closed during contract repair")
return result_text
def _lifecycle_status_for_goal(goal_status: Any) -> str | None:
"""Translate Codex goal state into the durable worker lifecycle."""
if goal_status == "active":
return "running"
if goal_status in {"paused", "blocked"}:
return "paused"
if goal_status in {"usageLimited", "budgetLimited"}:
return "suspended"
return None
def _goal_objective_from_metadata(metadata: Mapping[str, Any]) -> str:
value = metadata.get("goal_objective")
if isinstance(value, str) and value.strip():
return value
return bounded_goal_objective(str(metadata["task"]))
def _persist_worker_host_state(
directory: Path,
state: Mapping[str, Any],
changes: Mapping[str, Any],
) -> None:
persisted: dict[str, Any] = {"last_progress_at": utc_now(), **changes}
current_status = str(_read_metadata(directory).get("status"))
status_is_protected = current_status in (
_TERMINAL_OR_CANCELLING_STATUSES | {"detached", "paused", "recovering", "finalizing"}
)
goal = changes.get("goal")
if isinstance(goal, Mapping):
persisted.update(
goal_status=goal.get("status"),
goal_objective=goal.get("objective"),
goal_token_budget=goal.get("tokenBudget"),
goal_tokens_used=goal.get("tokensUsed", 0),
goal_time_used_seconds=goal.get("timeUsedSeconds", 0),
)
goal_lifecycle = _lifecycle_status_for_goal(goal.get("status"))
if goal_lifecycle == "suspended" and isinstance(state.get("active_turn_id"), str):
# A limit notification can precede turn/completed. The admitted
# turn still owns its scheduler and write leases until its terminal
# event is consumed or the host is suspended explicitly.
goal_lifecycle = "running"
if goal_lifecycle is not None and not status_is_protected:
persisted["status"] = goal_lifecycle
if "turn_failure" in changes:
persisted["failure"] = changes["turn_failure"]
if changes["turn_failure"] is None:
persisted["error"] = None
if "pending_request_count" in changes and not status_is_protected:
persisted["status"] = (
"waiting"
if int(changes["pending_request_count"]) > 0
else "running"
if current_status == "waiting"
else persisted.get("status", current_status)
)
elif not status_is_protected and isinstance(changes.get("active_turn_id"), str):
persisted["status"] = "finalizing" if state.get("finalize_requested") else "running"
update(directory, **persisted)
def _publish_worker_terminal(
*,
runner: WorkerRunner,
metadata: dict[str, Any],
session: Mapping[str, Any],
state: Mapping[str, Any],
control_stop: threading.Event,
control_thread: threading.Thread | None,
started: float,
exit_code: int,
) -> int:
"""Validate and publish the terminal result after the app-server loop ends."""
directory = runner.directory
cwd = Path(str(metadata["cwd"])).resolve()
result_path = directory / "result.md"
events_path = directory / "events.jsonl"
stderr_path = directory / "stderr.log"
control_stop.set()
if control_thread is not None:
with contextlib.suppress(Exception):
control_thread.join(timeout=2.0)
runner.terminate_child()
runner.app_server = None
elapsed = time.monotonic() - started
usage = _extract_usage(events_path)
observed_route = route_telemetry(metadata, session, events_path)
if runner.stop_requested or _read_metadata(directory).get("status") == "cancelling":
partial = _partial_result(directory, reason="worker cancelled")
finalize(
directory,
runner=runner,
status="cancelled",
elapsed_seconds=elapsed,
exit_code=exit_code,
usage=usage,
**partial,
)
return 130
warnings = route_telemetry_warnings(metadata, observed_route)
artifacts: list[dict[str, Any]] = []
patch: dict[str, Any] | None = None
if metadata["sandbox_mode"] == "workspace-write":
boundary_errors, artifacts, patch = capture_isolated_patch(metadata, directory, cwd)
if boundary_errors:
reason = "; ".join(boundary_errors)
taint_session(str(metadata["session_id"]), reason, job_id=str(metadata["job_id"]))
return _fail(
directory,
1,
"writable worker violated its mechanically enforced boundary: " + reason,
runner=runner,
usage=usage,
)
result_text = (
result_path.read_text(encoding="utf-8", errors="replace") if result_path.is_file() else ""
)
contract_valid: bool | None = None
contract_errors: list[str] = []
structured: Any = None
if metadata.get("output_contract") is not None:
structured, contract_valid, contract_errors, declared_artifacts = _validate_result_contract(
result_text,
metadata=metadata,
cwd=cwd,
events_path=events_path,
)
known_artifacts = {(item.get("relative_path"), item.get("sha256")) for item in artifacts}
artifacts.extend(
item
for item in declared_artifacts
if (item.get("relative_path"), item.get("sha256")) not in known_artifacts
)
if contract_errors:
warnings.append("output contract violation: " + "; ".join(contract_errors[:12]))
terminal_limit_status = state.get("terminal_limit_status")
if terminal_limit_status in {"usageLimited", "budgetLimited"} and contract_valid is True:
warnings.append(
f"goal reached {terminal_limit_status} after producing a fully validated strict "
"result; exact observed usage is retained"
)
current_status = str(_read_metadata(directory).get("status"))
if current_status == "stopped":
partial = _partial_result(directory, reason="worker stopped by controller")
final, cancelled = finalize(
directory,
runner=runner,
status="stopped",
elapsed_seconds=elapsed,
exit_code=1,
usage=usage,
**partial,
)
return 130 if cancelled else shell_exit_status(int(final.get("exit_code", 1)))
if exit_code != 0 or not result_text.strip():
stderr_tail = (
stderr_path.read_text(encoding="utf-8", errors="replace")[-5000:]
if stderr_path.is_file()
else ""
)
message = "Codex worker failed" if exit_code else "Codex worker produced no final result"
if stderr_tail:
message += ": " + stderr_tail
return _fail(
directory,
int(exit_code or 1),
message,
runner=runner,
warning="; ".join(warnings) or None,
usage=usage,
)
strict_contract_failure = (
contract_valid is False and metadata.get("contract_enforcement", "warn") == "strict"
)
status = (
"failed"
if strict_contract_failure
else "completed_with_warnings"
if warnings
else "completed"
)
changes: dict[str, Any] = {
"status": status,
"finished_at": utc_now(),
"elapsed_seconds": elapsed,
"exit_code": exit_code,
"usage": usage,
"route_telemetry": observed_route,
"contract_valid": contract_valid,
"contract_errors": contract_errors,
"warning": "; ".join(warnings) or None,
"artifacts": artifacts,
"patch": patch,
"goal_status": (
state.get("terminal_limit_status")
or (
state.get("goal", {}).get("status")
if isinstance(state.get("goal"), Mapping)
else None
)
),
"result_kind": "final",
"result_state": "unread",
"failure": None,
"error": None,
}
if structured is not None and contract_valid:
structured_path = directory / "result.json"
atomic_write_json(structured_path, structured)
changes["structured_result_path"] = str(structured_path)
if strict_contract_failure:
changes["error"] = "worker output did not satisfy its required result contract"
final, cancelled = finalize(directory, runner=runner, **changes)
final_status = str(final["status"])
append_audit(
str(session["session_id"]),
"agent_completed",
run_id=metadata["run_id"],
job_id=metadata["job_id"],
status=final_status,
exit_code=exit_code,
contract_valid=contract_valid,
elapsed_seconds=elapsed,
route_telemetry=observed_route,
)
return 130 if cancelled else shell_exit_status(int(final.get("exit_code", exit_code)))
def _run_worker_event_loop(
*,
runner: WorkerRunner,
directory: Path,
metadata: Mapping[str, Any],
session: Mapping[str, Any],
state: dict[str, Any],
state_lock: threading.RLock,
host: PersistentThreadHost,
connect_app_server: Callable[..., AppServerClient],
resume_recovered_work: Callable[[AppServerClient, str], None],
latest_history_result: Callable[[], str],
execution_mode: str,
terminal_output_schema: dict[str, Any] | None,
lifecycle_timeout: float,
control_failed: threading.Event,
control_errors: list[BaseException],
control_stop: threading.Event,
progress: dict[str, Any],
cwd: Path,
events_path: Path,
initial_result_text: str,
) -> tuple[str, int, float | None, bool, int | None]:
"""Drive one worker until a durable terminal result or recoverable suspension."""
result_text = initial_result_text
exit_code = 0
contract_repair_deadline: float | None = None
finalization_expired = False
last_heartbeat = 0.0
recovery_attempts = int(metadata.get("recovery_attempts", 0))
stall_seconds = int(metadata["stall_warning_seconds"])
if result_text:
return result_text, exit_code, contract_repair_deadline, finalization_expired, None
while not runner.stop_requested:
now = time.monotonic()
if control_failed.is_set():
error = control_errors[0]
raise AppServerError(
f"worker control socket failed: {type(error).__name__}: {error}"
) from error
if now - last_heartbeat >= 15.0:
goal = state.get("goal")
heartbeat: dict[str, Any] = {
"heartbeat_at": utc_now(),
"pending_request_count": len(
state["client"].pending_server_requests_for_thread(str(state["thread_id"]))
),
}
if isinstance(goal, Mapping):
heartbeat.update(
goal_status=goal.get("status"),
goal_tokens_used=goal.get("tokensUsed", 0),
goal_time_used_seconds=goal.get("timeUsedSeconds", 0),
)
update(directory, **heartbeat)
last_heartbeat = now
if not progress["stall_reported"] and now - progress["last_progress"] >= stall_seconds:
progress["stall_reported"] = True
update(directory, stall_warning_at=utc_now())
append_audit(
str(session["session_id"]),
"agent_stall_warning",
run_id=metadata["run_id"],
job_id=metadata["job_id"],
warning_seconds=stall_seconds,
)
if state.get("stop_requested"):
update(directory, status="stopped")
break
completed_turn: dict[str, Any] | None = None
paused = False
if state["turn_event"].wait(timeout=0.25):
with state_lock:
completed_turn = state.get("completed_turn")
state["completed_turn"] = None
state["turn_event"].clear()
paused = bool(state.get("paused"))
if runner.stop_requested:
break
if state.get("retire_after_pause"):
partial = _partial_result(directory, reason="worker cold-paused by controller")
update(
directory,
status="paused",
paused_at=utc_now(),
active_turn_id=None,
goal_status=("paused" if execution_mode == "goal" else None),
**partial,
)
control_stop.set()
state["client"].close()
runner.app_server = None
return "", exit_code, contract_repair_deadline, finalization_expired, 75
# Recover the transport before interpreting any turn or goal state. A
# completion/limit notification can arrive just before the socket closes
# while active_turn_id still names a turn already persisted by Codex.
# Finalization's waiting state must not mask that dead client either.
if not state["client"].alive:
recovered = False
last_recovery_error = "app-server transport closed"
pre_recovery_status = str(_read_metadata(directory).get("status"))
for incident_attempt, delay in enumerate(
APP_SERVER_RECOVERY_DELAYS_SECONDS,
start=1,
):
if runner.stop_requested:
break
previous = state.get("client")
if isinstance(previous, AppServerClient):
previous.close()
recovery_attempts += 1
update(
directory,
status="recovering",
recovery_attempts=recovery_attempts,
recovery_incident_attempt=incident_attempt,
)
deadline = time.monotonic() + delay
while time.monotonic() < deadline and not runner.stop_requested:
time.sleep(min(0.25, deadline - time.monotonic()))
if runner.stop_requested:
break
try:
recovered_client = connect_app_server(resume=True)
runner.app_server = recovered_client
with state_lock:
state["client"] = recovered_client
resume_recovered_work(recovered_client, pre_recovery_status)
update(
directory,
status=(
pre_recovery_status
if pre_recovery_status
in {"paused", "detached", "waiting", "finalizing"}
else "running"
),
recovery_error=None,
recovery_incident_attempt=0,
)
recovered = True
break
except (AppServerError, FileNotFoundError, OSError) as exc:
last_recovery_error = f"{type(exc).__name__}: {exc}"
update(directory, recovery_error=last_recovery_error)
if not recovered and not runner.stop_requested:
partial = _partial_result(
directory,
reason="app-server recovery attempts exhausted",
)
update(
directory,
status="suspended",
error="app-server recovery attempts exhausted: " + last_recovery_error,
**partial,
)
control_stop.set()
runner.app_server = None
return "", exit_code, contract_repair_deadline, finalization_expired, 75
finalization_state, finalization_result, finalization_exit, finalization_deadline = (
_advance_finalization(
host=host,
state=state,
state_lock=state_lock,
completed_turn=completed_turn,
execution_mode=execution_mode,
terminal_output_schema=terminal_output_schema,
lifecycle_timeout=lifecycle_timeout,
)
)
if finalization_state == "completed":
result_text = finalization_result
exit_code = finalization_exit
contract_repair_deadline = finalization_deadline
break
if finalization_state == "expired":
contract_repair_deadline = finalization_deadline
finalization_expired = True
update(
directory,
status="stopped",
finalization_expired_at=utc_now(),
)
break
if finalization_state == "waiting":
continue
if (
isinstance(completed_turn, Mapping)
and completed_turn.get("status") == "interrupted"
and paused
):
update(directory, status="paused", active_turn_id=None)
elif isinstance(completed_turn, Mapping) and completed_turn.get("status") == "failed":
failure = normalize_turn_failure(completed_turn) or {
"kind": "turn_failed",
"source": "turn/completed",
"turn_id": completed_turn.get("id"),
"turn_status": "failed",
"message": "app-server turn failed",
"retryable": False,
"observed_at": utc_now(),
}
partial = _partial_result(directory, reason=str(failure["message"]))
status = "suspended" if failure.get("retryable") else "failed"
update(
directory,
status=status,
suspended_at=utc_now() if status == "suspended" else None,
finished_at=utc_now() if status == "failed" else None,
error=str(failure["message"]),
failure=failure,
**partial,
)
control_stop.set()
runner.terminate_child()
runner.app_server = None
return (
"",
exit_code,
contract_repair_deadline,
finalization_expired,
75 if status == "suspended" else 1,
)
elif execution_mode == "turn" and isinstance(completed_turn, dict):
if completed_turn.get("status") == "failed":
exit_code = 1
result_text = _last_agent_message(completed_turn)
break
goal = state.get("goal")
if execution_mode == "goal" and isinstance(goal, Mapping):
goal_status = goal.get("status")
if goal_status == "complete":
# update_goal can make the goal complete before Codex emits
# turn/completed and the terminal agent message. Preserve
# that active turn before reading history or beginning a
# strict terminal-serialization turn.
if state.get("active_turn_id") is not None:
continue
result_text = latest_history_result()
if terminal_output_schema is not None:
finalization_deadline = time.monotonic() + int(
metadata["finalization_grace_seconds"]
)
with state_lock:
state["output_schema"] = terminal_output_schema
state["finalize_requested"] = True
state["finalization_deadline_monotonic"] = finalization_deadline
state["finalization_turn_id"] = None
state["finalization_prompt"] = (
"Terminal serialization turn. Do not investigate or call tools. "
"Return exactly one JSON document satisfying the compiled output "
"schema, using only evidence already present in this thread."
)
update(
directory,
status="finalizing",
finalization_started_at=utc_now(),
)
continue
break
if goal_status in {"usageLimited", "budgetLimited"}:
state["terminal_limit_status"] = goal_status
# A goal-limit notification may precede turn/completed by
# only milliseconds. Never interrupt or discard that
# already-admitted turn; classify its terminal message
# after the app server declares the turn complete.
if state.get("active_turn_id") is not None:
continue
terminal_turn: Any = completed_turn
if not isinstance(terminal_turn, Mapping):
turns = host.complete_history(timeout=lifecycle_timeout)
terminal_turn = turns[-1] if turns else None
candidate = _successful_current_turn_result(
terminal_turn,
state.get("last_turn_id"),
)
if _strict_result_candidate_is_valid(
candidate,
metadata=metadata,
cwd=cwd,
events_path=events_path,
):
result_text = candidate
break
partial = _partial_result(
directory,
reason=f"goal suspended with status {goal_status}",
)
update(
directory,
status="suspended",
error=f"goal suspended with status {goal_status}",
**partial,
)
control_stop.set()
state["client"].close()
runner.app_server = None
return "", exit_code, contract_repair_deadline, finalization_expired, 75
return result_text, exit_code, contract_repair_deadline, finalization_expired, None
def _run_worker(runner: WorkerRunner) -> int:
directory = runner.directory
metadata = _read_metadata(directory)
if metadata.get("worktree_root"):
atexit.register(_cleanup_worktree_on_exit, directory)
caller_token = os.environ.get("MMO_CALLER_TOKEN", "")
if not caller_token:
return _fail(
directory,
2,
"worker caller capability is missing from launch environment",
runner=runner,
)
session = read_session_record(session_dir(str(metadata["session_id"])))
if session.get("snapshot_hash") != metadata.get("snapshot_hash"):
return _fail(directory, 2, "job/session snapshot mismatch", runner=runner)
if session.get("current_run_id") != metadata.get("run_id"):
return _fail(directory, 2, "job belongs to an inactive session run", runner=runner)
agent_id = str(metadata["agent"])
home = session["homes"][agent_id]
cwd = Path(str(metadata["cwd"])).resolve()
prompt_path = directory / "prompt.txt"
result_path = directory / "result.md"
events_path = directory / "events.jsonl"
stderr_path = directory / "stderr.log"
socket_path = Path(str(metadata["app_server_socket_path"]))
expected_paths = {
"result_path": result_path,
"events_path": events_path,
"stderr_path": stderr_path,
}
unsafe_paths = [
key
for key, path in expected_paths.items()
if metadata.get(key) != str(path) or path.is_symlink()
]
if prompt_path.is_symlink() or not prompt_path.is_file():
unsafe_paths.append("prompt_path")
if not socket_path.is_absolute() or socket_path.is_symlink():
unsafe_paths.append("app_server_socket_path")
if unsafe_paths:
return _fail(
directory,
2,
"worker state contains unsafe artifact paths: " + ", ".join(unsafe_paths),
runner=runner,
)
runner.install_signal_handlers()
if begin_running(directory, runner=runner) is None:
append_audit(
session["session_id"],
"agent_cancelled_terminal",
run_id=metadata["run_id"],
job_id=metadata["job_id"],
phase="before_codex_launch",
)
return 130
start_event = (
"agent_resumed"
if metadata.get("status") == "recovering"
or isinstance(metadata.get("app_server_thread_id"), str)
else "agent_started"
)
append_audit(
session["session_id"],
start_event,
run_id=metadata["run_id"],
job_id=metadata["job_id"],
agent=agent_id,
parent_job_id=metadata.get("parent_job_id"),
)
codex_bin = str(session["codex_binary"])
command = app_server_listen_command(
codex_bin,
home.get("command_flags", []),
socket_path,
)
output_contract = metadata.get("output_contract")
terminal_output_schema = (
_codex_output_schema(output_contract)
if isinstance(output_contract, dict)
and metadata.get("contract_enforcement") == "strict"
and metadata.get("structured_output_supported", False)
else None
)
contract_transport = (
"validated_text"
if terminal_output_schema is None
else "native_schema"
if terminal_output_schema == output_contract
else "native_schema_projection"
)
update(directory, command=command, contract_transport=contract_transport)
if runner.stop_requested or _read_metadata(directory).get("status") == "cancelling":
finalize(
directory,
runner=runner,
status="cancelled",
warning="worker launch was cancelled before Codex started",
)
return 130
started = time.monotonic()
state_lock = threading.RLock()
control_stop = threading.Event()
execution_mode = str(metadata["execution_mode"])
recovery_finalize = bool(
metadata.get("finalize_requested_on_recovery")
or metadata.get("recovery_action") == "finalize"
)
state: dict[str, Any] = {
"client": None,
"thread_id": metadata.get("app_server_thread_id"),
"active_turn_id": metadata.get("active_turn_id"),
"last_turn_id": metadata.get("last_turn_id"),
"turn_start_pending": bool(metadata.get("turn_start_pending", False)),
"completed_turn": None,
"turn_event": threading.Event(),
"thread_status": metadata.get("thread_status"),
"goal": None,
"paused": metadata.get("status") == "paused",
"stop_requested": False,
"finalize_requested": recovery_finalize,
"finalization_deadline_monotonic": (
started + int(metadata["finalization_grace_seconds"]) if recovery_finalize else None
),
"finalization_turn_id": None,
"finalization_prompt": (metadata.get("recovery_prompt") if recovery_finalize else None),
"current_effort": metadata.get("reasoning_effort"),
"finalization_grace_seconds": int(metadata["finalization_grace_seconds"]),
"execution_mode": execution_mode,
"agent_run_ref": metadata["agent_run_ref"],
"recovery_action": metadata.get("recovery_action"),
"recovery_control_revision": metadata.get("recovery_control_revision"),
"recovery_controller_agent": metadata.get("recovery_controller_agent"),
"recovery_controller_job_id": metadata.get("recovery_controller_job_id"),
"recovery_control_dispatched": False,
"output_schema": (
terminal_output_schema if execution_mode == "turn" or recovery_finalize else None
),
"terminal_output_schema": terminal_output_schema,
"retire_after_pause": False,
}
lifecycle_timeout = float(
metadata.get(
"app_server_lifecycle_timeout_seconds",
APP_SERVER_LIFECYCLE_TIMEOUT_SECONDS,
)
)
progress: dict[str, Any] = {
"last_progress": time.monotonic(),
"stall_reported": False,
}
def persist_host_state(changes: dict[str, Any], _message: Mapping[str, Any]) -> None:
progress["last_progress"] = time.monotonic()
progress["stall_reported"] = False
_persist_worker_host_state(directory, state, changes)
host = PersistentThreadHost(
state=state,
state_lock=state_lock,
on_state_change=persist_host_state,
)
state["host"] = host
environment = session_environment(
session,
agent_id,
caller_job_id=metadata["job_id"],
caller_token=caller_token,
)
def connect_app_server(*, resume: bool) -> AppServerClient:
with state_lock:
prior_turn_id = state.get("last_turn_id") or state.get("active_turn_id")
observed = _read_metadata(directory)
host_alive = process_matches(
observed.get("app_server_pid"),
observed.get("app_server_start_token"),
)
if not host_alive:
# Worker hosts can be recreated long after session admission. Do
# not let an executable replaced in place silently change the
# experimental app-server protocol on the persisted thread.
require_app_server_codex_version(codex_bin)
if not host_alive and (socket_path.exists() or socket_path.is_symlink()):
if socket_path.is_symlink() or not socket_path.is_socket():
raise AppServerError("recorded worker app-server socket is unsafe")
socket_path.unlink()
client = AppServerClient(
socket_path=socket_path,
command=None if host_alive else command,
cwd=cwd,
env=environment,
events_path=events_path,
stderr_path=stderr_path,
approval_policy=str(metadata["approval_policy"]),
on_message=host.on_message,
)
def publish_host(pid: int) -> None:
start_token = process_start_token(pid)
if start_token is None:
raise AppServerError("worker app-server process could not be fingerprinted")
with file_lock(runtime_lock_path()):
current = _read_metadata(directory)
if current.get("status") in _TERMINAL_OR_CANCELLING_STATUSES:
raise AppServerError("worker admission closed during app-server bootstrap")
current.update(
app_server_pid=pid,
app_server_pgid=pid,
app_server_start_token=start_token,
app_server_process_group_isolated=True,
app_server_protocol="codex-app-server-v2-unix",
)
publish_job_record(directory, current)
runner.app_server = client
try:
client.start(
timeout=APP_SERVER_INITIALIZE_TIMEOUT_SECONDS,
on_started=publish_host if not host_alive else None,
)
host.attach_client(client)
dynamic_tools: list[dict[str, Any]] = []
if metadata["driver"] == "switchyard":
observed_switchyard = session.get("switchyard_version")
if not isinstance(observed_switchyard, str):
raise AppServerError("Switchyard session has no pinned gateway version")
if observed_switchyard == SWITCHYARD_MCP_NAMESPACE_BRIDGE_VERSION:
dynamic_tools = client.install_switchyard_mcp_bridge(timeout=lifecycle_timeout)
expected_thread_id: str | None = None
if resume:
mode = "resume"
expected_thread_id = str(state["thread_id"])
thread_params: dict[str, Any] = {
"threadId": expected_thread_id,
"cwd": str(cwd),
"sandbox": metadata["sandbox_mode"],
"approvalPolicy": metadata["approval_policy"],
"excludeTurns": False,
}
elif metadata.get("fork_thread_id"):
mode = "fork"
thread_params = {
"threadId": str(metadata["fork_thread_id"]),
"cwd": str(cwd),
"sandbox": metadata["sandbox_mode"],
"approvalPolicy": metadata["approval_policy"],
"ephemeral": False,
"deferGoalContinuation": True,
"excludeTurns": False,
}
else:
mode = "start"
thread_params = {
"cwd": str(cwd),
"sandbox": metadata["sandbox_mode"],
"approvalPolicy": metadata["approval_policy"],
"allowProviderModelFallback": False,
"ephemeral": False,
"historyMode": "paginated",
}
if dynamic_tools:
thread_params["dynamicTools"] = dynamic_tools
thread = host.open_thread(
mode,
thread_params,
timeout=lifecycle_timeout,
prior_turn_id=prior_turn_id,
expected_thread_id=expected_thread_id,
interrupt_stale=bool(resume and not host_alive and execution_mode == "turn"),
)
update(
directory,
app_server_thread_id=thread["id"],
active_turn_id=state.get("active_turn_id"),
last_turn_id=state.get("last_turn_id"),
turn_start_pending=bool(state.get("turn_start_pending")),
app_server_protocol="codex-app-server-v2-unix",
)
return client
except BaseException:
host.detach_client(client)
if client.process is not None:
client.stop_host()
else:
client.close()
if runner.app_server is client:
runner.app_server = None
raise
def latest_history_result() -> str:
turns = host.complete_history(timeout=lifecycle_timeout)
for turn in reversed(turns):
result = _last_agent_message(turn)
if result.strip():
return result
return ""
control_thread: threading.Thread | None = None
exit_code = 0
result_text = ""
contract_repair_deadline: float | None = None
finalization_expired = False
try:
resuming_thread = bool(state["thread_id"])
active_client = connect_app_server(resume=resuming_thread)
runner.app_server = active_client
control_ready = threading.Event()
control_failed = threading.Event()
control_errors: list[BaseException] = []
def serve_control() -> None:
try:
_serve_control(
directory,
state,
state_lock,
control_stop,
control_ready,
)
except BaseException as exc:
control_errors.append(exc)
control_failed.set()
control_ready.set()
control_thread = threading.Thread(
target=serve_control,
name=f"mmo-control-{metadata['job_id']}",
daemon=True,
)
control_thread.start()
if not control_ready.wait(timeout=APP_SERVER_INITIALIZE_TIMEOUT_SECONDS):
raise AppServerError("worker control socket did not become ready")
if control_errors:
error = control_errors[0]
raise AppServerError(
f"worker control socket startup failed: {type(error).__name__}: {error}"
) from error
prompt = prompt_path.read_text(encoding="utf-8")
def resume_recovered_work(client: AppServerClient, prior_status: str) -> None:
if execution_mode == "goal" and not isinstance(state.get("goal"), Mapping):
observed = _read_metadata(directory)
persisted_goal_status = observed.get("goal_status")
if persisted_goal_status in {
"complete",
"paused",
"blocked",
"usageLimited",
"budgetLimited",
}:
# Some hosts can resume a terminal persisted turn without
# returning its former goal object. MMO's durable goal
# record still owns lifecycle classification for that exact
# thread; restoring it avoids duplicating completed work.
with state_lock:
state["goal"] = {
"objective": observed.get("goal_objective"),
"status": persisted_goal_status,
"tokenBudget": observed.get("goal_token_budget"),
"tokensUsed": observed.get("goal_tokens_used", 0),
"timeUsedSeconds": observed.get("goal_time_used_seconds", 0),
}
if prior_status in {"paused", "waiting", "finalizing"}:
return
if (
execution_mode == "turn"
and state.get("active_turn_id") is None
and state.get("completed_turn") is None
):
_start_turn(
client,
state,
"Continue the original delegated task from the persisted app-server thread "
"after transport recovery. Use retained evidence and satisfy the original "
"result contract.",
[],
state_lock=state_lock,
)
elif execution_mode == "goal" and not isinstance(state.get("goal"), Mapping):
host.set_goal(
objective=_goal_objective_from_metadata(metadata),
status="paused",
token_budget=int(metadata["goal_token_budget"]),
timeout=lifecycle_timeout,
)
if state.get("active_turn_id") is None and state.get("completed_turn") is None:
host.start_turn(
_turn_input(
"Continue the original delegated objective from the persisted "
"app-server thread after transport recovery.",
[],
),
effort=state.get("current_effort"),
)
host.set_goal(status="active", timeout=lifecycle_timeout)
if recovery_finalize and metadata.get("goal_status") in {
"usageLimited",
"budgetLimited",
}:
state["terminal_limit_status"] = metadata["goal_status"]
if recovery_finalize:
turns = host.complete_history(timeout=lifecycle_timeout)
candidate = _successful_current_turn_result(
turns[-1] if turns else None,
state.get("last_turn_id"),
)
if _strict_result_candidate_is_valid(
candidate,
metadata=metadata,
cwd=cwd,
events_path=events_path,
):
result_text = candidate
if state.get("recovery_action") in {"continue", "finalize"}:
# From here onward the requested action is either accepted locally
# from exact-thread history or may send a mutating app-server call.
# A transport loss is therefore uncertain rather than safely failed.
state["recovery_control_dispatched"] = True
if result_text:
pass
elif execution_mode == "goal":
existing_goal = state.get("goal")
if not isinstance(existing_goal, Mapping):
host.set_goal(
objective=_goal_objective_from_metadata(metadata),
status="paused",
token_budget=int(metadata["goal_token_budget"]),
timeout=lifecycle_timeout,
)
if not recovery_finalize and state.get("active_turn_id") is None:
host.start_turn(
_turn_input(prompt, list(metadata.get("attachments", []))),
effort=state.get("current_effort"),
output_schema=None,
)
if not recovery_finalize:
active_goal = host.set_goal(status="active", timeout=lifecycle_timeout)
update(
directory,
status="running",
goal_status=active_goal.get("status"),
)
elif existing_goal.get("status") in {
"paused",
"blocked",
"usageLimited",
"budgetLimited",
} and metadata.get("continue_requested"):
active_goal = host.set_goal(status="active", timeout=lifecycle_timeout)
update(
directory,
status="running",
goal_status=active_goal.get("status"),
continue_requested=False,
)
elif recovery_finalize and existing_goal.get("status") != "paused":
host.set_goal(status="paused", timeout=lifecycle_timeout)
elif state.get("active_turn_id") is None and state.get("completed_turn") is None:
recovery_prompt = metadata.get("recovery_prompt")
_start_turn(
active_client,
state,
(
str(recovery_prompt)
if resuming_thread
and isinstance(recovery_prompt, str)
and recovery_prompt.strip()
else "Continue the original delegated task from the persisted app-server "
"thread. Use retained evidence and satisfy the original result contract."
if resuming_thread
else prompt
),
[] if resuming_thread else list(metadata.get("attachments", [])),
state_lock=state_lock,
)
update(directory, recovery_prompt=None)
recovery_control_applied = _settle_recovery_control(
directory,
state,
status="applied",
)
if recovery_control_applied:
append_audit(
str(metadata["session_id"]),
"agent_controlled",
caller_agent=state.get("recovery_controller_agent"),
caller_job_id=state.get("recovery_controller_job_id"),
target_job_id=metadata["job_id"],
action=state.get("recovery_action"),
control_revision=state.get("recovery_control_revision"),
delivery="durable_relaunch_ready",
)
(
result_text,
exit_code,
contract_repair_deadline,
finalization_expired,
suspension_exit,
) = _run_worker_event_loop(
runner=runner,
directory=directory,
metadata=metadata,
session=session,
state=state,
state_lock=state_lock,
host=host,
connect_app_server=connect_app_server,
resume_recovered_work=resume_recovered_work,
latest_history_result=latest_history_result,
execution_mode=execution_mode,
terminal_output_schema=terminal_output_schema,
lifecycle_timeout=lifecycle_timeout,
control_failed=control_failed,
control_errors=control_errors,
control_stop=control_stop,
progress=progress,
cwd=cwd,
events_path=events_path,
initial_result_text=result_text,
)
if suspension_exit is not None:
return suspension_exit
if runner.stop_requested:
raise KeyboardInterrupt
if not result_text and not finalization_expired:
result_text = latest_history_result()
result_path.write_text(result_text, encoding="utf-8")
os.chmod(result_path, 0o600)
if output_contract is not None and metadata.get("contract_enforcement") == "strict":
with state_lock:
state["output_schema"] = terminal_output_schema
result_text = _repair_strict_contract(
runner=runner,
directory=directory,
state=state,
state_lock=state_lock,
metadata=metadata,
output_contract=output_contract,
result_text=result_text,
result_path=result_path,
deadline=contract_repair_deadline,
)
# Contract repair is itself a persisted same-thread turn. Capture
# terminal history only after that optional turn so the immutable
# history artifact agrees with the result MMO actually validates and
# publishes. Expired finalization still relies on the lossless JSONL
# event trace and bounded partial evidence rather than extending the
# operator's deadline with another lifecycle request.
terminal_turns = (
[] if finalization_expired else host.complete_history(timeout=lifecycle_timeout)
)
atomic_write_json(
directory / "terminal-history.json",
{
"thread_id": host.thread_id,
"captured_at": utc_now(),
"turns": terminal_turns,
},
)
except FileNotFoundError as exc:
control_stop.set()
failure_exit = _file_not_found_exit(runner, state, codex_bin, exc)
runner.app_server = None
return failure_exit
except AppServerError as exc:
control_stop.set()
failure_exit = _app_server_error_exit(runner, state, started, exc)
runner.app_server = None
return failure_exit
except KeyboardInterrupt:
control_stop.set()
failure_exit = _cancelled_worker_exit(runner, started)
runner.app_server = None
return failure_exit
except Exception as exc:
_settle_recovery_control(
directory,
state,
status=("delivery_unknown" if state.get("recovery_control_dispatched") else "failed"),
error=f"{type(exc).__name__}: {exc}",
)
runner.terminate_child()
return _fail(
directory,
1,
f"worker runner failure: {type(exc).__name__}: {exc}",
runner=runner,
)
return _publish_worker_terminal(
runner=runner,
metadata=metadata,
session=session,
state=state,
control_stop=control_stop,
control_thread=control_thread,
started=started,
exit_code=exit_code,
)
def main() -> int:
if len(sys.argv) != 2:
print("usage: worker_runner.py JOB_DIR", file=sys.stderr)
return 2
return WorkerRunner(Path(sys.argv[1]).resolve()).run()
if __name__ == "__main__":
raise SystemExit(main())
Reference in New Issue View Git Blame Copy Permalink