1849 lines
78 KiB
Python
1849 lines
78 KiB
Python
|
|
#!/usr/bin/env python3
|
||
|
|
"""Long-lived controller for one root Codex app-server host and thread."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import contextlib
|
||
|
|
import hashlib
|
||
|
|
import secrets
|
||
|
|
import signal
|
||
|
|
import sys
|
||
|
|
import threading
|
||
|
|
import time
|
||
|
|
from collections.abc import Mapping
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
from mmo_app_server import (
|
||
|
|
APP_SERVER_INITIALIZE_TIMEOUT_SECONDS,
|
||
|
|
APP_SERVER_LIFECYCLE_TIMEOUT_SECONDS,
|
||
|
|
APP_SERVER_RECOVERY_DELAYS_SECONDS,
|
||
|
|
AppServerClient,
|
||
|
|
AppServerError,
|
||
|
|
PersistentThreadHost,
|
||
|
|
app_server_listen_command,
|
||
|
|
bounded_goal_objective,
|
||
|
|
completed_turn_presentable_text,
|
||
|
|
last_agent_message,
|
||
|
|
normalize_turn_failure,
|
||
|
|
require_app_server_codex_version,
|
||
|
|
retain_partial_evidence,
|
||
|
|
serve_control_socket,
|
||
|
|
turn_input,
|
||
|
|
validate_server_request_response,
|
||
|
|
)
|
||
|
|
from mmo_codex_home import session_environment
|
||
|
|
from mmo_snapshot import load_snapshot
|
||
|
|
from mmo_state import (
|
||
|
|
append_audit,
|
||
|
|
load_session_capabilities,
|
||
|
|
publish_session_record,
|
||
|
|
read_session_record,
|
||
|
|
revoke_session_capabilities,
|
||
|
|
runtime_lock_path,
|
||
|
|
session_dir,
|
||
|
|
terminate_recorded_process_group,
|
||
|
|
)
|
||
|
|
from mmo_util import (
|
||
|
|
atomic_write_json,
|
||
|
|
atomic_write_text,
|
||
|
|
file_lock,
|
||
|
|
package_version,
|
||
|
|
process_matches,
|
||
|
|
process_start_token,
|
||
|
|
utc_now,
|
||
|
|
)
|
||
|
|
from mmo_version import SWITCHYARD_MCP_NAMESPACE_BRIDGE_VERSION
|
||
|
|
|
||
|
|
_SHUTDOWN = threading.Event()
|
||
|
|
|
||
|
|
|
||
|
|
def _signal_handler(_signum: int, _frame: object) -> None:
|
||
|
|
_SHUTDOWN.set()
|
||
|
|
|
||
|
|
|
||
|
|
class RootRunner:
|
||
|
|
"""Own one durable root host; operator clients may attach and detach freely."""
|
||
|
|
|
||
|
|
def __init__(self, session_id: str) -> None:
|
||
|
|
self.directory = session_dir(session_id)
|
||
|
|
self.session_id = session_id
|
||
|
|
self.session = read_session_record(self.directory)
|
||
|
|
self.snapshot = load_snapshot(str(self.session["snapshot_hash"]))
|
||
|
|
self.agent_id = str(self.session["root_agent"])
|
||
|
|
self.agent = self.snapshot["resolved"]["agents"][self.agent_id]
|
||
|
|
self.events_path = self.directory / "root-events.jsonl"
|
||
|
|
self.stderr_path = self.directory / "root-stderr.log"
|
||
|
|
self.result_path = self.directory / "root-result.md"
|
||
|
|
self.partial_path = self.directory / "root-partial-result.md"
|
||
|
|
self.socket_path = Path(str(self.session["root_app_server_socket"]))
|
||
|
|
self.control_path = Path(str(self.session["root_control_socket"]))
|
||
|
|
self.state_lock = threading.RLock()
|
||
|
|
self.state: dict[str, Any] = {
|
||
|
|
"thread_id": self.session.get("root_thread_id"),
|
||
|
|
"active_turn_id": self.session.get("active_root_turn_id"),
|
||
|
|
"last_turn_id": self.session.get("root_last_turn_id"),
|
||
|
|
"turn_start_pending": bool(self.session.get("root_turn_start_pending", False)),
|
||
|
|
"control_revision": int(self.session.get("root_control_revision", 0)),
|
||
|
|
"finalize_requested": bool(self.session.get("root_finalizing", False)),
|
||
|
|
}
|
||
|
|
self.host = PersistentThreadHost(
|
||
|
|
state=self.state,
|
||
|
|
state_lock=self.state_lock,
|
||
|
|
on_state_change=self._on_state_change,
|
||
|
|
)
|
||
|
|
self.client: AppServerClient | None = None
|
||
|
|
self.stop_status: str | None = None
|
||
|
|
self.pause_exit = False
|
||
|
|
self.retryable_failure_detach_exit = False
|
||
|
|
self.control_stop = threading.Event()
|
||
|
|
self.last_progress = time.monotonic()
|
||
|
|
self.stall_reported = False
|
||
|
|
self.lifecycle_timeout = float(
|
||
|
|
self.session.get(
|
||
|
|
"root_app_server_lifecycle_timeout_seconds",
|
||
|
|
APP_SERVER_LIFECYCLE_TIMEOUT_SECONDS,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
self.finalize_turn_id: str | None = None
|
||
|
|
self.finalize_deadline: float | None = None
|
||
|
|
self.finalize_status = "completed"
|
||
|
|
self.bootstrap_goal_pending = bool(self.session.get("root_goal_bootstrap_pending", False))
|
||
|
|
|
||
|
|
def _has_durable_work(self) -> bool:
|
||
|
|
return bool(
|
||
|
|
self.host.active_turn_id
|
||
|
|
or self.host.last_turn_id
|
||
|
|
or self.state.get("turn_start_pending") is True
|
||
|
|
or isinstance(self.host.completed_turn, Mapping)
|
||
|
|
or isinstance(self.state.get("goal"), Mapping)
|
||
|
|
)
|
||
|
|
|
||
|
|
def _attached_root_client(self) -> bool:
|
||
|
|
"""Return whether the recorded foreground client still owns this session."""
|
||
|
|
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
return process_matches(
|
||
|
|
current.get("root_client_pid"),
|
||
|
|
current.get("root_client_start_token"),
|
||
|
|
)
|
||
|
|
|
||
|
|
def _interactive_goal_objective(self) -> str:
|
||
|
|
return bounded_goal_objective(
|
||
|
|
"Continue the operator's ongoing work in this persistent Codex MMO session. "
|
||
|
|
"Treat each accepted interactive message as guidance within that ongoing project. "
|
||
|
|
"Keep ownership of the critical path and integrate delegated evidence. Do not mark "
|
||
|
|
"the goal complete merely because one turn ends; complete it only when the overall "
|
||
|
|
"objective is genuinely achieved."
|
||
|
|
)
|
||
|
|
|
||
|
|
def _eligible_root_thread(self, thread: Mapping[str, Any]) -> bool:
|
||
|
|
"""Return whether one Codex thread is a persistent top-level root context."""
|
||
|
|
|
||
|
|
thread_id = thread.get("id")
|
||
|
|
cwd = thread.get("cwd")
|
||
|
|
return bool(
|
||
|
|
isinstance(thread_id, str)
|
||
|
|
and thread_id
|
||
|
|
and thread.get("ephemeral") is False
|
||
|
|
and thread.get("parentThreadId") is None
|
||
|
|
and thread.get("forkedFromId") is None
|
||
|
|
and thread.get("agentRole") is None
|
||
|
|
and thread.get("agentNickname") is None
|
||
|
|
and isinstance(cwd, str)
|
||
|
|
and Path(cwd).resolve() == Path(str(self.session["cwd"])).resolve()
|
||
|
|
)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _thread_sort_key(thread: Mapping[str, Any]) -> tuple[float, str]:
|
||
|
|
created = thread.get("createdAt")
|
||
|
|
timestamp = (
|
||
|
|
float(created)
|
||
|
|
if isinstance(created, (int, float)) and not isinstance(created, bool)
|
||
|
|
else 0.0
|
||
|
|
)
|
||
|
|
return timestamp, str(thread.get("id") or "")
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _thread_projection(thread: Mapping[str, Any]) -> dict[str, Any]:
|
||
|
|
turns = [value for value in thread.get("turns", []) if isinstance(value, Mapping)]
|
||
|
|
active = next(
|
||
|
|
(value for value in reversed(turns) if value.get("status") == "inProgress"),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
last = turns[-1] if turns else None
|
||
|
|
goal = thread.get("goal")
|
||
|
|
projection: dict[str, Any] = {
|
||
|
|
"root_thread_status": thread.get("status"),
|
||
|
|
"active_root_turn_id": (active.get("id") if isinstance(active, Mapping) else None),
|
||
|
|
"root_last_turn_id": last.get("id") if isinstance(last, Mapping) else None,
|
||
|
|
"root_turn_start_pending": False,
|
||
|
|
"root_pending_request_count": 0,
|
||
|
|
"root_token_usage": None,
|
||
|
|
"root_last_item_id": None,
|
||
|
|
"root_last_item_type": None,
|
||
|
|
"root_last_observability_event": None,
|
||
|
|
}
|
||
|
|
if isinstance(goal, Mapping):
|
||
|
|
projection.update(
|
||
|
|
root_goal_status=goal.get("status"),
|
||
|
|
root_goal_objective=goal.get("objective"),
|
||
|
|
root_goal_token_budget=goal.get("tokenBudget"),
|
||
|
|
root_goal_tokens_used=goal.get("tokensUsed", 0),
|
||
|
|
root_goal_time_used_seconds=goal.get("timeUsedSeconds", 0),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
projection.update(
|
||
|
|
root_goal_status=None,
|
||
|
|
root_goal_objective=None,
|
||
|
|
root_goal_token_budget=None,
|
||
|
|
root_goal_tokens_used=0,
|
||
|
|
root_goal_time_used_seconds=0,
|
||
|
|
)
|
||
|
|
return projection
|
||
|
|
|
||
|
|
def _commit_root_thread(
|
||
|
|
self,
|
||
|
|
thread: Mapping[str, Any],
|
||
|
|
*,
|
||
|
|
reason: str,
|
||
|
|
require_attached_client: bool,
|
||
|
|
) -> bool:
|
||
|
|
"""Atomically make one top-level Codex thread the current logical root."""
|
||
|
|
|
||
|
|
if not self._eligible_root_thread(thread):
|
||
|
|
return False
|
||
|
|
thread_id = str(thread["id"])
|
||
|
|
now = utc_now()
|
||
|
|
with file_lock(runtime_lock_path()):
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("status") not in {
|
||
|
|
"starting",
|
||
|
|
"running",
|
||
|
|
"detached",
|
||
|
|
"paused",
|
||
|
|
"suspended",
|
||
|
|
}:
|
||
|
|
return False
|
||
|
|
current_id = current.get("root_thread_id")
|
||
|
|
if current_id == thread_id:
|
||
|
|
current.update(self._thread_projection(thread))
|
||
|
|
current["last_active_at"] = now
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
self.session = current
|
||
|
|
return False
|
||
|
|
lineage = [dict(value) for value in current.get("root_thread_lineage", [])]
|
||
|
|
if thread_id in {value.get("thread_id") for value in lineage}:
|
||
|
|
return False
|
||
|
|
if require_attached_client:
|
||
|
|
with self.state_lock:
|
||
|
|
host_thread_id = self.state.get("thread_id")
|
||
|
|
host_active_turn_id = self.state.get("active_turn_id")
|
||
|
|
host_turn_start_pending = bool(self.state.get("turn_start_pending"))
|
||
|
|
if (
|
||
|
|
not isinstance(current_id, str)
|
||
|
|
or current_id != host_thread_id
|
||
|
|
or host_active_turn_id is not None
|
||
|
|
or host_turn_start_pending
|
||
|
|
or not process_matches(
|
||
|
|
current.get("root_client_pid"), current.get("root_client_start_token")
|
||
|
|
)
|
||
|
|
):
|
||
|
|
return False
|
||
|
|
generation = int(current.get("root_thread_generation", 0)) + 1
|
||
|
|
needs_interactive_goal = bool(
|
||
|
|
reason != "initial"
|
||
|
|
and current.get("session_kind") == "interactive"
|
||
|
|
and self.agent["execution_mode"] == "goal"
|
||
|
|
and not isinstance(thread.get("goal"), Mapping)
|
||
|
|
)
|
||
|
|
transition = {
|
||
|
|
"from_thread_id": current_id,
|
||
|
|
"to_thread_id": thread_id,
|
||
|
|
"generation": generation,
|
||
|
|
"observed_at": now,
|
||
|
|
"reason": reason,
|
||
|
|
}
|
||
|
|
current["root_thread_transition"] = transition
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
if lineage:
|
||
|
|
lineage[-1]["superseded_at"] = now
|
||
|
|
lineage[-1]["successor_thread_id"] = thread_id
|
||
|
|
lineage.append(
|
||
|
|
{
|
||
|
|
"generation": generation,
|
||
|
|
"thread_id": thread_id,
|
||
|
|
"codex_session_id": str(thread.get("sessionId") or thread_id),
|
||
|
|
"adopted_at": now,
|
||
|
|
"reason": reason,
|
||
|
|
"created_at": thread.get("createdAt"),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
current.update(
|
||
|
|
root_thread_id=thread_id,
|
||
|
|
root_thread_generation=generation,
|
||
|
|
root_thread_lineage=lineage,
|
||
|
|
root_thread_transition=None,
|
||
|
|
root_control_revision=int(current.get("root_control_revision", 0)) + 1,
|
||
|
|
root_last_progress_at=now,
|
||
|
|
last_active_at=now,
|
||
|
|
root_goal_bootstrap_pending=needs_interactive_goal,
|
||
|
|
**self._thread_projection(thread),
|
||
|
|
)
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["control_revision"] = int(current["root_control_revision"])
|
||
|
|
self.session = current
|
||
|
|
self.host.adopt_thread(thread)
|
||
|
|
self.last_progress = time.monotonic()
|
||
|
|
self.stall_reported = False
|
||
|
|
if needs_interactive_goal:
|
||
|
|
self.bootstrap_goal_pending = True
|
||
|
|
append_audit(
|
||
|
|
self.session_id,
|
||
|
|
"root_thread_adopted",
|
||
|
|
previous_thread_id=current_id,
|
||
|
|
root_thread_id=thread_id,
|
||
|
|
generation=generation,
|
||
|
|
reason=reason,
|
||
|
|
)
|
||
|
|
return True
|
||
|
|
|
||
|
|
def _record_opened_thread(self, thread: Mapping[str, Any], *, mode: str) -> None:
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("root_thread_id") is None:
|
||
|
|
if not self._commit_root_thread(
|
||
|
|
thread,
|
||
|
|
reason="initial",
|
||
|
|
require_attached_client=False,
|
||
|
|
):
|
||
|
|
raise AppServerError("initial root thread could not be recorded")
|
||
|
|
return
|
||
|
|
if current.get("root_thread_id") != thread.get("id"):
|
||
|
|
raise AppServerError("opened root thread disagrees with canonical session state")
|
||
|
|
self._commit_root_thread(
|
||
|
|
thread,
|
||
|
|
reason="resume" if mode == "resume" else "initial",
|
||
|
|
require_attached_client=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _list_root_threads(self, client: AppServerClient) -> list[dict[str, Any]]:
|
||
|
|
cursor: str | None = None
|
||
|
|
seen: set[str] = set()
|
||
|
|
threads: list[dict[str, Any]] = []
|
||
|
|
while True:
|
||
|
|
params: dict[str, Any] = {
|
||
|
|
"archived": False,
|
||
|
|
"limit": 100,
|
||
|
|
"sortDirection": "asc",
|
||
|
|
"sortKey": "created_at",
|
||
|
|
"useStateDbOnly": True,
|
||
|
|
}
|
||
|
|
if cursor is not None:
|
||
|
|
params["cursor"] = cursor
|
||
|
|
response = client.request("thread/list", params, timeout=self.lifecycle_timeout)
|
||
|
|
data = response.get("data") if isinstance(response, Mapping) else None
|
||
|
|
if not isinstance(data, list) or not all(isinstance(value, Mapping) for value in data):
|
||
|
|
raise AppServerError("thread/list returned invalid root-thread data")
|
||
|
|
threads.extend(dict(value) for value in data if self._eligible_root_thread(value))
|
||
|
|
next_cursor = response.get("nextCursor")
|
||
|
|
if next_cursor is None:
|
||
|
|
return sorted(threads, key=self._thread_sort_key)
|
||
|
|
if not isinstance(next_cursor, str) or next_cursor in seen:
|
||
|
|
raise AppServerError("thread/list returned an invalid root-thread cursor")
|
||
|
|
seen.add(next_cursor)
|
||
|
|
cursor = next_cursor
|
||
|
|
|
||
|
|
def _read_root_thread(
|
||
|
|
self,
|
||
|
|
client: AppServerClient,
|
||
|
|
summary: Mapping[str, Any],
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Resolve one list summary to authoritative turn and goal state."""
|
||
|
|
|
||
|
|
thread_id = str(summary["id"])
|
||
|
|
response = client.request(
|
||
|
|
"thread/read",
|
||
|
|
{"threadId": thread_id, "includeTurns": True},
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
)
|
||
|
|
thread = response.get("thread") if isinstance(response, Mapping) else None
|
||
|
|
if (
|
||
|
|
not isinstance(thread, Mapping)
|
||
|
|
or thread.get("id") != thread_id
|
||
|
|
or not self._eligible_root_thread(thread)
|
||
|
|
):
|
||
|
|
raise AppServerError("thread/read returned an invalid root successor")
|
||
|
|
return dict(thread)
|
||
|
|
|
||
|
|
def _recover_root_successors(self, client: AppServerClient) -> None:
|
||
|
|
current_id = self.session.get("root_thread_id")
|
||
|
|
if not isinstance(current_id, str):
|
||
|
|
return
|
||
|
|
threads = self._list_root_threads(client)
|
||
|
|
by_id = {str(value["id"]): value for value in threads}
|
||
|
|
current_thread = by_id.get(current_id)
|
||
|
|
if current_thread is not None:
|
||
|
|
current_key = self._thread_sort_key(current_thread)
|
||
|
|
else:
|
||
|
|
lineage = self.session.get("root_thread_lineage", [])
|
||
|
|
current_row = next(
|
||
|
|
(
|
||
|
|
value
|
||
|
|
for value in reversed(lineage)
|
||
|
|
if isinstance(value, Mapping) and value.get("thread_id") == current_id
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
created_at = current_row.get("created_at") if isinstance(current_row, Mapping) else None
|
||
|
|
if not isinstance(created_at, (int, float)) or isinstance(created_at, bool):
|
||
|
|
# State-db indexing may briefly lag the rollout that remains
|
||
|
|
# directly resumable by identity. Never trade that canonical
|
||
|
|
# path for an inferred successor without an ordering anchor.
|
||
|
|
return
|
||
|
|
current_key = float(created_at), current_id
|
||
|
|
transition = self.session.get("root_thread_transition")
|
||
|
|
transition_id = transition.get("to_thread_id") if isinstance(transition, Mapping) else None
|
||
|
|
if isinstance(transition_id, str):
|
||
|
|
# A staged transition is stronger evidence than timestamp ordering.
|
||
|
|
# Never skip or overwrite it merely because the state-db index has
|
||
|
|
# not exposed its target yet.
|
||
|
|
if transition_id not in by_id:
|
||
|
|
return
|
||
|
|
self._commit_root_thread(
|
||
|
|
self._read_root_thread(client, by_id[transition_id]),
|
||
|
|
reason="crash_recovery",
|
||
|
|
require_attached_client=False,
|
||
|
|
)
|
||
|
|
current_key = self._thread_sort_key(by_id[transition_id])
|
||
|
|
successors = [value for value in threads if self._thread_sort_key(value) > current_key]
|
||
|
|
for successor in successors:
|
||
|
|
self._commit_root_thread(
|
||
|
|
self._read_root_thread(client, successor),
|
||
|
|
reason="crash_recovery",
|
||
|
|
require_attached_client=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
def recoverable_descendants(self) -> list[dict[str, Any]]:
|
||
|
|
"""Return durable descendants that still require a live session."""
|
||
|
|
|
||
|
|
# Imported lazily to keep the standalone runner's transport boundary
|
||
|
|
# acyclic during module initialization.
|
||
|
|
from mmo_runtime import iter_jobs
|
||
|
|
from mmo_state import RECOVERABLE_JOB_STATUSES
|
||
|
|
|
||
|
|
run_id = self.session.get("current_run_id")
|
||
|
|
return [
|
||
|
|
job
|
||
|
|
for job in iter_jobs(strict=True)
|
||
|
|
if job.get("session_id") == self.session_id
|
||
|
|
and job.get("run_id") == run_id
|
||
|
|
and job.get("status") in RECOVERABLE_JOB_STATUSES
|
||
|
|
]
|
||
|
|
|
||
|
|
def refresh_native_runs(self) -> list[dict[str, Any]]:
|
||
|
|
"""Project every native descendant thread into stable MMO run identity."""
|
||
|
|
|
||
|
|
client = self.client
|
||
|
|
root_thread_id = self.host.thread_id
|
||
|
|
if client is None or root_thread_id is None:
|
||
|
|
return []
|
||
|
|
cursor: str | None = None
|
||
|
|
threads: list[dict[str, Any]] = []
|
||
|
|
seen_cursors: set[str] = set()
|
||
|
|
while True:
|
||
|
|
params: dict[str, Any] = {
|
||
|
|
"ancestorThreadId": root_thread_id,
|
||
|
|
"archived": False,
|
||
|
|
"limit": 100,
|
||
|
|
"sortDirection": "asc",
|
||
|
|
"sortKey": "created_at",
|
||
|
|
"sourceKinds": [
|
||
|
|
"subAgent",
|
||
|
|
"subAgentReview",
|
||
|
|
"subAgentCompact",
|
||
|
|
"subAgentThreadSpawn",
|
||
|
|
"subAgentOther",
|
||
|
|
],
|
||
|
|
"useStateDbOnly": True,
|
||
|
|
}
|
||
|
|
if cursor is not None:
|
||
|
|
params["cursor"] = cursor
|
||
|
|
response = client.request("thread/list", params)
|
||
|
|
data = response.get("data") if isinstance(response, Mapping) else None
|
||
|
|
if not isinstance(data, list) or not all(isinstance(item, Mapping) for item in data):
|
||
|
|
raise AppServerError("thread/list returned invalid native-agent data")
|
||
|
|
threads.extend(dict(item) for item in data)
|
||
|
|
next_cursor = response.get("nextCursor")
|
||
|
|
if next_cursor is None:
|
||
|
|
break
|
||
|
|
if not isinstance(next_cursor, str) or next_cursor in seen_cursors:
|
||
|
|
raise AppServerError("thread/list returned an invalid cursor sequence")
|
||
|
|
seen_cursors.add(next_cursor)
|
||
|
|
cursor = next_cursor
|
||
|
|
|
||
|
|
native_names = {
|
||
|
|
str(agent.get("native_name")): agent_id
|
||
|
|
for agent_id, agent in self.snapshot["resolved"]["agents"].items()
|
||
|
|
if "native" in agent.get("backends", [])
|
||
|
|
}
|
||
|
|
existing = self.session.get("root_native_runs", {})
|
||
|
|
existing_by_thread = (
|
||
|
|
{
|
||
|
|
str(value.get("thread_id")): dict(value)
|
||
|
|
for value in existing.values()
|
||
|
|
if isinstance(value, Mapping) and isinstance(value.get("thread_id"), str)
|
||
|
|
}
|
||
|
|
if isinstance(existing, Mapping)
|
||
|
|
else {}
|
||
|
|
)
|
||
|
|
projected: dict[str, dict[str, Any]] = {}
|
||
|
|
projected_thread_ids: set[str] = set()
|
||
|
|
for thread in threads:
|
||
|
|
thread_id = thread.get("id")
|
||
|
|
role = native_names.get(str(thread.get("agentRole")))
|
||
|
|
if not isinstance(thread_id, str) or role is None:
|
||
|
|
continue
|
||
|
|
prior = existing_by_thread.get(thread_id, {})
|
||
|
|
run_ref = prior.get("agent_run_ref")
|
||
|
|
if not isinstance(run_ref, str) or not run_ref.startswith("ar_"):
|
||
|
|
run_ref = "ar_" + secrets.token_urlsafe(24)
|
||
|
|
reported_status = thread.get("status")
|
||
|
|
reported_status_type = (
|
||
|
|
reported_status.get("type") if isinstance(reported_status, Mapping) else None
|
||
|
|
)
|
||
|
|
row = {
|
||
|
|
"agent_run_ref": run_ref,
|
||
|
|
"agent": role,
|
||
|
|
"backend": "native",
|
||
|
|
"thread_id": thread_id,
|
||
|
|
"parent_thread_id": thread.get("parentThreadId"),
|
||
|
|
"nickname": thread.get("agentNickname"),
|
||
|
|
"status": (
|
||
|
|
prior.get("status")
|
||
|
|
if prior.get("status") in {"paused", "stopped"}
|
||
|
|
or (prior.get("status") == "detached" and reported_status_type == "active")
|
||
|
|
else reported_status
|
||
|
|
),
|
||
|
|
"can_accept_direct_input": thread.get("canAcceptDirectInput"),
|
||
|
|
"created_at": thread.get("createdAt"),
|
||
|
|
"updated_at": thread.get("updatedAt"),
|
||
|
|
"control_revision": int(prior.get("control_revision", 0)),
|
||
|
|
}
|
||
|
|
projected[run_ref] = row
|
||
|
|
projected_thread_ids.add(thread_id)
|
||
|
|
for run_ref, row in existing.items() if isinstance(existing, Mapping) else ():
|
||
|
|
if (
|
||
|
|
isinstance(row, Mapping)
|
||
|
|
and row.get("status") == "stopped"
|
||
|
|
and row.get("thread_id") not in projected_thread_ids
|
||
|
|
):
|
||
|
|
projected[str(run_ref)] = dict(row)
|
||
|
|
self._update(root_native_runs=projected)
|
||
|
|
return list(projected.values())
|
||
|
|
|
||
|
|
def _update(self, **changes: Any) -> dict[str, Any]:
|
||
|
|
with file_lock(runtime_lock_path()):
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("status") in {"finishing", "stopping", "cancelling"} and changes.get(
|
||
|
|
"status"
|
||
|
|
) not in {"completed", "stopped", "failed", "cancelled"}:
|
||
|
|
changes.pop("status", None)
|
||
|
|
if (
|
||
|
|
current.get("status") in {"completed", "stopped", "failed", "cancelled"}
|
||
|
|
and "status" in changes
|
||
|
|
and changes.get("status") != current.get("status")
|
||
|
|
):
|
||
|
|
self.session = current
|
||
|
|
return current
|
||
|
|
current.update(changes)
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
self.session = current
|
||
|
|
return current
|
||
|
|
|
||
|
|
def _on_state_change(
|
||
|
|
self,
|
||
|
|
changes: dict[str, Any],
|
||
|
|
message: Mapping[str, Any],
|
||
|
|
) -> None:
|
||
|
|
started = changes.get("thread_started")
|
||
|
|
if isinstance(started, Mapping):
|
||
|
|
adopted = self._commit_root_thread(
|
||
|
|
started,
|
||
|
|
reason="fresh_context",
|
||
|
|
require_attached_client=True,
|
||
|
|
)
|
||
|
|
if not adopted:
|
||
|
|
current_id = read_session_record(self.directory).get("root_thread_id")
|
||
|
|
candidate_id = started.get("id")
|
||
|
|
if isinstance(current_id, str) and candidate_id != current_id:
|
||
|
|
append_audit(
|
||
|
|
self.session_id,
|
||
|
|
"root_thread_candidate_ignored",
|
||
|
|
root_thread_id=current_id,
|
||
|
|
candidate_thread_id=candidate_id,
|
||
|
|
)
|
||
|
|
return
|
||
|
|
now = utc_now()
|
||
|
|
persisted: dict[str, Any] = {"root_last_progress_at": now, "last_active_at": now}
|
||
|
|
self.last_progress = time.monotonic()
|
||
|
|
self.stall_reported = False
|
||
|
|
if "active_turn_id" in changes:
|
||
|
|
persisted["active_root_turn_id"] = changes["active_turn_id"]
|
||
|
|
if "last_turn_id" in changes:
|
||
|
|
persisted["root_last_turn_id"] = changes["last_turn_id"]
|
||
|
|
if "turn_start_pending" in changes:
|
||
|
|
persisted["root_turn_start_pending"] = bool(changes["turn_start_pending"])
|
||
|
|
if "pending_request_count" in changes:
|
||
|
|
persisted["root_pending_request_count"] = changes["pending_request_count"]
|
||
|
|
if "thread_status" in changes:
|
||
|
|
persisted["root_thread_status"] = changes["thread_status"]
|
||
|
|
if "token_usage" in changes:
|
||
|
|
persisted["root_token_usage"] = changes["token_usage"]
|
||
|
|
if "turn_failure" in changes:
|
||
|
|
persisted["failure"] = changes["turn_failure"]
|
||
|
|
if changes["turn_failure"] is None:
|
||
|
|
persisted["error"] = None
|
||
|
|
goal = changes.get("goal")
|
||
|
|
if isinstance(goal, Mapping):
|
||
|
|
persisted.update(
|
||
|
|
root_goal_status=goal.get("status"),
|
||
|
|
root_goal_objective=goal.get("objective"),
|
||
|
|
root_goal_token_budget=goal.get("tokenBudget"),
|
||
|
|
root_goal_tokens_used=goal.get("tokensUsed", 0),
|
||
|
|
root_goal_time_used_seconds=goal.get("timeUsedSeconds", 0),
|
||
|
|
)
|
||
|
|
goal_status = goal.get("status")
|
||
|
|
try:
|
||
|
|
session_status = str(read_session_record(self.directory).get("status"))
|
||
|
|
except Exception:
|
||
|
|
# Observability callbacks must not kill the transport reader if
|
||
|
|
# a concurrent atomic session replacement is briefly unreadable.
|
||
|
|
session_status = str(self.session.get("status"))
|
||
|
|
transition_active = session_status in {"finishing", "stopping", "cancelling"}
|
||
|
|
if (
|
||
|
|
goal_status == "active"
|
||
|
|
and session_status not in {"detached", "paused", "suspended"}
|
||
|
|
and not transition_active
|
||
|
|
):
|
||
|
|
persisted["status"] = "running"
|
||
|
|
elif (
|
||
|
|
goal_status == "paused"
|
||
|
|
and session_status not in {"detached", "suspended"}
|
||
|
|
and not self.bootstrap_goal_pending
|
||
|
|
and not transition_active
|
||
|
|
):
|
||
|
|
persisted["status"] = "paused"
|
||
|
|
elif (
|
||
|
|
goal_status == "blocked"
|
||
|
|
and session_status != "detached"
|
||
|
|
and not transition_active
|
||
|
|
and not isinstance(self.state.get("turn_failure"), Mapping)
|
||
|
|
):
|
||
|
|
persisted["status"] = "paused"
|
||
|
|
elif (
|
||
|
|
goal_status in {"usageLimited", "budgetLimited"}
|
||
|
|
and session_status != "detached"
|
||
|
|
and not transition_active
|
||
|
|
):
|
||
|
|
persisted["status"] = "suspended"
|
||
|
|
if isinstance(changes.get("active_turn_id"), str):
|
||
|
|
try:
|
||
|
|
session_status = str(read_session_record(self.directory).get("status"))
|
||
|
|
except Exception:
|
||
|
|
session_status = str(self.session.get("status"))
|
||
|
|
if session_status not in {
|
||
|
|
"detached",
|
||
|
|
"paused",
|
||
|
|
"suspended",
|
||
|
|
"finishing",
|
||
|
|
"stopping",
|
||
|
|
"cancelling",
|
||
|
|
}:
|
||
|
|
persisted["status"] = "running"
|
||
|
|
if "last_item" in changes:
|
||
|
|
item = changes["last_item"]
|
||
|
|
if isinstance(item, Mapping):
|
||
|
|
persisted["root_last_item_id"] = item.get("id")
|
||
|
|
persisted["root_last_item_type"] = item.get("type")
|
||
|
|
if "last_observability_event" in changes:
|
||
|
|
persisted["root_last_observability_event"] = changes["last_observability_event"]
|
||
|
|
method = message.get("method")
|
||
|
|
if method in {"mmo/turn/reset", "mmo/turn/accepted"}:
|
||
|
|
# These synthetic transitions participate in exact crash
|
||
|
|
# reconciliation. A turn must not be sent, or reported accepted,
|
||
|
|
# unless its recovery marker is durable.
|
||
|
|
self._update(**persisted)
|
||
|
|
else:
|
||
|
|
with contextlib.suppress(Exception):
|
||
|
|
self._update(**persisted)
|
||
|
|
if isinstance(method, str) and method in {
|
||
|
|
"thread/goal/updated",
|
||
|
|
"thread/status/changed",
|
||
|
|
"model/rerouted",
|
||
|
|
"model/verification",
|
||
|
|
"warning",
|
||
|
|
"error",
|
||
|
|
"account/rateLimits/updated",
|
||
|
|
}:
|
||
|
|
with contextlib.suppress(Exception):
|
||
|
|
append_audit(self.session_id, "root_app_server_event", method=method)
|
||
|
|
|
||
|
|
def _publish_app_server(self, pid: int) -> None:
|
||
|
|
token = process_start_token(pid)
|
||
|
|
if token is None:
|
||
|
|
raise AppServerError("root app-server process could not be fingerprinted")
|
||
|
|
with file_lock(runtime_lock_path()):
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("status") not in {
|
||
|
|
"starting",
|
||
|
|
"running",
|
||
|
|
"detached",
|
||
|
|
"paused",
|
||
|
|
"suspended",
|
||
|
|
}:
|
||
|
|
raise AppServerError("root admission closed during app-server bootstrap")
|
||
|
|
current.update(
|
||
|
|
root_app_server_pid=pid,
|
||
|
|
root_app_server_pgid=pid,
|
||
|
|
root_app_server_start_token=token,
|
||
|
|
root_app_server_process_group_isolated=True,
|
||
|
|
)
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
self.session = current
|
||
|
|
|
||
|
|
def _new_client(self, *, start_host: bool) -> AppServerClient:
|
||
|
|
environment = session_environment(self.session, self.agent_id)
|
||
|
|
command = None
|
||
|
|
if start_host:
|
||
|
|
flags = self.session["homes"][self.agent_id].get("command_flags", [])
|
||
|
|
command = app_server_listen_command(
|
||
|
|
str(self.session["codex_binary"]),
|
||
|
|
flags,
|
||
|
|
self.socket_path,
|
||
|
|
)
|
||
|
|
return AppServerClient(
|
||
|
|
socket_path=self.socket_path,
|
||
|
|
command=command,
|
||
|
|
cwd=Path(str(self.session["cwd"])),
|
||
|
|
env=environment,
|
||
|
|
events_path=self.events_path,
|
||
|
|
stderr_path=self.stderr_path,
|
||
|
|
approval_policy=str(self.agent["approval_policy"]),
|
||
|
|
on_message=self.host.on_message,
|
||
|
|
)
|
||
|
|
|
||
|
|
def connect(self) -> None:
|
||
|
|
pid = self.session.get("root_app_server_pid")
|
||
|
|
token = self.session.get("root_app_server_start_token")
|
||
|
|
host_alive = process_matches(pid, token)
|
||
|
|
if not host_alive:
|
||
|
|
# The executable at a pinned path can be replaced while a durable
|
||
|
|
# session is detached. Every replacement host must re-check the
|
||
|
|
# reviewed protocol release before it can touch the saved thread.
|
||
|
|
require_app_server_codex_version(str(self.session["codex_binary"]))
|
||
|
|
if not host_alive and (self.socket_path.exists() or self.socket_path.is_symlink()):
|
||
|
|
if self.socket_path.is_symlink() or not self.socket_path.is_socket():
|
||
|
|
raise AppServerError("recorded root app-server socket is unsafe")
|
||
|
|
self.socket_path.unlink()
|
||
|
|
client = self._new_client(start_host=not host_alive)
|
||
|
|
try:
|
||
|
|
client.start(
|
||
|
|
timeout=APP_SERVER_INITIALIZE_TIMEOUT_SECONDS,
|
||
|
|
on_started=self._publish_app_server if not host_alive else None,
|
||
|
|
)
|
||
|
|
self.host.attach_client(client)
|
||
|
|
self.client = client
|
||
|
|
self._recover_root_successors(client)
|
||
|
|
self.session = read_session_record(self.directory)
|
||
|
|
dynamic_tools: list[dict[str, Any]] = []
|
||
|
|
if self.agent["driver"] == "switchyard":
|
||
|
|
observed_switchyard = self.session.get("switchyard_version")
|
||
|
|
if not isinstance(observed_switchyard, str):
|
||
|
|
raise AppServerError("Switchyard session has no pinned gateway version")
|
||
|
|
if observed_switchyard == SWITCHYARD_MCP_NAMESPACE_BRIDGE_VERSION:
|
||
|
|
dynamic_tools = client.install_switchyard_mcp_bridge(
|
||
|
|
timeout=self.lifecycle_timeout
|
||
|
|
)
|
||
|
|
mode = "resume" if isinstance(self.session.get("root_thread_id"), str) else "start"
|
||
|
|
params: dict[str, Any]
|
||
|
|
if mode == "resume":
|
||
|
|
params = {
|
||
|
|
"threadId": self.session["root_thread_id"],
|
||
|
|
"cwd": self.session["cwd"],
|
||
|
|
"sandbox": self.session.get("root_sandbox_mode", self.agent["permissions"]),
|
||
|
|
"approvalPolicy": self.agent["approval_policy"],
|
||
|
|
"excludeTurns": False,
|
||
|
|
}
|
||
|
|
else:
|
||
|
|
params = {
|
||
|
|
"cwd": self.session["cwd"],
|
||
|
|
"sandbox": self.session.get("root_sandbox_mode", self.agent["permissions"]),
|
||
|
|
"approvalPolicy": self.agent["approval_policy"],
|
||
|
|
"allowProviderModelFallback": False,
|
||
|
|
"ephemeral": False,
|
||
|
|
"historyMode": "paginated",
|
||
|
|
}
|
||
|
|
if dynamic_tools:
|
||
|
|
params["dynamicTools"] = dynamic_tools
|
||
|
|
prior_turn_id = self.state.get("last_turn_id") or self.state.get("active_turn_id")
|
||
|
|
thread = self.host.open_thread(
|
||
|
|
mode,
|
||
|
|
params,
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
prior_turn_id=prior_turn_id,
|
||
|
|
expected_thread_id=(
|
||
|
|
str(self.session["root_thread_id"]) if mode == "resume" else None
|
||
|
|
),
|
||
|
|
)
|
||
|
|
self._record_opened_thread(thread, mode=mode)
|
||
|
|
self._update(
|
||
|
|
active_root_turn_id=self.state.get("active_turn_id"),
|
||
|
|
root_last_turn_id=self.state.get("last_turn_id"),
|
||
|
|
root_turn_start_pending=bool(self.state.get("turn_start_pending")),
|
||
|
|
root_app_server_protocol="codex-app-server-v2-unix",
|
||
|
|
root_recovery_error=None,
|
||
|
|
)
|
||
|
|
except BaseException:
|
||
|
|
if client.process is not None:
|
||
|
|
client.stop_host()
|
||
|
|
else:
|
||
|
|
client.close()
|
||
|
|
raise
|
||
|
|
|
||
|
|
def start_initial_work(self) -> None:
|
||
|
|
prompt = self.session.get("root_initial_prompt")
|
||
|
|
if not isinstance(prompt, str) or not prompt.strip():
|
||
|
|
if (
|
||
|
|
self.agent["execution_mode"] == "goal"
|
||
|
|
and self.session.get("session_kind") == "interactive"
|
||
|
|
and not isinstance(self.state.get("goal"), Mapping)
|
||
|
|
):
|
||
|
|
self.bootstrap_goal_pending = True
|
||
|
|
goal = self.host.set_goal(
|
||
|
|
objective=self._interactive_goal_objective(),
|
||
|
|
status="paused",
|
||
|
|
token_budget=int(self.agent["goal_token_budget"]),
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
)
|
||
|
|
self._update(
|
||
|
|
root_goal_status=goal["status"],
|
||
|
|
root_goal_objective=goal["objective"],
|
||
|
|
root_goal_token_budget=goal.get("tokenBudget"),
|
||
|
|
root_goal_bootstrap_pending=True,
|
||
|
|
)
|
||
|
|
self._resume_turn_work_if_needed(str(self.session.get("status")))
|
||
|
|
return
|
||
|
|
attachments = [
|
||
|
|
str(item)
|
||
|
|
for item in self.session.get("root_initial_attachments", [])
|
||
|
|
if isinstance(item, str)
|
||
|
|
]
|
||
|
|
if self.agent["execution_mode"] == "goal":
|
||
|
|
goal = self.host.set_goal(
|
||
|
|
objective=bounded_goal_objective(
|
||
|
|
str(self.session.get("root_goal_objective") or prompt)
|
||
|
|
),
|
||
|
|
status="paused",
|
||
|
|
token_budget=int(self.agent["goal_token_budget"]),
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
)
|
||
|
|
if self.host.active_turn_id is None and self.host.completed_turn is None:
|
||
|
|
self.host.start_turn(
|
||
|
|
turn_input(prompt.strip(), attachments),
|
||
|
|
effort=self.agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
goal = self.host.set_goal(
|
||
|
|
status="active",
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
)
|
||
|
|
self._update(
|
||
|
|
status="running",
|
||
|
|
root_goal_status=goal["status"],
|
||
|
|
root_goal_objective=goal["objective"],
|
||
|
|
root_goal_token_budget=goal.get("tokenBudget"),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
if self.host.active_turn_id is None and self.host.completed_turn is None:
|
||
|
|
self.host.start_turn(
|
||
|
|
turn_input(prompt.strip(), attachments),
|
||
|
|
effort=self.agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
self._update(status="running")
|
||
|
|
self._update(root_initial_prompt=None, root_initial_attachments=[])
|
||
|
|
|
||
|
|
def _activate_bootstrap_goal(self) -> None:
|
||
|
|
if not self.bootstrap_goal_pending or self.host.active_turn_id is None:
|
||
|
|
return
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("status") in {
|
||
|
|
"detached",
|
||
|
|
"paused",
|
||
|
|
"suspended",
|
||
|
|
"finishing",
|
||
|
|
"stopping",
|
||
|
|
"cancelling",
|
||
|
|
}:
|
||
|
|
return
|
||
|
|
goal = self.state.get("goal")
|
||
|
|
if not isinstance(goal, Mapping):
|
||
|
|
return
|
||
|
|
if goal.get("status") == "paused":
|
||
|
|
goal = self.host.set_goal(status="active", timeout=self.lifecycle_timeout)
|
||
|
|
self.bootstrap_goal_pending = False
|
||
|
|
self._update(
|
||
|
|
status="running",
|
||
|
|
root_goal_status=goal.get("status"),
|
||
|
|
root_goal_bootstrap_pending=False,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _ensure_bootstrap_goal(self) -> None:
|
||
|
|
"""Restore the ongoing interactive objective after adopting a fresh root thread."""
|
||
|
|
|
||
|
|
if not self.bootstrap_goal_pending or isinstance(self.state.get("goal"), Mapping):
|
||
|
|
return
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("status") in {
|
||
|
|
"finishing",
|
||
|
|
"stopping",
|
||
|
|
"cancelling",
|
||
|
|
"completed",
|
||
|
|
"stopped",
|
||
|
|
"failed",
|
||
|
|
"cancelled",
|
||
|
|
}:
|
||
|
|
return
|
||
|
|
goal = self.host.set_goal(
|
||
|
|
objective=self._interactive_goal_objective(),
|
||
|
|
status="paused",
|
||
|
|
token_budget=int(self.agent["goal_token_budget"]),
|
||
|
|
timeout=self.lifecycle_timeout,
|
||
|
|
)
|
||
|
|
self._update(
|
||
|
|
root_goal_status=goal["status"],
|
||
|
|
root_goal_objective=goal["objective"],
|
||
|
|
root_goal_token_budget=goal.get("tokenBudget"),
|
||
|
|
root_goal_bootstrap_pending=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _resume_turn_work_if_needed(self, lifecycle_status: str) -> None:
|
||
|
|
"""Continue an unfinished turn after host recovery without duplicating work."""
|
||
|
|
|
||
|
|
if (
|
||
|
|
self.agent["execution_mode"] != "turn"
|
||
|
|
or lifecycle_status == "paused"
|
||
|
|
or self.host.active_turn_id is not None
|
||
|
|
or self.host.completed_turn is not None
|
||
|
|
):
|
||
|
|
return
|
||
|
|
self.host.start_turn(
|
||
|
|
turn_input(
|
||
|
|
"Continue the original root task from this persisted app-server thread after "
|
||
|
|
"transport recovery. Use retained evidence and complete the original request.",
|
||
|
|
[],
|
||
|
|
),
|
||
|
|
effort=self.agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
|
||
|
|
def _claim_revision(self, request: Mapping[str, Any], target_thread_id: str) -> int:
|
||
|
|
expected = request.get("expected_revision")
|
||
|
|
root_thread_id = self.host.thread_id
|
||
|
|
if target_thread_id == root_thread_id:
|
||
|
|
with self.state_lock:
|
||
|
|
current = int(self.state.get("control_revision", 0))
|
||
|
|
if (
|
||
|
|
not isinstance(expected, int)
|
||
|
|
or isinstance(expected, bool)
|
||
|
|
or expected != current
|
||
|
|
):
|
||
|
|
raise RuntimeError(f"control revision conflict: expected {current}")
|
||
|
|
current += 1
|
||
|
|
self.state["control_revision"] = current
|
||
|
|
self._update(root_control_revision=current)
|
||
|
|
return current
|
||
|
|
runs = self.session.get("root_native_runs", {})
|
||
|
|
if not isinstance(runs, Mapping):
|
||
|
|
raise RuntimeError("native-agent run registry is unavailable")
|
||
|
|
selected = next(
|
||
|
|
(
|
||
|
|
(run_ref, dict(row))
|
||
|
|
for run_ref, row in runs.items()
|
||
|
|
if isinstance(row, Mapping) and row.get("thread_id") == target_thread_id
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
if selected is None:
|
||
|
|
raise RuntimeError("native-agent run is unavailable")
|
||
|
|
run_ref, row = selected
|
||
|
|
current = int(row.get("control_revision", 0))
|
||
|
|
if not isinstance(expected, int) or isinstance(expected, bool) or expected != current:
|
||
|
|
raise RuntimeError(f"control revision conflict: expected {current}")
|
||
|
|
current += 1
|
||
|
|
row["control_revision"] = current
|
||
|
|
updated = dict(runs)
|
||
|
|
updated[str(run_ref)] = row
|
||
|
|
self._update(root_native_runs=updated)
|
||
|
|
return current
|
||
|
|
|
||
|
|
def _native_row(self, target_thread_id: str) -> tuple[str, dict[str, Any]]:
|
||
|
|
runs = self.session.get("root_native_runs", {})
|
||
|
|
if isinstance(runs, Mapping):
|
||
|
|
for run_ref, row in runs.items():
|
||
|
|
if isinstance(row, Mapping) and row.get("thread_id") == target_thread_id:
|
||
|
|
return str(run_ref), dict(row)
|
||
|
|
raise RuntimeError("native-agent run is unavailable")
|
||
|
|
|
||
|
|
def _update_native_row(self, target_thread_id: str, **changes: Any) -> dict[str, Any]:
|
||
|
|
run_ref, row = self._native_row(target_thread_id)
|
||
|
|
row.update(changes)
|
||
|
|
runs = dict(self.session.get("root_native_runs", {}))
|
||
|
|
runs[run_ref] = row
|
||
|
|
self._update(root_native_runs=runs)
|
||
|
|
return row
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _start_native_turn(
|
||
|
|
client: AppServerClient,
|
||
|
|
thread_id: str,
|
||
|
|
prompt: str,
|
||
|
|
effort: Any,
|
||
|
|
) -> str:
|
||
|
|
response = client.request(
|
||
|
|
"turn/start",
|
||
|
|
{
|
||
|
|
"threadId": thread_id,
|
||
|
|
"input": turn_input(prompt, []),
|
||
|
|
"effort": effort,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
turn = response.get("turn") if isinstance(response, Mapping) else None
|
||
|
|
if not isinstance(turn, Mapping) or not isinstance(turn.get("id"), str):
|
||
|
|
raise RuntimeError("native turn/start returned no turn")
|
||
|
|
return str(turn["id"])
|
||
|
|
|
||
|
|
def _execute_control(self, request: Mapping[str, Any]) -> Any:
|
||
|
|
client = self.client
|
||
|
|
if client is None:
|
||
|
|
raise RuntimeError("root app-server client is unavailable")
|
||
|
|
action = str(request.get("action", ""))
|
||
|
|
mutating = action not in {"list", "inspect", "trace"}
|
||
|
|
thread_id = self.host.thread_id
|
||
|
|
if thread_id is None:
|
||
|
|
raise RuntimeError("root app-server thread is unavailable")
|
||
|
|
target_thread_id = str(request.get("target_thread_id") or thread_id)
|
||
|
|
root_target = target_thread_id == thread_id
|
||
|
|
if action == "list":
|
||
|
|
return {"agents": self.refresh_native_runs()}
|
||
|
|
if action == "finalize" and root_target:
|
||
|
|
terminal_status = str(request.get("terminal_status", "completed"))
|
||
|
|
if terminal_status not in {"completed", "stopped"}:
|
||
|
|
raise ValueError("root terminal_status must be completed or stopped")
|
||
|
|
native_row = None if root_target else self._native_row(target_thread_id)[1]
|
||
|
|
if root_target:
|
||
|
|
target_agent_id = self.agent_id
|
||
|
|
else:
|
||
|
|
if native_row is None:
|
||
|
|
raise RuntimeError("native-agent run is unavailable")
|
||
|
|
target_agent_id = str(native_row["agent"])
|
||
|
|
target_agent = self.snapshot["resolved"]["agents"][target_agent_id]
|
||
|
|
if root_target and action == "fork":
|
||
|
|
raise RuntimeError("the immutable root run cannot be forked as another root")
|
||
|
|
if (
|
||
|
|
not root_target
|
||
|
|
and mutating
|
||
|
|
and isinstance(native_row, Mapping)
|
||
|
|
and native_row.get("status") == "stopped"
|
||
|
|
):
|
||
|
|
raise RuntimeError("cannot control a stopped native run")
|
||
|
|
revision = (
|
||
|
|
self._claim_revision(request, target_thread_id)
|
||
|
|
if mutating
|
||
|
|
else int(
|
||
|
|
self.state.get("control_revision", 0)
|
||
|
|
if root_target
|
||
|
|
else (native_row or {}).get("control_revision", 0)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
thread_response = client.request(
|
||
|
|
"thread/read",
|
||
|
|
{"threadId": target_thread_id, "includeTurns": True},
|
||
|
|
)
|
||
|
|
target_thread = (
|
||
|
|
thread_response.get("thread") if isinstance(thread_response, Mapping) else None
|
||
|
|
)
|
||
|
|
if not isinstance(target_thread, Mapping) or target_thread.get("id") != target_thread_id:
|
||
|
|
raise RuntimeError("target app-server thread is unavailable")
|
||
|
|
active = next(
|
||
|
|
(
|
||
|
|
turn
|
||
|
|
for turn in reversed(target_thread.get("turns", []))
|
||
|
|
if isinstance(turn, Mapping) and turn.get("status") == "inProgress"
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
# The host owns the authoritative live root turn. Persisted thread
|
||
|
|
# history is authoritative for native children, but may lag an active
|
||
|
|
# root notification and must not cause a second root turn to start.
|
||
|
|
turn_id = (
|
||
|
|
self.host.active_turn_id
|
||
|
|
if root_target
|
||
|
|
else active.get("id")
|
||
|
|
if isinstance(active, Mapping)
|
||
|
|
else None
|
||
|
|
)
|
||
|
|
if action == "inspect":
|
||
|
|
pending_requests = client.pending_server_requests_for_thread(target_thread_id)
|
||
|
|
return {
|
||
|
|
"thread_id": target_thread_id,
|
||
|
|
"thread_status": target_thread.get("status"),
|
||
|
|
"agent": target_agent_id,
|
||
|
|
"backend": "root" if root_target else "native",
|
||
|
|
"agent_run_ref": (
|
||
|
|
self.session.get("root_agent_run_ref")
|
||
|
|
if root_target
|
||
|
|
else (native_row or {}).get("agent_run_ref")
|
||
|
|
),
|
||
|
|
"goal": self.state.get("goal") if root_target else None,
|
||
|
|
"active_turn_id": turn_id,
|
||
|
|
"pending_requests": pending_requests,
|
||
|
|
"control_revision": revision,
|
||
|
|
}
|
||
|
|
if action == "trace":
|
||
|
|
return {
|
||
|
|
"events_path": str(self.events_path),
|
||
|
|
"stderr_path": str(self.stderr_path),
|
||
|
|
"result_path": str(self.result_path),
|
||
|
|
}
|
||
|
|
if action == "steer":
|
||
|
|
if turn_id is None:
|
||
|
|
raise RuntimeError("root has no active turn to steer")
|
||
|
|
return client.request(
|
||
|
|
"turn/steer",
|
||
|
|
{
|
||
|
|
"threadId": target_thread_id,
|
||
|
|
"expectedTurnId": str(request.get("expected_turn_id") or turn_id),
|
||
|
|
"input": turn_input(str(request["input"]), []),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
if action in {"interrupt", "pause"}:
|
||
|
|
if action == "pause" and root_target and self.agent["execution_mode"] == "goal":
|
||
|
|
self.host.set_goal(status="paused")
|
||
|
|
result: Any = {"interrupted": False}
|
||
|
|
if turn_id is not None:
|
||
|
|
result = client.request(
|
||
|
|
"turn/interrupt",
|
||
|
|
{"threadId": target_thread_id, "turnId": turn_id},
|
||
|
|
)
|
||
|
|
if action == "pause":
|
||
|
|
if root_target:
|
||
|
|
self.bootstrap_goal_pending = False
|
||
|
|
self.finalize_deadline = None
|
||
|
|
self.finalize_turn_id = None
|
||
|
|
self.finalize_status = "completed"
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["finalize_requested"] = False
|
||
|
|
self._update(
|
||
|
|
status="paused",
|
||
|
|
root_goal_status=(
|
||
|
|
"paused" if self.agent["execution_mode"] == "goal" else None
|
||
|
|
),
|
||
|
|
root_finalizing=False,
|
||
|
|
root_goal_bootstrap_pending=False,
|
||
|
|
)
|
||
|
|
if request.get("retire_host") is True:
|
||
|
|
self.pause_exit = True
|
||
|
|
_SHUTDOWN.set()
|
||
|
|
else:
|
||
|
|
self._update_native_row(target_thread_id, status="paused")
|
||
|
|
elif target_agent["execution_mode"] == "turn":
|
||
|
|
if root_target:
|
||
|
|
self._update(status="paused")
|
||
|
|
else:
|
||
|
|
self._update_native_row(target_thread_id, status="paused")
|
||
|
|
return result
|
||
|
|
if action == "continue":
|
||
|
|
text = str(request.get("input") or "Continue the active objective.")
|
||
|
|
if root_target and self.agent["execution_mode"] == "goal":
|
||
|
|
self.bootstrap_goal_pending = False
|
||
|
|
token_budget = request.get("goal_token_budget")
|
||
|
|
goal = self.host.set_goal(
|
||
|
|
status="active",
|
||
|
|
token_budget=(
|
||
|
|
int(token_budget)
|
||
|
|
if isinstance(token_budget, int) and not isinstance(token_budget, bool)
|
||
|
|
else None
|
||
|
|
),
|
||
|
|
)
|
||
|
|
self.finalize_deadline = None
|
||
|
|
self.finalize_turn_id = None
|
||
|
|
self.finalize_status = "completed"
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["finalize_requested"] = False
|
||
|
|
self._update(
|
||
|
|
status="running",
|
||
|
|
root_goal_status=goal["status"],
|
||
|
|
root_finalizing=False,
|
||
|
|
root_completion_deferred=False,
|
||
|
|
root_completion_deferred_jobs=[],
|
||
|
|
root_goal_bootstrap_pending=False,
|
||
|
|
)
|
||
|
|
return {"goal": goal}
|
||
|
|
if turn_id is not None:
|
||
|
|
raise RuntimeError("root already has an active turn")
|
||
|
|
if root_target:
|
||
|
|
new_turn = self.host.start_turn(
|
||
|
|
turn_input(text, []),
|
||
|
|
effort=self.agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
response = client.request(
|
||
|
|
"turn/start",
|
||
|
|
{
|
||
|
|
"threadId": target_thread_id,
|
||
|
|
"input": turn_input(text, []),
|
||
|
|
"effort": target_agent.get("reasoning"),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
turn = response.get("turn") if isinstance(response, Mapping) else None
|
||
|
|
if not isinstance(turn, Mapping) or not isinstance(turn.get("id"), str):
|
||
|
|
raise RuntimeError("native turn/start returned no turn")
|
||
|
|
new_turn = str(turn["id"])
|
||
|
|
if root_target:
|
||
|
|
self.finalize_deadline = None
|
||
|
|
self.finalize_turn_id = None
|
||
|
|
self.finalize_status = "completed"
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["finalize_requested"] = False
|
||
|
|
self._update(
|
||
|
|
status="running",
|
||
|
|
root_finalizing=False,
|
||
|
|
root_completion_deferred=False,
|
||
|
|
root_completion_deferred_jobs=[],
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
self._update_native_row(target_thread_id, status="running")
|
||
|
|
return {"turn_id": new_turn}
|
||
|
|
if action == "detach":
|
||
|
|
if root_target:
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
failure = current.get("failure")
|
||
|
|
if (
|
||
|
|
isinstance(failure, Mapping)
|
||
|
|
and failure.get("retryable") is True
|
||
|
|
and self.host.active_turn_id is None
|
||
|
|
):
|
||
|
|
now = utc_now()
|
||
|
|
self.retryable_failure_detach_exit = True
|
||
|
|
self._update(
|
||
|
|
status="suspended",
|
||
|
|
suspended_at=now,
|
||
|
|
detached_at=now,
|
||
|
|
last_active_at=now,
|
||
|
|
)
|
||
|
|
_SHUTDOWN.set()
|
||
|
|
return {"detached": True, "status": "suspended"}
|
||
|
|
self._update(status="detached", detached_at=utc_now())
|
||
|
|
else:
|
||
|
|
self._update_native_row(
|
||
|
|
target_thread_id,
|
||
|
|
status="detached",
|
||
|
|
detached_at=utc_now(),
|
||
|
|
)
|
||
|
|
return {"detached": True}
|
||
|
|
if action == "stop":
|
||
|
|
if root_target and self.agent["execution_mode"] == "goal":
|
||
|
|
self.host.set_goal(status="paused")
|
||
|
|
if turn_id is not None:
|
||
|
|
with contextlib.suppress(AppServerError):
|
||
|
|
client.request(
|
||
|
|
"turn/interrupt",
|
||
|
|
{"threadId": target_thread_id, "turnId": turn_id},
|
||
|
|
)
|
||
|
|
if root_target:
|
||
|
|
self.stop_status = "stopped"
|
||
|
|
_SHUTDOWN.set()
|
||
|
|
else:
|
||
|
|
client.request("thread/archive", {"threadId": target_thread_id})
|
||
|
|
self._update_native_row(
|
||
|
|
target_thread_id,
|
||
|
|
status="stopped",
|
||
|
|
stopped_at=utc_now(),
|
||
|
|
)
|
||
|
|
return {"stopping": True}
|
||
|
|
if action == "finalize":
|
||
|
|
if root_target and self.agent["execution_mode"] == "goal":
|
||
|
|
self.host.set_goal(status="paused")
|
||
|
|
prompt = str(
|
||
|
|
request.get("input")
|
||
|
|
or "Return the best supported final result from evidence already in this thread."
|
||
|
|
)
|
||
|
|
if root_target:
|
||
|
|
self.finalize_status = terminal_status
|
||
|
|
self.finalize_deadline = time.monotonic() + int(
|
||
|
|
self.agent["finalization_grace_seconds"]
|
||
|
|
)
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["finalize_requested"] = True
|
||
|
|
self._update(
|
||
|
|
root_finalizing=True,
|
||
|
|
root_finalization_started_at=utc_now(),
|
||
|
|
)
|
||
|
|
if turn_id is not None:
|
||
|
|
result = client.request(
|
||
|
|
"turn/steer",
|
||
|
|
{
|
||
|
|
"threadId": target_thread_id,
|
||
|
|
"expectedTurnId": turn_id,
|
||
|
|
"input": turn_input(prompt, []),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
if root_target:
|
||
|
|
self.finalize_turn_id = str(turn_id)
|
||
|
|
return result
|
||
|
|
new_turn_id = (
|
||
|
|
self.host.start_turn(
|
||
|
|
turn_input(prompt, []),
|
||
|
|
effort=self.agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
if root_target
|
||
|
|
else self._start_native_turn(
|
||
|
|
client,
|
||
|
|
target_thread_id,
|
||
|
|
prompt,
|
||
|
|
target_agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
if root_target:
|
||
|
|
self.finalize_turn_id = str(new_turn_id)
|
||
|
|
return {"turn_id": new_turn_id}
|
||
|
|
if action == "compact":
|
||
|
|
return client.request("thread/compact/start", {"threadId": target_thread_id})
|
||
|
|
if action == "respond":
|
||
|
|
request_id = request["request_id"]
|
||
|
|
pending = next(
|
||
|
|
(
|
||
|
|
item
|
||
|
|
for item in client.pending_server_requests_for_thread(target_thread_id)
|
||
|
|
if type(item.get("id")) is type(request_id) and item.get("id") == request_id
|
||
|
|
),
|
||
|
|
None,
|
||
|
|
)
|
||
|
|
if not isinstance(pending, Mapping) or not isinstance(pending.get("method"), str):
|
||
|
|
raise RuntimeError("app-server request is not pending")
|
||
|
|
response = request.get("response")
|
||
|
|
if not isinstance(response, Mapping):
|
||
|
|
raise ValueError("app-server response must be an object")
|
||
|
|
validate_server_request_response(str(pending["method"]), response)
|
||
|
|
client.respond(request_id, response)
|
||
|
|
if root_target:
|
||
|
|
self._update(
|
||
|
|
root_pending_request_count=len(
|
||
|
|
client.pending_server_requests_for_thread(target_thread_id)
|
||
|
|
)
|
||
|
|
)
|
||
|
|
return {"responded": True}
|
||
|
|
if action == "set_effort":
|
||
|
|
effort = str(request["effort"])
|
||
|
|
if effort not in target_agent["allowed_reasoning_efforts"]:
|
||
|
|
raise ValueError("reasoning effort is outside the profile grant")
|
||
|
|
return client.request(
|
||
|
|
"thread/settings/update",
|
||
|
|
{"threadId": target_thread_id, "effort": effort},
|
||
|
|
)
|
||
|
|
if action == "fork":
|
||
|
|
runs = self.refresh_native_runs()
|
||
|
|
active_count = sum(
|
||
|
|
isinstance(row.get("status"), Mapping) and row["status"].get("type") == "active"
|
||
|
|
for row in runs
|
||
|
|
)
|
||
|
|
native_limit = int(
|
||
|
|
self.snapshot["resolved"]["coordination"]["native_max_concurrent_threads"]
|
||
|
|
)
|
||
|
|
if active_count >= native_limit:
|
||
|
|
raise RuntimeError(
|
||
|
|
f"native fork would exceed the compiled thread limit ({native_limit})"
|
||
|
|
)
|
||
|
|
response = client.request(
|
||
|
|
"thread/fork",
|
||
|
|
{
|
||
|
|
"threadId": target_thread_id,
|
||
|
|
"cwd": self.session["cwd"],
|
||
|
|
"sandbox": target_agent["permissions"],
|
||
|
|
"approvalPolicy": target_agent["approval_policy"],
|
||
|
|
"ephemeral": False,
|
||
|
|
"deferGoalContinuation": True,
|
||
|
|
"excludeTurns": False,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
forked = response.get("thread") if isinstance(response, Mapping) else None
|
||
|
|
if not isinstance(forked, Mapping) or not isinstance(forked.get("id"), str):
|
||
|
|
raise RuntimeError("native thread/fork returned no thread")
|
||
|
|
fork_thread_id = str(forked["id"])
|
||
|
|
fork_ref = "ar_" + secrets.token_urlsafe(24)
|
||
|
|
fork_row = {
|
||
|
|
"agent_run_ref": fork_ref,
|
||
|
|
"agent": target_agent_id,
|
||
|
|
"backend": "native",
|
||
|
|
"thread_id": fork_thread_id,
|
||
|
|
"parent_thread_id": forked.get("parentThreadId"),
|
||
|
|
"nickname": forked.get("agentNickname"),
|
||
|
|
"status": forked.get("status"),
|
||
|
|
"can_accept_direct_input": forked.get("canAcceptDirectInput"),
|
||
|
|
"created_at": forked.get("createdAt"),
|
||
|
|
"updated_at": forked.get("updatedAt"),
|
||
|
|
"control_revision": 0,
|
||
|
|
}
|
||
|
|
updated = dict(self.session.get("root_native_runs", {}))
|
||
|
|
updated[fork_ref] = fork_row
|
||
|
|
self._update(root_native_runs=updated)
|
||
|
|
new_turn_id = self._start_native_turn(
|
||
|
|
client,
|
||
|
|
fork_thread_id,
|
||
|
|
str(request["input"]),
|
||
|
|
target_agent.get("reasoning"),
|
||
|
|
)
|
||
|
|
return {"agent": fork_row, "turn_id": new_turn_id}
|
||
|
|
raise ValueError(f"unsupported root control action: {action}")
|
||
|
|
|
||
|
|
def serve_control(self) -> None:
|
||
|
|
def publish_ready() -> None:
|
||
|
|
self._update(root_control_socket_ready=True)
|
||
|
|
|
||
|
|
serve_control_socket(
|
||
|
|
self.control_path,
|
||
|
|
stop_event=self.control_stop,
|
||
|
|
handler=self._execute_control,
|
||
|
|
on_ready=publish_ready,
|
||
|
|
backlog=16,
|
||
|
|
connection_timeout=None,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _result_from_turns(self, turns: list[dict[str, Any]]) -> str:
|
||
|
|
for turn in reversed(turns):
|
||
|
|
if turn.get("status") != "completed":
|
||
|
|
continue
|
||
|
|
turn_id = turn.get("id")
|
||
|
|
if isinstance(turn_id, str):
|
||
|
|
result = completed_turn_presentable_text(self.events_path, turn_id)
|
||
|
|
if result.strip():
|
||
|
|
return result
|
||
|
|
return last_agent_message(turn)
|
||
|
|
return ""
|
||
|
|
|
||
|
|
def _publish_terminal(self, status: str, *, error: str | None = None) -> str:
|
||
|
|
if error is None and status == "failed" and isinstance(self.session.get("error"), str):
|
||
|
|
error = str(self.session["error"])
|
||
|
|
turns: list[dict[str, Any]] = []
|
||
|
|
history_timeout = self.lifecycle_timeout
|
||
|
|
if self.finalize_deadline is not None:
|
||
|
|
history_timeout = min(
|
||
|
|
history_timeout,
|
||
|
|
max(0.0, self.finalize_deadline - time.monotonic()),
|
||
|
|
)
|
||
|
|
if history_timeout > 0:
|
||
|
|
with contextlib.suppress(AppServerError):
|
||
|
|
turns = self.host.complete_history(timeout=history_timeout)
|
||
|
|
atomic_write_json(
|
||
|
|
self.directory / "root-terminal-history.json",
|
||
|
|
{
|
||
|
|
"thread_id": self.host.thread_id,
|
||
|
|
"captured_at": utc_now(),
|
||
|
|
"turns": turns,
|
||
|
|
},
|
||
|
|
)
|
||
|
|
result = self._result_from_turns(turns)
|
||
|
|
if status == "completed" and not result:
|
||
|
|
status = "failed"
|
||
|
|
error = error or "root completed without a readable terminal result"
|
||
|
|
if status == "completed" and result:
|
||
|
|
atomic_write_text(self.result_path, result, 0o600)
|
||
|
|
result_kind = "final"
|
||
|
|
elif self._has_durable_work():
|
||
|
|
partial = retain_partial_evidence(
|
||
|
|
self.session,
|
||
|
|
self.directory,
|
||
|
|
reason=error or f"root host ended with status {status}",
|
||
|
|
events_filename=self.events_path.name,
|
||
|
|
result_filename=self.result_path.name,
|
||
|
|
partial_filename=self.partial_path.name,
|
||
|
|
title="Partial root result",
|
||
|
|
)
|
||
|
|
result_kind = "partial"
|
||
|
|
self._update(**partial)
|
||
|
|
else:
|
||
|
|
result_kind = "none"
|
||
|
|
now = utc_now()
|
||
|
|
current = self._update(
|
||
|
|
status=status,
|
||
|
|
finished_at=now,
|
||
|
|
last_active_at=now,
|
||
|
|
exit_code=0 if status == "completed" else 130 if status == "cancelled" else 1,
|
||
|
|
error=error,
|
||
|
|
result_path=(
|
||
|
|
str(self.result_path)
|
||
|
|
if status == "completed" and result
|
||
|
|
else str(self.partial_path)
|
||
|
|
if result_kind == "partial"
|
||
|
|
else None
|
||
|
|
),
|
||
|
|
result_kind=result_kind,
|
||
|
|
root_finalizing=False,
|
||
|
|
root_completion_deferred=False,
|
||
|
|
root_completion_deferred_jobs=[],
|
||
|
|
root_control_socket_ready=False,
|
||
|
|
failure=None if status == "completed" else self.session.get("failure"),
|
||
|
|
suspended_at=None if status == "completed" else self.session.get("suspended_at"),
|
||
|
|
)
|
||
|
|
status = str(current["status"])
|
||
|
|
with file_lock(runtime_lock_path()):
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
if current.get("current_run_id"):
|
||
|
|
current["last_run_id"] = current["current_run_id"]
|
||
|
|
current["current_run_id"] = None
|
||
|
|
publish_session_record(self.directory, current, mirror_run=False)
|
||
|
|
revoke_session_capabilities(self.session_id)
|
||
|
|
append_audit(self.session_id, "root_host_terminal", status=status, error=error)
|
||
|
|
return status
|
||
|
|
|
||
|
|
def recover(self) -> bool:
|
||
|
|
if not self._has_durable_work():
|
||
|
|
return False
|
||
|
|
previous = self.client
|
||
|
|
if previous is not None:
|
||
|
|
previous.close()
|
||
|
|
for attempt, delay in enumerate(APP_SERVER_RECOVERY_DELAYS_SECONDS, start=1):
|
||
|
|
if _SHUTDOWN.wait(delay):
|
||
|
|
return False
|
||
|
|
self.session = read_session_record(self.directory)
|
||
|
|
prior_status = str(self.session.get("status"))
|
||
|
|
if prior_status in {
|
||
|
|
"finishing",
|
||
|
|
"stopping",
|
||
|
|
"cancelling",
|
||
|
|
"completed",
|
||
|
|
"stopped",
|
||
|
|
"failed",
|
||
|
|
"cancelled",
|
||
|
|
}:
|
||
|
|
return False
|
||
|
|
try:
|
||
|
|
self.connect()
|
||
|
|
self._resume_turn_work_if_needed(prior_status)
|
||
|
|
# Publish recovery under the same lock as resume admission.
|
||
|
|
# The attached client may already have advanced detached to
|
||
|
|
# running after observing the new app-server identity; never
|
||
|
|
# restore the stale pre-recovery status over that decision.
|
||
|
|
with file_lock(runtime_lock_path()):
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
current["root_recovery_attempts"] = attempt
|
||
|
|
current_status = str(current.get("status"))
|
||
|
|
if current_status not in {
|
||
|
|
"finishing",
|
||
|
|
"stopping",
|
||
|
|
"cancelling",
|
||
|
|
"completed",
|
||
|
|
"stopped",
|
||
|
|
"failed",
|
||
|
|
"cancelled",
|
||
|
|
}:
|
||
|
|
current["status"] = (
|
||
|
|
current_status
|
||
|
|
if current_status in {"paused", "detached"}
|
||
|
|
else "running"
|
||
|
|
)
|
||
|
|
publish_session_record(self.directory, current, mirror_run=True)
|
||
|
|
self.session = current
|
||
|
|
append_audit(self.session_id, "root_app_server_recovered", attempt=attempt)
|
||
|
|
return True
|
||
|
|
except (AppServerError, OSError) as exc:
|
||
|
|
self._update(root_recovery_error=f"{type(exc).__name__}: {exc}")
|
||
|
|
self._update(status="suspended")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _stop_recorded_app_server(self) -> None:
|
||
|
|
current = read_session_record(self.directory)
|
||
|
|
terminate_recorded_process_group(
|
||
|
|
current,
|
||
|
|
prefix="root_app_server",
|
||
|
|
grace_seconds=8.0,
|
||
|
|
)
|
||
|
|
|
||
|
|
def run(self) -> int:
|
||
|
|
load_session_capabilities(self.directory)
|
||
|
|
self._update(
|
||
|
|
root_runtime_package_version=package_version(),
|
||
|
|
root_runtime_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
||
|
|
)
|
||
|
|
try:
|
||
|
|
self.connect()
|
||
|
|
except (AppServerError, OSError):
|
||
|
|
if (
|
||
|
|
not isinstance(self.session.get("root_thread_id"), str)
|
||
|
|
or not self._has_durable_work()
|
||
|
|
):
|
||
|
|
raise
|
||
|
|
if not self.recover():
|
||
|
|
if isinstance(self.session.get("root_thread_id"), str):
|
||
|
|
return 75
|
||
|
|
raise
|
||
|
|
resume_status = self.session.get("resume_requested_from")
|
||
|
|
if resume_status in {"detached", "paused", "suspended"}:
|
||
|
|
failure = self.session.get("failure")
|
||
|
|
completed_turn = self.host.completed_turn
|
||
|
|
retrying_failed_turn = bool(
|
||
|
|
resume_status == "suspended"
|
||
|
|
and isinstance(failure, Mapping)
|
||
|
|
and failure.get("retryable") is True
|
||
|
|
and isinstance(completed_turn, Mapping)
|
||
|
|
and completed_turn.get("status") == "failed"
|
||
|
|
and completed_turn.get("id") == failure.get("turn_id")
|
||
|
|
)
|
||
|
|
if retrying_failed_turn:
|
||
|
|
# The failed turn remains immutable history, but it must not be
|
||
|
|
# reclassified as a fresh failure before the controller can
|
||
|
|
# accept an explicit continuation on the replacement host.
|
||
|
|
self.host.take_completed_turn()
|
||
|
|
with self.state_lock:
|
||
|
|
self.state["turn_failure"] = None
|
||
|
|
self._update(
|
||
|
|
status="paused",
|
||
|
|
active_root_turn_id=None,
|
||
|
|
resume_requested_from=None,
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
self._update(
|
||
|
|
status="paused" if resume_status == "paused" else "running",
|
||
|
|
resume_requested_from=None,
|
||
|
|
)
|
||
|
|
control = threading.Thread(target=self.serve_control, name="mmo-root-control", daemon=True)
|
||
|
|
control.start()
|
||
|
|
try:
|
||
|
|
while True:
|
||
|
|
try:
|
||
|
|
self.start_initial_work()
|
||
|
|
break
|
||
|
|
except AppServerError:
|
||
|
|
if not self._has_durable_work():
|
||
|
|
raise
|
||
|
|
if not self.recover():
|
||
|
|
return 75
|
||
|
|
atomic_write_json(
|
||
|
|
self.directory / "root-host-ready.json",
|
||
|
|
{
|
||
|
|
"thread_id": self.host.thread_id,
|
||
|
|
"socket": str(self.socket_path),
|
||
|
|
"ready_at": utc_now(),
|
||
|
|
},
|
||
|
|
)
|
||
|
|
stall_seconds = int(self.agent["stall_warning_seconds"])
|
||
|
|
while not _SHUTDOWN.wait(0.5):
|
||
|
|
client = self.client
|
||
|
|
if client is None or not client.alive:
|
||
|
|
if not self.recover():
|
||
|
|
# Exit only the lightweight controller. The session is
|
||
|
|
# suspended with its exact thread/evidence identity and
|
||
|
|
# can start a fresh controller on explicit continue.
|
||
|
|
return 75
|
||
|
|
continue
|
||
|
|
self._ensure_bootstrap_goal()
|
||
|
|
self._activate_bootstrap_goal()
|
||
|
|
if (
|
||
|
|
not self.stall_reported
|
||
|
|
and time.monotonic() - self.last_progress >= stall_seconds
|
||
|
|
):
|
||
|
|
self.stall_reported = True
|
||
|
|
self._update(root_stall_warning_at=utc_now())
|
||
|
|
append_audit(
|
||
|
|
self.session_id,
|
||
|
|
"root_stall_warning",
|
||
|
|
warning_seconds=stall_seconds,
|
||
|
|
)
|
||
|
|
completed_turn = self.host.completed_turn
|
||
|
|
if isinstance(completed_turn, Mapping) and completed_turn.get("status") == "failed":
|
||
|
|
failure = normalize_turn_failure(completed_turn) or {
|
||
|
|
"kind": "turn_failed",
|
||
|
|
"source": "turn/completed",
|
||
|
|
"turn_id": completed_turn.get("id"),
|
||
|
|
"turn_status": "failed",
|
||
|
|
"message": "app-server turn failed",
|
||
|
|
"retryable": False,
|
||
|
|
"observed_at": utc_now(),
|
||
|
|
}
|
||
|
|
partial = retain_partial_evidence(
|
||
|
|
self.session,
|
||
|
|
self.directory,
|
||
|
|
reason=str(failure["message"]),
|
||
|
|
events_filename="root-events.jsonl",
|
||
|
|
result_filename="root-result.md",
|
||
|
|
partial_filename="root-partial-result.md",
|
||
|
|
title="Partial root result",
|
||
|
|
)
|
||
|
|
if failure.get("retryable"):
|
||
|
|
if self._attached_root_client():
|
||
|
|
# A failed turn is immutable thread history, not a
|
||
|
|
# reason to destroy a healthy app-server underneath
|
||
|
|
# an attached TUI. Consume only this controller's
|
||
|
|
# completion marker; a later accepted turn clears
|
||
|
|
# the persisted failure through turn/started.
|
||
|
|
self.host.take_completed_turn()
|
||
|
|
now = utc_now()
|
||
|
|
self._update(
|
||
|
|
status="running",
|
||
|
|
last_active_at=now,
|
||
|
|
error=str(failure["message"]),
|
||
|
|
failure=failure,
|
||
|
|
**partial,
|
||
|
|
)
|
||
|
|
append_audit(
|
||
|
|
self.session_id,
|
||
|
|
"root_retryable_failure_client_retained",
|
||
|
|
turn_id=failure.get("turn_id"),
|
||
|
|
failure_kind=failure.get("kind"),
|
||
|
|
)
|
||
|
|
continue
|
||
|
|
self._update(
|
||
|
|
status="suspended",
|
||
|
|
suspended_at=utc_now(),
|
||
|
|
error=str(failure["message"]),
|
||
|
|
failure=failure,
|
||
|
|
**partial,
|
||
|
|
)
|
||
|
|
return 75
|
||
|
|
self._update(error=str(failure["message"]), failure=failure, **partial)
|
||
|
|
self.stop_status = "failed"
|
||
|
|
break
|
||
|
|
finalize_requested = bool(self.state.get("finalize_requested"))
|
||
|
|
if finalize_requested and isinstance(completed_turn, Mapping):
|
||
|
|
if (
|
||
|
|
self.finalize_turn_id is None
|
||
|
|
or completed_turn.get("id") == self.finalize_turn_id
|
||
|
|
):
|
||
|
|
self.stop_status = (
|
||
|
|
self.finalize_status
|
||
|
|
if completed_turn.get("status") == "completed"
|
||
|
|
else "stopped"
|
||
|
|
if self.finalize_status == "stopped"
|
||
|
|
else "failed"
|
||
|
|
)
|
||
|
|
break
|
||
|
|
if (
|
||
|
|
finalize_requested
|
||
|
|
and self.finalize_deadline is not None
|
||
|
|
and time.monotonic() >= self.finalize_deadline
|
||
|
|
):
|
||
|
|
self.stop_status = "stopped"
|
||
|
|
break
|
||
|
|
goal = self.state.get("goal")
|
||
|
|
if isinstance(goal, Mapping) and goal.get("status") == "complete":
|
||
|
|
# Codex may publish goal completion from update_goal while
|
||
|
|
# the terminal turn is still producing its final message.
|
||
|
|
# Goal state is not a substitute for authoritative turn
|
||
|
|
# completion; retiring the host here would discard that
|
||
|
|
# message and leave only an in-progress history record.
|
||
|
|
if self.host.active_turn_id is not None:
|
||
|
|
continue
|
||
|
|
descendants = self.recoverable_descendants()
|
||
|
|
if descendants:
|
||
|
|
recoverable_jobs = sorted(str(item["job_id"]) for item in descendants)
|
||
|
|
if (
|
||
|
|
not self.session.get("root_completion_deferred")
|
||
|
|
or sorted(
|
||
|
|
str(item)
|
||
|
|
for item in self.session.get("root_completion_deferred_jobs", [])
|
||
|
|
)
|
||
|
|
!= recoverable_jobs
|
||
|
|
):
|
||
|
|
now = utc_now()
|
||
|
|
self._update(
|
||
|
|
status="detached",
|
||
|
|
detached_at=now,
|
||
|
|
last_active_at=now,
|
||
|
|
root_completion_deferred=True,
|
||
|
|
root_completion_deferred_jobs=recoverable_jobs,
|
||
|
|
)
|
||
|
|
append_audit(
|
||
|
|
self.session_id,
|
||
|
|
"root_completion_deferred",
|
||
|
|
recoverable_jobs=recoverable_jobs,
|
||
|
|
)
|
||
|
|
continue
|
||
|
|
self.stop_status = "completed"
|
||
|
|
break
|
||
|
|
if self.agent["execution_mode"] == "turn" and completed_turn:
|
||
|
|
turn = completed_turn
|
||
|
|
if (
|
||
|
|
self.session.get("status") == "paused"
|
||
|
|
and isinstance(turn, Mapping)
|
||
|
|
and turn.get("status") == "interrupted"
|
||
|
|
):
|
||
|
|
self.host.take_completed_turn()
|
||
|
|
continue
|
||
|
|
self.stop_status = (
|
||
|
|
"completed"
|
||
|
|
if isinstance(turn, Mapping) and turn.get("status") == "completed"
|
||
|
|
else "failed"
|
||
|
|
)
|
||
|
|
break
|
||
|
|
if self.retryable_failure_detach_exit:
|
||
|
|
self._update(
|
||
|
|
status="suspended",
|
||
|
|
active_root_turn_id=None,
|
||
|
|
)
|
||
|
|
return 75
|
||
|
|
if self.pause_exit:
|
||
|
|
partial = retain_partial_evidence(
|
||
|
|
self.session,
|
||
|
|
self.directory,
|
||
|
|
reason="root cold-paused by operator",
|
||
|
|
events_filename="root-events.jsonl",
|
||
|
|
result_filename="root-result.md",
|
||
|
|
partial_filename="root-partial-result.md",
|
||
|
|
title="Partial root result",
|
||
|
|
)
|
||
|
|
self._update(
|
||
|
|
status="paused",
|
||
|
|
paused_at=utc_now(),
|
||
|
|
root_goal_status=("paused" if self.agent["execution_mode"] == "goal" else None),
|
||
|
|
active_root_turn_id=None,
|
||
|
|
**partial,
|
||
|
|
)
|
||
|
|
return 75
|
||
|
|
status = self.stop_status or "cancelled"
|
||
|
|
status = self._publish_terminal(status)
|
||
|
|
return 0 if status == "completed" else 130 if status == "cancelled" else 1
|
||
|
|
finally:
|
||
|
|
self.control_stop.set()
|
||
|
|
control.join(timeout=2.0)
|
||
|
|
if self.client is not None:
|
||
|
|
if self.client.process is not None:
|
||
|
|
self.client.stop_host()
|
||
|
|
else:
|
||
|
|
self.client.close()
|
||
|
|
self._stop_recorded_app_server()
|
||
|
|
self._update(
|
||
|
|
root_control_socket_ready=False,
|
||
|
|
root_app_server_pid=None,
|
||
|
|
root_app_server_pgid=None,
|
||
|
|
root_app_server_start_token=None,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
if len(sys.argv) != 2:
|
||
|
|
print("usage: root_runner.py SESSION_ID", file=sys.stderr)
|
||
|
|
return 2
|
||
|
|
signal.signal(signal.SIGTERM, _signal_handler)
|
||
|
|
signal.signal(signal.SIGINT, _signal_handler)
|
||
|
|
runner = RootRunner(sys.argv[1])
|
||
|
|
try:
|
||
|
|
return runner.run()
|
||
|
|
except Exception as exc:
|
||
|
|
with contextlib.suppress(Exception):
|
||
|
|
runner._publish_terminal("failed", error=f"{type(exc).__name__}: {exc}")
|
||
|
|
print(f"root runner failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main())
|