#!/usr/bin/env python3 """Lifecycle manager for route-set-addressed Switchyard gateway processes.""" from __future__ import annotations import contextlib import datetime as dt import json import os import re import shutil import socket import subprocess import time from collections.abc import Mapping from ipaddress import ip_address from pathlib import Path from typing import Any from urllib.parse import urlsplit from mmo_profiles import load_settings from mmo_snapshot import load_snapshot from mmo_state import ( ACTIVE_SESSION_STATUSES, RECOVERABLE_JOB_STATUSES, iter_job_records, iter_session_records, ) from mmo_util import ( atomic_write_json, config_root, file_lock, filtered_environment, http_json, http_ready, parse_env_file, port_available, process_matches, process_start_token, read_json, state_root, strict_json_loads, terminate_process_group, utc_now, ) from mmo_version import MMO_SCHEMA_VERSION _GATEWAY_PROCESSES: dict[int, subprocess.Popen[bytes]] = {} _GATEWAY_ADMISSION_GRACE_SECONDS = 60.0 def _reap_gateway(pid: int | None) -> None: """Reap terminal gateway children without dropping live handles.""" if not pid: return process = _GATEWAY_PROCESSES.get(int(pid)) if process is None: return try: process.wait(timeout=1.0) except subprocess.TimeoutExpired: return except OSError: pass _GATEWAY_PROCESSES.pop(int(pid), None) def gateways_root() -> Path: root = state_root() / "gateways" root.mkdir(parents=True, exist_ok=True, mode=0o700) return root def _validate_hash(value: str, label: str) -> str: if len(value) != 64 or any(char not in "0123456789abcdef" for char in value): raise ValueError(f"invalid {label}") return value def _gateway_key(snapshot_hash: str) -> str: snapshot = load_snapshot(_validate_hash(snapshot_hash, "snapshot hash")) return _validate_hash( str(snapshot["manifest"].get("gateway_hash") or snapshot_hash), "gateway hash", ) def _gateway_dir_for_key(gateway_hash: str) -> Path: path = gateways_root() / _validate_hash(gateway_hash, "gateway hash") path.mkdir(parents=True, exist_ok=True, mode=0o700) return path def gateway_dir(snapshot_hash: str) -> Path: """Return the route-set gateway directory for a profile snapshot.""" return _gateway_dir_for_key(_gateway_key(snapshot_hash)) def _gateway_state_path_for_key(gateway_hash: str) -> Path: return _gateway_dir_for_key(gateway_hash) / "gateway.json" def gateway_state_path(snapshot_hash: str) -> Path: return _gateway_state_path_for_key(_gateway_key(snapshot_hash)) def _binary(settings: Mapping[str, Any]) -> str: configured = str(settings.get("switchyard_bin", "switchyard-server")) resolved = shutil.which(configured) if resolved: return resolved path = Path(configured).expanduser() if path.is_file() and os.access(path, os.X_OK): return str(path.resolve()) raise FileNotFoundError(f"Switchyard binary not found: {configured}") def switchyard_version(binary: str | None = None) -> str: """Return the exact release reported by the configured gateway binary.""" resolved = binary or _binary(load_settings()) try: result = subprocess.run( [resolved, "--version"], capture_output=True, text=True, check=False, timeout=10, env=filtered_environment(), ) except (OSError, subprocess.TimeoutExpired) as exc: raise RuntimeError(f"unable to inspect Switchyard version at {resolved}: {exc}") from exc output = (result.stdout + "\n" + result.stderr).strip() match = re.fullmatch(r"switchyard-server ([0-9]+\.[0-9]+\.[0-9]+)", output) if result.returncode != 0 or match is None: raise RuntimeError( "Switchyard did not report a supported semantic version: " f"status={result.returncode}, output={output[-1000:]!r}" ) return match.group(1) def route_availability(snapshot: Mapping[str, Any]) -> dict[str, dict[str, Any]]: """Resolve current per-route availability without making workers startup-critical.""" credentials_path = config_root() / "credentials.env" file_values = parse_env_file(credentials_path) values = dict(file_values) values.update({key: value for key, value in os.environ.items() if value}) result: dict[str, dict[str, Any]] = {} for route_key, route in sorted(snapshot["resolved"]["routes"].items()): credentials = list(route.get("credential_envs", [])) selected = next((name for name in credentials if values.get(name)), None) available = selected is not None or not credentials reason = None if available else "missing credential: " + "/".join(credentials) if available and route.get("billing_mode") == "local" and route.get("base_url"): parsed = urlsplit(str(route["base_url"])) port = parsed.port or (443 if parsed.scheme == "https" else 80) try: with socket.create_connection((str(parsed.hostname), port), timeout=0.2): pass except OSError: available = False reason = f"local endpoint unavailable: {parsed.hostname}:{port}" result[route_key] = { "available": available, "selected_credential_env": selected, "reason": reason, } return result def _credentials(snapshot: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]: credentials_path = config_root() / "credentials.env" file_values = parse_env_file(credentials_path) values = dict(file_values) values.update({key: value for key, value in os.environ.items() if value}) root_agent = snapshot["resolved"]["agents"][snapshot["manifest"]["root_agent"]] root_route = str(root_agent["route"]) groups = snapshot["manifest"].get("credential_groups", []) resolved: dict[str, str] = {} missing: list[str] = [] for group in groups: alternatives = list(group.get("alternatives") or [group.get("target_env")]) source = next((name for name in alternatives if name and values.get(name)), None) target = str(group.get("target_env") or alternatives[0]) if source is None: route_key = str(group.get("route")) if route_key == root_route: missing.append("/".join(name for name in alternatives if name)) else: # Switchyard resolves client environment keys when parsing its # immutable route set. The supervisor prevents this sentinel # client from receiving work through the availability overlay. resolved[target] = f"mmo-unavailable-{route_key}" continue resolved[target] = values[source] return resolved, missing def _select_port(snapshot_hash: str, settings: Mapping[str, Any]) -> int: minimum = int(settings.get("gateway_port_min", 42000)) maximum = int(settings.get("gateway_port_max", 51999)) if not 1024 <= minimum <= maximum <= 65535: raise ValueError("invalid gateway port range in settings") span = maximum - minimum + 1 start = int(snapshot_hash[:16], 16) % span host = str(settings.get("gateway_host", "127.0.0.1")) for offset in range(span): port = minimum + ((start + offset) % span) if port_available(host, port): return port raise RuntimeError("no free port is available in the configured gateway range") def _health_url(host: str, port: int) -> str: address = f"[{host}]" if ":" in host and not host.startswith("[") else host return f"http://{address}:{port}/health" def _base_url(host: str, port: int) -> str: address = f"[{host}]" if ":" in host and not host.startswith("[") else host return f"http://{address}:{port}/v1" def _timestamp(value: Any, label: str) -> float: if not isinstance(value, str) or not value: raise ValueError(f"invalid {label}") try: parsed = dt.datetime.fromisoformat(value) except ValueError as exc: raise ValueError(f"invalid {label}") from exc if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=dt.UTC) return parsed.timestamp() def _read_gateway_state_by_key(gateway_hash: str) -> dict[str, Any] | None: path = _gateway_state_path_for_key(gateway_hash) if not path.is_file(): return None try: state = read_json(path) except (OSError, ValueError) as exc: return { "gateway_hash": gateway_hash, "status": "invalid", "error": f"invalid gateway state: {type(exc).__name__}: {exc}", } if not isinstance(state, dict): return { "gateway_hash": gateway_hash, "status": "invalid", "error": "invalid gateway state: root must be an object", } schema_version = state.get("schema_version") observed_switchyard_version = state.get("switchyard_version") snapshot_hashes = state.get("snapshot_hashes", []) profile_ids = state.get("profile_ids", []) host = state.get("host") port = state.get("port") endpoint_identity_valid = False if ( isinstance(host, str) and isinstance(port, int) and not isinstance(port, bool) and 1 <= port <= 65535 ): with contextlib.suppress(ValueError): endpoint_identity_valid = ( ip_address(host).is_loopback and state.get("base_url") == _base_url(host, port) and state.get("health_url") == _health_url(host, port) ) if ( not isinstance(schema_version, int) or isinstance(schema_version, bool) or schema_version != MMO_SCHEMA_VERSION or state.get("gateway_hash") != gateway_hash or not isinstance(observed_switchyard_version, str) or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", observed_switchyard_version) is None or not isinstance(state.get("pid"), int) or isinstance(state.get("pid"), bool) or not isinstance(state.get("process_start_token"), str) or not isinstance(snapshot_hashes, list) or not all(isinstance(value, str) for value in snapshot_hashes) or not isinstance(profile_ids, list) or not all(isinstance(value, str) for value in profile_ids) or not endpoint_identity_valid ): return { "gateway_hash": gateway_hash, "status": "invalid", "error": "invalid gateway state identity or process metadata", } try: _timestamp(state.get("last_used_at"), "gateway last-used timestamp") except ValueError: return { "gateway_hash": gateway_hash, "status": "invalid", "error": "invalid gateway state last-used timestamp", } pid = state.get("pid") if not process_matches(pid, state.get("process_start_token")): state["status"] = "stopped" return state health = state.get("health_url") state["status"] = "running" if isinstance(health, str) and http_ready(health) else "starting" return state def read_gateway_state(snapshot_hash: str) -> dict[str, Any] | None: return _read_gateway_state_by_key(_gateway_key(snapshot_hash)) def _record_gateway_lease( state: dict[str, Any], snapshot: Mapping[str, Any], gateway_hash: str ) -> dict[str, Any]: snapshot_hash = str(snapshot["manifest"]["snapshot_hash"]) profile_id = str(snapshot["manifest"]["profile_id"]) state["gateway_hash"] = gateway_hash state["snapshot_hashes"] = sorted(set(state.get("snapshot_hashes", [])) | {snapshot_hash}) state["profile_ids"] = sorted(set(state.get("profile_ids", [])) | {profile_id}) state["last_used_at"] = utc_now() atomic_write_json(_gateway_state_path_for_key(gateway_hash), state) return state def ensure_gateway(snapshot_hash: str) -> dict[str, Any] | None: snapshot = load_snapshot(snapshot_hash) if not snapshot["manifest"].get("gateway_required"): return None gateway_hash = _validate_hash( str(snapshot["manifest"].get("gateway_hash") or snapshot_hash), "gateway hash", ) settings = load_settings() binary = _binary(settings) observed_switchyard_version = switchyard_version(binary) resolved_availability = route_availability(snapshot) lock = gateways_root() / ".gateway.lock" with file_lock(lock): current = _read_gateway_state_by_key(gateway_hash) if current and current.get("status") == "invalid": raise RuntimeError(str(current.get("error"))) if ( current and current.get("status") == "running" and current.get("switchyard_version") == observed_switchyard_version and current.get("route_availability") == resolved_availability ): return _record_gateway_lease(current, snapshot, gateway_hash) if current and process_matches(current.get("pid"), current.get("process_start_token")): terminate_process_group(int(current["pid"])) _reap_gateway(current.get("pid")) credentials, missing = _credentials(snapshot) if missing: raise RuntimeError( "missing root-route credentials for this profile: " + ", ".join(sorted(missing)) ) host = str(settings.get("gateway_host", "127.0.0.1")) port = _select_port(gateway_hash, settings) directory = _gateway_dir_for_key(gateway_hash) log_path = directory / "gateway.log" routing_log_path = directory / "routing.jsonl" routes_path = Path(snapshot["directory"]) / "routes.toml" command = [ binary, "--config", str(routes_path), "--host", host, "--port", str(port), "--routing-log-file", str(routing_log_path), ] environment = filtered_environment( allow_sensitive=credentials, extra={ **credentials, "RUST_LOG": os.environ.get("RUST_LOG", "switchyard_server=info,libsy=info"), }, ) log_handle = log_path.open("ab", buffering=0) try: process = subprocess.Popen( command, stdin=subprocess.DEVNULL, stdout=log_handle, stderr=subprocess.STDOUT, close_fds=True, start_new_session=True, env=environment, cwd=directory, ) finally: log_handle.close() _GATEWAY_PROCESSES[process.pid] = process ready = False try: start_token = process_start_token(process.pid) if start_token is None: initial_returncode = process.poll() tail = "" with contextlib.suppress(OSError): tail = log_path.read_text(encoding="utf-8", errors="replace")[-4000:] error = ( f"Switchyard exited during startup with status {initial_returncode}" if initial_returncode is not None else "unable to fingerprint the Switchyard process" ) raise RuntimeError(error + (f"\n{tail}" if tail else "")) state = { "schema_version": MMO_SCHEMA_VERSION, "gateway_hash": gateway_hash, "snapshot_hash": snapshot_hash, "snapshot_hashes": [snapshot_hash], "profile_id": snapshot["manifest"]["profile_id"], "profile_ids": [snapshot["manifest"]["profile_id"]], "switchyard_version": observed_switchyard_version, "pid": process.pid, "process_start_token": start_token, "host": host, "port": port, "base_url": _base_url(host, port), "health_url": _health_url(host, port), "routes_path": str(routes_path), "log_path": str(log_path), "routing_log_path": str(routing_log_path), "command": command, "route_availability": resolved_availability, "started_at": utc_now(), "last_used_at": utc_now(), "status": "starting", } atomic_write_json(_gateway_state_path_for_key(gateway_hash), state) timeout = float(settings.get("gateway_start_timeout_seconds", 15)) deadline = time.monotonic() + max(1.0, timeout) while time.monotonic() < deadline: if process.poll() is not None: break if http_ready(state["health_url"], timeout=0.4): state["status"] = "running" state["ready_at"] = utc_now() atomic_write_json(_gateway_state_path_for_key(gateway_hash), state) ready = True return state time.sleep(0.1) tail = "" with contextlib.suppress(OSError): tail = log_path.read_text(encoding="utf-8", errors="replace")[-4000:] state["status"] = "failed" state["finished_at"] = utc_now() state["error"] = "Switchyard did not become healthy before the startup deadline" atomic_write_json(_gateway_state_path_for_key(gateway_hash), state) raise RuntimeError(state["error"] + (f"\n{tail}" if tail else "")) finally: if not ready: terminate_process_group(process.pid) _reap_gateway(process.pid) def stop_gateway(snapshot_hash: str) -> dict[str, Any]: gateway_hash = _gateway_key(snapshot_hash) lock = gateways_root() / ".gateway.lock" with file_lock(lock): state = _read_gateway_state_by_key(gateway_hash) if not state: return {"snapshot_hash": snapshot_hash, "status": "not_found"} if state.get("status") == "invalid": raise RuntimeError(str(state.get("error"))) pid = state.get("pid") if process_matches(pid, state.get("process_start_token")): if not isinstance(pid, int) or isinstance(pid, bool): raise RuntimeError("gateway state contains an invalid process id") terminate_process_group(int(pid)) _reap_gateway(pid) state["status"] = "stopped" state["stopped_at"] = utc_now() atomic_write_json(_gateway_state_path_for_key(gateway_hash), state) return state def gateway_status(snapshot_hash: str) -> dict[str, Any]: state = read_gateway_state(snapshot_hash) if state is None: return {"snapshot_hash": snapshot_hash, "status": "not_started"} return state def list_gateways() -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] for directory in sorted(gateways_root().iterdir()): if ( not directory.is_dir() or len(directory.name) != 64 or any(char not in "0123456789abcdef" for char in directory.name) ): continue state = _read_gateway_state_by_key(directory.name) if state: results.append(state) return results def gateway_models(snapshot_hash: str) -> Any: state = ensure_gateway(snapshot_hash) if state is None: return {"object": "list", "data": []} return http_json(state["base_url"].rstrip("/") + "/models") def dry_run_gateway(snapshot_hash: str) -> subprocess.CompletedProcess[str] | None: snapshot = load_snapshot(snapshot_hash) if not snapshot["manifest"].get("gateway_required"): return None settings = load_settings() credentials, missing = _credentials(snapshot) if missing: raise RuntimeError("missing credentials: " + ", ".join(missing)) binary = _binary(settings) environment = filtered_environment( allow_sensitive=credentials, extra={**credentials, "RUST_LOG": "error"}, ) return subprocess.run( [binary, "--config", str(Path(snapshot["directory"]) / "routes.toml"), "--dry-run"], env=environment, capture_output=True, text=True, check=False, timeout=max(1.0, float(settings.get("gateway_start_timeout_seconds", 15))), ) def route_telemetry( metadata: Mapping[str, Any], session: Mapping[str, Any], events_path: Path, ) -> dict[str, Any]: """Extract observed serving-route identity from Codex and Switchyard evidence.""" provider_slugs: list[str] = [] endpoint_tags: list[str] = [] routing_attempts: list[int] = [] retry_counts: list[int] = [] fallback_indices: list[int] = [] def add_unique(target: list[str], value: Any) -> None: if isinstance(value, str) and value.strip() and value not in target: target.append(value) def visit(value: Any, *, scope: str | None = None) -> None: if isinstance(value, dict): selected = value.get("selected") status = value.get("status") successful_attempt_record = ( isinstance(status, int) and not isinstance(status, bool) and 200 <= status < 300 ) for key, child in value.items(): normalized = str(key).casefold() if normalized in {"openrouter_metadata", "routing"}: visit(child, scope=normalized) continue if normalized in { "provider_slug", "serving_provider_slug", "actual_provider_slug", }: add_unique(provider_slugs, child) elif normalized == "provider" and scope in { "openrouter_metadata", "routing", }: if ( selected is True or successful_attempt_record or (scope == "routing" and selected is None) ): add_unique(provider_slugs, child) elif normalized == "provider_name" and ( scope in {"openrouter_metadata", "routing"} or "upstream_id" in value or "total_cost" in value ): add_unique(provider_slugs, child) elif normalized in {"provider_tag", "endpoint_tag", "serving_endpoint_tag"}: add_unique(endpoint_tags, child) elif normalized == "tag" and isinstance(value.get("selected"), bool): if value.get("selected"): add_unique(endpoint_tags, child) elif ( normalized == "attempt" and isinstance(child, int) and not isinstance(child, bool) and scope in {"openrouter_metadata", "routing"} ): routing_attempts.append(child) elif ( normalized in {"retry_count", "retries"} and isinstance(child, int) and not isinstance(child, bool) ): retry_counts.append(max(0, child)) elif ( normalized == "fallback_index" and isinstance(child, int) and not isinstance(child, bool) ): fallback_indices.append(max(0, child)) visit(child, scope=scope) elif isinstance(value, list): for child in value: visit(child, scope=scope) sources: list[str] = [] if events_path.is_file(): for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): with contextlib.suppress(json.JSONDecodeError, ValueError): visit(strict_json_loads(line)) sources.append("codex_events") routing_path = session.get("gateway_routing_log_path") if isinstance(routing_path, str) and Path(routing_path).is_file(): needles = { str(metadata.get("model_key") or ""), str(metadata.get("model") or ""), str(metadata.get("route") or ""), } for line in Path(routing_path).read_text(encoding="utf-8", errors="replace").splitlines(): if not any(needle and needle in line for needle in needles): continue with contextlib.suppress(json.JSONDecodeError, ValueError): visit(strict_json_loads(line)) sources.append("switchyard_routing_log") successful_attempt = max(routing_attempts) if routing_attempts else None observed_fallback_index = ( max(fallback_indices) if fallback_indices else max((attempt - 1 for attempt in routing_attempts if attempt >= 1), default=None) ) retries = max(retry_counts) if retry_counts else None requested_policy = metadata.get("requested_route_policy") return { "requested_policy": requested_policy, "actual_serving_provider_slugs": provider_slugs, "actual_serving_endpoint_tags": endpoint_tags, "successful_attempt": successful_attempt, "fallback_index": observed_fallback_index, "retries": retries, "retry_telemetry_complete": retries is not None, "sources": sources, "complete": requested_policy is None or bool(provider_slugs or endpoint_tags), } def route_telemetry_warnings( metadata: Mapping[str, Any], observation: Mapping[str, Any] ) -> list[str]: """Interpret provider-route evidence without leaking that policy into workers.""" if metadata.get("requested_route_policy") is not None and not observation.get("complete"): return [ "OpenRouter serving-provider telemetry is incomplete; requested policy is recorded " "but the actual endpoint was not present in captured events" ] return [] def _latest_record_timestamp( record: Mapping[str, Any], fields: tuple[str, ...], label: str ) -> float: observed: list[float] = [] for field in fields: value = record.get(field) if value is not None: observed.append(_timestamp(value, f"{label} {field}")) if not observed: raise ValueError(f"{label} has no lifecycle timestamp") return max(observed) def _session_gateway_hash(session: Mapping[str, Any], *, verify_snapshot: bool) -> str | None: recorded = session.get("gateway_hash") if recorded is None: if session.get("gateway_base_url") is not None: raise ValueError("session has a gateway endpoint without a gateway identity") return None gateway_hash = _validate_hash(str(recorded), "session gateway hash") if verify_snapshot: expected = _gateway_key(str(session["snapshot_hash"])) if gateway_hash != expected: raise ValueError("session gateway identity does not match its snapshot") return gateway_hash def _session_admission_timestamp(session: Mapping[str, Any]) -> float: return _timestamp( session.get("last_active_at") or session.get("run_created_at") or session.get("created_at"), "session admission timestamp", ) def _job_admission_timestamp(job: Mapping[str, Any]) -> float: return _timestamp( job.get("recovery_requested_at") or job.get("created_at"), "worker admission timestamp", ) def _session_retains_gateway(session: Mapping[str, Any], now: float) -> bool: status = session.get("status") if status not in ACTIVE_SESSION_STATUSES: return False if process_matches(session.get("root_pid"), session.get("root_start_token")) or process_matches( session.get("root_app_server_pid"), session.get("root_app_server_start_token") ): return True if status == "starting": admitted_at = _session_admission_timestamp(session) return now <= admitted_at + _GATEWAY_ADMISSION_GRACE_SECONDS return False def _job_retains_gateway(job: Mapping[str, Any], now: float) -> bool: status = job.get("status") if status not in RECOVERABLE_JOB_STATUSES: return False if process_matches(job.get("runner_pid"), job.get("runner_start_token")) or process_matches( job.get("app_server_pid"), job.get("app_server_start_token") ): return True if status in {"queued", "starting", "recovering"}: admitted_at = _job_admission_timestamp(job) return now <= admitted_at + _GATEWAY_ADMISSION_GRACE_SECONDS return False def stop_idle_gateways() -> list[str]: """Stop route-set gateways only after every durable execution host releases them.""" settings = load_settings() idle_seconds = int(settings.get("gateway_idle_timeout_seconds", 3600)) now = time.time() active_gateway_hashes: set[str] = set() last_consumer_at: dict[str, float] = {} session_gateway_hashes: dict[str, str | None] = {} try: sessions = iter_session_records(strict=True) jobs = iter_job_records(strict=True) for session in sessions: active = session.get("status") in ACTIVE_SESSION_STATUSES gateway_hash = _session_gateway_hash(session, verify_snapshot=active) session_id = str(session["session_id"]) session_gateway_hashes[session_id] = gateway_hash if gateway_hash is None: continue release_at = _latest_record_timestamp( session, ( "finished_at", "paused_at", "suspended_at", "detached_at", "last_active_at", "run_created_at", "created_at", ), f"session {session_id}", ) if session.get("status") == "starting": release_at = max( release_at, _session_admission_timestamp(session) + _GATEWAY_ADMISSION_GRACE_SECONDS, ) last_consumer_at[gateway_hash] = max( last_consumer_at.get(gateway_hash, 0.0), release_at ) if _session_retains_gateway(session, now): active_gateway_hashes.add(gateway_hash) for job in jobs: session_id = str(job["session_id"]) if session_id not in session_gateway_hashes: if job.get("status") in RECOVERABLE_JOB_STATUSES: raise ValueError(f"active worker {job['job_id']} refers to a missing session") continue gateway_hash = session_gateway_hashes[session_id] if gateway_hash is None: continue release_at = _latest_record_timestamp( job, ( "finished_at", "paused_at", "suspended_at", "last_progress_at", "created_at", ), f"worker {job['job_id']}", ) if job.get("status") in {"queued", "starting", "recovering"}: release_at = max( release_at, _job_admission_timestamp(job) + _GATEWAY_ADMISSION_GRACE_SECONDS, ) last_consumer_at[gateway_hash] = max( last_consumer_at.get(gateway_hash, 0.0), release_at ) if _job_retains_gateway(job, now): active_gateway_hashes.add(gateway_hash) except (KeyError, OSError, RuntimeError, ValueError) as exc: raise RuntimeError( "cannot determine gateway idleness from invalid durable execution state" ) from exc stopped: list[str] = [] for state in list_gateways(): gateway_hash = state.get("gateway_hash") or state.get("snapshot_hash") if not gateway_hash or gateway_hash in active_gateway_hashes: continue if state.get("status") == "invalid": raise RuntimeError( "cannot determine gateway idleness from invalid gateway state: " + str(state.get("error")) ) last_used_at = _timestamp(state.get("last_used_at"), "gateway last-used timestamp") idle_since = max(last_used_at, last_consumer_at.get(str(gateway_hash), 0.0)) if now - idle_since >= idle_seconds and state.get("status") in {"running", "starting"}: representative = str(state.get("snapshot_hash") or "") if representative: stop_gateway(representative) stopped.append(representative) return stopped