2005 lines
82 KiB
Python
2005 lines
82 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from unittest import mock
|
|
|
|
import mmo_eval
|
|
import mmoctl
|
|
from common import FAKE_CODEX, FAKE_SWITCHYARD, ROOT, RuntimeSandbox
|
|
from mmo_eval import (
|
|
_estimated_token_cost,
|
|
_event_usage,
|
|
_retry_count,
|
|
_run_validation,
|
|
compare_runs,
|
|
run_evaluation,
|
|
validate_suite,
|
|
)
|
|
from mmo_util import (
|
|
process_alive,
|
|
process_start_token,
|
|
read_json,
|
|
read_toml,
|
|
shell_exit_status,
|
|
toml_dumps,
|
|
)
|
|
from mmo_version import APP_SERVER_PROTOCOL_CODEX_VERSION, MMO_SCHEMA_VERSION
|
|
from mmoctl import _parser, doctor
|
|
from worker_runner import _extract_usage
|
|
|
|
import scripts.install as install_script
|
|
import scripts.uninstall as uninstall_script
|
|
from scripts.install import (
|
|
SWITCHYARD_BASELINE_VERSION,
|
|
default_bin_dir,
|
|
install_optional_tools,
|
|
toml_quote,
|
|
validate_destination_paths,
|
|
validate_existing_install,
|
|
)
|
|
from scripts.install import xdg_path as install_xdg_path
|
|
from scripts.uninstall import defaults as uninstall_defaults
|
|
from scripts.uninstall import validate_removal_target
|
|
from scripts.uninstall import xdg_path as uninstall_xdg_path
|
|
|
|
|
|
def _current_suite_text(
|
|
*,
|
|
suite_id: str,
|
|
profile: str,
|
|
highest_worker: str,
|
|
workers: tuple[str, ...],
|
|
fixture: str | None = None,
|
|
images: tuple[str, ...] = (),
|
|
forbidden_agents: tuple[str, ...] = (),
|
|
) -> str:
|
|
"""Build a compact matched-control suite for the active MMO generation."""
|
|
|
|
fixture_line = f'fixture = "{fixture}"\n' if fixture else ""
|
|
image_line = (
|
|
"images = [" + ", ".join(json.dumps(item) for item in images) + "]\n" if images else ""
|
|
)
|
|
forbidden_line = (
|
|
"forbidden_agents = [" + ", ".join(json.dumps(item) for item in forbidden_agents) + "]\n"
|
|
if forbidden_agents
|
|
else ""
|
|
)
|
|
ablations = "".join(
|
|
f'''\n[[variants]]
|
|
id = "without-{worker.replace("_", "-")}"
|
|
purpose = "Focused worker ablation."
|
|
topology = "full_without_worker"
|
|
worker = "{worker}"
|
|
comparison_class = "ablation"
|
|
'''
|
|
for worker in workers
|
|
)
|
|
return f'''schema_version = {MMO_SCHEMA_VERSION}
|
|
id = "{suite_id}"
|
|
profile = "{profile}"
|
|
name = "Focused current-generation suite"
|
|
description = "Matched-control fixture for runtime integration tests."
|
|
{fixture_line}development_trials = 1
|
|
release_trials = 1
|
|
|
|
[promotion]
|
|
primary_metric = "success_rate"
|
|
direction = "higher"
|
|
strongest_success_tolerance = 0.0
|
|
minimum_relative_improvement = 0.0
|
|
minimum_absolute_improvement = 0.0
|
|
worker_minimum_success_contribution = 0.0
|
|
worker_minimum_metric_contribution = 0.0
|
|
no_regression_higher_metrics = []
|
|
no_regression_lower_metrics = []
|
|
require_complete_api_cost = false
|
|
|
|
[[variants]]
|
|
id = "configured-root"
|
|
purpose = "Configured root alone."
|
|
topology = "root_only"
|
|
comparison_class = "configured_root_alone"
|
|
|
|
[[variants]]
|
|
id = "strongest-single"
|
|
purpose = "Strongest matched single-agent control."
|
|
topology = "root_only"
|
|
comparison_class = "strongest_single_agent"
|
|
|
|
[[variants]]
|
|
id = "access-single"
|
|
purpose = "Accessible service control."
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "chatgpt_codex"
|
|
|
|
[[variants]]
|
|
id = "root-plus-worker"
|
|
purpose = "Root plus the highest-value worker."
|
|
topology = "root_plus_worker"
|
|
worker = "{highest_worker}"
|
|
comparison_class = "root_plus_highest_value"
|
|
|
|
[[variants]]
|
|
id = "full-profile"
|
|
purpose = "Complete configured profile."
|
|
topology = "full"
|
|
comparison_class = "full_profile"
|
|
{ablations}
|
|
[[tasks]]
|
|
id = "one"
|
|
description = "Focused integration task."
|
|
sandbox = "read-only"
|
|
difficulty = "easy"
|
|
negative_control = false
|
|
wall_timeout_seconds = 60
|
|
validation_timeout_seconds = 30
|
|
prompt = "Inspect one bounded concern and end with EVAL_V_OK."
|
|
{image_line}
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_V_OK"]
|
|
forbidden_patterns = []
|
|
validation_commands = []
|
|
|
|
[tasks.orchestration_assertions]
|
|
{forbidden_line}min_peak_mcp_workers = 0
|
|
max_jobs = 0
|
|
max_contract_failures = 0
|
|
max_observed_mcp_wait_ratio = 0.1
|
|
'''
|
|
|
|
|
|
class InstallEvaluationTests(unittest.TestCase):
|
|
def test_orchestration_assertions_mark_absent_variant_roles_nonapplicable(self) -> None:
|
|
task = {"orchestration_assertions": {"required_agents": ["fresh_critic"]}}
|
|
absent = mmo_eval._orchestration_assertion_results(
|
|
task,
|
|
{"profile_agents": ["lead"], "agents": {}},
|
|
1.0,
|
|
)
|
|
self.assertTrue(absent["passed"])
|
|
self.assertEqual(absent["checks"][0]["not_applicable"], ["fresh_critic"])
|
|
|
|
applicable = mmo_eval._orchestration_assertion_results(
|
|
task,
|
|
{"profile_agents": ["lead", "fresh_critic"], "agents": {}},
|
|
1.0,
|
|
)
|
|
self.assertFalse(applicable["passed"])
|
|
|
|
def test_labs_report_hypothesis_gates_without_a_superiority_verdict(self) -> None:
|
|
variants = [
|
|
{"id": "root", "comparison_class": "configured_root_alone"},
|
|
{"id": "strongest", "comparison_class": "strongest_single_agent"},
|
|
{"id": "access", "comparison_class": "access_service_single_agent"},
|
|
{"id": "full", "comparison_class": "full_profile"},
|
|
]
|
|
|
|
def summary(identifier: str, scarce: float) -> dict[str, Any]:
|
|
return {
|
|
"id": identifier,
|
|
"success_rate": 1.0,
|
|
"outcome_metrics": {"scarce_tier_request_units": scarce},
|
|
"write_scope_conflicts": 0,
|
|
"contract_failures": 0,
|
|
"route_telemetry_incomplete": 0,
|
|
"cost_ledgers": {"actual_api_usd_complete": True},
|
|
}
|
|
|
|
verdict = mmo_eval._promotion_verdict(
|
|
resolved={
|
|
"profile": {"maturity": "lab", "root": "lead"},
|
|
"agents": {"lead": {"can_spawn": []}},
|
|
},
|
|
suite={
|
|
"promotion": {
|
|
"primary_metric": "scarce_tier_request_units",
|
|
"primary_baseline": "strongest_single_agent",
|
|
"direction": "lower",
|
|
"strongest_success_tolerance": 0.02,
|
|
"minimum_relative_improvement": 0.50,
|
|
"minimum_absolute_improvement": 1.0,
|
|
"require_complete_api_cost": True,
|
|
}
|
|
},
|
|
variants=variants,
|
|
summaries={
|
|
"root": summary("root", 0.0),
|
|
"strongest": summary("strongest", 3.0),
|
|
"access": summary("access", 2.0),
|
|
"full": summary("full", 1.0),
|
|
},
|
|
skipped=[],
|
|
)
|
|
self.assertFalse(verdict["eligible"])
|
|
self.assertIsNone(verdict["passed"])
|
|
self.assertTrue(verdict["hypothesis_passed"], verdict)
|
|
primary = next(
|
|
check for check in verdict["checks"] if check["name"] == "primary_metric_improvement"
|
|
)
|
|
self.assertEqual(primary["baseline_variant"], "strongest")
|
|
|
|
def test_variant_summary_counts_only_declared_scarce_model_requests(self) -> None:
|
|
empty_ledger = {
|
|
"actual_api_usd": 0.0,
|
|
"actual_api_usd_complete": True,
|
|
"api_equivalent_estimate_usd": 0.0,
|
|
"api_equivalent_estimate_complete": True,
|
|
"subscription_units": {},
|
|
"local_resource_seconds": {},
|
|
}
|
|
worker = {
|
|
"job_count": 3,
|
|
"statuses": {"completed": 3},
|
|
"agents": {},
|
|
"models": {"scarce-worker": 2, "economical-worker": 1},
|
|
"routes": {},
|
|
"makers": {},
|
|
"api_operators": {},
|
|
"access_products": {},
|
|
"gateway_drivers": {},
|
|
"serving_providers": {},
|
|
"serving_endpoints": {},
|
|
"usage": {},
|
|
"cost_ledgers": empty_ledger,
|
|
"terminal_results_accepted": 0,
|
|
"contract_failures": 0,
|
|
"write_scope_conflicts": 0,
|
|
"route_telemetry_incomplete": 0,
|
|
}
|
|
record = {
|
|
"passed": True,
|
|
"root_usage": {},
|
|
"worker_metrics": worker,
|
|
"root_identity": {
|
|
"model": "scarce-root",
|
|
"maker": "maker",
|
|
"route": "route",
|
|
"api_operator": "operator",
|
|
"access_product": "product",
|
|
"gateway_driver": "driver",
|
|
},
|
|
"outcome_metrics": {},
|
|
"root_cost_ledgers": empty_ledger,
|
|
"root_elapsed_seconds": 1.0,
|
|
"explicit_root_mcp_wait_seconds": 0.0,
|
|
"root_activity": {},
|
|
"orchestration_diagnostics_passed": True,
|
|
}
|
|
result = mmo_eval._variant_summary(
|
|
{
|
|
"id": "full",
|
|
"purpose": "test",
|
|
"comparison_class": "full_profile",
|
|
"topology": "full",
|
|
},
|
|
[record],
|
|
scarce_model_keys={"scarce-root", "scarce-worker"},
|
|
)
|
|
self.assertEqual(result["outcome_metrics"]["scarce_tier_request_units"], 3.0)
|
|
|
|
def test_integration_corrections_are_measured_from_canonical_artifacts(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
target = box.workspace / "result.py"
|
|
target.write_text("value = 1\n", encoding="utf-8")
|
|
expected_hash = hashlib.sha256(target.read_bytes()).hexdigest()
|
|
job = {
|
|
"result_state": "integrated",
|
|
"canonical_cwd": str(box.workspace),
|
|
"patch": {"changed_paths": ["result.py", "deleted.py"]},
|
|
"artifacts": [
|
|
{
|
|
"relative_path": "result.py",
|
|
"sha256": expected_hash,
|
|
}
|
|
],
|
|
}
|
|
unchanged = mmo_eval._integration_correction_metrics([job])
|
|
self.assertEqual(unchanged["integrated_patch_paths"], 2)
|
|
self.assertEqual(unchanged["integration_corrected_paths"], 0)
|
|
|
|
target.write_text("value = 2\n", encoding="utf-8")
|
|
(box.workspace / "deleted.py").write_text("restored = True\n", encoding="utf-8")
|
|
corrected = mmo_eval._integration_correction_metrics([job])
|
|
self.assertEqual(corrected["integration_corrected_paths"], 2)
|
|
self.assertEqual(corrected["integrated_jobs_with_corrections"], 1)
|
|
self.assertEqual(corrected["integration_correction_rate"], 1.0)
|
|
|
|
def test_root_activity_reports_timestamp_coverage_and_worker_overlap(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
events = box.root / "root-events.jsonl"
|
|
events.write_text(
|
|
"\n".join(
|
|
[
|
|
json.dumps({"timestamp": "2026-08-16T12:00:01+00:00"}),
|
|
json.dumps({"timestamp": "2026-08-16T12:00:06+00:00"}),
|
|
json.dumps({"event": "missing timestamp"}),
|
|
]
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
jobs = [
|
|
{
|
|
"session_id": "session-under-test",
|
|
"started_at": "2026-08-16T12:00:00+00:00",
|
|
"finished_at": "2026-08-16T12:00:05+00:00",
|
|
}
|
|
]
|
|
with mock.patch("mmo_eval.iter_jobs", return_value=jobs):
|
|
metrics = mmo_eval._root_activity_metrics(events, "session-under-test")
|
|
coverage = metrics["telemetry_coverage"]
|
|
self.assertEqual(coverage["event_count"], 3)
|
|
self.assertEqual(coverage["timestamped_event_count"], 2)
|
|
self.assertAlmostEqual(coverage["timestamp_coverage_ratio"], 2 / 3)
|
|
self.assertIsNone(coverage["activity_overlap_seconds"])
|
|
self.assertEqual(metrics["observed_root_activity_events_during_worker_execution"], 1)
|
|
|
|
def test_cost_ledgers_keep_api_subscription_and_local_units_separate(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
api_events = box.root / "api-events.jsonl"
|
|
api_events.write_text('{"usage":{"cost_usd":0.125}}\n', encoding="utf-8")
|
|
missing_api_events = box.root / "missing-api-events.jsonl"
|
|
missing_api_events.write_text('{"usage":{"input_tokens":10}}\n', encoding="utf-8")
|
|
empty_events = box.root / "empty-events.jsonl"
|
|
empty_events.write_text("", encoding="utf-8")
|
|
usage = {"input_tokens": 100, "output_tokens": 25}
|
|
model = {
|
|
"input_cost_per_million": 1.0,
|
|
"output_cost_per_million": 2.0,
|
|
}
|
|
api = mmo_eval._call_cost_ledgers(
|
|
usage=usage,
|
|
model=model,
|
|
route={"billing_mode": "api", "access_product": "openrouter_api"},
|
|
events_path=api_events,
|
|
elapsed_seconds=2.0,
|
|
)
|
|
subscription = mmo_eval._call_cost_ledgers(
|
|
usage=usage,
|
|
model=model,
|
|
route={"billing_mode": "subscription", "access_product": "zai_coding_plan"},
|
|
events_path=empty_events,
|
|
elapsed_seconds=3.0,
|
|
)
|
|
local = mmo_eval._call_cost_ledgers(
|
|
usage=usage,
|
|
model=model,
|
|
route={"billing_mode": "local", "access_product": "local_gpu"},
|
|
events_path=empty_events,
|
|
elapsed_seconds=4.5,
|
|
)
|
|
merged = mmo_eval._merge_cost_ledgers([api, subscription, local])
|
|
self.assertEqual(merged["actual_api_usd"], 0.125)
|
|
self.assertEqual(
|
|
merged["subscription_units"]["zai_coding_plan"],
|
|
{"request_units": 1, "observed_tokens": 125},
|
|
)
|
|
self.assertEqual(merged["local_resource_seconds"], {"local_gpu": 4.5})
|
|
|
|
incomplete_api = mmo_eval._call_cost_ledgers(
|
|
usage=usage,
|
|
model=model,
|
|
route={"billing_mode": "api", "access_product": "openrouter_api"},
|
|
events_path=missing_api_events,
|
|
elapsed_seconds=2.0,
|
|
)
|
|
incomplete = mmo_eval._merge_cost_ledgers([incomplete_api, subscription, local])
|
|
self.assertIsNone(incomplete["actual_api_usd"])
|
|
self.assertFalse(incomplete["actual_api_usd_complete"])
|
|
self.assertEqual(
|
|
incomplete["subscription_units"]["zai_coding_plan"]["request_units"], 1
|
|
)
|
|
self.assertEqual(incomplete["local_resource_seconds"]["local_gpu"], 4.5)
|
|
|
|
def test_doctor_uses_codex_login_status_as_builtin_credential_gate(self) -> None:
|
|
with RuntimeSandbox():
|
|
with (
|
|
mock.patch(
|
|
"mmo_diagnostics._codex_auth_status",
|
|
return_value={"passed": False, "exit_code": 1, "stderr": "Not logged in"},
|
|
),
|
|
mock.patch(
|
|
"mmo_diagnostics.app_server_protocol_status",
|
|
return_value={"passed": True},
|
|
),
|
|
):
|
|
result = doctor("codex-harness-team", live=False, probe=False, bindings={})
|
|
self.assertTrue(result["offline_ok"], result)
|
|
self.assertFalse(result["credentials_ok"], result)
|
|
self.assertFalse(result["passed"], result)
|
|
self.assertEqual(result["credentials"]["codex_login_status"]["stderr"], "Not logged in")
|
|
|
|
def test_doctor_rejects_keyring_only_auth_for_generated_codex_home(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
(box.base_codex_home / "auth.json").unlink()
|
|
with mock.patch(
|
|
"mmo_diagnostics._codex_auth_status",
|
|
return_value={"passed": True, "exit_code": 0, "stdout": "Logged in"},
|
|
):
|
|
result = doctor("codex-harness-team", live=False, probe=False, bindings={})
|
|
auth = result["credentials"]["builtin_auth"]["codex_chatgpt_builtin"]
|
|
self.assertFalse(result["credentials_ok"], result)
|
|
self.assertFalse(auth["usable_by_generated_home"])
|
|
self.assertIn("keyring-only", auth["reason"])
|
|
|
|
def test_doctor_gates_required_tool_mcp_transport_and_credentials(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
(box.config / "tool-mcp.d" / "servers.toml").write_text(
|
|
f"""schema_version = {MMO_SCHEMA_VERSION}
|
|
|
|
[tool_mcp_servers.missing_command]
|
|
transport = "stdio"
|
|
command = "/definitely/missing/codex-mmo-tool-mcp"
|
|
enabled_tools = ["inspect"]
|
|
default_tools_approval_mode = "approve"
|
|
|
|
[tool_mcp_servers.missing_credential]
|
|
transport = "streamable_http"
|
|
url = "https://example.invalid/mcp"
|
|
bearer_token_env_var = "IDA_MCP_TOKEN"
|
|
enabled_tools = ["search"]
|
|
default_tools_approval_mode = "prompt"
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
profile = box.root / "tool-mcp-doctor"
|
|
shutil.copytree(ROOT / "profiles" / "codex-harness-team", profile)
|
|
manifest = profile / "profile.toml"
|
|
profile_data = read_toml(manifest)
|
|
profile_data["agents"]["integrator"]["tool_mcp_servers"] = {
|
|
"missing_command": {},
|
|
"missing_credential": {},
|
|
}
|
|
manifest.write_text(toml_dumps(profile_data), encoding="utf-8")
|
|
|
|
with (
|
|
mock.patch.dict(os.environ, {"IDA_MCP_TOKEN": ""}),
|
|
mock.patch(
|
|
"mmo_diagnostics._codex_auth_status",
|
|
return_value={"passed": True, "exit_code": 0, "stdout": "Logged in"},
|
|
),
|
|
):
|
|
result = doctor(str(profile), live=False, probe=False, bindings={})
|
|
|
|
self.assertFalse(result["offline_ok"], result)
|
|
self.assertFalse(result["credentials_ok"], result)
|
|
self.assertFalse(result["passed"], result)
|
|
status = result["tool_mcp"]
|
|
self.assertFalse(status["transport_ok"])
|
|
self.assertFalse(status["credentials_ok"])
|
|
self.assertEqual(status["servers"]["missing_command"]["required_by"], ["integrator"])
|
|
self.assertFalse(
|
|
status["servers"]["missing_credential"]["environment"]["IDA_MCP_TOKEN"]
|
|
)
|
|
|
|
def test_current_codex_usage_fields_and_cached_cost_formula(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-usage-") as temporary_raw:
|
|
events = Path(temporary_raw) / "events.jsonl"
|
|
events.write_text(
|
|
"\n".join(
|
|
(
|
|
json.dumps(
|
|
{
|
|
"type": "turn.completed",
|
|
"usage": {
|
|
"input_tokens": 100,
|
|
"cached_input_tokens": 40,
|
|
"cache_write_input_tokens": 5,
|
|
"output_tokens": 10,
|
|
"reasoning_output_tokens": 4,
|
|
},
|
|
}
|
|
),
|
|
'{"usage":{"input_tokens":NaN}}',
|
|
"not-json",
|
|
)
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
expected = {
|
|
"input_tokens": 100,
|
|
"cached_input_tokens": 40,
|
|
"cache_write_input_tokens": 5,
|
|
"output_tokens": 10,
|
|
"reasoning_output_tokens": 4,
|
|
"event_count": 1,
|
|
}
|
|
self.assertEqual(_event_usage(events), expected)
|
|
self.assertEqual(_extract_usage(events), expected)
|
|
cumulative = {
|
|
"inputTokens": 120,
|
|
"cachedInputTokens": 30,
|
|
"cacheWriteInputTokens": 6,
|
|
"outputTokens": 20,
|
|
"reasoningOutputTokens": 8,
|
|
"totalTokens": 140,
|
|
}
|
|
usage_notification = {
|
|
"message": {
|
|
"method": "thread/tokenUsage/updated",
|
|
"params": {
|
|
"threadId": "thread-1",
|
|
"turnId": "turn-1",
|
|
"tokenUsage": {"last": cumulative, "total": cumulative},
|
|
},
|
|
}
|
|
}
|
|
events.write_text(
|
|
json.dumps(usage_notification) + "\n" + json.dumps(usage_notification) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
self.assertEqual(
|
|
_event_usage(events),
|
|
{
|
|
"input_tokens": 120,
|
|
"cached_input_tokens": 30,
|
|
"cache_write_input_tokens": 6,
|
|
"output_tokens": 20,
|
|
"reasoning_output_tokens": 8,
|
|
"total_tokens": 140,
|
|
"event_count": 2,
|
|
},
|
|
)
|
|
events.write_text(
|
|
'{"retry_count":true,"attempts":true,"retries":2}\n',
|
|
encoding="utf-8",
|
|
)
|
|
self.assertEqual(_retry_count(events), 2)
|
|
events.write_text('{"openrouter_metadata":{"attempt":3}}\n', encoding="utf-8")
|
|
self.assertEqual(_retry_count(events), 0)
|
|
|
|
model = {
|
|
"input_cost_per_million": 2.0,
|
|
"cached_input_cost_per_million": 0.5,
|
|
"cache_write_input_cost_per_million": 3.0,
|
|
"output_cost_per_million": 6.0,
|
|
}
|
|
self.assertAlmostEqual(_estimated_token_cost(expected, model) or 0, 0.000205)
|
|
self.assertIsNone(
|
|
_estimated_token_cost(
|
|
expected,
|
|
{
|
|
key: value
|
|
for key, value in model.items()
|
|
if key != "cached_input_cost_per_million"
|
|
},
|
|
)
|
|
)
|
|
self.assertIsNone(
|
|
_estimated_token_cost(
|
|
expected,
|
|
{
|
|
key: value
|
|
for key, value in model.items()
|
|
if key != "cache_write_input_cost_per_million"
|
|
},
|
|
)
|
|
)
|
|
self.assertIsNone(
|
|
_estimated_token_cost(
|
|
{
|
|
"input_tokens": 1,
|
|
"cached_input_tokens": 1,
|
|
"cache_write_input_tokens": 1,
|
|
"output_tokens": 0,
|
|
},
|
|
model,
|
|
)
|
|
)
|
|
|
|
def test_run_parser_separates_codex_owned_options(self) -> None:
|
|
parsed = _parser().parse_args(
|
|
[
|
|
"run",
|
|
"--profile",
|
|
"visual-engineering",
|
|
"--",
|
|
"--image",
|
|
"screenshot.png",
|
|
]
|
|
)
|
|
self.assertEqual(parsed.profile, "visual-engineering")
|
|
self.assertEqual(parsed.codex_args, ["--", "--image", "screenshot.png"])
|
|
with self.assertRaises(mmoctl.CLIUsageError):
|
|
_parser().parse_args(["jobs", "result", "job-id", "--max-chars", "499"])
|
|
with self.assertRaises(mmoctl.CLIUsageError):
|
|
_parser().parse_args(["catalog", "verify", "--zai-url", "https://example.invalid"])
|
|
accepted = _parser().parse_args(["jobs", "result", "job-id", "--max-chars", "500"])
|
|
self.assertEqual(accepted.max_chars, 500)
|
|
|
|
def test_resume_parser_and_dispatch_use_the_persistent_session_command(self) -> None:
|
|
parsed = _parser().parse_args(["resume", "--last", "--all", "--allow-tainted"])
|
|
self.assertTrue(parsed.last)
|
|
self.assertTrue(parsed.all_cwds)
|
|
self.assertTrue(parsed.allow_tainted)
|
|
runs = _parser().parse_args(["session", "runs", "session-id", "--limit", "3"])
|
|
self.assertEqual((runs.session_id, runs.limit), ("session-id", 3))
|
|
with mock.patch("mmoctl.iter_session_runs", return_value=[]) as history:
|
|
self.assertEqual(mmoctl.main(["session", "runs", "session-id"]), 0)
|
|
history.assert_called_once_with("session-id")
|
|
jobs = _parser().parse_args(["jobs", "list", "--run", "run-id"])
|
|
self.assertEqual(jobs.run, "run-id")
|
|
with (
|
|
mock.patch("mmoctl.resolve_resume_session", return_value="persistent-session") as pick,
|
|
mock.patch("mmoctl.resume_interactive", return_value=-2) as resume,
|
|
):
|
|
self.assertEqual(mmoctl.main(["resume", "--last"]), 130)
|
|
pick.assert_called_once()
|
|
resume.assert_called_once_with("persistent-session", allow_tainted=False)
|
|
with mock.patch("mmoctl.launch_interactive") as launch:
|
|
self.assertEqual(mmoctl.main(["run", "resume"]), 1)
|
|
launch.assert_not_called()
|
|
self.assertEqual(
|
|
mmoctl.main(["run", "--", "--no-alt-screen", "resume", "--last"]),
|
|
1,
|
|
)
|
|
launch.assert_not_called()
|
|
with mock.patch("mmoctl.launch_interactive", return_value=0) as launch:
|
|
self.assertEqual(mmoctl.main(["run", "--", "--model", "resume"]), 0)
|
|
self.assertEqual(launch.call_args.kwargs["codex_args"], ["--model", "resume"])
|
|
|
|
def test_runtime_bindings_reject_duplicate_agent_assignments(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "duplicate binding"):
|
|
mmoctl._bindings(
|
|
[
|
|
"invariant_designer=codex_chatgpt_builtin__gpt_5_6_sol",
|
|
"invariant_designer=codex_chatgpt_builtin__gpt_5_6_terra",
|
|
]
|
|
)
|
|
|
|
def test_profile_validation_reports_the_bound_model(self) -> None:
|
|
with RuntimeSandbox():
|
|
report = mmoctl.profile_validation_report(
|
|
"codex-harness-team",
|
|
{"invariant_designer": "codex_chatgpt_builtin__gpt_5_6_terra"},
|
|
)
|
|
self.assertEqual(
|
|
report["profile"]["agents"]["invariant_designer"]["model"],
|
|
"codex_chatgpt_builtin__gpt_5_6_terra",
|
|
)
|
|
|
|
def test_tool_mcp_cli_lists_shows_and_validates_operator_registry(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
registry = box.config / "tool-mcp.d" / "servers.toml"
|
|
registry.write_text(
|
|
f"""schema_version = {MMO_SCHEMA_VERSION}
|
|
|
|
[tool_mcp_servers.local_docs]
|
|
transport = "stdio"
|
|
command = "/bin/true"
|
|
enabled_tools = ["search"]
|
|
default_tools_approval_mode = "approve"
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
with mock.patch("mmoctl.emit_structured") as emit:
|
|
self.assertEqual(mmoctl.main(["--json", "tool-mcp", "list"]), 0)
|
|
listed = emit.call_args.args[1]
|
|
self.assertIn("local_docs", listed["servers"])
|
|
self.assertTrue(listed["servers"]["local_docs"]["ready"])
|
|
|
|
self.assertEqual(mmoctl.main(["tool-mcp", "show", "local_docs"]), 0)
|
|
shown = emit.call_args.args[1]
|
|
self.assertEqual(shown["definition"]["transport"], "stdio")
|
|
|
|
self.assertEqual(mmoctl.main(["tool-mcp", "validate"]), 0)
|
|
self.assertTrue(emit.call_args.args[1]["passed"])
|
|
|
|
self.assertEqual(mmoctl.main(["tool-mcp", "show", "missing"]), 1)
|
|
|
|
def test_cli_validation_evaluation_and_wait_exit_statuses_are_truthful(self) -> None:
|
|
with (
|
|
mock.patch("mmoctl.profile_validation_report", return_value={"valid": False}),
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(mmoctl.main(["profile", "validate", "broken"]), 1)
|
|
with (
|
|
mock.patch("mmoctl.validate_suite", return_value={"valid": False}),
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(mmoctl.main(["eval", "validate", "broken"]), 1)
|
|
with (
|
|
mock.patch(
|
|
"mmoctl.run_evaluation", return_value={"status": "completed_with_failures"}
|
|
) as run_eval,
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(
|
|
mmoctl.main(
|
|
[
|
|
"eval",
|
|
"run",
|
|
"--profile",
|
|
"p",
|
|
"--suite",
|
|
"s",
|
|
"--trial-mode",
|
|
"release",
|
|
]
|
|
),
|
|
1,
|
|
)
|
|
self.assertEqual(run_eval.call_args.kwargs["trial_mode"], "release")
|
|
with (
|
|
mock.patch(
|
|
"mmoctl.run_evaluation",
|
|
return_value={
|
|
"status": "completed",
|
|
"summary": {"promotion": {"eligible": True, "passed": False}},
|
|
},
|
|
),
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(
|
|
mmoctl.main(
|
|
[
|
|
"eval",
|
|
"run",
|
|
"--profile",
|
|
"p",
|
|
"--suite",
|
|
"s",
|
|
"--trial-mode",
|
|
"release",
|
|
]
|
|
),
|
|
1,
|
|
)
|
|
self.assertEqual(
|
|
mmoctl.main(["eval", "run", "--profile", "p", "--suite", "s"]),
|
|
0,
|
|
)
|
|
with (
|
|
mock.patch("mmoctl.wait_for_jobs", return_value={"unfinished": ["job"]}),
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(mmoctl.main(["jobs", "wait", "--session", "s", "job"]), 1)
|
|
|
|
def test_signal_terminated_children_use_shell_exit_statuses(self) -> None:
|
|
self.assertEqual(shell_exit_status(-2), 130)
|
|
self.assertEqual(shell_exit_status(-15), 143)
|
|
self.assertEqual(shell_exit_status(7), 7)
|
|
with mock.patch("mmoctl.launch_interactive", return_value=-2):
|
|
self.assertEqual(mmoctl.main(["run"]), 130)
|
|
with (
|
|
mock.patch("mmoctl.run_root_exec", return_value={"exit_code": -15, "result": ""}),
|
|
mock.patch("mmoctl.emit_json"),
|
|
):
|
|
self.assertEqual(mmoctl.main(["--json", "exec", "task"]), 143)
|
|
|
|
def test_catalog_codex_cli_uses_configured_context_and_explicit_options_imply_source(
|
|
self,
|
|
) -> None:
|
|
with RuntimeSandbox() as box:
|
|
configured = Path(os.environ.pop("MMO_CODEX_BIN"))
|
|
settings_path = box.config / "settings.toml"
|
|
settings_path.write_text(
|
|
settings_path.read_text(encoding="utf-8").replace(
|
|
'codex_bin = "codex"', f'codex_bin = "{configured.as_posix()}"'
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with (
|
|
mock.patch("mmoctl.verify_catalog", return_value={"passed": True}) as verify,
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(mmoctl.main(["catalog", "verify", "--codex"]), 0)
|
|
self.assertEqual(verify.call_args.kwargs["codex_binary"], str(configured))
|
|
self.assertEqual(verify.call_args.kwargs["codex_home"], box.base_codex_home)
|
|
self.assertTrue(verify.call_args.kwargs["include_codex"])
|
|
|
|
with (
|
|
mock.patch("mmoctl.verify_catalog", return_value={"passed": True}) as verify,
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(
|
|
mmoctl.main(
|
|
["catalog", "verify", "--opencode-url", "https://example.invalid/models"]
|
|
),
|
|
0,
|
|
)
|
|
self.assertTrue(verify.call_args.kwargs["remote"])
|
|
|
|
with (
|
|
mock.patch("mmoctl.verify_catalog", return_value={"passed": True}) as verify,
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(
|
|
mmoctl.main(
|
|
[
|
|
"catalog",
|
|
"verify",
|
|
"--opencode-zen-url",
|
|
"https://example.invalid/zen/models",
|
|
]
|
|
),
|
|
0,
|
|
)
|
|
self.assertTrue(verify.call_args.kwargs["remote"])
|
|
self.assertEqual(
|
|
verify.call_args.kwargs["opencode_zen_url"],
|
|
"https://example.invalid/zen/models",
|
|
)
|
|
|
|
with (
|
|
mock.patch("mmoctl.verify_catalog", return_value={"passed": True}) as verify,
|
|
mock.patch("mmoctl.emit_structured"),
|
|
):
|
|
self.assertEqual(
|
|
mmoctl.main(
|
|
[
|
|
"catalog",
|
|
"verify",
|
|
"--openrouter-url",
|
|
"https://example.invalid/models",
|
|
]
|
|
),
|
|
0,
|
|
)
|
|
self.assertTrue(verify.call_args.kwargs["remote"])
|
|
self.assertEqual(
|
|
verify.call_args.kwargs["openrouter_url"],
|
|
"https://example.invalid/models",
|
|
)
|
|
|
|
def test_doctor_probe_requires_live_mode(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "requires --live"):
|
|
doctor("codex-harness-team", live=False, probe=True, bindings={})
|
|
|
|
def test_optional_switchyard_install_pins_verified_top_level_version(self) -> None:
|
|
args = mock.Mock(
|
|
install_codex=False,
|
|
install_switchyard=True,
|
|
codex_bin="codex",
|
|
switchyard_bin="switchyard-server",
|
|
)
|
|
with (
|
|
mock.patch(
|
|
"scripts.install.shutil.which",
|
|
side_effect=[None, "/usr/bin/cargo", "/opt/bin/switchyard-server"],
|
|
),
|
|
mock.patch("scripts.install.subprocess.run") as run,
|
|
):
|
|
install_optional_tools(args)
|
|
self.assertEqual(
|
|
run.call_args.args[0],
|
|
[
|
|
"/usr/bin/cargo",
|
|
"install",
|
|
"--locked",
|
|
"--version",
|
|
SWITCHYARD_BASELINE_VERSION,
|
|
"switchyard-server",
|
|
],
|
|
)
|
|
|
|
def test_optional_codex_install_pins_and_verifies_the_exact_reviewed_version(self) -> None:
|
|
args = mock.Mock(
|
|
install_codex=True,
|
|
install_switchyard=False,
|
|
codex_bin="codex",
|
|
switchyard_bin="switchyard-server",
|
|
)
|
|
version = subprocess.CompletedProcess(
|
|
["/opt/bin/codex", "--version"],
|
|
0,
|
|
stdout=f"codex-cli {APP_SERVER_PROTOCOL_CODEX_VERSION}\n",
|
|
stderr="",
|
|
)
|
|
with (
|
|
mock.patch(
|
|
"scripts.install.shutil.which",
|
|
side_effect=[None, "/usr/bin/npm", "/opt/bin/codex"],
|
|
),
|
|
mock.patch(
|
|
"scripts.install.subprocess.run",
|
|
side_effect=[mock.Mock(returncode=0), version],
|
|
) as run,
|
|
):
|
|
install_optional_tools(args)
|
|
self.assertEqual(
|
|
run.call_args_list[0].args[0],
|
|
[
|
|
"/usr/bin/npm",
|
|
"install",
|
|
"-g",
|
|
f"@openai/codex@{APP_SERVER_PROTOCOL_CODEX_VERSION}",
|
|
],
|
|
)
|
|
self.assertEqual(run.call_args_list[1].args[0], ["/opt/bin/codex", "--version"])
|
|
|
|
def test_optional_codex_install_skips_an_already_matching_binary(self) -> None:
|
|
args = mock.Mock(
|
|
install_codex=True,
|
|
install_switchyard=False,
|
|
codex_bin="codex",
|
|
switchyard_bin="switchyard-server",
|
|
)
|
|
version = subprocess.CompletedProcess(
|
|
["/opt/bin/codex", "--version"],
|
|
0,
|
|
stdout=f"codex-cli {APP_SERVER_PROTOCOL_CODEX_VERSION}\n",
|
|
stderr="",
|
|
)
|
|
with (
|
|
mock.patch("scripts.install.shutil.which", return_value="/opt/bin/codex"),
|
|
mock.patch("scripts.install.subprocess.run", return_value=version) as run,
|
|
):
|
|
install_optional_tools(args)
|
|
run.assert_called_once()
|
|
|
|
def test_optional_codex_install_replaces_and_rejects_a_remaining_mismatch(self) -> None:
|
|
args = mock.Mock(
|
|
install_codex=True,
|
|
install_switchyard=False,
|
|
codex_bin="codex",
|
|
switchyard_bin="switchyard-server",
|
|
)
|
|
old = subprocess.CompletedProcess(
|
|
["/old/codex", "--version"], 0, stdout="codex-cli 0.148.0\n", stderr=""
|
|
)
|
|
still_old = subprocess.CompletedProcess(
|
|
["/old/codex", "--version"], 0, stdout="codex-cli 0.148.0\n", stderr=""
|
|
)
|
|
with (
|
|
mock.patch(
|
|
"scripts.install.shutil.which",
|
|
side_effect=["/old/codex", "/usr/bin/npm", "/old/codex"],
|
|
),
|
|
mock.patch(
|
|
"scripts.install.subprocess.run",
|
|
side_effect=[old, mock.Mock(returncode=0), still_old],
|
|
),
|
|
self.assertRaisesRegex(RuntimeError, "reviewed version 0.149.0"),
|
|
):
|
|
install_optional_tools(args)
|
|
|
|
def test_installer_rejects_missing_linked_or_special_payload_members(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-install-source-") as temporary:
|
|
root = Path(temporary)
|
|
(root / "docs").mkdir()
|
|
(root / "VERSION").write_text("test\n", encoding="utf-8")
|
|
outside = root / "outside.txt"
|
|
outside.write_text("must not be copied\n", encoding="utf-8")
|
|
(root / "docs" / "linked.txt").symlink_to(outside)
|
|
with (
|
|
mock.patch.object(install_script, "PACKAGE_ROOT", root),
|
|
mock.patch.object(install_script, "PAYLOAD_FILES", ("VERSION",)),
|
|
mock.patch.object(install_script, "PAYLOAD_DIRECTORIES", ("docs",)),
|
|
self.assertRaisesRegex(ValueError, "symbolic links"),
|
|
):
|
|
install_script.copy_payload(root / "destination")
|
|
self.assertFalse((root / "destination").exists())
|
|
|
|
(root / "docs" / "linked.txt").unlink()
|
|
with (
|
|
mock.patch.object(install_script, "PACKAGE_ROOT", root),
|
|
mock.patch.object(install_script, "PAYLOAD_FILES", ("VERSION", "LICENSE")),
|
|
mock.patch.object(install_script, "PAYLOAD_DIRECTORIES", ("docs",)),
|
|
self.assertRaisesRegex(FileNotFoundError, "LICENSE"),
|
|
):
|
|
install_script.copy_payload(root / "destination")
|
|
|
|
(root / "LICENSE").write_text("test license\n", encoding="utf-8")
|
|
os.mkfifo(root / "docs" / "named-pipe")
|
|
with (
|
|
mock.patch.object(install_script, "PACKAGE_ROOT", root),
|
|
mock.patch.object(install_script, "PAYLOAD_FILES", ("VERSION", "LICENSE")),
|
|
mock.patch.object(install_script, "PAYLOAD_DIRECTORIES", ("docs",)),
|
|
self.assertRaisesRegex(ValueError, "special files"),
|
|
):
|
|
install_script.copy_payload(root / "destination")
|
|
|
|
def test_installer_refuses_to_replace_an_unowned_nonempty_install_root(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-existing-install-") as temporary:
|
|
parent = Path(temporary)
|
|
root = parent / "unowned" / "install"
|
|
root.mkdir(parents=True)
|
|
sentinel = root / "sentinel.txt"
|
|
sentinel.write_text("preserve\n", encoding="utf-8")
|
|
with self.assertRaisesRegex(RuntimeError, "ownership manifest"):
|
|
validate_existing_install(root)
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "preserve\n")
|
|
|
|
empty = parent / "empty" / "install"
|
|
empty.mkdir(parents=True)
|
|
validate_existing_install(empty)
|
|
|
|
owned = parent / "owned" / "install"
|
|
(owned / "config").mkdir(parents=True)
|
|
(owned / "config" / "install-manifest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(owned),
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
validate_existing_install(owned)
|
|
|
|
ambiguous = parent / "ambiguous" / "install"
|
|
(ambiguous / "config").mkdir(parents=True)
|
|
(ambiguous / "config" / "install-manifest.json").write_text(
|
|
"{"
|
|
f'"schema_version": {MMO_SCHEMA_VERSION},'
|
|
'"package": "codex-multimodel-orchestrator",'
|
|
f'"install_root": {json.dumps(str(ambiguous))},'
|
|
f'"install_root": {json.dumps(str(owned))}'
|
|
"}",
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(RuntimeError, "ownership manifest is invalid"):
|
|
validate_existing_install(ambiguous)
|
|
|
|
boolean_schema = parent / "boolean-schema" / "install"
|
|
(boolean_schema / "config").mkdir(parents=True)
|
|
(boolean_schema / "config" / "install-manifest.json").write_text(
|
|
json.dumps(
|
|
{
|
|
"schema_version": True,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(boolean_schema),
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(RuntimeError, "does not match"):
|
|
validate_existing_install(boolean_schema)
|
|
|
|
def test_uninstaller_refuses_unowned_install_tree(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-unowned-parent-") as temporary:
|
|
root = Path(temporary) / "unowned" / "install"
|
|
root.mkdir(parents=True)
|
|
sentinel = root / "sentinel.txt"
|
|
sentinel.write_text("preserve\n", encoding="utf-8")
|
|
with (
|
|
mock.patch.object(
|
|
sys,
|
|
"argv",
|
|
["uninstall.py", "--install-root", str(root)],
|
|
),
|
|
self.assertRaisesRegex(RuntimeError, "ownership manifest"),
|
|
):
|
|
uninstall_script.main()
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "preserve\n")
|
|
|
|
manifest_path = root / "config" / "install-manifest.json"
|
|
manifest_path.parent.mkdir()
|
|
manifest_path.write_text(
|
|
json.dumps({"schema_version": MMO_SCHEMA_VERSION, "install_root": str(root)}),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertIsNone(uninstall_script._trusted_manifest(manifest_path, root))
|
|
|
|
def test_uninstaller_does_not_use_a_stale_owner_marker_for_a_recreated_install(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-stale-owner-") as temporary:
|
|
root = Path(temporary)
|
|
install = root / "install"
|
|
config = root / "config"
|
|
state = root / "state"
|
|
bin_dir = root / "bin"
|
|
for path in (install, config, state, bin_dir):
|
|
path.mkdir()
|
|
sentinel = install / "unrelated.txt"
|
|
sentinel.write_text("preserve unrelated replacement\n", encoding="utf-8")
|
|
retained = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(install),
|
|
"config_root": str(config),
|
|
"state_root": str(state),
|
|
"bin_dir": str(bin_dir),
|
|
}
|
|
(config / uninstall_script.OWNER_MANIFEST).write_text(
|
|
json.dumps(retained), encoding="utf-8"
|
|
)
|
|
with (
|
|
mock.patch.object(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"uninstall.py",
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(config),
|
|
"--state-root",
|
|
str(state),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
],
|
|
),
|
|
self.assertRaisesRegex(RuntimeError, "ownership manifest"),
|
|
):
|
|
uninstall_script.main()
|
|
self.assertEqual(
|
|
sentinel.read_text(encoding="utf-8"),
|
|
"preserve unrelated replacement\n",
|
|
)
|
|
|
|
def test_uninstaller_refuses_explicit_roots_outside_owned_manifest(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-owned-parent-") as temporary:
|
|
parent = Path(temporary)
|
|
install = parent / "owned" / "install"
|
|
config = parent / "owned" / "config"
|
|
state = parent / "owned" / "state"
|
|
bin_dir = parent / "owned" / "bin"
|
|
outside = parent / "outside" / "config"
|
|
for path in (install / "config", config, state, bin_dir, outside):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
manifest = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(install),
|
|
"config_root": str(config),
|
|
"state_root": str(state),
|
|
"bin_dir": str(bin_dir),
|
|
}
|
|
(install / "config" / "install-manifest.json").write_text(
|
|
json.dumps(manifest), encoding="utf-8"
|
|
)
|
|
sentinel = outside / "sentinel.txt"
|
|
sentinel.write_text("preserve\n", encoding="utf-8")
|
|
with (
|
|
mock.patch.object(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"uninstall.py",
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(outside),
|
|
"--purge-config",
|
|
],
|
|
),
|
|
self.assertRaisesRegex(RuntimeError, "outside the ownership manifest"),
|
|
):
|
|
uninstall_script.main()
|
|
self.assertEqual(sentinel.read_text(encoding="utf-8"), "preserve\n")
|
|
self.assertTrue(install.exists())
|
|
|
|
def test_installer_toml_quoting_handles_del_and_invalid_unicode(self) -> None:
|
|
self.assertEqual(toml_quote("before\x7fafter"), '"before\\u007Fafter"')
|
|
with self.assertRaisesRegex(ValueError, "Unicode scalar"):
|
|
toml_quote("\ud800")
|
|
|
|
def test_xdg_defaults_ignore_empty_and_relative_environment_values(self) -> None:
|
|
fallback = Path("/fallback/base")
|
|
for resolver in (install_xdg_path, uninstall_xdg_path):
|
|
with self.subTest(resolver=resolver.__module__):
|
|
for value in ("", "relative/path", "~/not-expanded"):
|
|
with mock.patch.dict(os.environ, {"XDG_DATA_HOME": value}):
|
|
self.assertEqual(resolver("XDG_DATA_HOME", fallback), fallback)
|
|
with mock.patch.dict(os.environ, {"XDG_DATA_HOME": "/absolute/base"}):
|
|
self.assertEqual(resolver("XDG_DATA_HOME", fallback), Path("/absolute/base"))
|
|
|
|
env = {
|
|
"XDG_DATA_HOME": "relative-data",
|
|
"XDG_CONFIG_HOME": "",
|
|
"XDG_STATE_HOME": "~/relative-state",
|
|
"XDG_BIN_HOME": "relative-bin",
|
|
}
|
|
with (
|
|
mock.patch.dict(os.environ, env),
|
|
mock.patch.object(Path, "home", return_value=Path("/home/tester")),
|
|
):
|
|
self.assertEqual(default_bin_dir(), Path("/home/tester/.local/bin"))
|
|
self.assertEqual(
|
|
uninstall_defaults(),
|
|
(
|
|
Path("/home/tester/.local/share/codex-mmo"),
|
|
Path("/home/tester/.config/codex-mmo"),
|
|
Path("/home/tester/.local/state/codex-mmo"),
|
|
Path("/home/tester/.local/bin"),
|
|
),
|
|
)
|
|
|
|
def test_proc_stat_parser_handles_spaces_and_closing_parentheses(self) -> None:
|
|
suffix = [
|
|
"Z",
|
|
"1",
|
|
"42",
|
|
"42",
|
|
"0",
|
|
"-1",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"0",
|
|
"20",
|
|
"0",
|
|
"1",
|
|
"0",
|
|
"987654",
|
|
]
|
|
record = "123 (worker name ) with a close) " + " ".join(suffix)
|
|
with (
|
|
mock.patch("mmo_util.Path.read_text", return_value=record),
|
|
mock.patch("mmo_util.os.kill") as kill,
|
|
):
|
|
self.assertFalse(process_alive(123))
|
|
kill.assert_not_called()
|
|
self.assertEqual(process_start_token(123), "987654")
|
|
|
|
def test_install_and_uninstall_reject_broad_recursive_targets(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "broad protected path"):
|
|
validate_destination_paths((Path("/tmp"), Path("/a/b")))
|
|
with self.assertRaisesRegex(ValueError, "broad protected state root"):
|
|
validate_removal_target(Path("/tmp"), "state root")
|
|
|
|
def test_matched_evaluation_and_separate_cost_ledgers(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
(box.config / "catalog.d" / "pricing.toml").write_text(
|
|
f"""schema_version = {MMO_SCHEMA_VERSION}
|
|
|
|
[models.codex_chatgpt_builtin__gpt_5_6_sol]
|
|
input_cost_per_million = 2.0
|
|
cached_input_cost_per_million = 0.5
|
|
cache_write_input_cost_per_million = 3.0
|
|
output_cost_per_million = 6.0
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
suite = box.root / "matched-eval-suite"
|
|
suite.mkdir()
|
|
(suite / "suite.toml").write_text(
|
|
_current_suite_text(
|
|
suite_id="matched-eval",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertTrue(validate_suite(suite)["valid"])
|
|
first = run_evaluation(
|
|
profile="codex-harness-team", suite=suite, wall_timeout_override=60
|
|
)
|
|
self.assertEqual(first["status"], "completed", first)
|
|
summary = first["summary"]
|
|
self.assertEqual(summary["matched_task_count"], 1)
|
|
self.assertEqual(summary["executed_trial_records"], 8)
|
|
self.assertIsNone(summary["aggregate_score"])
|
|
self.assertIn("not interchangeable", summary["aggregate_score_reason"])
|
|
|
|
full = summary["variant_summaries"]["full-profile"]
|
|
self.assertEqual(full["success_rate"], 1.0)
|
|
self.assertEqual(full["root_usage"]["cache_write_input_tokens"], 3)
|
|
self.assertEqual(full["root_usage"]["reasoning_output_tokens"], 11)
|
|
self.assertEqual(full["worker_usage"], {})
|
|
self.assertEqual(full["explicit_root_mcp_wait_seconds"], 0.0)
|
|
self.assertEqual(full["worker_jobs"], 0)
|
|
ledgers = full["cost_ledgers"]
|
|
self.assertEqual(ledgers["actual_api_usd"], 0.0)
|
|
self.assertTrue(ledgers["actual_api_usd_complete"])
|
|
self.assertAlmostEqual(ledgers["api_equivalent_estimate_usd"], 0.0003325)
|
|
self.assertEqual(ledgers["subscription_units"]["chatgpt_codex"]["request_units"], 1)
|
|
self.assertEqual(ledgers["local_resource_seconds"], {})
|
|
self.assertTrue(summary["promotion"]["passed"], summary["promotion"])
|
|
primary = next(
|
|
check
|
|
for check in summary["promotion"]["checks"]
|
|
if check["name"] == "primary_metric_improvement"
|
|
)
|
|
self.assertEqual(primary["improvement"]["relative"], 0.0)
|
|
self.assertFalse(primary["improvement"]["relative_unbounded"])
|
|
|
|
second = run_evaluation(
|
|
profile="codex-harness-team", suite=suite, wall_timeout_override=60
|
|
)
|
|
compared = compare_runs([first["run_id"], second["run_id"]])
|
|
self.assertEqual(len(compared["runs"]), 2)
|
|
self.assertIsNone(compared["warning"])
|
|
self.assertIsNone(compared["aggregate_score"])
|
|
self.assertEqual(
|
|
compared["ranked_by_full_profile_success_then_time"][0][
|
|
"full_profile_success_rate"
|
|
],
|
|
1.0,
|
|
)
|
|
|
|
def test_improvement_from_zero_is_strict_json_without_losing_gate_semantics(self) -> None:
|
|
improvement = mmo_eval._directed_improvement(0.0, 1.0, "higher")
|
|
self.assertEqual(improvement["absolute"], 1.0)
|
|
self.assertIsNone(improvement["relative"])
|
|
self.assertTrue(improvement["relative_unbounded"])
|
|
self.assertTrue(mmo_eval._passes_relative_improvement(improvement, 10.0))
|
|
json.dumps(improvement, allow_nan=False)
|
|
|
|
def test_stale_evaluation_generation_and_unknown_fields_are_rejected(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
suite = box.root / "closed-eval-suite"
|
|
suite.mkdir()
|
|
path = suite / "suite.toml"
|
|
valid = _current_suite_text(
|
|
suite_id="closed-eval",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
)
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
f"schema_version = {MMO_SCHEMA_VERSION}",
|
|
f"schema_version = {MMO_SCHEMA_VERSION - 1}",
|
|
1,
|
|
)
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "unsupported suite schema"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
'description = "Matched-control fixture for runtime integration tests."',
|
|
'description = "Matched-control fixture for runtime integration tests."\n'
|
|
"unknown_root = true",
|
|
1,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "unknown_root"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."',
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."\n'
|
|
'validation_command = "true"',
|
|
1,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "validation_command"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(
|
|
valid.replace(f"schema_version = {MMO_SCHEMA_VERSION}", "schema_version = true", 1)
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "unsupported suite schema"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(valid.replace('id = "one"', "id = 1", 1))
|
|
with self.assertRaisesRegex(ValueError, "task 0 id"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."',
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."\n'
|
|
'route_faults = { opencode_go_openai_chat = "timeout" }',
|
|
1,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertTrue(validate_suite(suite)["valid"])
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."',
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."\n'
|
|
'route_faults = { opencode_go_openai_chat = "unavailable" }',
|
|
1,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "route_faults must map"):
|
|
validate_suite(suite)
|
|
|
|
path.write_text(
|
|
valid.replace(
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."',
|
|
'prompt = "Inspect one bounded concern and end with EVAL_V_OK."\n'
|
|
'disabled_routes = ["opencode_go_openai_chat"]',
|
|
1,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "disabled_routes"):
|
|
validate_suite(suite)
|
|
|
|
fixture = suite / "fixture"
|
|
fixture.mkdir()
|
|
absolute = _current_suite_text(
|
|
suite_id="closed-eval",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
fixture=str(fixture),
|
|
)
|
|
path.write_text(absolute, encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "relative path"):
|
|
validate_suite(suite)
|
|
|
|
(fixture / "outside-link").symlink_to(box.root)
|
|
linked = _current_suite_text(
|
|
suite_id="closed-eval",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
fixture="fixture",
|
|
)
|
|
path.write_text(linked, encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "may not contain symlinks"):
|
|
validate_suite(suite)
|
|
(fixture / "outside-link").unlink()
|
|
|
|
for old_value, replacement in (
|
|
('expected_patterns = ["EVAL_V_OK"]', 'expected_patterns = [""]'),
|
|
("forbidden_patterns = []", 'forbidden_patterns = [" "]'),
|
|
("validation_commands = []", 'validation_commands = [" "]'),
|
|
):
|
|
with self.subTest(blank_field=old_value.split(" =", 1)[0]):
|
|
path.write_text(valid.replace(old_value, replacement, 1), encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "non-empty"):
|
|
validate_suite(suite)
|
|
|
|
run_id = mmo_eval._run_id("p" * 64, "s" * 64)
|
|
self.assertGreater(len(run_id), 80)
|
|
run_directory = box.state / "evaluations" / run_id
|
|
run_directory.mkdir(parents=True)
|
|
expected_run = {"run_id": run_id, "status": "completed"}
|
|
(run_directory / "run.json").write_text(json.dumps(expected_run), encoding="utf-8")
|
|
self.assertEqual(mmo_eval.load_run(run_id), expected_run)
|
|
with self.assertRaisesRegex(ValueError, "invalid evaluation run ID"):
|
|
mmo_eval.load_run("..")
|
|
|
|
def test_evaluation_images_and_closed_orchestration_diagnostics(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
suite = box.root / "image-eval-suite"
|
|
fixture = suite / "fixture"
|
|
fixture.mkdir(parents=True)
|
|
(fixture / "reference.png").write_bytes(b"deterministic fixture image bytes")
|
|
(suite / "suite.toml").write_text(
|
|
_current_suite_text(
|
|
suite_id="image-eval",
|
|
profile="visual-engineering",
|
|
highest_worker="visual_analyst",
|
|
workers=("visual_analyst", "visual_verifier"),
|
|
fixture="fixture",
|
|
images=("reference.png",),
|
|
forbidden_agents=("visual_verifier",),
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
self.assertTrue(validate_suite(suite)["valid"])
|
|
synthetic_events = box.root / "synthetic-root-events.jsonl"
|
|
synthetic_events.write_text("", encoding="utf-8")
|
|
root_result = {
|
|
"session": {"session_id": "synthetic-image-session"},
|
|
"status": "completed",
|
|
"exit_code": 0,
|
|
"elapsed_seconds": 1.0,
|
|
"events_path": str(synthetic_events),
|
|
"result": "visual evidence inspected EVAL_V_OK",
|
|
}
|
|
with mock.patch.object(mmo_eval, "run_root_exec", return_value=root_result) as run:
|
|
result = run_evaluation(profile="visual-engineering", suite=suite)
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertEqual(run.call_count, 7)
|
|
self.assertEqual(run.call_args.kwargs["images"], ["reference.png"])
|
|
self.assertTrue(
|
|
all(task["orchestration_assertions"]["passed"] for task in result["tasks"])
|
|
)
|
|
|
|
unknown_agent = (
|
|
(suite / "suite.toml")
|
|
.read_text(encoding="utf-8")
|
|
.replace(
|
|
'forbidden_agents = ["visual_verifier"]',
|
|
'forbidden_agents = ["missing_role"]',
|
|
)
|
|
)
|
|
(suite / "suite.toml").write_text(unknown_agent, encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "agents absent from profile"):
|
|
run_evaluation(profile="visual-engineering", suite=suite, dry_run=True)
|
|
(suite / "suite.toml").write_text(
|
|
unknown_agent.replace(
|
|
'forbidden_agents = ["missing_role"]',
|
|
'forbidden_agents = ["visual_verifier"]',
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
bad = (
|
|
(suite / "suite.toml")
|
|
.read_text(encoding="utf-8")
|
|
.replace('images = ["reference.png"]', 'images = ["../reference.png"]')
|
|
)
|
|
(suite / "suite.toml").write_text(bad, encoding="utf-8")
|
|
with self.assertRaisesRegex(ValueError, "supported relative fixture paths"):
|
|
validate_suite(suite)
|
|
|
|
def test_root_failures_are_recorded_and_validation_timeout_kills_children(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
suite = box.root / "root-failure-suite"
|
|
suite.mkdir()
|
|
(suite / "suite.toml").write_text(
|
|
_current_suite_text(
|
|
suite_id="root-failure",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
with mock.patch.object(
|
|
mmo_eval, "run_root_exec", side_effect=RuntimeError("synthetic root failure")
|
|
):
|
|
result = run_evaluation(
|
|
profile="codex-harness-team",
|
|
suite=suite,
|
|
wall_timeout_override=60,
|
|
)
|
|
self.assertEqual(result["status"], "completed_with_failures")
|
|
self.assertEqual(result["summary"]["executed_trial_records"], 8)
|
|
self.assertTrue(
|
|
all(
|
|
task["root_error"] == "RuntimeError: synthetic root failure"
|
|
and not task["passed"]
|
|
for task in result["tasks"]
|
|
)
|
|
)
|
|
persisted = read_json(box.state / "evaluations" / result["run_id"] / "run.json")
|
|
self.assertEqual(persisted["status"], "completed_with_failures")
|
|
self.assertFalse(persisted["summary"]["promotion"]["passed"])
|
|
|
|
validation = _run_validation(
|
|
'sleep 60 & child=$!; echo "$child" > child.pid; wait "$child"',
|
|
box.workspace,
|
|
1,
|
|
)
|
|
self.assertEqual(validation["exit_code"], 124, validation)
|
|
child_pid = int((box.workspace / "child.pid").read_text(encoding="utf-8"))
|
|
self.assertFalse(process_alive(child_pid))
|
|
|
|
def test_detached_evaluation_root_is_stopped_before_holdout_validation(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
suite = box.root / "detached-eval-suite"
|
|
suite.mkdir()
|
|
(suite / "suite.toml").write_text(
|
|
_current_suite_text(
|
|
suite_id="detached-eval",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
events = box.root / "detached-events.jsonl"
|
|
events.write_text("", encoding="utf-8")
|
|
root_result = {
|
|
"session": {"session_id": "detached-eval-session"},
|
|
"status": "detached",
|
|
"root_status": "harness_wall_detached",
|
|
"exit_code": 124,
|
|
"elapsed_seconds": 1.0,
|
|
"events_path": str(events),
|
|
"result": "partial evidence",
|
|
}
|
|
order: list[str] = []
|
|
|
|
def stop(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
order.append("stop")
|
|
return {"session": {"status": "stopped"}}
|
|
|
|
def holdout(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
order.append("holdout")
|
|
return {}
|
|
|
|
with (
|
|
mock.patch.object(mmo_eval, "run_root_exec", return_value=root_result),
|
|
mock.patch.object(mmo_eval, "stop_session", side_effect=stop) as stopped,
|
|
mock.patch.object(mmo_eval, "_install_holdout", side_effect=holdout),
|
|
):
|
|
result = run_evaluation(profile="codex-harness-team", suite=suite)
|
|
self.assertEqual(result["status"], "completed_with_failures")
|
|
self.assertGreater(stopped.call_count, 0)
|
|
self.assertEqual(len(order) % 2, 0)
|
|
self.assertTrue(
|
|
all(
|
|
order[index : index + 2] == ["stop", "holdout"]
|
|
for index in range(0, len(order), 2)
|
|
),
|
|
order,
|
|
)
|
|
|
|
def test_evaluation_variants_reuse_their_validated_immutable_snapshots(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
suite = box.root / "snapshot-pin-suite"
|
|
suite.mkdir()
|
|
(suite / "suite.toml").write_text(
|
|
_current_suite_text(
|
|
suite_id="snapshot-pin",
|
|
profile="codex-harness-team",
|
|
highest_worker="invariant_designer",
|
|
workers=("repo_scout", "invariant_designer", "fresh_critic"),
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
root_result = {
|
|
"session": {"session_id": "synthetic-session"},
|
|
"status": "completed",
|
|
"exit_code": 0,
|
|
"elapsed_seconds": 0.1,
|
|
"result": "done EVAL_V_OK",
|
|
"events_path": str(box.root / "missing-events.jsonl"),
|
|
}
|
|
with mock.patch.object(mmo_eval, "run_root_exec", return_value=root_result) as execute:
|
|
manifest = run_evaluation(profile="codex-harness-team", suite=suite)
|
|
self.assertEqual(execute.call_count, 8)
|
|
expected_by_variant = {
|
|
variant["id"]: variant["snapshot_hash"] for variant in manifest["variants"]
|
|
}
|
|
self.assertEqual(
|
|
{call.kwargs["snapshot_hash"] for call in execute.call_args_list},
|
|
set(expected_by_variant.values()),
|
|
)
|
|
for call in execute.call_args_list:
|
|
self.assertIsNone(call.kwargs["profile"])
|
|
self.assertNotIn("bindings", call.kwargs)
|
|
for record in manifest["tasks"]:
|
|
self.assertEqual(record["snapshot_hash"], expected_by_variant[record["variant_id"]])
|
|
|
|
def test_evaluation_ablations_reuse_weighted_capacity_derivation(self) -> None:
|
|
with RuntimeSandbox() as box:
|
|
profile = box.root / "weighted-evaluation-profile"
|
|
shutil.copytree(ROOT / "profiles" / "incident-hypothesis-triage", profile)
|
|
profile_path = profile / "profile.toml"
|
|
data = read_toml(profile_path)
|
|
data["id"] = "weighted-evaluation-profile"
|
|
data["catalog"] = "catalog.toml"
|
|
data["agents"]["evidence_runner"]["resource_group"] = "weighted-evaluation"
|
|
data["agents"]["evidence_runner"]["resource_units"] = 1
|
|
data["agents"]["causal_challenger"]["resource_group"] = "weighted-evaluation"
|
|
data["agents"]["causal_challenger"]["resource_units"] = 2
|
|
data["agents"]["causal_challenger"]["max_active"] = 2
|
|
data["coordination"]["max_active_agents"] = 3
|
|
profile_path.write_text(toml_dumps(data), encoding="utf-8")
|
|
(profile / "catalog.toml").write_text(
|
|
toml_dumps(
|
|
{
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"resources": {
|
|
"weighted-evaluation": {
|
|
"lock_key": "test:weighted-evaluation",
|
|
"max_active": 3,
|
|
}
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
snapshot = mmo_eval._variant_snapshot(
|
|
profile,
|
|
None,
|
|
{
|
|
"id": "without-evidence-runner",
|
|
"comparison_class": "ablation",
|
|
"topology": "full_without_worker",
|
|
"worker": "evidence_runner",
|
|
},
|
|
)
|
|
coordination = snapshot["resolved"]["coordination"]
|
|
self.assertEqual(coordination["max_active_agents"], 2)
|
|
self.assertEqual(coordination["feasible_max_active_agents"], 2)
|
|
self.assertNotIn("max_total_spawns", coordination)
|
|
self.assertNotIn("feasible_max_total_spawns", coordination)
|
|
agents = snapshot["resolved"]["agents"]
|
|
self.assertNotIn("evidence_runner", agents["incident_lead"]["controls"])
|
|
self.assertEqual(agents["evidence_runner"]["controls"], {})
|
|
|
|
def test_atomic_install_reinstall_validate_and_uninstall_with_spaces(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo installer test ") as temporary:
|
|
root = Path(temporary)
|
|
install = root / "install root"
|
|
config = root / "config root"
|
|
state = root / "state root"
|
|
bin_dir = root / "bin dir"
|
|
validation_sentinel = state / ".install-validation" / "sentinel.txt"
|
|
validation_sentinel.parent.mkdir(parents=True)
|
|
validation_sentinel.write_text("preserve\n", encoding="utf-8")
|
|
stale_owner = json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(install),
|
|
}
|
|
)
|
|
for retained_root in (config, state):
|
|
retained_root.mkdir(parents=True, exist_ok=True)
|
|
(retained_root / install_script.RETAINED_OWNER_MANIFEST).write_text(
|
|
stale_owner,
|
|
encoding="utf-8",
|
|
)
|
|
command = [
|
|
"python3",
|
|
str(ROOT / "scripts" / "install.py"),
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(config),
|
|
"--state-root",
|
|
str(state),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
"--codex-bin",
|
|
str(FAKE_CODEX),
|
|
"--switchyard-bin",
|
|
str(FAKE_SWITCHYARD),
|
|
]
|
|
first = subprocess.run(command, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(first.returncode, 0, first.stdout + first.stderr)
|
|
self.assertEqual(validation_sentinel.read_text(encoding="utf-8"), "preserve\n")
|
|
self.assertTrue((bin_dir / "codex-mmoctl").is_file())
|
|
self.assertTrue((install / "config" / "install-manifest.json").is_file())
|
|
self.assertTrue((config / "tool-mcp.d").is_dir())
|
|
self.assertFalse((config / install_script.RETAINED_OWNER_MANIFEST).exists())
|
|
self.assertFalse((state / install_script.RETAINED_OWNER_MANIFEST).exists())
|
|
|
|
primary_help = subprocess.run(
|
|
[str(bin_dir / "codex-mmo"), "--help"],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(primary_help.returncode, 0, primary_help.stdout + primary_help.stderr)
|
|
self.assertIn("Launch a new interactive session", primary_help.stdout)
|
|
self.assertNotIn("\x1b[", primary_help.stdout)
|
|
|
|
control_without_command = subprocess.run(
|
|
[str(bin_dir / "codex-mmoctl")],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(control_without_command.returncode, 2)
|
|
self.assertEqual(control_without_command.stdout, "")
|
|
self.assertIn("a command is required", control_without_command.stderr)
|
|
|
|
(config / "tool-mcp.d" / "servers.toml").write_text(
|
|
f"""schema_version = {MMO_SCHEMA_VERSION}
|
|
|
|
[tool_mcp_servers.local_docs]
|
|
transport = "stdio"
|
|
command = "/bin/true"
|
|
enabled_tools = ["search"]
|
|
default_tools_approval_mode = "approve"
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
tool_mcp = subprocess.run(
|
|
[str(bin_dir / "codex-mmo"), "tool-mcp", "list"],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(tool_mcp.returncode, 0, tool_mcp.stdout + tool_mcp.stderr)
|
|
self.assertIn("local_docs", tool_mcp.stdout)
|
|
|
|
credentials = config / "credentials.env"
|
|
credentials.write_text("ZAI_CODING_API_KEY=preserve-me\n", encoding="utf-8")
|
|
operator_profile = config / "profiles.d" / "operator-profile"
|
|
shutil.copytree(ROOT / "profiles" / "codex-harness-team", operator_profile)
|
|
operator_manifest = operator_profile / "profile.toml"
|
|
operator_data = read_toml(operator_manifest)
|
|
operator_data["id"] = "operator-profile"
|
|
operator_data["operator_private_field"] = "must survive core reinstall"
|
|
operator_manifest.write_text(toml_dumps(operator_data), encoding="utf-8")
|
|
operator_bytes = operator_manifest.read_bytes()
|
|
second = subprocess.run(command, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(second.returncode, 0, second.stdout + second.stderr)
|
|
self.assertEqual(
|
|
credentials.read_text(encoding="utf-8"),
|
|
"ZAI_CODING_API_KEY=preserve-me\n",
|
|
)
|
|
self.assertTrue(operator_manifest.is_file())
|
|
self.assertEqual(operator_manifest.read_bytes(), operator_bytes)
|
|
self.assertTrue(any((state / "backups").iterdir()))
|
|
|
|
operator_data.pop("operator_private_field")
|
|
operator_manifest.write_text(toml_dumps(operator_data), encoding="utf-8")
|
|
|
|
env = {**os.environ, "MMO_CODEX_BIN": str(FAKE_CODEX)}
|
|
validate = subprocess.run(
|
|
[str(bin_dir / "codex-mmoctl"), "--json", "validate", "--all-profiles"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env,
|
|
timeout=120,
|
|
)
|
|
self.assertEqual(validate.returncode, 0, validate.stdout + validate.stderr)
|
|
report = json.loads(validate.stdout)
|
|
self.assertTrue(report["passed"])
|
|
self.assertTrue(report["catalog"]["inventory"]["passed"])
|
|
|
|
uninstall = subprocess.run(
|
|
[str(bin_dir / "codex-mmo-uninstall")],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
self.assertEqual(uninstall.returncode, 0, uninstall.stdout + uninstall.stderr)
|
|
self.assertFalse(install.exists())
|
|
self.assertTrue(config.exists())
|
|
self.assertTrue(state.exists())
|
|
self.assertEqual(
|
|
credentials.read_text(encoding="utf-8"),
|
|
"ZAI_CODING_API_KEY=preserve-me\n",
|
|
)
|
|
self.assertFalse((bin_dir / "codex-mmoctl").exists())
|
|
|
|
purge = subprocess.run(
|
|
[
|
|
"python3",
|
|
str(ROOT / "scripts" / "uninstall.py"),
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(config),
|
|
"--state-root",
|
|
str(state),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
"--purge-config",
|
|
"--purge-state",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=60,
|
|
)
|
|
self.assertEqual(purge.returncode, 0, purge.stdout + purge.stderr)
|
|
self.assertFalse(config.exists())
|
|
self.assertFalse(state.exists())
|
|
|
|
def test_installed_wrappers_quote_shell_metacharacters(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-wrapper-quote-") as temporary:
|
|
root = Path(temporary)
|
|
install = root / "$MMO_WRAPPER_SEGMENT install"
|
|
config = root / "config"
|
|
state = root / "state"
|
|
bin_dir = root / "bin"
|
|
result = subprocess.run(
|
|
[
|
|
"python3",
|
|
str(ROOT / "scripts" / "install.py"),
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(config),
|
|
"--state-root",
|
|
str(state),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
"--no-validate",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=120,
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
env = {**os.environ, "MMO_WRAPPER_SEGMENT": "EXPANDED_BY_SHELL"}
|
|
version = subprocess.run(
|
|
[str(bin_dir / "codex-mmoctl"), "version"],
|
|
env=env,
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertEqual(version.returncode, 0, version.stdout + version.stderr)
|
|
self.assertEqual(version.stdout.strip(), (ROOT / "VERSION").read_text().strip())
|
|
|
|
def test_installer_preserves_unowned_wrapper_targets(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-wrapper-owner-") as temporary:
|
|
root = Path(temporary)
|
|
bin_dir = root / "bin"
|
|
bin_dir.mkdir()
|
|
existing = bin_dir / "codex-mmo"
|
|
existing.write_text("#!/bin/sh\necho unrelated\n", encoding="utf-8")
|
|
result = subprocess.run(
|
|
[
|
|
"python3",
|
|
str(ROOT / "scripts" / "install.py"),
|
|
"--install-root",
|
|
str(root / "install"),
|
|
"--config-root",
|
|
str(root / "config"),
|
|
"--state-root",
|
|
str(root / "state"),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
"--no-validate",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("unowned executable", result.stderr + result.stdout)
|
|
self.assertEqual(
|
|
existing.read_text(encoding="utf-8"),
|
|
"#!/bin/sh\necho unrelated\n",
|
|
)
|
|
self.assertFalse((root / "install").exists())
|
|
|
|
def test_uninstaller_refuses_manifest_redirects_despite_explicit_roots(self) -> None:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-uninstall-redirect-") as temporary:
|
|
root = Path(temporary)
|
|
install = root / "install"
|
|
config = root / "config"
|
|
state = root / "state"
|
|
bin_dir = root / "bin"
|
|
victim_config = root / "victim-config"
|
|
victim_state = root / "victim-state"
|
|
for path in (
|
|
install / "config",
|
|
config,
|
|
state,
|
|
bin_dir,
|
|
victim_config,
|
|
victim_state,
|
|
):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
(victim_config / "keep.txt").write_text("keep\n", encoding="utf-8")
|
|
(victim_state / "keep.txt").write_text("keep\n", encoding="utf-8")
|
|
manifest = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"install_root": str(install),
|
|
"config_root": str(victim_config),
|
|
"state_root": str(victim_state),
|
|
"bin_dir": str(bin_dir),
|
|
}
|
|
(install / "config" / "install-manifest.json").write_text(
|
|
json.dumps(manifest), encoding="utf-8"
|
|
)
|
|
result = subprocess.run(
|
|
[
|
|
"python3",
|
|
str(ROOT / "scripts" / "uninstall.py"),
|
|
"--install-root",
|
|
str(install),
|
|
"--config-root",
|
|
str(config),
|
|
"--state-root",
|
|
str(state),
|
|
"--bin-dir",
|
|
str(bin_dir),
|
|
"--purge-config",
|
|
"--purge-state",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=30,
|
|
)
|
|
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
self.assertIn("outside the ownership manifest", result.stderr + result.stdout)
|
|
self.assertTrue(install.exists())
|
|
self.assertTrue(config.exists())
|
|
self.assertTrue(state.exists())
|
|
self.assertTrue((victim_config / "keep.txt").is_file())
|
|
self.assertTrue((victim_state / "keep.txt").is_file())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|