😏
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build reviewed inventory snapshots from captured upstream documents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_inventory_snapshot import ( # noqa: E402
|
||||
build_codex_installed_snapshot,
|
||||
build_opencode_go_snapshot,
|
||||
build_opencode_zen_snapshot,
|
||||
build_openrouter_snapshot,
|
||||
rehash_inventory_snapshot,
|
||||
)
|
||||
from mmo_util import atomic_write_json, strict_json_loads # noqa: E402
|
||||
|
||||
|
||||
def _sha256(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Normalize a captured provider response into the common inventory schema"
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="inventory", required=True)
|
||||
openrouter = subparsers.add_parser("openrouter")
|
||||
openrouter.add_argument("--input", type=Path, required=True)
|
||||
openrouter.add_argument("--zdr-input", type=Path, required=True)
|
||||
openrouter.add_argument("--output", type=Path, required=True)
|
||||
openrouter.add_argument("--as-of", required=True)
|
||||
openrouter.add_argument("--retrieved-at", required=True)
|
||||
openrouter.add_argument("--source-url", required=True)
|
||||
openrouter.add_argument("--zdr-source-url", required=True)
|
||||
openrouter.add_argument(
|
||||
"--endpoint-selection",
|
||||
action="append",
|
||||
required=True,
|
||||
metavar="MODEL_ID=ENDPOINT_TAG",
|
||||
)
|
||||
codex = subparsers.add_parser("codex-installed")
|
||||
codex.add_argument("--input", type=Path, required=True)
|
||||
codex.add_argument("--reviewed-input", type=Path, required=True)
|
||||
codex.add_argument("--output", type=Path, required=True)
|
||||
codex.add_argument("--as-of", required=True)
|
||||
codex.add_argument("--retrieved-at", required=True)
|
||||
codex.add_argument("--source-url", required=True)
|
||||
zen = subparsers.add_parser("opencode-zen")
|
||||
zen.add_argument("--listing-input", type=Path, required=True)
|
||||
zen.add_argument("--models-dev-input", type=Path, required=True)
|
||||
zen.add_argument("--docs-input", type=Path, required=True)
|
||||
zen.add_argument("--output", type=Path, required=True)
|
||||
zen.add_argument("--as-of", required=True)
|
||||
zen.add_argument("--retrieved-at", required=True)
|
||||
zen.add_argument("--listing-url", required=True)
|
||||
zen.add_argument("--models-dev-url", required=True)
|
||||
zen.add_argument("--docs-url", required=True)
|
||||
go = subparsers.add_parser("opencode-go")
|
||||
go.add_argument("--listing-input", type=Path, required=True)
|
||||
go.add_argument("--models-dev-input", type=Path, required=True)
|
||||
go.add_argument("--docs-input", type=Path, required=True)
|
||||
go.add_argument("--output", type=Path, required=True)
|
||||
go.add_argument("--as-of", required=True)
|
||||
go.add_argument("--retrieved-at", required=True)
|
||||
go.add_argument("--listing-url", required=True)
|
||||
go.add_argument("--models-dev-url", required=True)
|
||||
go.add_argument("--docs-url", required=True)
|
||||
rehash = subparsers.add_parser("rehash")
|
||||
rehash.add_argument("--input", type=Path, required=True)
|
||||
rehash.add_argument("--output", type=Path, required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
if args.inventory == "codex-installed":
|
||||
raw = args.input.read_bytes()
|
||||
reviewed_raw = args.reviewed_input.read_bytes()
|
||||
snapshot = build_codex_installed_snapshot(
|
||||
strict_json_loads(raw),
|
||||
strict_json_loads(reviewed_raw),
|
||||
as_of=args.as_of,
|
||||
retrieved_at=args.retrieved_at,
|
||||
response_sha256=_sha256(raw),
|
||||
source_url=args.source_url,
|
||||
)
|
||||
input_hashes = {"codex": _sha256(raw), "reviewed": _sha256(reviewed_raw)}
|
||||
elif args.inventory == "openrouter":
|
||||
raw = args.input.read_bytes()
|
||||
zdr_raw = args.zdr_input.read_bytes()
|
||||
document = strict_json_loads(raw)
|
||||
endpoint_selections: dict[str, str] = {}
|
||||
for value in args.endpoint_selection:
|
||||
model_id, separator, endpoint_tag = value.partition("=")
|
||||
if not separator or not model_id or not endpoint_tag:
|
||||
raise ValueError("--endpoint-selection must use MODEL_ID=ENDPOINT_TAG")
|
||||
if model_id in endpoint_selections:
|
||||
raise ValueError(f"duplicate OpenRouter endpoint selection: {model_id}")
|
||||
endpoint_selections[model_id] = endpoint_tag
|
||||
snapshot = build_openrouter_snapshot(
|
||||
document,
|
||||
strict_json_loads(zdr_raw),
|
||||
as_of=args.as_of,
|
||||
retrieved_at=args.retrieved_at,
|
||||
response_sha256=_sha256(raw),
|
||||
zdr_response_sha256=_sha256(zdr_raw),
|
||||
source_url=args.source_url,
|
||||
zdr_source_url=args.zdr_source_url,
|
||||
endpoint_selections=endpoint_selections,
|
||||
)
|
||||
input_hashes = {"openrouter": _sha256(raw), "zdr": _sha256(zdr_raw)}
|
||||
elif args.inventory in {"opencode-go", "opencode-zen"}:
|
||||
listing_raw = args.listing_input.read_bytes()
|
||||
models_dev_raw = args.models_dev_input.read_bytes()
|
||||
docs_raw = args.docs_input.read_bytes()
|
||||
builder = (
|
||||
build_opencode_go_snapshot
|
||||
if args.inventory == "opencode-go"
|
||||
else build_opencode_zen_snapshot
|
||||
)
|
||||
snapshot = builder(
|
||||
strict_json_loads(listing_raw),
|
||||
strict_json_loads(models_dev_raw),
|
||||
docs_raw.decode("utf-8"),
|
||||
as_of=args.as_of,
|
||||
retrieved_at=args.retrieved_at,
|
||||
listing_sha256=_sha256(listing_raw),
|
||||
models_dev_sha256=_sha256(models_dev_raw),
|
||||
docs_sha256=_sha256(docs_raw),
|
||||
listing_url=args.listing_url,
|
||||
models_dev_url=args.models_dev_url,
|
||||
docs_url=args.docs_url,
|
||||
)
|
||||
input_hashes = {
|
||||
"docs": _sha256(docs_raw),
|
||||
"models_dev": _sha256(models_dev_raw),
|
||||
args.inventory.replace("-", "_"): _sha256(listing_raw),
|
||||
}
|
||||
else:
|
||||
raw = args.input.read_bytes()
|
||||
document = strict_json_loads(raw)
|
||||
snapshot = rehash_inventory_snapshot(document)
|
||||
input_hashes = {"snapshot": _sha256(raw)}
|
||||
atomic_write_json(args.output, snapshot, 0o644)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"inventory": snapshot["inventory"],
|
||||
"models": len(snapshot["models"]),
|
||||
"models_sha256": snapshot["models_sha256"],
|
||||
"output": str(args.output),
|
||||
"input_sha256": input_hashes,
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+441
@@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic, atomically published Codex MMO release archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_util import filtered_environment # noqa: E402
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scripts.verify_release import (
|
||||
REQUIRED_DIRECTORIES,
|
||||
REQUIRED_FILES,
|
||||
verify_archive,
|
||||
)
|
||||
else:
|
||||
from verify_release import ( # noqa: E402
|
||||
REQUIRED_DIRECTORIES,
|
||||
REQUIRED_FILES,
|
||||
verify_archive,
|
||||
)
|
||||
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION # noqa: E402
|
||||
|
||||
VERSION = PACKAGE_VERSION
|
||||
PACKAGE_NAME = f"codex-multimodel-orchestrator-{VERSION}"
|
||||
DEFAULT_EPOCH = 1786665600 # 2026-08-14T00:00:00Z
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
"dist",
|
||||
"build",
|
||||
}
|
||||
EXCLUDED_SUFFIXES = {".pyc", ".pyo"}
|
||||
MINIMUM_SOURCE_FILES = 100
|
||||
|
||||
|
||||
def source_paths() -> list[Path]:
|
||||
result: list[Path] = []
|
||||
for path in ROOT.rglob("*"):
|
||||
relative = path.relative_to(ROOT)
|
||||
if any(part in EXCLUDED_PARTS for part in relative.parts):
|
||||
continue
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode):
|
||||
raise RuntimeError(f"refusing to package symbolic link: {relative}")
|
||||
if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
|
||||
raise RuntimeError(f"refusing to package special file: {relative}")
|
||||
if path.suffix in EXCLUDED_SUFFIXES:
|
||||
continue
|
||||
if stat.S_ISREG(mode) and relative.as_posix() != "PACKAGE-MANIFEST.json":
|
||||
result.append(path)
|
||||
return sorted(result, key=lambda item: item.relative_to(ROOT).as_posix())
|
||||
|
||||
|
||||
def normalized_mode(source: Path) -> int:
|
||||
return 0o755 if source.stat().st_mode & stat.S_IXUSR else 0o644
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def assert_complete_source(paths: list[Path]) -> None:
|
||||
relative_files = {path.relative_to(ROOT).as_posix() for path in paths}
|
||||
missing_files = sorted(REQUIRED_FILES - relative_files)
|
||||
missing_directories = sorted(
|
||||
name for name in REQUIRED_DIRECTORIES if not (ROOT / name).is_dir()
|
||||
)
|
||||
if len(paths) < MINIMUM_SOURCE_FILES or missing_files or missing_directories:
|
||||
raise RuntimeError(
|
||||
"refusing to package an incomplete source tree: "
|
||||
f"file_count={len(paths)}, minimum={MINIMUM_SOURCE_FILES}, "
|
||||
f"missing_files={missing_files}, missing_directories={missing_directories}"
|
||||
)
|
||||
|
||||
|
||||
def stage_tree(destination: Path, epoch: int) -> Path:
|
||||
sources = source_paths()
|
||||
assert_complete_source(sources)
|
||||
top = destination / PACKAGE_NAME
|
||||
top.mkdir(parents=True)
|
||||
files: list[dict[str, Any]] = []
|
||||
for source in sources:
|
||||
relative = source.relative_to(ROOT)
|
||||
target = top / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, target)
|
||||
mode = normalized_mode(source)
|
||||
os.chmod(target, mode)
|
||||
os.utime(target, (epoch, epoch))
|
||||
files.append(
|
||||
{
|
||||
"path": relative.as_posix(),
|
||||
"mode": f"{mode:04o}",
|
||||
"size": target.stat().st_size,
|
||||
"sha256": sha256(target),
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": "codex-multimodel-orchestrator",
|
||||
"version": VERSION,
|
||||
"source_date_epoch": epoch,
|
||||
"manifest_excludes": ["PACKAGE-MANIFEST.json"],
|
||||
"file_count": len(files),
|
||||
"total_bytes": sum(int(item["size"]) for item in files),
|
||||
"files": files,
|
||||
}
|
||||
manifest_path = top / "PACKAGE-MANIFEST.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(manifest_path, 0o644)
|
||||
os.utime(manifest_path, (epoch, epoch))
|
||||
for directory in sorted((path for path in top.rglob("*") if path.is_dir()), reverse=True):
|
||||
os.chmod(directory, 0o755)
|
||||
os.utime(directory, (epoch, epoch))
|
||||
os.chmod(top, 0o755)
|
||||
os.utime(top, (epoch, epoch))
|
||||
return top
|
||||
|
||||
|
||||
def tree_entries(top: Path) -> list[Path]:
|
||||
return [top, *sorted(top.rglob("*"), key=lambda item: item.relative_to(top.parent).as_posix())]
|
||||
|
||||
|
||||
def build_tar_gz(top: Path, output: Path, epoch: int) -> None:
|
||||
with output.open("wb") as raw:
|
||||
with gzip.GzipFile(
|
||||
filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=epoch
|
||||
) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as archive:
|
||||
for path in tree_entries(top):
|
||||
arcname = path.relative_to(top.parent).as_posix()
|
||||
info = archive.gettarinfo(str(path), arcname=arcname)
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = "root"
|
||||
info.gname = "root"
|
||||
info.mtime = epoch
|
||||
info.mode = 0o755 if path.is_dir() else normalized_mode(path)
|
||||
if path.is_file():
|
||||
with path.open("rb") as handle:
|
||||
archive.addfile(info, handle)
|
||||
else:
|
||||
archive.addfile(info)
|
||||
|
||||
|
||||
def build_zip(top: Path, output: Path, epoch: int) -> None:
|
||||
timestamp = time.gmtime(max(epoch, 315532800))[:6]
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||||
for path in tree_entries(top):
|
||||
arcname = path.relative_to(top.parent).as_posix() + ("/" if path.is_dir() else "")
|
||||
info = zipfile.ZipInfo(arcname, timestamp)
|
||||
info.create_system = 3
|
||||
mode = 0o755 if path.is_dir() else normalized_mode(path)
|
||||
info.external_attr = (stat.S_IFDIR if path.is_dir() else stat.S_IFREG) << 16
|
||||
info.external_attr |= mode << 16
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.flag_bits |= 0x800
|
||||
archive.writestr(info, b"" if path.is_dir() else path.read_bytes())
|
||||
|
||||
|
||||
def build_once(destination: Path, epoch: int) -> tuple[Path, Path]:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
stage_root = destination / "stage"
|
||||
stage_root.mkdir()
|
||||
top = stage_tree(stage_root, epoch)
|
||||
tar_path = destination / f"{PACKAGE_NAME}-linux.tar.gz"
|
||||
zip_path = destination / f"{PACKAGE_NAME}-linux.zip"
|
||||
build_tar_gz(top, tar_path, epoch)
|
||||
build_zip(top, zip_path, epoch)
|
||||
return tar_path, zip_path
|
||||
|
||||
|
||||
def validate_source(report_path: Path | None = None) -> None:
|
||||
command = [sys.executable, str(ROOT / "scripts" / "validate_package.py")]
|
||||
if report_path:
|
||||
command.extend(("--report", str(report_path)))
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env=filtered_environment(extra={"PYTHONDONTWRITEBYTECODE": "1"}),
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError("package validation failed")
|
||||
|
||||
|
||||
def write_checksums(paths: Iterable[Path], output: Path) -> None:
|
||||
lines = [f"{sha256(path)} {path.name}" for path in sorted(paths, key=lambda item: item.name)]
|
||||
output.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _atomic_copy(source: Path, destination: Path) -> None:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, raw_temporary = tempfile.mkstemp(prefix=f".{destination.name}.tmp-", dir=destination.parent)
|
||||
temporary = Path(raw_temporary)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as output_handle, source.open("rb") as input_handle:
|
||||
shutil.copyfileobj(input_handle, output_handle, length=1024 * 1024)
|
||||
output_handle.flush()
|
||||
os.fsync(output_handle.fileno())
|
||||
os.chmod(temporary, 0o644)
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _publish_artifacts(
|
||||
sources: Iterable[tuple[Path, Path]],
|
||||
*,
|
||||
verify: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
"""Publish a release set and restore every prior artifact on failure."""
|
||||
|
||||
pairs = list(sources)
|
||||
destinations = [destination for _source, destination in pairs]
|
||||
if len(set(destinations)) != len(destinations):
|
||||
raise RuntimeError("release publication contains duplicate destinations")
|
||||
for source, destination in pairs:
|
||||
source_mode = source.lstat().st_mode
|
||||
if not stat.S_ISREG(source_mode):
|
||||
raise RuntimeError(f"release publication source is not a regular file: {source}")
|
||||
try:
|
||||
destination_mode = destination.lstat().st_mode
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not stat.S_ISREG(destination_mode):
|
||||
raise RuntimeError(f"refusing to replace a non-regular release artifact: {destination}")
|
||||
prepared: dict[Path, Path] = {}
|
||||
backups: dict[Path, Path] = {}
|
||||
published: list[Path] = []
|
||||
retained_backups: set[Path] = set()
|
||||
nonce = f"{os.getpid()}-{time.time_ns()}"
|
||||
try:
|
||||
for source, destination in pairs:
|
||||
temporary = destination.with_name(f".{destination.name}.publish-{nonce}")
|
||||
_atomic_copy(source, temporary)
|
||||
prepared[destination] = temporary
|
||||
if destination.is_file():
|
||||
backup = destination.with_name(f".{destination.name}.backup-{nonce}")
|
||||
_atomic_copy(destination, backup)
|
||||
backups[destination] = backup
|
||||
for _source, destination in pairs:
|
||||
os.replace(prepared[destination], destination)
|
||||
published.append(destination)
|
||||
if verify is not None:
|
||||
verify()
|
||||
except BaseException as publish_error:
|
||||
restoration_errors: list[str] = []
|
||||
for destination in reversed(published):
|
||||
saved = backups.get(destination)
|
||||
try:
|
||||
if saved and saved.exists():
|
||||
os.replace(saved, destination)
|
||||
else:
|
||||
destination.unlink(missing_ok=True)
|
||||
except BaseException as restore_error:
|
||||
if saved and saved.exists():
|
||||
retained_backups.add(saved)
|
||||
restoration_errors.append(
|
||||
f"{destination}: {type(restore_error).__name__}: {restore_error}"
|
||||
)
|
||||
if restoration_errors:
|
||||
retained = ", ".join(str(path) for path in sorted(retained_backups)) or "none"
|
||||
raise RuntimeError(
|
||||
"release publication failed and rollback was incomplete; "
|
||||
f"retained backups: {retained}; errors: {'; '.join(restoration_errors)}"
|
||||
) from publish_error
|
||||
raise
|
||||
finally:
|
||||
for path in prepared.values():
|
||||
path.unlink(missing_ok=True)
|
||||
for path in backups.values():
|
||||
if path not in retained_backups:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _verify_pair(tar_path: Path, zip_path: Path) -> list[dict[str, Any]]:
|
||||
results = [
|
||||
verify_archive(tar_path, source_tree=ROOT),
|
||||
verify_archive(zip_path, source_tree=ROOT),
|
||||
]
|
||||
counts = {
|
||||
(
|
||||
item["manifest_files"],
|
||||
item["total_files"],
|
||||
item["manifest_sha256"],
|
||||
item["tree_sha256"],
|
||||
)
|
||||
for item in results
|
||||
}
|
||||
if len(counts) != 1:
|
||||
raise RuntimeError("tar.gz and ZIP extracted trees are not identical")
|
||||
return results
|
||||
|
||||
|
||||
def assert_safe_output_directory(output: Path) -> None:
|
||||
if output == ROOT or ROOT in output.parents:
|
||||
raise RuntimeError(f"refusing to write release output inside the source tree: {output}")
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
value = argparse.ArgumentParser(description="Build reproducible Codex MMO release archives")
|
||||
value.add_argument("--output-dir", type=Path, default=ROOT.parent)
|
||||
value.add_argument(
|
||||
"--source-date-epoch",
|
||||
type=int,
|
||||
default=int(os.environ.get("SOURCE_DATE_EPOCH", DEFAULT_EPOCH)),
|
||||
)
|
||||
value.add_argument("--skip-validation", action="store_true")
|
||||
value.add_argument("--no-reproducibility-check", action="store_true")
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
if not 0 <= args.source_date_epoch <= 0xFFFFFFFF:
|
||||
raise ValueError("--source-date-epoch must be between 0 and 4294967295")
|
||||
output = args.output_dir.expanduser().resolve()
|
||||
assert_safe_output_directory(output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
validation_report = output / f"{PACKAGE_NAME}-VALIDATION.json"
|
||||
checksums = output / f"{PACKAGE_NAME}-SHA256SUMS.txt"
|
||||
integrity_report = output / f"{PACKAGE_NAME}-INTEGRITY.json"
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-release-primary-") as primary_raw:
|
||||
primary = Path(primary_raw)
|
||||
validation_stage: Path | None = None
|
||||
if not args.skip_validation:
|
||||
validation_stage = primary / validation_report.name
|
||||
validate_source(validation_stage)
|
||||
tar_path, zip_path = build_once(primary, args.source_date_epoch)
|
||||
verification = _verify_pair(tar_path, zip_path)
|
||||
|
||||
reproducible = True
|
||||
if not args.no_reproducibility_check:
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-release-repro-") as second_raw:
|
||||
second_tar, second_zip = build_once(Path(second_raw), args.source_date_epoch)
|
||||
_verify_pair(second_tar, second_zip)
|
||||
reproducible = sha256(tar_path) == sha256(second_tar) and sha256(
|
||||
zip_path
|
||||
) == sha256(second_zip)
|
||||
if not reproducible:
|
||||
raise RuntimeError("release build is not reproducible")
|
||||
|
||||
final_tar = output / tar_path.name
|
||||
final_zip = output / zip_path.name
|
||||
intended_verification = []
|
||||
for item, destination in zip(verification, (final_tar, final_zip), strict=True):
|
||||
intended_verification.append({**item, "archive": str(destination)})
|
||||
|
||||
checksum_stage = primary / checksums.name
|
||||
write_checksums((tar_path, zip_path), checksum_stage)
|
||||
integrity_stage = primary / integrity_report.name
|
||||
integrity = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": PACKAGE_NAME,
|
||||
"source_date_epoch": args.source_date_epoch,
|
||||
"source_files": len(source_paths()),
|
||||
"reproducible": reproducible,
|
||||
"archives": intended_verification,
|
||||
"checksums": checksums.name,
|
||||
}
|
||||
integrity_stage.write_text(
|
||||
json.dumps(integrity, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
artifacts = [
|
||||
(tar_path, final_tar),
|
||||
(zip_path, final_zip),
|
||||
(checksum_stage, checksums),
|
||||
(integrity_stage, integrity_report),
|
||||
]
|
||||
if validation_stage is not None:
|
||||
artifacts.append((validation_stage, validation_report))
|
||||
|
||||
def verify_published_set() -> None:
|
||||
for staged, published in artifacts:
|
||||
if sha256(staged) != sha256(published):
|
||||
raise RuntimeError(
|
||||
f"published bytes differ from verified artifact: {published.name}"
|
||||
)
|
||||
published_verification = _verify_pair(final_tar, final_zip)
|
||||
if intended_verification != published_verification:
|
||||
raise RuntimeError("published archives differ from verified build artifacts")
|
||||
|
||||
# Publish the archives and all generated release metadata as one
|
||||
# rollback-safe set. Final-name verification runs while backups remain
|
||||
# available, so a verification failure also restores the prior set.
|
||||
_publish_artifacts(artifacts, verify=verify_published_set)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"package": PACKAGE_NAME,
|
||||
"source_date_epoch": args.source_date_epoch,
|
||||
"source_files": len(source_paths()),
|
||||
"reproducible": reproducible,
|
||||
"tar_gz": {"path": str(final_tar), "sha256": sha256(final_tar)},
|
||||
"zip": {"path": str(final_zip), "sha256": sha256(final_zip)},
|
||||
"checksums": str(checksums),
|
||||
"integrity_report": str(integrity_report),
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+613
@@ -0,0 +1,613 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the bundled route/model catalog and upstream inventory.
|
||||
|
||||
The inventories are intentionally explicit and reviewable. `codex-mmo catalog
|
||||
verify --remote` compares dynamic provider listings against this baseline; it
|
||||
does not silently add models whose protocol and capabilities are unknown.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
from mmo_inventory_snapshot import load_inventory_snapshots # noqa: E402
|
||||
from mmo_profiles import validate_catalog_data # noqa: E402
|
||||
from mmo_util import atomic_write_json, atomic_write_text, toml_dumps # noqa: E402
|
||||
from mmo_version import MMO_SCHEMA_VERSION # noqa: E402
|
||||
|
||||
INVENTORY_SNAPSHOT_ROOT = ROOT / "config" / "inventory-snapshots"
|
||||
|
||||
|
||||
def route(**values: Any) -> dict[str, Any]:
|
||||
return values
|
||||
|
||||
|
||||
def model(
|
||||
route_key: str,
|
||||
upstream_id: str,
|
||||
maker: str,
|
||||
display_name: str,
|
||||
*,
|
||||
description: str,
|
||||
context: int,
|
||||
output: int | None = None,
|
||||
reasoning: list[str] | None = None,
|
||||
default_reasoning: str | None = None,
|
||||
modalities: list[str] | None = None,
|
||||
output_modalities: list[str] | None = None,
|
||||
tool_calling: bool = True,
|
||||
parallel_tool_calls: bool = True,
|
||||
summaries: bool = False,
|
||||
structured: bool = True,
|
||||
resource: str | None = None,
|
||||
kind: str = "chat",
|
||||
agent_compatible: bool = True,
|
||||
availability: str = "current",
|
||||
confidence: str = "documented",
|
||||
source: str,
|
||||
inventory: str | None = None,
|
||||
input_cost: float | None = None,
|
||||
output_cost: float | None = None,
|
||||
cached_input_cost: float | None = None,
|
||||
cache_write_input_cost: float | None = None,
|
||||
unit_cost: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
values: dict[str, Any] = {
|
||||
"route": route_key,
|
||||
"upstream_id": upstream_id,
|
||||
"maker": maker,
|
||||
"display_name": display_name,
|
||||
"description": description,
|
||||
"kind": kind,
|
||||
"agent_compatible": agent_compatible,
|
||||
"context_window": context,
|
||||
"reasoning_levels": reasoning or (["none"] if not agent_compatible else ["high"]),
|
||||
"default_reasoning": default_reasoning
|
||||
or ("none" if not agent_compatible else (reasoning or ["high"])[-1]),
|
||||
"modalities": modalities or ["text"],
|
||||
"output_modalities": output_modalities or ["text"],
|
||||
"tool_calling": tool_calling,
|
||||
"parallel_tool_calls": parallel_tool_calls,
|
||||
"supports_reasoning_summaries": summaries,
|
||||
"structured_output": structured,
|
||||
"availability": availability,
|
||||
"capability_confidence": confidence,
|
||||
"source": source,
|
||||
}
|
||||
if output is not None:
|
||||
values["max_output_tokens"] = output
|
||||
if resource:
|
||||
values["resource_group"] = resource
|
||||
if inventory:
|
||||
values["inventory"] = inventory
|
||||
if input_cost is not None:
|
||||
values["input_cost_per_million"] = input_cost
|
||||
if output_cost is not None:
|
||||
values["output_cost_per_million"] = output_cost
|
||||
if cached_input_cost is not None:
|
||||
values["cached_input_cost_per_million"] = cached_input_cost
|
||||
if cache_write_input_cost is not None:
|
||||
values["cache_write_input_cost_per_million"] = cache_write_input_cost
|
||||
if unit_cost is not None:
|
||||
values["unit_cost_usd"] = unit_cost
|
||||
return values
|
||||
|
||||
|
||||
routes: dict[str, dict[str, Any]] = {
|
||||
"zai_coding_responses": route(
|
||||
driver="switchyard",
|
||||
name="Z.AI Coding Plan Responses",
|
||||
api_operator="zai",
|
||||
access_product="zai_coding_plan",
|
||||
wire_protocol="openai_responses",
|
||||
billing_mode="subscription",
|
||||
base_url="https://api.z.ai/api/v1",
|
||||
credential_envs=["ZAI_CODING_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="zai_coding_plan",
|
||||
inventory="zai-coding-plan",
|
||||
),
|
||||
"zai_coding_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="Z.AI Coding Plan OpenAI-compatible",
|
||||
api_operator="zai",
|
||||
access_product="zai_coding_plan",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="subscription",
|
||||
base_url="https://api.z.ai/api/coding/paas/v4",
|
||||
credential_envs=["ZAI_CODING_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="zai_coding_plan",
|
||||
inventory="zai-coding-plan",
|
||||
),
|
||||
"zai_coding_anthropic_messages": route(
|
||||
driver="catalog_only",
|
||||
name="Z.AI Coding Plan Anthropic-compatible (catalog only)",
|
||||
api_operator="zai",
|
||||
access_product="zai_coding_plan",
|
||||
wire_protocol="anthropic_messages",
|
||||
billing_mode="subscription",
|
||||
base_url="https://api.z.ai/api/anthropic",
|
||||
credential_envs=["ZAI_CODING_API_KEY"],
|
||||
auth="bearer",
|
||||
transport_modalities=["text"],
|
||||
transport_output_modalities=["text"],
|
||||
tool_calling=False,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="zai_coding_plan",
|
||||
inventory="zai-coding-plan",
|
||||
),
|
||||
"zai_general_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="Z.AI General API",
|
||||
api_operator="zai",
|
||||
access_product="zai_general_api",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="api",
|
||||
base_url="https://api.z.ai/api/paas/v4",
|
||||
credential_envs=["ZAI_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text", "image", "video", "file"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="zai_api",
|
||||
inventory="zai-api",
|
||||
),
|
||||
"zai_general_catalog": route(
|
||||
driver="catalog_only",
|
||||
name="Z.AI media and specialist API catalog",
|
||||
api_operator="zai",
|
||||
access_product="zai_general_api",
|
||||
wire_protocol="catalog_only",
|
||||
billing_mode="catalog_only",
|
||||
transport_modalities=["text", "image", "video", "audio", "file"],
|
||||
tool_calling=False,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="zai_api",
|
||||
inventory="zai-api",
|
||||
),
|
||||
"opencode_zen_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Zen Chat Completions",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_zen",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="api",
|
||||
base_url="https://opencode.ai/zen/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="opencode_zen",
|
||||
inventory="opencode-zen",
|
||||
),
|
||||
"opencode_zen_responses": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Zen Responses",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_zen",
|
||||
wire_protocol="openai_responses",
|
||||
billing_mode="api",
|
||||
base_url="https://opencode.ai/zen/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="opencode_zen",
|
||||
inventory="opencode-zen",
|
||||
),
|
||||
"opencode_zen_anthropic_messages": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Zen Anthropic Messages",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_zen",
|
||||
wire_protocol="anthropic_messages",
|
||||
billing_mode="api",
|
||||
base_url="https://opencode.ai/zen/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="opencode_zen",
|
||||
inventory="opencode-zen",
|
||||
),
|
||||
"opencode_zen_google_catalog": route(
|
||||
driver="catalog_only",
|
||||
name="OpenCode Zen Google-native models (catalog only)",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_zen",
|
||||
wire_protocol="catalog_only",
|
||||
billing_mode="catalog_only",
|
||||
transport_modalities=["text", "image", "video", "audio", "file"],
|
||||
transport_output_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="opencode_zen",
|
||||
inventory="opencode-zen",
|
||||
),
|
||||
# OpenCode Go uses different endpoints for different model families.
|
||||
"opencode_go_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Go Chat Completions",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_go",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="subscription",
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="opencode_go",
|
||||
inventory="opencode-go",
|
||||
),
|
||||
"opencode_go_responses": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Go Responses",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_go",
|
||||
wire_protocol="openai_responses",
|
||||
billing_mode="subscription",
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="opencode_go",
|
||||
inventory="opencode-go",
|
||||
),
|
||||
"opencode_go_anthropic_messages": route(
|
||||
driver="switchyard",
|
||||
name="OpenCode Go Anthropic Messages",
|
||||
api_operator="opencode",
|
||||
access_product="opencode_go",
|
||||
wire_protocol="anthropic_messages",
|
||||
billing_mode="subscription",
|
||||
base_url="https://opencode.ai/zen/go/v1",
|
||||
credential_envs=["OPENCODE_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="opencode_go",
|
||||
inventory="opencode-go",
|
||||
),
|
||||
"openrouter_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="OpenRouter Chat Completions",
|
||||
api_operator="openrouter",
|
||||
access_product="openrouter_api",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="api",
|
||||
base_url="https://openrouter.ai/api/v1",
|
||||
credential_envs=["OPENROUTER_API_KEY"],
|
||||
extra_headers={"X-OpenRouter-Metadata": "enabled"},
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="openrouter",
|
||||
inventory="openrouter",
|
||||
),
|
||||
"llama_cpp_local_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="Local llama.cpp",
|
||||
api_operator="local",
|
||||
access_product="llama_cpp",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="local",
|
||||
base_url="http://127.0.0.1:8001/v1",
|
||||
max_retries=0,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="local_gpu_0",
|
||||
),
|
||||
"codex_chatgpt_builtin": route(
|
||||
driver="codex_builtin",
|
||||
name="Built-in Codex with ChatGPT authentication",
|
||||
api_operator="openai",
|
||||
access_product="chatgpt_codex",
|
||||
wire_protocol="codex_builtin",
|
||||
billing_mode="chatgpt_subscription",
|
||||
provider_id="openai",
|
||||
auth="chatgpt",
|
||||
transport_modalities=["text", "image"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="chatgpt_subscription",
|
||||
inventory="openai-codex",
|
||||
),
|
||||
"openai_api_responses": route(
|
||||
driver="switchyard",
|
||||
name="OpenAI API through Switchyard",
|
||||
api_operator="openai",
|
||||
access_product="openai_api",
|
||||
wire_protocol="openai_responses",
|
||||
billing_mode="api",
|
||||
base_url="https://api.openai.com/v1",
|
||||
credential_envs=["OPENAI_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text", "image"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="openai_api",
|
||||
),
|
||||
"anthropic_api_messages": route(
|
||||
driver="switchyard",
|
||||
name="Anthropic API through Switchyard",
|
||||
api_operator="anthropic",
|
||||
access_product="anthropic_api",
|
||||
wire_protocol="anthropic_messages",
|
||||
billing_mode="api",
|
||||
base_url="https://api.anthropic.com/v1",
|
||||
credential_envs=["ANTHROPIC_API_KEY"],
|
||||
max_retries=1,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=True,
|
||||
resource_group="anthropic_api",
|
||||
),
|
||||
"ollama_local_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="Local Ollama through Switchyard",
|
||||
api_operator="local",
|
||||
access_product="ollama",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="local",
|
||||
base_url="http://127.0.0.1:11434/v1",
|
||||
max_retries=0,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="local_gpu_0",
|
||||
),
|
||||
"lmstudio_local_openai_chat": route(
|
||||
driver="switchyard",
|
||||
name="Local LM Studio through Switchyard",
|
||||
api_operator="local",
|
||||
access_product="lmstudio",
|
||||
wire_protocol="openai_chat",
|
||||
billing_mode="local",
|
||||
base_url="http://127.0.0.1:1234/v1",
|
||||
max_retries=0,
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="local_gpu_0",
|
||||
),
|
||||
"ollama_codex_oss": route(
|
||||
driver="codex_oss",
|
||||
name="Codex native Ollama OSS mode",
|
||||
api_operator="local",
|
||||
access_product="ollama",
|
||||
wire_protocol="codex_oss",
|
||||
billing_mode="local",
|
||||
provider_id="ollama",
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="local_gpu_0",
|
||||
),
|
||||
"lmstudio_codex_oss": route(
|
||||
driver="codex_oss",
|
||||
name="Codex native LM Studio OSS mode",
|
||||
api_operator="local",
|
||||
access_product="lmstudio",
|
||||
wire_protocol="codex_oss",
|
||||
billing_mode="local",
|
||||
provider_id="lmstudio",
|
||||
transport_modalities=["text"],
|
||||
tool_calling=True,
|
||||
parallel_tool_calls=False,
|
||||
resource_group="local_gpu_0",
|
||||
),
|
||||
}
|
||||
|
||||
resources: dict[str, dict[str, Any]] = {
|
||||
"opencode_zen": {
|
||||
"description": "OpenCode Zen request capacity",
|
||||
"lock_key": "provider:opencode-zen",
|
||||
"max_active": 8,
|
||||
},
|
||||
"zai_coding_plan": {
|
||||
"description": "Z.AI Coding Plan request capacity",
|
||||
"lock_key": "provider:zai-coding-plan",
|
||||
"max_active": 4,
|
||||
},
|
||||
"zai_api": {
|
||||
"description": "Z.AI general API request capacity",
|
||||
"lock_key": "provider:zai-api",
|
||||
"max_active": 8,
|
||||
},
|
||||
"opencode_go": {
|
||||
"description": "OpenCode Go request capacity",
|
||||
"lock_key": "provider:opencode-go",
|
||||
"max_active": 6,
|
||||
},
|
||||
"openrouter": {
|
||||
"description": "OpenRouter request capacity",
|
||||
"lock_key": "provider:openrouter",
|
||||
"max_active": 8,
|
||||
},
|
||||
"local_gpu_0": {
|
||||
"description": "One local GPU/model-server slot",
|
||||
"lock_key": "local-gpu:0",
|
||||
"max_active": 1,
|
||||
},
|
||||
"chatgpt_subscription": {
|
||||
"description": "Built-in ChatGPT/Codex account concurrency",
|
||||
"lock_key": "provider:chatgpt",
|
||||
"max_active": 4,
|
||||
},
|
||||
"openai_api": {
|
||||
"description": "OpenAI API concurrency",
|
||||
"lock_key": "provider:openai-api",
|
||||
"max_active": 8,
|
||||
},
|
||||
"anthropic_api": {
|
||||
"description": "Anthropic API concurrency",
|
||||
"lock_key": "provider:anthropic-api",
|
||||
"max_active": 8,
|
||||
},
|
||||
}
|
||||
|
||||
models: dict[str, dict[str, Any]] = {}
|
||||
binding_keys: dict[tuple[str, str], str] = {}
|
||||
inventory_snapshots = load_inventory_snapshots(INVENTORY_SNAPSHOT_ROOT)
|
||||
inventory_as_of = max(str(snapshot["as_of"]) for snapshot in inventory_snapshots)
|
||||
for snapshot in inventory_snapshots:
|
||||
inventory_id = str(snapshot["inventory"])
|
||||
for key, record in snapshot["models"].items():
|
||||
if key in models:
|
||||
raise ValueError(f"duplicate catalog model key across inventory snapshots: {key}")
|
||||
catalog_record = record["catalog"]
|
||||
if catalog_record.get("inventory") != inventory_id:
|
||||
raise ValueError(f"snapshot {inventory_id} model {key} has mismatched inventory")
|
||||
binding = (str(catalog_record["route"]), str(catalog_record["upstream_id"]))
|
||||
if binding in binding_keys:
|
||||
raise ValueError(
|
||||
f"duplicate route/upstream binding across inventory snapshots: "
|
||||
f"{binding_keys[binding]!r} and {key!r} both select {binding!r}"
|
||||
)
|
||||
binding_keys[binding] = key
|
||||
models[key] = dict(catalog_record)
|
||||
|
||||
# Project-local deployments are curated configuration, not upstream inventory.
|
||||
models["llama_cpp_local_openai_chat__qwen3_5_9b"] = model(
|
||||
"llama_cpp_local_openai_chat",
|
||||
"qwen3.5-9b",
|
||||
"qwen",
|
||||
"Qwen3.5-9B local",
|
||||
description="Project-capped text-only 32K/8K local evidence deployment; the upstream model is natively 262K and multimodal",
|
||||
context=32_768,
|
||||
output=8_192,
|
||||
reasoning=["none"],
|
||||
default_reasoning="none",
|
||||
parallel_tool_calls=False,
|
||||
summaries=False,
|
||||
structured=False,
|
||||
resource="local_gpu_0",
|
||||
source="qwen35-model-card",
|
||||
confidence="project-capped-deployment",
|
||||
)
|
||||
|
||||
catalog = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"routes": routes,
|
||||
"models": models,
|
||||
"resources": resources,
|
||||
}
|
||||
validate_catalog_data(catalog, label="generated catalog")
|
||||
|
||||
inventory: dict[str, Any] = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"as_of": inventory_as_of,
|
||||
"sources": {
|
||||
"openai-api-models": "https://developers.openai.com/api/docs/models",
|
||||
"qwen35-model-card": "https://huggingface.co/Qwen/Qwen3.5-9B",
|
||||
"llama-cpp-server": "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md",
|
||||
},
|
||||
"inventories": {},
|
||||
}
|
||||
for snapshot in inventory_snapshots:
|
||||
inventory_id = str(snapshot["inventory"])
|
||||
for source_id, source_url in snapshot["sources"].items():
|
||||
existing = inventory["sources"].get(source_id)
|
||||
if existing is not None and existing != source_url:
|
||||
raise ValueError(f"conflicting URL for inventory source {source_id}")
|
||||
inventory["sources"][source_id] = source_url
|
||||
catalog_keys = sorted(snapshot["models"])
|
||||
model_ids = sorted(
|
||||
{str(snapshot["models"][key]["catalog"]["upstream_id"]) for key in catalog_keys}
|
||||
)
|
||||
entry: dict[str, Any] = {
|
||||
"as_of": snapshot["as_of"],
|
||||
"snapshot": f"config/inventory-snapshots/{inventory_id}.json",
|
||||
"adapter": snapshot["adapter"],
|
||||
"fingerprint_fields": snapshot["fingerprint_fields"],
|
||||
"models_sha256": snapshot["models_sha256"],
|
||||
"dynamic": snapshot["dynamic"],
|
||||
"expected_count": len(model_ids),
|
||||
"models": model_ids,
|
||||
"catalog_keys": catalog_keys,
|
||||
}
|
||||
entry.update(snapshot["discovery"])
|
||||
if snapshot["captures"]:
|
||||
entry["captures"] = snapshot["captures"]
|
||||
inventory["inventories"][inventory_id] = entry
|
||||
|
||||
HEADER = f"""# Generated by scripts/generate_catalog.py. Do not edit this file directly.\n# Add local routes/models under ~/.config/codex-mmo/catalog.d/*.toml.\n# Inventory baseline: config/upstream-inventory.json (as of {inventory_as_of}).\n\n"""
|
||||
|
||||
|
||||
def _rendered_outputs() -> dict[Path, str]:
|
||||
return {
|
||||
ROOT / "config" / "catalog.toml": HEADER + toml_dumps(catalog),
|
||||
ROOT / "config" / "upstream-inventory.json": json.dumps(
|
||||
inventory, ensure_ascii=False, indent=2, sort_keys=True, allow_nan=False
|
||||
)
|
||||
+ "\n",
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate the bundled model catalog")
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="fail if generated files differ without modifying the source tree",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
outputs = _rendered_outputs()
|
||||
changed = [
|
||||
path.relative_to(ROOT).as_posix()
|
||||
for path, expected in outputs.items()
|
||||
if not path.is_file() or path.read_text(encoding="utf-8") != expected
|
||||
]
|
||||
if not args.check:
|
||||
atomic_write_text(
|
||||
ROOT / "config" / "catalog.toml",
|
||||
outputs[ROOT / "config" / "catalog.toml"],
|
||||
0o644,
|
||||
)
|
||||
atomic_write_json(ROOT / "config" / "upstream-inventory.json", inventory, 0o644)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"routes": len(routes),
|
||||
"models": len(models),
|
||||
"resources": len(resources),
|
||||
"inventory_models": {
|
||||
key: len(value["models"])
|
||||
for key, value in sorted(inventory["inventories"].items())
|
||||
},
|
||||
"changed": changed,
|
||||
"passed": not changed if args.check else True,
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 1 if args.check and changed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+773
@@ -0,0 +1,773 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Atomic per-user installer for Codex Multi-Model Orchestrator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from collections.abc import Iterable, Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(PACKAGE_ROOT / "libexec"))
|
||||
|
||||
from mmo_version import ( # noqa: E402
|
||||
APP_SERVER_PROTOCOL_CODEX_VERSION,
|
||||
MMO_SCHEMA_VERSION,
|
||||
PACKAGE_VERSION,
|
||||
SWITCHYARD_BASELINE_VERSION,
|
||||
)
|
||||
|
||||
VERSION = PACKAGE_VERSION
|
||||
|
||||
PAYLOAD_FILES = (
|
||||
"LICENSE",
|
||||
"VERSION",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"VALIDATION.md",
|
||||
"PACKAGE-MANIFEST.json",
|
||||
)
|
||||
PAYLOAD_DIRECTORIES = (
|
||||
"config",
|
||||
"docs",
|
||||
"evals",
|
||||
"libexec",
|
||||
"profiles",
|
||||
"scripts",
|
||||
"tests",
|
||||
)
|
||||
|
||||
RETAINED_OWNER_MANIFEST = ".codex-mmo-install-owner.json"
|
||||
|
||||
|
||||
def filtered_environment(*, extra: dict[str, str] | None = None) -> dict[str, str]:
|
||||
sensitive = re.compile(
|
||||
r"(?:^|_)(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|COOKIE|AUTH|PRIVATE)(?:_|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
result = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if not key.startswith("MMO_") and not sensitive.search(key)
|
||||
}
|
||||
if extra:
|
||||
result.update(extra)
|
||||
return result
|
||||
|
||||
|
||||
def toml_quote(value: str) -> str:
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise ValueError("generated TOML strings must contain valid Unicode scalar values") from exc
|
||||
return json.dumps(value, ensure_ascii=False, allow_nan=False).replace("\x7f", "\\u007F")
|
||||
|
||||
|
||||
def xdg_path(variable: str, fallback: Path) -> Path:
|
||||
"""Resolve an XDG base directory, ignoring empty or relative values."""
|
||||
|
||||
raw = os.environ.get(variable)
|
||||
if raw:
|
||||
candidate = Path(raw)
|
||||
if candidate.is_absolute():
|
||||
return candidate
|
||||
return fallback
|
||||
|
||||
|
||||
def default_install_root() -> Path:
|
||||
return xdg_path("XDG_DATA_HOME", Path.home() / ".local" / "share") / "codex-mmo"
|
||||
|
||||
|
||||
def default_config_root() -> Path:
|
||||
return xdg_path("XDG_CONFIG_HOME", Path.home() / ".config") / "codex-mmo"
|
||||
|
||||
|
||||
def default_state_root() -> Path:
|
||||
return xdg_path("XDG_STATE_HOME", Path.home() / ".local" / "state") / "codex-mmo"
|
||||
|
||||
|
||||
def default_bin_dir() -> Path:
|
||||
return xdg_path("XDG_BIN_HOME", Path.home() / ".local" / "bin")
|
||||
|
||||
|
||||
def atomic_write(path: Path, data: str, mode: int = 0o600) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, mode)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(temporary)
|
||||
|
||||
|
||||
def remove_tree(path: Path) -> None:
|
||||
"""Remove a tree that may contain immutable snapshot permissions."""
|
||||
|
||||
try:
|
||||
root_mode = path.lstat().st_mode
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if stat.S_ISLNK(root_mode):
|
||||
path.unlink()
|
||||
return
|
||||
if not stat.S_ISDIR(root_mode):
|
||||
raise RuntimeError(f"refusing to recursively remove non-directory: {path}")
|
||||
for item in path.rglob("*"):
|
||||
with contextlib.suppress(OSError):
|
||||
mode = item.lstat().st_mode
|
||||
if not stat.S_ISLNK(mode):
|
||||
os.chmod(item, 0o700 if stat.S_ISDIR(mode) else 0o600)
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(path, 0o700)
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def validate_destination_paths(paths: Iterable[Path]) -> None:
|
||||
values = list(paths)
|
||||
protected = {Path("/"), Path.home().resolve()}
|
||||
for path in values:
|
||||
if path in protected or len(path.parts) <= 2:
|
||||
raise ValueError(f"refusing to install into broad protected path: {path}")
|
||||
for index, left in enumerate(values):
|
||||
for right in values[index + 1 :]:
|
||||
if left == right or left in right.parents or right in left.parents:
|
||||
raise ValueError(f"installation paths must not overlap: {left} and {right}")
|
||||
|
||||
|
||||
def _reject_json_constant(value: str) -> None:
|
||||
raise ValueError(f"non-standard JSON constant: {value}")
|
||||
|
||||
|
||||
def _finite_json_float(value: str) -> float:
|
||||
parsed = float(value)
|
||||
if not math.isfinite(parsed):
|
||||
raise ValueError(f"JSON number is outside the supported finite range: {value}")
|
||||
return parsed
|
||||
|
||||
|
||||
def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
||||
result: dict[str, object] = {}
|
||||
for key, item in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON object member: {key!r}")
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def _validate_json_unicode(value: object) -> None:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise ValueError("JSON strings must contain valid Unicode scalar values") from exc
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
_validate_json_unicode(item)
|
||||
elif isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
_validate_json_unicode(key)
|
||||
_validate_json_unicode(item)
|
||||
|
||||
|
||||
def _strict_json_loads(value: str) -> object:
|
||||
parsed = json.loads(
|
||||
value,
|
||||
parse_constant=_reject_json_constant,
|
||||
parse_float=_finite_json_float,
|
||||
object_pairs_hook=_unique_json_object,
|
||||
)
|
||||
_validate_json_unicode(parsed)
|
||||
return parsed
|
||||
|
||||
|
||||
def validate_existing_install(path: Path) -> None:
|
||||
"""Refuse to replace a nonempty tree without this installer's ownership record."""
|
||||
|
||||
if not path.exists():
|
||||
return
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise RuntimeError(f"existing install root is not a real directory: {path}")
|
||||
if next(path.iterdir(), None) is None:
|
||||
return
|
||||
manifest_path = path / "config" / "install-manifest.json"
|
||||
if manifest_path.is_symlink() or not manifest_path.is_file():
|
||||
raise RuntimeError(
|
||||
f"refusing to replace a nonempty install root without its ownership manifest: {path}"
|
||||
)
|
||||
try:
|
||||
data = _strict_json_loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
||||
raise RuntimeError(
|
||||
f"existing install ownership manifest is invalid: {manifest_path}"
|
||||
) from exc
|
||||
recorded = data.get("install_root") if isinstance(data, dict) else None
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or not isinstance(data.get("schema_version"), int)
|
||||
or isinstance(data.get("schema_version"), bool)
|
||||
or data.get("schema_version") != MMO_SCHEMA_VERSION
|
||||
or data.get("package") != "codex-multimodel-orchestrator"
|
||||
or not isinstance(recorded, str)
|
||||
or not Path(recorded).expanduser().is_absolute()
|
||||
or Path(recorded).expanduser().resolve(strict=False) != path
|
||||
):
|
||||
raise RuntimeError(f"existing install ownership manifest does not match: {path}")
|
||||
|
||||
|
||||
def validate_wrapper_targets(bin_dir: Path) -> None:
|
||||
marker = "Installed by codex-multimodel-orchestrator"
|
||||
for name in ("codex-mmoctl", "codex-mmo", "codex-mmo-uninstall"):
|
||||
path = bin_dir / name
|
||||
if not path.exists() and not path.is_symlink():
|
||||
continue
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise FileExistsError(f"refusing to replace non-regular wrapper target: {path}")
|
||||
try:
|
||||
owned = marker in path.read_text(encoding="utf-8", errors="replace")[:500]
|
||||
except OSError as exc:
|
||||
raise FileExistsError(f"unable to verify existing wrapper ownership: {path}") from exc
|
||||
if not owned:
|
||||
raise FileExistsError(f"refusing to replace unowned executable: {path}")
|
||||
|
||||
|
||||
def _ignore(_directory: str, names: list[str]) -> set[str]:
|
||||
ignored = {
|
||||
name
|
||||
for name in names
|
||||
if name in {"__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache"}
|
||||
}
|
||||
ignored.update(name for name in names if name.endswith((".pyc", ".pyo")))
|
||||
return ignored
|
||||
|
||||
|
||||
def validate_payload_source() -> None:
|
||||
"""Require a complete, static source tree before copying an installer payload."""
|
||||
|
||||
for name in PAYLOAD_FILES:
|
||||
source = PACKAGE_ROOT / name
|
||||
try:
|
||||
mode = source.lstat().st_mode
|
||||
except FileNotFoundError as exc:
|
||||
raise FileNotFoundError(f"required installer payload file is missing: {name}") from exc
|
||||
if stat.S_ISLNK(mode) or not stat.S_ISREG(mode):
|
||||
raise ValueError(f"installer payload file must be regular and not a symlink: {name}")
|
||||
for name in PAYLOAD_DIRECTORIES:
|
||||
source = PACKAGE_ROOT / name
|
||||
try:
|
||||
mode = source.lstat().st_mode
|
||||
except FileNotFoundError as exc:
|
||||
raise FileNotFoundError(
|
||||
f"required installer payload directory is missing: {name}"
|
||||
) from exc
|
||||
if stat.S_ISLNK(mode) or not stat.S_ISDIR(mode):
|
||||
raise ValueError(f"installer payload directory must be real and not a symlink: {name}")
|
||||
for item in source.rglob("*"):
|
||||
relative = item.relative_to(PACKAGE_ROOT)
|
||||
item_mode = item.lstat().st_mode
|
||||
if stat.S_ISLNK(item_mode):
|
||||
raise ValueError(f"symbolic links are not allowed in installer payload: {relative}")
|
||||
if not (stat.S_ISREG(item_mode) or stat.S_ISDIR(item_mode)):
|
||||
raise ValueError(f"special files are not allowed in installer payload: {relative}")
|
||||
|
||||
|
||||
def copy_payload(destination: Path) -> None:
|
||||
validate_payload_source()
|
||||
destination.mkdir(parents=True, exist_ok=False)
|
||||
for name in PAYLOAD_FILES:
|
||||
source = PACKAGE_ROOT / name
|
||||
shutil.copy2(source, destination / name)
|
||||
for name in PAYLOAD_DIRECTORIES:
|
||||
source = PACKAGE_ROOT / name
|
||||
shutil.copytree(source, destination / name, ignore=_ignore)
|
||||
for path in destination.rglob("*"):
|
||||
if path.is_dir():
|
||||
os.chmod(path, 0o755)
|
||||
elif path.suffix == ".py":
|
||||
os.chmod(path, 0o755 if os.access(path, os.X_OK) else 0o644)
|
||||
else:
|
||||
os.chmod(path, 0o644)
|
||||
|
||||
|
||||
def wrapper(
|
||||
python: Path,
|
||||
entrypoint: Path,
|
||||
fixed: Iterable[str] = (),
|
||||
*,
|
||||
environment: Mapping[str, str] | None = None,
|
||||
) -> str:
|
||||
words = [
|
||||
shlex.quote(str(python)),
|
||||
shlex.quote(str(entrypoint)),
|
||||
*[shlex.quote(item) for item in fixed],
|
||||
]
|
||||
command = " ".join(words)
|
||||
exports = "".join(
|
||||
f"export {name}={shlex.quote(value)}\n"
|
||||
for name, value in sorted((environment or {}).items())
|
||||
)
|
||||
return f"""#!/usr/bin/env bash
|
||||
# Installed by codex-multimodel-orchestrator.
|
||||
set -euo pipefail
|
||||
{exports}exec {command} "$@"
|
||||
"""
|
||||
|
||||
|
||||
def main_wrapper(python: Path, entrypoint: Path) -> str:
|
||||
return wrapper(
|
||||
python,
|
||||
entrypoint,
|
||||
environment={"MMO_CLI_ENTRYPOINT": "codex-mmo"},
|
||||
)
|
||||
|
||||
|
||||
def install_optional_tools(args: argparse.Namespace) -> None:
|
||||
if args.install_codex:
|
||||
codex = shutil.which(args.codex_bin)
|
||||
observed = ""
|
||||
if codex:
|
||||
result = subprocess.run(
|
||||
[codex, "--version"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env=filtered_environment(),
|
||||
)
|
||||
if result.returncode == 0:
|
||||
observed = result.stdout.strip().removeprefix("codex-cli ").strip()
|
||||
if observed != APP_SERVER_PROTOCOL_CODEX_VERSION:
|
||||
npm = shutil.which("npm")
|
||||
if not npm:
|
||||
raise RuntimeError("--install-codex requires npm")
|
||||
subprocess.run(
|
||||
[
|
||||
npm,
|
||||
"install",
|
||||
"-g",
|
||||
f"@openai/codex@{APP_SERVER_PROTOCOL_CODEX_VERSION}",
|
||||
],
|
||||
check=True,
|
||||
env=filtered_environment(),
|
||||
)
|
||||
codex = shutil.which(args.codex_bin)
|
||||
if not codex:
|
||||
raise RuntimeError(
|
||||
f"npm installation did not provide requested Codex binary {args.codex_bin!r}"
|
||||
)
|
||||
result = subprocess.run(
|
||||
[codex, "--version"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env=filtered_environment(),
|
||||
)
|
||||
observed = (
|
||||
result.stdout.strip().removeprefix("codex-cli ").strip()
|
||||
if result.returncode == 0
|
||||
else ""
|
||||
)
|
||||
if observed != APP_SERVER_PROTOCOL_CODEX_VERSION:
|
||||
raise RuntimeError(
|
||||
"npm installed Codex but the requested binary does not expose the reviewed "
|
||||
f"version {APP_SERVER_PROTOCOL_CODEX_VERSION}: observed "
|
||||
f"{observed or 'unavailable'} at {codex}"
|
||||
)
|
||||
if args.install_switchyard and not shutil.which(args.switchyard_bin):
|
||||
cargo = shutil.which("cargo")
|
||||
if not cargo:
|
||||
raise RuntimeError("--install-switchyard requires Cargo")
|
||||
subprocess.run(
|
||||
[
|
||||
cargo,
|
||||
"install",
|
||||
"--locked",
|
||||
"--version",
|
||||
SWITCHYARD_BASELINE_VERSION,
|
||||
"switchyard-server",
|
||||
],
|
||||
check=True,
|
||||
env=filtered_environment(),
|
||||
)
|
||||
if not shutil.which(args.switchyard_bin):
|
||||
raise RuntimeError(
|
||||
"Cargo installation did not provide requested Switchyard binary "
|
||||
f"{args.switchyard_bin!r}"
|
||||
)
|
||||
|
||||
|
||||
def validate_stage(stage: Path, config_root: Path, state_root: Path) -> None:
|
||||
validation_state = Path(tempfile.mkdtemp(prefix=".install-validation-", dir=state_root))
|
||||
env = filtered_environment(
|
||||
extra={
|
||||
"MMO_INSTALL_ROOT": str(stage),
|
||||
"MMO_CONFIG_ROOT": str(config_root),
|
||||
"MMO_STATE_ROOT": str(validation_state),
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONPATH": str(stage / "libexec"),
|
||||
}
|
||||
)
|
||||
code = r"""
|
||||
from mmo_catalog import local_inventory_report
|
||||
from mmo_profiles import builtin_profiles_root, resolve_profile
|
||||
from mmo_snapshot import compile_profile
|
||||
profile_directories = sorted(
|
||||
path
|
||||
for path in builtin_profiles_root().iterdir()
|
||||
if path.is_dir() and (path / "profile.toml").is_file()
|
||||
)
|
||||
profiles = {}
|
||||
for directory in profile_directories:
|
||||
resolved = resolve_profile(directory)
|
||||
profile_id = resolved["profile"]["id"]
|
||||
if profile_id in profiles:
|
||||
raise RuntimeError(f"duplicate bundled profile id: {profile_id}")
|
||||
profiles[profile_id] = (directory, resolved)
|
||||
expected_profiles = {
|
||||
"adaptive-engineering",
|
||||
"access-efficient-escalation-lab",
|
||||
"bounded-research-organization-lab",
|
||||
"codex-harness-team",
|
||||
"competing-implementations-lab",
|
||||
"contract-first-refactoring",
|
||||
"high-confidence-debugging",
|
||||
"incident-hypothesis-triage",
|
||||
"research-backed-engineering",
|
||||
"route-resilience-lab",
|
||||
"secure-change",
|
||||
"visual-engineering",
|
||||
}
|
||||
if set(profiles) != expected_profiles:
|
||||
raise RuntimeError(
|
||||
f"bundled profile set differs from the release contract: {sorted(profiles)}"
|
||||
)
|
||||
for profile_id in sorted(profiles):
|
||||
directory, resolved = profiles[profile_id]
|
||||
if resolved["profile"]["root"] not in resolved["agents"]:
|
||||
raise RuntimeError(f"profile root is missing from agents: {profile_id}")
|
||||
compile_profile(directory)
|
||||
report = local_inventory_report()
|
||||
if not report["passed"]:
|
||||
raise RuntimeError(f"local inventory validation failed: {report}")
|
||||
print(len(profiles))
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", code],
|
||||
env=env,
|
||||
cwd=stage,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
finally:
|
||||
if validation_state.exists():
|
||||
remove_tree(validation_state)
|
||||
if result.returncode:
|
||||
raise RuntimeError("staged package validation failed:\n" + result.stdout + result.stderr)
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
options: dict[str, Any] = {
|
||||
"description": "Install or atomically replace one per-user Codex MMO runtime.",
|
||||
"epilog": (
|
||||
"Existing schema-8 configuration and state are retained. Run the uninstaller "
|
||||
"with explicit purge flags when permanent removal is intended."
|
||||
),
|
||||
"allow_abbrev": False,
|
||||
}
|
||||
if sys.version_info >= (3, 14):
|
||||
options["color"] = False
|
||||
value = argparse.ArgumentParser(**options)
|
||||
value.add_argument(
|
||||
"--version", action="version", version=VERSION, help="print the package version"
|
||||
)
|
||||
value.add_argument(
|
||||
"--install-root",
|
||||
type=Path,
|
||||
default=default_install_root(),
|
||||
help="runtime payload destination (default: XDG data home/codex-mmo)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--config-root",
|
||||
type=Path,
|
||||
default=default_config_root(),
|
||||
help="operator configuration destination (default: XDG config home/codex-mmo)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--state-root",
|
||||
type=Path,
|
||||
default=default_state_root(),
|
||||
help="session and job state destination (default: XDG state home/codex-mmo)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--bin-dir",
|
||||
type=Path,
|
||||
default=default_bin_dir(),
|
||||
help="installed command directory (default: XDG_BIN_HOME or ~/.local/bin)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--codex-bin", default="codex", help="Codex executable name or path (default: codex)"
|
||||
)
|
||||
value.add_argument(
|
||||
"--switchyard-bin",
|
||||
default="switchyard-server",
|
||||
help="Switchyard executable name or path (default: switchyard-server)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--install-codex",
|
||||
action="store_true",
|
||||
help="install the reviewed Codex CLI with npm when the requested binary is missing",
|
||||
)
|
||||
value.add_argument(
|
||||
"--install-switchyard",
|
||||
action="store_true",
|
||||
help="install the pinned Switchyard release with Cargo when missing",
|
||||
)
|
||||
value.add_argument(
|
||||
"--no-validate",
|
||||
action="store_true",
|
||||
help="skip staged package validation (development use only)",
|
||||
)
|
||||
value.add_argument(
|
||||
"--debug", action="store_true", help="show a Python traceback when installation fails"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if sys.version_info < (3, 11): # noqa: UP036 - bootstrap a clear unsupported-Python error
|
||||
raise RuntimeError("Python 3.11 or newer is required")
|
||||
args = parser().parse_args()
|
||||
for name in ("install_root", "config_root", "state_root", "bin_dir"):
|
||||
setattr(args, name, getattr(args, name).expanduser().resolve())
|
||||
validate_destination_paths((args.install_root, args.config_root, args.state_root, args.bin_dir))
|
||||
validate_existing_install(args.install_root)
|
||||
validate_wrapper_targets(args.bin_dir)
|
||||
install_optional_tools(args)
|
||||
|
||||
args.config_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
args.state_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
args.bin_dir.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||
(args.config_root / "profiles.d").mkdir(mode=0o700, exist_ok=True)
|
||||
(args.config_root / "catalog.d").mkdir(mode=0o700, exist_ok=True)
|
||||
(args.config_root / "tool-mcp.d").mkdir(mode=0o700, exist_ok=True)
|
||||
|
||||
retained_owner_markers = tuple(
|
||||
root / RETAINED_OWNER_MANIFEST for root in (args.config_root, args.state_root)
|
||||
)
|
||||
for marker in retained_owner_markers:
|
||||
if marker.is_symlink():
|
||||
raise RuntimeError(f"refusing retained ownership marker symlink: {marker}")
|
||||
if marker.exists() and not marker.is_file():
|
||||
raise RuntimeError(f"retained ownership marker is not a regular file: {marker}")
|
||||
|
||||
settings_path = args.config_root / "settings.toml"
|
||||
if not settings_path.exists():
|
||||
settings = (PACKAGE_ROOT / "config" / "settings.toml").read_text(encoding="utf-8")
|
||||
settings = settings.replace(
|
||||
'codex_bin = "codex"',
|
||||
f"codex_bin = {toml_quote(args.codex_bin)}",
|
||||
)
|
||||
settings = settings.replace(
|
||||
'switchyard_bin = "switchyard-server"',
|
||||
f"switchyard_bin = {toml_quote(args.switchyard_bin)}",
|
||||
)
|
||||
atomic_write(settings_path, settings, 0o600)
|
||||
|
||||
credentials_path = args.config_root / "credentials.env"
|
||||
credentials_existed = credentials_path.exists()
|
||||
if not credentials_existed:
|
||||
atomic_write(
|
||||
credentials_path,
|
||||
(PACKAGE_ROOT / "config" / "credentials.env.example").read_text(encoding="utf-8"),
|
||||
0o600,
|
||||
)
|
||||
|
||||
stage_parent = args.install_root.parent
|
||||
stage_parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||
stage = Path(tempfile.mkdtemp(prefix=f".{args.install_root.name}.staging-", dir=stage_parent))
|
||||
backup: Path | None = None
|
||||
old_install: Path | None = None
|
||||
wrapper_paths = tuple(
|
||||
args.bin_dir / name for name in ("codex-mmoctl", "codex-mmo", "codex-mmo-uninstall")
|
||||
)
|
||||
previous_wrappers: dict[Path, tuple[str, int] | None] = {}
|
||||
try:
|
||||
remove_tree(stage)
|
||||
copy_payload(stage)
|
||||
runtime = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": "codex-multimodel-orchestrator",
|
||||
"package_version": VERSION,
|
||||
"install_root": str(args.install_root),
|
||||
"config_root": str(args.config_root),
|
||||
"state_root": str(args.state_root),
|
||||
"bin_dir": str(args.bin_dir),
|
||||
"python_bin": str(Path(sys.executable).resolve()),
|
||||
"installed_at": dt.datetime.now(dt.UTC).isoformat(timespec="seconds"),
|
||||
}
|
||||
atomic_write(
|
||||
stage / "config" / "runtime.json",
|
||||
json.dumps(runtime, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
0o644,
|
||||
)
|
||||
manifest = {
|
||||
**runtime,
|
||||
"source_root": str(PACKAGE_ROOT),
|
||||
"wrappers": [
|
||||
str(args.bin_dir / "codex-mmo"),
|
||||
str(args.bin_dir / "codex-mmoctl"),
|
||||
str(args.bin_dir / "codex-mmo-uninstall"),
|
||||
],
|
||||
}
|
||||
atomic_write(
|
||||
stage / "config" / "install-manifest.json",
|
||||
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
0o644,
|
||||
)
|
||||
if not args.no_validate:
|
||||
validate_stage(stage, args.config_root, args.state_root)
|
||||
|
||||
python = Path(sys.executable).resolve()
|
||||
wrapper_contents = {
|
||||
args.bin_dir / "codex-mmoctl": wrapper(
|
||||
python,
|
||||
args.install_root / "libexec" / "mmoctl.py",
|
||||
environment={"MMO_CLI_ENTRYPOINT": "codex-mmoctl"},
|
||||
),
|
||||
args.bin_dir / "codex-mmo": main_wrapper(
|
||||
python, args.install_root / "libexec" / "mmoctl.py"
|
||||
),
|
||||
args.bin_dir / "codex-mmo-uninstall": wrapper(
|
||||
python,
|
||||
args.install_root / "scripts" / "uninstall.py",
|
||||
(
|
||||
"--install-root",
|
||||
str(args.install_root),
|
||||
"--config-root",
|
||||
str(args.config_root),
|
||||
"--state-root",
|
||||
str(args.state_root),
|
||||
"--bin-dir",
|
||||
str(args.bin_dir),
|
||||
),
|
||||
),
|
||||
}
|
||||
for path in wrapper_paths:
|
||||
previous_wrappers[path] = (
|
||||
(path.read_text(encoding="utf-8"), path.stat().st_mode & 0o777)
|
||||
if path.is_file()
|
||||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
# Wrapper targets use the stable install path, so they can be
|
||||
# staged before the brief atomic payload swap and rolled back if
|
||||
# any write or activation step fails.
|
||||
for path, contents in wrapper_contents.items():
|
||||
atomic_write(path, contents, 0o755)
|
||||
if args.install_root.exists():
|
||||
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%d-%H%M%S-%f")
|
||||
backup_path = args.state_root / "backups" / f"install-{stamp}"
|
||||
backup_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(args.install_root, backup_path, symlinks=True)
|
||||
backup = backup_path
|
||||
old_path = args.install_root.with_name(
|
||||
f".{args.install_root.name}.previous-{os.getpid()}-{stamp}"
|
||||
)
|
||||
os.replace(args.install_root, old_path)
|
||||
old_install = old_path
|
||||
try:
|
||||
os.replace(stage, args.install_root)
|
||||
except BaseException:
|
||||
if old_install and old_install.exists() and not args.install_root.exists():
|
||||
os.replace(old_install, args.install_root)
|
||||
raise
|
||||
except BaseException:
|
||||
for path, previous in previous_wrappers.items():
|
||||
if previous is None:
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
atomic_write(path, previous[0], previous[1])
|
||||
raise
|
||||
if old_install and old_install.exists():
|
||||
try:
|
||||
remove_tree(old_install)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
print(
|
||||
f"warning: installed successfully but could not remove {old_install}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
# Retained ownership markers authorize a later purge only while the
|
||||
# payload is absent. Once a fresh payload is active its embedded
|
||||
# manifest is authoritative, and leaving an older marker behind would
|
||||
# retain an obsolete MMO schema in the active config/state roots.
|
||||
for marker in retained_owner_markers:
|
||||
marker.unlink(missing_ok=True)
|
||||
finally:
|
||||
if stage.exists():
|
||||
with contextlib.suppress(OSError):
|
||||
remove_tree(stage)
|
||||
|
||||
print(f"Codex Multi-Model Orchestrator {VERSION} installed.")
|
||||
print(f" executable: {args.bin_dir / 'codex-mmo'}")
|
||||
print(f" control: {args.bin_dir / 'codex-mmoctl'}")
|
||||
print(f" install: {args.install_root}")
|
||||
print(f" config: {args.config_root}")
|
||||
print(f" state: {args.state_root}")
|
||||
if backup:
|
||||
print(f" backup: {backup}")
|
||||
print()
|
||||
print(f"Configure provider credentials in {credentials_path}")
|
||||
print("Authenticate built-in Codex models with: codex-mmo auth login")
|
||||
print("Validate with: codex-mmo validate --all-profiles")
|
||||
print("Live-check with: codex-mmo doctor --live")
|
||||
print("Launch with: codex-mmo")
|
||||
path_entries = {
|
||||
Path(item).expanduser().resolve(strict=False)
|
||||
for item in os.environ.get("PATH", "").split(os.pathsep)
|
||||
if item
|
||||
}
|
||||
if args.bin_dir not in path_entries:
|
||||
print(
|
||||
f"warning: {args.bin_dir} is not on PATH; add it before invoking codex-mmo",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def cli_main() -> int:
|
||||
try:
|
||||
return main()
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
if "--debug" in sys.argv[1:]:
|
||||
print(f"exception: {type(exc).__name__}", file=sys.stderr)
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(cli_main())
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run each offline integration-test module in an isolated process group."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_util import filtered_environment, terminate_process_group # noqa: E402
|
||||
|
||||
DEFAULT_MODULES = (
|
||||
"tests.test_catalog_profiles",
|
||||
"tests.test_runtime",
|
||||
"tests.test_runtime_advanced",
|
||||
"tests.test_tui_metadata",
|
||||
"tests.test_install_eval",
|
||||
"tests.test_cli_ux",
|
||||
"tests.test_release",
|
||||
)
|
||||
_CURRENT_PROCESS: subprocess.Popen[str] | None = None
|
||||
|
||||
|
||||
def _terminate_group(process: subprocess.Popen[str]) -> None:
|
||||
terminate_process_group(process.pid, grace_seconds=3.0)
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
|
||||
def _handle_termination(_signum: int, _frame: object) -> None:
|
||||
if _CURRENT_PROCESS is not None:
|
||||
_terminate_group(_CURRENT_PROCESS)
|
||||
raise KeyboardInterrupt
|
||||
|
||||
|
||||
def run_module(module: str, timeout: float) -> tuple[bool, float]:
|
||||
global _CURRENT_PROCESS
|
||||
environment = filtered_environment(
|
||||
extra={
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONWARNINGS": "error::ResourceWarning",
|
||||
"PYTHONPATH": os.pathsep.join((str(ROOT / "libexec"), str(ROOT / "tests"))),
|
||||
}
|
||||
)
|
||||
command = [sys.executable, "-m", "unittest", "-v", module]
|
||||
print(f"\n=== {module} ===", flush=True)
|
||||
started = time.monotonic()
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env=environment,
|
||||
text=True,
|
||||
start_new_session=True,
|
||||
)
|
||||
_CURRENT_PROCESS = process
|
||||
try:
|
||||
try:
|
||||
return_code = process.wait(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(
|
||||
f"ERROR: {module} exceeded {timeout:g} seconds; terminating its process group",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
return False, time.monotonic() - started
|
||||
finally:
|
||||
# A test process can exit while leaving a child in its isolated group.
|
||||
# Retire that group on success, timeout, interruption, and wait errors.
|
||||
_terminate_group(process)
|
||||
_CURRENT_PROCESS = None
|
||||
return return_code == 0, time.monotonic() - started
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run Codex MMO offline tests with per-module process isolation"
|
||||
)
|
||||
parser.add_argument("modules", nargs="*", default=list(DEFAULT_MODULES))
|
||||
parser.add_argument("--timeout", type=float, default=300.0)
|
||||
args = parser.parse_args()
|
||||
if args.timeout <= 0:
|
||||
parser.error("--timeout must be positive")
|
||||
|
||||
signal.signal(signal.SIGTERM, _handle_termination)
|
||||
signal.signal(signal.SIGINT, _handle_termination)
|
||||
|
||||
failures: list[str] = []
|
||||
durations: dict[str, float] = {}
|
||||
for module in args.modules:
|
||||
passed, duration = run_module(module, args.timeout)
|
||||
durations[module] = duration
|
||||
if not passed:
|
||||
failures.append(module)
|
||||
|
||||
print("\n=== isolated suite summary ===")
|
||||
for module, duration in durations.items():
|
||||
status = "PASS" if module not in failures else "FAIL"
|
||||
print(f"{status:4} {duration:8.2f}s {module}")
|
||||
if failures:
|
||||
print(
|
||||
"failed modules: " + ", ".join(failures),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
print(f"all {len(durations)} modules passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+333
@@ -0,0 +1,333 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Uninstall Codex MMO without deleting user state unless explicitly requested."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
PACKAGE_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(PACKAGE_ROOT / "libexec"))
|
||||
|
||||
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION # noqa: E402
|
||||
|
||||
OWNER_MANIFEST = ".codex-mmo-install-owner.json"
|
||||
|
||||
|
||||
def _reject_json_constant(value: str) -> Any:
|
||||
raise ValueError(f"non-standard JSON constant: {value}")
|
||||
|
||||
|
||||
def _finite_json_float(value: str) -> float:
|
||||
parsed = float(value)
|
||||
if not math.isfinite(parsed):
|
||||
raise ValueError(f"JSON number is outside the supported finite range: {value}")
|
||||
return parsed
|
||||
|
||||
|
||||
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, item in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON object member: {key!r}")
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def _validate_json_unicode(value: Any) -> None:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise ValueError("JSON strings must contain valid Unicode scalar values") from exc
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
_validate_json_unicode(item)
|
||||
elif isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
_validate_json_unicode(key)
|
||||
_validate_json_unicode(item)
|
||||
|
||||
|
||||
def _strict_json_loads(value: str) -> Any:
|
||||
parsed = json.loads(
|
||||
value,
|
||||
parse_constant=_reject_json_constant,
|
||||
parse_float=_finite_json_float,
|
||||
object_pairs_hook=_unique_json_object,
|
||||
)
|
||||
_validate_json_unicode(parsed)
|
||||
return parsed
|
||||
|
||||
|
||||
def xdg_path(variable: str, fallback: Path) -> Path:
|
||||
"""Resolve an XDG base directory, ignoring empty or relative values."""
|
||||
|
||||
raw = os.environ.get(variable)
|
||||
if raw:
|
||||
candidate = Path(raw)
|
||||
if candidate.is_absolute():
|
||||
return candidate
|
||||
return fallback
|
||||
|
||||
|
||||
def defaults() -> tuple[Path, Path, Path, Path]:
|
||||
install = xdg_path("XDG_DATA_HOME", Path.home() / ".local" / "share") / "codex-mmo"
|
||||
config = xdg_path("XDG_CONFIG_HOME", Path.home() / ".config") / "codex-mmo"
|
||||
state = xdg_path("XDG_STATE_HOME", Path.home() / ".local" / "state") / "codex-mmo"
|
||||
bin_dir = xdg_path("XDG_BIN_HOME", Path.home() / ".local" / "bin")
|
||||
return install, config, state, bin_dir
|
||||
|
||||
|
||||
def owned_wrapper(path: Path) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
try:
|
||||
return (
|
||||
"Installed by codex-multimodel-orchestrator"
|
||||
in path.read_text(encoding="utf-8", errors="replace")[:500]
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def absolute_path(path: Path) -> Path:
|
||||
"""Return a lexical absolute path without following the deletion target."""
|
||||
|
||||
return Path(os.path.abspath(path.expanduser()))
|
||||
|
||||
|
||||
def remove_tree(path: Path) -> None:
|
||||
"""Remove an owned tree even when compiled snapshots are read-only."""
|
||||
|
||||
try:
|
||||
mode = path.lstat().st_mode
|
||||
except FileNotFoundError:
|
||||
return
|
||||
if stat.S_ISLNK(mode):
|
||||
path.unlink()
|
||||
return
|
||||
if not stat.S_ISDIR(mode):
|
||||
raise RuntimeError(f"refusing to recursively remove non-directory: {path}")
|
||||
if path.resolve(strict=False) != path:
|
||||
raise RuntimeError(f"refusing to remove a path reached through a symlink: {path}")
|
||||
for item in path.rglob("*"):
|
||||
with contextlib.suppress(OSError):
|
||||
item_mode = item.lstat().st_mode
|
||||
if not stat.S_ISLNK(item_mode):
|
||||
os.chmod(item, 0o700 if stat.S_ISDIR(item_mode) else 0o600)
|
||||
with contextlib.suppress(OSError):
|
||||
os.chmod(path, 0o700)
|
||||
shutil.rmtree(path)
|
||||
|
||||
|
||||
def validate_removal_target(path: Path, label: str) -> None:
|
||||
protected = {Path("/"), Path.home().resolve()}
|
||||
if path in protected or len(path.parts) <= 2:
|
||||
raise ValueError(f"refusing to remove broad protected {label}: {path}")
|
||||
if path.resolve(strict=False) != path:
|
||||
raise ValueError(f"refusing {label} reached through a symlink: {path}")
|
||||
|
||||
|
||||
def _option_was_supplied(name: str) -> bool:
|
||||
return any(item == name or item.startswith(f"{name}=") for item in sys.argv[1:])
|
||||
|
||||
|
||||
def _trusted_manifest(path: Path, install_root: Path) -> dict[str, Any] | None:
|
||||
if path.is_symlink() or not path.is_file():
|
||||
return None
|
||||
try:
|
||||
data = _strict_json_loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
schema_version = data.get("schema_version") if isinstance(data, dict) else None
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or not isinstance(schema_version, int)
|
||||
or isinstance(schema_version, bool)
|
||||
or schema_version != MMO_SCHEMA_VERSION
|
||||
):
|
||||
return None
|
||||
recorded = data.get("install_root")
|
||||
if (
|
||||
not isinstance(recorded, str)
|
||||
or not Path(recorded).expanduser().is_absolute()
|
||||
or absolute_path(Path(recorded)) != install_root
|
||||
):
|
||||
return None
|
||||
if data.get("package") != "codex-multimodel-orchestrator":
|
||||
return None
|
||||
return data
|
||||
|
||||
|
||||
def _write_owner_manifest(root: Path, data: dict[str, Any]) -> None:
|
||||
"""Persist deletion ownership when config/state survives package removal."""
|
||||
|
||||
if not root.is_dir() or root.is_symlink():
|
||||
return
|
||||
destination = root / OWNER_MANIFEST
|
||||
fd, raw_temporary = tempfile.mkstemp(prefix=f".{OWNER_MANIFEST}.", dir=root)
|
||||
temporary = Path(raw_temporary)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(data, handle, indent=2, sort_keys=True, allow_nan=False)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
temporary.unlink()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
install, config, state, bin_dir = defaults()
|
||||
parser_options: dict[str, Any] = {
|
||||
"description": "Remove the owned Codex MMO runtime while retaining user data by default.",
|
||||
"epilog": (
|
||||
"Configuration, credentials, sessions, jobs, evaluations, and logs remain unless "
|
||||
"their explicit purge flags are supplied. Ownership manifests guard every removal."
|
||||
),
|
||||
"allow_abbrev": False,
|
||||
}
|
||||
if sys.version_info >= (3, 14):
|
||||
parser_options["color"] = False
|
||||
parser = argparse.ArgumentParser(**parser_options)
|
||||
parser.add_argument(
|
||||
"--version", action="version", version=PACKAGE_VERSION, help="print the package version"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--install-root", type=Path, default=install, help="owned runtime payload to remove"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config-root",
|
||||
type=Path,
|
||||
default=config,
|
||||
help="configuration root recorded by the installer",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state-root", type=Path, default=state, help="state root recorded by the installer"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bin-dir", type=Path, default=bin_dir, help="directory containing installed wrappers"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--purge-config",
|
||||
action="store_true",
|
||||
help="also permanently remove configuration and credentials",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--purge-state",
|
||||
action="store_true",
|
||||
help="also permanently remove sessions, jobs, evaluations, and logs",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug", action="store_true", help="show a Python traceback when removal fails"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
for name in ("install_root", "config_root", "state_root", "bin_dir"):
|
||||
setattr(args, name, absolute_path(getattr(args, name)))
|
||||
|
||||
install_manifest = args.install_root / "config" / "install-manifest.json"
|
||||
# A retained config/state marker authorizes a later purge only after the
|
||||
# payload root is gone. If anything has appeared at the install path again,
|
||||
# require that live tree's own manifest so a stale marker cannot authorize
|
||||
# deletion of unrelated replacement content.
|
||||
manifests: tuple[Path, ...]
|
||||
if args.install_root.exists() or args.install_root.is_symlink():
|
||||
manifests = (install_manifest,)
|
||||
else:
|
||||
manifests = (
|
||||
args.config_root / OWNER_MANIFEST,
|
||||
args.state_root / OWNER_MANIFEST,
|
||||
)
|
||||
data = next(
|
||||
(
|
||||
candidate
|
||||
for path in manifests
|
||||
if (candidate := _trusted_manifest(path, args.install_root)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
if data is None:
|
||||
raise RuntimeError(
|
||||
"refusing to remove an install root without its valid current-generation "
|
||||
"ownership manifest: "
|
||||
f"{args.install_root}"
|
||||
)
|
||||
for option, attribute in (
|
||||
("--config-root", "config_root"),
|
||||
("--state-root", "state_root"),
|
||||
("--bin-dir", "bin_dir"),
|
||||
):
|
||||
value = data.get(attribute)
|
||||
if not isinstance(value, str) or not Path(value).expanduser().is_absolute():
|
||||
raise RuntimeError(f"ownership manifest is missing {attribute}")
|
||||
recorded = absolute_path(Path(value))
|
||||
if _option_was_supplied(option):
|
||||
if getattr(args, attribute) != recorded:
|
||||
raise RuntimeError(
|
||||
f"refusing {option} outside the ownership manifest: "
|
||||
f"{getattr(args, attribute)} != {recorded}"
|
||||
)
|
||||
else:
|
||||
setattr(args, attribute, recorded)
|
||||
|
||||
validate_removal_target(args.install_root, "install root")
|
||||
validate_removal_target(args.config_root, "configuration root")
|
||||
validate_removal_target(args.state_root, "state root")
|
||||
if args.bin_dir.resolve(strict=False) != args.bin_dir:
|
||||
raise ValueError(f"refusing binary directory reached through a symlink: {args.bin_dir}")
|
||||
|
||||
# Preserve a location-bound ownership record before removing the only
|
||||
# installed copy. This supports a later explicit purge without relaxing
|
||||
# deletion checks for arbitrary directories.
|
||||
if not args.purge_config:
|
||||
_write_owner_manifest(args.config_root, data)
|
||||
if not args.purge_state:
|
||||
_write_owner_manifest(args.state_root, data)
|
||||
|
||||
for name in ("codex-mmo", "codex-mmoctl", "codex-mmo-uninstall"):
|
||||
path = args.bin_dir / name
|
||||
if owned_wrapper(path):
|
||||
path.unlink()
|
||||
remove_tree(args.install_root)
|
||||
if args.purge_config:
|
||||
remove_tree(args.config_root)
|
||||
if args.purge_state:
|
||||
remove_tree(args.state_root)
|
||||
|
||||
print("Codex Multi-Model Orchestrator removed.")
|
||||
if not args.purge_config and args.config_root.exists():
|
||||
print(f"Configuration retained at {args.config_root}; use --purge-config to remove it.")
|
||||
if not args.purge_state and args.state_root.exists():
|
||||
print(f"State retained at {args.state_root}; use --purge-state to remove it.")
|
||||
return 0
|
||||
|
||||
|
||||
def cli_main() -> int:
|
||||
try:
|
||||
return main()
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
if "--debug" in sys.argv[1:]:
|
||||
print(f"exception: {type(exc).__name__}", file=sys.stderr)
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(cli_main())
|
||||
Executable
+661
@@ -0,0 +1,661 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Release acceptance validator for Codex Multi-Model Orchestrator.
|
||||
|
||||
The validator intentionally uses only the Python standard library. It checks the
|
||||
static package, all profile compositions, the complete catalog baseline,
|
||||
generated snapshots, source syntax, installable-profile safety, release hygiene,
|
||||
and the offline integration suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import py_compile
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.dont_write_bytecode = True
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_catalog import catalog_summary, local_inventory_report # noqa: E402
|
||||
from mmo_eval import discover_suites, validate_suite # noqa: E402
|
||||
from mmo_profiles import ( # noqa: E402
|
||||
discover_profiles,
|
||||
resolve_profile,
|
||||
validate_profile_pack_tree,
|
||||
)
|
||||
from mmo_snapshot import compile_profile # noqa: E402
|
||||
from mmo_util import ( # noqa: E402
|
||||
atomic_write_json,
|
||||
filtered_environment,
|
||||
read_toml,
|
||||
sha256_file,
|
||||
strict_json_loads,
|
||||
terminate_process_group,
|
||||
utc_now,
|
||||
)
|
||||
from mmo_version import MMO_SCHEMA_VERSION # noqa: E402
|
||||
|
||||
EXPECTED_PROFILES = {
|
||||
"adaptive-engineering",
|
||||
"access-efficient-escalation-lab",
|
||||
"bounded-research-organization-lab",
|
||||
"codex-harness-team",
|
||||
"competing-implementations-lab",
|
||||
"contract-first-refactoring",
|
||||
"high-confidence-debugging",
|
||||
"incident-hypothesis-triage",
|
||||
"research-backed-engineering",
|
||||
"route-resilience-lab",
|
||||
"secure-change",
|
||||
"visual-engineering",
|
||||
}
|
||||
EXPECTED_ORCHESTRATION = {"mcp", "hybrid"}
|
||||
EXPECTED_EVALUATION_SUITES = {
|
||||
"access-efficient",
|
||||
"adaptive-change",
|
||||
"bounded-research",
|
||||
"codex-harness",
|
||||
"competing-implementations",
|
||||
"contract-refactoring",
|
||||
"debugging-confidence",
|
||||
"incident-triage",
|
||||
"research-currentness",
|
||||
"route-resilience",
|
||||
"security-assurance",
|
||||
"visual-conformance",
|
||||
}
|
||||
REQUIRED_ROOT_FILES = {
|
||||
"LICENSE",
|
||||
"VERSION",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"VALIDATION.md",
|
||||
"Makefile",
|
||||
"PACKAGE-MANIFEST.json",
|
||||
"install.sh",
|
||||
"uninstall.sh",
|
||||
}
|
||||
REQUIRED_DOCS = {
|
||||
"ACCEPTANCE.md",
|
||||
"ARCHITECTURE.md",
|
||||
"CATALOG.md",
|
||||
"CLI.md",
|
||||
"EVALUATION.md",
|
||||
"EXTERNAL-VERACITY.md",
|
||||
"INSTALLATION.md",
|
||||
"OPTIMIZATION.md",
|
||||
"PLAN-COVERAGE.md",
|
||||
"ORCHESTRATION_BACKENDS.md",
|
||||
"PROFILE_SCHEMA.md",
|
||||
"PROFILES.md",
|
||||
"PROVIDER_DRIVERS.md",
|
||||
"SECURITY.md",
|
||||
"TROUBLESHOOTING.md",
|
||||
"TOOL_MCP.md",
|
||||
}
|
||||
IGNORED_PARTS = {
|
||||
".git",
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
"dist",
|
||||
"build",
|
||||
}
|
||||
TEXT_SUFFIXES = {
|
||||
"",
|
||||
".md",
|
||||
".txt",
|
||||
".toml",
|
||||
".json",
|
||||
".py",
|
||||
".sh",
|
||||
".in",
|
||||
".example",
|
||||
".gitignore",
|
||||
".license",
|
||||
}
|
||||
SECRET_PATTERNS = {
|
||||
"private key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA )?PRIVATE KEY-----"),
|
||||
"OpenAI-style secret": re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
|
||||
"GitHub token": re.compile(r"\bgh[pousr]_[A-Za-z0-9]{30,}\b"),
|
||||
"AWS access key": re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
started_at: str = field(default_factory=utc_now)
|
||||
checks: dict[str, Any] = field(default_factory=dict)
|
||||
errors: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
def check(self, name: str, passed: bool, **details: Any) -> None:
|
||||
self.checks[name] = {"passed": bool(passed), **details}
|
||||
if not passed:
|
||||
self.errors.append(f"{name} failed")
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
self.errors.append(message)
|
||||
|
||||
def warning(self, message: str) -> None:
|
||||
self.warnings.append(message)
|
||||
|
||||
def finish(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": "codex-multimodel-orchestrator",
|
||||
"version": (ROOT / "VERSION").read_text(encoding="utf-8").strip(),
|
||||
"started_at": self.started_at,
|
||||
"finished_at": utc_now(),
|
||||
"passed": not self.errors and all(item.get("passed") for item in self.checks.values()),
|
||||
"checks": self.checks,
|
||||
"warnings": self.warnings,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
|
||||
def _source_files() -> list[Path]:
|
||||
files: list[Path] = []
|
||||
for path in ROOT.rglob("*"):
|
||||
if any(part in IGNORED_PARTS for part in path.relative_to(ROOT).parts):
|
||||
continue
|
||||
if path.is_file() and not path.is_symlink():
|
||||
files.append(path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def _digest(paths: Iterable[Path]) -> dict[str, str]:
|
||||
return {path.relative_to(ROOT).as_posix(): sha256_file(path) for path in sorted(paths)}
|
||||
|
||||
|
||||
def _check_structure(report: Report) -> None:
|
||||
missing_root = sorted(name for name in REQUIRED_ROOT_FILES if not (ROOT / name).is_file())
|
||||
missing_docs = sorted(name for name in REQUIRED_DOCS if not (ROOT / "docs" / name).is_file())
|
||||
report.check(
|
||||
"required_files",
|
||||
not missing_root and not missing_docs,
|
||||
missing_root=missing_root,
|
||||
missing_docs=missing_docs,
|
||||
)
|
||||
|
||||
caches = sorted(
|
||||
path.relative_to(ROOT).as_posix()
|
||||
for path in ROOT.rglob("*")
|
||||
if path.name == "__pycache__" or path.suffix in {".pyc", ".pyo"}
|
||||
)
|
||||
report.check("release_hygiene_no_bytecode", not caches, paths=caches)
|
||||
|
||||
symlinks = sorted(
|
||||
path.relative_to(ROOT).as_posix() for path in ROOT.rglob("*") if path.is_symlink()
|
||||
)
|
||||
report.check("release_hygiene_no_symlinks", not symlinks, paths=symlinks)
|
||||
|
||||
special = sorted(
|
||||
path.relative_to(ROOT).as_posix()
|
||||
for path in ROOT.rglob("*")
|
||||
if not any(part in IGNORED_PARTS for part in path.relative_to(ROOT).parts)
|
||||
and not (
|
||||
stat.S_ISREG(path.lstat().st_mode)
|
||||
or stat.S_ISDIR(path.lstat().st_mode)
|
||||
or stat.S_ISLNK(path.lstat().st_mode)
|
||||
)
|
||||
)
|
||||
report.check("release_hygiene_no_special_files", not special, paths=special)
|
||||
|
||||
|
||||
def _check_source_manifest(report: Report) -> None:
|
||||
path = ROOT / "PACKAGE-MANIFEST.json"
|
||||
errors: list[str] = []
|
||||
try:
|
||||
manifest = strict_json_loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
||||
report.check("source_manifest", False, errors=[f"unable to read manifest: {exc}"])
|
||||
return
|
||||
if not isinstance(manifest, dict):
|
||||
report.check("source_manifest", False, errors=["manifest root must be an object"])
|
||||
return
|
||||
expected_manifest_fields = {
|
||||
"schema_version",
|
||||
"package",
|
||||
"version",
|
||||
"source_date_epoch",
|
||||
"manifest_excludes",
|
||||
"file_count",
|
||||
"total_bytes",
|
||||
"files",
|
||||
}
|
||||
if set(manifest) != expected_manifest_fields:
|
||||
errors.append("manifest root has missing or unexpected fields")
|
||||
schema_version = manifest.get("schema_version")
|
||||
if (
|
||||
not isinstance(schema_version, int)
|
||||
or isinstance(schema_version, bool)
|
||||
or schema_version != MMO_SCHEMA_VERSION
|
||||
):
|
||||
errors.append(f"schema_version must be {MMO_SCHEMA_VERSION}")
|
||||
if manifest.get("package") != "codex-multimodel-orchestrator":
|
||||
errors.append("package identity is invalid")
|
||||
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
||||
if manifest.get("version") != version:
|
||||
errors.append("manifest version does not match VERSION")
|
||||
if manifest.get("manifest_excludes") != ["PACKAGE-MANIFEST.json"]:
|
||||
errors.append("manifest_excludes is invalid")
|
||||
epoch = manifest.get("source_date_epoch")
|
||||
if not isinstance(epoch, int) or isinstance(epoch, bool) or not 0 <= epoch <= 0xFFFFFFFF:
|
||||
errors.append("source_date_epoch is invalid")
|
||||
entries = manifest.get("files")
|
||||
expected: dict[str, dict[str, Any]] = {}
|
||||
if not isinstance(entries, list):
|
||||
errors.append("files must be an array")
|
||||
else:
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):
|
||||
errors.append("files contains an invalid entry")
|
||||
continue
|
||||
relative = entry["path"]
|
||||
if set(entry) != {"path", "mode", "size", "sha256"}:
|
||||
errors.append(f"manifest entry has missing or unexpected fields: {relative}")
|
||||
if entry.get("mode") not in {"0644", "0755"}:
|
||||
errors.append(f"manifest entry has an invalid mode: {relative}")
|
||||
size = entry.get("size")
|
||||
if not isinstance(size, int) or isinstance(size, bool) or size < 0:
|
||||
errors.append(f"manifest entry has an invalid size: {relative}")
|
||||
digest = entry.get("sha256")
|
||||
if (
|
||||
not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
errors.append(f"manifest entry has an invalid hash: {relative}")
|
||||
if relative in expected:
|
||||
errors.append(f"duplicate manifest path: {relative}")
|
||||
continue
|
||||
expected[relative] = {
|
||||
"mode": entry.get("mode"),
|
||||
"size": entry.get("size"),
|
||||
"sha256": entry.get("sha256"),
|
||||
}
|
||||
actual: dict[str, dict[str, Any]] = {}
|
||||
for source in _source_files():
|
||||
relative = source.relative_to(ROOT).as_posix()
|
||||
if relative == "PACKAGE-MANIFEST.json":
|
||||
continue
|
||||
mode = "0755" if source.stat().st_mode & stat.S_IXUSR else "0644"
|
||||
actual[relative] = {
|
||||
"mode": mode,
|
||||
"size": source.stat().st_size,
|
||||
"sha256": sha256_file(source),
|
||||
}
|
||||
if expected != actual:
|
||||
errors.append("manifest file inventory differs from the source tree")
|
||||
file_count = manifest.get("file_count")
|
||||
if not isinstance(file_count, int) or isinstance(file_count, bool) or file_count != len(actual):
|
||||
errors.append("file_count differs from the source tree")
|
||||
total_bytes = manifest.get("total_bytes")
|
||||
if (
|
||||
not isinstance(total_bytes, int)
|
||||
or isinstance(total_bytes, bool)
|
||||
or total_bytes != sum(item["size"] for item in actual.values())
|
||||
):
|
||||
errors.append("total_bytes differs from the source tree")
|
||||
report.check(
|
||||
"source_manifest",
|
||||
not errors,
|
||||
errors=errors,
|
||||
expected_files=len(expected),
|
||||
actual_files=len(actual),
|
||||
missing=sorted(set(actual) - set(expected)),
|
||||
extra=sorted(set(expected) - set(actual)),
|
||||
changed=sorted(key for key in set(expected) & set(actual) if expected[key] != actual[key]),
|
||||
)
|
||||
|
||||
|
||||
def _check_syntax(report: Report) -> None:
|
||||
python_errors: list[str] = []
|
||||
python_files = [path for path in _source_files() if path.suffix == ".py"]
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-pycompile-") as temporary:
|
||||
compiled = Path(temporary)
|
||||
for index, path in enumerate(python_files):
|
||||
try:
|
||||
ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
py_compile.compile(str(path), cfile=str(compiled / f"{index}.pyc"), doraise=True)
|
||||
except Exception as exc: # validation boundary
|
||||
python_errors.append(f"{path.relative_to(ROOT)}: {type(exc).__name__}: {exc}")
|
||||
report.check("python_syntax", not python_errors, files=len(python_files), errors=python_errors)
|
||||
|
||||
shell_errors: list[str] = []
|
||||
shell_files = [path for path in _source_files() if path.suffix in {".sh", ".in"}]
|
||||
for path in shell_files:
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(path)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
env=filtered_environment(),
|
||||
)
|
||||
if result.returncode:
|
||||
shell_errors.append(f"{path.relative_to(ROOT)}: {result.stderr.strip()}")
|
||||
report.check("shell_syntax", not shell_errors, files=len(shell_files), errors=shell_errors)
|
||||
|
||||
parse_errors: list[str] = []
|
||||
toml_files = [path for path in _source_files() if path.suffix == ".toml"]
|
||||
json_files = [path for path in _source_files() if path.suffix == ".json"]
|
||||
for path in toml_files:
|
||||
try:
|
||||
read_toml(path)
|
||||
except Exception as exc:
|
||||
parse_errors.append(f"{path.relative_to(ROOT)}: {type(exc).__name__}: {exc}")
|
||||
for path in json_files:
|
||||
try:
|
||||
strict_json_loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
parse_errors.append(f"{path.relative_to(ROOT)}: {type(exc).__name__}: {exc}")
|
||||
report.check(
|
||||
"configuration_syntax",
|
||||
not parse_errors,
|
||||
toml_files=len(toml_files),
|
||||
json_files=len(json_files),
|
||||
errors=parse_errors,
|
||||
)
|
||||
|
||||
|
||||
def _check_static_safety(report: Report) -> None:
|
||||
unresolved: list[str] = []
|
||||
markers: list[str] = []
|
||||
secrets: list[str] = []
|
||||
executable_profile_files: list[str] = []
|
||||
for path in _source_files():
|
||||
relative = path.relative_to(ROOT)
|
||||
if relative.parts and relative.parts[0] == "profiles" and os.access(path, os.X_OK):
|
||||
executable_profile_files.append(relative.as_posix())
|
||||
suffix = path.suffix.lower()
|
||||
if suffix not in TEXT_SUFFIXES and path.name not in {
|
||||
"Makefile",
|
||||
".gitignore",
|
||||
"LICENSE",
|
||||
"VERSION",
|
||||
}:
|
||||
continue
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
# The validator contains the literal patterns it searches for. Exclude
|
||||
# this file from content-marker scans so it does not report itself.
|
||||
if path.name != "validate_package.py":
|
||||
if "@@" in text and not (
|
||||
relative.parts and relative.parts[0] == "bin" and path.suffix == ".in"
|
||||
):
|
||||
unresolved.append(relative.as_posix())
|
||||
if re.search(r"(?m)\b(?:TODO|FIXME|XXX|HACK)\b", text):
|
||||
markers.append(relative.as_posix())
|
||||
for label, pattern in SECRET_PATTERNS.items():
|
||||
if pattern.search(text):
|
||||
secrets.append(f"{relative.as_posix()}: {label}")
|
||||
report.check("no_unresolved_placeholders", not unresolved, paths=unresolved)
|
||||
report.check("no_development_markers", not markers, paths=markers)
|
||||
report.check("secret_scan", not secrets, findings=secrets)
|
||||
report.check(
|
||||
"profile_packs_are_non_executable",
|
||||
not executable_profile_files,
|
||||
paths=executable_profile_files,
|
||||
)
|
||||
|
||||
|
||||
def _sandbox_environment(temporary: Path) -> dict[str, str]:
|
||||
config = temporary / "config"
|
||||
state = temporary / "state"
|
||||
base_home = temporary / "codex-home"
|
||||
for path in (
|
||||
config / "profiles.d",
|
||||
config / "catalog.d",
|
||||
config / "tool-mcp.d",
|
||||
state,
|
||||
base_home,
|
||||
):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
settings = (ROOT / "config" / "settings.toml").read_text(encoding="utf-8")
|
||||
settings = settings.replace(
|
||||
'base_codex_home = "~/.codex"', f"base_codex_home = {json.dumps(str(base_home))}"
|
||||
)
|
||||
(config / "settings.toml").write_text(settings, encoding="utf-8")
|
||||
(config / "credentials.env").write_text("", encoding="utf-8")
|
||||
return {
|
||||
**filtered_environment(),
|
||||
"MMO_INSTALL_ROOT": str(ROOT),
|
||||
"MMO_CONFIG_ROOT": str(config),
|
||||
"MMO_STATE_ROOT": str(state),
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONPATH": os.pathsep.join((str(ROOT / "libexec"), str(ROOT / "tests"))),
|
||||
}
|
||||
|
||||
|
||||
def _check_profiles_and_catalog(report: Report) -> None:
|
||||
previous_environment = dict(os.environ)
|
||||
try:
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-acceptance-") as temporary_raw:
|
||||
env = _sandbox_environment(Path(temporary_raw))
|
||||
os.environ.clear()
|
||||
os.environ.update(env)
|
||||
|
||||
profiles = discover_profiles()
|
||||
profile_ids = set(profiles)
|
||||
profile_errors: list[str] = []
|
||||
modes: set[str] = set()
|
||||
native_count = 0
|
||||
mcp_count = 0
|
||||
hybrid_count = 0
|
||||
snapshots: dict[str, str] = {}
|
||||
low_trust: list[str] = []
|
||||
for profile_id in sorted(profile_ids):
|
||||
directory = Path(profiles[profile_id]["path"])
|
||||
tree_errors = validate_profile_pack_tree(directory)
|
||||
if tree_errors:
|
||||
profile_errors.extend(f"{profile_id}: {item}" for item in tree_errors)
|
||||
continue
|
||||
try:
|
||||
resolved = resolve_profile(profile_id)
|
||||
snapshot = compile_profile(profile_id)
|
||||
except Exception as exc:
|
||||
profile_errors.append(f"{profile_id}: {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
snapshots[profile_id] = snapshot["manifest"]["snapshot_hash"]
|
||||
mode = resolved["coordination"]["orchestration"]
|
||||
modes.add(mode)
|
||||
native_count += len(resolved["capabilities"]["native_agents"])
|
||||
mcp_count += len(resolved["capabilities"]["mcp_agents"])
|
||||
hybrid_count += int(mode == "hybrid")
|
||||
root = resolved["profile"]["root"]
|
||||
if resolved["agents"][root]["kind"] != "root":
|
||||
profile_errors.append(f"{profile_id}: configured root is not kind=root")
|
||||
if not resolved.get("smoke", {}).get("tasks"):
|
||||
profile_errors.append(f"{profile_id}: no smoke tasks")
|
||||
for agent_id, agent in resolved["agents"].items():
|
||||
if agent["trust"] == "low":
|
||||
low_trust.append(f"{profile_id}:{agent_id}")
|
||||
if not (
|
||||
agent["permissions"] == "read-only"
|
||||
and agent["verification"] == "always"
|
||||
and agent["backends"] == ["mcp"]
|
||||
and agent["output_contract_schema"] is not None
|
||||
):
|
||||
profile_errors.append(
|
||||
f"{profile_id}:{agent_id}: low-trust boundary is not mechanical"
|
||||
)
|
||||
report.check(
|
||||
"profile_inventory",
|
||||
profile_ids == EXPECTED_PROFILES,
|
||||
expected=sorted(EXPECTED_PROFILES),
|
||||
actual=sorted(profile_ids),
|
||||
)
|
||||
report.check(
|
||||
"profile_semantics",
|
||||
not profile_errors
|
||||
and modes == EXPECTED_ORCHESTRATION
|
||||
and native_count > 0
|
||||
and mcp_count > 0
|
||||
and hybrid_count > 0
|
||||
and bool(low_trust),
|
||||
errors=profile_errors,
|
||||
orchestration_modes=sorted(modes),
|
||||
native_agents=native_count,
|
||||
mcp_agents=mcp_count,
|
||||
hybrid_profiles=hybrid_count,
|
||||
low_trust_agents=low_trust,
|
||||
snapshots=snapshots,
|
||||
)
|
||||
|
||||
summary = catalog_summary()
|
||||
inventory = local_inventory_report()
|
||||
inventory_counts = {
|
||||
key: value["actual_count"] for key, value in inventory["inventories"].items()
|
||||
}
|
||||
catalog_ok = (
|
||||
inventory["passed"]
|
||||
and summary["routes"] == 21
|
||||
and summary["models"] == 560
|
||||
and summary["agent_compatible_models"] == 462
|
||||
and summary["resources"] == 9
|
||||
and inventory_counts.get("opencode-go") == 29
|
||||
and inventory_counts.get("opencode-zen") == 64
|
||||
and inventory_counts.get("openrouter") == 422
|
||||
and inventory_counts.get("zai-api") == 35
|
||||
and inventory_counts.get("zai-coding-plan") == 3
|
||||
and inventory_counts.get("openai-codex") == 6
|
||||
)
|
||||
report.check(
|
||||
"complete_model_catalog",
|
||||
catalog_ok,
|
||||
summary=summary,
|
||||
inventory=inventory,
|
||||
)
|
||||
|
||||
suite_results = {suite: validate_suite(suite) for suite in discover_suites()}
|
||||
report.check(
|
||||
"evaluation_suites",
|
||||
set(suite_results) == EXPECTED_EVALUATION_SUITES
|
||||
and all(item["valid"] for item in suite_results.values()),
|
||||
expected=sorted(EXPECTED_EVALUATION_SUITES),
|
||||
suites=suite_results,
|
||||
)
|
||||
finally:
|
||||
os.environ.clear()
|
||||
os.environ.update(previous_environment)
|
||||
|
||||
|
||||
def _check_catalog_reproducibility(report: Report) -> None:
|
||||
tracked = [ROOT / "config" / "catalog.toml", ROOT / "config" / "upstream-inventory.json"]
|
||||
before = _digest(tracked)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(ROOT / "scripts" / "generate_catalog.py"), "--check"],
|
||||
cwd=ROOT,
|
||||
env=filtered_environment(extra={"PYTHONDONTWRITEBYTECODE": "1"}),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
after = _digest(tracked)
|
||||
report.check(
|
||||
"catalog_generation_is_reproducible",
|
||||
result.returncode == 0 and before == after,
|
||||
exit_code=result.returncode,
|
||||
changed=sorted(
|
||||
key for key in set(before) | set(after) if before.get(key) != after.get(key)
|
||||
),
|
||||
stdout=result.stdout[-4000:],
|
||||
stderr=result.stderr[-4000:],
|
||||
)
|
||||
|
||||
|
||||
def _run_tests(report: Report) -> None:
|
||||
started = time.monotonic()
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, str(ROOT / "scripts" / "run_tests.py")],
|
||||
cwd=ROOT,
|
||||
env=filtered_environment(
|
||||
extra={
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONWARNINGS": "error::ResourceWarning",
|
||||
"PYTHONPATH": os.pathsep.join((str(ROOT / "libexec"), str(ROOT / "tests"))),
|
||||
}
|
||||
),
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
start_new_session=True,
|
||||
)
|
||||
timed_out = False
|
||||
try:
|
||||
stdout, stderr = process.communicate(timeout=2100)
|
||||
except subprocess.TimeoutExpired:
|
||||
timed_out = True
|
||||
terminate_process_group(process.pid)
|
||||
stdout, stderr = process.communicate()
|
||||
report.check(
|
||||
"offline_integration_tests",
|
||||
process.returncode == 0 and not timed_out,
|
||||
exit_code=process.returncode,
|
||||
timed_out=timed_out,
|
||||
duration_seconds=time.monotonic() - started,
|
||||
stdout=stdout[-12000:],
|
||||
stderr=stderr[-12000:],
|
||||
)
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Validate a Codex MMO source/release tree")
|
||||
parser.add_argument(
|
||||
"--skip-tests", action="store_true", help="skip the offline integration suite"
|
||||
)
|
||||
parser.add_argument("--report", type=Path, help="write the full JSON report to this path")
|
||||
parser.add_argument("--json", action="store_true", help="emit the full JSON report on stdout")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
report = Report()
|
||||
_check_structure(report)
|
||||
_check_source_manifest(report)
|
||||
_check_syntax(report)
|
||||
_check_static_safety(report)
|
||||
_check_profiles_and_catalog(report)
|
||||
_check_catalog_reproducibility(report)
|
||||
if not args.skip_tests:
|
||||
_run_tests(report)
|
||||
result = report.finish()
|
||||
if args.report:
|
||||
atomic_write_json(args.report, result, 0o644)
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False))
|
||||
else:
|
||||
state = "PASS" if result["passed"] else "FAIL"
|
||||
print(f"Codex MMO package validation: {state}")
|
||||
for name, item in result["checks"].items():
|
||||
print(f" {'PASS' if item['passed'] else 'FAIL'} {name}")
|
||||
for warning in result["warnings"]:
|
||||
print(f" WARN {warning}")
|
||||
for error in result["errors"]:
|
||||
print(f" ERROR {error}")
|
||||
if args.report:
|
||||
print(f"Report: {args.report}")
|
||||
return 0 if result["passed"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+609
@@ -0,0 +1,609 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify Codex MMO release archives by safe extraction and manifest checks.
|
||||
|
||||
The verifier is deliberately independent from the archive builder. It rejects
|
||||
traversal, links, special files, duplicate members, incomplete package trees,
|
||||
extra files, missing files, hash mismatches, size mismatches, and mode drift.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_version import MMO_SCHEMA_VERSION # noqa: E402
|
||||
|
||||
REQUIRED_FILES = {
|
||||
"VERSION",
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"install.sh",
|
||||
"uninstall.sh",
|
||||
"config/catalog.toml",
|
||||
"config/inventory-snapshots/openai-codex.json",
|
||||
"config/inventory-snapshots/opencode-go.json",
|
||||
"config/inventory-snapshots/opencode-zen.json",
|
||||
"config/inventory-snapshots/openrouter.json",
|
||||
"config/inventory-snapshots/zai-api.json",
|
||||
"config/inventory-snapshots/zai-coding-plan.json",
|
||||
"scripts/build_release.py",
|
||||
"scripts/build_inventory_snapshot.py",
|
||||
"scripts/validate_package.py",
|
||||
"scripts/verify_release.py",
|
||||
"libexec/mmo_app_server.py",
|
||||
"libexec/mmo_catalog_data.py",
|
||||
"libexec/mmo_codex_home.py",
|
||||
"libexec/mmo_diagnostics.py",
|
||||
"libexec/mmo_runtime.py",
|
||||
"libexec/mmo_state.py",
|
||||
"libexec/mmo_workspace.py",
|
||||
"libexec/mmo_inventory_snapshot.py",
|
||||
"libexec/mmo_mcp.py",
|
||||
"libexec/mmo_tool_mcp.py",
|
||||
"libexec/root_runner.py",
|
||||
"libexec/worker_runner.py",
|
||||
"profiles/README.md",
|
||||
}
|
||||
REQUIRED_DIRECTORIES = {"bin", "config", "docs", "evals", "libexec", "profiles", "scripts", "tests"}
|
||||
MAX_ARCHIVE_MEMBERS = 20_000
|
||||
MAX_ARCHIVE_MEMBER_BYTES = 512 * 1024 * 1024
|
||||
MAX_EXPANDED_BYTES = 1_000_000_000
|
||||
MAX_ZIP_COMPRESSION_RATIO = 1_000
|
||||
|
||||
|
||||
def _reject_json_constant(value: str) -> Any:
|
||||
raise ValueError(f"non-standard JSON constant: {value}")
|
||||
|
||||
|
||||
def _finite_json_float(value: str) -> float:
|
||||
parsed = float(value)
|
||||
if not math.isfinite(parsed):
|
||||
raise ValueError(f"JSON number is outside the supported finite range: {value}")
|
||||
return parsed
|
||||
|
||||
|
||||
def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, item in pairs:
|
||||
if key in result:
|
||||
raise ValueError(f"duplicate JSON object member: {key!r}")
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def _validate_json_unicode(value: Any) -> None:
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
except UnicodeEncodeError as exc:
|
||||
raise ValueError("JSON strings must contain valid Unicode scalar values") from exc
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
_validate_json_unicode(item)
|
||||
elif isinstance(value, dict):
|
||||
for key, item in value.items():
|
||||
_validate_json_unicode(key)
|
||||
_validate_json_unicode(item)
|
||||
|
||||
|
||||
def _strict_json_loads(value: str) -> Any:
|
||||
parsed = json.loads(
|
||||
value,
|
||||
parse_constant=_reject_json_constant,
|
||||
parse_float=_finite_json_float,
|
||||
object_pairs_hook=_unique_json_object,
|
||||
)
|
||||
_validate_json_unicode(parsed)
|
||||
return parsed
|
||||
|
||||
|
||||
class VerificationError(RuntimeError):
|
||||
"""Raised when a release archive fails integrity verification."""
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _safe_member_path(name: str) -> PurePosixPath:
|
||||
if not name or "\x00" in name or "\\" in name or name.startswith("/"):
|
||||
raise VerificationError(f"unsafe archive member path: {name!r}")
|
||||
value = name[:-1] if name.endswith("/") else name
|
||||
parts = value.split("/")
|
||||
if not parts or any(part in {"", ".", ".."} for part in parts):
|
||||
raise VerificationError(f"unsafe archive member path: {name!r}")
|
||||
return PurePosixPath(*parts)
|
||||
|
||||
|
||||
def _ensure_member_limits(names: Iterable[str], sizes: Iterable[int]) -> None:
|
||||
names_list = list(names)
|
||||
sizes_list = list(sizes)
|
||||
if len(names_list) != len(sizes_list):
|
||||
raise VerificationError("archive member metadata is inconsistent")
|
||||
if len(names_list) > MAX_ARCHIVE_MEMBERS:
|
||||
raise VerificationError(f"archive contains too many members: {len(names_list)}")
|
||||
if any(size < 0 for size in sizes_list):
|
||||
raise VerificationError("archive contains a negative member size")
|
||||
expanded = sum(sizes_list)
|
||||
if expanded > MAX_EXPANDED_BYTES:
|
||||
raise VerificationError(f"archive expands beyond safety limit: {expanded} bytes")
|
||||
if len(set(names_list)) != len(names_list):
|
||||
raise VerificationError("archive contains duplicate member names")
|
||||
|
||||
|
||||
def _copy_limited(source: Any, output: Any, maximum: int, label: str) -> int:
|
||||
copied = 0
|
||||
while True:
|
||||
block = source.read(min(1024 * 1024, maximum - copied + 1))
|
||||
if not block:
|
||||
return copied
|
||||
copied += len(block)
|
||||
if copied > maximum:
|
||||
raise VerificationError(f"archive member exceeds size limit: {label}")
|
||||
output.write(block)
|
||||
|
||||
|
||||
def _register_member(
|
||||
seen: dict[str, bool], relative: PurePosixPath, *, is_directory: bool, label: str
|
||||
) -> None:
|
||||
key = relative.as_posix()
|
||||
if key in seen:
|
||||
raise VerificationError(f"archive aliases the same extraction path: {label}")
|
||||
parts = relative.parts
|
||||
for index in range(1, len(parts)):
|
||||
parent = "/".join(parts[:index])
|
||||
if seen.get(parent) is False:
|
||||
raise VerificationError(f"archive places a member beneath a file: {label}")
|
||||
if not is_directory and any(existing.startswith(f"{key}/") for existing in seen):
|
||||
raise VerificationError(f"archive replaces a populated directory with a file: {label}")
|
||||
seen[key] = is_directory
|
||||
|
||||
|
||||
def _validate_explicit_directories(seen: dict[str, bool]) -> None:
|
||||
"""Require exactly one explicit entry for every payload parent directory."""
|
||||
|
||||
expected: set[str] = set()
|
||||
for name, is_directory in seen.items():
|
||||
if is_directory:
|
||||
continue
|
||||
parts = name.split("/")
|
||||
expected.update("/".join(parts[:index]) for index in range(1, len(parts)))
|
||||
actual = {name for name, is_directory in seen.items() if is_directory}
|
||||
missing = sorted(expected - actual)
|
||||
extra = sorted(actual - expected)
|
||||
if missing or extra:
|
||||
raise VerificationError(
|
||||
f"archive directory set differs from payload parents; missing={missing}, extra={extra}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_permission_mode(
|
||||
archive_type: str, label: str, mode: int, *, is_directory: bool
|
||||
) -> None:
|
||||
allowed = {0o755} if is_directory else {0o644, 0o755}
|
||||
if mode not in allowed:
|
||||
expected = "0755" if is_directory else "0644 or 0755"
|
||||
raise VerificationError(
|
||||
f"{archive_type} member has invalid permission mode: "
|
||||
f"{label}: {mode:04o}; expected {expected}"
|
||||
)
|
||||
|
||||
|
||||
def _extract_zip(archive_path: Path, destination: Path) -> None:
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
infos = archive.infolist()
|
||||
_ensure_member_limits((item.filename for item in infos), (item.file_size for item in infos))
|
||||
relative_paths = [_safe_member_path(item.filename) for item in infos]
|
||||
canonical_names = [path.as_posix() for path in relative_paths]
|
||||
if len(set(canonical_names)) != len(canonical_names):
|
||||
raise VerificationError("archive aliases the same extraction path")
|
||||
seen: dict[str, bool] = {}
|
||||
for item, relative in zip(infos, relative_paths, strict=True):
|
||||
if item.create_system != 3:
|
||||
raise VerificationError(f"ZIP member lacks Unix mode metadata: {item.filename}")
|
||||
unix_mode = (item.external_attr >> 16) & 0xFFFF
|
||||
file_type = stat.S_IFMT(unix_mode)
|
||||
if file_type not in {stat.S_IFREG, stat.S_IFDIR}:
|
||||
raise VerificationError(f"ZIP contains a link or special file: {item.filename}")
|
||||
is_directory = file_type == stat.S_IFDIR
|
||||
if item.is_dir() != is_directory:
|
||||
raise VerificationError(
|
||||
f"ZIP member path and type metadata disagree: {item.filename}"
|
||||
)
|
||||
permission_mode = unix_mode & 0o7777
|
||||
_validate_permission_mode(
|
||||
"ZIP", item.filename, permission_mode, is_directory=is_directory
|
||||
)
|
||||
_register_member(seen, relative, is_directory=is_directory, label=item.filename)
|
||||
if item.flag_bits & 0x1:
|
||||
raise VerificationError(f"ZIP contains an encrypted member: {item.filename}")
|
||||
if item.file_size > MAX_ARCHIVE_MEMBER_BYTES:
|
||||
raise VerificationError(f"ZIP member exceeds size limit: {item.filename}")
|
||||
if not is_directory and item.file_size and item.compress_size == 0:
|
||||
raise VerificationError(f"ZIP member has invalid compressed size: {item.filename}")
|
||||
if (
|
||||
item.compress_size > 0
|
||||
and item.file_size / item.compress_size > MAX_ZIP_COMPRESSION_RATIO
|
||||
):
|
||||
raise VerificationError(
|
||||
f"ZIP member has suspicious compression ratio: {item.filename}"
|
||||
)
|
||||
target = destination.joinpath(*relative.parts)
|
||||
if is_directory:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(target, permission_mode)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with archive.open(item) as source, target.open("wb") as output:
|
||||
copied = _copy_limited(source, output, MAX_ARCHIVE_MEMBER_BYTES, item.filename)
|
||||
if copied != item.file_size:
|
||||
raise VerificationError(f"ZIP member size mismatch: {item.filename}")
|
||||
os.chmod(target, permission_mode)
|
||||
_validate_explicit_directories(seen)
|
||||
|
||||
|
||||
def _extract_tar(archive_path: Path, destination: Path) -> None:
|
||||
with tarfile.open(archive_path, mode="r|gz") as archive:
|
||||
seen: dict[str, bool] = {}
|
||||
member_count = 0
|
||||
expanded_bytes = 0
|
||||
for item in archive:
|
||||
member_count += 1
|
||||
if member_count > MAX_ARCHIVE_MEMBERS:
|
||||
raise VerificationError(f"archive contains too many members: {member_count}")
|
||||
if item.size < 0:
|
||||
raise VerificationError(f"tar member has a negative size: {item.name}")
|
||||
expanded_bytes += item.size
|
||||
if expanded_bytes > MAX_EXPANDED_BYTES:
|
||||
raise VerificationError(
|
||||
f"archive expands beyond safety limit: {expanded_bytes} bytes"
|
||||
)
|
||||
relative = _safe_member_path(item.name)
|
||||
if not (item.isdir() or item.isfile()):
|
||||
raise VerificationError(f"tar contains a link or special file: {item.name}")
|
||||
permission_mode = item.mode & 0o7777
|
||||
if item.mode != permission_mode:
|
||||
raise VerificationError(f"tar member has invalid mode metadata: {item.name}")
|
||||
_validate_permission_mode("tar", item.name, permission_mode, is_directory=item.isdir())
|
||||
_register_member(seen, relative, is_directory=item.isdir(), label=item.name)
|
||||
if item.size > MAX_ARCHIVE_MEMBER_BYTES:
|
||||
raise VerificationError(f"tar member exceeds size limit: {item.name}")
|
||||
target = destination.joinpath(*relative.parts)
|
||||
if item.isdir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
os.chmod(target, permission_mode)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
source = archive.extractfile(item)
|
||||
if source is None:
|
||||
raise VerificationError(f"unable to read tar member: {item.name}")
|
||||
with source, target.open("wb") as output:
|
||||
copied = _copy_limited(source, output, MAX_ARCHIVE_MEMBER_BYTES, item.name)
|
||||
if copied != item.size:
|
||||
raise VerificationError(f"tar member size mismatch: {item.name}")
|
||||
os.chmod(target, permission_mode)
|
||||
_validate_explicit_directories(seen)
|
||||
|
||||
|
||||
def _source_inventory(source_tree: Path) -> dict[str, dict[str, Any]]:
|
||||
excluded_parts = {
|
||||
".git",
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
"dist",
|
||||
"build",
|
||||
}
|
||||
inventory: dict[str, dict[str, Any]] = {}
|
||||
for path in sorted(source_tree.rglob("*")):
|
||||
relative = path.relative_to(source_tree)
|
||||
if any(part in excluded_parts for part in relative.parts):
|
||||
continue
|
||||
mode_bits = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode_bits):
|
||||
raise VerificationError(f"source tree contains a symbolic link: {relative}")
|
||||
if not (stat.S_ISREG(mode_bits) or stat.S_ISDIR(mode_bits)):
|
||||
raise VerificationError(f"source tree contains a special file: {relative}")
|
||||
if (
|
||||
relative.as_posix() == "PACKAGE-MANIFEST.json"
|
||||
or not path.is_file()
|
||||
or path.suffix in {".pyc", ".pyo"}
|
||||
):
|
||||
continue
|
||||
mode = 0o755 if path.stat().st_mode & stat.S_IXUSR else 0o644
|
||||
inventory[relative.as_posix()] = {
|
||||
"size": path.stat().st_size,
|
||||
"sha256": sha256(path),
|
||||
"mode": f"{mode:04o}",
|
||||
}
|
||||
return inventory
|
||||
|
||||
|
||||
def _tree_hash(package_root: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
files = sorted(
|
||||
(path for path in package_root.rglob("*") if path.is_file() and not path.is_symlink()),
|
||||
key=lambda item: item.relative_to(package_root).as_posix(),
|
||||
)
|
||||
for path in files:
|
||||
relative = path.relative_to(package_root).as_posix().encode("utf-8")
|
||||
mode = ("0755" if path.stat().st_mode & stat.S_IXUSR else "0644").encode("ascii")
|
||||
size = str(path.stat().st_size).encode("ascii")
|
||||
for value in (relative, mode, size):
|
||||
digest.update(len(value).to_bytes(8, "big"))
|
||||
digest.update(value)
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _verify_extracted(package_root: Path, source_tree: Path | None = None) -> dict[str, Any]:
|
||||
manifest_path = package_root / "PACKAGE-MANIFEST.json"
|
||||
if not manifest_path.is_file():
|
||||
raise VerificationError("PACKAGE-MANIFEST.json is missing")
|
||||
try:
|
||||
manifest = _strict_json_loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, ValueError) as exc:
|
||||
raise VerificationError(f"invalid package manifest: {exc}") from exc
|
||||
if not isinstance(manifest, dict):
|
||||
raise VerificationError("package manifest root must be an object")
|
||||
expected_manifest_fields = {
|
||||
"schema_version",
|
||||
"package",
|
||||
"version",
|
||||
"source_date_epoch",
|
||||
"manifest_excludes",
|
||||
"file_count",
|
||||
"total_bytes",
|
||||
"files",
|
||||
}
|
||||
if set(manifest) != expected_manifest_fields:
|
||||
raise VerificationError("package manifest has missing or unexpected fields")
|
||||
schema_version = manifest.get("schema_version")
|
||||
if (
|
||||
not isinstance(schema_version, int)
|
||||
or isinstance(schema_version, bool)
|
||||
or schema_version != MMO_SCHEMA_VERSION
|
||||
or manifest.get("package") != "codex-multimodel-orchestrator"
|
||||
):
|
||||
raise VerificationError("package manifest identity is invalid")
|
||||
version = (
|
||||
(package_root / "VERSION").read_text(encoding="utf-8").strip()
|
||||
if (package_root / "VERSION").is_file()
|
||||
else ""
|
||||
)
|
||||
if manifest.get("version") != version:
|
||||
raise VerificationError("manifest version does not match VERSION")
|
||||
expected_root = f"codex-multimodel-orchestrator-{version}"
|
||||
if package_root.name != expected_root:
|
||||
raise VerificationError(
|
||||
f"archive top-level directory must be {expected_root!r}, got {package_root.name!r}"
|
||||
)
|
||||
entries = manifest.get("files")
|
||||
if not isinstance(entries, list) or not entries:
|
||||
raise VerificationError("manifest file list is empty or invalid")
|
||||
if manifest.get("manifest_excludes") != ["PACKAGE-MANIFEST.json"]:
|
||||
raise VerificationError("manifest exclusion declaration is invalid")
|
||||
epoch = manifest.get("source_date_epoch")
|
||||
if not isinstance(epoch, int) or isinstance(epoch, bool) or not 0 <= epoch <= 0xFFFFFFFF:
|
||||
raise VerificationError("manifest source_date_epoch is invalid")
|
||||
file_count = manifest.get("file_count")
|
||||
if (
|
||||
not isinstance(file_count, int)
|
||||
or isinstance(file_count, bool)
|
||||
or file_count != len(entries)
|
||||
):
|
||||
raise VerificationError("manifest file_count does not match files array")
|
||||
total_bytes = manifest.get("total_bytes")
|
||||
if not isinstance(total_bytes, int) or isinstance(total_bytes, bool) or total_bytes < 0:
|
||||
raise VerificationError("manifest total_bytes is invalid")
|
||||
|
||||
manifest_inventory: dict[str, dict[str, Any]] = {}
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get("path"), str):
|
||||
raise VerificationError("manifest contains an invalid file entry")
|
||||
relative = _safe_member_path(entry["path"]).as_posix()
|
||||
if relative in manifest_inventory or relative == "PACKAGE-MANIFEST.json":
|
||||
raise VerificationError(f"manifest contains a duplicate or reserved path: {relative}")
|
||||
if set(entry) != {"path", "mode", "size", "sha256"}:
|
||||
raise VerificationError(f"manifest entry has unexpected fields: {relative}")
|
||||
if entry.get("mode") not in {"0644", "0755"}:
|
||||
raise VerificationError(f"manifest entry has invalid mode: {relative}")
|
||||
if (
|
||||
not isinstance(entry.get("size"), int)
|
||||
or isinstance(entry.get("size"), bool)
|
||||
or entry["size"] < 0
|
||||
):
|
||||
raise VerificationError(f"manifest entry has invalid size: {relative}")
|
||||
expected_hash = entry.get("sha256")
|
||||
if (
|
||||
not isinstance(expected_hash, str)
|
||||
or len(expected_hash) != 64
|
||||
or any(char not in "0123456789abcdef" for char in expected_hash)
|
||||
):
|
||||
raise VerificationError(f"manifest entry has invalid hash: {relative}")
|
||||
manifest_inventory[relative] = entry
|
||||
|
||||
actual_files = {
|
||||
path.relative_to(package_root).as_posix()
|
||||
for path in package_root.rglob("*")
|
||||
if path.is_file() and not path.is_symlink()
|
||||
}
|
||||
expected_files = set(manifest_inventory) | {"PACKAGE-MANIFEST.json"}
|
||||
missing = sorted(expected_files - actual_files)
|
||||
extra = sorted(actual_files - expected_files)
|
||||
if missing or extra:
|
||||
raise VerificationError(
|
||||
f"archive file set differs from manifest; missing={missing}, extra={extra}"
|
||||
)
|
||||
|
||||
missing_required = sorted(REQUIRED_FILES - actual_files)
|
||||
missing_directories = sorted(
|
||||
name for name in REQUIRED_DIRECTORIES if not (package_root / name).is_dir()
|
||||
)
|
||||
if missing_required or missing_directories:
|
||||
raise VerificationError(
|
||||
f"archive is not a complete source release; missing_files={missing_required}, "
|
||||
f"missing_directories={missing_directories}"
|
||||
)
|
||||
|
||||
mismatches: list[str] = []
|
||||
for relative, entry in sorted(manifest_inventory.items()):
|
||||
path = package_root / relative
|
||||
actual_mode = stat.S_IMODE(path.stat().st_mode)
|
||||
expected_mode = str(entry.get("mode", ""))
|
||||
expected_size = entry.get("size")
|
||||
expected_hash = entry.get("sha256")
|
||||
if path.stat().st_size != expected_size:
|
||||
mismatches.append(f"{relative}: size")
|
||||
if sha256(path) != expected_hash:
|
||||
mismatches.append(f"{relative}: sha256")
|
||||
if f"{actual_mode:04o}" != expected_mode:
|
||||
mismatches.append(f"{relative}: mode")
|
||||
actual_payload_bytes = sum(
|
||||
(package_root / relative).stat().st_size for relative in manifest_inventory
|
||||
)
|
||||
if manifest.get("total_bytes") != actual_payload_bytes:
|
||||
mismatches.append(
|
||||
f"manifest total_bytes {manifest.get('total_bytes')!r} != {actual_payload_bytes}"
|
||||
)
|
||||
if mismatches:
|
||||
raise VerificationError(f"manifest verification failed: {mismatches[:20]}")
|
||||
|
||||
if source_tree is not None:
|
||||
source_inventory = _source_inventory(source_tree)
|
||||
if set(source_inventory) != set(manifest_inventory):
|
||||
missing_from_archive = sorted(set(source_inventory) - set(manifest_inventory))
|
||||
extra_in_archive = sorted(set(manifest_inventory) - set(source_inventory))
|
||||
raise VerificationError(
|
||||
f"archive differs from source tree; missing={missing_from_archive}, extra={extra_in_archive}"
|
||||
)
|
||||
source_mismatches = [
|
||||
relative
|
||||
for relative in sorted(source_inventory)
|
||||
if source_inventory[relative]
|
||||
!= {
|
||||
"size": manifest_inventory[relative].get("size"),
|
||||
"sha256": manifest_inventory[relative].get("sha256"),
|
||||
"mode": manifest_inventory[relative].get("mode"),
|
||||
}
|
||||
]
|
||||
if source_mismatches:
|
||||
raise VerificationError(
|
||||
f"archive content differs from source tree: {source_mismatches[:20]}"
|
||||
)
|
||||
|
||||
return {
|
||||
"package": manifest["package"],
|
||||
"version": manifest["version"],
|
||||
"manifest_files": len(manifest_inventory),
|
||||
"total_files": len(actual_files),
|
||||
"expanded_bytes": sum((package_root / item).stat().st_size for item in actual_files),
|
||||
"manifest_sha256": sha256(manifest_path),
|
||||
"tree_sha256": _tree_hash(package_root),
|
||||
}
|
||||
|
||||
|
||||
def verify_archive(archive_path: Path, source_tree: Path | None = None) -> dict[str, Any]:
|
||||
archive_path = archive_path.expanduser().resolve()
|
||||
if not archive_path.is_file():
|
||||
raise VerificationError(f"archive does not exist: {archive_path}")
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-verify-release-") as temporary:
|
||||
destination = Path(temporary)
|
||||
if archive_path.name.endswith(".tar.gz"):
|
||||
_extract_tar(archive_path, destination)
|
||||
elif archive_path.suffix == ".zip":
|
||||
_extract_zip(archive_path, destination)
|
||||
else:
|
||||
raise VerificationError(f"unsupported archive type: {archive_path.name}")
|
||||
roots = [path for path in destination.iterdir() if path.is_dir()]
|
||||
stray = [path for path in destination.iterdir() if not path.is_dir()]
|
||||
if len(roots) != 1 or stray:
|
||||
raise VerificationError("archive must contain exactly one top-level package directory")
|
||||
package_root = roots[0]
|
||||
result = _verify_extracted(package_root, source_tree=source_tree)
|
||||
result.update(
|
||||
{
|
||||
"archive": str(archive_path),
|
||||
"archive_bytes": archive_path.stat().st_size,
|
||||
"archive_sha256": sha256(archive_path),
|
||||
"top_level": package_root.name,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Verify Codex MMO release archives")
|
||||
parser.add_argument("archives", nargs="+", type=Path)
|
||||
parser.add_argument("--source-tree", type=Path)
|
||||
parser.add_argument("--report", type=Path)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def _atomic_write_text(path: Path, value: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(value)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o644)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(temporary)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = _parser().parse_args()
|
||||
source = args.source_tree.expanduser().resolve() if args.source_tree else None
|
||||
results: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
for archive in args.archives:
|
||||
try:
|
||||
results.append(verify_archive(archive, source_tree=source))
|
||||
except Exception as exc: # verification boundary
|
||||
errors.append(f"{archive}: {type(exc).__name__}: {exc}")
|
||||
report = {"passed": not errors, "results": results, "errors": errors}
|
||||
if args.report:
|
||||
_atomic_write_text(
|
||||
args.report,
|
||||
json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
)
|
||||
if args.json or errors:
|
||||
print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False))
|
||||
else:
|
||||
for item in results:
|
||||
print(
|
||||
f"PASS {Path(item['archive']).name}: {item['manifest_files']} manifest files, "
|
||||
f"{item['total_files']} total files, sha256={item['archive_sha256']}"
|
||||
)
|
||||
return 0 if not errors else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user