#!/usr/bin/env python3 """Verify Codex MMO release archives by safe extraction and manifest checks. The verifier is deliberately independent from the archive builder. It rejects traversal, links, special files, duplicate members, incomplete package trees, extra files, missing files, hash mismatches, size mismatches, and mode drift. """ from __future__ import annotations import argparse import contextlib import hashlib import json import math import os import stat import sys import tarfile import tempfile import zipfile from collections.abc import Iterable from pathlib import Path, PurePosixPath from typing import Any ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "libexec")) from mmo_version import MMO_SCHEMA_VERSION # noqa: E402 REQUIRED_FILES = { "VERSION", "README.md", "LICENSE", "install.sh", "uninstall.sh", "config/catalog.toml", "config/inventory-snapshots/openai-codex.json", "config/inventory-snapshots/opencode-go.json", "config/inventory-snapshots/opencode-zen.json", "config/inventory-snapshots/openrouter.json", "config/inventory-snapshots/zai-api.json", "config/inventory-snapshots/zai-coding-plan.json", "scripts/build_release.py", "scripts/build_inventory_snapshot.py", "scripts/validate_package.py", "scripts/verify_release.py", "libexec/mmo_app_server.py", "libexec/mmo_catalog_data.py", "libexec/mmo_codex_home.py", "libexec/mmo_diagnostics.py", "libexec/mmo_runtime.py", "libexec/mmo_state.py", "libexec/mmo_workspace.py", "libexec/mmo_inventory_snapshot.py", "libexec/mmo_mcp.py", "libexec/mmo_tool_mcp.py", "libexec/root_runner.py", "libexec/worker_runner.py", "profiles/README.md", } REQUIRED_DIRECTORIES = {"bin", "config", "docs", "evals", "libexec", "profiles", "scripts", "tests"} MAX_ARCHIVE_MEMBERS = 20_000 MAX_ARCHIVE_MEMBER_BYTES = 512 * 1024 * 1024 MAX_EXPANDED_BYTES = 1_000_000_000 MAX_ZIP_COMPRESSION_RATIO = 1_000 def _reject_json_constant(value: str) -> Any: raise ValueError(f"non-standard JSON constant: {value}") def _finite_json_float(value: str) -> float: parsed = float(value) if not math.isfinite(parsed): raise ValueError(f"JSON number is outside the supported finite range: {value}") return parsed def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: result: dict[str, Any] = {} for key, item in pairs: if key in result: raise ValueError(f"duplicate JSON object member: {key!r}") result[key] = item return result def _validate_json_unicode(value: Any) -> None: if isinstance(value, str): try: value.encode("utf-8") except UnicodeEncodeError as exc: raise ValueError("JSON strings must contain valid Unicode scalar values") from exc elif isinstance(value, list): for item in value: _validate_json_unicode(item) elif isinstance(value, dict): for key, item in value.items(): _validate_json_unicode(key) _validate_json_unicode(item) def _strict_json_loads(value: str) -> Any: parsed = json.loads( value, parse_constant=_reject_json_constant, parse_float=_finite_json_float, object_pairs_hook=_unique_json_object, ) _validate_json_unicode(parsed) return parsed class VerificationError(RuntimeError): """Raised when a release archive fails integrity verification.""" def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _safe_member_path(name: str) -> PurePosixPath: if not name or "\x00" in name or "\\" in name or name.startswith("/"): raise VerificationError(f"unsafe archive member path: {name!r}") value = name[:-1] if name.endswith("/") else name parts = value.split("/") if not parts or any(part in {"", ".", ".."} for part in parts): raise VerificationError(f"unsafe archive member path: {name!r}") return PurePosixPath(*parts) def _ensure_member_limits(names: Iterable[str], sizes: Iterable[int]) -> None: names_list = list(names) sizes_list = list(sizes) if len(names_list) != len(sizes_list): raise VerificationError("archive member metadata is inconsistent") if len(names_list) > MAX_ARCHIVE_MEMBERS: raise VerificationError(f"archive contains too many members: {len(names_list)}") if any(size < 0 for size in sizes_list): raise VerificationError("archive contains a negative member size") expanded = sum(sizes_list) if expanded > MAX_EXPANDED_BYTES: raise VerificationError(f"archive expands beyond safety limit: {expanded} bytes") if len(set(names_list)) != len(names_list): raise VerificationError("archive contains duplicate member names") def _copy_limited(source: Any, output: Any, maximum: int, label: str) -> int: copied = 0 while True: block = source.read(min(1024 * 1024, maximum - copied + 1)) if not block: return copied copied += len(block) if copied > maximum: raise VerificationError(f"archive member exceeds size limit: {label}") output.write(block) def _register_member( seen: dict[str, bool], relative: PurePosixPath, *, is_directory: bool, label: str ) -> None: key = relative.as_posix() if key in seen: raise VerificationError(f"archive aliases the same extraction path: {label}") parts = relative.parts for index in range(1, len(parts)): parent = "/".join(parts[:index]) if seen.get(parent) is False: raise VerificationError(f"archive places a member beneath a file: {label}") if not is_directory and any(existing.startswith(f"{key}/") for existing in seen): raise VerificationError(f"archive replaces a populated directory with a file: {label}") seen[key] = is_directory def _validate_explicit_directories(seen: dict[str, bool]) -> None: """Require exactly one explicit entry for every payload parent directory.""" expected: set[str] = set() for name, is_directory in seen.items(): if is_directory: continue parts = name.split("/") expected.update("/".join(parts[:index]) for index in range(1, len(parts))) actual = {name for name, is_directory in seen.items() if is_directory} missing = sorted(expected - actual) extra = sorted(actual - expected) if missing or extra: raise VerificationError( f"archive directory set differs from payload parents; missing={missing}, extra={extra}" ) def _validate_permission_mode( archive_type: str, label: str, mode: int, *, is_directory: bool ) -> None: allowed = {0o755} if is_directory else {0o644, 0o755} if mode not in allowed: expected = "0755" if is_directory else "0644 or 0755" raise VerificationError( f"{archive_type} member has invalid permission mode: " f"{label}: {mode:04o}; expected {expected}" ) def _extract_zip(archive_path: Path, destination: Path) -> None: with zipfile.ZipFile(archive_path) as archive: infos = archive.infolist() _ensure_member_limits((item.filename for item in infos), (item.file_size for item in infos)) relative_paths = [_safe_member_path(item.filename) for item in infos] canonical_names = [path.as_posix() for path in relative_paths] if len(set(canonical_names)) != len(canonical_names): raise VerificationError("archive aliases the same extraction path") seen: dict[str, bool] = {} for item, relative in zip(infos, relative_paths, strict=True): if item.create_system != 3: raise VerificationError(f"ZIP member lacks Unix mode metadata: {item.filename}") unix_mode = (item.external_attr >> 16) & 0xFFFF file_type = stat.S_IFMT(unix_mode) if file_type not in {stat.S_IFREG, stat.S_IFDIR}: raise VerificationError(f"ZIP contains a link or special file: {item.filename}") is_directory = file_type == stat.S_IFDIR if item.is_dir() != is_directory: raise VerificationError( f"ZIP member path and type metadata disagree: {item.filename}" ) permission_mode = unix_mode & 0o7777 _validate_permission_mode( "ZIP", item.filename, permission_mode, is_directory=is_directory ) _register_member(seen, relative, is_directory=is_directory, label=item.filename) if item.flag_bits & 0x1: raise VerificationError(f"ZIP contains an encrypted member: {item.filename}") if item.file_size > MAX_ARCHIVE_MEMBER_BYTES: raise VerificationError(f"ZIP member exceeds size limit: {item.filename}") if not is_directory and item.file_size and item.compress_size == 0: raise VerificationError(f"ZIP member has invalid compressed size: {item.filename}") if ( item.compress_size > 0 and item.file_size / item.compress_size > MAX_ZIP_COMPRESSION_RATIO ): raise VerificationError( f"ZIP member has suspicious compression ratio: {item.filename}" ) target = destination.joinpath(*relative.parts) if is_directory: target.mkdir(parents=True, exist_ok=True) os.chmod(target, permission_mode) continue target.parent.mkdir(parents=True, exist_ok=True) with archive.open(item) as source, target.open("wb") as output: copied = _copy_limited(source, output, MAX_ARCHIVE_MEMBER_BYTES, item.filename) if copied != item.file_size: raise VerificationError(f"ZIP member size mismatch: {item.filename}") os.chmod(target, permission_mode) _validate_explicit_directories(seen) def _extract_tar(archive_path: Path, destination: Path) -> None: with tarfile.open(archive_path, mode="r|gz") as archive: seen: dict[str, bool] = {} member_count = 0 expanded_bytes = 0 for item in archive: member_count += 1 if member_count > MAX_ARCHIVE_MEMBERS: raise VerificationError(f"archive contains too many members: {member_count}") if item.size < 0: raise VerificationError(f"tar member has a negative size: {item.name}") expanded_bytes += item.size if expanded_bytes > MAX_EXPANDED_BYTES: raise VerificationError( f"archive expands beyond safety limit: {expanded_bytes} bytes" ) relative = _safe_member_path(item.name) if not (item.isdir() or item.isfile()): raise VerificationError(f"tar contains a link or special file: {item.name}") permission_mode = item.mode & 0o7777 if item.mode != permission_mode: raise VerificationError(f"tar member has invalid mode metadata: {item.name}") _validate_permission_mode("tar", item.name, permission_mode, is_directory=item.isdir()) _register_member(seen, relative, is_directory=item.isdir(), label=item.name) if item.size > MAX_ARCHIVE_MEMBER_BYTES: raise VerificationError(f"tar member exceeds size limit: {item.name}") target = destination.joinpath(*relative.parts) if item.isdir(): target.mkdir(parents=True, exist_ok=True) os.chmod(target, permission_mode) continue target.parent.mkdir(parents=True, exist_ok=True) source = archive.extractfile(item) if source is None: raise VerificationError(f"unable to read tar member: {item.name}") with source, target.open("wb") as output: copied = _copy_limited(source, output, MAX_ARCHIVE_MEMBER_BYTES, item.name) if copied != item.size: raise VerificationError(f"tar member size mismatch: {item.name}") os.chmod(target, permission_mode) _validate_explicit_directories(seen) def _source_inventory(source_tree: Path) -> dict[str, dict[str, Any]]: excluded_parts = { ".git", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "dist", "build", } inventory: dict[str, dict[str, Any]] = {} for path in sorted(source_tree.rglob("*")): relative = path.relative_to(source_tree) if any(part in excluded_parts for part in relative.parts): continue mode_bits = path.lstat().st_mode if stat.S_ISLNK(mode_bits): raise VerificationError(f"source tree contains a symbolic link: {relative}") if not (stat.S_ISREG(mode_bits) or stat.S_ISDIR(mode_bits)): raise VerificationError(f"source tree contains a special file: {relative}") if ( relative.as_posix() == "PACKAGE-MANIFEST.json" or not path.is_file() or path.suffix in {".pyc", ".pyo"} ): continue mode = 0o755 if path.stat().st_mode & stat.S_IXUSR else 0o644 inventory[relative.as_posix()] = { "size": path.stat().st_size, "sha256": sha256(path), "mode": f"{mode:04o}", } return inventory def _tree_hash(package_root: Path) -> str: digest = hashlib.sha256() files = sorted( (path for path in package_root.rglob("*") if path.is_file() and not path.is_symlink()), key=lambda item: item.relative_to(package_root).as_posix(), ) for path in files: relative = path.relative_to(package_root).as_posix().encode("utf-8") mode = ("0755" if path.stat().st_mode & stat.S_IXUSR else "0644").encode("ascii") size = str(path.stat().st_size).encode("ascii") for value in (relative, mode, size): digest.update(len(value).to_bytes(8, "big")) digest.update(value) with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _verify_extracted(package_root: Path, source_tree: Path | None = None) -> dict[str, Any]: manifest_path = package_root / "PACKAGE-MANIFEST.json" if not manifest_path.is_file(): raise VerificationError("PACKAGE-MANIFEST.json is missing") try: manifest = _strict_json_loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError, ValueError) as exc: raise VerificationError(f"invalid package manifest: {exc}") from exc if not isinstance(manifest, dict): raise VerificationError("package manifest root must be an object") expected_manifest_fields = { "schema_version", "package", "version", "source_date_epoch", "manifest_excludes", "file_count", "total_bytes", "files", } if set(manifest) != expected_manifest_fields: raise VerificationError("package manifest has missing or unexpected fields") schema_version = manifest.get("schema_version") if ( not isinstance(schema_version, int) or isinstance(schema_version, bool) or schema_version != MMO_SCHEMA_VERSION or manifest.get("package") != "codex-multimodel-orchestrator" ): raise VerificationError("package manifest identity is invalid") version = ( (package_root / "VERSION").read_text(encoding="utf-8").strip() if (package_root / "VERSION").is_file() else "" ) if manifest.get("version") != version: raise VerificationError("manifest version does not match VERSION") expected_root = f"codex-multimodel-orchestrator-{version}" if package_root.name != expected_root: raise VerificationError( f"archive top-level directory must be {expected_root!r}, got {package_root.name!r}" ) entries = manifest.get("files") if not isinstance(entries, list) or not entries: raise VerificationError("manifest file list is empty or invalid") if manifest.get("manifest_excludes") != ["PACKAGE-MANIFEST.json"]: raise VerificationError("manifest exclusion declaration is invalid") epoch = manifest.get("source_date_epoch") if not isinstance(epoch, int) or isinstance(epoch, bool) or not 0 <= epoch <= 0xFFFFFFFF: raise VerificationError("manifest source_date_epoch is invalid") file_count = manifest.get("file_count") if ( not isinstance(file_count, int) or isinstance(file_count, bool) or file_count != len(entries) ): raise VerificationError("manifest file_count does not match files array") total_bytes = manifest.get("total_bytes") if not isinstance(total_bytes, int) or isinstance(total_bytes, bool) or total_bytes < 0: raise VerificationError("manifest total_bytes is invalid") manifest_inventory: dict[str, dict[str, Any]] = {} for entry in entries: if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): raise VerificationError("manifest contains an invalid file entry") relative = _safe_member_path(entry["path"]).as_posix() if relative in manifest_inventory or relative == "PACKAGE-MANIFEST.json": raise VerificationError(f"manifest contains a duplicate or reserved path: {relative}") if set(entry) != {"path", "mode", "size", "sha256"}: raise VerificationError(f"manifest entry has unexpected fields: {relative}") if entry.get("mode") not in {"0644", "0755"}: raise VerificationError(f"manifest entry has invalid mode: {relative}") if ( not isinstance(entry.get("size"), int) or isinstance(entry.get("size"), bool) or entry["size"] < 0 ): raise VerificationError(f"manifest entry has invalid size: {relative}") expected_hash = entry.get("sha256") if ( not isinstance(expected_hash, str) or len(expected_hash) != 64 or any(char not in "0123456789abcdef" for char in expected_hash) ): raise VerificationError(f"manifest entry has invalid hash: {relative}") manifest_inventory[relative] = entry actual_files = { path.relative_to(package_root).as_posix() for path in package_root.rglob("*") if path.is_file() and not path.is_symlink() } expected_files = set(manifest_inventory) | {"PACKAGE-MANIFEST.json"} missing = sorted(expected_files - actual_files) extra = sorted(actual_files - expected_files) if missing or extra: raise VerificationError( f"archive file set differs from manifest; missing={missing}, extra={extra}" ) missing_required = sorted(REQUIRED_FILES - actual_files) missing_directories = sorted( name for name in REQUIRED_DIRECTORIES if not (package_root / name).is_dir() ) if missing_required or missing_directories: raise VerificationError( f"archive is not a complete source release; missing_files={missing_required}, " f"missing_directories={missing_directories}" ) mismatches: list[str] = [] for relative, entry in sorted(manifest_inventory.items()): path = package_root / relative actual_mode = stat.S_IMODE(path.stat().st_mode) expected_mode = str(entry.get("mode", "")) expected_size = entry.get("size") expected_hash = entry.get("sha256") if path.stat().st_size != expected_size: mismatches.append(f"{relative}: size") if sha256(path) != expected_hash: mismatches.append(f"{relative}: sha256") if f"{actual_mode:04o}" != expected_mode: mismatches.append(f"{relative}: mode") actual_payload_bytes = sum( (package_root / relative).stat().st_size for relative in manifest_inventory ) if manifest.get("total_bytes") != actual_payload_bytes: mismatches.append( f"manifest total_bytes {manifest.get('total_bytes')!r} != {actual_payload_bytes}" ) if mismatches: raise VerificationError(f"manifest verification failed: {mismatches[:20]}") if source_tree is not None: source_inventory = _source_inventory(source_tree) if set(source_inventory) != set(manifest_inventory): missing_from_archive = sorted(set(source_inventory) - set(manifest_inventory)) extra_in_archive = sorted(set(manifest_inventory) - set(source_inventory)) raise VerificationError( f"archive differs from source tree; missing={missing_from_archive}, extra={extra_in_archive}" ) source_mismatches = [ relative for relative in sorted(source_inventory) if source_inventory[relative] != { "size": manifest_inventory[relative].get("size"), "sha256": manifest_inventory[relative].get("sha256"), "mode": manifest_inventory[relative].get("mode"), } ] if source_mismatches: raise VerificationError( f"archive content differs from source tree: {source_mismatches[:20]}" ) return { "package": manifest["package"], "version": manifest["version"], "manifest_files": len(manifest_inventory), "total_files": len(actual_files), "expanded_bytes": sum((package_root / item).stat().st_size for item in actual_files), "manifest_sha256": sha256(manifest_path), "tree_sha256": _tree_hash(package_root), } def verify_archive(archive_path: Path, source_tree: Path | None = None) -> dict[str, Any]: archive_path = archive_path.expanduser().resolve() if not archive_path.is_file(): raise VerificationError(f"archive does not exist: {archive_path}") with tempfile.TemporaryDirectory(prefix="mmo-verify-release-") as temporary: destination = Path(temporary) if archive_path.name.endswith(".tar.gz"): _extract_tar(archive_path, destination) elif archive_path.suffix == ".zip": _extract_zip(archive_path, destination) else: raise VerificationError(f"unsupported archive type: {archive_path.name}") roots = [path for path in destination.iterdir() if path.is_dir()] stray = [path for path in destination.iterdir() if not path.is_dir()] if len(roots) != 1 or stray: raise VerificationError("archive must contain exactly one top-level package directory") package_root = roots[0] result = _verify_extracted(package_root, source_tree=source_tree) result.update( { "archive": str(archive_path), "archive_bytes": archive_path.stat().st_size, "archive_sha256": sha256(archive_path), "top_level": package_root.name, } ) return result def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Verify Codex MMO release archives") parser.add_argument("archives", nargs="+", type=Path) parser.add_argument("--source-tree", type=Path) parser.add_argument("--report", type=Path) parser.add_argument("--json", action="store_true") return parser def _atomic_write_text(path: Path, value: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) try: with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: handle.write(value) handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, 0o644) os.replace(temporary, path) finally: with contextlib.suppress(FileNotFoundError): os.unlink(temporary) def main() -> int: args = _parser().parse_args() source = args.source_tree.expanduser().resolve() if args.source_tree else None results: list[dict[str, Any]] = [] errors: list[str] = [] for archive in args.archives: try: results.append(verify_archive(archive, source_tree=source)) except Exception as exc: # verification boundary errors.append(f"{archive}: {type(exc).__name__}: {exc}") report = {"passed": not errors, "results": results, "errors": errors} if args.report: _atomic_write_text( args.report, json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + "\n", ) if args.json or errors: print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) else: for item in results: print( f"PASS {Path(item['archive']).name}: {item['manifest_files']} manifest files, " f"{item['total_files']} total files, sha256={item['archive_sha256']}" ) return 0 if not errors else 1 if __name__ == "__main__": raise SystemExit(main())