442 lines
17 KiB
Python
Executable File
442 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Build deterministic, atomically published Codex MMO release archives."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import zipfile
|
|
from collections.abc import Callable, Iterable
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
sys.path.insert(0, str(ROOT / "libexec"))
|
|
|
|
from mmo_util import filtered_environment # noqa: E402
|
|
|
|
if TYPE_CHECKING:
|
|
from scripts.verify_release import (
|
|
REQUIRED_DIRECTORIES,
|
|
REQUIRED_FILES,
|
|
verify_archive,
|
|
)
|
|
else:
|
|
from verify_release import ( # noqa: E402
|
|
REQUIRED_DIRECTORIES,
|
|
REQUIRED_FILES,
|
|
verify_archive,
|
|
)
|
|
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION # noqa: E402
|
|
|
|
VERSION = PACKAGE_VERSION
|
|
PACKAGE_NAME = f"codex-multimodel-orchestrator-{VERSION}"
|
|
DEFAULT_EPOCH = 1786665600 # 2026-08-14T00:00:00Z
|
|
EXCLUDED_PARTS = {
|
|
".git",
|
|
"__pycache__",
|
|
".pytest_cache",
|
|
".mypy_cache",
|
|
".ruff_cache",
|
|
"dist",
|
|
"build",
|
|
}
|
|
EXCLUDED_SUFFIXES = {".pyc", ".pyo"}
|
|
MINIMUM_SOURCE_FILES = 100
|
|
|
|
|
|
def source_paths() -> list[Path]:
|
|
result: list[Path] = []
|
|
for path in ROOT.rglob("*"):
|
|
relative = path.relative_to(ROOT)
|
|
if any(part in EXCLUDED_PARTS for part in relative.parts):
|
|
continue
|
|
mode = path.lstat().st_mode
|
|
if stat.S_ISLNK(mode):
|
|
raise RuntimeError(f"refusing to package symbolic link: {relative}")
|
|
if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
|
|
raise RuntimeError(f"refusing to package special file: {relative}")
|
|
if path.suffix in EXCLUDED_SUFFIXES:
|
|
continue
|
|
if stat.S_ISREG(mode) and relative.as_posix() != "PACKAGE-MANIFEST.json":
|
|
result.append(path)
|
|
return sorted(result, key=lambda item: item.relative_to(ROOT).as_posix())
|
|
|
|
|
|
def normalized_mode(source: Path) -> int:
|
|
return 0o755 if source.stat().st_mode & stat.S_IXUSR else 0o644
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def assert_complete_source(paths: list[Path]) -> None:
|
|
relative_files = {path.relative_to(ROOT).as_posix() for path in paths}
|
|
missing_files = sorted(REQUIRED_FILES - relative_files)
|
|
missing_directories = sorted(
|
|
name for name in REQUIRED_DIRECTORIES if not (ROOT / name).is_dir()
|
|
)
|
|
if len(paths) < MINIMUM_SOURCE_FILES or missing_files or missing_directories:
|
|
raise RuntimeError(
|
|
"refusing to package an incomplete source tree: "
|
|
f"file_count={len(paths)}, minimum={MINIMUM_SOURCE_FILES}, "
|
|
f"missing_files={missing_files}, missing_directories={missing_directories}"
|
|
)
|
|
|
|
|
|
def stage_tree(destination: Path, epoch: int) -> Path:
|
|
sources = source_paths()
|
|
assert_complete_source(sources)
|
|
top = destination / PACKAGE_NAME
|
|
top.mkdir(parents=True)
|
|
files: list[dict[str, Any]] = []
|
|
for source in sources:
|
|
relative = source.relative_to(ROOT)
|
|
target = top / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(source, target)
|
|
mode = normalized_mode(source)
|
|
os.chmod(target, mode)
|
|
os.utime(target, (epoch, epoch))
|
|
files.append(
|
|
{
|
|
"path": relative.as_posix(),
|
|
"mode": f"{mode:04o}",
|
|
"size": target.stat().st_size,
|
|
"sha256": sha256(target),
|
|
}
|
|
)
|
|
manifest = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": "codex-multimodel-orchestrator",
|
|
"version": VERSION,
|
|
"source_date_epoch": epoch,
|
|
"manifest_excludes": ["PACKAGE-MANIFEST.json"],
|
|
"file_count": len(files),
|
|
"total_bytes": sum(int(item["size"]) for item in files),
|
|
"files": files,
|
|
}
|
|
manifest_path = top / "PACKAGE-MANIFEST.json"
|
|
manifest_path.write_text(
|
|
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
os.chmod(manifest_path, 0o644)
|
|
os.utime(manifest_path, (epoch, epoch))
|
|
for directory in sorted((path for path in top.rglob("*") if path.is_dir()), reverse=True):
|
|
os.chmod(directory, 0o755)
|
|
os.utime(directory, (epoch, epoch))
|
|
os.chmod(top, 0o755)
|
|
os.utime(top, (epoch, epoch))
|
|
return top
|
|
|
|
|
|
def tree_entries(top: Path) -> list[Path]:
|
|
return [top, *sorted(top.rglob("*"), key=lambda item: item.relative_to(top.parent).as_posix())]
|
|
|
|
|
|
def build_tar_gz(top: Path, output: Path, epoch: int) -> None:
|
|
with output.open("wb") as raw:
|
|
with gzip.GzipFile(
|
|
filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=epoch
|
|
) as compressed:
|
|
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as archive:
|
|
for path in tree_entries(top):
|
|
arcname = path.relative_to(top.parent).as_posix()
|
|
info = archive.gettarinfo(str(path), arcname=arcname)
|
|
info.uid = 0
|
|
info.gid = 0
|
|
info.uname = "root"
|
|
info.gname = "root"
|
|
info.mtime = epoch
|
|
info.mode = 0o755 if path.is_dir() else normalized_mode(path)
|
|
if path.is_file():
|
|
with path.open("rb") as handle:
|
|
archive.addfile(info, handle)
|
|
else:
|
|
archive.addfile(info)
|
|
|
|
|
|
def build_zip(top: Path, output: Path, epoch: int) -> None:
|
|
timestamp = time.gmtime(max(epoch, 315532800))[:6]
|
|
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
|
for path in tree_entries(top):
|
|
arcname = path.relative_to(top.parent).as_posix() + ("/" if path.is_dir() else "")
|
|
info = zipfile.ZipInfo(arcname, timestamp)
|
|
info.create_system = 3
|
|
mode = 0o755 if path.is_dir() else normalized_mode(path)
|
|
info.external_attr = (stat.S_IFDIR if path.is_dir() else stat.S_IFREG) << 16
|
|
info.external_attr |= mode << 16
|
|
info.compress_type = zipfile.ZIP_DEFLATED
|
|
info.flag_bits |= 0x800
|
|
archive.writestr(info, b"" if path.is_dir() else path.read_bytes())
|
|
|
|
|
|
def build_once(destination: Path, epoch: int) -> tuple[Path, Path]:
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
stage_root = destination / "stage"
|
|
stage_root.mkdir()
|
|
top = stage_tree(stage_root, epoch)
|
|
tar_path = destination / f"{PACKAGE_NAME}-linux.tar.gz"
|
|
zip_path = destination / f"{PACKAGE_NAME}-linux.zip"
|
|
build_tar_gz(top, tar_path, epoch)
|
|
build_zip(top, zip_path, epoch)
|
|
return tar_path, zip_path
|
|
|
|
|
|
def validate_source(report_path: Path | None = None) -> None:
|
|
command = [sys.executable, str(ROOT / "scripts" / "validate_package.py")]
|
|
if report_path:
|
|
command.extend(("--report", str(report_path)))
|
|
result = subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
env=filtered_environment(extra={"PYTHONDONTWRITEBYTECODE": "1"}),
|
|
check=False,
|
|
)
|
|
if result.returncode:
|
|
raise RuntimeError("package validation failed")
|
|
|
|
|
|
def write_checksums(paths: Iterable[Path], output: Path) -> None:
|
|
lines = [f"{sha256(path)} {path.name}" for path in sorted(paths, key=lambda item: item.name)]
|
|
output.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def _atomic_copy(source: Path, destination: Path) -> None:
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, raw_temporary = tempfile.mkstemp(prefix=f".{destination.name}.tmp-", dir=destination.parent)
|
|
temporary = Path(raw_temporary)
|
|
try:
|
|
with os.fdopen(fd, "wb") as output_handle, source.open("rb") as input_handle:
|
|
shutil.copyfileobj(input_handle, output_handle, length=1024 * 1024)
|
|
output_handle.flush()
|
|
os.fsync(output_handle.fileno())
|
|
os.chmod(temporary, 0o644)
|
|
os.replace(temporary, destination)
|
|
finally:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def _publish_artifacts(
|
|
sources: Iterable[tuple[Path, Path]],
|
|
*,
|
|
verify: Callable[[], None] | None = None,
|
|
) -> None:
|
|
"""Publish a release set and restore every prior artifact on failure."""
|
|
|
|
pairs = list(sources)
|
|
destinations = [destination for _source, destination in pairs]
|
|
if len(set(destinations)) != len(destinations):
|
|
raise RuntimeError("release publication contains duplicate destinations")
|
|
for source, destination in pairs:
|
|
source_mode = source.lstat().st_mode
|
|
if not stat.S_ISREG(source_mode):
|
|
raise RuntimeError(f"release publication source is not a regular file: {source}")
|
|
try:
|
|
destination_mode = destination.lstat().st_mode
|
|
except FileNotFoundError:
|
|
continue
|
|
if not stat.S_ISREG(destination_mode):
|
|
raise RuntimeError(f"refusing to replace a non-regular release artifact: {destination}")
|
|
prepared: dict[Path, Path] = {}
|
|
backups: dict[Path, Path] = {}
|
|
published: list[Path] = []
|
|
retained_backups: set[Path] = set()
|
|
nonce = f"{os.getpid()}-{time.time_ns()}"
|
|
try:
|
|
for source, destination in pairs:
|
|
temporary = destination.with_name(f".{destination.name}.publish-{nonce}")
|
|
_atomic_copy(source, temporary)
|
|
prepared[destination] = temporary
|
|
if destination.is_file():
|
|
backup = destination.with_name(f".{destination.name}.backup-{nonce}")
|
|
_atomic_copy(destination, backup)
|
|
backups[destination] = backup
|
|
for _source, destination in pairs:
|
|
os.replace(prepared[destination], destination)
|
|
published.append(destination)
|
|
if verify is not None:
|
|
verify()
|
|
except BaseException as publish_error:
|
|
restoration_errors: list[str] = []
|
|
for destination in reversed(published):
|
|
saved = backups.get(destination)
|
|
try:
|
|
if saved and saved.exists():
|
|
os.replace(saved, destination)
|
|
else:
|
|
destination.unlink(missing_ok=True)
|
|
except BaseException as restore_error:
|
|
if saved and saved.exists():
|
|
retained_backups.add(saved)
|
|
restoration_errors.append(
|
|
f"{destination}: {type(restore_error).__name__}: {restore_error}"
|
|
)
|
|
if restoration_errors:
|
|
retained = ", ".join(str(path) for path in sorted(retained_backups)) or "none"
|
|
raise RuntimeError(
|
|
"release publication failed and rollback was incomplete; "
|
|
f"retained backups: {retained}; errors: {'; '.join(restoration_errors)}"
|
|
) from publish_error
|
|
raise
|
|
finally:
|
|
for path in prepared.values():
|
|
path.unlink(missing_ok=True)
|
|
for path in backups.values():
|
|
if path not in retained_backups:
|
|
path.unlink(missing_ok=True)
|
|
|
|
|
|
def _verify_pair(tar_path: Path, zip_path: Path) -> list[dict[str, Any]]:
|
|
results = [
|
|
verify_archive(tar_path, source_tree=ROOT),
|
|
verify_archive(zip_path, source_tree=ROOT),
|
|
]
|
|
counts = {
|
|
(
|
|
item["manifest_files"],
|
|
item["total_files"],
|
|
item["manifest_sha256"],
|
|
item["tree_sha256"],
|
|
)
|
|
for item in results
|
|
}
|
|
if len(counts) != 1:
|
|
raise RuntimeError("tar.gz and ZIP extracted trees are not identical")
|
|
return results
|
|
|
|
|
|
def assert_safe_output_directory(output: Path) -> None:
|
|
if output == ROOT or ROOT in output.parents:
|
|
raise RuntimeError(f"refusing to write release output inside the source tree: {output}")
|
|
|
|
|
|
def parser() -> argparse.ArgumentParser:
|
|
value = argparse.ArgumentParser(description="Build reproducible Codex MMO release archives")
|
|
value.add_argument("--output-dir", type=Path, default=ROOT.parent)
|
|
value.add_argument(
|
|
"--source-date-epoch",
|
|
type=int,
|
|
default=int(os.environ.get("SOURCE_DATE_EPOCH", DEFAULT_EPOCH)),
|
|
)
|
|
value.add_argument("--skip-validation", action="store_true")
|
|
value.add_argument("--no-reproducibility-check", action="store_true")
|
|
return value
|
|
|
|
|
|
def main() -> int:
|
|
args = parser().parse_args()
|
|
if not 0 <= args.source_date_epoch <= 0xFFFFFFFF:
|
|
raise ValueError("--source-date-epoch must be between 0 and 4294967295")
|
|
output = args.output_dir.expanduser().resolve()
|
|
assert_safe_output_directory(output)
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
validation_report = output / f"{PACKAGE_NAME}-VALIDATION.json"
|
|
checksums = output / f"{PACKAGE_NAME}-SHA256SUMS.txt"
|
|
integrity_report = output / f"{PACKAGE_NAME}-INTEGRITY.json"
|
|
with tempfile.TemporaryDirectory(prefix="mmo-release-primary-") as primary_raw:
|
|
primary = Path(primary_raw)
|
|
validation_stage: Path | None = None
|
|
if not args.skip_validation:
|
|
validation_stage = primary / validation_report.name
|
|
validate_source(validation_stage)
|
|
tar_path, zip_path = build_once(primary, args.source_date_epoch)
|
|
verification = _verify_pair(tar_path, zip_path)
|
|
|
|
reproducible = True
|
|
if not args.no_reproducibility_check:
|
|
with tempfile.TemporaryDirectory(prefix="mmo-release-repro-") as second_raw:
|
|
second_tar, second_zip = build_once(Path(second_raw), args.source_date_epoch)
|
|
_verify_pair(second_tar, second_zip)
|
|
reproducible = sha256(tar_path) == sha256(second_tar) and sha256(
|
|
zip_path
|
|
) == sha256(second_zip)
|
|
if not reproducible:
|
|
raise RuntimeError("release build is not reproducible")
|
|
|
|
final_tar = output / tar_path.name
|
|
final_zip = output / zip_path.name
|
|
intended_verification = []
|
|
for item, destination in zip(verification, (final_tar, final_zip), strict=True):
|
|
intended_verification.append({**item, "archive": str(destination)})
|
|
|
|
checksum_stage = primary / checksums.name
|
|
write_checksums((tar_path, zip_path), checksum_stage)
|
|
integrity_stage = primary / integrity_report.name
|
|
integrity = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"package": PACKAGE_NAME,
|
|
"source_date_epoch": args.source_date_epoch,
|
|
"source_files": len(source_paths()),
|
|
"reproducible": reproducible,
|
|
"archives": intended_verification,
|
|
"checksums": checksums.name,
|
|
}
|
|
integrity_stage.write_text(
|
|
json.dumps(integrity, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
artifacts = [
|
|
(tar_path, final_tar),
|
|
(zip_path, final_zip),
|
|
(checksum_stage, checksums),
|
|
(integrity_stage, integrity_report),
|
|
]
|
|
if validation_stage is not None:
|
|
artifacts.append((validation_stage, validation_report))
|
|
|
|
def verify_published_set() -> None:
|
|
for staged, published in artifacts:
|
|
if sha256(staged) != sha256(published):
|
|
raise RuntimeError(
|
|
f"published bytes differ from verified artifact: {published.name}"
|
|
)
|
|
published_verification = _verify_pair(final_tar, final_zip)
|
|
if intended_verification != published_verification:
|
|
raise RuntimeError("published archives differ from verified build artifacts")
|
|
|
|
# Publish the archives and all generated release metadata as one
|
|
# rollback-safe set. Final-name verification runs while backups remain
|
|
# available, so a verification failure also restores the prior set.
|
|
_publish_artifacts(artifacts, verify=verify_published_set)
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"package": PACKAGE_NAME,
|
|
"source_date_epoch": args.source_date_epoch,
|
|
"source_files": len(source_paths()),
|
|
"reproducible": reproducible,
|
|
"tar_gz": {"path": str(final_tar), "sha256": sha256(final_tar)},
|
|
"zip": {"path": str(final_zip), "sha256": sha256(final_zip)},
|
|
"checksums": str(checksums),
|
|
"integrity_report": str(integrity_report),
|
|
},
|
|
indent=2,
|
|
sort_keys=True,
|
|
allow_nan=False,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|