😏
This commit is contained in:
Executable
+441
@@ -0,0 +1,441 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic, atomically published Codex MMO release archives."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import zipfile
|
||||
from collections.abc import Callable, Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "libexec"))
|
||||
|
||||
from mmo_util import filtered_environment # noqa: E402
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scripts.verify_release import (
|
||||
REQUIRED_DIRECTORIES,
|
||||
REQUIRED_FILES,
|
||||
verify_archive,
|
||||
)
|
||||
else:
|
||||
from verify_release import ( # noqa: E402
|
||||
REQUIRED_DIRECTORIES,
|
||||
REQUIRED_FILES,
|
||||
verify_archive,
|
||||
)
|
||||
from mmo_version import MMO_SCHEMA_VERSION, PACKAGE_VERSION # noqa: E402
|
||||
|
||||
VERSION = PACKAGE_VERSION
|
||||
PACKAGE_NAME = f"codex-multimodel-orchestrator-{VERSION}"
|
||||
DEFAULT_EPOCH = 1786665600 # 2026-08-14T00:00:00Z
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
"dist",
|
||||
"build",
|
||||
}
|
||||
EXCLUDED_SUFFIXES = {".pyc", ".pyo"}
|
||||
MINIMUM_SOURCE_FILES = 100
|
||||
|
||||
|
||||
def source_paths() -> list[Path]:
|
||||
result: list[Path] = []
|
||||
for path in ROOT.rglob("*"):
|
||||
relative = path.relative_to(ROOT)
|
||||
if any(part in EXCLUDED_PARTS for part in relative.parts):
|
||||
continue
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode):
|
||||
raise RuntimeError(f"refusing to package symbolic link: {relative}")
|
||||
if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
|
||||
raise RuntimeError(f"refusing to package special file: {relative}")
|
||||
if path.suffix in EXCLUDED_SUFFIXES:
|
||||
continue
|
||||
if stat.S_ISREG(mode) and relative.as_posix() != "PACKAGE-MANIFEST.json":
|
||||
result.append(path)
|
||||
return sorted(result, key=lambda item: item.relative_to(ROOT).as_posix())
|
||||
|
||||
|
||||
def normalized_mode(source: Path) -> int:
|
||||
return 0o755 if source.stat().st_mode & stat.S_IXUSR else 0o644
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def assert_complete_source(paths: list[Path]) -> None:
|
||||
relative_files = {path.relative_to(ROOT).as_posix() for path in paths}
|
||||
missing_files = sorted(REQUIRED_FILES - relative_files)
|
||||
missing_directories = sorted(
|
||||
name for name in REQUIRED_DIRECTORIES if not (ROOT / name).is_dir()
|
||||
)
|
||||
if len(paths) < MINIMUM_SOURCE_FILES or missing_files or missing_directories:
|
||||
raise RuntimeError(
|
||||
"refusing to package an incomplete source tree: "
|
||||
f"file_count={len(paths)}, minimum={MINIMUM_SOURCE_FILES}, "
|
||||
f"missing_files={missing_files}, missing_directories={missing_directories}"
|
||||
)
|
||||
|
||||
|
||||
def stage_tree(destination: Path, epoch: int) -> Path:
|
||||
sources = source_paths()
|
||||
assert_complete_source(sources)
|
||||
top = destination / PACKAGE_NAME
|
||||
top.mkdir(parents=True)
|
||||
files: list[dict[str, Any]] = []
|
||||
for source in sources:
|
||||
relative = source.relative_to(ROOT)
|
||||
target = top / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, target)
|
||||
mode = normalized_mode(source)
|
||||
os.chmod(target, mode)
|
||||
os.utime(target, (epoch, epoch))
|
||||
files.append(
|
||||
{
|
||||
"path": relative.as_posix(),
|
||||
"mode": f"{mode:04o}",
|
||||
"size": target.stat().st_size,
|
||||
"sha256": sha256(target),
|
||||
}
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": "codex-multimodel-orchestrator",
|
||||
"version": VERSION,
|
||||
"source_date_epoch": epoch,
|
||||
"manifest_excludes": ["PACKAGE-MANIFEST.json"],
|
||||
"file_count": len(files),
|
||||
"total_bytes": sum(int(item["size"]) for item in files),
|
||||
"files": files,
|
||||
}
|
||||
manifest_path = top / "PACKAGE-MANIFEST.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(manifest_path, 0o644)
|
||||
os.utime(manifest_path, (epoch, epoch))
|
||||
for directory in sorted((path for path in top.rglob("*") if path.is_dir()), reverse=True):
|
||||
os.chmod(directory, 0o755)
|
||||
os.utime(directory, (epoch, epoch))
|
||||
os.chmod(top, 0o755)
|
||||
os.utime(top, (epoch, epoch))
|
||||
return top
|
||||
|
||||
|
||||
def tree_entries(top: Path) -> list[Path]:
|
||||
return [top, *sorted(top.rglob("*"), key=lambda item: item.relative_to(top.parent).as_posix())]
|
||||
|
||||
|
||||
def build_tar_gz(top: Path, output: Path, epoch: int) -> None:
|
||||
with output.open("wb") as raw:
|
||||
with gzip.GzipFile(
|
||||
filename="", mode="wb", fileobj=raw, compresslevel=9, mtime=epoch
|
||||
) as compressed:
|
||||
with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as archive:
|
||||
for path in tree_entries(top):
|
||||
arcname = path.relative_to(top.parent).as_posix()
|
||||
info = archive.gettarinfo(str(path), arcname=arcname)
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = "root"
|
||||
info.gname = "root"
|
||||
info.mtime = epoch
|
||||
info.mode = 0o755 if path.is_dir() else normalized_mode(path)
|
||||
if path.is_file():
|
||||
with path.open("rb") as handle:
|
||||
archive.addfile(info, handle)
|
||||
else:
|
||||
archive.addfile(info)
|
||||
|
||||
|
||||
def build_zip(top: Path, output: Path, epoch: int) -> None:
|
||||
timestamp = time.gmtime(max(epoch, 315532800))[:6]
|
||||
with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||||
for path in tree_entries(top):
|
||||
arcname = path.relative_to(top.parent).as_posix() + ("/" if path.is_dir() else "")
|
||||
info = zipfile.ZipInfo(arcname, timestamp)
|
||||
info.create_system = 3
|
||||
mode = 0o755 if path.is_dir() else normalized_mode(path)
|
||||
info.external_attr = (stat.S_IFDIR if path.is_dir() else stat.S_IFREG) << 16
|
||||
info.external_attr |= mode << 16
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.flag_bits |= 0x800
|
||||
archive.writestr(info, b"" if path.is_dir() else path.read_bytes())
|
||||
|
||||
|
||||
def build_once(destination: Path, epoch: int) -> tuple[Path, Path]:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
stage_root = destination / "stage"
|
||||
stage_root.mkdir()
|
||||
top = stage_tree(stage_root, epoch)
|
||||
tar_path = destination / f"{PACKAGE_NAME}-linux.tar.gz"
|
||||
zip_path = destination / f"{PACKAGE_NAME}-linux.zip"
|
||||
build_tar_gz(top, tar_path, epoch)
|
||||
build_zip(top, zip_path, epoch)
|
||||
return tar_path, zip_path
|
||||
|
||||
|
||||
def validate_source(report_path: Path | None = None) -> None:
|
||||
command = [sys.executable, str(ROOT / "scripts" / "validate_package.py")]
|
||||
if report_path:
|
||||
command.extend(("--report", str(report_path)))
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
env=filtered_environment(extra={"PYTHONDONTWRITEBYTECODE": "1"}),
|
||||
check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise RuntimeError("package validation failed")
|
||||
|
||||
|
||||
def write_checksums(paths: Iterable[Path], output: Path) -> None:
|
||||
lines = [f"{sha256(path)} {path.name}" for path in sorted(paths, key=lambda item: item.name)]
|
||||
output.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _atomic_copy(source: Path, destination: Path) -> None:
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, raw_temporary = tempfile.mkstemp(prefix=f".{destination.name}.tmp-", dir=destination.parent)
|
||||
temporary = Path(raw_temporary)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as output_handle, source.open("rb") as input_handle:
|
||||
shutil.copyfileobj(input_handle, output_handle, length=1024 * 1024)
|
||||
output_handle.flush()
|
||||
os.fsync(output_handle.fileno())
|
||||
os.chmod(temporary, 0o644)
|
||||
os.replace(temporary, destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _publish_artifacts(
|
||||
sources: Iterable[tuple[Path, Path]],
|
||||
*,
|
||||
verify: Callable[[], None] | None = None,
|
||||
) -> None:
|
||||
"""Publish a release set and restore every prior artifact on failure."""
|
||||
|
||||
pairs = list(sources)
|
||||
destinations = [destination for _source, destination in pairs]
|
||||
if len(set(destinations)) != len(destinations):
|
||||
raise RuntimeError("release publication contains duplicate destinations")
|
||||
for source, destination in pairs:
|
||||
source_mode = source.lstat().st_mode
|
||||
if not stat.S_ISREG(source_mode):
|
||||
raise RuntimeError(f"release publication source is not a regular file: {source}")
|
||||
try:
|
||||
destination_mode = destination.lstat().st_mode
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not stat.S_ISREG(destination_mode):
|
||||
raise RuntimeError(f"refusing to replace a non-regular release artifact: {destination}")
|
||||
prepared: dict[Path, Path] = {}
|
||||
backups: dict[Path, Path] = {}
|
||||
published: list[Path] = []
|
||||
retained_backups: set[Path] = set()
|
||||
nonce = f"{os.getpid()}-{time.time_ns()}"
|
||||
try:
|
||||
for source, destination in pairs:
|
||||
temporary = destination.with_name(f".{destination.name}.publish-{nonce}")
|
||||
_atomic_copy(source, temporary)
|
||||
prepared[destination] = temporary
|
||||
if destination.is_file():
|
||||
backup = destination.with_name(f".{destination.name}.backup-{nonce}")
|
||||
_atomic_copy(destination, backup)
|
||||
backups[destination] = backup
|
||||
for _source, destination in pairs:
|
||||
os.replace(prepared[destination], destination)
|
||||
published.append(destination)
|
||||
if verify is not None:
|
||||
verify()
|
||||
except BaseException as publish_error:
|
||||
restoration_errors: list[str] = []
|
||||
for destination in reversed(published):
|
||||
saved = backups.get(destination)
|
||||
try:
|
||||
if saved and saved.exists():
|
||||
os.replace(saved, destination)
|
||||
else:
|
||||
destination.unlink(missing_ok=True)
|
||||
except BaseException as restore_error:
|
||||
if saved and saved.exists():
|
||||
retained_backups.add(saved)
|
||||
restoration_errors.append(
|
||||
f"{destination}: {type(restore_error).__name__}: {restore_error}"
|
||||
)
|
||||
if restoration_errors:
|
||||
retained = ", ".join(str(path) for path in sorted(retained_backups)) or "none"
|
||||
raise RuntimeError(
|
||||
"release publication failed and rollback was incomplete; "
|
||||
f"retained backups: {retained}; errors: {'; '.join(restoration_errors)}"
|
||||
) from publish_error
|
||||
raise
|
||||
finally:
|
||||
for path in prepared.values():
|
||||
path.unlink(missing_ok=True)
|
||||
for path in backups.values():
|
||||
if path not in retained_backups:
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _verify_pair(tar_path: Path, zip_path: Path) -> list[dict[str, Any]]:
|
||||
results = [
|
||||
verify_archive(tar_path, source_tree=ROOT),
|
||||
verify_archive(zip_path, source_tree=ROOT),
|
||||
]
|
||||
counts = {
|
||||
(
|
||||
item["manifest_files"],
|
||||
item["total_files"],
|
||||
item["manifest_sha256"],
|
||||
item["tree_sha256"],
|
||||
)
|
||||
for item in results
|
||||
}
|
||||
if len(counts) != 1:
|
||||
raise RuntimeError("tar.gz and ZIP extracted trees are not identical")
|
||||
return results
|
||||
|
||||
|
||||
def assert_safe_output_directory(output: Path) -> None:
|
||||
if output == ROOT or ROOT in output.parents:
|
||||
raise RuntimeError(f"refusing to write release output inside the source tree: {output}")
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
value = argparse.ArgumentParser(description="Build reproducible Codex MMO release archives")
|
||||
value.add_argument("--output-dir", type=Path, default=ROOT.parent)
|
||||
value.add_argument(
|
||||
"--source-date-epoch",
|
||||
type=int,
|
||||
default=int(os.environ.get("SOURCE_DATE_EPOCH", DEFAULT_EPOCH)),
|
||||
)
|
||||
value.add_argument("--skip-validation", action="store_true")
|
||||
value.add_argument("--no-reproducibility-check", action="store_true")
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
if not 0 <= args.source_date_epoch <= 0xFFFFFFFF:
|
||||
raise ValueError("--source-date-epoch must be between 0 and 4294967295")
|
||||
output = args.output_dir.expanduser().resolve()
|
||||
assert_safe_output_directory(output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
validation_report = output / f"{PACKAGE_NAME}-VALIDATION.json"
|
||||
checksums = output / f"{PACKAGE_NAME}-SHA256SUMS.txt"
|
||||
integrity_report = output / f"{PACKAGE_NAME}-INTEGRITY.json"
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-release-primary-") as primary_raw:
|
||||
primary = Path(primary_raw)
|
||||
validation_stage: Path | None = None
|
||||
if not args.skip_validation:
|
||||
validation_stage = primary / validation_report.name
|
||||
validate_source(validation_stage)
|
||||
tar_path, zip_path = build_once(primary, args.source_date_epoch)
|
||||
verification = _verify_pair(tar_path, zip_path)
|
||||
|
||||
reproducible = True
|
||||
if not args.no_reproducibility_check:
|
||||
with tempfile.TemporaryDirectory(prefix="mmo-release-repro-") as second_raw:
|
||||
second_tar, second_zip = build_once(Path(second_raw), args.source_date_epoch)
|
||||
_verify_pair(second_tar, second_zip)
|
||||
reproducible = sha256(tar_path) == sha256(second_tar) and sha256(
|
||||
zip_path
|
||||
) == sha256(second_zip)
|
||||
if not reproducible:
|
||||
raise RuntimeError("release build is not reproducible")
|
||||
|
||||
final_tar = output / tar_path.name
|
||||
final_zip = output / zip_path.name
|
||||
intended_verification = []
|
||||
for item, destination in zip(verification, (final_tar, final_zip), strict=True):
|
||||
intended_verification.append({**item, "archive": str(destination)})
|
||||
|
||||
checksum_stage = primary / checksums.name
|
||||
write_checksums((tar_path, zip_path), checksum_stage)
|
||||
integrity_stage = primary / integrity_report.name
|
||||
integrity = {
|
||||
"schema_version": MMO_SCHEMA_VERSION,
|
||||
"package": PACKAGE_NAME,
|
||||
"source_date_epoch": args.source_date_epoch,
|
||||
"source_files": len(source_paths()),
|
||||
"reproducible": reproducible,
|
||||
"archives": intended_verification,
|
||||
"checksums": checksums.name,
|
||||
}
|
||||
integrity_stage.write_text(
|
||||
json.dumps(integrity, indent=2, sort_keys=True, allow_nan=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
artifacts = [
|
||||
(tar_path, final_tar),
|
||||
(zip_path, final_zip),
|
||||
(checksum_stage, checksums),
|
||||
(integrity_stage, integrity_report),
|
||||
]
|
||||
if validation_stage is not None:
|
||||
artifacts.append((validation_stage, validation_report))
|
||||
|
||||
def verify_published_set() -> None:
|
||||
for staged, published in artifacts:
|
||||
if sha256(staged) != sha256(published):
|
||||
raise RuntimeError(
|
||||
f"published bytes differ from verified artifact: {published.name}"
|
||||
)
|
||||
published_verification = _verify_pair(final_tar, final_zip)
|
||||
if intended_verification != published_verification:
|
||||
raise RuntimeError("published archives differ from verified build artifacts")
|
||||
|
||||
# Publish the archives and all generated release metadata as one
|
||||
# rollback-safe set. Final-name verification runs while backups remain
|
||||
# available, so a verification failure also restores the prior set.
|
||||
_publish_artifacts(artifacts, verify=verify_published_set)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"package": PACKAGE_NAME,
|
||||
"source_date_epoch": args.source_date_epoch,
|
||||
"source_files": len(source_paths()),
|
||||
"reproducible": reproducible,
|
||||
"tar_gz": {"path": str(final_tar), "sha256": sha256(final_tar)},
|
||||
"zip": {"path": str(final_zip), "sha256": sha256(final_zip)},
|
||||
"checksums": str(checksums),
|
||||
"integrity_report": str(integrity_report),
|
||||
},
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
allow_nan=False,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user