2239 lines
93 KiB
Python
2239 lines
93 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable profile evaluation, metrics extraction, and run comparison."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import copy
|
|
import datetime as dt
|
|
import json
|
|
import math
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import uuid
|
|
from collections import Counter
|
|
from collections.abc import Callable, Mapping, Sequence
|
|
from decimal import Decimal, InvalidOperation
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from mmo_gateway import route_availability
|
|
from mmo_profiles import derive_coordination_capacities
|
|
from mmo_runtime import cancel_session, iter_jobs, load_session, run_root_exec, stop_session
|
|
from mmo_snapshot import compile_profile, compile_resolved_profile
|
|
from mmo_util import (
|
|
atomic_write_json,
|
|
config_root,
|
|
event_usage,
|
|
filtered_environment,
|
|
install_root,
|
|
read_json,
|
|
read_json_object,
|
|
read_toml,
|
|
safe_name,
|
|
sha256_bytes,
|
|
stable_hash,
|
|
state_root,
|
|
strict_json_loads,
|
|
terminate_process_group,
|
|
utc_now,
|
|
validate_id,
|
|
)
|
|
from mmo_version import MMO_SCHEMA_VERSION
|
|
|
|
SUITE_FIELDS = {
|
|
"schema_version",
|
|
"id",
|
|
"name",
|
|
"description",
|
|
"profile",
|
|
"fixture",
|
|
"development_trials",
|
|
"release_trials",
|
|
"promotion",
|
|
"variants",
|
|
"tasks",
|
|
}
|
|
PROMOTION_FIELDS = {
|
|
"primary_metric",
|
|
"primary_baseline",
|
|
"direction",
|
|
"strongest_success_tolerance",
|
|
"minimum_relative_improvement",
|
|
"minimum_absolute_improvement",
|
|
"worker_minimum_success_contribution",
|
|
"worker_minimum_metric_contribution",
|
|
"no_regression_higher_metrics",
|
|
"no_regression_lower_metrics",
|
|
"require_complete_api_cost",
|
|
"scarce_model_keys",
|
|
}
|
|
TASK_FIELDS = {
|
|
"id",
|
|
"description",
|
|
"sandbox",
|
|
"wall_timeout_seconds",
|
|
"validation_timeout_seconds",
|
|
"prompt",
|
|
"images",
|
|
"difficulty",
|
|
"negative_control",
|
|
"route_faults",
|
|
"outcome_assertions",
|
|
"orchestration_assertions",
|
|
}
|
|
ROUTE_FAULT_TYPES = {"credential_loss", "rate_limit", "timeout"}
|
|
OUTCOME_ASSERTION_FIELDS = {
|
|
"expected_patterns",
|
|
"forbidden_patterns",
|
|
"validation_commands",
|
|
}
|
|
ORCHESTRATION_ASSERTION_FIELDS = {
|
|
"required_agents",
|
|
"forbidden_agents",
|
|
"min_peak_mcp_workers",
|
|
"max_jobs",
|
|
"min_result_acceptance_rate",
|
|
"max_contract_failures",
|
|
"max_observed_mcp_wait_ratio",
|
|
}
|
|
VARIANT_FIELDS = {
|
|
"id",
|
|
"purpose",
|
|
"profile",
|
|
"bindings",
|
|
"topology",
|
|
"worker",
|
|
"comparison_class",
|
|
"access_product",
|
|
}
|
|
VARIANT_TOPOLOGIES = {"root_only", "root_plus_worker", "full", "full_without_worker"}
|
|
VARIANT_COMPARISON_CLASSES = {
|
|
"configured_root_alone",
|
|
"strongest_single_agent",
|
|
"access_service_single_agent",
|
|
"root_plus_highest_value",
|
|
"full_profile",
|
|
"ablation",
|
|
"control",
|
|
}
|
|
EVALUATION_IMAGE_SUFFIXES = {".gif", ".jpeg", ".jpg", ".png", ".webp"}
|
|
RUN_ID_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,199}")
|
|
|
|
|
|
def _reject_unknown_fields(value: Mapping[str, Any], allowed: set[str], label: str) -> None:
|
|
unknown = sorted(set(value) - allowed)
|
|
if unknown:
|
|
raise ValueError(f"{label} has unknown fields: {', '.join(unknown)}")
|
|
|
|
|
|
def _required_string(value: Any, label: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise ValueError(f"{label} must be a non-empty string")
|
|
return value
|
|
|
|
|
|
def _required_id(value: Any, label: str) -> str:
|
|
return validate_id(_required_string(value, label), label)
|
|
|
|
|
|
def builtin_evals_root() -> Path:
|
|
return install_root() / "evals"
|
|
|
|
|
|
def user_evals_root() -> Path:
|
|
return config_root() / "evals.d"
|
|
|
|
|
|
def evaluations_state_root() -> Path:
|
|
root = state_root() / "evaluations"
|
|
root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
return root
|
|
|
|
|
|
def discover_suites() -> dict[str, dict[str, Any]]:
|
|
results: dict[str, dict[str, Any]] = {}
|
|
for source, root in (("builtin", builtin_evals_root()), ("user", user_evals_root())):
|
|
if not root.is_dir():
|
|
continue
|
|
for directory in sorted(path for path in root.iterdir() if path.is_dir()):
|
|
path = directory / "suite.toml"
|
|
if not path.is_file():
|
|
continue
|
|
with contextlib.suppress(Exception):
|
|
data = read_toml(path)
|
|
suite_id = _required_id(data["id"], "evaluation suite id")
|
|
results[suite_id] = {
|
|
"id": suite_id,
|
|
"name": data.get("name", suite_id),
|
|
"description": data.get("description", ""),
|
|
"source": source,
|
|
"path": str(directory),
|
|
"task_count": len(data.get("tasks", [])),
|
|
}
|
|
return results
|
|
|
|
|
|
def resolve_suite(value: str | Path) -> tuple[Path, dict[str, Any]]:
|
|
candidate = Path(value).expanduser()
|
|
if candidate.exists():
|
|
directory = candidate.resolve()
|
|
if directory.is_file():
|
|
directory = directory.parent
|
|
else:
|
|
suite_id = validate_id(str(value), "evaluation suite id")
|
|
suites = discover_suites()
|
|
if suite_id not in suites:
|
|
raise FileNotFoundError(f"unknown evaluation suite: {suite_id}")
|
|
directory = Path(suites[suite_id]["path"])
|
|
path = directory / "suite.toml"
|
|
data = read_toml(path)
|
|
_reject_unknown_fields(data, SUITE_FIELDS, f"evaluation suite {path}")
|
|
schema_version = data.get("schema_version")
|
|
if (
|
|
not isinstance(schema_version, int)
|
|
or isinstance(schema_version, bool)
|
|
or schema_version != MMO_SCHEMA_VERSION
|
|
):
|
|
raise ValueError(f"unsupported suite schema in {path}")
|
|
_required_id(data.get("id"), "evaluation suite id")
|
|
_required_id(data.get("profile"), "evaluation suite profile")
|
|
for field in ("name", "description"):
|
|
if field in data and not isinstance(data[field], str):
|
|
raise ValueError(f"evaluation suite {field} must be a string")
|
|
tasks = data.get("tasks")
|
|
if not isinstance(tasks, list) or not tasks:
|
|
raise ValueError("evaluation suite must contain [[tasks]]")
|
|
seen: set[str] = set()
|
|
for field, default in (("development_trials", 3), ("release_trials", 5)):
|
|
trials = data.get(field, default)
|
|
if not isinstance(trials, int) or isinstance(trials, bool) or not 1 <= trials <= 20:
|
|
raise ValueError(f"evaluation suite {field} must be an integer from 1 to 20")
|
|
variants = data.get("variants", [])
|
|
if not isinstance(variants, list):
|
|
raise ValueError("evaluation suite variants must be an array of tables")
|
|
variant_ids: set[str] = set()
|
|
for index, variant in enumerate(variants):
|
|
if not isinstance(variant, Mapping):
|
|
raise ValueError(f"evaluation variant {index} must be a table")
|
|
_reject_unknown_fields(variant, VARIANT_FIELDS, f"evaluation variant {index}")
|
|
variant_id = _required_id(variant.get("id"), f"evaluation variant {index} id")
|
|
if variant_id in variant_ids:
|
|
raise ValueError(f"duplicate evaluation variant id: {variant_id}")
|
|
variant_ids.add(variant_id)
|
|
if not isinstance(variant.get("purpose"), str) or not variant["purpose"].strip():
|
|
raise ValueError(f"evaluation variant {variant_id}: purpose is required")
|
|
if "profile" in variant and not isinstance(variant["profile"], str):
|
|
raise ValueError(f"evaluation variant {variant_id}: profile must be a string")
|
|
bindings = variant.get("bindings", {})
|
|
if not isinstance(bindings, Mapping) or not all(
|
|
isinstance(key, str) and isinstance(value, str) for key, value in bindings.items()
|
|
):
|
|
raise ValueError(f"evaluation variant {variant_id}: bindings must be a string table")
|
|
topology = variant.get("topology")
|
|
if topology not in VARIANT_TOPOLOGIES:
|
|
raise ValueError(
|
|
f"evaluation variant {variant_id}: topology must be one of "
|
|
f"{sorted(VARIANT_TOPOLOGIES)}"
|
|
)
|
|
comparison_class = variant.get("comparison_class")
|
|
if comparison_class not in VARIANT_COMPARISON_CLASSES:
|
|
raise ValueError(
|
|
f"evaluation variant {variant_id}: comparison_class must be one of "
|
|
f"{sorted(VARIANT_COMPARISON_CLASSES)}"
|
|
)
|
|
if (topology == "full_without_worker") != (comparison_class == "ablation"):
|
|
raise ValueError(
|
|
f"evaluation variant {variant_id}: full_without_worker topology and "
|
|
"ablation comparison_class must be used together"
|
|
)
|
|
worker = variant.get("worker")
|
|
if topology in {"root_plus_worker", "full_without_worker"}:
|
|
_required_id(worker, f"evaluation variant {variant_id} worker")
|
|
elif worker is not None:
|
|
raise ValueError(
|
|
f"evaluation variant {variant_id}: worker is allowed only for "
|
|
"root_plus_worker topology"
|
|
)
|
|
access_product = variant.get("access_product")
|
|
if comparison_class == "access_service_single_agent":
|
|
_required_id(
|
|
access_product,
|
|
f"evaluation variant {variant_id} access_product",
|
|
)
|
|
elif access_product is not None:
|
|
raise ValueError(
|
|
f"evaluation variant {variant_id}: access_product is reserved for "
|
|
"access-service controls"
|
|
)
|
|
required_comparisons = {
|
|
"configured_root_alone",
|
|
"strongest_single_agent",
|
|
"root_plus_highest_value",
|
|
"full_profile",
|
|
}
|
|
comparison_classes = {str(variant.get("comparison_class")) for variant in variants}
|
|
missing_comparisons = sorted(required_comparisons - comparison_classes)
|
|
if missing_comparisons:
|
|
raise ValueError(
|
|
"evaluation suite lacks required matched comparison classes: "
|
|
+ ", ".join(missing_comparisons)
|
|
)
|
|
if "access_service_single_agent" not in comparison_classes:
|
|
raise ValueError(
|
|
"evaluation suite requires at least one access_service_single_agent variant"
|
|
)
|
|
promotion = data.get("promotion")
|
|
if not isinstance(promotion, Mapping):
|
|
raise ValueError("evaluation suite promotion table is required")
|
|
_reject_unknown_fields(promotion, PROMOTION_FIELDS, "evaluation promotion")
|
|
primary_metric = _required_id(
|
|
promotion.get("primary_metric"), "evaluation promotion primary_metric"
|
|
)
|
|
if primary_metric == "score":
|
|
raise ValueError("promotion primary_metric must be success_rate, not legacy score")
|
|
primary_baseline = promotion.get("primary_baseline", "configured_root_alone")
|
|
if primary_baseline not in {"configured_root_alone", "strongest_single_agent"}:
|
|
raise ValueError(
|
|
"evaluation promotion primary_baseline must be configured_root_alone or "
|
|
"strongest_single_agent"
|
|
)
|
|
if promotion.get("direction") not in {"higher", "lower"}:
|
|
raise ValueError("evaluation promotion direction must be higher or lower")
|
|
for field, default_number in (
|
|
("strongest_success_tolerance", 0.02),
|
|
("minimum_relative_improvement", 0.10),
|
|
("minimum_absolute_improvement", 0.05),
|
|
("worker_minimum_success_contribution", 0.02),
|
|
("worker_minimum_metric_contribution", 0.10),
|
|
):
|
|
number = promotion.get(field, default_number)
|
|
if (
|
|
not isinstance(number, (int, float))
|
|
or isinstance(number, bool)
|
|
or not math.isfinite(float(number))
|
|
or float(number) < 0
|
|
):
|
|
raise ValueError(f"evaluation promotion {field} must be a non-negative number")
|
|
metric_sets: list[set[str]] = []
|
|
for field in ("no_regression_higher_metrics", "no_regression_lower_metrics"):
|
|
metrics = promotion.get(field, [])
|
|
if not isinstance(metrics, list) or not all(
|
|
isinstance(metric, str) and metric.strip() for metric in metrics
|
|
):
|
|
raise ValueError(f"evaluation promotion {field} must be strings")
|
|
normalized = {validate_id(metric, f"evaluation promotion {field}") for metric in metrics}
|
|
if len(normalized) != len(metrics):
|
|
raise ValueError(f"evaluation promotion {field} contains duplicates")
|
|
metric_sets.append(normalized)
|
|
if metric_sets[0] & metric_sets[1]:
|
|
raise ValueError("a non-regression metric cannot have both directions")
|
|
scarce_model_keys = promotion.get("scarce_model_keys", [])
|
|
if not isinstance(scarce_model_keys, list) or not all(
|
|
isinstance(key, str) and key.strip() for key in scarce_model_keys
|
|
):
|
|
raise ValueError("evaluation promotion scarce_model_keys must be an array of model keys")
|
|
if len(scarce_model_keys) != len(set(scarce_model_keys)):
|
|
raise ValueError("evaluation promotion scarce_model_keys contains duplicates")
|
|
if primary_metric == "scarce_tier_request_units" and not scarce_model_keys:
|
|
raise ValueError(
|
|
"evaluation promotion scarce_tier_request_units requires scarce_model_keys"
|
|
)
|
|
if not isinstance(promotion.get("require_complete_api_cost", True), bool):
|
|
raise ValueError("evaluation promotion require_complete_api_cost must be boolean")
|
|
for index, task in enumerate(tasks):
|
|
if not isinstance(task, Mapping):
|
|
raise ValueError(f"evaluation task {index} must be a table")
|
|
_reject_unknown_fields(task, TASK_FIELDS, f"evaluation task {index}")
|
|
task_id = _required_id(task.get("id"), f"task {index} id")
|
|
if task_id in seen:
|
|
raise ValueError(f"duplicate evaluation task id: {task_id}")
|
|
seen.add(task_id)
|
|
if not isinstance(task.get("prompt"), str) or not task["prompt"].strip():
|
|
raise ValueError(f"task {task_id}: prompt is required")
|
|
if "description" in task and not isinstance(task["description"], str):
|
|
raise ValueError(f"task {task_id}: description must be a string")
|
|
sandbox = task.get("sandbox", "read-only")
|
|
if not isinstance(sandbox, str) or sandbox not in {"read-only", "workspace-write"}:
|
|
raise ValueError(f"task {task_id}: invalid sandbox")
|
|
for field, default, maximum in (
|
|
("wall_timeout_seconds", 1800, 172_800),
|
|
("validation_timeout_seconds", 300, 86_400),
|
|
):
|
|
timeout = task.get(field, default)
|
|
if (
|
|
not isinstance(timeout, int)
|
|
or isinstance(timeout, bool)
|
|
or not 1 <= timeout <= maximum
|
|
):
|
|
raise ValueError(f"task {task_id}: {field} must be an integer from 1 to {maximum}")
|
|
for field in ("images",):
|
|
value = task.get(field, [])
|
|
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
|
raise ValueError(f"task {task_id}: {field} must be an array of strings")
|
|
if any(not item.strip() for item in value):
|
|
raise ValueError(f"task {task_id}: {field} entries must be non-empty strings")
|
|
difficulty = task.get("difficulty", "medium")
|
|
if difficulty not in {"easy", "medium", "hard", "adversarial"}:
|
|
raise ValueError(f"task {task_id}: invalid difficulty")
|
|
if not isinstance(task.get("negative_control", False), bool):
|
|
raise ValueError(f"task {task_id}: negative_control must be boolean")
|
|
route_faults = task.get("route_faults", {})
|
|
if not isinstance(route_faults, Mapping) or not all(
|
|
isinstance(route, str)
|
|
and route.strip()
|
|
and isinstance(fault, str)
|
|
and fault in ROUTE_FAULT_TYPES
|
|
for route, fault in route_faults.items()
|
|
):
|
|
raise ValueError(
|
|
f"task {task_id}: route_faults must map route IDs to one of "
|
|
f"{sorted(ROUTE_FAULT_TYPES)}"
|
|
)
|
|
for route in route_faults:
|
|
validate_id(route, f"task {task_id} disabled route")
|
|
for image in task.get("images", []):
|
|
image_path = Path(image)
|
|
if (
|
|
image_path.is_absolute()
|
|
or ".." in image_path.parts
|
|
or image_path.suffix.lower() not in EVALUATION_IMAGE_SUFFIXES
|
|
):
|
|
raise ValueError(f"task {task_id}: images must be supported relative fixture paths")
|
|
outcomes = task.get("outcome_assertions")
|
|
if not isinstance(outcomes, Mapping):
|
|
raise ValueError(f"task {task_id}: outcome_assertions table is required")
|
|
_reject_unknown_fields(outcomes, OUTCOME_ASSERTION_FIELDS, f"task {task_id} outcomes")
|
|
if not any(outcomes.get(field) for field in OUTCOME_ASSERTION_FIELDS):
|
|
raise ValueError(f"task {task_id}: outcome_assertions cannot be empty")
|
|
for field in OUTCOME_ASSERTION_FIELDS:
|
|
entries = outcomes.get(field, [])
|
|
if not isinstance(entries, list) or not all(isinstance(item, str) for item in entries):
|
|
raise ValueError(f"task {task_id}: {field} must be an array of strings")
|
|
if any(not item.strip() for item in entries):
|
|
raise ValueError(f"task {task_id}: {field} entries must be non-empty")
|
|
if field != "validation_commands":
|
|
for pattern in entries:
|
|
re.compile(pattern)
|
|
assertions = task.get("orchestration_assertions", {})
|
|
if not isinstance(assertions, Mapping):
|
|
raise ValueError(f"task {task_id}: orchestration_assertions must be a table")
|
|
_reject_unknown_fields(
|
|
assertions,
|
|
ORCHESTRATION_ASSERTION_FIELDS,
|
|
f"task {task_id} orchestration_assertions",
|
|
)
|
|
for field in ("required_agents", "forbidden_agents"):
|
|
agents = assertions.get(field, [])
|
|
if not isinstance(agents, list) or not all(isinstance(item, str) for item in agents):
|
|
raise ValueError(f"task {task_id}: {field} must be an array of agent IDs")
|
|
if len(agents) != len(set(agents)):
|
|
raise ValueError(f"task {task_id}: {field} contains duplicates")
|
|
for agent in agents:
|
|
validate_id(agent, f"task {task_id} {field} agent")
|
|
overlap = set(assertions.get("required_agents", [])) & set(
|
|
assertions.get("forbidden_agents", [])
|
|
)
|
|
if overlap:
|
|
raise ValueError(
|
|
f"task {task_id}: agents cannot be both required and forbidden: {sorted(overlap)}"
|
|
)
|
|
for field in ("min_peak_mcp_workers", "max_jobs", "max_contract_failures"):
|
|
if field not in assertions:
|
|
continue
|
|
number = assertions[field]
|
|
if not isinstance(number, int) or isinstance(number, bool) or number < 0:
|
|
raise ValueError(f"task {task_id}: {field} must be a non-negative integer")
|
|
for field in ("min_result_acceptance_rate", "max_observed_mcp_wait_ratio"):
|
|
if field not in assertions:
|
|
continue
|
|
number = assertions[field]
|
|
if (
|
|
not isinstance(number, (int, float))
|
|
or isinstance(number, bool)
|
|
or not math.isfinite(float(number))
|
|
or not 0 <= float(number) <= 1
|
|
):
|
|
raise ValueError(f"task {task_id}: {field} must be a number from 0 to 1")
|
|
fixture = data.get("fixture")
|
|
fixture_path: Path | None = None
|
|
if fixture is not None:
|
|
if not isinstance(fixture, str) or not fixture.strip():
|
|
raise ValueError("suite fixture must be a non-empty relative path")
|
|
if Path(fixture).is_absolute():
|
|
raise ValueError("suite fixture must be a non-empty relative path")
|
|
candidate = directory / fixture
|
|
if candidate.is_symlink():
|
|
raise ValueError("suite fixture may not be a symbolic link")
|
|
fixture_path = candidate.resolve()
|
|
if directory not in fixture_path.parents or not fixture_path.is_dir():
|
|
raise ValueError("suite fixture must be a directory inside the suite")
|
|
for member in fixture_path.rglob("*"):
|
|
relative = member.relative_to(fixture_path)
|
|
if member.is_symlink():
|
|
raise ValueError(f"evaluation fixtures may not contain symlinks: {relative}")
|
|
if not member.is_dir() and not member.is_file():
|
|
raise ValueError(f"evaluation fixtures may not contain special files: {relative}")
|
|
for task in tasks:
|
|
if task.get("images") and fixture_path is None:
|
|
raise ValueError(f"task {task['id']}: images require a suite fixture")
|
|
for image in task.get("images", []):
|
|
assert fixture_path is not None
|
|
candidate = fixture_path / image
|
|
if candidate.is_symlink():
|
|
raise ValueError(f"task {task['id']}: image may not be a symbolic link: {image}")
|
|
resolved_image = candidate.resolve()
|
|
if fixture_path not in resolved_image.parents or not resolved_image.is_file():
|
|
raise ValueError(f"task {task['id']}: image is not a fixture file: {image}")
|
|
task_ids = {str(task["id"]) for task in tasks}
|
|
for auxiliary_name in ("holdout", "mutations"):
|
|
auxiliary_root = directory / auxiliary_name
|
|
if not auxiliary_root.exists():
|
|
continue
|
|
if auxiliary_root.is_symlink() or not auxiliary_root.is_dir():
|
|
raise ValueError(f"evaluation {auxiliary_name} must be a static directory")
|
|
for member in auxiliary_root.rglob("*"):
|
|
relative = member.relative_to(auxiliary_root)
|
|
if member.is_symlink():
|
|
raise ValueError(
|
|
f"evaluation {auxiliary_name} may not contain symlinks: {relative}"
|
|
)
|
|
if not member.is_dir() and not member.is_file():
|
|
raise ValueError(
|
|
f"evaluation {auxiliary_name} may not contain special files: {relative}"
|
|
)
|
|
if relative.parts and relative.parts[0] not in task_ids:
|
|
raise ValueError(
|
|
f"evaluation {auxiliary_name} references unknown task: {relative.parts[0]}"
|
|
)
|
|
if auxiliary_name == "mutations" and member.is_file() and member.suffix != ".patch":
|
|
raise ValueError(f"evaluation mutations must be .patch files: {relative}")
|
|
return directory, data
|
|
|
|
|
|
def validate_suite(value: str | Path) -> dict[str, Any]:
|
|
directory, data = resolve_suite(value)
|
|
return {
|
|
"id": data["id"],
|
|
"name": data.get("name", data["id"]),
|
|
"path": str(directory),
|
|
"tasks": [task["id"] for task in data["tasks"]],
|
|
"hidden_mutation_tasks": sorted(path.name for path in (directory / "mutations").iterdir())
|
|
if (directory / "mutations").is_dir()
|
|
else [],
|
|
"holdout_tasks": sorted(path.name for path in (directory / "holdout").iterdir())
|
|
if (directory / "holdout").is_dir()
|
|
else [],
|
|
"valid": True,
|
|
}
|
|
|
|
|
|
def _variant_snapshot(
|
|
configured_profile: str | Path,
|
|
configured_bindings: Mapping[str, str] | None,
|
|
variant: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Compile one matched evaluation topology as an immutable v2 snapshot."""
|
|
|
|
selected_profile: str | Path = variant.get("profile") or configured_profile
|
|
bindings = dict(configured_bindings or {})
|
|
bindings.update(dict(variant.get("bindings") or {}))
|
|
base = compile_profile(selected_profile, bindings=bindings)
|
|
topology = str(variant["topology"])
|
|
if topology == "full":
|
|
return base
|
|
|
|
resolved = copy.deepcopy(base["resolved"])
|
|
root_id = str(resolved["profile"]["root"])
|
|
worker = variant.get("worker")
|
|
if topology == "root_only":
|
|
resolved["agents"][root_id]["can_spawn"] = []
|
|
resolved["coordination"].update(
|
|
{
|
|
"max_active_agents": 1,
|
|
"max_depth": 0,
|
|
"max_active_writers": 0,
|
|
"feasible_max_active_agents": 1,
|
|
}
|
|
)
|
|
elif topology == "root_plus_worker":
|
|
if not isinstance(worker, str) or worker not in resolved["agents"]:
|
|
raise ValueError(
|
|
f"evaluation variant {variant['id']}: unknown root-plus worker {worker!r}"
|
|
)
|
|
if worker not in resolved["agents"][root_id]["can_spawn"]:
|
|
raise ValueError(
|
|
f"evaluation variant {variant['id']}: root cannot spawn worker {worker!r}"
|
|
)
|
|
resolved["agents"][root_id]["can_spawn"] = [worker]
|
|
resolved["agents"][worker]["can_spawn"] = []
|
|
writable = (
|
|
resolved["agents"][worker]["permissions"] == "workspace-write"
|
|
and "mcp" in resolved["agents"][worker]["backends"]
|
|
)
|
|
resolved["coordination"].update(
|
|
{
|
|
"max_active_agents": 2,
|
|
"max_depth": 1,
|
|
"max_children_per_agent": 1,
|
|
"max_active_writers": 1 if writable else 0,
|
|
"feasible_max_active_agents": 2,
|
|
}
|
|
)
|
|
for agent_id, agent in resolved["agents"].items():
|
|
if agent_id not in {root_id, worker}:
|
|
agent["can_spawn"] = []
|
|
else:
|
|
if not isinstance(worker, str) or worker not in resolved["agents"]:
|
|
raise ValueError(
|
|
f"evaluation variant {variant['id']}: unknown ablated worker {worker!r}"
|
|
)
|
|
if worker == root_id:
|
|
raise ValueError(f"evaluation variant {variant['id']}: root cannot be ablated")
|
|
for agent in resolved["agents"].values():
|
|
agent["can_spawn"] = [child for child in agent["can_spawn"] if child != worker]
|
|
resolved["agents"][worker]["can_spawn"] = []
|
|
|
|
# A matched topology may not retain dormant control edges to roles that
|
|
# its spawn graph cannot create. Prune both outgoing spawn and control
|
|
# authority to the exact root-reachable role set before compiling guidance
|
|
# and Agent-MCP tools for the variant.
|
|
reachable = {root_id}
|
|
frontier = [root_id]
|
|
while frontier:
|
|
parent = frontier.pop()
|
|
for child in resolved["agents"][parent]["can_spawn"]:
|
|
if child not in reachable:
|
|
reachable.add(child)
|
|
frontier.append(child)
|
|
for agent_id, agent in resolved["agents"].items():
|
|
if agent_id not in reachable:
|
|
agent["can_spawn"] = []
|
|
agent["controls"] = {}
|
|
continue
|
|
agent["controls"] = {
|
|
target: grant
|
|
for target, grant in agent.get("controls", {}).items()
|
|
if target in reachable
|
|
}
|
|
coordination = resolved["coordination"]
|
|
capacities = derive_coordination_capacities(
|
|
resolved["agents"],
|
|
resolved["resources"],
|
|
root_agent=root_id,
|
|
max_depth=int(coordination["max_depth"]),
|
|
)
|
|
feasible_active = int(capacities["feasible_max_active_agents"])
|
|
if resolved["agents"][root_id]["can_spawn"] and feasible_active < 2:
|
|
raise ValueError(
|
|
f"evaluation variant {variant['id']}: no configured worker can run within "
|
|
"the root's resource reservation"
|
|
)
|
|
coordination.update(
|
|
{
|
|
"max_active_agents": min(int(coordination["max_active_agents"]), feasible_active),
|
|
"max_active_writers": min(
|
|
int(coordination["max_active_writers"]), int(capacities["writable_slots"])
|
|
),
|
|
"feasible_max_active_agents": feasible_active,
|
|
}
|
|
)
|
|
resolved["evaluation_variant"] = {
|
|
"id": variant["id"],
|
|
"comparison_class": variant["comparison_class"],
|
|
"topology": topology,
|
|
"worker": worker,
|
|
"source_snapshot_hash": base["manifest"]["snapshot_hash"],
|
|
}
|
|
hash_payload = copy.deepcopy(resolved)
|
|
hash_payload.pop("logical_hash", None)
|
|
resolved["logical_hash"] = stable_hash(hash_payload)
|
|
return compile_resolved_profile(resolved)
|
|
|
|
|
|
def _run_id(profile_id: str, suite_id: str) -> str:
|
|
stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
return f"{stamp}-{safe_name(profile_id)}-{safe_name(suite_id)}-{uuid.uuid4().hex[:8]}"
|
|
|
|
|
|
def _copy_fixture(source: Path | None, destination: Path) -> None:
|
|
destination.mkdir(parents=True, exist_ok=False)
|
|
if source is None:
|
|
return
|
|
for path in source.rglob("*"):
|
|
relative = path.relative_to(source)
|
|
target = destination / relative
|
|
if path.is_symlink():
|
|
raise ValueError(f"evaluation fixtures may not contain symlinks: {relative}")
|
|
if path.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
elif path.is_file():
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(path, target)
|
|
else:
|
|
raise ValueError(f"evaluation fixtures may not contain special files: {relative}")
|
|
|
|
|
|
def _copy_overlay(source: Path, destination: Path) -> list[str]:
|
|
copied: list[str] = []
|
|
for path in sorted(source.rglob("*")):
|
|
relative = path.relative_to(source)
|
|
target = destination / relative
|
|
if path.is_symlink():
|
|
raise ValueError(f"evaluation holdouts may not contain symlinks: {relative}")
|
|
if path.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
elif path.is_file():
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(path, target)
|
|
copied.append(relative.as_posix())
|
|
else:
|
|
raise ValueError(f"evaluation holdouts may not contain special files: {relative}")
|
|
return copied
|
|
|
|
|
|
def _prepare_hidden_trial(
|
|
suite_dir: Path, task_id: str, trial: int, workspace: Path
|
|
) -> dict[str, Any]:
|
|
"""Apply one deterministic hidden mutation shared by every matched variant."""
|
|
|
|
root = suite_dir / "mutations" / task_id
|
|
candidates = sorted(root.glob("*.patch")) if root.is_dir() else []
|
|
if not candidates:
|
|
return {"configured": False, "selected": None, "sha256": None}
|
|
selected = candidates[(trial - 1) % len(candidates)].resolve()
|
|
completed = subprocess.run(
|
|
["git", "apply", "--binary", str(selected)],
|
|
cwd=workspace,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
env=filtered_environment(extra={"GIT_TERMINAL_PROMPT": "0"}),
|
|
)
|
|
if completed.returncode != 0:
|
|
raise RuntimeError(
|
|
f"hidden mutation failed for {task_id}: {selected.name}: {completed.stderr.strip()}"
|
|
)
|
|
return {
|
|
"configured": True,
|
|
"selected": selected.name,
|
|
"sha256": sha256_bytes(selected.read_bytes()),
|
|
"candidate_count": len(candidates),
|
|
}
|
|
|
|
|
|
def _initialize_fixture_repository(workspace: Path) -> None:
|
|
"""Create the Git boundary required by isolated writable Agent MCP workers."""
|
|
|
|
commands = (
|
|
["git", "init", "--quiet"],
|
|
["git", "config", "user.name", "MMO Evaluation"],
|
|
["git", "config", "user.email", "eval@invalid.example"],
|
|
["git", "add", "--all"],
|
|
["git", "commit", "--quiet", "--allow-empty", "-m", "matched evaluation fixture"],
|
|
)
|
|
for command in commands:
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=workspace,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
env=filtered_environment(extra={"GIT_TERMINAL_PROMPT": "0"}),
|
|
)
|
|
if completed.returncode != 0:
|
|
raise RuntimeError(
|
|
f"unable to initialize evaluation fixture repository: {completed.stderr.strip()}"
|
|
)
|
|
|
|
|
|
def _install_holdout(suite_dir: Path, task_id: str, workspace: Path) -> dict[str, Any]:
|
|
source = suite_dir / "holdout" / task_id
|
|
if not source.is_dir():
|
|
return {"configured": False, "files": []}
|
|
files = _copy_overlay(source, workspace)
|
|
return {"configured": True, "files": files}
|
|
|
|
|
|
def _run_validation(command: str, cwd: Path, timeout: int) -> dict[str, Any]:
|
|
started = dt.datetime.now(dt.UTC)
|
|
process = subprocess.Popen(
|
|
["sh", "-lc", command],
|
|
cwd=cwd,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
start_new_session=True,
|
|
env=filtered_environment(extra={"GIT_TERMINAL_PROMPT": "0", "NO_COLOR": "1"}),
|
|
)
|
|
try:
|
|
stdout, stderr = process.communicate(timeout=timeout)
|
|
return {
|
|
"command": command,
|
|
"exit_code": process.returncode,
|
|
"passed": process.returncode == 0,
|
|
"stdout": stdout[-12000:],
|
|
"stderr": stderr[-12000:],
|
|
"elapsed_seconds": (dt.datetime.now(dt.UTC) - started).total_seconds(),
|
|
}
|
|
except subprocess.TimeoutExpired:
|
|
terminate_process_group(process.pid)
|
|
stdout, stderr = process.communicate()
|
|
return {
|
|
"command": command,
|
|
"exit_code": 124,
|
|
"passed": False,
|
|
"stdout": stdout[-12000:],
|
|
"stderr": stderr[-12000:],
|
|
"elapsed_seconds": (dt.datetime.now(dt.UTC) - started).total_seconds(),
|
|
"error": "validation command timed out",
|
|
}
|
|
|
|
|
|
def _numeric_validation_metrics(validations: Sequence[Mapping[str, Any]]) -> dict[str, float]:
|
|
"""Read typed numeric metrics from a validator's final JSON object."""
|
|
|
|
buckets: dict[str, list[float]] = {}
|
|
for validation in validations:
|
|
stdout = str(validation.get("stdout") or "")
|
|
payload: Mapping[str, Any] | None = None
|
|
for line in reversed(stdout.splitlines()):
|
|
with contextlib.suppress(json.JSONDecodeError, ValueError):
|
|
candidate = strict_json_loads(line)
|
|
if isinstance(candidate, Mapping):
|
|
payload = candidate
|
|
break
|
|
if payload is None or not isinstance(payload.get("metrics"), Mapping):
|
|
continue
|
|
for name, value in payload["metrics"].items():
|
|
if (
|
|
isinstance(name, str)
|
|
and validate_id(name, "validator metric")
|
|
and isinstance(value, (int, float))
|
|
and not isinstance(value, bool)
|
|
and math.isfinite(float(value))
|
|
):
|
|
buckets.setdefault(name, []).append(float(value))
|
|
return {name: sum(values) / len(values) for name, values in sorted(buckets.items())}
|
|
|
|
|
|
def _pattern_results(output: str, task: Mapping[str, Any]) -> dict[str, Any]:
|
|
assertions = task["outcome_assertions"]
|
|
expected = [
|
|
{"pattern": pattern, "matched": bool(re.search(pattern, output, flags=re.MULTILINE))}
|
|
for pattern in assertions.get("expected_patterns", [])
|
|
]
|
|
forbidden = [
|
|
{"pattern": pattern, "matched": bool(re.search(pattern, output, flags=re.MULTILINE))}
|
|
for pattern in assertions.get("forbidden_patterns", [])
|
|
]
|
|
return {
|
|
"expected": expected,
|
|
"forbidden": forbidden,
|
|
"passed": all(item["matched"] for item in expected)
|
|
and not any(item["matched"] for item in forbidden),
|
|
}
|
|
|
|
|
|
def _jsonl_rows(path: Path) -> list[dict[str, Any]]:
|
|
rows: list[dict[str, Any]] = []
|
|
if not path.is_file():
|
|
return rows
|
|
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
for line in handle:
|
|
with contextlib.suppress(json.JSONDecodeError, ValueError):
|
|
value = strict_json_loads(line)
|
|
if isinstance(value, dict):
|
|
rows.append(value)
|
|
return rows
|
|
|
|
|
|
def _event_usage(path: Path) -> dict[str, int]:
|
|
return event_usage(path)
|
|
|
|
|
|
def _retry_count(path: Path) -> int:
|
|
observed = 0
|
|
|
|
def visit(value: Any) -> None:
|
|
nonlocal observed
|
|
if isinstance(value, Mapping):
|
|
for key, child in value.items():
|
|
normalized = str(key).lower()
|
|
if (
|
|
normalized in {"retry_count", "retries"}
|
|
and isinstance(child, int)
|
|
and not isinstance(child, bool)
|
|
):
|
|
observed += max(0, child)
|
|
visit(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
visit(child)
|
|
|
|
for row in _jsonl_rows(path):
|
|
visit(row)
|
|
return observed
|
|
|
|
|
|
def _estimated_token_cost(usage: Mapping[str, Any], model: Mapping[str, Any]) -> float | None:
|
|
input_price = model.get("input_cost_per_million")
|
|
output_price = model.get("output_cost_per_million")
|
|
if input_price is None or output_price is None:
|
|
return None
|
|
input_tokens = usage.get("input_tokens", 0)
|
|
cached_tokens = usage.get("cached_input_tokens", 0)
|
|
cache_write_tokens = usage.get("cache_write_input_tokens", 0)
|
|
output_tokens = usage.get("output_tokens", 0)
|
|
if any(
|
|
not isinstance(value, int) or isinstance(value, bool) or value < 0
|
|
for value in (input_tokens, cached_tokens, cache_write_tokens, output_tokens)
|
|
):
|
|
return None
|
|
if cached_tokens + cache_write_tokens > input_tokens:
|
|
return None
|
|
cached_price = model.get("cached_input_cost_per_million")
|
|
cache_write_price = model.get("cache_write_input_cost_per_million")
|
|
if cached_tokens and cached_price is None:
|
|
return None
|
|
if cache_write_tokens and cache_write_price is None:
|
|
return None
|
|
try:
|
|
amount = (
|
|
Decimal(input_tokens - cached_tokens - cache_write_tokens) * Decimal(str(input_price))
|
|
+ Decimal(cached_tokens) * Decimal(str(cached_price or 0))
|
|
+ Decimal(cache_write_tokens) * Decimal(str(cache_write_price or 0))
|
|
+ Decimal(output_tokens) * Decimal(str(output_price))
|
|
) / Decimal(1_000_000)
|
|
result = float(amount)
|
|
except (InvalidOperation, OverflowError, ValueError):
|
|
return None
|
|
return result if math.isfinite(result) else None
|
|
|
|
|
|
def _observed_api_cost(path: Path) -> float | None:
|
|
maxima: list[float] = []
|
|
|
|
def visit(value: Any) -> None:
|
|
if isinstance(value, Mapping):
|
|
for key, child in value.items():
|
|
if (
|
|
str(key).casefold()
|
|
in {
|
|
"cost",
|
|
"cost_usd",
|
|
"total_cost",
|
|
"total_cost_usd",
|
|
}
|
|
and isinstance(child, (int, float))
|
|
and not isinstance(child, bool)
|
|
):
|
|
amount = float(child)
|
|
if math.isfinite(amount) and amount >= 0:
|
|
maxima.append(amount)
|
|
visit(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
visit(child)
|
|
|
|
for row in _jsonl_rows(path):
|
|
visit(row)
|
|
return max(maxima) if maxima else None
|
|
|
|
|
|
def _call_cost_ledgers(
|
|
*,
|
|
usage: Mapping[str, Any],
|
|
model: Mapping[str, Any],
|
|
route: Mapping[str, Any],
|
|
events_path: Path,
|
|
elapsed_seconds: float,
|
|
) -> dict[str, Any]:
|
|
billing = str(route["billing_mode"])
|
|
access_product = str(route["access_product"])
|
|
tokens = sum(
|
|
int(usage.get(field, 0))
|
|
for field in ("input_tokens", "output_tokens")
|
|
if isinstance(usage.get(field, 0), int) and not isinstance(usage.get(field, 0), bool)
|
|
)
|
|
actual = _observed_api_cost(events_path) if billing == "api" else None
|
|
estimate = _estimated_token_cost(usage, model)
|
|
subscription: dict[str, Any] = {}
|
|
local: dict[str, float] = {}
|
|
if billing in {"subscription", "chatgpt_subscription"}:
|
|
subscription[access_product] = {"request_units": 1, "observed_tokens": tokens}
|
|
if billing == "local":
|
|
local[access_product] = max(0.0, elapsed_seconds)
|
|
return {
|
|
"actual_api_usd": actual,
|
|
"actual_api_usd_complete": billing != "api" or actual is not None,
|
|
"api_equivalent_estimate_usd": estimate,
|
|
"api_equivalent_estimate_complete": estimate is not None,
|
|
"subscription_units": subscription,
|
|
"local_resource_seconds": local,
|
|
}
|
|
|
|
|
|
def _merge_cost_ledgers(items: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
actual = 0.0
|
|
estimated = 0.0
|
|
actual_complete = True
|
|
estimated_complete = True
|
|
subscription: dict[str, Counter[str]] = {}
|
|
local: Counter[str] = Counter()
|
|
for item in items:
|
|
actual_complete = actual_complete and bool(item.get("actual_api_usd_complete"))
|
|
estimated_complete = estimated_complete and bool(
|
|
item.get("api_equivalent_estimate_complete")
|
|
)
|
|
if item.get("actual_api_usd") is not None:
|
|
actual += float(item["actual_api_usd"])
|
|
if item.get("api_equivalent_estimate_usd") is not None:
|
|
estimated += float(item["api_equivalent_estimate_usd"])
|
|
for product, units in item.get("subscription_units", {}).items():
|
|
bucket = subscription.setdefault(str(product), Counter())
|
|
bucket.update({str(key): int(value) for key, value in units.items()})
|
|
local.update(
|
|
{
|
|
str(product): float(seconds)
|
|
for product, seconds in item.get("local_resource_seconds", {}).items()
|
|
}
|
|
)
|
|
return {
|
|
"actual_api_usd": actual if actual_complete else None,
|
|
"actual_api_usd_complete": actual_complete,
|
|
"api_equivalent_estimate_usd": estimated if estimated_complete else None,
|
|
"api_equivalent_estimate_complete": estimated_complete,
|
|
"subscription_units": {
|
|
product: dict(units) for product, units in sorted(subscription.items())
|
|
},
|
|
"local_resource_seconds": dict(sorted(local.items())),
|
|
}
|
|
|
|
|
|
def _audit_metrics(session_id: str, jobs: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
audit_path = state_root() / "sessions" / session_id / "audit.jsonl"
|
|
rows = _jsonl_rows(audit_path)
|
|
root_wait_seconds = 0.0
|
|
worker_wait_seconds = 0.0
|
|
read: set[str] = set()
|
|
cancel_reasons: Counter[str] = Counter()
|
|
scope_conflicts = 0
|
|
for row in rows:
|
|
event = row.get("event")
|
|
if event == "agents_wait_finished":
|
|
seconds = float(row.get("waited_seconds") or 0.0)
|
|
if row.get("caller_job_id"):
|
|
worker_wait_seconds += seconds
|
|
else:
|
|
root_wait_seconds += seconds
|
|
elif event == "agent_result_read" and row.get("terminal"):
|
|
target = row.get("target_job_id")
|
|
if target:
|
|
read.add(str(target))
|
|
elif event == "agent_cancelled":
|
|
reason = str(row.get("reason") or "unspecified")
|
|
cancel_reasons[reason] += 1
|
|
elif (
|
|
event in {"spawn_rejected", "spawn_batch_rejected"}
|
|
and row.get("reason") == "write_scope_conflict"
|
|
):
|
|
scope_conflicts += 1
|
|
terminal = {
|
|
str(job.get("job_id"))
|
|
for job in jobs
|
|
if job.get("status") not in {"queued", "running", "cancelling"}
|
|
}
|
|
accepted = {
|
|
str(job.get("job_id"))
|
|
for job in jobs
|
|
if job.get("result_state") in {"accepted", "integrated"}
|
|
}
|
|
integrated = {str(job.get("job_id")) for job in jobs if job.get("result_state") == "integrated"}
|
|
return {
|
|
"explicit_root_mcp_wait_seconds": root_wait_seconds,
|
|
"explicit_worker_mcp_wait_seconds": worker_wait_seconds,
|
|
"terminal_results_read": len(terminal & read),
|
|
"result_read_rate": len(terminal & read) / len(terminal) if terminal else None,
|
|
"terminal_results_accepted": len(terminal & accepted),
|
|
"result_acceptance_rate": len(terminal & accepted) / len(terminal) if terminal else None,
|
|
"patches_integrated": len(integrated),
|
|
"cancel_reasons": dict(cancel_reasons),
|
|
"cancelled_duplicates": sum(
|
|
count for reason, count in cancel_reasons.items() if "duplicate" in reason.lower()
|
|
),
|
|
"write_scope_conflicts": scope_conflicts,
|
|
"audit_event_count": len(rows),
|
|
}
|
|
|
|
|
|
def _timestamp(value: Any) -> dt.datetime | None:
|
|
if not isinstance(value, str):
|
|
return None
|
|
with contextlib.suppress(ValueError):
|
|
parsed = dt.datetime.fromisoformat(value)
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=dt.UTC)
|
|
return parsed.astimezone(dt.UTC)
|
|
return None
|
|
|
|
|
|
def _interval_metrics(session_id: str, jobs: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
intervals: list[tuple[dt.datetime, dt.datetime]] = []
|
|
terminal_times: list[dt.datetime] = []
|
|
for job in jobs:
|
|
started = _timestamp(job.get("started_at"))
|
|
finished = _timestamp(job.get("finished_at"))
|
|
if finished is not None:
|
|
terminal_times.append(finished)
|
|
if started is not None and finished is not None and finished >= started:
|
|
intervals.append((started, finished))
|
|
|
|
events: dict[dt.datetime, int] = {}
|
|
for started, finished in intervals:
|
|
events[started] = events.get(started, 0) + 1
|
|
events[finished] = events.get(finished, 0) - 1
|
|
active = 0
|
|
peak = 0
|
|
overlap_seconds = 0.0
|
|
previous: dt.datetime | None = None
|
|
for stamp in sorted(events):
|
|
if previous is not None and active >= 2:
|
|
overlap_seconds += max(0.0, (stamp - previous).total_seconds())
|
|
active += events[stamp]
|
|
peak = max(peak, active)
|
|
previous = stamp
|
|
|
|
session_started = None
|
|
with contextlib.suppress(FileNotFoundError, RuntimeError, ValueError):
|
|
session_started = _timestamp(load_session(session_id).get("started_at"))
|
|
first_terminal = min(terminal_times) if terminal_times else None
|
|
time_to_first = (
|
|
max(0.0, (first_terminal - session_started).total_seconds())
|
|
if first_terminal is not None and session_started is not None
|
|
else None
|
|
)
|
|
return {
|
|
"peak_mcp_workers": peak,
|
|
"worker_overlap_seconds": overlap_seconds,
|
|
"time_to_first_terminal_result_seconds": time_to_first,
|
|
}
|
|
|
|
|
|
def _root_activity_metrics(events_path: Path, session_id: str) -> dict[str, Any]:
|
|
rows = _jsonl_rows(events_path)
|
|
timestamps = [stamp for row in rows if (stamp := _timestamp(row.get("timestamp"))) is not None]
|
|
jobs = [job for job in iter_jobs(strict=True) if job.get("session_id") == session_id]
|
|
intervals = [
|
|
(started, finished)
|
|
for job in jobs
|
|
if (started := _timestamp(job.get("started_at"))) is not None
|
|
and (finished := _timestamp(job.get("finished_at"))) is not None
|
|
]
|
|
overlap_events = sum(
|
|
any(started <= stamp <= finished for started, finished in intervals) for stamp in timestamps
|
|
)
|
|
return {
|
|
"telemetry_coverage": {
|
|
"event_count": len(rows),
|
|
"timestamped_event_count": len(timestamps),
|
|
"timestamp_coverage_ratio": len(timestamps) / len(rows) if rows else None,
|
|
"activity_overlap_seconds": None,
|
|
"activity_overlap_seconds_reason": (
|
|
"point events do not establish continuous root activity intervals"
|
|
),
|
|
},
|
|
"observed_root_activity_events_during_worker_execution": overlap_events,
|
|
}
|
|
|
|
|
|
def _integration_correction_metrics(jobs: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
integrated_paths = 0
|
|
corrected_paths = 0
|
|
corrected_jobs = 0
|
|
for job in jobs:
|
|
if job.get("result_state") != "integrated":
|
|
continue
|
|
canonical_cwd = Path(str(job.get("canonical_cwd", ""))).resolve(strict=False)
|
|
patch = job.get("patch")
|
|
changed = patch.get("changed_paths", []) if isinstance(patch, Mapping) else []
|
|
artifacts = {
|
|
str(item.get("relative_path")): str(item.get("sha256"))
|
|
for item in job.get("artifacts", [])
|
|
if isinstance(item, Mapping)
|
|
and isinstance(item.get("relative_path"), str)
|
|
and isinstance(item.get("sha256"), str)
|
|
}
|
|
job_corrected = False
|
|
for raw_path in changed:
|
|
if not isinstance(raw_path, str):
|
|
continue
|
|
relative = Path(raw_path)
|
|
if relative.is_absolute() or ".." in relative.parts:
|
|
continue
|
|
integrated_paths += 1
|
|
target = (canonical_cwd / relative).resolve(strict=False)
|
|
expected_hash = artifacts.get(relative.as_posix())
|
|
if expected_hash is None:
|
|
corrected = target.exists()
|
|
else:
|
|
corrected = (
|
|
not target.is_file()
|
|
or target.is_symlink()
|
|
or sha256_bytes(target.read_bytes()) != expected_hash
|
|
)
|
|
if corrected:
|
|
corrected_paths += 1
|
|
job_corrected = True
|
|
corrected_jobs += int(job_corrected)
|
|
return {
|
|
"integrated_patch_paths": integrated_paths,
|
|
"integration_corrected_paths": corrected_paths,
|
|
"integrated_jobs_with_corrections": corrected_jobs,
|
|
"integration_correction_rate": (
|
|
corrected_paths / integrated_paths if integrated_paths else None
|
|
),
|
|
}
|
|
|
|
|
|
def _session_job_metrics(session_id: str, resolved: Mapping[str, Any]) -> dict[str, Any]:
|
|
jobs = [job for job in iter_jobs(strict=True) if job.get("session_id") == session_id]
|
|
statuses = Counter(str(job.get("status")) for job in jobs)
|
|
agents = Counter(str(job.get("agent")) for job in jobs)
|
|
models = Counter(str(job.get("model")) for job in jobs)
|
|
routes = Counter(str(job.get("route")) for job in jobs)
|
|
makers = Counter(str(job.get("maker")) for job in jobs)
|
|
api_operators: Counter[str] = Counter()
|
|
access_products: Counter[str] = Counter()
|
|
gateway_drivers: Counter[str] = Counter()
|
|
serving_providers: Counter[str] = Counter()
|
|
serving_endpoints: Counter[str] = Counter()
|
|
route_telemetry_incomplete = 0
|
|
usage: Counter[str] = Counter()
|
|
warnings = 0
|
|
contract_failures = 0
|
|
conflicts = 0
|
|
cost_ledgers: list[dict[str, Any]] = []
|
|
for job in jobs:
|
|
if job.get("warning"):
|
|
warnings += 1
|
|
if job.get("contract_valid") is False:
|
|
contract_failures += 1
|
|
result_path = Path(str(job.get("structured_result_path", "")))
|
|
if result_path.is_file():
|
|
with contextlib.suppress(Exception):
|
|
value = read_json(result_path)
|
|
if value.get("status") == "conflict" or value.get("conflicts"):
|
|
conflicts += 1
|
|
for key, value in (job.get("usage") or {}).items():
|
|
if isinstance(value, int) and not isinstance(value, bool):
|
|
usage[key] += value
|
|
model_key = job.get("model_key")
|
|
model = resolved["models"].get(model_key, {})
|
|
route = resolved["routes"].get(model.get("route"), {})
|
|
if model and route:
|
|
api_operators[str(route["api_operator"])] += 1
|
|
access_products[str(route["access_product"])] += 1
|
|
gateway_drivers[str(route["driver"])] += 1
|
|
route_telemetry = job.get("route_telemetry") or {}
|
|
for value in route_telemetry.get("actual_serving_provider_slugs", []):
|
|
serving_providers[str(value)] += 1
|
|
for value in route_telemetry.get("actual_serving_endpoint_tags", []):
|
|
serving_endpoints[str(value)] += 1
|
|
if job.get("requested_route_policy") is not None and not route_telemetry.get(
|
|
"complete"
|
|
):
|
|
route_telemetry_incomplete += 1
|
|
cost_ledgers.append(
|
|
_call_cost_ledgers(
|
|
usage=job.get("usage") or {},
|
|
model=model,
|
|
route=route,
|
|
events_path=Path(str(job.get("events_path", ""))),
|
|
elapsed_seconds=float(job.get("elapsed_seconds") or 0.0),
|
|
)
|
|
)
|
|
terminal = sum(
|
|
value for key, value in statuses.items() if key not in {"queued", "running", "cancelling"}
|
|
)
|
|
successful = statuses.get("completed", 0) + statuses.get("completed_with_warnings", 0)
|
|
audit = _audit_metrics(session_id, jobs)
|
|
intervals = _interval_metrics(session_id, jobs)
|
|
integration_corrections = _integration_correction_metrics(jobs)
|
|
retries_observed = sum(_retry_count(Path(str(job.get("events_path", "")))) for job in jobs)
|
|
root = str(resolved["profile"]["root"])
|
|
reachable = {root}
|
|
frontier = [root]
|
|
while frontier:
|
|
parent = frontier.pop()
|
|
for child in resolved["agents"][parent].get("can_spawn", []):
|
|
if child not in reachable:
|
|
reachable.add(child)
|
|
frontier.append(child)
|
|
return {
|
|
"profile_agents": sorted(reachable),
|
|
"job_count": len(jobs),
|
|
"statuses": dict(statuses),
|
|
"agents": dict(agents),
|
|
"models": dict(models),
|
|
"routes": dict(routes),
|
|
"makers": dict(makers),
|
|
"api_operators": dict(api_operators),
|
|
"access_products": dict(access_products),
|
|
"gateway_drivers": dict(gateway_drivers),
|
|
"serving_providers": dict(serving_providers),
|
|
"serving_endpoints": dict(serving_endpoints),
|
|
"route_telemetry_incomplete": route_telemetry_incomplete,
|
|
"usage": dict(usage),
|
|
"worker_success_rate": successful / terminal if terminal else None,
|
|
"warnings": warnings,
|
|
"contract_failures": contract_failures,
|
|
"reported_conflicts": conflicts,
|
|
"retries_observed": retries_observed,
|
|
**audit,
|
|
**intervals,
|
|
**integration_corrections,
|
|
"cost_ledgers": _merge_cost_ledgers(cost_ledgers),
|
|
}
|
|
|
|
|
|
def _orchestration_assertion_results(
|
|
task: Mapping[str, Any], worker_metrics: Mapping[str, Any], root_elapsed: float
|
|
) -> dict[str, Any]:
|
|
assertions = task.get("orchestration_assertions", {})
|
|
checks: list[dict[str, Any]] = []
|
|
|
|
def add(name: str, expected: Any, actual: Any, passed: bool) -> None:
|
|
checks.append({"name": name, "expected": expected, "actual": actual, "passed": passed})
|
|
|
|
observed_agents = set(worker_metrics.get("agents", {}))
|
|
required = set(assertions.get("required_agents", []))
|
|
forbidden = set(assertions.get("forbidden_agents", []))
|
|
applicable_required = required & set(worker_metrics.get("profile_agents", observed_agents))
|
|
absent_required = required - set(worker_metrics.get("profile_agents", observed_agents))
|
|
if required:
|
|
add(
|
|
"required_agents",
|
|
sorted(required),
|
|
sorted(observed_agents),
|
|
applicable_required <= observed_agents,
|
|
)
|
|
if absent_required:
|
|
checks[-1]["not_applicable"] = sorted(absent_required)
|
|
if forbidden:
|
|
add(
|
|
"forbidden_agents",
|
|
sorted(forbidden),
|
|
sorted(observed_agents),
|
|
not bool(forbidden & observed_agents),
|
|
)
|
|
for field, metric, comparison in (
|
|
("min_peak_mcp_workers", "peak_mcp_workers", lambda actual, expected: actual >= expected),
|
|
("max_jobs", "job_count", lambda actual, expected: actual <= expected),
|
|
("max_contract_failures", "contract_failures", lambda actual, expected: actual <= expected),
|
|
):
|
|
if field in assertions:
|
|
actual_count = int(worker_metrics.get(metric) or 0)
|
|
expected_count = int(assertions[field])
|
|
add(
|
|
field,
|
|
expected_count,
|
|
actual_count,
|
|
comparison(actual_count, expected_count),
|
|
)
|
|
if "min_result_acceptance_rate" in assertions:
|
|
actual_rate = worker_metrics.get("result_acceptance_rate")
|
|
expected_rate = float(assertions["min_result_acceptance_rate"])
|
|
add(
|
|
"min_result_acceptance_rate",
|
|
expected_rate,
|
|
actual_rate,
|
|
actual_rate is not None and float(actual_rate) >= expected_rate,
|
|
)
|
|
if "max_observed_mcp_wait_ratio" in assertions:
|
|
wait_seconds = min(
|
|
root_elapsed, float(worker_metrics.get("explicit_root_mcp_wait_seconds") or 0.0)
|
|
)
|
|
actual_wait_ratio = wait_seconds / root_elapsed if root_elapsed else 0.0
|
|
expected_wait_ratio = float(assertions["max_observed_mcp_wait_ratio"])
|
|
add(
|
|
"max_observed_mcp_wait_ratio",
|
|
expected_wait_ratio,
|
|
actual_wait_ratio,
|
|
actual_wait_ratio <= expected_wait_ratio,
|
|
)
|
|
return {"checks": checks, "passed": all(check["passed"] for check in checks)}
|
|
|
|
|
|
def _variant_root_availability(snapshot: Mapping[str, Any]) -> dict[str, Any]:
|
|
resolved = snapshot["resolved"]
|
|
root = resolved["agents"][resolved["profile"]["root"]]
|
|
status = route_availability(snapshot)[root["route"]]
|
|
return {"route": root["route"], **status}
|
|
|
|
|
|
def _mean(values: Sequence[float]) -> float | None:
|
|
return sum(values) / len(values) if values else None
|
|
|
|
|
|
def _variant_summary(
|
|
variant: Mapping[str, Any],
|
|
records: Sequence[Mapping[str, Any]],
|
|
*,
|
|
scarce_model_keys: set[str] | None = None,
|
|
) -> dict[str, Any]:
|
|
passed = sum(bool(record["passed"]) for record in records)
|
|
root_usage: Counter[str] = Counter()
|
|
worker_usage: Counter[str] = Counter()
|
|
worker_agents: Counter[str] = Counter()
|
|
identities: dict[str, Counter[str]] = {
|
|
name: Counter()
|
|
for name in (
|
|
"makers",
|
|
"api_operators",
|
|
"access_products",
|
|
"gateway_drivers",
|
|
"routes",
|
|
"serving_providers",
|
|
"serving_endpoints",
|
|
)
|
|
}
|
|
metric_values: dict[str, list[float]] = {}
|
|
cost_items: list[Mapping[str, Any]] = []
|
|
terminal_jobs = 0
|
|
accepted_results = 0
|
|
first_results: list[float] = []
|
|
scarcity = scarce_model_keys or set()
|
|
scarce_requests = 0
|
|
for record in records:
|
|
root_usage.update(record.get("root_usage") or {})
|
|
worker = record["worker_metrics"]
|
|
worker_usage.update(worker.get("usage") or {})
|
|
worker_agents.update(worker.get("agents") or {})
|
|
root_identity = record["root_identity"]
|
|
if str(root_identity["model"]) in scarcity:
|
|
scarce_requests += 1
|
|
scarce_requests += sum(
|
|
int(count)
|
|
for model, count in (worker.get("models") or {}).items()
|
|
if str(model) in scarcity
|
|
)
|
|
for field in ("maker", "api_operator", "access_product", "gateway_driver", "route"):
|
|
identities[{"maker": "makers", "route": "routes"}.get(field, field + "s")][
|
|
str(root_identity[field])
|
|
] += 1
|
|
for field in identities:
|
|
if field in {"serving_providers", "serving_endpoints"}:
|
|
identities[field].update(worker.get(field) or {})
|
|
elif field == "routes":
|
|
identities[field].update(worker.get("routes") or {})
|
|
else:
|
|
identities[field].update(worker.get(field) or {})
|
|
for name, value in (record.get("outcome_metrics") or {}).items():
|
|
metric_values.setdefault(str(name), []).append(float(value))
|
|
cost_items.extend((record["root_cost_ledgers"], worker["cost_ledgers"]))
|
|
terminal_jobs += sum(
|
|
int(count)
|
|
for status, count in worker.get("statuses", {}).items()
|
|
if status not in {"queued", "running", "cancelling"}
|
|
)
|
|
accepted_results += int(worker.get("terminal_results_accepted") or 0)
|
|
if worker.get("time_to_first_terminal_result_seconds") is not None:
|
|
first_results.append(float(worker["time_to_first_terminal_result_seconds"]))
|
|
success_rate = passed / len(records) if records else None
|
|
metrics = {name: _mean(values) for name, values in sorted(metric_values.items())}
|
|
metrics["success_rate"] = success_rate
|
|
root_seconds = sum(float(record.get("root_elapsed_seconds") or 0.0) for record in records)
|
|
wait_seconds = sum(
|
|
float(record.get("explicit_root_mcp_wait_seconds") or 0.0) for record in records
|
|
)
|
|
cost_ledgers = _merge_cost_ledgers(cost_items)
|
|
subscription_request_units = sum(
|
|
int(units.get("request_units", 0)) for units in cost_ledgers["subscription_units"].values()
|
|
)
|
|
local_resource_seconds = sum(cost_ledgers["local_resource_seconds"].values())
|
|
metrics["subscription_request_units"] = float(subscription_request_units)
|
|
metrics["scarce_tier_request_units"] = float(scarce_requests)
|
|
metrics["local_resource_seconds"] = float(local_resource_seconds)
|
|
if cost_ledgers["actual_api_usd"] is not None:
|
|
metrics["actual_api_usd"] = float(cost_ledgers["actual_api_usd"])
|
|
if cost_ledgers["api_equivalent_estimate_usd"] is not None:
|
|
metrics["api_equivalent_estimate_usd"] = float(cost_ledgers["api_equivalent_estimate_usd"])
|
|
integrated_patch_paths = sum(
|
|
int(record["worker_metrics"].get("integrated_patch_paths") or 0) for record in records
|
|
)
|
|
integration_corrected_paths = sum(
|
|
int(record["worker_metrics"].get("integration_corrected_paths") or 0) for record in records
|
|
)
|
|
return {
|
|
"id": variant["id"],
|
|
"purpose": variant["purpose"],
|
|
"comparison_class": variant["comparison_class"],
|
|
"topology": variant["topology"],
|
|
"worker": variant.get("worker"),
|
|
"access_product": variant.get("access_product"),
|
|
"trial_records": len(records),
|
|
"passed": passed,
|
|
"failed": len(records) - passed,
|
|
"success_rate": success_rate,
|
|
"outcome_metrics": metrics,
|
|
"mean_root_elapsed_seconds": root_seconds / len(records) if records else None,
|
|
"explicit_root_mcp_wait_seconds": wait_seconds,
|
|
"explicit_root_mcp_wait_ratio": wait_seconds / root_seconds if root_seconds else 0.0,
|
|
"orchestration_diagnostic_pass_rate": (
|
|
sum(bool(record["orchestration_diagnostics_passed"]) for record in records)
|
|
/ len(records)
|
|
if records
|
|
else None
|
|
),
|
|
"worker_jobs": sum(int(record["worker_metrics"]["job_count"]) for record in records),
|
|
"contract_failures": sum(
|
|
int(record["worker_metrics"]["contract_failures"]) for record in records
|
|
),
|
|
"write_scope_conflicts": sum(
|
|
int(record["worker_metrics"]["write_scope_conflicts"]) for record in records
|
|
),
|
|
"integrated_patch_paths": integrated_patch_paths,
|
|
"integration_corrected_paths": integration_corrected_paths,
|
|
"integrated_jobs_with_corrections": sum(
|
|
int(record["worker_metrics"].get("integrated_jobs_with_corrections") or 0)
|
|
for record in records
|
|
),
|
|
"integration_correction_rate": (
|
|
integration_corrected_paths / integrated_patch_paths if integrated_patch_paths else None
|
|
),
|
|
"route_telemetry_incomplete": sum(
|
|
int(record["worker_metrics"].get("route_telemetry_incomplete") or 0)
|
|
for record in records
|
|
),
|
|
"result_acceptance_rate": (accepted_results / terminal_jobs if terminal_jobs else None),
|
|
"mean_time_to_first_terminal_result_seconds": _mean(first_results),
|
|
"root_activity_telemetry": [record.get("root_activity") for record in records],
|
|
"root_usage": dict(root_usage),
|
|
"worker_usage": dict(worker_usage),
|
|
"worker_agents": dict(worker_agents),
|
|
"usage": dict(root_usage + worker_usage),
|
|
"cost_ledgers": cost_ledgers,
|
|
"diversity": {name: dict(counter) for name, counter in identities.items()},
|
|
}
|
|
|
|
|
|
def _directed_improvement(
|
|
baseline: float, candidate: float, direction: str
|
|
) -> dict[str, float | bool | None]:
|
|
absolute = candidate - baseline if direction == "higher" else baseline - candidate
|
|
if baseline:
|
|
relative: float | None = absolute / abs(baseline)
|
|
relative_unbounded = False
|
|
elif absolute > 0:
|
|
# Improvement from an exact zero baseline is positive but has no finite
|
|
# percentage representation. Keep the JSON document strict and expose
|
|
# that boundary explicitly instead of serializing Infinity.
|
|
relative = None
|
|
relative_unbounded = True
|
|
else:
|
|
relative = 0.0 if absolute == 0 else None
|
|
relative_unbounded = False
|
|
return {
|
|
"absolute": absolute,
|
|
"relative": relative,
|
|
"relative_unbounded": relative_unbounded,
|
|
}
|
|
|
|
|
|
def _passes_relative_improvement(
|
|
improvement: Mapping[str, float | bool | None], minimum: float
|
|
) -> bool:
|
|
if improvement.get("relative_unbounded") is True:
|
|
return True
|
|
relative = improvement.get("relative")
|
|
return (
|
|
isinstance(relative, (int, float))
|
|
and not isinstance(relative, bool)
|
|
and relative >= minimum
|
|
)
|
|
|
|
|
|
def _promotion_verdict(
|
|
*,
|
|
resolved: Mapping[str, Any],
|
|
suite: Mapping[str, Any],
|
|
variants: Sequence[Mapping[str, Any]],
|
|
summaries: Mapping[str, Mapping[str, Any]],
|
|
skipped: Sequence[Mapping[str, Any]],
|
|
) -> dict[str, Any]:
|
|
is_lab = resolved["profile"]["maturity"] == "lab"
|
|
config = suite["promotion"]
|
|
checks: list[dict[str, Any]] = []
|
|
|
|
def finish() -> dict[str, Any]:
|
|
hypothesis_passed = all(check["passed"] for check in checks)
|
|
if is_lab:
|
|
return {
|
|
"eligible": False,
|
|
"passed": None,
|
|
"status": "experimental_lab",
|
|
"reason": "labs report hypothesis checks but make no bundled superiority claim",
|
|
"hypothesis_passed": hypothesis_passed,
|
|
"checks": checks,
|
|
}
|
|
return {"eligible": True, "passed": hypothesis_passed, "checks": checks}
|
|
|
|
def add(name: str, passed: bool, **evidence: Any) -> None:
|
|
checks.append({"name": name, "passed": passed, **evidence})
|
|
|
|
by_class: dict[str, list[Mapping[str, Any]]] = {}
|
|
for variant in variants:
|
|
summary = summaries.get(str(variant["id"]))
|
|
if summary is not None:
|
|
by_class.setdefault(str(variant["comparison_class"]), []).append(summary)
|
|
full_candidates = by_class.get("full_profile", [])
|
|
root_candidates = by_class.get("configured_root_alone", [])
|
|
if len(full_candidates) != 1 or len(root_candidates) != 1:
|
|
add(
|
|
"unique_full_and_configured_root",
|
|
False,
|
|
full_count=len(full_candidates),
|
|
configured_root_count=len(root_candidates),
|
|
)
|
|
return finish()
|
|
full = full_candidates[0]
|
|
root = root_candidates[0]
|
|
add(
|
|
"full_profile_has_success",
|
|
float(full["success_rate"] or 0.0) > 0.0,
|
|
full_success_rate=full["success_rate"],
|
|
)
|
|
single_candidates = [
|
|
*by_class.get("strongest_single_agent", []),
|
|
*by_class.get("access_service_single_agent", []),
|
|
root,
|
|
]
|
|
strongest = max(single_candidates, key=lambda item: float(item["success_rate"] or 0.0))
|
|
tolerance = float(config.get("strongest_success_tolerance", 0.02))
|
|
add(
|
|
"strongest_single_success_tolerance",
|
|
float(full["success_rate"] or 0.0) + tolerance >= float(strongest["success_rate"] or 0.0),
|
|
full_success_rate=full["success_rate"],
|
|
strongest_variant=strongest["id"],
|
|
strongest_success_rate=strongest["success_rate"],
|
|
tolerance=tolerance,
|
|
)
|
|
metric = str(config["primary_metric"])
|
|
direction = str(config["direction"])
|
|
full_metric = full["outcome_metrics"].get(metric)
|
|
baseline_class = str(config.get("primary_baseline", "configured_root_alone"))
|
|
baseline_candidates = by_class.get(baseline_class, [])
|
|
baseline = root if baseline_class == "configured_root_alone" else None
|
|
if baseline is None and len(baseline_candidates) == 1:
|
|
baseline = baseline_candidates[0]
|
|
baseline_metric = baseline["outcome_metrics"].get(metric) if baseline else None
|
|
if full_metric is None or baseline_metric is None:
|
|
add(
|
|
"primary_metric_improvement",
|
|
False,
|
|
metric=metric,
|
|
baseline_class=baseline_class,
|
|
reason="metric missing from full or primary-baseline validation output",
|
|
)
|
|
else:
|
|
assert baseline is not None
|
|
improvement = _directed_improvement(float(baseline_metric), float(full_metric), direction)
|
|
relative_gate = float(config.get("minimum_relative_improvement", 0.10))
|
|
absolute_gate = float(config.get("minimum_absolute_improvement", 0.05))
|
|
add(
|
|
"primary_metric_improvement",
|
|
_passes_relative_improvement(improvement, relative_gate)
|
|
or float(improvement["absolute"] or 0.0) >= absolute_gate,
|
|
metric=metric,
|
|
direction=direction,
|
|
baseline_class=baseline_class,
|
|
baseline_variant=baseline["id"],
|
|
baseline=baseline_metric,
|
|
full_profile=full_metric,
|
|
improvement=improvement,
|
|
minimum_relative=relative_gate,
|
|
minimum_absolute=absolute_gate,
|
|
)
|
|
for field, expected in (
|
|
("write_scope_conflicts", 0),
|
|
("contract_failures", 0),
|
|
("route_telemetry_incomplete", 0),
|
|
):
|
|
add(f"no_{field}", int(full[field]) == expected, actual=full[field], expected=expected)
|
|
if bool(config.get("require_complete_api_cost", True)):
|
|
complete = bool(full["cost_ledgers"]["actual_api_usd_complete"])
|
|
add("complete_actual_api_cost", complete, ledger=full["cost_ledgers"])
|
|
for field, metric_direction in (
|
|
("no_regression_higher_metrics", "higher"),
|
|
("no_regression_lower_metrics", "lower"),
|
|
):
|
|
for name in config.get(field, []):
|
|
candidate = full["outcome_metrics"].get(name)
|
|
baseline = strongest["outcome_metrics"].get(name)
|
|
if candidate is None or baseline is None:
|
|
add(
|
|
f"non_regression_{name}",
|
|
False,
|
|
reason="metric missing",
|
|
full=candidate,
|
|
strongest=baseline,
|
|
)
|
|
else:
|
|
passed = (
|
|
float(candidate) >= float(baseline)
|
|
if metric_direction == "higher"
|
|
else float(candidate) <= float(baseline)
|
|
)
|
|
add(
|
|
f"non_regression_{name}",
|
|
passed,
|
|
direction=metric_direction,
|
|
full=candidate,
|
|
strongest=baseline,
|
|
)
|
|
root_id = str(resolved["profile"]["root"])
|
|
reachable = {root_id}
|
|
frontier = [root_id]
|
|
while frontier:
|
|
parent = frontier.pop()
|
|
for child in resolved["agents"][parent]["can_spawn"]:
|
|
if child not in reachable:
|
|
reachable.add(child)
|
|
frontier.append(child)
|
|
expected_workers = reachable - {root_id}
|
|
ablations = {str(item["worker"]): item for item in by_class.get("ablation", [])}
|
|
missing_ablations = sorted(expected_workers - set(ablations))
|
|
add(
|
|
"complete_worker_ablation_coverage",
|
|
not missing_ablations,
|
|
expected_workers=sorted(expected_workers),
|
|
missing=missing_ablations,
|
|
)
|
|
for worker, ablation in sorted(ablations.items()):
|
|
success_contribution = float(full["success_rate"] or 0.0) - float(
|
|
ablation["success_rate"] or 0.0
|
|
)
|
|
ablated_metric = ablation["outcome_metrics"].get(metric)
|
|
contribution = None
|
|
if full_metric is not None and ablated_metric is not None:
|
|
contribution = _directed_improvement(
|
|
float(ablated_metric), float(full_metric), direction
|
|
)
|
|
success_gate = float(config.get("worker_minimum_success_contribution", 0.02))
|
|
metric_gate = float(config.get("worker_minimum_metric_contribution", 0.10))
|
|
add(
|
|
f"worker_ablation_{worker}",
|
|
success_contribution >= success_gate
|
|
or (
|
|
contribution is not None and _passes_relative_improvement(contribution, metric_gate)
|
|
),
|
|
success_contribution=success_contribution,
|
|
metric_contribution=contribution,
|
|
minimum_success=success_gate,
|
|
minimum_metric_relative=metric_gate,
|
|
)
|
|
unavailable_access = [
|
|
item for item in skipped if item.get("comparison_class") == "access_service_single_agent"
|
|
]
|
|
executed_access = by_class.get("access_service_single_agent", [])
|
|
add(
|
|
"accessible_service_controls_executed",
|
|
bool(executed_access),
|
|
executed=[item["id"] for item in executed_access],
|
|
unavailable=unavailable_access,
|
|
)
|
|
return finish()
|
|
|
|
|
|
def _run_evaluation(
|
|
*,
|
|
profile: str | Path,
|
|
suite: str | Path,
|
|
bindings: Mapping[str, str] | None = None,
|
|
wall_timeout_override: int | None = None,
|
|
dry_run: bool = False,
|
|
trial_mode: str = "development",
|
|
progress: Callable[[str], None] | None = None,
|
|
_failure_context: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if wall_timeout_override is not None and (
|
|
not isinstance(wall_timeout_override, int)
|
|
or isinstance(wall_timeout_override, bool)
|
|
or not 1 <= wall_timeout_override <= 172_800
|
|
):
|
|
raise ValueError("wall-timeout override must be an integer from 1 to 172800")
|
|
if trial_mode not in {"development", "release"}:
|
|
raise ValueError("trial_mode must be development or release")
|
|
if progress:
|
|
progress(f"Resolving evaluation suite {suite} and profile {profile}...")
|
|
suite_dir, suite_data = resolve_suite(suite)
|
|
configured_snapshot = compile_profile(profile, bindings=bindings)
|
|
configured_resolved = configured_snapshot["resolved"]
|
|
if configured_resolved["profile"]["id"] != suite_data["profile"]:
|
|
raise ValueError(
|
|
f"evaluation suite {suite_data['id']!r} requires profile "
|
|
f"{suite_data['profile']!r}, not {configured_resolved['profile']['id']!r}"
|
|
)
|
|
configured_agents = set(configured_resolved["agents"])
|
|
for task in suite_data["tasks"]:
|
|
assertions = task.get("orchestration_assertions", {})
|
|
asserted_agents = set(assertions.get("required_agents", [])) | set(
|
|
assertions.get("forbidden_agents", [])
|
|
)
|
|
unknown_agents = sorted(asserted_agents - configured_agents)
|
|
if unknown_agents:
|
|
raise ValueError(
|
|
f"task {task['id']}: orchestration assertion agents absent from profile: "
|
|
+ ", ".join(unknown_agents)
|
|
)
|
|
configured_availability = _variant_root_availability(configured_snapshot)
|
|
if not configured_availability["available"]:
|
|
raise RuntimeError(
|
|
f"configured root route {configured_availability['route']!r} is unavailable: "
|
|
f"{configured_availability['reason']}"
|
|
)
|
|
variant_entries = []
|
|
for variant in suite_data["variants"]:
|
|
snapshot = _variant_snapshot(profile, bindings, variant)
|
|
variant_entries.append((variant, snapshot, _variant_root_availability(snapshot)))
|
|
known_variant_models = {
|
|
model_key
|
|
for _variant, snapshot, _availability in variant_entries
|
|
for model_key in snapshot["resolved"]["models"]
|
|
}
|
|
unknown_scarce_models = sorted(
|
|
set(suite_data["promotion"].get("scarce_model_keys", [])) - known_variant_models
|
|
)
|
|
if unknown_scarce_models:
|
|
raise ValueError(
|
|
"evaluation promotion scarce_model_keys are absent from every matched variant: "
|
|
+ ", ".join(unknown_scarce_models)
|
|
)
|
|
trial_count = int(
|
|
suite_data["release_trials" if trial_mode == "release" else "development_trials"]
|
|
)
|
|
run_id = _run_id(configured_resolved["profile"]["id"], suite_data["id"])
|
|
directory = evaluations_state_root() / run_id
|
|
directory.mkdir(mode=0o700)
|
|
fixture = suite_data.get("fixture")
|
|
fixture_source = (suite_dir / str(fixture)).resolve() if fixture else None
|
|
manifest: dict[str, Any] = {
|
|
"schema_version": MMO_SCHEMA_VERSION,
|
|
"run_id": run_id,
|
|
"profile_id": configured_resolved["profile"]["id"],
|
|
"profile_version": configured_resolved["profile"]["version"],
|
|
"snapshot_hash": configured_snapshot["manifest"]["snapshot_hash"],
|
|
"suite_id": suite_data["id"],
|
|
"suite_name": suite_data.get("name", suite_data["id"]),
|
|
"bindings": dict(bindings or {}),
|
|
"trial_mode": trial_mode,
|
|
"trial_count": trial_count,
|
|
"promotion_policy": dict(suite_data["promotion"]),
|
|
"variants": [
|
|
{
|
|
**dict(variant),
|
|
"snapshot_hash": variant_snapshot["manifest"]["snapshot_hash"],
|
|
"resolved_profile_id": variant_snapshot["resolved"]["profile"]["id"],
|
|
"root_route_availability": availability,
|
|
}
|
|
for variant, variant_snapshot, availability in variant_entries
|
|
],
|
|
"skipped_variants": [
|
|
{
|
|
"id": variant["id"],
|
|
"comparison_class": variant["comparison_class"],
|
|
"access_product": variant.get("access_product"),
|
|
"reason": availability["reason"],
|
|
"root_route": availability["route"],
|
|
}
|
|
for variant, _snapshot, availability in variant_entries
|
|
if not availability["available"]
|
|
],
|
|
"created_at": utc_now(),
|
|
"status": "validated" if dry_run else "running",
|
|
"tasks": [],
|
|
}
|
|
atomic_write_json(directory / "run.json", manifest)
|
|
_failure_context.update(directory=directory, manifest=manifest)
|
|
if dry_run:
|
|
manifest["tasks"] = [
|
|
{
|
|
"id": task["id"],
|
|
"variant_id": variant["id"],
|
|
"comparison_class": variant["comparison_class"],
|
|
"trial": trial,
|
|
"snapshot_hash": variant_snapshot["manifest"]["snapshot_hash"],
|
|
"sandbox": task.get("sandbox", "read-only"),
|
|
"images": list(task.get("images", [])),
|
|
"route_faults": dict(task.get("route_faults", {})),
|
|
"outcome_assertions": dict(task["outcome_assertions"]),
|
|
"orchestration_assertions": dict(task.get("orchestration_assertions", {})),
|
|
"validated": True,
|
|
}
|
|
for task in suite_data["tasks"]
|
|
for variant, variant_snapshot, availability in variant_entries
|
|
if availability["available"]
|
|
for trial in range(1, trial_count + 1)
|
|
]
|
|
manifest["finished_at"] = utc_now()
|
|
atomic_write_json(directory / "run.json", manifest)
|
|
if progress:
|
|
progress(f"Validated {len(manifest['tasks'])} planned evaluation trial(s).")
|
|
return manifest
|
|
|
|
total_trials = sum(
|
|
1
|
|
for _task in suite_data["tasks"]
|
|
for _variant, _snapshot, availability in variant_entries
|
|
if availability["available"]
|
|
for _trial in range(1, trial_count + 1)
|
|
)
|
|
trial_index = 0
|
|
for task in suite_data["tasks"]:
|
|
for variant, snapshot, availability in variant_entries:
|
|
if not availability["available"]:
|
|
continue
|
|
for trial in range(1, trial_count + 1):
|
|
trial_index += 1
|
|
if progress:
|
|
progress(
|
|
f"Evaluation trial {trial_index}/{total_trials}: "
|
|
f"task={task['id']} variant={variant['id']} trial={trial}."
|
|
)
|
|
resolved = snapshot["resolved"]
|
|
task_dir = (
|
|
directory / "tasks" / task["id"] / str(variant["id"]) / f"trial-{trial:02d}"
|
|
)
|
|
workspace = task_dir / "workspace"
|
|
task_dir.mkdir(parents=True, exist_ok=True)
|
|
_copy_fixture(fixture_source, workspace)
|
|
mutation = _prepare_hidden_trial(suite_dir, task["id"], trial, workspace)
|
|
_initialize_fixture_repository(workspace)
|
|
timeout = wall_timeout_override or int(task.get("wall_timeout_seconds", 1800))
|
|
root_exception: str | None = None
|
|
try:
|
|
root_result = run_root_exec(
|
|
profile=None,
|
|
cwd=workspace,
|
|
prompt=task["prompt"],
|
|
images=task.get("images", []),
|
|
snapshot_hash=snapshot["manifest"]["snapshot_hash"],
|
|
wall_timeout_seconds=timeout,
|
|
sandbox_mode=task.get("sandbox", "read-only"),
|
|
label=(
|
|
f"eval-{suite_data['id']}-{task['id']}-{variant['id']}-trial-{trial}"
|
|
),
|
|
route_faults={
|
|
route: fault
|
|
for route, fault in task.get("route_faults", {}).items()
|
|
if route in snapshot["resolved"]["routes"]
|
|
},
|
|
)
|
|
except Exception as exc:
|
|
root_exception = f"{type(exc).__name__}: {exc}"
|
|
failed_session_id = getattr(exc, "mmo_session_id", None)
|
|
failed_session_status = getattr(exc, "mmo_session_status", None)
|
|
root_result = {
|
|
"status": (
|
|
"detached"
|
|
if failed_session_status == "detached"
|
|
else "failed_before_or_during_root_execution"
|
|
),
|
|
"root_status": "failed_before_or_during_root_execution",
|
|
"exit_code": 1,
|
|
"elapsed_seconds": 0.0,
|
|
"events_path": "",
|
|
"result": "",
|
|
}
|
|
if isinstance(failed_session_id, str):
|
|
root_result["session"] = {"session_id": failed_session_id}
|
|
session_id = (
|
|
str(root_result["session"]["session_id"])
|
|
if isinstance(root_result.get("session"), Mapping)
|
|
else ""
|
|
)
|
|
# A harness wall limit detaches the durable app-server session;
|
|
# it does not terminate it. Stop it before installing holdouts
|
|
# or running validators so detached agents cannot keep mutating
|
|
# the evaluation workspace behind the harness.
|
|
if root_result.get("status") == "detached" and session_id:
|
|
try:
|
|
root_result["harness_cleanup"] = stop_session(session_id, grace_seconds=0)
|
|
except Exception as stop_error:
|
|
root_result["harness_cleanup_error"] = (
|
|
f"{type(stop_error).__name__}: {stop_error}"
|
|
)
|
|
try:
|
|
root_result["harness_cancel"] = cancel_session(session_id)
|
|
except Exception as cancel_error:
|
|
raise RuntimeError(
|
|
"evaluation cannot install holdouts while detached agents may "
|
|
"still be running; graceful stop and immediate cancellation "
|
|
"both failed: "
|
|
f"{type(stop_error).__name__}: {stop_error}; "
|
|
f"{type(cancel_error).__name__}: {cancel_error}"
|
|
) from cancel_error
|
|
(workspace / ".mmo-eval-final.txt").write_text(
|
|
str(root_result.get("result") or ""), encoding="utf-8"
|
|
)
|
|
patterns = _pattern_results(root_result.get("result", ""), task)
|
|
holdout = _install_holdout(suite_dir, task["id"], workspace)
|
|
validations = [
|
|
_run_validation(
|
|
command,
|
|
workspace,
|
|
int(task.get("validation_timeout_seconds", 300)),
|
|
)
|
|
for command in task["outcome_assertions"].get("validation_commands", [])
|
|
]
|
|
outcome_metrics = _numeric_validation_metrics(validations)
|
|
worker_metrics = _session_job_metrics(session_id, resolved)
|
|
events_path = Path(str(root_result.get("events_path", "")))
|
|
root_usage = _event_usage(events_path)
|
|
root_agent = resolved["agents"][resolved["profile"]["root"]]
|
|
root_model = resolved["models"][root_agent["model"]]
|
|
root_route = resolved["routes"][root_model["route"]]
|
|
root_retries = _retry_count(events_path)
|
|
root_elapsed = float(root_result.get("elapsed_seconds") or 0.0)
|
|
root_cost_ledgers = (
|
|
_call_cost_ledgers(
|
|
usage=root_usage,
|
|
model=root_model,
|
|
route=root_route,
|
|
events_path=events_path,
|
|
elapsed_seconds=root_elapsed,
|
|
)
|
|
if root_exception is None
|
|
else _merge_cost_ledgers([])
|
|
)
|
|
observed_wait = min(
|
|
root_elapsed,
|
|
float(worker_metrics.get("explicit_root_mcp_wait_seconds") or 0.0),
|
|
)
|
|
root_activity = _root_activity_metrics(events_path, session_id)
|
|
orchestration_assertions = _orchestration_assertion_results(
|
|
task, worker_metrics, root_elapsed
|
|
)
|
|
task_passed = (
|
|
root_result.get("exit_code") == 0
|
|
and patterns["passed"]
|
|
and all(item["passed"] for item in validations)
|
|
)
|
|
record = {
|
|
"id": task["id"],
|
|
"variant_id": variant["id"],
|
|
"comparison_class": variant["comparison_class"],
|
|
"topology": variant["topology"],
|
|
"access_product": variant.get("access_product"),
|
|
"trial": trial,
|
|
"difficulty": task.get("difficulty", "medium"),
|
|
"negative_control": bool(task.get("negative_control", False)),
|
|
"route_faults": dict(task.get("route_faults", {})),
|
|
"hidden_mutation": mutation,
|
|
"holdout": holdout,
|
|
"snapshot_hash": snapshot["manifest"]["snapshot_hash"],
|
|
"resolved_profile_id": resolved["profile"]["id"],
|
|
"description": task.get("description", ""),
|
|
"workspace": str(workspace),
|
|
"session_id": session_id,
|
|
"root_status": root_result.get("status"),
|
|
"root_error": root_exception,
|
|
"root_exit_code": root_result.get("exit_code"),
|
|
"root_elapsed_seconds": root_elapsed,
|
|
"explicit_root_mcp_wait_seconds": observed_wait,
|
|
"explicit_root_mcp_wait_ratio": (
|
|
observed_wait / root_elapsed if root_elapsed else 0.0
|
|
),
|
|
"root_activity": root_activity,
|
|
"root_usage": root_usage,
|
|
"root_cost_ledgers": root_cost_ledgers,
|
|
"root_retries_observed": root_retries,
|
|
"root_identity": {
|
|
"model": root_agent["model"],
|
|
"maker": root_model["maker"],
|
|
"route": root_model["route"],
|
|
"api_operator": root_route["api_operator"],
|
|
"access_product": root_route["access_product"],
|
|
"gateway_driver": root_route["driver"],
|
|
},
|
|
"result": root_result.get("result", ""),
|
|
"patterns": patterns,
|
|
"validation": validations,
|
|
"outcome_metrics": outcome_metrics,
|
|
"worker_metrics": worker_metrics,
|
|
"outcome_assertions": {
|
|
"patterns": patterns,
|
|
"validation": validations,
|
|
"passed": patterns["passed"]
|
|
and all(item["passed"] for item in validations),
|
|
},
|
|
"orchestration_assertions": orchestration_assertions,
|
|
"orchestration_diagnostics_passed": orchestration_assertions["passed"],
|
|
"passed": task_passed,
|
|
}
|
|
atomic_write_json(task_dir / "result.json", record)
|
|
manifest["tasks"].append(record)
|
|
atomic_write_json(directory / "run.json", manifest)
|
|
|
|
if progress:
|
|
progress("Aggregating evaluation results and promotion checks...")
|
|
variant_summaries = {
|
|
str(variant["id"]): _variant_summary(
|
|
variant,
|
|
[record for record in manifest["tasks"] if record["variant_id"] == variant["id"]],
|
|
scarce_model_keys=set(suite_data["promotion"].get("scarce_model_keys", [])),
|
|
)
|
|
for variant, _snapshot, availability in variant_entries
|
|
if availability["available"]
|
|
}
|
|
promotion = _promotion_verdict(
|
|
resolved=configured_resolved,
|
|
suite=suite_data,
|
|
variants=suite_data["variants"],
|
|
summaries=variant_summaries,
|
|
skipped=manifest["skipped_variants"],
|
|
)
|
|
manifest["summary"] = {
|
|
"matched_task_count": len(suite_data["tasks"]),
|
|
"executed_trial_records": len(manifest["tasks"]),
|
|
"variant_summaries": variant_summaries,
|
|
"promotion": promotion,
|
|
"aggregate_score": None,
|
|
"aggregate_score_reason": (
|
|
"comparison variants are controls, not interchangeable tasks; use per-variant "
|
|
"outcomes and the promotion verdict"
|
|
),
|
|
}
|
|
manifest["status"] = (
|
|
"completed"
|
|
if manifest["tasks"] and all(bool(record["passed"]) for record in manifest["tasks"])
|
|
else "completed_with_failures"
|
|
)
|
|
manifest["finished_at"] = utc_now()
|
|
atomic_write_json(directory / "run.json", manifest)
|
|
return manifest
|
|
|
|
|
|
def run_evaluation(
|
|
*,
|
|
profile: str | Path,
|
|
suite: str | Path,
|
|
bindings: Mapping[str, str] | None = None,
|
|
wall_timeout_override: int | None = None,
|
|
dry_run: bool = False,
|
|
trial_mode: str = "development",
|
|
progress: Callable[[str], None] | None = None,
|
|
) -> dict[str, Any]:
|
|
failure_context: dict[str, Any] = {}
|
|
try:
|
|
return _run_evaluation(
|
|
profile=profile,
|
|
suite=suite,
|
|
bindings=bindings,
|
|
wall_timeout_override=wall_timeout_override,
|
|
dry_run=dry_run,
|
|
trial_mode=trial_mode,
|
|
progress=progress,
|
|
_failure_context=failure_context,
|
|
)
|
|
except BaseException as exc:
|
|
manifest = failure_context.get("manifest")
|
|
directory = failure_context.get("directory")
|
|
if isinstance(manifest, dict) and isinstance(directory, Path):
|
|
manifest["status"] = "failed"
|
|
manifest["finished_at"] = utc_now()
|
|
manifest["error"] = f"{type(exc).__name__}: {exc}"
|
|
with contextlib.suppress(Exception):
|
|
atomic_write_json(directory / "run.json", manifest)
|
|
raise
|
|
|
|
|
|
def list_runs(limit: int = 100) -> list[dict[str, Any]]:
|
|
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 1:
|
|
raise ValueError("evaluation run limit must be a positive integer")
|
|
results: list[dict[str, Any]] = []
|
|
for directory in sorted(evaluations_state_root().iterdir(), reverse=True):
|
|
path = directory / "run.json"
|
|
if not path.is_file():
|
|
continue
|
|
with contextlib.suppress(Exception):
|
|
data = read_json_object(path, label="evaluation run state")
|
|
results.append(
|
|
{
|
|
"run_id": data.get("run_id"),
|
|
"profile_id": data.get("profile_id"),
|
|
"suite_id": data.get("suite_id"),
|
|
"status": data.get("status"),
|
|
"created_at": data.get("created_at"),
|
|
"summary": data.get("summary"),
|
|
}
|
|
)
|
|
if len(results) >= limit:
|
|
break
|
|
return results
|
|
|
|
|
|
def load_run(run_id: str) -> dict[str, Any]:
|
|
if not isinstance(run_id, str) or RUN_ID_PATTERN.fullmatch(run_id) is None:
|
|
raise ValueError(f"invalid evaluation run ID: {run_id!r}")
|
|
path = evaluations_state_root() / run_id / "run.json"
|
|
if not path.is_file():
|
|
raise FileNotFoundError(f"unknown evaluation run: {run_id}")
|
|
return read_json_object(path, label="evaluation run state")
|
|
|
|
|
|
def compare_runs(run_ids: Sequence[str]) -> dict[str, Any]:
|
|
if len(run_ids) < 2:
|
|
raise ValueError("compare requires at least two run IDs")
|
|
runs = [load_run(item) for item in run_ids]
|
|
suite_ids = {item.get("suite_id") for item in runs}
|
|
warning = None
|
|
if len(suite_ids) != 1:
|
|
warning = "runs use different suites; scores are not directly comparable"
|
|
rows: list[dict[str, Any]] = []
|
|
for run in runs:
|
|
summary = run.get("summary") or {}
|
|
variants = summary.get("variant_summaries") or {}
|
|
full: Mapping[str, Any] = next(
|
|
(
|
|
value
|
|
for value in variants.values()
|
|
if value.get("comparison_class") == "full_profile"
|
|
),
|
|
{},
|
|
)
|
|
rows.append(
|
|
{
|
|
"run_id": run["run_id"],
|
|
"profile_id": run["profile_id"],
|
|
"suite_id": run["suite_id"],
|
|
"status": run["status"],
|
|
"promotion": summary.get("promotion"),
|
|
"full_profile_success_rate": full.get("success_rate"),
|
|
"full_profile_outcome_metrics": full.get("outcome_metrics"),
|
|
"full_profile_mean_root_elapsed_seconds": full.get("mean_root_elapsed_seconds"),
|
|
"full_profile_cost_ledgers": full.get("cost_ledgers"),
|
|
"variant_summaries": variants,
|
|
}
|
|
)
|
|
ranked = sorted(
|
|
rows,
|
|
key=lambda item: (
|
|
-(
|
|
float(item["full_profile_success_rate"])
|
|
if item["full_profile_success_rate"] is not None
|
|
else -1.0
|
|
),
|
|
float(item["full_profile_mean_root_elapsed_seconds"] or float("inf")),
|
|
),
|
|
)
|
|
return {
|
|
"runs": rows,
|
|
"ranked_by_full_profile_success_then_time": ranked,
|
|
"warning": warning,
|
|
"aggregate_score": None,
|
|
"aggregate_score_reason": "controls and full profiles are not pooled into one score",
|
|
}
|