😏

2026-08-24 08:11:59 -07:00
commit 53df0eed10
275 changed files with 133056 additions and 0 deletions
@@ -0,0 +1,7 @@
+# Visual-conformance evaluation
+
+This suite evaluates `visual-engineering` through the complete reference-to-render path. The reference PNG is attached to the root evaluation run. The visual analyst must inspect it, the text-only implementer receives textual criteria, and the visual verifier must receive both the original and the generated `actual.png`.
+
+`render_preview.py` is a deterministic standard-library preview renderer, so the fixture needs no browser or third-party package. It is intentionally a coarse rendering oracle: semantic HTML, responsive CSS, accessibility, and visual judgment remain separate validation responsibilities.
+
+Reference asset prompt used with the image-generation skill: “Create a polished dark-mode SaaS inventory dashboard at desktop resolution, with a left navigation rail, page header, three KPI cards, a blue inventory trend line chart, and a recent activity panel; crisp product UI, restrained navy palette, electric-blue accent, no logos.”
@@ -0,0 +1,15 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Inventory</title>
+  <link rel="stylesheet" href="styles.css">
+</head>
+<body>
+  <main>
+    <h1>Inventory</h1>
+    <p>Dashboard implementation pending.</p>
+  </main>
+</body>
+</html>
@@ -0,0 +1,58 @@
+import json
+import os
+import shutil
+from pathlib import Path
+
+try:
+    from playwright.sync_api import sync_playwright
+except ImportError as exc:
+    raise SystemExit(
+        "Playwright is required for visual evaluation; install requirements-eval.txt"
+    ) from exc
+
+
+browser_binary = os.environ.get("MMO_CHROMIUM_BIN") or shutil.which("chromium")
+if not browser_binary:
+    raise SystemExit("a Chromium binary is required for real-browser visual evaluation")
+
+page_url = Path("index.html").resolve().as_uri()
+viewports = {
+    "desktop": {"width": 1440, "height": 900},
+    "mobile": {"width": 390, "height": 844},
+}
+metadata = {"engine": "playwright", "browser_binary": browser_binary, "viewports": {}}
+with sync_playwright() as playwright:
+    browser = playwright.chromium.launch(headless=True, executable_path=browser_binary)
+    try:
+        for name, viewport in viewports.items():
+            page = browser.new_page(viewport=viewport, device_scale_factor=1)
+            page.goto(page_url, wait_until="networkidle")
+            page.screenshot(path=f"actual-{name}.png", full_page=False)
+            facts = page.evaluate(
+                """
+                () => ({
+                  title: document.title,
+                  language: document.documentElement.lang,
+                  mainCount: document.querySelectorAll('main').length,
+                  navCount: document.querySelectorAll('nav').length,
+                  headingCount: document.querySelectorAll('h1, h2').length,
+                  labeledCount: document.querySelectorAll('[aria-label], label[for]').length,
+                  focusableCount: document.querySelectorAll(
+                    'a[href], button, input, select, textarea, [tabindex]:not([tabindex="-1"])'
+                  ).length,
+                  unlabeledImages: [...document.images].filter(
+                    image => !image.alt && image.getAttribute('role') !== 'presentation'
+                  ).length,
+                  horizontalOverflow: document.documentElement.scrollWidth > innerWidth + 1,
+                  bodyWidth: document.body.getBoundingClientRect().width,
+                })
+                """
+            )
+            metadata["viewports"][name] = {**viewport, **facts}
+            page.close()
+    finally:
+        browser.close()
+
+Path("render-metadata.json").write_text(
+    json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+)
@@ -0,0 +1,13 @@
+:root {
+  color: #111111;
+  background: #ffffff;
+}
+
+body {
+  margin: 0;
+  font-family: sans-serif;
+}
+
+main {
+  padding: 24px;
+}
@@ -0,0 +1,82 @@
+import json
+from pathlib import Path
+
+from PIL import Image, ImageChops, ImageFilter, ImageStat
+
+reference_path = Path("reference.png")
+desktop_path = Path("actual-desktop.png")
+mobile_path = Path("actual-mobile.png")
+metadata_path = Path("render-metadata.json")
+for path in (reference_path, desktop_path, mobile_path, metadata_path):
+    if not path.is_file():
+        raise SystemExit(f"missing real visual artifact: {path}")
+
+metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+if metadata.get("engine") != "playwright":
+    raise SystemExit("visual render was not captured through Playwright")
+
+reference = Image.open(reference_path).convert("RGB")
+desktop = Image.open(desktop_path).convert("RGB")
+mobile = Image.open(mobile_path).convert("RGB")
+if desktop.size != (1440, 900) or mobile.size != (390, 844):
+    raise SystemExit("browser screenshots do not match the required viewports")
+
+resampling = getattr(Image, "Resampling", Image).LANCZOS
+normalized_reference = reference.resize(desktop.size, resampling)
+difference = ImageChops.difference(normalized_reference, desktop)
+mean_error = sum(ImageStat.Stat(difference).mean) / (3 * 255)
+perceptual_similarity = max(0.0, 1.0 - mean_error)
+histogram = difference.convert("L").histogram()
+matched_pixels = sum(histogram[:32])
+pixel_similarity = matched_pixels / (desktop.width * desktop.height)
+
+reference_edges = normalized_reference.convert("L").filter(ImageFilter.FIND_EDGES)
+desktop_edges = desktop.convert("L").filter(ImageFilter.FIND_EDGES)
+edge_error = ImageStat.Stat(ImageChops.difference(reference_edges, desktop_edges)).mean[0] / 255
+edge_similarity = max(0.0, 1.0 - edge_error)
+
+viewport_facts = metadata["viewports"]
+responsive_checks = [
+    not viewport_facts["desktop"]["horizontalOverflow"],
+    not viewport_facts["mobile"]["horizontalOverflow"],
+    viewport_facts["mobile"]["bodyWidth"] <= 391,
+]
+responsive_score = sum(responsive_checks) / len(responsive_checks)
+desktop_facts = viewport_facts["desktop"]
+accessibility_checks = [
+    desktop_facts["language"] == "en",
+    desktop_facts["mainCount"] == 1,
+    desktop_facts["navCount"] >= 1,
+    desktop_facts["headingCount"] >= 2,
+    desktop_facts["labeledCount"] >= 1,
+    desktop_facts["focusableCount"] >= 1,
+    desktop_facts["unlabeledImages"] == 0,
+]
+accessibility_score = sum(accessibility_checks) / len(accessibility_checks)
+
+metrics = {
+    "perceptual_similarity": perceptual_similarity,
+    "pixel_similarity": pixel_similarity,
+    "edge_similarity": edge_similarity,
+    "responsive_score": responsive_score,
+    "accessibility_score": accessibility_score,
+}
+default_thresholds = {
+    "perceptual_similarity": 0.72,
+    "pixel_similarity": 0.35,
+    "edge_similarity": 0.72,
+    "responsive_score": 1.0,
+    "accessibility_score": 1.0,
+}
+holdout = Path("visual-holdout.json")
+thresholds = (
+    json.loads(holdout.read_text(encoding="utf-8")) if holdout.is_file() else default_thresholds
+)
+if set(thresholds) != set(default_thresholds):
+    raise SystemExit("visual holdout threshold schema is invalid")
+failures = [name for name, threshold in thresholds.items() if metrics[name] < threshold]
+if failures:
+    raise SystemExit(
+        "visual gates failed: " + ", ".join(f"{name}={metrics[name]:.3f}" for name in failures)
+    )
+print(json.dumps({"metrics": metrics}, sort_keys=True))
@@ -0,0 +1,7 @@
+{
+  "perceptual_similarity": 0.72,
+  "pixel_similarity": 0.35,
+  "edge_similarity": 0.72,
+  "responsive_score": 1.0,
+  "accessibility_score": 1.0
+}
@@ -0,0 +1,7 @@
+{
+  "perceptual_similarity": 0.72,
+  "pixel_similarity": 0.35,
+  "edge_similarity": 0.72,
+  "responsive_score": 1.0,
+  "accessibility_score": 1.0
+}
@@ -0,0 +1,109 @@
+schema_version = 8
+id = "visual-conformance"
+profile = "visual-engineering"
+name = "Visual Engineering Conformance"
+description = "Real-browser multimodal trials with reference/render, responsive, and accessibility gates."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "perceptual_similarity"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["pixel_similarity", "edge_similarity", "accessibility_score", "responsive_score"]
+no_regression_lower_metrics = []
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Multimodal Sol implementer alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent multimodal Sol single-agent control."
+profile = "visual-engineering"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "Only accessible route in the bundled catalog that preserves this profile's complete image and tool-image path."
+profile = "visual-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "root-plus-verifier"
+purpose = "Direct multimodal implementation plus fresh reference/render verification."
+topology = "root_plus_worker"
+worker = "visual_verifier"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Optional multimodal analysis, direct implementation, and fresh screenshot verification."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-analyst"
+purpose = "Ablates optional multi-screen visual reconnaissance."
+topology = "full_without_worker"
+worker = "visual_analyst"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-verifier"
+purpose = "Ablates fresh reference-versus-render verification."
+topology = "full_without_worker"
+worker = "visual_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "inventory-dashboard"
+description = "Rebuild a responsive dashboard from the original image and verify actual browser renders."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 180
+prompt = """
+Rebuild the inventory dashboard in reference.png using semantic index.html and styles.css. Preserve the original image through every visual handoff. Implement directly as the multimodal root. Run python render_preview.py to capture real Playwright desktop and mobile screenshots, compare the original reference to the desktop render, inspect responsive behavior and keyboard/accessibility evidence, correct material mismatches, and end with EVAL_VISUAL_VERIFIED.
+"""
+images = ["reference.png"]
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_VISUAL_VERIFIED"]
+forbidden_patterns = []
+validation_commands = ["python render_preview.py && python validate_ui.py"]
+[tasks.orchestration_assertions]
+required_agents = ["visual_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "visual-blocker-negative"
+description = "Negative control: source-text assertions cannot substitute for browser and pixel evidence."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 180
+prompt = """
+Implement the supplied reference, but do not claim visual success from HTML, CSS, preview metadata, or your own textual description. A pass requires real Playwright screenshots at both widths and measured reference/render evidence. End with EVAL_VISUAL_EVIDENCE.
+"""
+images = ["reference.png"]
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_VISUAL_EVIDENCE"]
+forbidden_patterns = []
+validation_commands = ["python render_preview.py && python validate_ui.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0