import json from pathlib import Path from PIL import Image, ImageChops, ImageFilter, ImageStat reference_path = Path("reference.png") desktop_path = Path("actual-desktop.png") mobile_path = Path("actual-mobile.png") metadata_path = Path("render-metadata.json") for path in (reference_path, desktop_path, mobile_path, metadata_path): if not path.is_file(): raise SystemExit(f"missing real visual artifact: {path}") metadata = json.loads(metadata_path.read_text(encoding="utf-8")) if metadata.get("engine") != "playwright": raise SystemExit("visual render was not captured through Playwright") reference = Image.open(reference_path).convert("RGB") desktop = Image.open(desktop_path).convert("RGB") mobile = Image.open(mobile_path).convert("RGB") if desktop.size != (1440, 900) or mobile.size != (390, 844): raise SystemExit("browser screenshots do not match the required viewports") resampling = getattr(Image, "Resampling", Image).LANCZOS normalized_reference = reference.resize(desktop.size, resampling) difference = ImageChops.difference(normalized_reference, desktop) mean_error = sum(ImageStat.Stat(difference).mean) / (3 * 255) perceptual_similarity = max(0.0, 1.0 - mean_error) histogram = difference.convert("L").histogram() matched_pixels = sum(histogram[:32]) pixel_similarity = matched_pixels / (desktop.width * desktop.height) reference_edges = normalized_reference.convert("L").filter(ImageFilter.FIND_EDGES) desktop_edges = desktop.convert("L").filter(ImageFilter.FIND_EDGES) edge_error = ImageStat.Stat(ImageChops.difference(reference_edges, desktop_edges)).mean[0] / 255 edge_similarity = max(0.0, 1.0 - edge_error) viewport_facts = metadata["viewports"] responsive_checks = [ not viewport_facts["desktop"]["horizontalOverflow"], not viewport_facts["mobile"]["horizontalOverflow"], viewport_facts["mobile"]["bodyWidth"] <= 391, ] responsive_score = sum(responsive_checks) / len(responsive_checks) desktop_facts = viewport_facts["desktop"] accessibility_checks = [ desktop_facts["language"] == "en", desktop_facts["mainCount"] == 1, desktop_facts["navCount"] >= 1, desktop_facts["headingCount"] >= 2, desktop_facts["labeledCount"] >= 1, desktop_facts["focusableCount"] >= 1, desktop_facts["unlabeledImages"] == 0, ] accessibility_score = sum(accessibility_checks) / len(accessibility_checks) metrics = { "perceptual_similarity": perceptual_similarity, "pixel_similarity": pixel_similarity, "edge_similarity": edge_similarity, "responsive_score": responsive_score, "accessibility_score": accessibility_score, } default_thresholds = { "perceptual_similarity": 0.72, "pixel_similarity": 0.35, "edge_similarity": 0.72, "responsive_score": 1.0, "accessibility_score": 1.0, } holdout = Path("visual-holdout.json") thresholds = ( json.loads(holdout.read_text(encoding="utf-8")) if holdout.is_file() else default_thresholds ) if set(thresholds) != set(default_thresholds): raise SystemExit("visual holdout threshold schema is invalid") failures = [name for name, threshold in thresholds.items() if metrics[name] < threshold] if failures: raise SystemExit( "visual gates failed: " + ", ".join(f"{name}={metrics[name]:.3f}" for name in failures) ) print(json.dumps({"metrics": metrics}, sort_keys=True))