83 lines
3.3 KiB
Python
83 lines
3.3 KiB
Python
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from PIL import Image, ImageChops, ImageFilter, ImageStat
|
||
|
|
|
||
|
|
reference_path = Path("reference.png")
|
||
|
|
desktop_path = Path("actual-desktop.png")
|
||
|
|
mobile_path = Path("actual-mobile.png")
|
||
|
|
metadata_path = Path("render-metadata.json")
|
||
|
|
for path in (reference_path, desktop_path, mobile_path, metadata_path):
|
||
|
|
if not path.is_file():
|
||
|
|
raise SystemExit(f"missing real visual artifact: {path}")
|
||
|
|
|
||
|
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||
|
|
if metadata.get("engine") != "playwright":
|
||
|
|
raise SystemExit("visual render was not captured through Playwright")
|
||
|
|
|
||
|
|
reference = Image.open(reference_path).convert("RGB")
|
||
|
|
desktop = Image.open(desktop_path).convert("RGB")
|
||
|
|
mobile = Image.open(mobile_path).convert("RGB")
|
||
|
|
if desktop.size != (1440, 900) or mobile.size != (390, 844):
|
||
|
|
raise SystemExit("browser screenshots do not match the required viewports")
|
||
|
|
|
||
|
|
resampling = getattr(Image, "Resampling", Image).LANCZOS
|
||
|
|
normalized_reference = reference.resize(desktop.size, resampling)
|
||
|
|
difference = ImageChops.difference(normalized_reference, desktop)
|
||
|
|
mean_error = sum(ImageStat.Stat(difference).mean) / (3 * 255)
|
||
|
|
perceptual_similarity = max(0.0, 1.0 - mean_error)
|
||
|
|
histogram = difference.convert("L").histogram()
|
||
|
|
matched_pixels = sum(histogram[:32])
|
||
|
|
pixel_similarity = matched_pixels / (desktop.width * desktop.height)
|
||
|
|
|
||
|
|
reference_edges = normalized_reference.convert("L").filter(ImageFilter.FIND_EDGES)
|
||
|
|
desktop_edges = desktop.convert("L").filter(ImageFilter.FIND_EDGES)
|
||
|
|
edge_error = ImageStat.Stat(ImageChops.difference(reference_edges, desktop_edges)).mean[0] / 255
|
||
|
|
edge_similarity = max(0.0, 1.0 - edge_error)
|
||
|
|
|
||
|
|
viewport_facts = metadata["viewports"]
|
||
|
|
responsive_checks = [
|
||
|
|
not viewport_facts["desktop"]["horizontalOverflow"],
|
||
|
|
not viewport_facts["mobile"]["horizontalOverflow"],
|
||
|
|
viewport_facts["mobile"]["bodyWidth"] <= 391,
|
||
|
|
]
|
||
|
|
responsive_score = sum(responsive_checks) / len(responsive_checks)
|
||
|
|
desktop_facts = viewport_facts["desktop"]
|
||
|
|
accessibility_checks = [
|
||
|
|
desktop_facts["language"] == "en",
|
||
|
|
desktop_facts["mainCount"] == 1,
|
||
|
|
desktop_facts["navCount"] >= 1,
|
||
|
|
desktop_facts["headingCount"] >= 2,
|
||
|
|
desktop_facts["labeledCount"] >= 1,
|
||
|
|
desktop_facts["focusableCount"] >= 1,
|
||
|
|
desktop_facts["unlabeledImages"] == 0,
|
||
|
|
]
|
||
|
|
accessibility_score = sum(accessibility_checks) / len(accessibility_checks)
|
||
|
|
|
||
|
|
metrics = {
|
||
|
|
"perceptual_similarity": perceptual_similarity,
|
||
|
|
"pixel_similarity": pixel_similarity,
|
||
|
|
"edge_similarity": edge_similarity,
|
||
|
|
"responsive_score": responsive_score,
|
||
|
|
"accessibility_score": accessibility_score,
|
||
|
|
}
|
||
|
|
default_thresholds = {
|
||
|
|
"perceptual_similarity": 0.72,
|
||
|
|
"pixel_similarity": 0.35,
|
||
|
|
"edge_similarity": 0.72,
|
||
|
|
"responsive_score": 1.0,
|
||
|
|
"accessibility_score": 1.0,
|
||
|
|
}
|
||
|
|
holdout = Path("visual-holdout.json")
|
||
|
|
thresholds = (
|
||
|
|
json.loads(holdout.read_text(encoding="utf-8")) if holdout.is_file() else default_thresholds
|
||
|
|
)
|
||
|
|
if set(thresholds) != set(default_thresholds):
|
||
|
|
raise SystemExit("visual holdout threshold schema is invalid")
|
||
|
|
failures = [name for name, threshold in thresholds.items() if metrics[name] < threshold]
|
||
|
|
if failures:
|
||
|
|
raise SystemExit(
|
||
|
|
"visual gates failed: " + ", ".join(f"{name}={metrics[name]:.3f}" for name in failures)
|
||
|
|
)
|
||
|
|
print(json.dumps({"metrics": metrics}, sort_keys=True))
|