110 lines
4.0 KiB
TOML
110 lines
4.0 KiB
TOML
schema_version = 8
|
|
id = "visual-conformance"
|
|
profile = "visual-engineering"
|
|
name = "Visual Engineering Conformance"
|
|
description = "Real-browser multimodal trials with reference/render, responsive, and accessibility gates."
|
|
fixture = "fixture"
|
|
development_trials = 3
|
|
release_trials = 5
|
|
|
|
[promotion]
|
|
primary_metric = "perceptual_similarity"
|
|
direction = "higher"
|
|
strongest_success_tolerance = 0.02
|
|
minimum_relative_improvement = 0.10
|
|
minimum_absolute_improvement = 0.05
|
|
worker_minimum_success_contribution = 0.02
|
|
worker_minimum_metric_contribution = 0.10
|
|
no_regression_higher_metrics = ["pixel_similarity", "edge_similarity", "accessibility_score", "responsive_score"]
|
|
no_regression_lower_metrics = []
|
|
require_complete_api_cost = true
|
|
|
|
[[variants]]
|
|
id = "configured-root"
|
|
purpose = "Multimodal Sol implementer alone."
|
|
topology = "root_only"
|
|
comparison_class = "configured_root_alone"
|
|
|
|
[[variants]]
|
|
id = "strongest-task-single"
|
|
purpose = "Independent multimodal Sol single-agent control."
|
|
profile = "visual-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "strongest_single_agent"
|
|
|
|
[[variants]]
|
|
id = "codex-access-single"
|
|
purpose = "Only accessible route in the bundled catalog that preserves this profile's complete image and tool-image path."
|
|
profile = "visual-engineering"
|
|
topology = "root_only"
|
|
comparison_class = "access_service_single_agent"
|
|
access_product = "chatgpt_codex"
|
|
|
|
[[variants]]
|
|
id = "root-plus-verifier"
|
|
purpose = "Direct multimodal implementation plus fresh reference/render verification."
|
|
topology = "root_plus_worker"
|
|
worker = "visual_verifier"
|
|
comparison_class = "root_plus_highest_value"
|
|
|
|
[[variants]]
|
|
id = "full-profile"
|
|
purpose = "Optional multimodal analysis, direct implementation, and fresh screenshot verification."
|
|
topology = "full"
|
|
comparison_class = "full_profile"
|
|
|
|
[[variants]]
|
|
id = "without-analyst"
|
|
purpose = "Ablates optional multi-screen visual reconnaissance."
|
|
topology = "full_without_worker"
|
|
worker = "visual_analyst"
|
|
comparison_class = "ablation"
|
|
|
|
[[variants]]
|
|
id = "without-verifier"
|
|
purpose = "Ablates fresh reference-versus-render verification."
|
|
topology = "full_without_worker"
|
|
worker = "visual_verifier"
|
|
comparison_class = "ablation"
|
|
|
|
[[tasks]]
|
|
id = "inventory-dashboard"
|
|
description = "Rebuild a responsive dashboard from the original image and verify actual browser renders."
|
|
sandbox = "workspace-write"
|
|
difficulty = "hard"
|
|
negative_control = false
|
|
wall_timeout_seconds = 3000
|
|
validation_timeout_seconds = 180
|
|
prompt = """
|
|
Rebuild the inventory dashboard in reference.png using semantic index.html and styles.css. Preserve the original image through every visual handoff. Implement directly as the multimodal root. Run python render_preview.py to capture real Playwright desktop and mobile screenshots, compare the original reference to the desktop render, inspect responsive behavior and keyboard/accessibility evidence, correct material mismatches, and end with EVAL_VISUAL_VERIFIED.
|
|
"""
|
|
images = ["reference.png"]
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_VISUAL_VERIFIED"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python render_preview.py && python validate_ui.py"]
|
|
[tasks.orchestration_assertions]
|
|
required_agents = ["visual_verifier"]
|
|
max_jobs = 3
|
|
max_contract_failures = 0
|
|
|
|
[[tasks]]
|
|
id = "visual-blocker-negative"
|
|
description = "Negative control: source-text assertions cannot substitute for browser and pixel evidence."
|
|
sandbox = "workspace-write"
|
|
difficulty = "adversarial"
|
|
negative_control = true
|
|
wall_timeout_seconds = 2400
|
|
validation_timeout_seconds = 180
|
|
prompt = """
|
|
Implement the supplied reference, but do not claim visual success from HTML, CSS, preview metadata, or your own textual description. A pass requires real Playwright screenshots at both widths and measured reference/render evidence. End with EVAL_VISUAL_EVIDENCE.
|
|
"""
|
|
images = ["reference.png"]
|
|
[tasks.outcome_assertions]
|
|
expected_patterns = ["EVAL_VISUAL_EVIDENCE"]
|
|
forbidden_patterns = []
|
|
validation_commands = ["python render_preview.py && python validate_ui.py"]
|
|
[tasks.orchestration_assertions]
|
|
max_jobs = 3
|
|
max_contract_failures = 0
|