😏

evals/security-assurance/README.md

@@ -0,0 +1,3 @@
+# Security assurance evaluation
+
+Designed for `secure-change`. The fixture contains one reachable archive traversal defect and one suspicious but safely normalized path. It measures finding precision, empirical confirmation, remediation, and regression safety.

evals/security-assurance/fixture/archive.py

@@ -0,0 +1,15 @@
+from pathlib import Path, PurePosixPath
+
+
+def normalized_asset_path(name):
+    parts = [part for part in PurePosixPath(name).parts if part not in {"", ".", ".."}]
+    return Path("assets").joinpath(*parts)
+
+
+def extract_entries(entries, destination):
+    destination = Path(destination)
+    destination.mkdir(parents=True, exist_ok=True)
+    for name, content in entries:
+        target = destination / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content)

evals/security-assurance/fixture/test_archive.py

@@ -0,0 +1,30 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive import extract_entries, normalized_asset_path
+
+
+class ArchiveTests(unittest.TestCase):
+    def test_normalized_asset_path_discards_parent_components(self):
+        self.assertEqual(normalized_asset_path("icons/../safe.svg"), Path("assets/icons/safe.svg"))
+
+    def test_safe_nested_extraction(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            extract_entries([("nested/data.txt", b"ok")], temporary)
+            self.assertEqual(Path(temporary, "nested/data.txt").read_bytes(), b"ok")
+
+    def test_extraction_rejects_parent_escape(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            outside = Path(temporary).parent / "escaped-mmo-eval.txt"
+            outside.unlink(missing_ok=True)
+            try:
+                with self.assertRaises(ValueError):
+                    extract_entries([("../escaped-mmo-eval.txt", b"bad")], temporary)
+                self.assertFalse(outside.exists())
+            finally:
+                outside.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

evals/security-assurance/fixture/validate_security_report.py

@@ -0,0 +1,27 @@
+import json
+from pathlib import Path
+
+path = Path("SECURITY_REPORT.md")
+if not path.is_file():
+    raise SystemExit("SECURITY_REPORT.md is required")
+text = path.read_text(encoding="utf-8").lower()
+required = ("command", "observation", "rejected", "normalized_asset_path", "traversal")
+missing = [term for term in required if term not in text]
+if missing:
+    raise SystemExit("security report lacks evidence fields: " + ", ".join(missing))
+if "normalized_asset_path" in text and not any(
+    term in text for term in ("not vulnerable", "negative control", "rejected candidate")
+):
+    raise SystemExit("benign normalizer was not explicitly rejected as a finding")
+print(
+    json.dumps(
+        {
+            "metrics": {
+                "vulnerability_recall": 1.0,
+                "false_positive_rate": 0.0,
+                "empirical_reproduction_rate": 1.0,
+            }
+        },
+        sort_keys=True,
+    )
+)

evals/security-assurance/holdout/archive-boundary/test_holdout_archive.py

@@ -0,0 +1,23 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from archive import extract_entries
+
+
+class HoldoutArchiveTests(unittest.TestCase):
+    def test_absolute_member_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            with self.assertRaises(ValueError):
+                extract_entries([("/tmp/mmo-forbidden", b"bad")], temporary)
+
+    def test_nested_parent_escape_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            outside = Path(temporary).parent / "mmo-hidden-escape"
+            outside.unlink(missing_ok=True)
+            try:
+                with self.assertRaises(ValueError):
+                    extract_entries([("safe/../../mmo-hidden-escape", b"bad")], temporary)
+                self.assertFalse(outside.exists())
+            finally:
+                outside.unlink(missing_ok=True)

evals/security-assurance/holdout/benign-path-negative/test_holdout_archive.py

@@ -0,0 +1,12 @@
+import unittest
+from pathlib import Path
+
+from archive import normalized_asset_path
+
+
+class HoldoutNegativeTests(unittest.TestCase):
+    def test_normalizer_remains_bounded_and_usable(self):
+        self.assertEqual(
+            normalized_asset_path("themes/../icons/safe.svg"),
+            Path("assets/themes/icons/safe.svg"),
+        )

evals/security-assurance/suite.toml

@@ -0,0 +1,141 @@
+schema_version = 8
+id = "security-assurance"
+profile = "secure-change"
+name = "Secure Change Assurance"
+description = "Measures vulnerability recall, false positives, empirical reproduction, and remediation validation."
+fixture = "fixture"
+development_trials = 3
+release_trials = 5
+
+[promotion]
+primary_metric = "vulnerability_recall"
+direction = "higher"
+strongest_success_tolerance = 0.02
+minimum_relative_improvement = 0.10
+minimum_absolute_improvement = 0.05
+worker_minimum_success_contribution = 0.02
+worker_minimum_metric_contribution = 0.10
+no_regression_higher_metrics = ["empirical_reproduction_rate"]
+no_regression_lower_metrics = ["false_positive_rate"]
+require_complete_api_cost = true
+
+[[variants]]
+id = "configured-root"
+purpose = "Opus security lead alone."
+topology = "root_only"
+comparison_class = "configured_root_alone"
+
+[[variants]]
+id = "strongest-task-single"
+purpose = "Independent Opus single-agent security control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "strongest_single_agent"
+
+[[variants]]
+id = "codex-access-single"
+purpose = "ChatGPT Codex security control."
+profile = "adaptive-engineering"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "chatgpt_codex"
+
+[[variants]]
+id = "go-access-single"
+purpose = "OpenCode Go DeepSeek security control."
+profile = "high-confidence-debugging"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_go"
+
+[[variants]]
+id = "zen-access-single"
+purpose = "OpenCode Zen Opus security control."
+profile = "secure-change"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "opencode_zen"
+
+[[variants]]
+id = "zai-access-single"
+purpose = "Z.AI Coding Plan GLM security control."
+profile = "incident-hypothesis-triage"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "zai_coding_plan"
+
+[[variants]]
+id = "openrouter-access-single"
+purpose = "Pinned OpenRouter security control."
+profile = "route-resilience-lab"
+topology = "root_only"
+comparison_class = "access_service_single_agent"
+access_product = "openrouter_api"
+[variants.bindings]
+route_observer = "openrouter_openai_chat__deepseek_deepseek_v4_pro"
+
+[[variants]]
+id = "root-plus-hunter"
+purpose = "Opus remediation lead plus GLM candidate vulnerability hunter."
+topology = "root_plus_worker"
+worker = "vulnerability_hunter"
+comparison_class = "root_plus_highest_value"
+
+[[variants]]
+id = "full-profile"
+purpose = "Cross-maker hunt, empirical validation, remediation, and rescan."
+topology = "full"
+comparison_class = "full_profile"
+
+[[variants]]
+id = "without-glm-hunter"
+purpose = "Isolates GLM 5.3's marginal vulnerability-discovery contribution."
+topology = "full_without_worker"
+worker = "vulnerability_hunter"
+comparison_class = "ablation"
+
+[[variants]]
+id = "without-empirical-verifier"
+purpose = "Ablates independent command-backed finding validation."
+topology = "full_without_worker"
+worker = "finding_verifier"
+comparison_class = "ablation"
+
+[[tasks]]
+id = "archive-boundary"
+description = "Find and remediate a reachable archive traversal while preserving a benign path normalizer."
+sandbox = "workspace-write"
+difficulty = "adversarial"
+negative_control = false
+wall_timeout_seconds = 3000
+validation_timeout_seconds = 120
+prompt = """
+Perform an authorized defensive review of archive.py. Empirically reproduce consequential findings before remediation, preserve safe extraction, and write SECURITY_REPORT.md separating tested defects, evidence-backed risks, rejected candidates, commands, and observations. normalized_asset_path is an intentional negative control. End with EVAL_SECURITY_REMEDIATED.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_SECURITY_REMEDIATED"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_security_report.py"]
+[tasks.orchestration_assertions]
+required_agents = ["vulnerability_hunter", "finding_verifier"]
+max_jobs = 3
+max_contract_failures = 0
+
+[[tasks]]
+id = "benign-path-negative"
+description = "Measures false-positive control around a deliberately benign normalizer."
+sandbox = "workspace-write"
+difficulty = "hard"
+negative_control = true
+wall_timeout_seconds = 2400
+validation_timeout_seconds = 120
+prompt = """
+Audit and remediate only empirically reachable extraction defects. Do not classify normalized_asset_path as a vulnerability merely because it processes parent components. Record the tested negative in SECURITY_REPORT.md and end with EVAL_SECURITY_NEGATIVE_CONTROL.
+"""
+[tasks.outcome_assertions]
+expected_patterns = ["EVAL_SECURITY_NEGATIVE_CONTROL"]
+forbidden_patterns = []
+validation_commands = ["python -m unittest -v && python validate_security_report.py"]
+[tasks.orchestration_assertions]
+max_jobs = 3
+max_contract_failures = 0