😏

evals/debugging-confidence/holdout/misleading-cache-symptom/test_holdout_policy.py

@@ -0,0 +1,17 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_true_to_false_toggle_invalidates(self):
+        policy = Policy({"preview": True})
+        self.assertTrue(policy.allows("u1", "preview"))
+        policy.set_flag("preview", False)
+        self.assertFalse(policy.allows("u1", "preview"))
+
+    def test_repeated_toggles_do_not_reuse_stale_value(self):
+        policy = Policy({"preview": False})
+        for value in (True, False, True):
+            policy.set_flag("preview", value)
+            self.assertEqual(policy.allows("u1", "preview"), value)

evals/debugging-confidence/holdout/no-speculative-rewrite/test_holdout_policy.py

@@ -0,0 +1,12 @@
+import unittest
+
+from policy import Policy
+
+
+class HoldoutPolicyTests(unittest.TestCase):
+    def test_unrelated_flag_change_preserves_correct_answers(self):
+        policy = Policy({"a": True, "b": False})
+        self.assertTrue(policy.allows("u", "a"))
+        policy.set_flag("b", True)
+        self.assertTrue(policy.allows("u", "a"))
+        self.assertTrue(policy.allows("u", "b"))