diff --git a/policy.py b/policy.py
--- a/policy.py
+++ b/policy.py
@@ -5,6 +5,8 @@ class Policy:
 
     def set_flag(self, name, enabled):
         self.flags[name] = bool(enabled)
+        for key in [key for key in self._decision_cache if key[0] == name]:
+            del self._decision_cache[key]
 
     def allows(self, user_id, flag):
         key = (user_id, flag)