diff --git a/src/core/button.rs b/src/core/button.rs
index fcf4361..6ad2c04 100644
--- a/src/core/button.rs
+++ b/src/core/button.rs
@@ -239,11 +239,11 @@ impl<D: Display, K: Keyboard> Button<D, K> {
         if self.modifiers[modifier - 1] == old {
             self.modifiers[modifier - 1] = new;
 
-            if new == ModState::Released {
+            if new == ModState::Released && !Layout::is_keysym_modifier(modifier) {
                 self.kbd.release(Layout::modifier_keysym(modifier));
             }
 
-            if old == ModState::Released {
+            if old == ModState::Released && !Layout::is_keysym_modifier(modifier) {
                 self.kbd.press(Layout::modifier_keysym(modifier));
             }