diff --git a/src/core/button.rs b/src/core/button.rs
index 6ad2c04..2f17007 100644
--- a/src/core/button.rs
+++ b/src/core/button.rs
@@ -268,8 +268,8 @@ impl<D: Display, K: Keyboard> Button<D, K> {
             let changed = self.update_modifier(modifier,
                                                ModState::Released,
                                                ModState::Pressed);
-            if changed {
-                draw = draw + DrawOperation::from_modifier_edge(modifier);
+            if changed && Layout::is_label_modifier(modifier) {
+                draw = draw + DrawOperation::Labels;
             }
 
             self.update_modifier(modifier,