diff --git a/src/core/button.rs b/src/core/button.rs
index dd0a499..e2baf8a 100644
--- a/src/core/button.rs
+++ b/src/core/button.rs
@@ -430,6 +430,24 @@ impl<D: Display, K: Keyboard> Button<D, K> {
         Self::draw(draw, &self.gfx, &self.layout, Some(key), &self.modifiers);
     }
 
+    fn is_normal_key_pressed(&self) -> bool
+    {
+        for id in &self.timers {
+            if *id == PRESSES_MAX {
+                continue;
+            }
+
+            let press = self.presses[*id].as_ref().unwrap();
+            let key = self.layout.locate_key(press.x1, press.y1).unwrap();
+
+            if key.parts[press.part].modifier_id() == 0 {
+                return true;
+            }
+        }
+
+        false
+    }
+
     /*
      * Release the given modifier key. If the key being pressed is not a
      * modifier key (i.e. modifier = 0), release all latched modifier keys.
@@ -443,6 +461,24 @@ impl<D: Display, K: Keyboard> Button<D, K> {
                                  ModState::Pressed,
                                  ModState::Latched);
 
+            self.update_modifier(modifier,
+                                 ModState::HeldPressed,
+                                 ModState::Held);
+
+            self.update_modifier(modifier,
+                                 ModState::HeldLockedPressed,
+                                 ModState::HeldLocked);
+
+            if self.is_normal_key_pressed() {
+                self.update_modifier(modifier,
+                                     ModState::Held,
+                                     ModState::Latched);
+
+                self.update_modifier(modifier,
+                                     ModState::HeldLocked,
+                                     ModState::Locked);
+            }
+
             let changed = self.update_modifier(modifier,
                                                ModState::Held,
                                                ModState::Released);
@@ -456,20 +492,6 @@ impl<D: Display, K: Keyboard> Button<D, K> {
             if changed && Layout::is_label_modifier(modifier) {
                 draw = draw + DrawOperation::Labels;
             }
-
-            let changed = self.update_modifier(modifier,
-                                               ModState::HeldPressed,
-                                               ModState::Released);
-            if changed && Layout::is_label_modifier(modifier) {
-                draw = draw + DrawOperation::Labels;
-            }
-
-            let changed = self.update_modifier(modifier,
-                                               ModState::HeldLockedPressed,
-                                               ModState::Released);
-            if changed && Layout::is_label_modifier(modifier) {
-                draw = draw + DrawOperation::Labels;
-            }
         } else {
             for modifier in 1..=MODIFIERS_MAX {
                 let changed = self.update_modifier(modifier,