diff --git a/src/audio_display.cpp b/src/audio_display.cpp
index 9713c5add..865284a0b 100644
--- a/src/audio_display.cpp
+++ b/src/audio_display.cpp
@@ -1049,7 +1049,10 @@ void AudioDisplay::OnMouseEvent(wxMouseEvent& event)
 	// very expensive, and aegisub’s work queue handling is bad,
 	// so limit mouse event rate to ~200 Hz
 	long ts = event.GetTimestamp();
-	if ((ts - last_event) < 5) return;
+	if (!event.IsButton() && (ts - last_event) < 5) {
+		event.Skip();
+		return;
+	}
 	last_event = ts;
 
 	// If we have focus, we get mouse move events on Mac even when the mouse is
diff --git a/src/visual_tool.cpp b/src/visual_tool.cpp
index c8863eddb..ee471d2de 100644
--- a/src/visual_tool.cpp
+++ b/src/visual_tool.cpp
@@ -176,7 +176,10 @@ void VisualTool<FeatureType>::OnMouseEvent(wxMouseEvent &event) {
 	// very expensive, and aegisub’s work queue handling is bad,
 	// so limit mouse event rate to ~200 Hz
 	long ts = event.GetTimestamp();
-	if ((ts - lastEvent) < 5) return;
+	if (!event.IsButton() && (ts - lastEvent) < 5) {
+		event.Skip();
+		return;
+	}
 	lastEvent = ts;
 
 	bool left_click = event.LeftDown();