diff --git a/aegisub/src/audio_display.cpp b/aegisub/src/audio_display.cpp
index 595cc19ff..641fd2657 100644
--- a/aegisub/src/audio_display.cpp
+++ b/aegisub/src/audio_display.cpp
@@ -1027,10 +1027,6 @@ void AudioDisplay::OnPaint(wxPaintEvent&)
 
 void AudioDisplay::SetDraggedObject(AudioDisplayInteractionObject *new_obj)
 {
-	// Special case for audio markers being dragged: they use a temporary wrapper object
-	// which must be deleted when it is no longer used.
-	delete dynamic_cast<AudioMarkerInteractionObject*>(dragged_object);
-
 	dragged_object = new_obj;
 
 	if (dragged_object && !HasCapture())
@@ -1162,7 +1158,8 @@ void AudioDisplay::OnMouseEvent(wxMouseEvent& event)
 		if (marker)
 		{
 			RemoveTrackCursor();
-			SetDraggedObject(new AudioMarkerInteractionObject(marker, timing, this, controller, wxMOUSE_BTN_LEFT));
+			audio_marker.reset(new AudioMarkerInteractionObject(marker, timing, this, controller, wxMOUSE_BTN_LEFT));
+			SetDraggedObject(audio_marker.get());
 			return;
 		}
 	}
@@ -1175,7 +1172,8 @@ void AudioDisplay::OnMouseEvent(wxMouseEvent& event)
 		if (marker)
 		{
 			RemoveTrackCursor();
-			SetDraggedObject(new AudioMarkerInteractionObject(marker, timing, this, controller, wxMOUSE_BTN_RIGHT));
+			audio_marker.reset(new AudioMarkerInteractionObject(marker, timing, this, controller, wxMOUSE_BTN_RIGHT));
+			SetDraggedObject(audio_marker.get());
 			return;
 		}
 	}
diff --git a/aegisub/src/audio_display.h b/aegisub/src/audio_display.h
index d9d9d6cfc..413fdbbca 100644
--- a/aegisub/src/audio_display.h
+++ b/aegisub/src/audio_display.h
@@ -58,6 +58,7 @@ class AudioProvider;
 class AudioDisplayScrollbar;
 class AudioDisplayTimeline;
 class AudioDisplaySelection;
+class AudioMarkerInteractionObject;
 
 /// @class AudioDisplayInteractionObject
 /// @brief Interface for objects on the audio display that can respond to mouse events
@@ -122,6 +123,9 @@ class AudioDisplay: public wxWindow {
 	/// Timeline helper object
 	agi::scoped_ptr<AudioDisplayTimeline> timeline;
 
+	/// The interaction object for the last-dragged audio marker
+	agi::scoped_ptr<AudioMarkerInteractionObject> audio_marker;
+
 
 	/// Current object on display being dragged, if any
 	AudioDisplayInteractionObject *dragged_object;