diff --git a/aegisub/src/base_grid.cpp b/aegisub/src/base_grid.cpp
index 6af9ce078..ceb4c3ae3 100644
--- a/aegisub/src/base_grid.cpp
+++ b/aegisub/src/base_grid.cpp
@@ -773,7 +773,9 @@ void BaseGrid::OnMouseEvent(wxMouseEvent &event) {
 
 		// Toggle selected
 		if (click && ctrl && !shift && !alt) {
-			SelectRow(row,true,!IsInSelection(row,0));
+			bool isSel = IsInSelection(row,0);
+			if (isSel && selection.size() == 1) return;
+			SelectRow(row,true,!isSel);
 			if (dlg == GetActiveLine()) {
 				SetActiveLine(GetDialogue(GetFirstSelRow()));
 			}