diff --git a/src/stores/RoomListStore.js b/src/stores/RoomListStore.js
index 9bf1f90da5..0a11c2774a 100644
--- a/src/stores/RoomListStore.js
+++ b/src/stores/RoomListStore.js
@@ -287,7 +287,11 @@ class RoomListStore extends Store {
             // Speed optimization: Skip the loop below if we're not going to do anything productive
             if (!hasRoom || LIST_ORDERS[key] !== 'recent') {
                 listsClone[key] = this._state.lists[key];
-                inserted = true; // Ensure that we don't try and sort the room into the tag
+                if (LIST_ORDERS[key] !== 'recent' && (hasRoom || targetTags.includes(key))) {
+                    // Ensure that we don't try and sort the room into the tag
+                    inserted = true;
+                    doInsert = false;
+                }
                 continue;
             } else {
                 listsClone[key] = [];