diff --git a/res/css/structures/_TagPanel.scss b/res/css/structures/_TagPanel.scss
index b03d36a592..8efd0796f5 100644
--- a/res/css/structures/_TagPanel.scss
+++ b/res/css/structures/_TagPanel.scss
@@ -68,7 +68,7 @@ limitations under the License.
 }
 .mx_TagPanel .mx_TagPanel_tagTileContainer > div {
     height: 40px;
-    padding: 5px 0 4px 0;
+    padding: 10px 0 9px 0;
 }
 
 .mx_TagPanel .mx_TagTile {
@@ -82,21 +82,15 @@ limitations under the License.
     // opacity: 1;
 }
 
-.mx_TagPanel .mx_TagTile.mx_TagTile_selected .mx_TagTile_avatar .mx_BaseAvatar {
+.mx_TagPanel .mx_TagTile.mx_TagTile_selected::before {
+    content: '';
+    height: 56px;
     background-color: $accent-color;
-    border-radius: 40px;
-
-    /* In case this is a "initial" avatar */
-    display: block;
-    height: 40px;
-    width: 40px;
-}
-
-.mx_TagPanel .mx_TagTile_selected .mx_BaseAvatar_image {
-    border: 3px solid $accent-color;
-    height: 40px;
-    width: 40px;
-    box-sizing: border-box;
+    width: 5px;
+    position: absolute;
+    left: -15px;
+    border-radius: 0 2px 2px 0;
+    top: -8px; // (56 - 40)/2
 }
 
 .mx_TagPanel .mx_TagTile.mx_AccessibleButton:focus {
diff --git a/src/stores/TagOrderStore.js b/src/stores/TagOrderStore.js
index 48a8817270..973d27f4e7 100644
--- a/src/stores/TagOrderStore.js
+++ b/src/stores/TagOrderStore.js
@@ -141,8 +141,13 @@ class TagOrderStore extends Store {
                             newTags = [...this._state.selectedTags, payload.tag];
                         }
                     } else {
-                        // Select individual tag
-                        newTags = [payload.tag];
+                        if (this._state.selectedTags.length === 1 && this._state.selectedTags.includes(payload.tag)) {
+                            // Existing (only) selected tag is being normally clicked again, clear tags
+                            newTags = [];
+                        } else {
+                            // Select individual tag
+                            newTags = [payload.tag];
+                        }
                     }
                     // Only set the anchor tag if the tag was previously unselected, otherwise
                     // the next range starts with an unselected tag.