Improve stability of Playwright screenshot tests (#11983)

playwright/e2e/editing/editing.spec.ts

@@ -139,7 +139,7 @@ test.describe("Editing", () => {
         ).toBeVisible();
 
         // Take a snapshot of the dialog
-        await expect(dialog).toHaveScreenshot("message-edit-history-dialog.png", {
+        await expect(dialog).toMatchScreenshot("message-edit-history-dialog.png", {
             mask: [page.locator(".mx_MessageTimestamp")],
         });
 

playwright/e2e/login/soft_logout.spec.ts

@@ -118,11 +118,13 @@ async function interceptRequestsWithSoftLogout(page: Page, user: Credentials): P
         await route.continue();
     });
 
+    const promise = page.waitForResponse((resp) => resp.url().includes("/sync") && resp.status() === 401);
+
     // do something to make the active /sync return: create a new room
     await page.evaluate(() => {
         // don't wait for this to complete: it probably won't, because of the broken sync
         window.mxMatrixClientPeg.get().createRoom({});
     });
 
-    await page.waitForResponse((resp) => resp.url().includes("/sync") && resp.status() === 401);
+    await promise;
 }

playwright/e2e/register/email.spec.ts

@@ -71,7 +71,7 @@ test.describe("Email Registration", async () => {
         await page.getByRole("button", { name: "Register" }).click();
 
         await expect(page.getByText("Check your email to continue")).toBeVisible();
-        await expect(page).toHaveScreenshot("registration_check_your_email.png", screenshotOptions);
+        await expect(page).toMatchScreenshot("registration_check_your_email.png", screenshotOptions);
         await checkA11y();
 
         await expect(page.getByText("An error was encountered when sending the email")).not.toBeVisible();

playwright/e2e/register/register.spec.ts

@@ -27,7 +27,7 @@ test.describe("Registration", () => {
         await page.getByRole("button", { name: "Edit", exact: true }).click();
         await expect(page.getByRole("button", { name: "Continue", exact: true })).toBeVisible();
 
-        await expect(page.locator(".mx_Dialog")).toHaveScreenshot("server-picker.png");
+        await expect(page.locator(".mx_Dialog")).toMatchScreenshot("server-picker.png");
         await checkA11y();
 
         await page.getByRole("textbox", { name: "Other homeserver" }).fill(homeserver.config.baseUrl);
@@ -38,7 +38,7 @@ test.describe("Registration", () => {
         await expect(page.getByRole("textbox", { name: "Username", exact: true })).toBeVisible();
         // Hide the server text as it contains the randomly allocated Homeserver port
         const screenshotOptions = { mask: [page.locator(".mx_ServerPicker_server")] };
-        await expect(page).toHaveScreenshot("registration.png", screenshotOptions);
+        await expect(page).toMatchScreenshot("registration.png", screenshotOptions);
         await checkA11y();
 
         await page.getByRole("textbox", { name: "Username", exact: true }).fill("alice");
@@ -48,12 +48,12 @@ test.describe("Registration", () => {
 
         const dialog = page.getByRole("dialog");
         await expect(dialog).toBeVisible();
-        await expect(page).toHaveScreenshot("email-prompt.png", screenshotOptions);
+        await expect(page).toMatchScreenshot("email-prompt.png", screenshotOptions);
         await checkA11y();
         await dialog.getByRole("button", { name: "Continue", exact: true }).click();
 
         await expect(page.locator(".mx_InteractiveAuthEntryComponents_termsPolicy")).toBeVisible();
-        await expect(page).toHaveScreenshot("terms-prompt.png", screenshotOptions);
+        await expect(page).toMatchScreenshot("terms-prompt.png", screenshotOptions);
         await checkA11y();
 
         const termsPolicy = page.locator(".mx_InteractiveAuthEntryComponents_termsPolicy");
@@ -63,7 +63,7 @@ test.describe("Registration", () => {
         await page.getByRole("button", { name: "Accept", exact: true }).click();
 
         await expect(page.locator(".mx_UseCaseSelection_skip")).toBeVisible();
-        await expect(page).toHaveScreenshot("use-case-selection.png", screenshotOptions);
+        await expect(page).toMatchScreenshot("use-case-selection.png", screenshotOptions);
         await checkA11y();
         await page.getByRole("button", { name: "Skip", exact: true }).click();
 

playwright/e2e/right-panel/file-panel.spec.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { type Page } from "@playwright/test";
+import { Download, type Page } from "@playwright/test";
 
 import { test, expect } from "../../element-web-test";
 import { viewRoomSummaryByName } from "./utils";
@@ -53,7 +53,7 @@ test.describe("FilePanel", () => {
             await expect(page.locator(".mx_FilePanel_empty")).toBeVisible();
 
             // Take a snapshot of RightPanel - fix https://github.com/vector-im/element-web/issues/25332
-            await expect(page.locator(".mx_RightPanel")).toHaveScreenshot("empty.png");
+            await expect(page.locator(".mx_RightPanel")).toMatchScreenshot("empty.png");
         });
 
         test("should list tiles on the panel", async ({ page }) => {
@@ -135,17 +135,9 @@ test.describe("FilePanel", () => {
             await expect(senderDetails.locator(".mx_MessageTimestamp")).toBeVisible();
 
             // Take a snapshot of file tiles list on FilePanel
-            // XXX: We remove the RM as masking it in different locations causes a false positive
-            await page.evaluate(() => {
-                document.querySelectorAll(".mx_MessagePanel_myReadMarker").forEach((e) => e.remove());
-            });
-            await expect(filePanelMessageList).toHaveScreenshot("file-tiles-list.png", {
-                // Exclude timestamps, profiles, avatars & flaky seek bar from snapshot
-                mask: [
-                    page.locator(
-                        ".mx_MessageTimestamp, .mx_DisambiguatedProfile, .mx_BaseAvatar, .mx_AudioPlayer_seek",
-                    ),
-                ],
+            await expect(filePanelMessageList).toMatchScreenshot("file-tiles-list.png", {
+                // Exclude timestamps, profile & flaky seek bar from snapshot
+                mask: [page.locator(".mx_MessageTimestamp, .mx_DisambiguatedProfile, .mx_AudioPlayer_seek")],
             });
         });
 
@@ -210,18 +202,22 @@ test.describe("FilePanel", () => {
             const link = imageBody.locator(".mx_MFileBody_download a");
 
             const newPagePromise = context.waitForEvent("page");
-            // const downloadPromise = page.waitForEvent("download");
+
+            const downloadPromise = new Promise<Download>((resolve) => {
+                page.once("download", resolve);
+            });
 
             // Click the anchor link (not the image itself)
             await link.click();
 
             const newPage = await newPagePromise;
-            // XXX: Clicking the link opens the image in a new tab on some browsers rather than downloading, so handle that case
-            await expect(newPage).toHaveURL(/.+\/_matrix\/media\/\w+\/download\/localhost\/\w+/);
-            // .catch(async () => {
-            //     const download = await downloadPromise;
-            //     expect(download.suggestedFilename()).toBe("riot.png");
-            // });
+            // XXX: Clicking the link opens the image in a new tab on some browsers rather than downloading
+            await expect(newPage)
+                .toHaveURL(/.+\/_matrix\/media\/\w+\/download\/localhost\/\w+/)
+                .catch(async () => {
+                    const download = await downloadPromise;
+                    expect(download.suggestedFilename()).toBe("riot.png");
+                });
         });
     });
 });

playwright/e2e/right-panel/notification-panel.spec.ts

@@ -38,6 +38,6 @@ test.describe("NotificationPanel", () => {
         await expect(page.locator(".mx_NotificationPanel_empty")).toBeVisible();
 
         // Take a snapshot of RightPanel
-        await expect(page.locator(".mx_RightPanel")).toHaveScreenshot("empty.png");
+        await expect(page.locator(".mx_RightPanel")).toMatchScreenshot("empty.png");
     });
 });

playwright/e2e/right-panel/right-panel.spec.ts

@@ -67,9 +67,7 @@ test.describe("RightPanel", () => {
             await expect(page.locator(".mx_RightPanel")).not.toBeVisible();
             await page.getByRole("button", { name: "Room info" }).click();
 
-            await expect(page.locator(".mx_RightPanel")).toHaveScreenshot("with-name-and-address.png", {
-                mask: [page.locator(".mx_BaseAvatar")],
-            });
+            await expect(page.locator(".mx_RightPanel")).toMatchScreenshot("with-name-and-address.png");
         });
 
         test("should handle clicking add widgets", async ({ page, app }) => {

playwright/e2e/settings/appearance-user-settings-tab.spec.ts

@@ -33,9 +33,7 @@ test.describe("Appearance user settings tab", () => {
         // Assert that "Hide advanced" link button is rendered
         await expect(tab.getByRole("button", { name: "Hide advanced" })).toBeVisible();
 
-        await expect(tab).toHaveScreenshot("appearance-tab.png", {
-            mask: [tab.locator(".mx_DisambiguatedProfile_displayName, .mx_BaseAvatar")],
-        });
+        await expect(tab).toMatchScreenshot("appearance-tab.png");
     });
 
     test("should support switching layouts", async ({ page, user, app }) => {
@@ -94,7 +92,7 @@ test.describe("Appearance user settings tab", () => {
             fontSliderSection.locator("output .mx_Slider_selection_label", { hasText: String(MIN_FONT_SIZE) }),
         ).toBeVisible();
 
-        await expect(fontSliderSection).toHaveScreenshot(`font-slider-${MIN_FONT_SIZE}.png`);
+        await expect(fontSliderSection).toMatchScreenshot(`font-slider-${MIN_FONT_SIZE}.png`);
 
         // Click the right position of the slider
         await slider.click({ position: { x: 572, y: 10 } });
@@ -106,7 +104,7 @@ test.describe("Appearance user settings tab", () => {
             fontSliderSection.locator("output .mx_Slider_selection_label", { hasText: String(MAX_FONT_SIZE) }),
         ).toBeVisible();
 
-        await expect(fontSliderSection).toHaveScreenshot(`font-slider-${MAX_FONT_SIZE}.png`);
+        await expect(fontSliderSection).toMatchScreenshot(`font-slider-${MAX_FONT_SIZE}.png`);
     });
 
     test("should disable font size slider when custom font size is used", async ({ page, app, user }) => {

playwright/e2e/settings/general-room-settings-tab.spec.ts

@@ -34,7 +34,7 @@ test.describe("General room settings tab", () => {
         // Assert that "Show less" details element is rendered
         await expect(settings.getByText("Show less")).toBeVisible();
 
-        await expect(settings).toHaveScreenshot();
+        await expect(settings).toMatchScreenshot();
 
         // Click the "Show less" details element
         await settings.getByText("Show less").click();

playwright/e2e/settings/general-user-settings-tab.spec.ts

@@ -35,7 +35,7 @@ test.describe("General user settings tab", () => {
     });
 
     test("should be rendered properly", async ({ uut }) => {
-        await expect(uut).toHaveScreenshot("general.png", {
+        await expect(uut).toMatchScreenshot("general.png", {
             // Exclude userId from snapshots
             mask: [uut.locator(".mx_ProfileSettings_profile_controls > p")],
         });

playwright/e2e/settings/preferences-user-settings-tab.spec.ts

@@ -26,6 +26,6 @@ test.describe("Preferences user settings tab", () => {
 
         // Assert that the top heading is rendered
         await expect(tab.getByRole("heading", { name: "Preferences" })).toBeVisible();
-        await expect(tab).toHaveScreenshot();
+        await expect(tab).toMatchScreenshot();
     });
 });

playwright/e2e/settings/security-user-settings-tab.spec.ts

@@ -44,7 +44,7 @@ test.describe("Security user settings tab", () => {
             test("should be rendered properly", async ({ app, page }) => {
                 const tab = await app.settings.openUserSettings("Security");
                 await tab.getByRole("button", { name: "Learn more" }).click();
-                await expect(page.locator(".mx_AnalyticsLearnMoreDialog_wrapper .mx_Dialog")).toHaveScreenshot();
+                await expect(page.locator(".mx_AnalyticsLearnMoreDialog_wrapper .mx_Dialog")).toMatchScreenshot();
             });
         });
     });

playwright/e2e/user-onboarding/user-onboarding-new.spec.ts

@@ -35,7 +35,7 @@ test.describe("User Onboarding (new user)", () => {
     });
 
     test("page is shown and preference exists", async ({ page, app }) => {
-        await expect(page.locator(".mx_UserOnboardingPage")).toHaveScreenshot();
+        await expect(page.locator(".mx_UserOnboardingPage")).toMatchScreenshot();
         await app.settings.openUserSettings("Preferences");
         await expect(page.getByText("Show shortcut to welcome checklist above the room list")).toBeVisible();
     });
@@ -45,7 +45,7 @@ test.describe("User Onboarding (new user)", () => {
         await expect(
             page.getByRole("dialog").getByRole("heading", { level: 2, name: "Download Element" }),
         ).toBeVisible();
-        await expect(page.locator(".mx_Dialog")).toHaveScreenshot();
+        await expect(page.locator(".mx_Dialog")).toMatchScreenshot();
     });
 
     test("using find friends action should increase progress", async ({ page, homeserver }) => {

playwright/e2e/user-view/user-view.spec.ts

@@ -28,8 +28,8 @@ test.describe("UserView", () => {
         const rightPanel = page.getByRole("complementary");
         await expect(rightPanel.getByRole("heading", { name: bot.credentials.displayName, exact: true })).toBeVisible();
         await expect(rightPanel.getByText("1 session")).toBeVisible();
-        await expect(rightPanel).toHaveScreenshot("user-info.png", {
-            mask: [page.locator(".mx_BaseAvatar, .mx_UserInfo_profile_mxid")],
+        await expect(rightPanel).toMatchScreenshot("user-info.png", {
+            mask: [page.locator(".mx_UserInfo_profile_mxid")],
         });
     });
 });

playwright/element-web-test.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { test as base, expect, Locator } from "@playwright/test";
+import { test as base, expect as baseExpect, Locator, Page, ExpectMatcherState, ElementHandle } from "@playwright/test";
 import AxeBuilder from "@axe-core/playwright";
 import _ from "lodash";
 
@@ -197,6 +197,35 @@ export const test = base.extend<
     },
 });
 
-test.use({});
+export const expect = baseExpect.extend({
+    async toMatchScreenshot(this: ExpectMatcherState, receiver: Page | Locator, ...args) {
+        const page = "page" in receiver ? receiver.page() : receiver;
 
-export { expect };
+        // We add a custom style tag before taking screenshots
+        const style = (await page.addStyleTag({
+            content: `
+                .mx_MessagePanel_myReadMarker {
+                    display: none !important;
+                }
+                .mx_RoomView_MessageList {
+                    height: auto !important;
+                }
+                .mx_DisambiguatedProfile_displayName {
+                    color: var(--cpd-color-blue-1200) !important;
+                }
+                .mx_BaseAvatar {
+                    background-color: var(--cpd-color-fuchsia-1200) !important;
+                    color: white !important;
+                }
+                .mx_ReplyChain {
+                    border-left-color: var(--cpd-color-blue-1200) !important;
+                }
+            `,
+        })) as ElementHandle<Element>;
+
+        await baseExpect(receiver).toHaveScreenshot(...args);
+
+        await style.evaluate((tag) => tag.remove());
+        return { pass: true, message: () => "", name: "toMatchScreenshot" };
+    },
+});

res/css/_common.pcss

@@ -66,14 +66,6 @@ limitations under the License.
     --cpd-font-family-sans: $font-family;
 }
 
-@media only percy {
-    :root {
-        --percy-color-avatar: $username-variant2-color;
-        --percy-color-displayName: $username-variant1-color;
-        --percy-color-replyChain-border: $username-variant1-color;
-    }
-}
-
 @media (prefers-reduced-motion) {
     :root {
         --transition-short: 0;

res/css/views/avatars/_BaseAvatar.pcss

@@ -14,15 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-/* Screenshot test specific CSS */
-@media only percy {
-    /* Stick the default room avatar colour, so it doesn't cause a false diff on the screenshot */
-    .mx_BaseAvatar {
-        background-color: var(--percy-color-avatar) !important;
-        color: white !important;
-    }
-}
-
 button.mx_BaseAvatar {
     /* <button> is a form element and by default it uses the user agent (browser) styling.
        We want it to inherit the font-family and line-height from its parent.

res/css/views/elements/_ReplyChain.pcss

@@ -64,11 +64,3 @@ limitations under the License.
         --username-color: $username-variant8-color;
     }
 }
-
-/* Percy screenshot test specific CSS */
-@media only percy {
-    .mx_ReplyChain {
-        /* Override the colour in percy tests for screenshot consistency */
-        border-left-color: var(--percy-color-replyChain-border) !important;
-    }
-}

res/css/views/messages/_DisambiguatedProfile.pcss

@@ -34,10 +34,3 @@ limitations under the License.
         font-size: var(--cpd-font-size-body-sm);
     }
 }
-
-@media only percy {
-    .mx_DisambiguatedProfile_displayName {
-        /* Override the colour in percy tests for screenshot consistency */
-        color: var(--percy-color-displayName) !important;
-    }
-}