Use Playwright snapshot utility instead of Percy (#11922)

* Install playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add foundations for writing tests under Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* .gitignore juggling

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add tsconfig and fix eslint rules

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add docker & synapse plugins

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add login.spec.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Wire up fixture which sets up ElementAppPage & bakes config.json

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove launch test, it has served its purpose

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove test which has been ported to Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test not cleaning up after itself

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Move registerUser to the Homeserver interface

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove unused fixture param

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove redundant launch test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add newline

* Run both legacy & rust crypto tests in Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove redundant comment

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Create plugin for mail-hog

* Move injectAxe into element-web-test.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Switch out axe-playwright for @axe-core/playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Migrate email.spec.ts from Cypress to Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* prettier

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use Playwright snapshot utility over Percy

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove commented our Percy badge as we're unlikely to want to go back

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
Co-authored-by: R Midhun Suresh <hi@midhun.dev>

README.md

@@ -3,14 +3,6 @@
 ![Static Analysis](https://github.com/matrix-org/matrix-react-sdk/actions/workflows/static_analysis.yaml/badge.svg)
 [![Knapsack Pro Parallel CI builds for Cypress Test - Legacy Crypto](https://img.shields.io/badge/Knapsack%20Pro-Parallel%20%2F%20Cypress%20Test%20--%20Legacy%20Crypto-%230074ff)](https://knapsackpro.com/dashboard/organizations/3882/projects/2469/test_suites/3724/builds?utm_campaign=organization-id-3882&utm_content=test-suite-id-3724&utm_medium=readme&utm_source=knapsack-pro-badge&utm_term=project-id-2469)
 [![Knapsack Pro Parallel CI builds for Cypress Test - Rust Crypto](https://img.shields.io/badge/Knapsack%20Pro-Parallel%20%2F%20Cypress%20Test%20--%20Rust%20Crypto-%230074ff)](https://knapsackpro.com/dashboard/organizations/3882/projects/2469/test_suites/3729/builds?utm_campaign=organization-id-3882&utm_content=test-suite-id-3729&utm_medium=readme&utm_source=knapsack-pro-badge&utm_term=project-id-2469)
-
-<!--
-Percy is disabled while we're figuring out https://github.com/vector-im/wat-internal/issues/36
-and https://github.com/vector-im/wat-internal/issues/56. We're hoping to turn it back on or switch
-to an alternative in the future.
-[![This project is using Percy.io for visual regression testing.](https://percy.io/static/images/percy-badge.svg)](https://percy.io/dfde73bd/matrix-react-sdk)
--->
-
 [![Localazy](https://img.shields.io/endpoint?url=https%3A%2F%2Fconnect.localazy.com%2Fstatus%2Felement-web%2Fdata%3Fcontent%3Dall%26title%3Dlocalazy%26logo%3Dtrue)](https://localazy.com/p/element-web)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=matrix-react-sdk&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=matrix-react-sdk)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=matrix-react-sdk&metric=coverage)](https://sonarcloud.io/summary/new_code?id=matrix-react-sdk)

playwright.config.ts

@@ -22,7 +22,6 @@ const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080";
 
 export default defineConfig<TestOptions>({
     use: {
-        headless: false,
         viewport: { width: 1280, height: 720 },
         ignoreHTTPSErrors: true,
         video: "on-first-retry",

playwright/.gitignore

@@ -1,3 +1,6 @@
 /test-results/
 /html-report/
 /synapselogs/
+# Only commit snapshots from Linux
+/e2e/*/*-snapshots/*.png
+!/e2e/*/*-snapshots/*-linux.png

playwright/README.md

@@ -0,0 +1,15 @@
+To update snapshots you will need to run Playwright on a Linux machine.
+If you have access to docker then you can use the following:
+
+```shell
+docker run \
+    --rm \
+    --network host \
+    -v $(pwd)/../:/work/ \
+    -v playwright-ew-node-modules:/work/element-web/node_modules \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /tmp/:/tmp/ \
+    -w /work/matrix-react-sdk \
+    -it mcr.microsoft.com/playwright:v1.40.0-jammy \
+    sh -c "apt-get update && apt-get -y install docker.io && yarn --cwd ../element-web install && npx playwright test --update-snapshots --reporter line"
+```

playwright/e2e/register/email.spec.ts

@@ -59,7 +59,7 @@ test.describe("Email Registration", async () => {
     }) => {
         await expect(page.getByRole("textbox", { name: "Username" })).toBeVisible();
         // Hide the server text as it contains the randomly allocated Homeserver port
-        // const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }"; // XXX: Percy
+        const screenshotOptions = { mask: [page.locator(".mx_ServerPicker_server")] };
 
         await page.getByRole("textbox", { name: "Username" }).fill("alice");
         await page.getByPlaceholder("Password", { exact: true }).fill("totally a great password");
@@ -68,7 +68,7 @@ test.describe("Email Registration", async () => {
         await page.getByRole("button", { name: "Register" }).click();
 
         await expect(page.getByText("Check your email to continue")).toBeVisible();
-        // cy.percySnapshot("Registration check your email", { percyCSS }); // XXX: Percy
+        await expect(page).toHaveScreenshot("registration_check_your_email.png", screenshotOptions);
         await checkA11y();
 
         await expect(page.getByText("An error was encountered when sending the email")).not.toBeVisible();

playwright/plugins/synapse/index.ts

@@ -114,7 +114,7 @@ export class Synapse implements Homeserver, HomeserverInstance {
      * one of the templates in the playwright/plugins/synapsedocker/templates
      * directory.
      *
-     * Any value in opts.variables that is set to `{{HOST_DOCKER_INTERNAL}}'
+     * Any value in `opts.variables` that is set to `{{HOST_DOCKER_INTERNAL}}'
      * will be replaced with 'host.docker.internal' (if we are on Docker) or
      * 'host.containers.internal' if we are on Podman.
      */