Migrate email.spec.ts from Cypress to Playwright (#11920)

Co-authored-by: R Midhun Suresh <hi@midhun.dev>

cypress/e2e/register/email.spec.ts

@@ -1,93 +0,0 @@
-/*
-Copyright 2023 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/// <reference types="cypress" />
-
-import { HomeserverInstance } from "../../plugins/utils/homeserver";
-import { Mailhog } from "../../support/mailhog";
-
-describe("Email Registration", () => {
-    let homeserver: HomeserverInstance;
-    let mailhog: Mailhog;
-
-    beforeEach(() => {
-        cy.startMailhog().then((_mailhog) => {
-            mailhog = _mailhog;
-            cy.startHomeserver({
-                template: "email",
-                variables: {
-                    SMTP_HOST: "{{HOST_DOCKER_INTERNAL}}", // This will get replaced in synapseStart
-                    SMTP_PORT: _mailhog.instance.smtpPort,
-                },
-            }).then((_homeserver) => {
-                homeserver = _homeserver;
-
-                cy.intercept(
-                    { method: "GET", pathname: "/config.json" },
-                    {
-                        body: {
-                            default_server_config: {
-                                "m.homeserver": {
-                                    base_url: homeserver.baseUrl,
-                                },
-                                "m.identity_server": {
-                                    base_url: "https://server.invalid",
-                                },
-                            },
-                        },
-                    },
-                );
-                cy.visit("/#/register");
-                cy.injectAxe();
-            });
-        });
-    });
-
-    afterEach(() => {
-        cy.stopHomeserver(homeserver);
-        cy.stopMailhog(mailhog);
-    });
-
-    it("registers an account and lands on the use case selection screen", () => {
-        cy.findByRole("textbox", { name: "Username" }).should("be.visible");
-        // Hide the server text as it contains the randomly allocated Homeserver port
-        const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }";
-
-        cy.findByRole("textbox", { name: "Username" }).type("alice");
-        cy.findByPlaceholderText("Password").type("totally a great password");
-        cy.findByPlaceholderText("Confirm password").type("totally a great password");
-        cy.findByPlaceholderText("Email").type("alice@email.com");
-        cy.findByRole("button", { name: "Register" }).click();
-
-        cy.findByText("Check your email to continue").should("be.visible");
-        cy.percySnapshot("Registration check your email", { percyCSS });
-        cy.checkA11y();
-
-        cy.findByText("An error was encountered when sending the email").should("not.exist");
-
-        cy.waitForPromise(async () => {
-            const messages = await mailhog.api.messages();
-            expect(messages.items).to.have.length(1);
-            expect(messages.items[0].to).to.eq("alice@email.com");
-            const [link] = messages.items[0].text.match(/http.+/);
-            return link;
-        }).as("emailLink");
-
-        cy.get<string>("@emailLink").then((link) => cy.request(link));
-
-        cy.get(".mx_UseCaseSelection_skip", { timeout: 30000 }).should("exist");
-    });
-});

cypress/plugins/index.ts

@@ -27,7 +27,6 @@ import { webserver } from "./webserver";
 import { docker } from "./docker";
 import { log } from "./log";
 import { oAuthServer } from "./oauth_server";
-import { mailhogDocker } from "./mailhog";
 
 /**
  * @type {Cypress.PluginConfig}
@@ -35,7 +34,7 @@ import { mailhogDocker } from "./mailhog";
 export default function (on: PluginEvents, config: PluginConfigOptions) {
     initPlugins(
         on,
-        [docker, synapseDocker, dendriteDocker, slidingSyncProxyDocker, webserver, oAuthServer, log, mailhogDocker],
+        [docker, synapseDocker, dendriteDocker, slidingSyncProxyDocker, webserver, oAuthServer, log],
         config,
     );
     installLogsPrinter(on, {

cypress/support/e2e.ts

@@ -40,7 +40,6 @@ import "./network";
 import "./composer";
 import "./proxy";
 import "./axe";
-import "./mailhog";
 import "./promise";
 
 installLogsCollector({

package.json

@@ -50,13 +50,13 @@
         "lint:js-fix": "eslint --fix src test cypress && prettier --loglevel=warn --write .",
         "lint:types": "tsc --noEmit --jsx react && tsc --noEmit --jsx react -p cypress",
         "lint:style": "stylelint \"res/css/**/*.pcss\"",
-        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
         "test": "jest",
         "test:cypress": "cypress run",
         "test:cypress:open": "cypress open",
         "test:playwright": "playwright test",
         "test:playwright:open": "yarn test:playwright --ui",
-        "coverage": "yarn test --coverage"
+        "coverage": "yarn test --coverage",
+        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'"
     },
     "resolutions": {
         "@types/react-dom": "17.0.21",
@@ -130,8 +130,6 @@
         "what-input": "^5.2.10"
     },
     "devDependencies": {
-        "@action-validator/cli": "^0.5.3",
-        "@action-validator/core": "^0.5.3",
         "@babel/cli": "^7.12.10",
         "@babel/core": "^7.12.10",
         "@babel/eslint-parser": "^7.12.10",
@@ -187,7 +185,6 @@
         "@typescript-eslint/parser": "^5.6.0",
         "allchange": "^1.1.0",
         "axe-core": "4.8.2",
-        "axe-playwright": "^1.2.3",
         "babel-jest": "^29.0.0",
         "blob-polyfill": "^7.0.0",
         "cypress": "^12.0.0",
@@ -228,7 +225,10 @@
         "stylelint-config-standard": "^34.0.0",
         "stylelint-scss": "^5.0.0",
         "ts-node": "^10.9.1",
-        "typescript": "5.1.6"
+        "typescript": "5.1.6",
+        "@axe-core/playwright": "^4.8.1",
+        "@action-validator/core": "^0.5.3",
+        "@action-validator/cli": "^0.5.3"
     },
     "peerDependencies": {
         "postcss": "^8.4.19",

playwright/e2e/login/login.spec.ts

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { checkA11y, injectAxe } from "axe-playwright";
-
 import { test, expect } from "../../element-web-test";
 
 test.describe("Consent", () => {
@@ -30,9 +28,11 @@ test.describe("Consent", () => {
             await page.goto("/#/login");
         });
 
-        test("logs in with an existing account and lands on the home screen", async ({ page, homeserver }) => {
+        test("logs in with an existing account and lands on the home screen", async ({
-            await injectAxe(page);
+            page,
-
+            homeserver,
+            checkA11y,
+        }) => {
             // first pick the homeserver, as otherwise the user picker won't be visible
             await page.getByRole("button", { name: "Edit" }).click();
             await page.getByRole("textbox", { name: "Other homeserver" }).fill(homeserver.config.baseUrl);
@@ -66,7 +66,7 @@ test.describe("Consent", () => {
             await expect(page.getByRole("textbox", { name: "Username" })).toBeVisible();
             // Disabled because flaky - see https://github.com/vector-im/element-web/issues/24688
             // cy.percySnapshot("Login");
-            await checkA11y(page);
+            await checkA11y();
 
             await page.getByRole("textbox", { name: "Username" }).fill(username);
             await page.getByPlaceholder("Password").fill(password);

playwright/e2e/register/email.spec.ts

@@ -0,0 +1,84 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect } from "../../element-web-test";
+import { MailHogServer } from "../../plugins/mailhog";
+
+test.describe("Email Registration", async () => {
+    test.use({
+        // eslint-disable-next-line no-empty-pattern
+        mailhog: async ({}, use) => {
+            const mailhog = new MailHogServer();
+            const instance = await mailhog.start();
+            await use(instance);
+            await mailhog.stop();
+        },
+        startHomeserverOpts: ({ mailhog }, use) =>
+            use({
+                template: "email",
+                variables: {
+                    SMTP_HOST: "{{HOST_DOCKER_INTERNAL}}", // This will get replaced in synapseStart
+                    SMTP_PORT: mailhog.instance.smtpPort,
+                },
+            }),
+        config: ({ homeserver }, use) =>
+            use({
+                default_server_config: {
+                    "m.homeserver": {
+                        base_url: homeserver.config.baseUrl,
+                    },
+                    "m.identity_server": {
+                        base_url: "https://server.invalid",
+                    },
+                },
+            }),
+    });
+
+    test.beforeEach(async ({ page }) => {
+        await page.goto("/#/register");
+    });
+
+    test("registers an account and lands on the use case selection screen", async ({
+        page,
+        mailhog,
+        request,
+        checkA11y,
+    }) => {
+        await expect(page.getByRole("textbox", { name: "Username" })).toBeVisible();
+        // Hide the server text as it contains the randomly allocated Homeserver port
+        // const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }"; // XXX: Percy
+
+        await page.getByRole("textbox", { name: "Username" }).fill("alice");
+        await page.getByPlaceholder("Password", { exact: true }).fill("totally a great password");
+        await page.getByPlaceholder("Confirm password").fill("totally a great password");
+        await page.getByPlaceholder("Email").fill("alice@email.com");
+        await page.getByRole("button", { name: "Register" }).click();
+
+        await expect(page.getByText("Check your email to continue")).toBeVisible();
+        // cy.percySnapshot("Registration check your email", { percyCSS }); // XXX: Percy
+        await checkA11y();
+
+        await expect(page.getByText("An error was encountered when sending the email")).not.toBeVisible();
+
+        const messages = await mailhog.api.messages();
+        expect(messages.items).toHaveLength(1);
+        expect(messages.items[0].to).toEqual("alice@email.com");
+        const [emailLink] = messages.items[0].text.match(/http.+/);
+        await request.get(emailLink); // "Click" the link in the email
+
+        await expect(page.locator(".mx_UseCaseSelection_skip")).toBeVisible();
+    });
+});

playwright/element-web-test.ts

@@ -14,12 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { test as base } from "@playwright/test";
+import { test as base, expect } from "@playwright/test";
+import AxeBuilder from "@axe-core/playwright";
 
+import type mailhog from "mailhog";
+import type { IConfigOptions } from "../src/IConfigOptions";
 import { HomeserverInstance, StartHomeserverOpts } from "./plugins/utils/homeserver";
 import { Synapse } from "./plugins/synapse";
+import { Instance } from "./plugins/mailhog";
 
-const CONFIG_JSON = {
+const CONFIG_JSON: Partial<IConfigOptions> = {
     // This is deliberately quite a minimal config.json, so that we can test that the default settings
     // actually work.
     //
@@ -41,9 +45,12 @@ export type TestOptions = {
 
 export const test = base.extend<
     TestOptions & {
+        axe: AxeBuilder;
+        checkA11y: () => Promise<void>;
         config: typeof CONFIG_JSON;
         startHomeserverOpts: StartHomeserverOpts | string;
         homeserver: HomeserverInstance;
+        mailhog?: { api: mailhog.API; instance: Instance };
     }
 >({
     crypto: ["legacy", { option: true }],
@@ -72,6 +79,21 @@ export const test = base.extend<
         await use(await server.start(opts));
         await server.stop();
     },
+
+    axe: async ({ page }, use) => {
+        await use(new AxeBuilder({ page }));
+    },
+    checkA11y: async ({ axe }, use, testInfo) =>
+        use(async () => {
+            const results = await axe.analyze();
+
+            await testInfo.attach("accessibility-scan-results", {
+                body: JSON.stringify(results, null, 2),
+                contentType: "application/json",
+            });
+
+            expect(results.violations).toEqual([]);
+        }),
 });
 
 test.use({});

playwright/plugins/mailhog/index.ts

@@ -0,0 +1,55 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import mailhog from "mailhog";
+
+import { getFreePort } from "../utils/port";
+import { Docker } from "../docker";
+
+export interface Instance {
+    host: string;
+    smtpPort: number;
+    httpPort: number;
+    containerId: string;
+}
+
+export class MailHogServer {
+    private readonly docker: Docker = new Docker();
+    private instance?: Instance;
+
+    async start(): Promise<{ api: mailhog.API; instance: Instance }> {
+        if (this.instance) throw new Error("Mailhog server is already running!");
+        const smtpPort = await getFreePort();
+        const httpPort = await getFreePort();
+        console.log(`Starting mailhog...`);
+        const containerId = await this.docker.run({
+            image: "mailhog/mailhog:latest",
+            containerName: `react-sdk-cypress-mailhog`,
+            params: ["--rm", "-p", `${smtpPort}:1025/tcp`, "-p", `${httpPort}:8025/tcp`],
+        });
+        console.log(`Started mailhog on ports smtp=${smtpPort} http=${httpPort}.`);
+        const host = await this.docker.getContainerIp();
+        this.instance = { smtpPort, httpPort, containerId, host };
+        return { api: mailhog({ host: "localhost", port: httpPort }), instance: this.instance };
+    }
+
+    async stop(): Promise<void> {
+        if (!this.instance) throw new Error("Missing existing mailhog instance, did you call stop() before start()?");
+        await this.docker.stop();
+        console.log(`Stopped mailhog id ${this.instance.containerId}.`);
+        this.instance = undefined;
+    }
+}

playwright/tsconfig.json

@@ -3,7 +3,6 @@
         "target": "es2016",
         "jsx": "react",
         "lib": ["es2021", "dom", "dom.iterable"],
-        "types": ["axe-playwright"],
         "resolveJsonModule": true,
         "esModuleInterop": true,
         "moduleResolution": "node",

yarn.lock

@@ -57,6 +57,13 @@
     "@jridgewell/gen-mapping" "^0.3.0"
     "@jridgewell/trace-mapping" "^0.3.9"
 
+"@axe-core/playwright@^4.8.1":
+  version "4.8.1"
+  resolved "https://registry.yarnpkg.com/@axe-core/playwright/-/playwright-4.8.1.tgz#2785f73eb9f0ba1d003387f85730f235e0b424ac"
+  integrity sha512-KC1X++UdRAwMLRvB+BIKFheyLHUnbJTL0t0Wbv6TJMozn2V2QyEtAcN6jyUiudtGiLUGhHCtj/eWorBnVZ4dAA==
+  dependencies:
+    axe-core "~4.8.2"
+
 "@babel/cli@^7.12.10":
   version "7.23.0"
   resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.23.0.tgz#1d7f37c44d4117c67df46749e0c86e11a58cc64b"
@@ -3446,7 +3453,7 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.12.0.tgz#ce1c9d143389679e253b314241ea9aa5cec980d3"
   integrity sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==
 
-axe-core@4.8.2, axe-core@^4.5.1:
+axe-core@4.8.2, axe-core@~4.8.2:
   version "4.8.2"
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.8.2.tgz#2f6f3cde40935825cf4465e3c1c9e77b240ff6ae"
   integrity sha512-/dlp0fxyM3R8YW7MFzaHWXrf4zzbr0vaYb23VBFCl83R7nWNPg/yaQw2Dc8jzCMmDVLhSdzH8MjrsuIUuvX+6g==
@@ -3456,23 +3463,6 @@ axe-core@=4.7.0:
   resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.7.0.tgz#34ba5a48a8b564f67e103f0aa5768d76e15bbbbf"
   integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==
 
-axe-html-reporter@2.2.3:
-  version "2.2.3"
-  resolved "https://registry.yarnpkg.com/axe-html-reporter/-/axe-html-reporter-2.2.3.tgz#2d56e239fe9bd1f09ba0735d94596bf79dd389a7"
-  integrity sha512-io8aCEt4fJvv43W+33n3zEa8rdplH5Ti2v5fOnth3GBKLhLHarNs7jj46xGfpnGnpaNrz23/tXPHC3HbwTzwwA==
-  dependencies:
-    mustache "^4.0.1"
-    rimraf "^3.0.2"
-
-axe-playwright@^1.2.3:
-  version "1.2.3"
-  resolved "https://registry.yarnpkg.com/axe-playwright/-/axe-playwright-1.2.3.tgz#b590b4edf3898ed5784c4932cbad2937115b31f2"
-  integrity sha512-bTxCTNp3kx6sQRMjmuLv8pG3+v+kOCvFXATim1+XUXzW6ykulbbuJdQfgB+vQPNAF9uvYbW2qrv9pg81ZSFV/A==
-  dependencies:
-    axe-core "^4.5.1"
-    axe-html-reporter "2.2.3"
-    picocolors "^1.0.0"
-
 axios-retry@^3.7.0:
   version "3.9.1"
   resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.9.1.tgz#c8924a8781c8e0a2c5244abf773deb7566b3830d"
@@ -7782,11 +7772,6 @@ murmurhash-js@^1.0.0:
   resolved "https://registry.yarnpkg.com/murmurhash-js/-/murmurhash-js-1.0.0.tgz#b06278e21fc6c37fa5313732b0412bcb6ae15f51"
   integrity sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==
 
-mustache@^4.0.1:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64"
-  integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==
-
 nanoid@^3.3.6:
   version "3.3.6"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c"