Migrate register.spec.ts from Cypress to Playwright (#11942)

* Install playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add foundations for writing tests under Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* .gitignore juggling

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add tsconfig and fix eslint rules

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add docker & synapse plugins

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add login.spec.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Wire up fixture which sets up ElementAppPage & bakes config.json

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove launch test, it has served its purpose

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove test which has been ported to Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test not cleaning up after itself

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Move registerUser to the Homeserver interface

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove unused fixture param

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove redundant launch test

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add newline

* Run both legacy & rust crypto tests in Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove redundant comment

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Create plugin for mail-hog

* Move injectAxe into element-web-test.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Switch out axe-playwright for @axe-core/playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Migrate email.spec.ts from Cypress to Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* prettier

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Use Playwright snapshot utility over Percy

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Remove commented our Percy badge as we're unlikely to want to go back

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Migrate user-onboarding-old.spec.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Migrate user-onboarding-new.spec.ts

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add screenshots

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix bad merge

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix test and re-enable

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Run linters on playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Make typescript happier

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Fix types

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update typescript

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Migrate register.spec.ts from Cypress to Playwright

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Add screenshots

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update screenshots

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update import

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
Co-authored-by: R Midhun Suresh <hi@midhun.dev>

cypress/e2e/register/register.spec.ts

@@ -1,152 +0,0 @@
-/*
-Copyright 2022 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/// <reference types="cypress" />
-
-import { HomeserverInstance } from "../../plugins/utils/homeserver";
-import { checkDeviceIsCrossSigned } from "../crypto/utils";
-
-describe("Registration", () => {
-    let homeserver: HomeserverInstance;
-
-    beforeEach(() => {
-        cy.visit("/#/register");
-        cy.startHomeserver("consent").then((data) => {
-            homeserver = data;
-        });
-    });
-
-    afterEach(() => {
-        cy.stopHomeserver(homeserver);
-    });
-
-    it("registers an account and lands on the home screen", () => {
-        cy.injectAxe();
-
-        cy.findByRole("button", { name: "Edit", timeout: 15000 }).click();
-        cy.findByRole("button", { name: "Continue" }).should("be.visible");
-        // Only snapshot the server picker otherwise in the background `matrix.org` may or may not be available
-        cy.get(".mx_Dialog").percySnapshotElement("Server Picker", { widths: [516] });
-        cy.checkA11y(undefined, {
-            rules: {
-                // Axe is unhappy with the configuration error's contrast here
-                "link-in-text-block": {
-                    enabled: false,
-                },
-            },
-        });
-
-        cy.findByRole("textbox", { name: "Other homeserver" }).type(homeserver.baseUrl);
-        cy.findByRole("button", { name: "Continue" }).click();
-        // wait for the dialog to go away
-        cy.get(".mx_ServerPickerDialog").should("not.exist");
-
-        cy.findByRole("textbox", { name: "Username" }).should("be.visible");
-        // Hide the server text as it contains the randomly allocated Homeserver port
-        const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }";
-        cy.percySnapshot("Registration", { percyCSS });
-        cy.checkA11y();
-
-        cy.findByRole("textbox", { name: "Username" }).type("alice");
-        cy.findByPlaceholderText("Password").type("totally a great password");
-        cy.findByPlaceholderText("Confirm password").type("totally a great password");
-        cy.findByRole("button", { name: "Register" }).click();
-
-        cy.get(".mx_RegistrationEmailPromptDialog").should("be.visible");
-        cy.percySnapshot("Registration email prompt", { percyCSS });
-        cy.checkA11y();
-        cy.get(".mx_RegistrationEmailPromptDialog").within(() => {
-            cy.findByRole("button", { name: "Continue" }).click();
-        });
-
-        cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy").should("be.visible");
-        cy.percySnapshot("Registration terms prompt", { percyCSS });
-        cy.checkA11y();
-
-        cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy").within(() => {
-            cy.findByRole("checkbox").click(); // Click the checkbox before privacy policy anchor link
-            cy.findByLabelText("Privacy Policy").should("be.visible");
-        });
-
-        cy.findByRole("button", { name: "Accept" }).click();
-
-        cy.get(".mx_UseCaseSelection_skip", { timeout: 30000 }).should("exist");
-        cy.percySnapshot("Use-case selection screen");
-        cy.checkA11y();
-        cy.findByRole("button", { name: "Skip" }).click();
-
-        cy.url().should("contain", "/#/home");
-
-        /*
-         * Cross-signing checks
-         */
-
-        // check that the device considers itself verified
-        cy.findByRole("button", { name: "User menu" }).click();
-        cy.findByRole("menuitem", { name: "All settings" }).click();
-        cy.findByRole("tab", { name: "Sessions" }).click();
-        cy.findByTestId("current-session-section").within(() => {
-            cy.findByTestId("device-metadata-isVerified").should("have.text", "Verified");
-        });
-
-        // check that cross-signing keys have been uploaded.
-        checkDeviceIsCrossSigned();
-    });
-
-    it("should require username to fulfil requirements and be available", () => {
-        cy.findByRole("button", { name: "Edit", timeout: 15000 }).click();
-        cy.findByRole("button", { name: "Continue" }).should("be.visible");
-        cy.findByRole("textbox", { name: "Other homeserver" }).type(homeserver.baseUrl);
-        cy.findByRole("button", { name: "Continue" }).click();
-        // wait for the dialog to go away
-        cy.get(".mx_ServerPickerDialog").should("not.exist");
-
-        cy.findByRole("textbox", { name: "Username" }).should("be.visible");
-
-        cy.intercept("**/_matrix/client/*/register/available?username=_alice", {
-            statusCode: 400,
-            headers: {
-                "Content-Type": "application/json",
-            },
-            body: {
-                errcode: "M_INVALID_USERNAME",
-                error: "User ID may not begin with _",
-            },
-        });
-        cy.findByRole("textbox", { name: "Username" }).type("_alice");
-        cy.get(".mx_Field_tooltip")
-            .should("have.class", "mx_Tooltip_visible")
-            .should("contain.text", "Some characters not allowed");
-
-        cy.intercept("**/_matrix/client/*/register/available?username=bob", {
-            statusCode: 400,
-            headers: {
-                "Content-Type": "application/json",
-            },
-            body: {
-                errcode: "M_USER_IN_USE",
-                error: "The desired username is already taken",
-            },
-        });
-        cy.findByRole("textbox", { name: "Username" }).type("{selectAll}{backspace}bob");
-        cy.get(".mx_Field_tooltip")
-            .should("have.class", "mx_Tooltip_visible")
-            .should("contain.text", "Someone already has that username");
-
-        cy.findByRole("textbox", { name: "Username" }).type("{selectAll}{backspace}foobar");
-        cy.get(".mx_Field_tooltip").should("not.have.class", "mx_Tooltip_visible");
-    });
-});

playwright.config.ts

@@ -39,11 +39,11 @@ export default defineConfig<TestOptions>({
     projects: [
         {
             name: "Legacy Crypto",
-            use: { crypto: "legacy" },
+            use: { cryptoBackend: "legacy" },
         },
         {
             name: "Rust Crypto",
-            use: { crypto: "rust" },
+            use: { cryptoBackend: "rust" },
         },
     ],
     snapshotDir: "playwright/snapshots",

playwright/e2e/register/register.spec.ts

@@ -0,0 +1,124 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect } from "../../element-web-test";
+
+test.describe("Registration", () => {
+    test.use({ startHomeserverOpts: "consent" });
+
+    test.beforeEach(async ({ page }) => {
+        await page.goto("/#/register");
+    });
+
+    test("registers an account and lands on the home screen", async ({ homeserver, page, checkA11y, crypto }) => {
+        await page.getByRole("button", { name: "Edit", exact: true }).click();
+        await expect(page.getByRole("button", { name: "Continue", exact: true })).toBeVisible();
+
+        await expect(page.locator(".mx_Dialog")).toHaveScreenshot("server-picker.png");
+        await checkA11y();
+
+        await page.getByRole("textbox", { name: "Other homeserver" }).fill(homeserver.config.baseUrl);
+        await page.getByRole("button", { name: "Continue", exact: true }).click();
+        // wait for the dialog to go away
+        await expect(page.getByRole("dialog")).not.toBeVisible();
+
+        await expect(page.getByRole("textbox", { name: "Username", exact: true })).toBeVisible();
+        // Hide the server text as it contains the randomly allocated Homeserver port
+        const screenshotOptions = { mask: [page.locator(".mx_ServerPicker_server")] };
+        await expect(page).toHaveScreenshot("registration.png", screenshotOptions);
+        await checkA11y();
+
+        await page.getByRole("textbox", { name: "Username", exact: true }).fill("alice");
+        await page.getByPlaceholder("Password", { exact: true }).fill("totally a great password");
+        await page.getByPlaceholder("Confirm password", { exact: true }).fill("totally a great password");
+        await page.getByRole("button", { name: "Register", exact: true }).click();
+
+        const dialog = page.getByRole("dialog");
+        await expect(dialog).toBeVisible();
+        await expect(page).toHaveScreenshot("email-prompt.png", screenshotOptions);
+        await checkA11y();
+        await dialog.getByRole("button", { name: "Continue", exact: true }).click();
+
+        await expect(page.locator(".mx_InteractiveAuthEntryComponents_termsPolicy")).toBeVisible();
+        await expect(page).toHaveScreenshot("terms-prompt.png", screenshotOptions);
+        await checkA11y();
+
+        const termsPolicy = page.locator(".mx_InteractiveAuthEntryComponents_termsPolicy");
+        await termsPolicy.getByRole("checkbox").click(); // Click the checkbox before terms of service anchor link
+        await expect(termsPolicy.getByLabel("Privacy Policy")).toBeVisible();
+
+        await page.getByRole("button", { name: "Accept", exact: true }).click();
+
+        await expect(page.locator(".mx_UseCaseSelection_skip")).toBeVisible();
+        await expect(page).toHaveScreenshot("use-case-selection.png", screenshotOptions);
+        await checkA11y();
+        await page.getByRole("button", { name: "Skip", exact: true }).click();
+
+        await expect(page).toHaveURL(/\/#\/home$/);
+
+        /*
+         * Cross-signing checks
+         */
+        // check that the device considers itself verified
+        await page.getByRole("button", { name: "User menu", exact: true }).click();
+        await page.getByRole("menuitem", { name: "All settings", exact: true }).click();
+        await page.getByRole("tab", { name: "Sessions", exact: true }).click();
+        await expect(page.getByTestId("current-session-section").getByTestId("device-metadata-isVerified")).toHaveText(
+            "Verified",
+        );
+
+        // check that cross-signing keys have been uploaded.
+        await crypto.assertDeviceIsCrossSigned();
+    });
+
+    test("should require username to fulfil requirements and be available", async ({ homeserver, page }) => {
+        await page.getByRole("button", { name: "Edit", exact: true }).click();
+        await expect(page.getByRole("button", { name: "Continue", exact: true })).toBeVisible();
+        await page.getByRole("textbox", { name: "Other homeserver" }).fill(homeserver.config.baseUrl);
+        await page.getByRole("button", { name: "Continue", exact: true }).click();
+        // wait for the dialog to go away
+        await expect(page.getByRole("dialog")).not.toBeVisible();
+
+        await expect(page.getByRole("textbox", { name: "Username", exact: true })).toBeVisible();
+
+        await page.route("**/_matrix/client/*/register/available?username=_alice", async (route) => {
+            await route.fulfill({
+                status: 400,
+                json: {
+                    errcode: "M_INVALID_USERNAME",
+                    error: "User ID may not begin with _",
+                },
+            });
+        });
+        await page.getByRole("textbox", { name: "Username", exact: true }).fill("_alice");
+        await expect(page.getByRole("alert").filter({ hasText: "Some characters not allowed" })).toBeVisible();
+
+        await page.route("**/_matrix/client/*/register/available?username=bob", async (route) => {
+            await route.fulfill({
+                status: 400,
+                json: {
+                    errcode: "M_USER_IN_USE",
+                    error: "The desired username is already taken",
+                },
+            });
+        });
+        await page.getByRole("textbox", { name: "Username", exact: true }).fill("bob");
+        await expect(page.getByRole("alert").filter({ hasText: "Someone already has that username" })).toBeVisible();
+
+        await page.getByRole("textbox", { name: "Username", exact: true }).fill("foobar");
+        await expect(page.getByRole("alert")).not.toBeVisible();
+    });
+});

playwright/element-web-test.ts

@@ -26,6 +26,7 @@ import { Dendrite, Pinecone } from "./plugins/homeserver/dendrite";
 import { Instance } from "./plugins/mailhog";
 import { ElementAppPage } from "./pages/ElementAppPage";
 import { OAuthServer } from "./plugins/oauth_server";
+import { Crypto } from "./pages/crypto";
 import { Toasts } from "./pages/toasts";
 
 const CONFIG_JSON: Partial<IConfigOptions> = {
@@ -45,7 +46,7 @@ const CONFIG_JSON: Partial<IConfigOptions> = {
 };
 
 export type TestOptions = {
-    crypto: "legacy" | "rust";
+    cryptoBackend: "legacy" | "rust";
 };
 
 export const test = base.extend<
@@ -62,15 +63,16 @@ export const test = base.extend<
         displayName?: string;
         app: ElementAppPage;
         mailhog?: { api: mailhog.API; instance: Instance };
+        crypto: Crypto;
         toasts: Toasts;
     }
 >({
-    crypto: ["legacy", { option: true }],
+    cryptoBackend: ["legacy", { option: true }],
     config: CONFIG_JSON,
-    page: async ({ context, page, config, crypto }, use) => {
+    page: async ({ context, page, config, cryptoBackend }, use) => {
         await context.route(`http://localhost:8080/config.json*`, async (route) => {
             const json = { ...CONFIG_JSON, ...config };
-            if (crypto === "rust") {
+            if (cryptoBackend === "rust") {
                 json["features"] = {
                     ...json["features"],
                     feature_rust_crypto: true,
@@ -163,6 +165,9 @@ export const test = base.extend<
     app: async ({ page }, use) => {
         await use(new ElementAppPage(page));
     },
+    crypto: async ({ page, homeserver, request }, use) => {
+        await use(new Crypto(page, homeserver, request));
+    },
     toasts: async ({ page }, use) => {
         await use(new Toasts(page));
     },

playwright/pages/crypto.ts

@@ -0,0 +1,57 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { APIRequestContext, Page, expect } from "@playwright/test";
+
+import { HomeserverInstance } from "../plugins/homeserver";
+
+export class Crypto {
+    public constructor(
+        private page: Page,
+        private homeserver: HomeserverInstance,
+        private request: APIRequestContext,
+    ) {}
+
+    /**
+     * Check that the user has published cross-signing keys, and that the user's device has been cross-signed.
+     */
+    public async assertDeviceIsCrossSigned(): Promise<void> {
+        const { userId, deviceId, accessToken } = await this.page.evaluate(() => ({
+            userId: window.mxMatrixClientPeg.get().getUserId(),
+            deviceId: window.mxMatrixClientPeg.get().getDeviceId(),
+            accessToken: window.mxMatrixClientPeg.get().getAccessToken(),
+        }));
+
+        const res = await this.request.post(`${this.homeserver.config.baseUrl}/_matrix/client/v3/keys/query`, {
+            headers: { Authorization: `Bearer ${accessToken}` },
+            data: { device_keys: { [userId]: [] } },
+        });
+        const json = await res.json();
+
+        // there should be three cross-signing keys
+        expect(json.master_keys[userId]).toHaveProperty("keys");
+        expect(json.self_signing_keys[userId]).toHaveProperty("keys");
+        expect(json.user_signing_keys[userId]).toHaveProperty("keys");
+
+        // and the device should be signed by the self-signing key
+        const selfSigningKeyId = Object.keys(json.self_signing_keys[userId].keys)[0];
+
+        expect(json.device_keys[userId][deviceId]).toBeDefined();
+
+        const myDeviceSignatures = json.device_keys[userId][deviceId].signatures[userId];
+        expect(myDeviceSignatures[selfSigningKeyId]).toBeDefined();
+    }
+}