From 0f1f056503db4a1e09be9a0885d1aeab0b119827 Mon Sep 17 00:00:00 2001 From: Michael Telatynski <7t3chguy@gmail.com> Date: Tue, 21 Nov 2023 17:33:32 +0000 Subject: [PATCH] Add Playwright end to end testing (#11912) * Install playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add foundations for writing tests under Playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * .gitignore juggling Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add tsconfig and fix eslint rules Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --- .eslintrc.js | 16 ++- .github/workflows/end-to-end-tests.yaml | 142 ++++++++++++++++++++++++ package.json | 3 + playwright.config.ts | 39 +++++++ playwright/.gitignore | 2 + playwright/e2e/launch.spec.ts | 29 +++++ playwright/tsconfig.json | 13 +++ yarn.lock | 26 +++++ 8 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/end-to-end-tests.yaml create mode 100644 playwright.config.ts create mode 100644 playwright/.gitignore create mode 100644 playwright/e2e/launch.spec.ts create mode 100644 playwright/tsconfig.json diff --git a/.eslintrc.js b/.eslintrc.js index 4434aecfdf..f0720a0252 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -169,7 +169,7 @@ module.exports = { }, overrides: [ { - files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts"], + files: ["src/**/*.{ts,tsx}", "test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"], extends: ["plugin:matrix-org/typescript", "plugin:matrix-org/react"], rules: { "@typescript-eslint/explicit-function-return-type": [ @@ -233,7 +233,7 @@ module.exports = { }, }, { - files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts"], + files: ["test/**/*.{ts,tsx}", "cypress/**/*.ts", "playwright/**/*.ts"], extends: ["plugin:matrix-org/jest"], rules: { // We don't need super strict typing in test utilities @@ -267,6 +267,18 @@ module.exports = { "jest/no-done-callback": "off", }, }, + { + files: ["playwright/**/*.ts"], + parserOptions: { + project: ["./playwright/tsconfig.json"], + }, + rules: { + // Cypress "promises" work differently - disable some related rules + "jest/valid-expect": "off", + "jest/valid-expect-in-promise": "off", + "jest/no-done-callback": "off", + }, + }, ], settings: { react: { diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml new file mode 100644 index 0000000000..8a0801e715 --- /dev/null +++ b/.github/workflows/end-to-end-tests.yaml @@ -0,0 +1,142 @@ +# Triggers after the layered build has finished, taking the artifact and running Playwright on it +name: End to End Tests +on: + workflow_run: + workflows: ["Element Web - Build"] + types: + - completed + +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.run_id }} + cancel-in-progress: ${{ github.event.workflow_run.event == 'pull_request' }} + +jobs: + prepare: + name: Prepare + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + permissions: + statuses: write + steps: + # We create the status here and then update it to success/failure in the `report` stage + # This provides an easy link to this workflow_run from the PR before the tests are done. + - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + state: pending + context: ${{ github.workflow }} / end-to-end-tests + sha: ${{ github.event.workflow_run.head_sha }} + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + + tests: + name: "Run Tests" + needs: prepare + runs-on: ubuntu-latest + permissions: + actions: read + issues: read + pull-requests: read + environment: EndToEndTests + strategy: + fail-fast: false + matrix: + # Run 2 instances in Parallel + ci_node_total: [2] + ci_node_index: [0, 1] + steps: + - uses: browser-actions/setup-chrome@803ef6dfb4fdf22089c9563225d95e4a515820a0 # v1 + - run: echo "BROWSER_PATH=$(which chrome)" >> $GITHUB_ENV + + # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action + # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess: + - name: 📥 Download artifact + uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2 + with: + run_id: ${{ github.event.workflow_run.id }} + name: previewbuild + path: webapp + + # The workflow_run.head_sha is the sha of the head commit but the element-web was built using a simulated + # merge commit - https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request + # so use the sha from the tarball for the checkout of the tests + # to make sure we get a matching set of code and tests. + - name: Grab sha from webapp + id: sha + run: | + echo "sha=$(cat webapp/sha)" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v4 + with: + # XXX: We're checking out untrusted code in a secure context + # We need to be careful to not trust anything this code outputs/may do + # + # Note that (in the absence of a `react-sdk-repository` input), + # we check out from the default repository, which is (for this workflow) the + # *target* repository for the pull request. + # + ref: ${{ steps.sha.outputs.sha }} + persist-credentials: false + path: matrix-react-sdk + repository: ${{ inputs.react-sdk-repository || github.repository }} + + - uses: actions/setup-node@v3 + with: + cache: "yarn" + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Install Playwright browsers + run: yarn playwright install --with-deps + + - name: Run Playwright tests + run: yarn playwright test --shard ${{ matrix.ci_node_index }}/${{ matrix.ci_node_total }} + + - name: Upload blob report to GitHub Actions Artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: all-blob-reports + path: blob-report + retention-days: 1 + + report: + name: Report results + needs: tests + runs-on: ubuntu-latest + if: always() + permissions: + statuses: write + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + cache: "yarn" + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v3 + with: + name: all-blob-reports + path: all-blob-reports + + - name: Merge into HTML Report + run: yarn playwright merge-reports --reporter=html,github ./all-blob-reports + + - name: Upload HTML report + uses: actions/upload-artifact@v3 + with: + name: html-report--attempt-${{ github.run_attempt }} + path: playwright-report + retention-days: 14 + + - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1 + with: + authToken: ${{ secrets.GITHUB_TOKEN }} + state: ${{ needs.tests.result == 'success' && 'success' || 'failure' }} + context: ${{ github.workflow }} / end-to-end-tests + sha: ${{ github.event.workflow_run.head_sha }} + target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/package.json b/package.json index d6e1fbc91e..01abd04fa9 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,8 @@ "test": "jest", "test:cypress": "cypress run", "test:cypress:open": "cypress open", + "test:playwright": "playwright test", + "test:playwright:open": "yarn test:playwright --ui", "coverage": "yarn test --coverage" }, "resolutions": { @@ -146,6 +148,7 @@ "@peculiar/webcrypto": "^1.4.3", "@percy/cli": "^1.11.0", "@percy/cypress": "^3.1.2", + "@playwright/test": "^1.40.0", "@testing-library/cypress": "^9.0.0", "@testing-library/dom": "^9.0.0", "@testing-library/jest-dom": "^6.0.0", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000000..5844443427 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,39 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import type { PlaywrightTestConfig } from "@playwright/test"; + +const baseURL = process.env["BASE_URL"] ?? "http://localhost:8080"; + +const config: PlaywrightTestConfig = { + use: { + headless: false, + viewport: { width: 1280, height: 720 }, + ignoreHTTPSErrors: true, + video: "on-first-retry", + baseURL, + }, + webServer: { + command: process.env.CI ? "npx serve -p 8080 -L ../webapp" : "yarn --cwd ../element-web start", + url: `${baseURL}/config.json`, + reuseExistingServer: true, + }, + testDir: "playwright/e2e", + outputDir: "playwright/test-results", + workers: 1, + reporter: process.env.CI ? "blob" : [["html", { outputFolder: "playwright/html-report" }]], +}; +export default config; diff --git a/playwright/.gitignore b/playwright/.gitignore new file mode 100644 index 0000000000..ab8ff9582a --- /dev/null +++ b/playwright/.gitignore @@ -0,0 +1,2 @@ +/test-results/ +/html-report/ diff --git a/playwright/e2e/launch.spec.ts b/playwright/e2e/launch.spec.ts new file mode 100644 index 0000000000..b14fac143d --- /dev/null +++ b/playwright/e2e/launch.spec.ts @@ -0,0 +1,29 @@ +/* +Copyright 2023 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { test, expect } from "@playwright/test"; + +test.describe("App launch", () => { + test.beforeEach(async ({ page }) => { + await page.goto("/"); + }); + + test("should launch and render the welcome view successfully", async ({ page }) => { + await page.locator("#matrixchat").waitFor(); + await page.locator(".mx_Welcome").waitFor(); + await expect(page).toHaveURL("http://localhost:8080/#/welcome"); + }); +}); diff --git a/playwright/tsconfig.json b/playwright/tsconfig.json new file mode 100644 index 0000000000..6eb5bac738 --- /dev/null +++ b/playwright/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "es2016", + "jsx": "react", + "lib": ["es2021", "dom", "dom.iterable"], + "types": ["axe-playwright"], + "resolveJsonModule": true, + "esModuleInterop": true, + "moduleResolution": "node", + "module": "es2022" + }, + "include": ["**/*.ts"] +} diff --git a/yarn.lock b/yarn.lock index 18057a5aec..a727768770 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2181,6 +2181,13 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@playwright/test@^1.40.0": + version "1.40.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.40.0.tgz#d06c506977dd7863aa16e07f2136351ecc1be6ed" + integrity sha512-PdW+kn4eV99iP5gxWNSDQCbhMaDVej+RXL5xr6t04nbKLCBwYtA046t7ofoczHOm8u6c+45hpDKQVZqtqwkeQg== + dependencies: + playwright "1.40.0" + "@radix-ui/primitive@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@radix-ui/primitive/-/primitive-1.0.1.tgz#e46f9958b35d10e9f6dc71c497305c22e3e55dbd" @@ -5696,6 +5703,11 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + fsevents@^2.3.2, fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -8191,6 +8203,20 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" +playwright-core@1.40.0: + version "1.40.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.40.0.tgz#82f61e5504cb3097803b6f8bbd98190dd34bdf14" + integrity sha512-fvKewVJpGeca8t0ipM56jkVSU6Eo0RmFvQ/MaCQNDYm+sdvKkMBBWTE1FdeMqIdumRaXXjZChWHvIzCGM/tA/Q== + +playwright@1.40.0: + version "1.40.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.40.0.tgz#2a1824b9fe5c4fe52ed53db9ea68003543a99df0" + integrity sha512-gyHAgQjiDf1m34Xpwzaqb76KgfzYrhK7iih+2IzcOCoZWr/8ZqmdBw+t0RU85ZmfJMgtgAiNtBQ/KS2325INXw== + dependencies: + playwright-core "1.40.0" + optionalDependencies: + fsevents "2.3.2" + pluralize@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"