speed up playwright and add visual regression tests (#1638)

This diff:
- tweaks how playwright runs in CI to have it go a bit faster
- uploads nice browsable reports to S3 for looking at playwright
failures and traces
- adds visual regression testing to playwright

### Change Type

- [x] `tests` — Changes to any test code only[^2]

### Test Plan

- [ ] Unit Tests
- [x] End to end tests

### Release Notes

--

---------

Co-authored-by: huppy-bot[bot] <128400622+huppy-bot[bot]@users.noreply.github.com>

.github/workflows/playwright-update-snapshots.yml

@@ -0,0 +1,82 @@
+# Adapted from https://mmazzarolo.com/blog/2022-09-09-visual-regression-testing-with-playwright-and-github-actions/
+
+# This workflow's goal is forcing an update of the reference snapshots used
+# by Playwright tests. Add the 'update-snapshots' label to a PR.
+# From a high-level perspective, it works like this:
+# 1. Because of a GitHub Action limitation, this workflow is triggered on every
+#    label added to a PR. We manually interrupt it unless the label is
+#    "update-snapshots".
+# 2. Use the GitHub API to grab the information about the branch name and SHA of
+#    the latest commit of the current pull request.
+# 3. Remove the label from the PR.
+# 4. Update the Playwright reference snapshots based on the UI of this branch.
+# 5. Commit the newly generated Playwright reference snapshots into this branch.
+name: Update playwright snapshots
+
+on:
+  pull_request:
+    types: [labeled]
+
+env:
+  CI: 1
+  PRINT_GITHUB_ANNOTATIONS: 1
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  update_snapshots:
+    name: 'Run'
+    timeout-minutes: 60
+    runs-on: ubuntu-latest-16-cores-open
+    if: github.event.label.name == 'update-snapshots'
+
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Remove the update-snapshots label
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          labels: update-snapshots
+
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 5
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'yarn'
+          cache-dependency-path: 'public-yarn.lock'
+
+      - name: Enable corepack
+        run: corepack enable
+
+      - name: Install dependencies
+        run: yarn
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium chrome
+
+      - name: Run Playwright tests & update snapshots
+        run: 'yarn e2e --update-snapshots'
+        working-directory: 'apps/examples'
+
+      - name: Commit and push changes
+        if: always()
+        run: |
+          git config --global user.name 'huppy-bot[bot]'
+          git config --global user.email '128400622+huppy-bot[bot]@users.noreply.github.com'
+          git add -A
+          git commit --no-verify -m '[automated] update snapshots'
+          git pull --rebase
+          git push

.github/workflows/playwright.yml

@@ -18,7 +18,7 @@ jobs:
   build:
     name: 'End to end tests'
     timeout-minutes: 60
-    runs-on: ubuntu-latest-16-cores-open # TODO: this should probably run on multiple OSes
+    runs-on: ubuntu-latest-16-cores-open
 
     steps:
       - name: Check out code
@@ -37,20 +37,34 @@ jobs:
       - name: Install dependencies
         run: yarn
 
-      - name: Build all projects
-        # the sed pipe makes sure that github annotations come through without
-        # turbo's prefix
-        run: "yarn build | sed -E 's/^.*? ::/::/'"
-
       - name: Install Playwright browsers
-        run: npx playwright install --with-deps
+        run: npx playwright install --with-deps chromium chrome
 
       - name: Run Playwright tests
-        run: yarn e2e
+        run: 'yarn e2e'
+        working-directory: apps/examples
 
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-report
-          path: playwright-report/
+          path: apps/examples/playwright-report
           retention-days: 30
+
+      - uses: shallwefootball/s3-upload-action@master
+        if: always()
+        name: Upload S3
+        id: s3
+        with:
+          aws_key_id: ${{ secrets.AWS_S3_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_S3_SECRET_KEY}}
+          aws_bucket: playwright-reports.tldraw.xyz
+          source_dir: apps/examples/playwright-report
+
+      - name: Log report to summary
+        if: always()
+        run: |
+          report_url="https://playwright-reports.tldraw.xyz/${{ steps.s3.outputs.object_key }}/index.html"
+          echo "Report: $report_url"
+          echo "## Playwright report" >> $GITHUB_STEP_SUMMARY
+          echo "* $report_url" >> $GITHUB_STEP_SUMMARY

.gitignore

@@ -79,4 +79,4 @@ apps/examples/www/index.js
 apps/examples/build.esbuild.json
 
 apps/examples/e2e/test-results
-apps/examples/playwright-report/index.html
+apps/examples/playwright-report

apps/examples/e2e/playwright.config.ts

@@ -1,6 +1,7 @@
 import type { PlaywrightTestConfig } from '@playwright/test'
 import { devices } from '@playwright/test'
 import { config as _config } from 'dotenv'
+import path from 'path'
 /**
  * Read environment variables from file.
  * https://github.com/motdotla/dotenv
@@ -23,19 +24,18 @@ const config: PlaywrightTestConfig = {
 		 */
 		timeout: 2000,
 		toHaveScreenshot: {
-			maxDiffPixelRatio: 0.15,
+			maxDiffPixelRatio: 0.001,
+			threshold: 0.01,
 		},
 	},
 	/* Run tests in files in parallel */
 	fullyParallel: true,
 	/* Fail the build on CI if you accidentally left test.only in the source code. */
-	forbidOnly: !!process.env.CI,
+	forbidOnly: false, // !!process.env.CI,
 	/* Retry on CI only */
 	retries: process.env.CI ? 1 : 0,
-	/* Opt out of parallel tests on CI. */
-	workers: process.env.CI ? 1 : undefined,
 	/* Reporter to use. See https://playwright.dev/docs/test-reporters */
-	// reporter: 'html',
+	reporter: process.env.CI ? [['list'], ['github'], ['html', { open: 'never' }]] : 'list',
 	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
 	use: {
 		/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
@@ -45,7 +45,7 @@ const config: PlaywrightTestConfig = {
 
 		/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
 		trace: 'on-first-retry',
-		headless: true, // !!process.env.CI,
+		headless: true, // !process.env.CI,
 	},
 
 	/* Configure projects for major browsers */
@@ -101,6 +101,7 @@ const config: PlaywrightTestConfig = {
 		command: 'yarn dev',
 		port: 5420,
 		reuseExistingServer: !process.env.CI,
+		cwd: path.join(__dirname, '../../..'),
 	},
 }
 

apps/examples/e2e/tests/export-snapshots.spec.ts

@@ -0,0 +1,214 @@
+import test, { Page, expect } from '@playwright/test'
+import { Editor, TLShapeId, TLShapePartial } from '@tldraw/tldraw'
+import { assert } from '@tldraw/utils'
+import { rename, writeFile } from 'fs/promises'
+import { setupPage } from '../shared-e2e'
+
+let page: Page
+declare const editor: Editor
+
+// this is currently skipped as we can't enforce it on CI. i'm going to enable it in a follow-up though!
+test.describe('Export snapshots', () => {
+	test.beforeAll(async ({ browser }) => {
+		page = await browser.newPage()
+	})
+	test.beforeEach(async () => {
+		await setupPage(page)
+	})
+
+	const snapshots = {} as Record<string, TLShapePartial[]>
+
+	for (const fill of ['none', 'semi', 'solid', 'pattern']) {
+		snapshots[`geo fill=${fill}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'geo',
+				props: {
+					fill,
+					color: 'green',
+					w: 100,
+					h: 100,
+				},
+			},
+		]
+
+		snapshots[`arrow fill=${fill}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'arrow',
+				props: {
+					color: 'light-green',
+					fill: fill,
+					arrowheadStart: 'square',
+					arrowheadEnd: 'dot',
+					start: { type: 'point', x: 0, y: 0 },
+					end: { type: 'point', x: 100, y: 100 },
+					bend: 20,
+				},
+			},
+		]
+
+		snapshots[`draw fill=${fill}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'draw',
+				props: {
+					color: 'light-violet',
+					fill: fill,
+					segments: [
+						{
+							type: 'straight',
+							points: [{ x: 0, y: 0 }],
+						},
+						{
+							type: 'straight',
+							points: [
+								{ x: 0, y: 0 },
+								{ x: 100, y: 0 },
+							],
+						},
+						{
+							type: 'straight',
+							points: [
+								{ x: 100, y: 0 },
+								{ x: 0, y: 100 },
+							],
+						},
+						{
+							type: 'straight',
+							points: [
+								{ x: 0, y: 100 },
+								{ x: 100, y: 100 },
+							],
+						},
+						{
+							type: 'straight',
+							points: [
+								{ x: 100, y: 100 },
+								{ x: 0, y: 0 },
+							],
+						},
+					],
+					isClosed: true,
+					isComplete: true,
+				},
+			},
+		]
+	}
+
+	for (const font of ['draw', 'sans', 'serif', 'mono']) {
+		snapshots[`geo font=${font}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'geo',
+				props: {
+					text: 'test',
+					color: 'blue',
+					font,
+					w: 100,
+					h: 100,
+				},
+			},
+		]
+
+		snapshots[`arrow font=${font}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'arrow',
+				props: {
+					color: 'blue',
+					fill: 'solid',
+					arrowheadStart: 'square',
+					arrowheadEnd: 'arrow',
+					font,
+					start: { type: 'point', x: 0, y: 0 },
+					end: { type: 'point', x: 100, y: 100 },
+					bend: 20,
+					text: 'test',
+				},
+			},
+		]
+
+		snapshots[`arrow font=${font}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'arrow',
+				props: {
+					color: 'blue',
+					fill: 'solid',
+					arrowheadStart: 'square',
+					arrowheadEnd: 'arrow',
+					font,
+					start: { type: 'point', x: 0, y: 0 },
+					end: { type: 'point', x: 100, y: 100 },
+					bend: 20,
+					text: 'test',
+				},
+			},
+		]
+
+		snapshots[`note font=${font}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'note',
+				props: {
+					color: 'violet',
+					font,
+					text: 'test',
+				},
+			},
+		]
+
+		snapshots[`text font=${font}`] = [
+			{
+				id: 'shape:testShape' as TLShapeId,
+				type: 'text',
+				props: {
+					color: 'red',
+					font,
+					text: 'test',
+				},
+			},
+		]
+	}
+
+	for (const [name, shapes] of Object.entries(snapshots)) {
+		test(`Exports with ${name}`, async () => {
+			await page.evaluate((shapes) => {
+				editor
+					.updateInstanceState({ exportBackground: false })
+					.selectAll()
+					.deleteShapes()
+					.createShapes(shapes)
+			}, shapes)
+
+			const downloadEvent = page.waitForEvent('download')
+			await page.click('[data-testid="main.menu"]')
+			await page.click('[data-testid="menu-item.edit"]')
+			await page.click('[data-testid="menu-item.export-as"]')
+			await page.click('[data-testid="menu-item.export-as-svg"]')
+
+			const download = await downloadEvent
+			const path = await download.path()
+			assert(path)
+			await rename(path, path + '.svg')
+			await writeFile(
+				path + '.html',
+				`
+                    <!DOCTYPE html>
+                    <meta charset="utf-8" />
+                    <meta name="viewport" content="width=device-width, initial-scale=1" />
+                    <img src="${path}.svg" />
+                `,
+				'utf-8'
+			)
+
+			await page.goto(`file://${path}.html`)
+			const clip = await page.$eval('img', (img) => img.getBoundingClientRect())
+			await expect(page).toHaveScreenshot({
+				omitBackground: true,
+				clip,
+			})
+		})
+	}
+})