diff --git a/.github/workflows/element-build-and-test.yaml b/.github/workflows/element-build-and-test.yaml index 9b3d0f373a..07a7ed0f5d 100644 --- a/.github/workflows/element-build-and-test.yaml +++ b/.github/workflows/element-build-and-test.yaml @@ -88,6 +88,20 @@ jobs: cypress/videos cypress/synapselogs + - name: Store benchmark result + if: github.ref == 'refs/heads/develop' + uses: matrix-org/github-action-benchmark@jsperfentry-1 + with: + name: Cypress measurements + tool: 'jsperformanceentry' + output-file-path: cypress/performance/measurements.json + # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/ + benchmark-data-dir-path: cypress/bench + fail-on-alert: false + comment-on-alert: false + github-token: ${{ secrets.DEPLOY_GH_PAGES }} + auto-push: ${{ github.ref == 'refs/heads/develop' }} + app-tests: name: Element Web Integration Tests runs-on: ubuntu-latest diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 6c663a0e01..7008791607 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -40,20 +40,18 @@ jobs: test/end-to-end-tests/synapse/installations/consent/homeserver.log retention-days: 14 - - name: Download previous benchmark data - uses: actions/cache@v1 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - - name: Store benchmark result + if: github.ref == 'refs/heads/develop' uses: matrix-org/github-action-benchmark@jsperfentry-1 with: tool: 'jsperformanceentry' output-file-path: test/end-to-end-tests/performance-entries.json + # This is the default dashboard path. It's included here anyway to + # make the difference from the Cypress variant in + # `element-build-and-test.yaml` more obvious. + # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/dev/bench/ + benchmark-data-dir-path: dev/bench fail-on-alert: false comment-on-alert: false - # Only temporary to monitor where failures occur - alert-comment-cc-users: '@gsouquet' github-token: ${{ secrets.DEPLOY_GH_PAGES }} auto-push: ${{ github.ref == 'refs/heads/develop' }} diff --git a/.gitignore b/.gitignore index 8e14ba9057..e360df7767 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ package-lock.json # These could have files in them but don't currently # Cypress will still auto-create them though... /cypress/fixtures +/cypress/performance diff --git a/cypress/global.d.ts b/cypress/global.d.ts index d0fb732778..efbb255b08 100644 --- a/cypress/global.d.ts +++ b/cypress/global.d.ts @@ -18,6 +18,7 @@ import "matrix-js-sdk/src/@types/global"; import type { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client"; import type { RoomMemberEvent } from "matrix-js-sdk/src/models/room-member"; import type { MatrixDispatcher } from "../src/dispatcher/dispatcher"; +import type PerformanceMonitor from "../src/performance"; declare global { // eslint-disable-next-line @typescript-eslint/no-namespace @@ -27,6 +28,7 @@ declare global { matrixClient?: MatrixClient; }; mxDispatcher: MatrixDispatcher; + mxPerformanceMonitor: PerformanceMonitor; beforeReload?: boolean; // for detecting reloads // Partial type for the matrix-js-sdk module, exported by browser-matrix matrixcs: { @@ -38,7 +40,11 @@ declare global { } interface Window { - mxDispatcher: MatrixDispatcher; // to appease the MatrixDispatcher import + // to appease the MatrixDispatcher import + mxDispatcher: MatrixDispatcher; + // to appease the PerformanceMonitor import + mxPerformanceMonitor: PerformanceMonitor; + mxPerformanceEntryNames: any; } } diff --git a/cypress/integration/1-register/register.spec.ts b/cypress/integration/1-register/register.spec.ts index f61a10e304..b470932c61 100644 --- a/cypress/integration/1-register/register.spec.ts +++ b/cypress/integration/1-register/register.spec.ts @@ -42,11 +42,15 @@ describe("Registration", () => { cy.get("#mx_RegistrationForm_username").type("alice"); cy.get("#mx_RegistrationForm_password").type("totally a great password"); cy.get("#mx_RegistrationForm_passwordConfirm").type("totally a great password"); + cy.startMeasuring("create-account"); cy.get(".mx_Login_submit").click(); cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click(); + cy.stopMeasuring("create-account"); cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click(); + cy.startMeasuring("from-submit-to-home"); cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click(); cy.url().should('contain', '/#/home'); + cy.stopMeasuring("from-submit-to-home"); }); }); diff --git a/cypress/integration/2-login/login.spec.ts b/cypress/integration/2-login/login.spec.ts index 9fb7ba4792..ea8eefa573 100644 --- a/cypress/integration/2-login/login.spec.ts +++ b/cypress/integration/2-login/login.spec.ts @@ -49,9 +49,11 @@ describe("Login", () => { cy.get("#mx_LoginForm_username").type(username); cy.get("#mx_LoginForm_password").type(password); + cy.startMeasuring("from-submit-to-home"); cy.get(".mx_Login_submit").click(); cy.url().should('contain', '/#/home'); + cy.stopMeasuring("from-submit-to-home"); }); }); }); diff --git a/cypress/integration/4-create-room/create-room.spec.ts b/cypress/integration/4-create-room/create-room.spec.ts index d6abab814d..9bf38194d9 100644 --- a/cypress/integration/4-create-room/create-room.spec.ts +++ b/cypress/integration/4-create-room/create-room.spec.ts @@ -54,10 +54,12 @@ describe("Create Room", () => { // Fill room address cy.get('[label="Room address"]').type("test-room-1"); // Submit + cy.startMeasuring("from-submit-to-room"); cy.get(".mx_Dialog_primary").click(); }); cy.url().should("contain", "/#/room/#test-room-1:localhost"); + cy.stopMeasuring("from-submit-to-room"); cy.get(".mx_RoomHeader_nametext").contains(name); cy.get(".mx_RoomHeader_topic").contains(topic); }); diff --git a/cypress/plugins/index.ts b/cypress/plugins/index.ts index 9438d13606..eab5441c20 100644 --- a/cypress/plugins/index.ts +++ b/cypress/plugins/index.ts @@ -16,13 +16,15 @@ limitations under the License. /// -import { synapseDocker } from "./synapsedocker"; import PluginEvents = Cypress.PluginEvents; import PluginConfigOptions = Cypress.PluginConfigOptions; +import { performance } from "./performance"; +import { synapseDocker } from "./synapsedocker"; /** * @type {Cypress.PluginConfig} */ export default function(on: PluginEvents, config: PluginConfigOptions) { + performance(on, config); synapseDocker(on, config); } diff --git a/cypress/plugins/performance.ts b/cypress/plugins/performance.ts new file mode 100644 index 0000000000..c6bd3e4ce9 --- /dev/null +++ b/cypress/plugins/performance.ts @@ -0,0 +1,47 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/// + +import * as path from "path"; +import * as fse from "fs-extra"; + +import PluginEvents = Cypress.PluginEvents; +import PluginConfigOptions = Cypress.PluginConfigOptions; + +// This holds all the performance measurements throughout the run +let bufferedMeasurements: PerformanceEntry[] = []; + +function addMeasurements(measurements: PerformanceEntry[]): void { + bufferedMeasurements = bufferedMeasurements.concat(measurements); + return null; +} + +async function writeMeasurementsFile() { + try { + const measurementsPath = path.join("cypress", "performance", "measurements.json"); + await fse.outputJSON(measurementsPath, bufferedMeasurements, { + spaces: 4, + }); + } finally { + bufferedMeasurements = []; + } +} + +export function performance(on: PluginEvents, config: PluginConfigOptions) { + on("task", { addMeasurements }); + on("after:run", writeMeasurementsFile); +} diff --git a/cypress/plugins/synapsedocker/index.ts b/cypress/plugins/synapsedocker/index.ts index af8ddac73c..292c74ee67 100644 --- a/cypress/plugins/synapsedocker/index.ts +++ b/cypress/plugins/synapsedocker/index.ts @@ -201,7 +201,7 @@ async function synapseStop(id: string): Promise { synapses.delete(id); console.log(`Stopped synapse id ${id}.`); - // cypres deliberately fails if you return 'undefined', so + // cypress deliberately fails if you return 'undefined', so // return null to signal all is well and we've handled the task. return null; } diff --git a/cypress/support/bot.ts b/cypress/support/bot.ts index 3ba7b89cde..a2488c0081 100644 --- a/cypress/support/bot.ts +++ b/cypress/support/bot.ts @@ -40,7 +40,7 @@ Cypress.Commands.add("getBot", (synapse: SynapseInstance, displayName?: string): const username = Cypress._.uniqueId("userId_"); const password = Cypress._.uniqueId("password_"); return cy.registerUser(synapse, username, password, displayName).then(credentials => { - return cy.window().then(win => { + return cy.window({ log: false }).then(win => { const cli = new win.matrixcs.MatrixClient({ baseUrl: synapse.baseUrl, userId: credentials.userId, diff --git a/cypress/support/client.ts b/cypress/support/client.ts index eef3aa1086..682f3ee426 100644 --- a/cypress/support/client.ts +++ b/cypress/support/client.ts @@ -46,11 +46,11 @@ declare global { } Cypress.Commands.add("getClient", (): Chainable => { - return cy.window().then(win => win.mxMatrixClientPeg.matrixClient); + return cy.window({ log: false }).then(win => win.mxMatrixClientPeg.matrixClient); }); Cypress.Commands.add("createRoom", (options: ICreateRoomOpts): Chainable => { - return cy.window().then(async win => { + return cy.window({ log: false }).then(async win => { const cli = win.mxMatrixClientPeg.matrixClient; const resp = await cli.createRoom(options); const roomId = resp.room_id; diff --git a/cypress/support/index.ts b/cypress/support/index.ts index 6535d3a0b1..197b2ddc0c 100644 --- a/cypress/support/index.ts +++ b/cypress/support/index.ts @@ -16,6 +16,7 @@ limitations under the License. /// +import "./performance"; import "./synapse"; import "./login"; import "./client"; diff --git a/cypress/support/login.ts b/cypress/support/login.ts index 90e5c5dbdc..50be88ae67 100644 --- a/cypress/support/login.ts +++ b/cypress/support/login.ts @@ -43,7 +43,7 @@ declare global { Cypress.Commands.add("initTestUser", (synapse: SynapseInstance, displayName: string): Chainable => { // XXX: work around Cypress not clearing IDB between tests - cy.window().then(win => { + cy.window({ log: false }).then(win => { win.indexedDB.databases().then(databases => { databases.forEach(database => { win.indexedDB.deleteDatabase(database.name); @@ -73,7 +73,7 @@ Cypress.Commands.add("initTestUser", (synapse: SynapseInstance, displayName: str }, }); }).then(response => { - cy.window().then(win => { + cy.window({ log: false }).then(win => { // Seed the localStorage with the required credentials win.localStorage.setItem("mx_hs_url", synapse.baseUrl); win.localStorage.setItem("mx_user_id", response.body.user_id); diff --git a/cypress/support/performance.ts b/cypress/support/performance.ts new file mode 100644 index 0000000000..bbd1fe217d --- /dev/null +++ b/cypress/support/performance.ts @@ -0,0 +1,74 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/// + +import Chainable = Cypress.Chainable; +import AUTWindow = Cypress.AUTWindow; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Cypress { + interface Chainable { + /** + * Start measuring the duration of some task. + * @param task The task name. + */ + startMeasuring(task: string): Chainable; + /** + * Stop measuring the duration of some task. + * The duration is reported in the Cypress log. + * @param task The task name. + */ + stopMeasuring(task: string): Chainable; + } + } +} + +function getPrefix(task: string): string { + return `cy:${Cypress.spec.name.split(".")[0]}:${task}`; +} + +function startMeasuring(task: string): Chainable { + return cy.window({ log: false }).then((win) => { + win.mxPerformanceMonitor.start(getPrefix(task)); + }); +} + +function stopMeasuring(task: string): Chainable { + return cy.window({ log: false }).then((win) => { + const measure = win.mxPerformanceMonitor.stop(getPrefix(task)); + cy.log(`**${task}** ${measure.duration} ms`); + }); +} + +Cypress.Commands.add("startMeasuring", startMeasuring); +Cypress.Commands.add("stopMeasuring", stopMeasuring); + +Cypress.on("window:before:unload", (event: BeforeUnloadEvent) => { + const doc = event.target as Document; + if (doc.location.href === "about:blank") return; + const win = doc.defaultView as AUTWindow; + if (!win.mxPerformanceMonitor) return; + const entries = win.mxPerformanceMonitor.getEntries().filter(entry => { + return entry.name.startsWith("cy:"); + }); + if (!entries || entries.length === 0) return; + cy.task("addMeasurements", entries); +}); + +// Needed to make this file a module +export { }; diff --git a/cypress/support/synapse.ts b/cypress/support/synapse.ts index aa1ba085f5..5696e8c015 100644 --- a/cypress/support/synapse.ts +++ b/cypress/support/synapse.ts @@ -63,7 +63,7 @@ function startSynapse(template: string): Chainable { function stopSynapse(synapse?: SynapseInstance): Chainable { if (!synapse) return; // Navigate away from app to stop the background network requests which will race with Synapse shutting down - return cy.window().then((win) => { + return cy.window({ log: false }).then((win) => { win.location.href = 'about:blank'; cy.task("synapseStop", synapse.synapseId); }); diff --git a/src/performance/index.ts b/src/performance/index.ts index 35319c23f0..9ea8dbd215 100644 --- a/src/performance/index.ts +++ b/src/performance/index.ts @@ -71,7 +71,7 @@ export default class PerformanceMonitor { * with the start marker * @param name Name of the recording * @param id Specify an identifier appended to the measurement name - * @returns {void} + * @returns The measurement */ stop(name: string, id?: string): PerformanceEntry { if (!this.supportsPerformanceApi()) {