Rework the Cypress & Percy CI (#8691)

* Split Cypress out into its own workflow

* Improve PR Details job to use github-script and output labels

* Fix wrongly using github.ref in workflow_run actions which always refer to develop

* Update pr-details to be far more generic

* Tweak how we fill command-prefix

* Tweak cypress job to pass more params & fix if condition

* Bring in external changes

* Add docs

* Use new composite action, and an action to update a status check based on this workflow run

* Iterate approach

.github/workflows/cypress.yaml

@@ -0,0 +1,101 @@
+# Triggers after the layered build has finished, taking the artifact and running cypress on it
+name: Cypress End to End Tests
+on:
+  workflow_run:
+    workflows: [ "Element Web - Build" ]
+    types:
+      - completed
+jobs:
+  # This job cannot have a pretty name due to https://github.com/haya14busa/action-workflow_run-status/issues/158
+  cypress:
+    if: github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      # Wire up the status check for this workflow_run action
+      - uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 # v1.0.0
+
+      - id: prdetails
+        if: github.event.workflow_run.event == 'pull_request'
+        uses: matrix-org/pr-details-action@v1.1
+        with:
+          owner: ${{ github.event.workflow_run.head_repository.owner.login }}
+          branch: ${{ github.event.workflow_run.head_branch }}
+
+      - uses: actions/checkout@v2
+
+      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
+      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
+      - name: 📥 Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: element-build-and-test.yaml
+          run_id: ${{ github.event.workflow_run.id }}
+          name: previewbuild
+          path: webapp
+
+      - name: Get commit details
+        if: github.event.workflow_run.event == 'pull_request'
+        run: |
+          echo "COMMIT_INFO_MESSAGE=\"$(git log --format=%B -n 1 ${{ github.event.workflow_run.head_sha }})\"" >> $GITHUB_ENV
+          echo "COMMIT_INFO_AUTHOR=$(git log --format='%an' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV
+          echo "COMMIT_INFO_EMAIL=$(git log --format='%ae' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV
+          # Only run Percy when it is demanded or on develop
+          if [[ "${{ (contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy') || github.ref == 'refs/heads/develop') }}" == "false" ]]; then
+            echo "PERCY_ENABLE=0" >> $GITHUB_ENV
+          fi
+
+      - name: Run Cypress tests
+        uses: cypress-io/github-action@v2
+        with:
+          # The built-in Electron runner seems to grind to a halt trying
+          # to run the tests, so use chrome.
+          browser: chrome
+          start: npx serve -p 8080 webapp
+          wait-on: 'http://localhost:8080'
+          record: true
+          command-prefix: 'yarn percy exec --'
+        env:
+          # pass the Dashboard record key as an environment variable
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          # tell Cypress more details about the context of this run
+          COMMIT_INFO_BRANCH: ${{ github.event.workflow_run.head_branch }}
+          COMMIT_INFO_SHA: ${{ github.event.workflow_run.head_sha }}
+          COMMIT_INFO_REMOTE: ${{ github.repositoryUrl }}
+
+          # pass the Percy token as an environment variable
+          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+          # Use existing chromium rather than downloading another
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser
+          # tell Percy more details about the context of this run
+          PERCY_BRANCH: ${{ github.event.workflow_run.head_branch }}
+          PERCY_COMMIT: ${{ github.event.workflow_run.head_sha }}
+          PERCY_PULL_REQUEST: ${{ steps.prdetails.outputs.pr_id }}
+          # pass GitHub token to allow accurately detecting a build vs a re-run build
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # make Node's os.tmpdir() return something where we actually have permissions
+          TMPDIR: ${{ runner.temp }}
+
+      - name: Upload Artifact
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: cypress-results
+          path: |
+            cypress/screenshots
+            cypress/videos
+            cypress/synapselogs
+
+      - name: Store benchmark result
+        if: github.ref == 'refs/heads/develop'
+        uses: matrix-org/github-action-benchmark@jsperfentry-1
+        with:
+          name: Cypress measurements
+          tool: 'jsperformanceentry'
+          output-file-path: cypress/performance/measurements.json
+          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
+          benchmark-data-dir-path: cypress/bench
+          fail-on-alert: false
+          comment-on-alert: false
+          github-token: ${{ secrets.DEPLOY_GH_PAGES }}
+          auto-push: ${{ github.ref == 'refs/heads/develop' }}

.github/workflows/element-build-and-test.yaml

@@ -1,125 +0,0 @@
-# Produce a build of element-web with this version of react-sdk
-# and any matching branches of element-web and js-sdk, output it
-# as an artifact and run integration tests.
-name: Element Web - Build and Test
-on:
-  pull_request: { }
-  push:
-    branches: [ develop, master ]
-  repository_dispatch:
-    types: [ upstream-sdk-notify ]
-env:
-  # These must be set for fetchdep.sh to get the right branch
-  REPOSITORY: ${{ github.repository }}
-  PR_NUMBER: ${{ github.event.pull_request.number }}
-jobs:
-  build:
-    name: "Build Element-Web"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-
-      - name: Fetch layered build
-        id: layered_build
-        run: |
-          scripts/ci/layered.sh
-          JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD)
-          REACT_SHA=$(git rev-parse --short=12 HEAD)
-          VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD)
-          echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA"
-
-      - name: Copy config
-        run: cp element.io/develop/config.json config.json
-        working-directory: ./element-web
-
-      - name: Build
-        env:
-          CI_PACKAGE: true
-          VERSION: "${{ steps.layered_build.outputs.VERSION }}"
-        run: yarn build
-        working-directory: ./element-web
-
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: previewbuild
-          path: element-web/webapp
-          # We'll only use this in a triggered job, then we're done with it
-          retention-days: 1
-
-  cypress:
-    name: "Cypress End to End Tests"
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Download build
-        uses: actions/download-artifact@v3
-        with:
-          name: previewbuild
-          path: webapp
-
-      - name: Run Cypress tests
-        uses: cypress-io/github-action@v2
-        with:
-          # The built-in Electron runner seems to grind to a halt trying
-          # to run the tests, so use chrome.
-          browser: chrome
-          start: npx serve -p 8080 webapp
-          wait-on: 'http://localhost:8080'
-          record: true
-          command-prefix: 'yarn percy exec --'
-        env:
-          # pass the Dashboard record key as an environment variable
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          # pass the Percy token as an environment variable
-          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
-          # Use existing chromium rather than downloading another
-          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
-          PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser
-          # pass GitHub token to allow accurately detecting a build vs a re-run build
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # make Node's os.tmpdir() return something where we actually have permissions
-          TMPDIR: ${{ runner.temp }}
-
-      - name: Upload Artifact
-        if: failure()
-        uses: actions/upload-artifact@v2
-        with:
-          name: cypress-results
-          path: |
-            cypress/screenshots
-            cypress/videos
-            cypress/synapselogs
-
-      - name: Store benchmark result
-        if: github.ref == 'refs/heads/develop'
-        uses: matrix-org/github-action-benchmark@jsperfentry-1
-        with:
-          name: Cypress measurements
-          tool: 'jsperformanceentry'
-          output-file-path: cypress/performance/measurements.json
-          # The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
-          benchmark-data-dir-path: cypress/bench
-          fail-on-alert: false
-          comment-on-alert: false
-          github-token: ${{ secrets.DEPLOY_GH_PAGES }}
-          auto-push: ${{ github.ref == 'refs/heads/develop' }}
-
-  app-tests:
-    name: Element Web Integration Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-
-      - name: Run tests
-        run: "./scripts/ci/app-tests.sh"

.github/workflows/element-web.yaml

@@ -0,0 +1,52 @@
+# Produce a build of element-web with this version of react-sdk
+# and any matching branches of element-web and js-sdk, output it
+# as an artifact and run integration tests.
+name: Element Web - Build
+on:
+  pull_request: { }
+  push:
+    branches: [ develop, master ]
+  repository_dispatch:
+    types: [ upstream-sdk-notify ]
+env:
+  # These must be set for fetchdep.sh to get the right branch
+  REPOSITORY: ${{ github.repository }}
+  PR_NUMBER: ${{ github.event.pull_request.number }}
+jobs:
+  build:
+    name: "Build Element-Web"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Fetch layered build
+        id: layered_build
+        run: |
+          scripts/ci/layered.sh
+          JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD)
+          REACT_SHA=$(git rev-parse --short=12 HEAD)
+          VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD)
+          echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA"
+
+      - name: Copy config
+        run: cp element.io/develop/config.json config.json
+        working-directory: ./element-web
+
+      - name: Build
+        env:
+          CI_PACKAGE: true
+          VERSION: "${{ steps.layered_build.outputs.VERSION }}"
+        run: yarn build
+        working-directory: ./element-web
+
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: previewbuild
+          path: element-web/webapp
+          # We'll only use this in a triggered job, then we're done with it
+          retention-days: 1

.github/workflows/netlify.yaml

@@ -3,7 +3,7 @@
 name: Upload Preview Build to Netlify
 on:
   workflow_run:
-    workflows: [ "Element Web - Build and Test" ]
+    workflows: [ "Element Web - Build" ]
     types:
       - completed
 jobs:

.github/workflows/tests.yml

@@ -35,3 +35,16 @@ jobs:
           path: |
             coverage
             !coverage/lcov-report
+
+  app-tests:
+    name: Element Web Integration Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
+
+      - name: Run tests
+        run: "./scripts/ci/app-tests.sh"

docs/cypress.md

@@ -171,9 +171,8 @@ should generally try to adhere to them.
 ## Percy Visual Testing
 We also support visual testing via Percy, this extracts the DOM from Cypress and renders it using custom renderers
 for Safari, Firefox, Chrome & Edge, allowing us to spot visual regressions before they become release regressions.
-Right now we run it as part of the standard Pull Request CI automation but due to only having 25k screenshots/month,
+Each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this can quickly be exhausted and
-and each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this could quickly be exhausted and
+so we only run Percy testing on `develop` and PRs which are labelled `X-Needs-Percy`.
-at that point we would likely run it on a CRON interval or before releases.
 
 To record a snapshot use `cy.percySnapshot()`, you may have to pass `percyCSS` into the 2nd argument to hide certain
 elements which contain dynamic/generated data to avoid them cause false positives in the Percy screenshot diffs.