Rework the Cypress & Percy CI (#8691)
* Split Cypress out into its own workflow * Improve PR Details job to use github-script and output labels * Fix wrongly using github.ref in workflow_run actions which always refer to develop * Update pr-details to be far more generic * Tweak how we fill command-prefix * Tweak cypress job to pass more params & fix if condition * Bring in external changes * Add docs * Use new composite action, and an action to update a status check based on this workflow run * Iterate approach
This commit is contained in:
Michael Telatynski
GitHub
parent
bba9189820
commit
fa8c346dfa
6 changed files with 169 additions and 129 deletions
101
.github/workflows/cypress.yaml
vendored
Normal file
101
.github/workflows/cypress.yaml
vendored
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# Triggers after the layered build has finished, taking the artifact and running cypress on it
|
||||
name: Cypress End to End Tests
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [ "Element Web - Build" ]
|
||||
types:
|
||||
- completed
|
||||
jobs:
|
||||
# This job cannot have a pretty name due to https://github.com/haya14busa/action-workflow_run-status/issues/158
|
||||
cypress:
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Wire up the status check for this workflow_run action
|
||||
- uses: haya14busa/action-workflow_run-status@967ed83efa565c257675ed70cfe5231f062ddd94 # v1.0.0
|
||||
|
||||
- id: prdetails
|
||||
if: github.event.workflow_run.event == 'pull_request'
|
||||
uses: matrix-org/pr-details-action@v1.1
|
||||
with:
|
||||
owner: ${{ github.event.workflow_run.head_repository.owner.login }}
|
||||
branch: ${{ github.event.workflow_run.head_branch }}
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
# There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
|
||||
# (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
|
||||
- name: 📥 Download artifact
|
||||
uses: dawidd6/action-download-artifact@v2
|
||||
with:
|
||||
workflow: element-build-and-test.yaml
|
||||
run_id: ${{ github.event.workflow_run.id }}
|
||||
name: previewbuild
|
||||
path: webapp
|
||||
|
||||
- name: Get commit details
|
||||
if: github.event.workflow_run.event == 'pull_request'
|
||||
run: |
|
||||
echo "COMMIT_INFO_MESSAGE=\"$(git log --format=%B -n 1 ${{ github.event.workflow_run.head_sha }})\"" >> $GITHUB_ENV
|
||||
echo "COMMIT_INFO_AUTHOR=$(git log --format='%an' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV
|
||||
echo "COMMIT_INFO_EMAIL=$(git log --format='%ae' -n 1 ${{ github.event.workflow_run.head_sha }})" >> $GITHUB_ENV
|
||||
# Only run Percy when it is demanded or on develop
|
||||
if [[ "${{ (contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy') || github.ref == 'refs/heads/develop') }}" == "false" ]]; then
|
||||
echo "PERCY_ENABLE=0" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Run Cypress tests
|
||||
uses: cypress-io/github-action@v2
|
||||
with:
|
||||
# The built-in Electron runner seems to grind to a halt trying
|
||||
# to run the tests, so use chrome.
|
||||
browser: chrome
|
||||
start: npx serve -p 8080 webapp
|
||||
wait-on: 'http://localhost:8080'
|
||||
record: true
|
||||
command-prefix: 'yarn percy exec --'
|
||||
env:
|
||||
# pass the Dashboard record key as an environment variable
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
# tell Cypress more details about the context of this run
|
||||
COMMIT_INFO_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
COMMIT_INFO_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
COMMIT_INFO_REMOTE: ${{ github.repositoryUrl }}
|
||||
|
||||
# pass the Percy token as an environment variable
|
||||
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
|
||||
# Use existing chromium rather than downloading another
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
|
||||
PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser
|
||||
# tell Percy more details about the context of this run
|
||||
PERCY_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
PERCY_COMMIT: ${{ github.event.workflow_run.head_sha }}
|
||||
PERCY_PULL_REQUEST: ${{ steps.prdetails.outputs.pr_id }}
|
||||
# pass GitHub token to allow accurately detecting a build vs a re-run build
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# make Node's os.tmpdir() return something where we actually have permissions
|
||||
TMPDIR: ${{ runner.temp }}
|
||||
|
||||
- name: Upload Artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cypress-results
|
||||
path: |
|
||||
cypress/screenshots
|
||||
cypress/videos
|
||||
cypress/synapselogs
|
||||
|
||||
- name: Store benchmark result
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
uses: matrix-org/github-action-benchmark@jsperfentry-1
|
||||
with:
|
||||
name: Cypress measurements
|
||||
tool: 'jsperformanceentry'
|
||||
output-file-path: cypress/performance/measurements.json
|
||||
# The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
|
||||
benchmark-data-dir-path: cypress/bench
|
||||
fail-on-alert: false
|
||||
comment-on-alert: false
|
||||
github-token: ${{ secrets.DEPLOY_GH_PAGES }}
|
||||
auto-push: ${{ github.ref == 'refs/heads/develop' }}
|
||||
125
.github/workflows/element-build-and-test.yaml
vendored
125
.github/workflows/element-build-and-test.yaml
vendored
|
|
@ -1,125 +0,0 @@
|
|||
# Produce a build of element-web with this version of react-sdk
|
||||
# and any matching branches of element-web and js-sdk, output it
|
||||
# as an artifact and run integration tests.
|
||||
name: Element Web - Build and Test
|
||||
on:
|
||||
pull_request: { }
|
||||
push:
|
||||
branches: [ develop, master ]
|
||||
repository_dispatch:
|
||||
types: [ upstream-sdk-notify ]
|
||||
env:
|
||||
# These must be set for fetchdep.sh to get the right branch
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
jobs:
|
||||
build:
|
||||
name: "Build Element-Web"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Fetch layered build
|
||||
id: layered_build
|
||||
run: |
|
||||
scripts/ci/layered.sh
|
||||
JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD)
|
||||
REACT_SHA=$(git rev-parse --short=12 HEAD)
|
||||
VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD)
|
||||
echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA"
|
||||
|
||||
- name: Copy config
|
||||
run: cp element.io/develop/config.json config.json
|
||||
working-directory: ./element-web
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
CI_PACKAGE: true
|
||||
VERSION: "${{ steps.layered_build.outputs.VERSION }}"
|
||||
run: yarn build
|
||||
working-directory: ./element-web
|
||||
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: previewbuild
|
||||
path: element-web/webapp
|
||||
# We'll only use this in a triggered job, then we're done with it
|
||||
retention-days: 1
|
||||
|
||||
cypress:
|
||||
name: "Cypress End to End Tests"
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Download build
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: previewbuild
|
||||
path: webapp
|
||||
|
||||
- name: Run Cypress tests
|
||||
uses: cypress-io/github-action@v2
|
||||
with:
|
||||
# The built-in Electron runner seems to grind to a halt trying
|
||||
# to run the tests, so use chrome.
|
||||
browser: chrome
|
||||
start: npx serve -p 8080 webapp
|
||||
wait-on: 'http://localhost:8080'
|
||||
record: true
|
||||
command-prefix: 'yarn percy exec --'
|
||||
env:
|
||||
# pass the Dashboard record key as an environment variable
|
||||
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
|
||||
# pass the Percy token as an environment variable
|
||||
PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
|
||||
# Use existing chromium rather than downloading another
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
|
||||
PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser
|
||||
# pass GitHub token to allow accurately detecting a build vs a re-run build
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# make Node's os.tmpdir() return something where we actually have permissions
|
||||
TMPDIR: ${{ runner.temp }}
|
||||
|
||||
- name: Upload Artifact
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: cypress-results
|
||||
path: |
|
||||
cypress/screenshots
|
||||
cypress/videos
|
||||
cypress/synapselogs
|
||||
|
||||
- name: Store benchmark result
|
||||
if: github.ref == 'refs/heads/develop'
|
||||
uses: matrix-org/github-action-benchmark@jsperfentry-1
|
||||
with:
|
||||
name: Cypress measurements
|
||||
tool: 'jsperformanceentry'
|
||||
output-file-path: cypress/performance/measurements.json
|
||||
# The dashboard is available at https://matrix-org.github.io/matrix-react-sdk/cypress/bench/
|
||||
benchmark-data-dir-path: cypress/bench
|
||||
fail-on-alert: false
|
||||
comment-on-alert: false
|
||||
github-token: ${{ secrets.DEPLOY_GH_PAGES }}
|
||||
auto-push: ${{ github.ref == 'refs/heads/develop' }}
|
||||
|
||||
app-tests:
|
||||
name: Element Web Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Run tests
|
||||
run: "./scripts/ci/app-tests.sh"
|
||||
52
.github/workflows/element-web.yaml
vendored
Normal file
52
.github/workflows/element-web.yaml
vendored
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Produce a build of element-web with this version of react-sdk
|
||||
# and any matching branches of element-web and js-sdk, output it
|
||||
# as an artifact and run integration tests.
|
||||
name: Element Web - Build
|
||||
on:
|
||||
pull_request: { }
|
||||
push:
|
||||
branches: [ develop, master ]
|
||||
repository_dispatch:
|
||||
types: [ upstream-sdk-notify ]
|
||||
env:
|
||||
# These must be set for fetchdep.sh to get the right branch
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
jobs:
|
||||
build:
|
||||
name: "Build Element-Web"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Fetch layered build
|
||||
id: layered_build
|
||||
run: |
|
||||
scripts/ci/layered.sh
|
||||
JSSDK_SHA=$(git -C matrix-js-sdk rev-parse --short=12 HEAD)
|
||||
REACT_SHA=$(git rev-parse --short=12 HEAD)
|
||||
VECTOR_SHA=$(git -C element-web rev-parse --short=12 HEAD)
|
||||
echo "::set-output name=VERSION::$VECTOR_SHA-react-$REACT_SHA-js-$JSSDK_SHA"
|
||||
|
||||
- name: Copy config
|
||||
run: cp element.io/develop/config.json config.json
|
||||
working-directory: ./element-web
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
CI_PACKAGE: true
|
||||
VERSION: "${{ steps.layered_build.outputs.VERSION }}"
|
||||
run: yarn build
|
||||
working-directory: ./element-web
|
||||
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: previewbuild
|
||||
path: element-web/webapp
|
||||
# We'll only use this in a triggered job, then we're done with it
|
||||
retention-days: 1
|
||||
2
.github/workflows/netlify.yaml
vendored
2
.github/workflows/netlify.yaml
vendored
|
|
@ -3,7 +3,7 @@
|
|||
name: Upload Preview Build to Netlify
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [ "Element Web - Build and Test" ]
|
||||
workflows: [ "Element Web - Build" ]
|
||||
types:
|
||||
- completed
|
||||
jobs:
|
||||
|
|
|
|||
13
.github/workflows/tests.yml
vendored
13
.github/workflows/tests.yml
vendored
|
|
@ -35,3 +35,16 @@ jobs:
|
|||
path: |
|
||||
coverage
|
||||
!coverage/lcov-report
|
||||
|
||||
app-tests:
|
||||
name: Element Web Integration Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Run tests
|
||||
run: "./scripts/ci/app-tests.sh"
|
||||
|
|
|
|||
|
|
@ -171,9 +171,8 @@ should generally try to adhere to them.
|
|||
## Percy Visual Testing
|
||||
We also support visual testing via Percy, this extracts the DOM from Cypress and renders it using custom renderers
|
||||
for Safari, Firefox, Chrome & Edge, allowing us to spot visual regressions before they become release regressions.
|
||||
Right now we run it as part of the standard Pull Request CI automation but due to only having 25k screenshots/month,
|
||||
and each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this could quickly be exhausted and
|
||||
at that point we would likely run it on a CRON interval or before releases.
|
||||
Each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this can quickly be exhausted and
|
||||
so we only run Percy testing on `develop` and PRs which are labelled `X-Needs-Percy`.
|
||||
|
||||
To record a snapshot use `cy.percySnapshot()`, you may have to pass `percyCSS` into the 2nd argument to hide certain
|
||||
elements which contain dynamic/generated data to avoid them cause false positives in the Percy screenshot diffs.
|
||||
|
|
|
|||
Loading…
Reference in a new issue