PrivateCoffee/element-web
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

Report cypress test results into testrail. (#9685)

Browse source 
* Reduce cypress size/scope

* Emit cypress reports in junit-compatible format.

* Clean implementation of upload

* Append a warning if not all tests that testrail knows of are run.

* Only run testrail if on develop (like percy).

* Re-enable full test run.

* Update cypress.config.ts

* Ensure SUITE_ID is passed.

* Fix quoting on SUITE_ID, add environment

* Update .github/workflows/cypress.yaml

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

* re-add mocha-junit-reporter (was dropped in the merge)

* Prettifier on newly created file.

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Michael Kaye 2022-12-15 12:04:03 +00:00 committed by GitHub
parent ab560bba40
commit b728b27435
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 141 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
.github/workflows/cypress.yaml vendored
View file

@ -22,6 +22,7 @@ jobs:
commit_author: ${{ steps.commit.outputs.author }}
commit_email: ${{ steps.commit.outputs.email }}
percy_enable: ${{ steps.percy.outputs.value || '1' }}
testrail_enable: ${{ steps.testrail.outputs.value || '1' }}
steps:
# We create the status here and then update it to success/failure in the `report` stage
# This provides an easy link to this workflow_run from the PR before Cypress is done.
@ -63,6 +64,14 @@ jobs:
!contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy')
run: echo "::set-output name=value::0"
# Only run Testrail when it is demanded or on develop
- name: Disable Testrail if not needed
id: testrail
if: |
github.event.workflow_run.event == 'pull_request' &&
!contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Send-Testrail')
run: echo "::set-output name=value::0"
- name: Generate unique ID 💎
id: uuid
run: echo "::set-output name=value::sha-$GITHUB_SHA-time-$(date +"%s")"
@ -114,6 +123,7 @@ jobs:
#parallel: true
#command-prefix: 'yarn percy exec --parallel --'
command-prefix: "yarn percy exec --"
config: '{"reporter":"cypress-multi-reporters", "reporterOptions": { "configFile": "cypress-ci-reporter-config.json" } }'
ci-build-id: ${{ needs.prepare.outputs.uuid }}
env:
# pass the Dashboard record key as an environment variable
@ -158,6 +168,12 @@ jobs:
cypress/videos
cypress/synapselogs
- name: Upload reports
if: always()
uses: actions/upload-artifact@v2
with:
name: cypress-junit
path: cypress/results
report:
name: Report results
needs: tests
@ -173,3 +189,68 @@ jobs:
context: ${{ github.workflow }} / cypress (${{ github.event.workflow_run.event }} => ${{ github.event_name }})
sha: ${{ github.event.workflow_run.head_sha }}
target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
testrail:
name: Report results to testrail
needs:
- prepare
- tests
environment: Testrail
runs-on: ubuntu-latest
if: ${{ needs.prepare.outputs.testrail_enable }}
steps:
- uses: actions/download-artifact@v3
with:
name: cypress-junit
- name: Prepare testrail upload config
id: testrailprep
env:
TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
TEST_ID: ${{ github.run_id }}
TESTRAIL_URL: https://elementqa.testrail.io
TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
run: |
echo '{"name": "element-web cypress '$TEST_ID'", "suite_id": '$TESTRAIL_SUITE_ID' }' > body.json # TODO add description with more context?
RUN_ID=`curl -X POST -d @body.json -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/add_run/$TESTRAIL_PROJECT_ID" | jq '.id'`
PROJECT_NAME=`curl -X GET -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/get_project/$TESTRAIL_PROJECT_ID" | jq '.name'`
echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
echo "project_name=$PROJECT_NAME" >> $GITHUB_OUTPUT
- name: setup python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- run: pip install trcli
- name: Upload junit files
env:
TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
TESTRAIL_URL: https://elementqa.testrail.io
TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
run: |
for file in results-*.xml; do
echo "Handling $file"
trcli -y -h $TESTRAIL_URL \
--project-id $TESTRAIL_PROJECT_ID \
--project ${{ steps.testrailprep.outputs.project_name }} \
--username $TESTRAIL_USER \
--password $TESTRAIL_API_KEY \
parse_junit \
--run-id $TESTRAIL_RUN_ID \
--suite-id $TESTRAIL_SUITE_ID \
--title "if you see this please check cypress build for run id not being provisioned" \
-f $file
done
- name: Close test run
id: testrailpost
if: always()
env:
TESTRAIL_URL: https://elementqa.testrail.io
TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
run: |
CLOSE_RESPONSE=`curl -X POST -d '{}' -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/close_run/$TESTRAIL_RUN_ID"`
if [ ! "0" == "`echo $CLOSE_RESPONSE | jq .untested_count`" ] ; then echo "::warning title=Missing Tests::Testrail reported some cypress tests were not run. $CLOSE_RESPONSE"; fi

6
cypress-ci-reporter-config.json Normal file
View file

@ -0,0 +1,6 @@
{
"reporterEnabled": "spec, mocha-junit-reporter",
"mochaJunitReporterReporterOptions": {
"mochaFile": "cypress/results/results-[hash].xml"
}
}

2
package.json
View file

@ -179,6 +179,7 @@
"chokidar": "^3.5.1",
"cypress": "^11.0.0",
"cypress-axe": "^1.0.0",
"cypress-multi-reporters": "^1.6.1",
"cypress-real-events": "^1.7.1",
"enzyme": "^3.11.0",
"enzyme-to-json": "^3.6.2",
@ -202,6 +203,7 @@
"jest-raw-loader": "^1.0.1",
"matrix-mock-request": "^2.5.0",
"matrix-web-i18n": "^1.3.0",
"mocha-junit-reporter": "^2.2.0",
"node-fetch": "2",
"postcss-scss": "^4.0.4",
"prettier": "2.8.0",

53
yarn.lock
View file

@ -3047,6 +3047,11 @@ char-regex@^1.0.2:
resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
charenc@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==
check-more-types@^2.24.0:
version "2.24.0"
resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
@ -3369,6 +3374,11 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
shebang-command "^2.0.0"
which "^2.0.1"
crypt@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==
css-box-model@^1.2.0:
version "1.2.1"
resolved "https://registry.yarnpkg.com/css-box-model/-/css-box-model-1.2.1.tgz#59951d3b81fd6b2074a62d49444415b0d2b4d7c1"
@ -3444,6 +3454,14 @@ cypress-axe@^1.0.0:
resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-1.0.0.tgz#ab4e9486eaa3bb956a90a1ae40d52df42827b4f0"
integrity sha512-QBlNMAd5eZoyhG8RGGR/pLtpHGkvgWXm2tkP68scJ+AjYiNNOlJihxoEwH93RT+rWOLrefw4iWwEx8kpEcrvJA==
cypress-multi-reporters@^1.6.1:
version "1.6.1"
resolved "https://registry.yarnpkg.com/cypress-multi-reporters/-/cypress-multi-reporters-1.6.1.tgz#515b891f6c80e0700068efb03ab9d55388399c95"
integrity sha512-FPeC0xWF1N6Myrwc2m7KC0xxlrtG8+x4hlsPFBDRWP8u/veR2x90pGaH3BuJfweV7xoQ4Zo85Qjhu3fgZGrBQQ==
dependencies:
debug "^4.1.1"
lodash "^4.17.15"
cypress-real-events@^1.7.1:
version "1.7.4"
resolved "https://registry.yarnpkg.com/cypress-real-events/-/cypress-real-events-1.7.4.tgz#87780ee0f6669ee30ce52016c4bfc94d094a6e3d"
@ -5183,6 +5201,11 @@ is-boolean-object@^1.0.1, is-boolean-object@^1.1.0:
call-bind "^1.0.2"
has-tostringtag "^1.0.0"
is-buffer@^1.1.5, is-buffer@~1.1.6:
version "1.1.6"
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
is-builtin-module@^3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.0.tgz#bb0310dfe881f144ca83f30100ceb10cf58835e0"
@ -6403,6 +6426,15 @@ matrix-widget-api@^1.0.0, matrix-widget-api@^1.1.1:
"@types/events" "^3.0.0"
events "^3.2.0"
md5@^2.3.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f"
integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==
dependencies:
charenc "0.0.2"
crypt "0.0.2"
is-buffer "~1.1.6"
mdurl@~1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
@ -6513,11 +6545,30 @@ minimist@>=1.2.2, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
mixin-deep@^1.2.0:
version "1.3.2"
resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566"
integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==
dependencies:
for-in "^1.0.2"
is-extendable "^1.0.1"
mkdirp@1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
mocha-junit-reporter@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/mocha-junit-reporter/-/mocha-junit-reporter-2.2.0.tgz#2663aaf25a98989ac9080c92b19e54209e539f67"
integrity sha512-W83Ddf94nfLiTBl24aS8IVyFvO8aRDLlCvb+cKb/VEaN5dEbcqu3CXiTe8MQK2DvzS7oKE1RsFTxzN302GGbDQ==
dependencies:
debug "^4.3.4"
md5 "^2.3.0"
mkdirp "~1.0.4"
strip-ansi "^6.0.1"
xml "^1.0.1"
moo-color@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/moo-color/-/moo-color-1.0.3.tgz#d56435f8359c8284d83ac58016df7427febece74"
@ -8729,7 +8780,7 @@ xml-name-validator@^4.0.0:
resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"
integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==
xml@1.0.1:
xml@1.0.1, xml@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==