From b728b2743565bfc1d21f3a2d0dac965578a71fa6 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Thu, 15 Dec 2022 12:04:03 +0000 Subject: [PATCH] Report cypress test results into testrail. (#9685) * Reduce cypress size/scope * Emit cypress reports in junit-compatible format. * Clean implementation of upload * Append a warning if not all tests that testrail knows of are run. * Only run testrail if on develop (like percy). * Re-enable full test run. * Update cypress.config.ts * Ensure SUITE_ID is passed. * Fix quoting on SUITE_ID, add environment * Update .github/workflows/cypress.yaml Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> * re-add mocha-junit-reporter (was dropped in the merge) * Prettifier on newly created file. Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> --- .github/workflows/cypress.yaml | 81 +++++++++++++++++++++++++++++++++ cypress-ci-reporter-config.json | 6 +++ package.json | 2 + yarn.lock | 53 ++++++++++++++++++++- 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 cypress-ci-reporter-config.json diff --git a/.github/workflows/cypress.yaml b/.github/workflows/cypress.yaml index a6eba119b6..c8fbfd9009 100644 --- a/.github/workflows/cypress.yaml +++ b/.github/workflows/cypress.yaml @@ -22,6 +22,7 @@ jobs: commit_author: ${{ steps.commit.outputs.author }} commit_email: ${{ steps.commit.outputs.email }} percy_enable: ${{ steps.percy.outputs.value || '1' }} + testrail_enable: ${{ steps.testrail.outputs.value || '1' }} steps: # We create the status here and then update it to success/failure in the `report` stage # This provides an easy link to this workflow_run from the PR before Cypress is done. @@ -63,6 +64,14 @@ jobs: !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy') run: echo "::set-output name=value::0" + # Only run Testrail when it is demanded or on develop + - name: Disable Testrail if not needed + id: testrail + if: | + github.event.workflow_run.event == 'pull_request' && + !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Send-Testrail') + run: echo "::set-output name=value::0" + - name: Generate unique ID 💎 id: uuid run: echo "::set-output name=value::sha-$GITHUB_SHA-time-$(date +"%s")" @@ -114,6 +123,7 @@ jobs: #parallel: true #command-prefix: 'yarn percy exec --parallel --' command-prefix: "yarn percy exec --" + config: '{"reporter":"cypress-multi-reporters", "reporterOptions": { "configFile": "cypress-ci-reporter-config.json" } }' ci-build-id: ${{ needs.prepare.outputs.uuid }} env: # pass the Dashboard record key as an environment variable @@ -158,6 +168,12 @@ jobs: cypress/videos cypress/synapselogs + - name: Upload reports + if: always() + uses: actions/upload-artifact@v2 + with: + name: cypress-junit + path: cypress/results report: name: Report results needs: tests @@ -173,3 +189,68 @@ jobs: context: ${{ github.workflow }} / cypress (${{ github.event.workflow_run.event }} => ${{ github.event_name }}) sha: ${{ github.event.workflow_run.head_sha }} target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + testrail: + name: Report results to testrail + needs: + - prepare + - tests + environment: Testrail + runs-on: ubuntu-latest + if: ${{ needs.prepare.outputs.testrail_enable }} + steps: + - uses: actions/download-artifact@v3 + with: + name: cypress-junit + - name: Prepare testrail upload config + id: testrailprep + env: + TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }} + TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }} + TEST_ID: ${{ github.run_id }} + TESTRAIL_URL: https://elementqa.testrail.io + TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }} + TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} + run: | + echo '{"name": "element-web cypress '$TEST_ID'", "suite_id": '$TESTRAIL_SUITE_ID' }' > body.json # TODO add description with more context? + RUN_ID=`curl -X POST -d @body.json -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/add_run/$TESTRAIL_PROJECT_ID" | jq '.id'` + PROJECT_NAME=`curl -X GET -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/get_project/$TESTRAIL_PROJECT_ID" | jq '.name'` + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + echo "project_name=$PROJECT_NAME" >> $GITHUB_OUTPUT + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - run: pip install trcli + - name: Upload junit files + env: + TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }} + TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }} + TESTRAIL_URL: https://elementqa.testrail.io + TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }} + TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} + TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }} + run: | + for file in results-*.xml; do + echo "Handling $file" + trcli -y -h $TESTRAIL_URL \ + --project-id $TESTRAIL_PROJECT_ID \ + --project ${{ steps.testrailprep.outputs.project_name }} \ + --username $TESTRAIL_USER \ + --password $TESTRAIL_API_KEY \ + parse_junit \ + --run-id $TESTRAIL_RUN_ID \ + --suite-id $TESTRAIL_SUITE_ID \ + --title "if you see this please check cypress build for run id not being provisioned" \ + -f $file + done + - name: Close test run + id: testrailpost + if: always() + env: + TESTRAIL_URL: https://elementqa.testrail.io + TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }} + TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }} + TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }} + run: | + CLOSE_RESPONSE=`curl -X POST -d '{}' -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/close_run/$TESTRAIL_RUN_ID"` + if [ ! "0" == "`echo $CLOSE_RESPONSE | jq .untested_count`" ] ; then echo "::warning title=Missing Tests::Testrail reported some cypress tests were not run. $CLOSE_RESPONSE"; fi diff --git a/cypress-ci-reporter-config.json b/cypress-ci-reporter-config.json new file mode 100644 index 0000000000..091f37b37e --- /dev/null +++ b/cypress-ci-reporter-config.json @@ -0,0 +1,6 @@ +{ + "reporterEnabled": "spec, mocha-junit-reporter", + "mochaJunitReporterReporterOptions": { + "mochaFile": "cypress/results/results-[hash].xml" + } +} diff --git a/package.json b/package.json index 37e7ffe7a4..01bb51aa8c 100644 --- a/package.json +++ b/package.json @@ -179,6 +179,7 @@ "chokidar": "^3.5.1", "cypress": "^11.0.0", "cypress-axe": "^1.0.0", + "cypress-multi-reporters": "^1.6.1", "cypress-real-events": "^1.7.1", "enzyme": "^3.11.0", "enzyme-to-json": "^3.6.2", @@ -202,6 +203,7 @@ "jest-raw-loader": "^1.0.1", "matrix-mock-request": "^2.5.0", "matrix-web-i18n": "^1.3.0", + "mocha-junit-reporter": "^2.2.0", "node-fetch": "2", "postcss-scss": "^4.0.4", "prettier": "2.8.0", diff --git a/yarn.lock b/yarn.lock index 932aef3134..ae4bd1f88d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3047,6 +3047,11 @@ char-regex@^1.0.2: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== +charenc@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" + integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== + check-more-types@^2.24.0: version "2.24.0" resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" @@ -3369,6 +3374,11 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +crypt@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" + integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== + css-box-model@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/css-box-model/-/css-box-model-1.2.1.tgz#59951d3b81fd6b2074a62d49444415b0d2b4d7c1" @@ -3444,6 +3454,14 @@ cypress-axe@^1.0.0: resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-1.0.0.tgz#ab4e9486eaa3bb956a90a1ae40d52df42827b4f0" integrity sha512-QBlNMAd5eZoyhG8RGGR/pLtpHGkvgWXm2tkP68scJ+AjYiNNOlJihxoEwH93RT+rWOLrefw4iWwEx8kpEcrvJA== +cypress-multi-reporters@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/cypress-multi-reporters/-/cypress-multi-reporters-1.6.1.tgz#515b891f6c80e0700068efb03ab9d55388399c95" + integrity sha512-FPeC0xWF1N6Myrwc2m7KC0xxlrtG8+x4hlsPFBDRWP8u/veR2x90pGaH3BuJfweV7xoQ4Zo85Qjhu3fgZGrBQQ== + dependencies: + debug "^4.1.1" + lodash "^4.17.15" + cypress-real-events@^1.7.1: version "1.7.4" resolved "https://registry.yarnpkg.com/cypress-real-events/-/cypress-real-events-1.7.4.tgz#87780ee0f6669ee30ce52016c4bfc94d094a6e3d" @@ -5183,6 +5201,11 @@ is-boolean-object@^1.0.1, is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@^1.1.5, is-buffer@~1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-builtin-module@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.0.tgz#bb0310dfe881f144ca83f30100ceb10cf58835e0" @@ -6403,6 +6426,15 @@ matrix-widget-api@^1.0.0, matrix-widget-api@^1.1.1: "@types/events" "^3.0.0" events "^3.2.0" +md5@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" + integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== + dependencies: + charenc "0.0.2" + crypt "0.0.2" + is-buffer "~1.1.6" + mdurl@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e" @@ -6513,11 +6545,30 @@ minimist@>=1.2.2, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== +mixin-deep@^1.2.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566" + integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA== + dependencies: + for-in "^1.0.2" + is-extendable "^1.0.1" + mkdirp@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mocha-junit-reporter@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/mocha-junit-reporter/-/mocha-junit-reporter-2.2.0.tgz#2663aaf25a98989ac9080c92b19e54209e539f67" + integrity sha512-W83Ddf94nfLiTBl24aS8IVyFvO8aRDLlCvb+cKb/VEaN5dEbcqu3CXiTe8MQK2DvzS7oKE1RsFTxzN302GGbDQ== + dependencies: + debug "^4.3.4" + md5 "^2.3.0" + mkdirp "~1.0.4" + strip-ansi "^6.0.1" + xml "^1.0.1" + moo-color@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/moo-color/-/moo-color-1.0.3.tgz#d56435f8359c8284d83ac58016df7427febece74" @@ -8729,7 +8780,7 @@ xml-name-validator@^4.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== -xml@1.0.1: +xml@1.0.1, xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==