Report cypress test results into testrail. (#9685)

* Reduce cypress size/scope

* Emit cypress reports in junit-compatible format.

* Clean implementation of upload

* Append a warning if not all tests that testrail knows of are run.

* Only run testrail if on develop (like percy).

* Re-enable full test run.

* Update cypress.config.ts

* Ensure SUITE_ID is passed.

* Fix quoting on SUITE_ID, add environment

* Update .github/workflows/cypress.yaml

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

* re-add mocha-junit-reporter (was dropped in the merge)

* Prettifier on newly created file.

Co-authored-by: Michael Telatynski <7t3chguy@gmail.com>

.github/workflows/cypress.yaml

@@ -22,6 +22,7 @@ jobs:
             commit_author: ${{ steps.commit.outputs.author }}
             commit_email: ${{ steps.commit.outputs.email }}
             percy_enable: ${{ steps.percy.outputs.value || '1' }}
+            testrail_enable: ${{ steps.testrail.outputs.value || '1' }}
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
             # This provides an easy link to this workflow_run from the PR before Cypress is done.
@@ -63,6 +64,14 @@ jobs:
                   !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Needs-Percy')
               run: echo "::set-output name=value::0"
 
+            # Only run Testrail when it is demanded or on develop
+            - name: Disable Testrail if not needed
+              id: testrail
+              if: |
+                  github.event.workflow_run.event == 'pull_request' &&
+                  !contains(fromJSON(steps.prdetails.outputs.data).labels.*.name, 'X-Send-Testrail')
+              run: echo "::set-output name=value::0"
+
             - name: Generate unique ID 💎
               id: uuid
               run: echo "::set-output name=value::sha-$GITHUB_SHA-time-$(date +"%s")"
@@ -114,6 +123,7 @@ jobs:
                       #parallel: true
                       #command-prefix: 'yarn percy exec --parallel --'
                   command-prefix: "yarn percy exec --"
+                  config: '{"reporter":"cypress-multi-reporters", "reporterOptions": { "configFile": "cypress-ci-reporter-config.json" } }'
                   ci-build-id: ${{ needs.prepare.outputs.uuid }}
               env:
                   # pass the Dashboard record key as an environment variable
@@ -158,6 +168,12 @@ jobs:
                       cypress/videos
                       cypress/synapselogs
 
+            - name: Upload reports
+              if: always()
+              uses: actions/upload-artifact@v2
+              with:
+                  name: cypress-junit
+                  path: cypress/results
     report:
         name: Report results
         needs: tests
@@ -173,3 +189,68 @@ jobs:
                   context: ${{ github.workflow }} / cypress (${{ github.event.workflow_run.event }} => ${{ github.event_name }})
                   sha: ${{ github.event.workflow_run.head_sha }}
                   target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    testrail:
+        name: Report results to testrail
+        needs:
+            - prepare
+            - tests
+        environment: Testrail
+        runs-on: ubuntu-latest
+        if: ${{ needs.prepare.outputs.testrail_enable }}
+        steps:
+            - uses: actions/download-artifact@v3
+              with:
+                  name: cypress-junit
+            - name: Prepare testrail upload config
+              id: testrailprep
+              env:
+                  TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
+                  TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
+                  TEST_ID: ${{ github.run_id }}
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+              run: |
+                  echo '{"name": "element-web cypress '$TEST_ID'", "suite_id": '$TESTRAIL_SUITE_ID' }' > body.json # TODO add description with more context?
+                  RUN_ID=`curl -X POST -d @body.json -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/add_run/$TESTRAIL_PROJECT_ID" | jq '.id'`
+                  PROJECT_NAME=`curl -X GET -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/get_project/$TESTRAIL_PROJECT_ID" | jq '.name'`
+                  echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT
+                  echo "project_name=$PROJECT_NAME" >> $GITHUB_OUTPUT
+            - name: setup python
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.11"
+            - run: pip install trcli
+            - name: Upload junit files
+              env:
+                  TESTRAIL_PROJECT_ID: ${{ secrets.TESTRAIL_PROJECT_ID }}
+                  TESTRAIL_SUITE_ID: ${{ secrets.TESTRAIL_SUITE_ID }}
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+                  TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
+              run: |
+                  for file in results-*.xml; do
+                      echo "Handling $file"
+                      trcli -y -h $TESTRAIL_URL \
+                                  --project-id $TESTRAIL_PROJECT_ID \
+                                  --project ${{ steps.testrailprep.outputs.project_name }} \
+                                  --username $TESTRAIL_USER \
+                                  --password $TESTRAIL_API_KEY \
+                                  parse_junit \
+                                  --run-id $TESTRAIL_RUN_ID \
+                                  --suite-id $TESTRAIL_SUITE_ID \
+                                  --title "if you see this please check cypress build for run id not being provisioned" \
+                                  -f $file
+                  done
+            - name: Close test run
+              id: testrailpost
+              if: always()
+              env:
+                  TESTRAIL_URL: https://elementqa.testrail.io
+                  TESTRAIL_USER: ${{ secrets.TESTRAIL_USER }}
+                  TESTRAIL_API_KEY: ${{ secrets.TESTRAIL_API_KEY }}
+                  TESTRAIL_RUN_ID: ${{ steps.testrailprep.outputs.run_id }}
+              run: |
+                  CLOSE_RESPONSE=`curl -X POST -d '{}' -u "$TESTRAIL_USER:$TESTRAIL_API_KEY" -H "Content-Type: application/json" "$TESTRAIL_URL/index.php?/api/v2/close_run/$TESTRAIL_RUN_ID"`
+                  if [ ! "0" == "`echo $CLOSE_RESPONSE | jq .untested_count`" ] ; then echo "::warning title=Missing Tests::Testrail reported some cypress tests were not run. $CLOSE_RESPONSE"; fi

cypress-ci-reporter-config.json

@@ -0,0 +1,6 @@
+{
+    "reporterEnabled": "spec, mocha-junit-reporter",
+    "mochaJunitReporterReporterOptions": {
+        "mochaFile": "cypress/results/results-[hash].xml"
+    }
+}

package.json

@@ -179,6 +179,7 @@
         "chokidar": "^3.5.1",
         "cypress": "^11.0.0",
         "cypress-axe": "^1.0.0",
+        "cypress-multi-reporters": "^1.6.1",
         "cypress-real-events": "^1.7.1",
         "enzyme": "^3.11.0",
         "enzyme-to-json": "^3.6.2",
@@ -202,6 +203,7 @@
         "jest-raw-loader": "^1.0.1",
         "matrix-mock-request": "^2.5.0",
         "matrix-web-i18n": "^1.3.0",
+        "mocha-junit-reporter": "^2.2.0",
         "node-fetch": "2",
         "postcss-scss": "^4.0.4",
         "prettier": "2.8.0",

yarn.lock

@@ -3047,6 +3047,11 @@ char-regex@^1.0.2:
   resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf"
   integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
 
+charenc@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
+  integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==
+
 check-more-types@^2.24.0:
   version "2.24.0"
   resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600"
@@ -3369,6 +3374,11 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+crypt@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
+  integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==
+
 css-box-model@^1.2.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/css-box-model/-/css-box-model-1.2.1.tgz#59951d3b81fd6b2074a62d49444415b0d2b4d7c1"
@@ -3444,6 +3454,14 @@ cypress-axe@^1.0.0:
   resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-1.0.0.tgz#ab4e9486eaa3bb956a90a1ae40d52df42827b4f0"
   integrity sha512-QBlNMAd5eZoyhG8RGGR/pLtpHGkvgWXm2tkP68scJ+AjYiNNOlJihxoEwH93RT+rWOLrefw4iWwEx8kpEcrvJA==
 
+cypress-multi-reporters@^1.6.1:
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/cypress-multi-reporters/-/cypress-multi-reporters-1.6.1.tgz#515b891f6c80e0700068efb03ab9d55388399c95"
+  integrity sha512-FPeC0xWF1N6Myrwc2m7KC0xxlrtG8+x4hlsPFBDRWP8u/veR2x90pGaH3BuJfweV7xoQ4Zo85Qjhu3fgZGrBQQ==
+  dependencies:
+    debug "^4.1.1"
+    lodash "^4.17.15"
+
 cypress-real-events@^1.7.1:
   version "1.7.4"
   resolved "https://registry.yarnpkg.com/cypress-real-events/-/cypress-real-events-1.7.4.tgz#87780ee0f6669ee30ce52016c4bfc94d094a6e3d"
@@ -5183,6 +5201,11 @@ is-boolean-object@^1.0.1, is-boolean-object@^1.1.0:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
+is-buffer@^1.1.5, is-buffer@~1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
+  integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==
+
 is-builtin-module@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.0.tgz#bb0310dfe881f144ca83f30100ceb10cf58835e0"
@@ -6403,6 +6426,15 @@ matrix-widget-api@^1.0.0, matrix-widget-api@^1.1.1:
     "@types/events" "^3.0.0"
     events "^3.2.0"
 
+md5@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f"
+  integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==
+  dependencies:
+    charenc "0.0.2"
+    crypt "0.0.2"
+    is-buffer "~1.1.6"
+
 mdurl@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
@@ -6513,11 +6545,30 @@ minimist@>=1.2.2, minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18"
   integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==
 
+mixin-deep@^1.2.0:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566"
+  integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==
+  dependencies:
+    for-in "^1.0.2"
+    is-extendable "^1.0.1"
+
 mkdirp@1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
+mocha-junit-reporter@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/mocha-junit-reporter/-/mocha-junit-reporter-2.2.0.tgz#2663aaf25a98989ac9080c92b19e54209e539f67"
+  integrity sha512-W83Ddf94nfLiTBl24aS8IVyFvO8aRDLlCvb+cKb/VEaN5dEbcqu3CXiTe8MQK2DvzS7oKE1RsFTxzN302GGbDQ==
+  dependencies:
+    debug "^4.3.4"
+    md5 "^2.3.0"
+    mkdirp "~1.0.4"
+    strip-ansi "^6.0.1"
+    xml "^1.0.1"
+
 moo-color@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/moo-color/-/moo-color-1.0.3.tgz#d56435f8359c8284d83ac58016df7427febece74"
@@ -8729,7 +8780,7 @@ xml-name-validator@^4.0.0:
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835"
   integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==
 
-xml@1.0.1:
+xml@1.0.1, xml@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"
   integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==