[Github] Improve test workflow to merge blob reports from all shards (#6687)

* [Github] Improved test workflow to merge blobs into 1 big test

* fixed workflow not checking out + test failure behavior

* fixed the thingy to still merge reports when a shard fails

* Fixed workflow being skipped

* Fixed inverted conditional

* Removed failing test demo

* updated workflow to hopefully not clone entire repo

* Add .nvmrc to sparse-checkout configuration

* fail workflow output test

* Include custom reporters, vite/vitest config in sparse checkout selection

* Add src/plugins/vite to sparse checkout

* Revert to checking out full repo and remove change to instruct test

Unfortunately using sparse checkout won't work

* Remove redundant comment

* Try not using recursive submodules when cloning locales

* re-addede recurse submoudules

* actually disabled it

* Remove extra newline

---------

Co-authored-by: NightKev <34855794+DayKev@users.noreply.github.com>
Co-authored-by: Sirz Benjie <142067137+SirzBenjie@users.noreply.github.com>

.github/workflows/test-shard-template.yml

@@ -3,27 +3,19 @@ name: Test Template
 on:
   workflow_call:
     inputs:
-      project:
-        required: true
-        type: string
       shard:
         required: true
         type: number
       totalShards:
         required: true
         type: number
-      skip:
-        required: true
-        type: boolean
-        default: false
 
 jobs:
   test:
-    # We can't use dynmically named jobs until https://github.com/orgs/community/discussions/13261 is implemented
+    # We can't use dynamically named jobs until https://github.com/orgs/community/discussions/13261 is implemented
     name: Shard
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    if: ${{ !inputs.skip }}
     steps:
       - name: Check out Git repository
         uses: actions/checkout@v4.2.2
@@ -43,4 +35,20 @@ jobs:
         run: pnpm i
 
       - name: Run tests
-        run: pnpm test:silent --shard=${{ inputs.shard }}/${{ inputs.totalShards }}
+        run: >
+          pnpm test:silent
+          --shard=${{ inputs.shard }}/${{ inputs.totalShards }}
+          --reporter=blob
+          --outputFile=test-results/blob-${{ inputs.shard }}.json || true
+
+      # NB: This CANNOT be made into a job output due to not being extractable from the matrix:
+      # https://github.com/orgs/community/discussions/17245
+      - name: Upload test result blobs
+        uses: actions/upload-artifact@v4
+        with:
+          name: shard-${{ inputs.shard }}-blob
+          path: test-results/blob-${{ inputs.shard }}.json
+          retention-days: 1 # don't need to keep for very long since they get merged afterwards
+          overwrite: true
+          if-no-files-found: error
+        if: ${{ !cancelled() }}

.github/workflows/tests.yml

@@ -26,7 +26,7 @@ jobs:
     outputs:
       all: ${{ steps.filter.outputs.all }}
     steps:
-      - name: checkout
+      - name: Checkout GitHub repository
         uses: actions/checkout@v4
         with:
           sparse-checkout: |
@@ -41,6 +41,7 @@ jobs:
   run-tests:
     name: Run Tests
     needs: check-path-change-filter
+    if: ${{ needs.check-path-change-filter.outputs.all == 'true'}}
     strategy:
       # don't stop upon 1 shard failing
       fail-fast: false
@@ -48,7 +49,39 @@ jobs:
         shard: [1, 2, 3, 4, 5]
     uses: ./.github/workflows/test-shard-template.yml
     with:
-      project: main
       shard: ${{ matrix.shard }}
       totalShards: 5
-      skip: ${{ needs.check-path-change-filter.outputs.all != 'true'}}
+
+  check-results:
+    name: Check Test Results
+    timeout-minutes: 8
+    needs: run-tests
+    runs-on: ubuntu-latest
+    if: ${{ needs.run-tests.result != 'skipped' && needs.run-tests.result != 'cancelled' }}
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v4.2.2
+        with:
+          submodules: "recursive"
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+
+      - name: Install Packages
+        run: pnpm i
+
+      - name: Download blob artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: shard-?-blob
+          path: test-results
+          merge-multiple: true
+
+      - name: Merge blobs
+        run: pnpm test:silent --merge-reports=test-results