Setup Flaky Test monitoring (#774)

Adds an upload step so that we upload all our test results. Validated on [webapp](https://app.trunk-staging.io/trunk-staging-org/trunk-io/plugins/test-analytics?statuses=Broken%2CFlaky%2CNew). The next step once the results seem to be reliable would be to turn on test quarantining and run with that action (note that it does not support Windows).
trunk-io · May 9, 2024 · 6cfcfea · 6cfcfea
1 parent 22b9e81
commit 6cfcfea
Show file tree

Hide file tree

Showing 11 changed files with 137 additions and 6 deletions.
diff --git a/.github/actions/action_tests/action.yaml b/.github/actions/action_tests/action.yaml
@@ -54,6 +54,7 @@ runs:
       shell: bash
       working-directory: ${{ inputs.path }}
       env:
+        JEST_SUITE_NAME: Action Tests
         PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
 
@@ -64,10 +65,23 @@ runs:
         breakpoint-id: trunk-plugins-action-tests
         shell: bash
         working-directory: ${{ inputs.path }}
-        trunk-token: ${{ inputs.trunk-token  }}
+        trunk-token: ${{ inputs.trunk-token }}
         org: trunk-staging-org
         run:
           npm test ${{ inputs.append-args }} ${{ env.PLATFORM_APPEND_ARGS }} --passWithNoTests --ci
       env:
+        JEST_SUITE_NAME: Action Tests
         PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
+
+    - name: Upload results
+      # TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
+      if: "!cancelled() && runner.os == 'Linux'"
+      uses: trunk-io/analytics-uploader@main
+      with:
+        junit-paths: junit.xml
+        org-slug: trunk-staging-org
+        token: ${{ inputs.trunk-token }}
+      continue-on-error: true
+      env:
+        TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io
diff --git a/.github/actions/linter_tests/action.yaml b/.github/actions/linter_tests/action.yaml
@@ -24,8 +24,12 @@ inputs:
     description: Token to login for sourcery test
     required: true
   trunk-token:
-    description: CI debugger api token
+    description: CI debugger api token (org token)
     required: true
+  ref-type:
+    description: release or main
+    required: false
+    default: main
 
 runs:
   # TODO(Tyler): See if this can be converted to a js action
@@ -91,7 +95,10 @@ runs:
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
         SOURCERY_TOKEN: ${{ inputs.sourcery-token }}
         # Debug recurrent eslint circular JSON errors
-        DEBUG: Driver:eslint:*,Driver:cspell:*,Driver:nixpkgs-fmt:*
+        DEBUG: Driver:eslint:*,Driver:nixpkgs-fmt:*
+        JEST_SUITE_NAME: Linter Tests
+        JEST_JUNIT_SUITE_NAME:
+          "{title} ${{ runner.os }} ${{ inputs.ref-type }} ${{ inputs.linter-version }}"
 
     - name: Run plugin tests
       if: runner.os != 'Windows'
@@ -101,7 +108,7 @@ runs:
         breakpoint-id: trunk-plugins-linter-tests
         shell: bash
         working-directory: ${{ inputs.path }}
-        trunk-token: ${{ inputs.trunk-token  }}
+        trunk-token: ${{ inputs.trunk-token }}
         org: trunk-staging-org
         run: npm test ${{ inputs.append-args }} ${{ env.PLATFORM_APPEND_ARGS }} --ci --runInBand
       env:
@@ -110,3 +117,18 @@ runs:
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
         SOURCERY_TOKEN: ${{ inputs.sourcery-token }}
         DEBUG: Driver:nixpkgs-fmt:*, Driver:eslint:*
+        JEST_SUITE_NAME: Linter Tests
+        JEST_JUNIT_SUITE_NAME:
+          "{title} ${{ runner.os }} ${{ inputs.ref-type }} ${{ inputs.linter-version }}"
+
+    - name: Upload results
+      # TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
+      if: "!cancelled() && runner.os == 'Linux'"
+      uses: trunk-io/analytics-uploader@main
+      with:
+        junit-paths: junit.xml
+        org-slug: trunk-staging-org
+        token: ${{ inputs.trunk-token }}
+      continue-on-error: true
+      env:
+        TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io
diff --git a/.github/actions/tool_tests/action.yaml b/.github/actions/tool_tests/action.yaml
@@ -17,8 +17,12 @@ inputs:
     required: false
     default: tools --
   trunk-token:
-    description: CI debugger api token
+    description: CI debugger api token (org token)
     required: true
+  ref-type:
+    description: release or main
+    required: false
+    default: main
 
 runs:
   # TODO(Tyler): See if this can be converted to a js action
@@ -56,6 +60,8 @@ runs:
       env:
         PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
+        JEST_SUITE_NAME: Tool Tests
+        JEST_JUNIT_SUITE_NAME: "{title} ${{ runner.os }} ${{ inputs.ref-type }}"
 
     - name: Run plugin tests
       if: runner.os != 'Windows'
@@ -71,3 +77,17 @@ runs:
       env:
         PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
         PLUGINS_TEST_CLI_PATH: ${{ env.CLI_PATH }}
+        JEST_SUITE_NAME: Tool Tests
+        JEST_JUNIT_SUITE_NAME: "{title} ${{ runner.os }} ${{ inputs.ref-type }}"
+
+    - name: Upload results
+      # TODO(Tyler): Add upload on MacOS/Windows once the action supports it.
+      if: "!cancelled() && runner.os == 'Linux'"
+      uses: trunk-io/analytics-uploader@main
+      with:
+        junit-paths: junit.xml
+        org-slug: trunk-staging-org
+        token: ${{ inputs.trunk-token }}
+      continue-on-error: true
+      env:
+        TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -66,6 +66,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: ${{ matrix.linter-version }}
+          ref-type: main
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 
@@ -173,6 +174,7 @@ jobs:
         with:
           linter-version: ${{ matrix.linter-version }}
           append-args: linters -- --json --outputFile=${{ matrix.results-file }}-res.json
+          ref-type: release
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
 

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -150,6 +150,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: KnownGoodVersion
+          ref-type: main
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           append-args:
             ${{ needs.detect_changes.outputs.all-linters }} ${{
@@ -164,6 +165,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: Latest
+          ref-type: main
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           append-args: ${{ needs.detect_changes.outputs.linters-files }}
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
@@ -255,6 +257,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: Latest
+          ref-type: main
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           cli-path: ${{ github.workspace }}\trunk.ps1
           append-args: ${{needs.detect_changes.outputs.linters-files }} -- --maxWorkers=5

diff --git a/.github/workflows/repo_tests.reusable.yaml b/.github/workflows/repo_tests.reusable.yaml
@@ -38,5 +38,17 @@ jobs:
       - name: Run repo tests
         run: npm test tests/repo_tests --ci
         env:
+          JEST_SUITE_NAME: Repo Tests
           PLUGINS_TEST_CLI_VERSION: ${{ inputs.cli-version }}
           PLUGINS_TEST_CLI_PATH: ${{ inputs.cli-path }}
+
+      - name: Upload results
+        if: "!cancelled()"
+        uses: trunk-io/analytics-uploader@main
+        with:
+          junit-paths: junit.xml
+          org-slug: trunk-staging-org
+          token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}
+        continue-on-error: true
+        env:
+          TRUNK_PUBLIC_API_ADDRESS: https://api.trunk-staging.io
diff --git a/.github/workflows/upload_results.reusable.yaml b/.github/workflows/upload_results.reusable.yaml
@@ -255,6 +255,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: Latest
+          ref-type: main
           append-args: ${{ needs.upload_test_results.outputs.reruns }} -- -u
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           trunk-token: ${{ secrets.TRUNK_DEBUGGER_TOKEN }}

diff --git a/.github/workflows/windows_nightly.yaml b/.github/workflows/windows_nightly.yaml
@@ -44,6 +44,7 @@ jobs:
         uses: ./.github/actions/linter_tests
         with:
           linter-version: ${{ matrix.linter-version }}
+          ref-type: main
           sourcery-token: ${{ secrets.TRUNK_SOURCERY_TOKEN }}
           cli-path: ${{ github.workspace }}\trunk.ps1
           # manually specify more parallelism to avoid bottlenecks

diff --git a/jest.config.json b/jest.config.json
@@ -2,6 +2,6 @@
   "preset": "ts-jest",
   "modulePaths": ["<rootDir>"],
   "setupFilesAfterEnv": ["<rootDir>/tests/jest_setup.ts"],
-  "reporters": ["<rootDir>/tests/reporter/index.js", "default"],
+  "reporters": ["<rootDir>/tests/reporter/index.js", "default", "jest-junit"],
   "testPathIgnorePatterns": ["<rootDir>/node_modules/", "<rootDir>/repo-tools/"]
 }
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -30,6 +30,7 @@
     "eslint-plugin-simple-import-sort": "^12.1.0",
     "fast-sort": "^3.2.0",
     "jest": "^29.3.1",
+    "jest-junit": "^16.0.0",
     "jest-specific-snapshot": "^8.0.0",
     "semver": "^7.6.0",
     "simple-git": "^3.24.0",