fix: report results correctly when the browser crashes mid-test

cli/CHANGELOG.md

@@ -7,6 +7,10 @@ _Released 09/19/2023 (PENDING)_
 
  - Introduces new layout for Runs page providing additional run information. Addresses [#27203](https://github.com/cypress-io/cypress/issues/27203).
 
+**Bugfixes:**
+
+ - Enables test replay for executed specs in runs that have a spec that causes a browser crash. Addressed in [#27786](https://github.com/cypress-io/cypress/pull/27786)
+
 ## 13.2.0
 
 _Released 09/12/2023_

packages/server/lib/modes/run.ts

@@ -21,12 +21,13 @@ import random from '../util/random'
 import system from '../util/system'
 import chromePolicyCheck from '../util/chrome_policy_check'
 import type { SpecWithRelativeRoot, SpecFile, TestingType, OpenProjectLaunchOpts, FoundBrowser, BrowserVideoController, VideoRecording, ProcessOptions } from '@packages/types'
-import type { Cfg } from '../project-base'
+import type { Cfg, ProjectBase } from '../project-base'
 import type { Browser } from '../browsers/types'
 import * as printResults from '../util/print-run'
 import type { ProtocolManager } from '../cloud/protocol'
 import { telemetry } from '@packages/telemetry'
 import { CypressRunResult, createPublicBrowser, createPublicConfig, createPublicRunResults, createPublicSpec, createPublicSpecResults } from './results'
+import { endAfterError, exitEarly } from '../util/graceful_crash_handling'
 
 type SetScreenshotMetadata = (data: TakeScreenshotProps) => void
 type ScreenshotMetadata = ReturnType<typeof screenshotMetadata>
@@ -36,16 +37,9 @@ type BeforeSpecRun = any
 type AfterSpecRun = any
 type Project = NonNullable<ReturnType<typeof openProject['getProject']>>
 
-let exitEarly = (err) => {
-  debug('set early exit error: %s', err.stack)
-
-  earlyExitErr = err
-}
-let earlyExitErr: Error
 let currentSetScreenshotMetadata: SetScreenshotMetadata
 
 const debug = Debug('cypress:server:run')
-
 const DELAY_TO_LET_VIDEO_FINISH_MS = 1000
 
 const relativeSpecPattern = (projectRoot, pattern) => {
@@ -411,54 +405,22 @@ function launchBrowser (options: { browser: Browser, spec: SpecWithRelativeRoot,
   return openProject.launch(browser, spec, browserOpts)
 }
 
-function listenForProjectEnd (project, exit): Bluebird<any> {
+function listenForProjectEnd (project: ProjectBase, exit: boolean): Promise<any> {
   if (globalThis.CY_TEST_MOCK?.listenForProjectEnd) return Bluebird.resolve(globalThis.CY_TEST_MOCK.listenForProjectEnd)
 
-  return new Bluebird((resolve, reject) => {
-    if (exit === false) {
-      resolve = () => {
-        console.log('not exiting due to options.exit being false')
-      }
-    }
-
-    const onEarlyExit = function (err) {
-      if (err.isFatalApiErr) {
-        return reject(err)
-      }
-
-      console.log('')
-      errors.log(err)
-
-      const obj = {
-        error: errors.stripAnsi(err.message),
-        stats: {
-          failures: 1,
-          tests: 0,
-          passes: 0,
-          pending: 0,
-          suites: 0,
-          skipped: 0,
-          wallClockDuration: 0,
-          wallClockStartedAt: new Date().toJSON(),
-          wallClockEndedAt: new Date().toJSON(),
-        },
-      }
-
-      return resolve(obj)
-    }
-
-    project.once('end', (results) => resolve(results))
-
-    // if we already received a reason to exit early, go ahead and do it
-    if (earlyExitErr) {
-      return onEarlyExit(earlyExitErr)
-    }
-
-    // otherwise override exitEarly so we exit as soon as there is a reason
-    exitEarly = (err) => {
-      onEarlyExit(err)
-    }
-  })
+  return Promise.race([
+    new Promise((resolve) => {
+      project.once('end', (results) => {
+        debug('project ended with results %O', results)
+        if (exit === false) {
+          console.log('not exiting due to options.exit being false')
+        } else {
+          resolve(results)
+        }
+      })
+    }),
+    endAfterError(project, exit),
+  ])
 }
 
 async function waitForBrowserToConnect (options: { project: Project, socketId: string, onError: (err: Error) => void, spec: SpecWithRelativeRoot, isFirstSpecInBrowser: boolean, testingType: string, experimentalSingleTabRunMode: boolean, browser: Browser, screenshots: ScreenshotMetadata[], projectRoot: string, shouldLaunchNewTab: boolean, webSecurity: boolean, videoRecording?: VideoRecording, protocolManager?: ProtocolManager }) {

packages/server/lib/project-base.ts

@@ -376,7 +376,6 @@ export class ProjectBase extends EE {
       },
 
       onMocha: async (event, runnable) => {
-        debug('onMocha', event)
         // bail if we dont have a
         // reporter instance
         if (!reporterInstance) {
@@ -385,7 +384,12 @@ export class ProjectBase extends EE {
 
         reporterInstance.emit(event, runnable)
 
-        if (event === 'end') {
+        if (event === 'test:before:run') {
+          this.emit('test:before:run', {
+            runnable,
+            previousResults: reporterInstance?.results() || {},
+          })
+        } else if (event === 'end') {
           const [stats = {}] = await Promise.all([
             (reporterInstance != null ? reporterInstance.end() : undefined),
             this.server.end(),

packages/server/lib/types/reporter.ts

@@ -0,0 +1,48 @@
+interface ReporterTestAttempt {
+  state: 'skipped' | 'failed' | 'passed'
+  error: any
+  timings: any
+  failedFromHookId: any
+  wallClockStartedAt: Date
+  wallClockDuration: number
+  videoTimestamp: any
+}
+interface ReporterTest {
+  testId: string
+  title: string[]
+  state: 'skipped' | 'passed' | 'failed'
+  body: string
+  displayError: any
+  attempts: ReporterTestAttempt[]
+}
+
+export interface BaseReporterResults {
+  error?: string
+  stats: {
+    failures: number
+    tests: number
+    passes: number
+    pending: number
+    suites: number
+    skipped: number
+    wallClockDuration: number
+    wallClockStartedAt: string
+    wallClockEndedAt: string
+  }
+}
+
+export interface ReporterResults extends BaseReporterResults {
+  reporter: string
+  reporterStats: {
+    suites: number
+    tests: number
+    passes: number
+    pending: number
+    failures: number
+    start: string
+    end: string
+    duration: number
+  }
+  hooks: any[]
+  tests: ReporterTest[]
+}

packages/server/lib/util/graceful_crash_handling.ts

@@ -0,0 +1,132 @@
+import type { ProjectBase } from '../project-base'
+import type { BaseReporterResults, ReporterResults } from '../types/reporter'
+import * as errors from '../errors'
+import Debug from 'debug'
+
+const debug = Debug('cypress:util:crash_handling')
+
+let earlyExitError: Error
+
+let earlyExit = (err: Error) => {
+  debug('set early exit error: %s', err.stack)
+
+  earlyExitError = err
+}
+
+const patchRunResultsAfterCrash = (error: Error, reporterResults: ReporterResults, mostRecentRunnable: any): ReporterResults => {
+  const endTime: number = reporterResults?.stats?.wallClockEndedAt ? Date.parse(reporterResults?.stats?.wallClockEndedAt) : new Date().getTime()
+  const wallClockDuration = reporterResults?.stats?.wallClockStartedAt ?
+    endTime - Date.parse(reporterResults.stats.wallClockStartedAt) : 0
+  const endTimeStamp = new Date(endTime).toJSON()
+
+  // in crash situations, the most recent report will not have the triggering test
+  // so the results are manually patched, which produces the expected exit=1 and
+  // terminal output indicating the failed test
+  return {
+    ...reporterResults,
+    stats: {
+      ...reporterResults?.stats,
+      wallClockEndedAt: endTimeStamp,
+      wallClockDuration,
+      failures: (reporterResults?.stats?.failures ?? 0) + 1,
+      skipped: (reporterResults?.stats?.skipped ?? 1) - 1,
+    },
+    reporterStats: {
+      ...reporterResults?.reporterStats,
+      tests: (reporterResults?.reporterStats?.tests ?? 0) + 1, // crashed test does not increment this value
+      end: reporterResults?.reporterStats?.end || endTimeStamp,
+      duration: wallClockDuration,
+      failures: (reporterResults?.reporterStats?.failures ?? 0) + 1,
+    },
+    tests: (reporterResults?.tests || []).map((test) => {
+      if (test.testId === mostRecentRunnable.id) {
+        return {
+          ...test,
+          state: 'failed',
+          attempts: [
+            ...test.attempts.slice(0, -1),
+            {
+              ...test.attempts[test.attempts.length - 1],
+              state: 'failed',
+            },
+          ],
+        }
+      }
+
+      return test
+    }),
+    error: errors.stripAnsi(error.message),
+  }
+}
+
+const defaultStats = (error: Error): BaseReporterResults => {
+  return {
+    error: errors.stripAnsi(error.message),
+    stats: {
+      failures: 1,
+      tests: 0,
+      passes: 0,
+      pending: 0,
+      suites: 0,
+      skipped: 0,
+      wallClockDuration: 0,
+      wallClockStartedAt: new Date().toJSON(),
+      wallClockEndedAt: new Date().toJSON(),
+    },
+  }
+}
+
+export const endAfterError = (project: ProjectBase, exit: boolean): Promise<any> => {
+  let pendingRunnable: any
+  let intermediateStats: ReporterResults
+
+  project.on('test:before:run', ({
+    runnable,
+    previousResults,
+  }) => {
+    debug('preparing to run test, previous stats reported as %O', previousResults)
+
+    intermediateStats = previousResults
+    pendingRunnable = runnable
+  })
+
+  return new Promise((resolve, reject) => {
+    const patchedResolve = exit === false ? () => {
+      // eslint-disable-next-line no-console
+      console.log('not exiting due to options.exit being false')
+    } : resolve
+
+    const handleEarlyExit = (error: Error & { isFatalApiErr?: boolean }) => {
+      if (error.isFatalApiErr) {
+        debug('handling fatal api error', error)
+        reject(error)
+      } else {
+        debug('patching results and resolving')
+        // eslint-disable-next-line no-console
+        console.log('')
+        errors.log(error)
+        const results = (intermediateStats && pendingRunnable) ?
+          patchRunResultsAfterCrash(error, intermediateStats, pendingRunnable) :
+          defaultStats(error)
+
+        debug('resolving with patched results %O', results)
+        patchedResolve(results)
+      }
+    }
+
+    earlyExit = (error) => {
+      debug('handling early exit with error', error)
+      handleEarlyExit(error)
+    }
+
+    if (earlyExitError) {
+      handleEarlyExit(earlyExitError)
+    }
+  })
+}
+
+export const exitEarly = (error) => {
+  debug('exit early called', error)
+
+  return earlyExit(error)
+}