feat: add support for configuring failure threshold (#1879)

Author: WickyNilliams

Diff for: .changeset/unlucky-turtles-fold.md

@@ -0,0 +1,5 @@
+---
+'@web/test-runner-visual-regression': minor
+---
+
+Add visual regression config option to set a failure threshold in pixels or percent

Diff for: packages/test-runner-visual-regression/src/config.ts

@@ -28,6 +28,7 @@ export type OptionalImage = Buffer | undefined | Promise<Buffer | undefined>;
 
 export interface DiffResult {
   diffPercentage: number;
+  diffPixels: number;
   diffImage: Buffer;
   error: string;
 }
@@ -60,6 +61,18 @@ export interface VisualRegressionPluginOptions {
    */
   diffOptions: PixelMatchOptions;
 
+  /**
+   * The threshold after which a diff is considered a failure, depending on the failureThresholdType.
+   * For `failureThresholdType` of "percentage", this should be a number between 0-100.
+   * For `failureThresholdType` of "pixels", this should be a positive integer.
+   */
+  failureThreshold: number;
+
+  /**
+   * The type of threshold that would trigger a failure.
+   */
+  failureThresholdType: 'percent' | 'pixel';
+
   /**
    * Returns the name of the baseline image file. The name
    * is a path relative to the baseDir
@@ -119,6 +132,9 @@ export const defaultOptions: VisualRegressionPluginOptions = {
   baseDir: 'screenshots',
   diffOptions: {},
 
+  failureThreshold: 0,
+  failureThresholdType: 'percent',
+
   getBaselineName: ({ browser, name }) => path.join(browser, 'baseline', name),
   getDiffName: ({ browser, name }) => path.join(browser, 'failed', `${name}-diff`),
   getFailedName: ({ browser, name }) => path.join(browser, 'failed', name),

Diff for: packages/test-runner-visual-regression/src/pixelMatchDiff.ts

@@ -12,7 +12,7 @@ export function pixelMatchDiff({ baselineImage, image, options }: DiffArgs): Dif
   if (basePng.width !== png.width || basePng.height !== png.height) {
     error =
       `Screenshot is not the same width and height as the baseline. ` +
-      `Baseline: { width: ${basePng.width}, height: ${basePng.height} }` +
+      `Baseline: { width: ${basePng.width}, height: ${basePng.height} } ` +
       `Screenshot: { width: ${png.width}, height: ${png.height} }`;
     width = Math.max(basePng.width, png.width);
     height = Math.max(basePng.height, png.height);
@@ -33,5 +33,6 @@ export function pixelMatchDiff({ baselineImage, image, options }: DiffArgs): Dif
     error,
     diffImage: PNG.sync.write(diff),
     diffPercentage,
+    diffPixels: numDiffPixels,
   };
 }

Diff for: packages/test-runner-visual-regression/src/visualDiffCommand.ts

@@ -1,6 +1,6 @@
 import path from 'path';
 
-import { VisualRegressionPluginOptions } from './config';
+import { VisualRegressionPluginOptions, DiffResult } from './config';
 import { VisualRegressionError } from './VisualRegressionError';
 
 function resolveImagePath(baseDir: string, name: string) {
@@ -22,6 +22,30 @@ export interface VisualDiffCommandContext {
   testFile: string;
 }
 
+function passesFailureThreshold(
+  { diffPercentage, diffPixels }: DiffResult,
+  { failureThresholdType, failureThreshold }: VisualRegressionPluginOptions,
+): { passed: boolean; message?: string } {
+  if (failureThresholdType === 'percent') {
+    return diffPercentage <= failureThreshold
+      ? { passed: true }
+      : {
+          passed: false,
+          // if diff is suitably small, output raw value, otherwise to two decimal points.
+          // this avoids outputting a failure value of "0.00%"
+          message: `${diffPercentage < 0.005 ? diffPercentage : diffPercentage.toFixed(2)}%`,
+        };
+  }
+
+  if (failureThresholdType === 'pixel') {
+    return diffPixels <= failureThreshold
+      ? { passed: true }
+      : { passed: false, message: `${diffPixels} pixels` };
+  }
+
+  throw new VisualRegressionError(`Unrecognized failureThresholdType: ${failureThresholdType}`);
+}
+
 export async function visualDiffCommand(
   options: VisualRegressionPluginOptions,
   image: Buffer,
@@ -79,22 +103,28 @@ export async function visualDiffCommand(
     };
   }
 
-  const { diffImage, diffPercentage, error } = await options.getImageDiff({
+  const result = await options.getImageDiff({
     name,
     baselineImage,
     image,
     options: options.diffOptions,
   });
 
+  const { error, diffImage } = result;
+
   if (error) {
     // The diff has failed, be sure to save the new image.
     await saveFailed();
     await saveDiff();
 
-    throw new VisualRegressionError(error);
+    return {
+      passed: false,
+      errorMessage: error,
+      diffPercentage: -1,
+    };
   }
 
-  const passed = diffPercentage === 0;
+  const { passed, message } = passesFailureThreshold(result, options);
 
   if (!passed) {
     await saveDiff();
@@ -104,14 +134,9 @@ export async function visualDiffCommand(
     await saveFailed();
   }
 
-  // if diff is suitably small, output raw value, otherwise to two decimal points.
-  // this avoids outputting a message like "New screenshot is 0.00% different"
-  const diffPercentageToDisplay =
-    diffPercentage < 0.005 ? diffPercentage : diffPercentage.toFixed(2);
-
   return {
     errorMessage: !passed
-      ? `Visual diff failed. New screenshot is ${diffPercentageToDisplay}% different.\nSee diff for details: ${diffFilePath}`
+      ? `Visual diff failed. New screenshot is ${message} different.\nSee diff for details: ${diffFilePath}`
       : undefined,
     diffPercentage: -1,
     passed,