fix: toJSON recursive error serialization (fix #5848) (#5884)

Author: eddienubes

Diff for: packages/utils/src/error.ts

@@ -45,7 +45,7 @@ export function serializeError(val: any, seen = new WeakMap()): any {
   if (typeof val.asymmetricMatch === 'function')
     return `${val.toString()} ${format(val.sample)}`
   if (typeof val.toJSON === 'function')
-    return val.toJSON()
+    return serializeError(val.toJSON(), seen)
 
   if (seen.has(val))
     return seen.get(val)

Diff for: test/core/test/serialize.test.ts

@@ -168,4 +168,32 @@ describe('error serialize', () => {
       message: 'test',
     })
   })
+
+  it('should serialize when toJSON does not return JSON-compatible object', () => {
+    class TestClass {}
+
+    const error = {
+      key: 'value',
+      // Still not serialized, thus second round of serialization is needed
+      toJSON() {
+        return {
+          key: 'value',
+          obj: {
+            fn: () => {},
+            class: TestClass,
+          },
+        }
+      },
+    }
+
+    const serialized = serializeError(error)
+
+    expect(serialized).toEqual({
+      key: 'value',
+      obj: {
+        fn: 'Function<fn>',
+        class: 'Function<TestClass>',
+      },
+    })
+  })
 })