fix: use instanceof instead of constructor name in formatErrors (#16089)

## What

Fixes `formatErrors` silently dropping the `data` field from `APIError`
responses in production builds.

Closes #16050

## Why

`formatErrors` used `proto.constructor.name` string comparison to
identify `APIError` and `ValidationError` instances. In production,
webpack/terser minifies class names (e.g., `APIError` → `t`), causing
the check to fail. This silently drops `error.data`, breaking any
feature that relies on it — for example, 2FA flows that check
`data.requires2FA`.

The `APIErrorName`/`ValidationErrorName` module variables were designed
to handle this via dynamic reassignment in constructors, but minifiers
can inline the initial string value at import sites, defeating the
mechanism.

## Changes

- Replaced `proto.constructor.name === APIErrorName` /
`ValidationErrorName` checks with `instanceof APIError` / `instanceof
ValidationError`
- For the Mongoose `ValidationError` fallback (which isn't a Payload
class), switched to checking `incoming.name === 'ValidationError'` — a
string property that Mongoose sets explicitly and isn't affected by
minification
- Removed unused `APIErrorName` / `ValidationErrorName` imports

## Why instanceof is safe here

The original code avoided `instanceof` citing
[TypeScript#13965](microsoft/TypeScript#13965),
where `instanceof` fails for classes extending `Error` when transpiled
to ES5. However:

- Payload's `tsconfig` targets **ES2022**, so native class syntax is
preserved
- `instanceof` works correctly with ES2015+ classes extending `Error`
- The original workaround itself was broken by minification — a worse
failure mode

## Testing

- `npx tsc --noEmit` passes with no errors
- Verified that Payload's `ValidationError` (which has `data`) is caught
by the `instanceof` branch and never falls through to the Mongoose
branch
- The Mongoose `ValidationError` check uses the `name` property which
Mongoose sets explicitly (`this.name = 'ValidationError'`) — not
affected by minification

---------

Co-authored-by: Paul Popus <paul@payloadcms.com>

Author: mahmoodhamdi

packages/payload/src/errors/APIError.ts

@@ -1,7 +1,7 @@
 import { status as httpStatus } from 'http-status'
 
-// This gets dynamically reassigned during compilation
-export let APIErrorName = 'APIError'
+/** @deprecated Use `instanceof APIError` instead of name comparison. */
+export const APIErrorName = 'APIError'
 
 class ExtendableError<TData extends object = { [key: string]: unknown }> extends Error {
   data: TData
@@ -17,7 +17,6 @@ class ExtendableError<TData extends object = { [key: string]: unknown }> extends
       // show data in cause
       cause: data,
     })
-    APIErrorName = this.constructor.name
     this.name = this.constructor.name
     this.message = message
     this.status = status

packages/payload/src/errors/ValidationError.ts

@@ -8,8 +8,8 @@ import type { PayloadRequest } from '../types/index.js'
 
 import { APIError } from './APIError.js'
 
-// This gets dynamically reassigned during compilation
-export let ValidationErrorName = 'ValidationError'
+/** @deprecated Use `instanceof ValidationError` instead of name comparison. */
+export const ValidationErrorName = 'ValidationError'
 
 export type ValidationFieldError = {
   label?: LabelFunction | StaticLabel
@@ -75,7 +75,5 @@ export class ValidationError extends APIError<{
       httpStatus.BAD_REQUEST,
       results,
     )
-
-    ValidationErrorName = this.constructor.name
   }
 }

packages/payload/src/utilities/formatErrors.spec.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+
+import { APIError } from '../errors/APIError.js'
+import { ValidationError } from '../errors/ValidationError.js'
+import { formatErrors } from './formatErrors.js'
+
+describe('formatErrors', () => {
+  it('should format a Payload ValidationError', () => {
+    const err = new ValidationError({
+      errors: [{ message: 'Field is required', path: 'title' }],
+    })
+
+    const result = formatErrors(err)
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]).toMatchObject({
+      name: 'ValidationError',
+      message: expect.stringContaining('title'),
+      data: { errors: [{ message: 'Field is required', path: 'title' }] },
+    })
+  })
+
+  it('should format a Payload APIError', () => {
+    const err = new APIError('Something went wrong', 400, { detail: 'bad input' }, true)
+
+    const result = formatErrors(err)
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]).toMatchObject({
+      name: 'APIError',
+      message: 'Something went wrong',
+      data: { detail: 'bad input' },
+    })
+  })
+
+  it('should format a Mongoose-style ValidationError', () => {
+    const mongooseError = {
+      name: 'ValidationError',
+      errors: {
+        email: { path: 'email', message: 'is invalid' },
+      },
+    }
+
+    const result = formatErrors(mongooseError as any)
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]).toMatchObject({ field: 'email', message: 'is invalid' })
+  })
+
+  it('should format an array message error', () => {
+    const err = { message: [{ message: 'item one' }, { message: 'item two' }] }
+
+    const result = formatErrors(err as any)
+
+    expect(result.errors).toHaveLength(2)
+    expect(result.errors[0]).toMatchObject({ message: 'item one' })
+    expect(result.errors[1]).toMatchObject({ message: 'item two' })
+  })
+
+  it('should format a named non-Payload error', () => {
+    const err = new Error('Unexpected failure')
+
+    const result = formatErrors(err as any)
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]!.message).toBe('Unexpected failure')
+  })
+
+  it('should format a Payload APIError with no data', () => {
+    const err = new APIError('Server error')
+
+    const result = formatErrors(err)
+
+    expect(result.errors).toHaveLength(1)
+    expect(result.errors[0]).toStrictEqual({ message: 'Server error' })
+  })
+
+  it('should return unknown error for null/undefined input', () => {
+    expect(formatErrors(null as any).errors[0]!.message).toBe('An unknown error occurred.')
+    expect(formatErrors(undefined as any).errors[0]!.message).toBe('An unknown error occurred.')
+  })
+})

packages/payload/src/utilities/formatErrors.ts

@@ -1,33 +1,30 @@
 import type { ErrorResult } from '../config/types.js'
-import type { APIError } from '../errors/APIError.js'
 
-import { APIErrorName } from '../errors/APIError.js'
-import { ValidationErrorName } from '../errors/ValidationError.js'
+import { APIError } from '../errors/APIError.js'
+import { ValidationError } from '../errors/ValidationError.js'
 
 export const formatErrors = (incoming: { [key: string]: unknown } | APIError): ErrorResult => {
   if (incoming) {
-    // Cannot use `instanceof` to check error type: https://github.com/microsoft/TypeScript/issues/13965
-    // Instead, get the prototype of the incoming error and check its constructor name
-    const proto = Object.getPrototypeOf(incoming)
-
     // Payload 'ValidationError' and 'APIError'
-    if (
-      (proto.constructor.name === ValidationErrorName || proto.constructor.name === APIErrorName) &&
-      incoming.data
-    ) {
+    if ((incoming instanceof ValidationError || incoming instanceof APIError) && incoming.data) {
       return {
         errors: [
           {
-            name: incoming.name as string,
+            name: incoming.name,
             data: incoming.data as Record<string, unknown>,
-            message: incoming.message as string,
+            message: incoming.message,
           },
         ],
       }
     }
 
     // Mongoose 'ValidationError': https://mongoosejs.com/docs/api/error.html#Error.ValidationError
-    if (proto.constructor.name === ValidationErrorName && 'errors' in incoming && incoming.errors) {
+    if (
+      'name' in incoming &&
+      incoming.name === 'ValidationError' &&
+      'errors' in incoming &&
+      incoming.errors
+    ) {
       return {
         errors: Object.keys(incoming.errors).reduce(
           (acc, key) => {

packages/plugin-nested-docs/src/hooks/resaveChildren.ts

@@ -1,6 +1,6 @@
-import type { CollectionAfterChangeHook, JsonObject, ValidationError } from 'payload'
+import type { CollectionAfterChangeHook, JsonObject } from 'payload'
 
-import { APIError, ValidationErrorName } from 'payload'
+import { APIError, ValidationError } from 'payload'
 
 import type { NestedDocsPluginConfig } from '../types.js'
 
@@ -91,10 +91,7 @@ export const resaveChildren =
         )
         req.payload.logger.error(err)
 
-        if (
-          (err as ValidationError)?.name === ValidationErrorName &&
-          (err as ValidationError)?.data?.errors?.length
-        ) {
+        if (err instanceof ValidationError && err.data?.errors?.length) {
           throw new APIError(
             'Could not publish or save changes: One or more children are invalid.',
             400,