Skip to content

Commit 9b36db5

Browse files
wadakatuwadakatu
authored
Merge pull request #26 from studio-design/feat/report-all-validation-errors
New Feature: Report all validation errors instead of stopping at the first one
2 parents 4404bd0 + c180e2e commit 9b36db5

6 files changed

Lines changed: 280 additions & 2 deletions

File tree

‎README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,10 @@ This creates `config/openapi-contract-testing.php`:
8080
```php
8181
return [
8282
'default_spec' => '', // e.g., 'front'
83+
84+
// Maximum number of validation errors to report per response.
85+
// 0 = unlimited (reports all errors).
86+
'max_errors' => 20,
8387
];
8488
```
8589

@@ -140,6 +144,23 @@ $result = $validator->validate(
140144
$this->assertTrue($result->isValid(), $result->errorMessage());
141145
```
142146

147+
#### Controlling the number of validation errors
148+
149+
By default, up to **20** validation errors are reported per response. You can change this via the constructor:
150+
151+
```php
152+
// Report up to 5 errors
153+
$validator = new OpenApiResponseValidator(maxErrors: 5);
154+
155+
// Report all errors (unlimited)
156+
$validator = new OpenApiResponseValidator(maxErrors: 0);
157+
158+
// Stop at first error (pre-v0.x default)
159+
$validator = new OpenApiResponseValidator(maxErrors: 1);
160+
```
161+
162+
For Laravel, set the `max_errors` key in `config/openapi-contract-testing.php`.
163+
143164
## Coverage Report
144165

145166
After running tests, the PHPUnit extension prints a coverage report:
@@ -211,9 +232,12 @@ The package auto-detects the OAS version from the `openapi` field and handles sc
211232

212233
Main validator class. Validates a response body against the spec.
213234

235+
The constructor accepts a `maxErrors` parameter (default: `20`) that limits how many validation errors the underlying JSON Schema validator collects. Use `0` for unlimited, `1` to stop at the first error.
236+
214237
The optional `responseContentType` parameter enables content negotiation: when provided, non-JSON content types (e.g., `text/html`) are checked for spec presence only, while JSON-compatible types proceed to full schema validation.
215238

216239
```php
240+
$validator = new OpenApiResponseValidator(maxErrors: 20);
217241
$result = $validator->validate(
218242
specName: 'front',
219243
method: 'GET',

‎src/Laravel/ValidatesOpenApiSchema.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
use Studio\OpenApiContractTesting\OpenApiCoverageTracker;
1111
use Studio\OpenApiContractTesting\OpenApiResponseValidator;
1212

13+
use function is_numeric;
1314
use function is_string;
1415
use function str_contains;
1516
use function strtolower;
@@ -51,7 +52,10 @@ protected function assertResponseMatchesOpenApiSchema(
5152

5253
$contentType = $response->headers->get('Content-Type', '');
5354

54-
$validator = new OpenApiResponseValidator();
55+
$maxErrors = config('openapi-contract-testing.max_errors', 20);
56+
$validator = new OpenApiResponseValidator(
57+
maxErrors: is_numeric($maxErrors) ? (int) $maxErrors : 20,
58+
);
5559
$result = $validator->validate(
5660
$specName,
5761
$resolvedMethod,

‎src/Laravel/config.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,8 @@
44

55
return [
66
'default_spec' => '',
7+
8+
// Maximum number of validation errors to report per response.
9+
// 0 = unlimited (reports all errors).
10+
'max_errors' => 20,
711
];

‎src/OpenApiResponseValidator.php‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,34 @@
55
namespace Studio\OpenApiContractTesting;
66

77
use const JSON_THROW_ON_ERROR;
8+
use const PHP_INT_MAX;
89

10+
use InvalidArgumentException;
911
use Opis\JsonSchema\Errors\ErrorFormatter;
1012
use Opis\JsonSchema\Validator;
1113

1214
use function array_keys;
1315
use function implode;
1416
use function json_decode;
1517
use function json_encode;
18+
use function sprintf;
1619
use function str_ends_with;
1720
use function strstr;
1821
use function strtolower;
1922
use function trim;
2023

2124
final class OpenApiResponseValidator
2225
{
26+
public function __construct(
27+
private readonly int $maxErrors = 20,
28+
) {
29+
if ($this->maxErrors < 0) {
30+
throw new InvalidArgumentException(
31+
sprintf('maxErrors must be 0 (unlimited) or a positive integer, got %d.', $this->maxErrors),
32+
);
33+
}
34+
}
35+
2336
public function validate(
2437
string $specName,
2538
string $method,
@@ -134,7 +147,11 @@ public function validate(
134147
JSON_THROW_ON_ERROR,
135148
);
136149

137-
$validator = new Validator();
150+
$resolvedMaxErrors = $this->maxErrors === 0 ? PHP_INT_MAX : $this->maxErrors;
151+
$validator = new Validator(
152+
max_errors: $resolvedMaxErrors,
153+
stop_at_first_error: $resolvedMaxErrors === 1,
154+
);
138155
$result = $validator->validate($dataObject, $schemaObject);
139156

140157
if ($result->isValid()) {

‎tests/Unit/OpenApiResponseValidatorTest.php‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,16 @@
44

55
namespace Studio\OpenApiContractTesting\Tests\Unit;
66

7+
use InvalidArgumentException;
78
use PHPUnit\Framework\Attributes\Test;
89
use PHPUnit\Framework\TestCase;
910
use Studio\OpenApiContractTesting\OpenApiResponseValidator;
1011
use Studio\OpenApiContractTesting\OpenApiSpecLoader;
1112

13+
use function array_map;
14+
use function count;
15+
use function range;
16+
1217
class OpenApiResponseValidatorTest extends TestCase
1318
{
1419
private OpenApiResponseValidator $validator;
@@ -584,6 +589,140 @@ public function v31_no_content_response_passes(): void
584589
$this->assertTrue($result->isValid());
585590
}
586591

592+
// ========================================
593+
// maxErrors tests
594+
// ========================================
595+
596+
#[Test]
597+
public function default_max_errors_reports_multiple_errors(): void
598+
{
599+
$result = $this->validator->validate(
600+
'petstore-3.0',
601+
'GET',
602+
'/v1/pets',
603+
200,
604+
[
605+
'data' => [
606+
['id' => 'not-an-int', 'name' => 123],
607+
['id' => 'also-not-an-int', 'name' => 456],
608+
],
609+
],
610+
);
611+
612+
$this->assertFalse($result->isValid());
613+
$this->assertGreaterThan(1, count($result->errors()));
614+
}
615+
616+
#[Test]
617+
public function max_errors_caps_reported_errors_to_configured_limit(): void
618+
{
619+
$items = array_map(
620+
static fn(int $i) => ['id' => 'str-' . $i, 'name' => $i],
621+
range(1, 50),
622+
);
623+
624+
$capped = new OpenApiResponseValidator(maxErrors: 5);
625+
$cappedResult = $capped->validate(
626+
'petstore-3.0',
627+
'GET',
628+
'/v1/pets',
629+
200,
630+
['data' => $items],
631+
);
632+
633+
$unlimited = new OpenApiResponseValidator(maxErrors: 0);
634+
$unlimitedResult = $unlimited->validate(
635+
'petstore-3.0',
636+
'GET',
637+
'/v1/pets',
638+
200,
639+
['data' => $items],
640+
);
641+
642+
$this->assertFalse($cappedResult->isValid());
643+
$this->assertFalse($unlimitedResult->isValid());
644+
$this->assertLessThan(
645+
count($unlimitedResult->errors()),
646+
count($cappedResult->errors()),
647+
);
648+
}
649+
650+
#[Test]
651+
public function max_errors_one_limits_to_single_error(): void
652+
{
653+
$validator = new OpenApiResponseValidator(maxErrors: 1);
654+
$result = $validator->validate(
655+
'petstore-3.0',
656+
'GET',
657+
'/v1/pets',
658+
200,
659+
[
660+
'data' => [
661+
['id' => 'not-an-int', 'name' => 123],
662+
['id' => 'also-not-an-int', 'name' => 456],
663+
],
664+
],
665+
);
666+
667+
$this->assertFalse($result->isValid());
668+
$this->assertCount(1, $result->errors());
669+
}
670+
671+
#[Test]
672+
public function max_errors_two_reports_more_than_one_error(): void
673+
{
674+
$items = array_map(
675+
static fn(int $i) => ['id' => 'str-' . $i, 'name' => $i],
676+
range(1, 50),
677+
);
678+
679+
$validator = new OpenApiResponseValidator(maxErrors: 2);
680+
$result = $validator->validate(
681+
'petstore-3.0',
682+
'GET',
683+
'/v1/pets',
684+
200,
685+
['data' => $items],
686+
);
687+
688+
$this->assertFalse($result->isValid());
689+
$this->assertGreaterThan(1, count($result->errors()));
690+
}
691+
692+
#[Test]
693+
public function max_errors_zero_reports_all_errors(): void
694+
{
695+
$items = array_map(
696+
static fn(int $i) => ['id' => 'str-' . $i, 'name' => $i],
697+
range(1, 50),
698+
);
699+
700+
$validator = new OpenApiResponseValidator(maxErrors: 0);
701+
$result = $validator->validate(
702+
'petstore-3.0',
703+
'GET',
704+
'/v1/pets',
705+
200,
706+
['data' => $items],
707+
);
708+
709+
$this->assertFalse($result->isValid());
710+
$this->assertGreaterThan(20, count($result->errors()));
711+
}
712+
713+
#[Test]
714+
public function negative_max_errors_throws_exception(): void
715+
{
716+
$this->expectException(InvalidArgumentException::class);
717+
$this->expectExceptionMessage('maxErrors must be 0 (unlimited) or a positive integer, got -1.');
718+
719+
new OpenApiResponseValidator(maxErrors: -1);
720+
}
721+
722+
// ========================================
723+
// Strip prefix tests
724+
// ========================================
725+
587726
#[Test]
588727
public function v30_strip_prefixes_applied(): void
589728
{

‎tests/Unit/ValidatesOpenApiSchemaDefaultSpecTest.php‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,12 @@
1515
use Studio\OpenApiContractTesting\OpenApiSpecLoader;
1616
use Studio\OpenApiContractTesting\Tests\Helpers\CreatesTestResponse;
1717

18+
use function array_filter;
19+
use function count;
20+
use function explode;
1821
use function json_encode;
22+
use function str_starts_with;
23+
use function trim;
1924

2025
// Load namespace-level config() mock before the trait resolves the function call.
2126
require_once __DIR__ . '/../Helpers/LaravelConfigMock.php';
@@ -100,4 +105,89 @@ public function configured_default_spec_validates_successfully(): void
100105
'/v1/pets',
101106
);
102107
}
108+
109+
// ========================================
110+
// max_errors config tests
111+
// ========================================
112+
113+
#[Test]
114+
public function max_errors_config_limits_reported_errors(): void
115+
{
116+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.default_spec'] = 'petstore-3.0';
117+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.max_errors'] = 1;
118+
119+
$body = (string) json_encode(
120+
['data' => [['id' => 'bad', 'name' => 123], ['id' => 'bad', 'name' => 456]]],
121+
JSON_THROW_ON_ERROR,
122+
);
123+
$response = $this->makeTestResponse($body, 200);
124+
125+
try {
126+
$this->assertResponseMatchesOpenApiSchema(
127+
$response,
128+
HttpMethod::GET,
129+
'/v1/pets',
130+
);
131+
$this->fail('Expected AssertionFailedError was not thrown.');
132+
} catch (AssertionFailedError $e) {
133+
// With max_errors=1, the error message should contain exactly one schema error line.
134+
// The error format is "[path] message", so count lines starting with "[".
135+
$lines = explode("\n", $e->getMessage());
136+
$errorLines = array_filter($lines, static fn(string $line) => str_starts_with(trim($line), '['));
137+
$this->assertCount(1, $errorLines);
138+
}
139+
}
140+
141+
#[Test]
142+
public function string_numeric_max_errors_config_is_cast_to_int(): void
143+
{
144+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.default_spec'] = 'petstore-3.0';
145+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.max_errors'] = '1';
146+
147+
$body = (string) json_encode(
148+
['data' => [['id' => 'bad', 'name' => 123], ['id' => 'bad', 'name' => 456]]],
149+
JSON_THROW_ON_ERROR,
150+
);
151+
$response = $this->makeTestResponse($body, 200);
152+
153+
try {
154+
$this->assertResponseMatchesOpenApiSchema(
155+
$response,
156+
HttpMethod::GET,
157+
'/v1/pets',
158+
);
159+
$this->fail('Expected AssertionFailedError was not thrown.');
160+
} catch (AssertionFailedError $e) {
161+
$lines = explode("\n", $e->getMessage());
162+
$errorLines = array_filter($lines, static fn(string $line) => str_starts_with(trim($line), '['));
163+
$this->assertCount(1, $errorLines);
164+
}
165+
}
166+
167+
#[Test]
168+
public function non_numeric_max_errors_config_falls_back_to_default(): void
169+
{
170+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.default_spec'] = 'petstore-3.0';
171+
$GLOBALS['__openapi_testing_config']['openapi-contract-testing.max_errors'] = 'not-a-number';
172+
173+
$body = (string) json_encode(
174+
['data' => [['id' => 'bad', 'name' => 123], ['id' => 'bad', 'name' => 456]]],
175+
JSON_THROW_ON_ERROR,
176+
);
177+
$response = $this->makeTestResponse($body, 200);
178+
179+
try {
180+
$this->assertResponseMatchesOpenApiSchema(
181+
$response,
182+
HttpMethod::GET,
183+
'/v1/pets',
184+
);
185+
$this->fail('Expected AssertionFailedError was not thrown.');
186+
} catch (AssertionFailedError $e) {
187+
// Falls back to default of 20, so multiple errors should be reported
188+
$lines = explode("\n", $e->getMessage());
189+
$errorLines = array_filter($lines, static fn(string $line) => str_starts_with(trim($line), '['));
190+
$this->assertGreaterThan(1, count($errorLines));
191+
}
192+
}
103193
}

0 commit comments

Comments
 (0)