diff --git a/README.md b/README.md index da9347d..bec7ac1 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,35 @@ Notes: - `auto_assert` accepts boolean-compatible values (`true`/`false`/`"1"`/`"0"`/`"true"`/`"false"`) so `'auto_assert' => env('OPENAPI_AUTO_ASSERT')` works. Unrecognized values fail the test loudly with a clear message, not silently. - Streamed responses (`StreamedResponse`, binary downloads) cause `getContent()` to return `false`, which fails auto-assert with a clear message. If you use `auto_assert=true` on tests that exercise streams, scope the config change per-test or fall back to explicit manual asserts. +#### Opting out with `#[SkipOpenApi]` + +Some tests intentionally return responses that violate the spec (error-injection tests, experimental endpoints with a not-yet-finalized contract, etc.). For these, use the `#[SkipOpenApi]` attribute to opt out of auto-assert without turning the feature off globally: + +```php +use Studio\OpenApiContractTesting\SkipOpenApi; + +class ExperimentalApiTest extends TestCase +{ + use ValidatesOpenApiSchema; + + #[Test] + #[SkipOpenApi(reason: 'endpoint is behind an experimental flag')] + public function test_experimental_endpoint(): void + { + $this->get('/v1/experimental'); // auto-assert is skipped + } +} +``` + +The attribute can also be applied at the class level to skip every method in that class. A method-level `#[SkipOpenApi]` fully shadows the class-level one — only the method-level attribute (and its `reason`) is inspected. + +Notes: + +- `#[SkipOpenApi]` suppresses **auto-assert only**. Explicit calls to `assertResponseMatchesOpenApiSchema()` still run — the assertion is the user's direct intent. +- When auto-assert is skipped and no explicit assertion is made, no coverage is recorded for that request (the endpoint is treated as uncovered in the report). If you call `assertResponseMatchesOpenApiSchema()` explicitly on a skipped test, validation runs and coverage is recorded as usual. +- If a test is marked `#[SkipOpenApi]` and still calls `assertResponseMatchesOpenApiSchema()` explicitly, an advisory warning is written to `STDERR` and a user deprecation is raised to flag the contradictory intent. The assertion is not suppressed — fix the cause by removing either the attribute or the explicit call. +- The attribute is resolved via reflection on the direct class only; a class-level `#[SkipOpenApi]` on an abstract parent is **not** inherited by subclasses. Apply the attribute on each concrete test class (or per method) instead. + ## Coverage Report After running tests, the PHPUnit extension prints a coverage report. The output format is controlled by the `console_output` parameter (or `OPENAPI_CONSOLE_OUTPUT` environment variable). diff --git a/src/Laravel/ValidatesOpenApiSchema.php b/src/Laravel/ValidatesOpenApiSchema.php index 809eaed..47f0525 100644 --- a/src/Laravel/ValidatesOpenApiSchema.php +++ b/src/Laravel/ValidatesOpenApiSchema.php @@ -4,8 +4,10 @@ namespace Studio\OpenApiContractTesting\Laravel; +use const E_USER_DEPRECATED; use const FILTER_NULL_ON_FAILURE; use const FILTER_VALIDATE_BOOLEAN; +use const STDERR; use Illuminate\Testing\TestResponse; use JsonException; @@ -13,11 +15,14 @@ use Studio\OpenApiContractTesting\OpenApiCoverageTracker; use Studio\OpenApiContractTesting\OpenApiResponseValidator; use Studio\OpenApiContractTesting\OpenApiSpecResolver; +use Studio\OpenApiContractTesting\SkipOpenApi; +use Studio\OpenApiContractTesting\SkipOpenApiResolver; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use WeakMap; use function filter_var; +use function fwrite; use function get_debug_type; use function is_numeric; use function is_string; @@ -25,17 +30,32 @@ use function str_contains; use function strtolower; use function strtoupper; +use function trigger_error; use function var_export; trait ValidatesOpenApiSchema { use OpenApiSpecResolver; + use SkipOpenApiResolver; private static ?OpenApiResponseValidator $cachedValidator = null; private static ?int $cachedMaxErrors = null; /** @var null|WeakMap> */ private static ?WeakMap $validatedResponses = null; + /** + * Receives the warning emitted when a test marked #[SkipOpenApi] still + * calls assertResponseMatchesOpenApiSchema() explicitly. The explicit + * assertion always runs regardless — this is an advisory nudge that the + * two signals contradict each other. + * + * Defaults to writing to STDERR and emitting an E_USER_DEPRECATED via + * trigger_error(). Tests can swap it to capture warnings in-memory. + * + * @var null|callable(string): void + */ + private static $skipWarningHandler; + public static function resetValidatorCache(): void { self::$cachedValidator = null; @@ -76,6 +96,13 @@ protected function maybeAutoAssertOpenApiSchema( return; } + // #[SkipOpenApi] opts the test out of auto-assert entirely — no + // validation, no coverage recording. Explicit calls to + // assertResponseMatchesOpenApiSchema() still run but emit a warning. + if ($this->findSkipOpenApiAttribute() !== null) { + return; + } + $this->assertResponseMatchesOpenApiSchema($response, $method, $path); } @@ -100,6 +127,11 @@ protected function assertResponseMatchesOpenApiSchema( ?HttpMethod $method = null, ?string $path = null, ): void { + $skipAttribute = $this->findSkipOpenApiAttribute(); + if ($skipAttribute !== null) { + $this->emitSkipOpenApiWarning($skipAttribute); + } + $resolvedMethod = $method !== null ? $method->value : app('request')->getMethod(); $resolvedPath = $path ?? app('request')->getPathInfo(); @@ -188,6 +220,34 @@ private static function getOrCreateValidator(): OpenApiResponseValidator return self::$cachedValidator; } + private function emitSkipOpenApiWarning(SkipOpenApi $attribute): void + { + $reason = $attribute->reason; + $message = sprintf( + '%s::%s is marked #[SkipOpenApi%s] but called assertResponseMatchesOpenApiSchema() explicitly. ' + . 'The assertion will run. Remove the attribute or the explicit call to clarify intent.', + static::class, + $this->name(), // @phpstan-ignore method.notFound + $reason !== '' ? sprintf('(reason: %s)', var_export($reason, true)) : '', + ); + + $handler = self::$skipWarningHandler; + if ($handler !== null) { + $handler($message); + + return; + } + + // STDERR guarantees the message body is visible in CI regardless of + // PHPUnit's `displayDetailsOnTestsThatTriggerDeprecations` setting — + // without it, the default config would only show a "1 deprecation" + // tally and hide the actual contradictory-intent message. + fwrite(STDERR, sprintf("\n[openapi-contract-testing] %s\n", $message)); + // trigger_error still fires so PHPUnit counts the deprecation and + // surfaces it in the run summary for downstream tools to detect. + trigger_error($message, E_USER_DEPRECATED); + } + private function isAutoAssertEnabled(): bool { $raw = config('openapi-contract-testing.auto_assert', false); diff --git a/src/SkipOpenApi.php b/src/SkipOpenApi.php new file mode 100644 index 0000000..1c23fb7 --- /dev/null +++ b/src/SkipOpenApi.php @@ -0,0 +1,15 @@ +findSkipOpenApiAttribute() !== null; + } + + private function findSkipOpenApiAttribute(): ?SkipOpenApi + { + $methodName = $this->name(); // @phpstan-ignore method.notFound + $refMethod = new ReflectionMethod($this, $methodName); + $methodAttrs = $refMethod->getAttributes(SkipOpenApi::class); + if ($methodAttrs !== []) { + return $methodAttrs[0]->newInstance(); + } + + $refClass = new ReflectionClass($this); + $classAttrs = $refClass->getAttributes(SkipOpenApi::class); + if ($classAttrs !== []) { + return $classAttrs[0]->newInstance(); + } + + return null; + } +} diff --git a/tests/Integration/Laravel/AutoAssertIntegrationTest.php b/tests/Integration/Laravel/AutoAssertIntegrationTest.php index db32949..7eef78d 100644 --- a/tests/Integration/Laravel/AutoAssertIntegrationTest.php +++ b/tests/Integration/Laravel/AutoAssertIntegrationTest.php @@ -13,6 +13,7 @@ use Studio\OpenApiContractTesting\OpenApiCoverageTracker; use Studio\OpenApiContractTesting\OpenApiSpec; use Studio\OpenApiContractTesting\OpenApiSpecLoader; +use Studio\OpenApiContractTesting\SkipOpenApi; use function dirname; @@ -163,6 +164,36 @@ public function method_level_attribute_resolves_spec_for_auto_assert(): void $this->assertArrayNotHasKey('petstore-3.0', $covered); } + #[Test] + #[SkipOpenApi(reason: 'intentional spec violation for test')] + public function skip_open_api_attribute_opts_method_out_of_auto_assert(): void + { + // Would normally fail auto-assert because ?bad=1 returns {wrong_key:...} + // which violates the spec. #[SkipOpenApi] must prevent that failure + // AND stop coverage recording. + config()->set('openapi-contract-testing.auto_assert', true); + + $response = $this->get('/v1/pets?bad=1'); + $response->assertOk(); + + $this->assertArrayNotHasKey('petstore-3.0', OpenApiCoverageTracker::getCovered()); + } + + #[Test] + #[SkipOpenApi] + public function skip_open_api_attribute_opts_post_out_of_auto_assert(): void + { + // Guard against a regression that only checks skip on GET. The hook + // is verb-agnostic in theory, but a bug that validated POST bodies + // before consulting skip would only be caught here. + config()->set('openapi-contract-testing.auto_assert', true); + + $response = $this->postJson('/v1/pets', ['name' => 'Buddy']); + $response->assertCreated(); + + $this->assertArrayNotHasKey('petstore-3.0', OpenApiCoverageTracker::getCovered()); + } + /** @return array */ protected function getPackageProviders($app): array { diff --git a/tests/Unit/SkipOpenApiResolverClassLevelTest.php b/tests/Unit/SkipOpenApiResolverClassLevelTest.php new file mode 100644 index 0000000..9b791db --- /dev/null +++ b/tests/Unit/SkipOpenApiResolverClassLevelTest.php @@ -0,0 +1,31 @@ +assertTrue($this->shouldSkipOpenApi()); + $this->assertSame('class-level', $this->findSkipOpenApiAttribute()->reason); + } + + #[Test] + #[SkipOpenApi(reason: 'method-level')] + public function method_level_reason_overrides_class_level(): void + { + $this->assertTrue($this->shouldSkipOpenApi()); + $this->assertSame('method-level', $this->findSkipOpenApiAttribute()->reason); + } +} diff --git a/tests/Unit/SkipOpenApiResolverTest.php b/tests/Unit/SkipOpenApiResolverTest.php new file mode 100644 index 0000000..c3df8db --- /dev/null +++ b/tests/Unit/SkipOpenApiResolverTest.php @@ -0,0 +1,38 @@ +assertFalse($this->shouldSkipOpenApi()); + $this->assertNull($this->findSkipOpenApiAttribute()); + } + + #[Test] + #[SkipOpenApi] + public function method_level_attribute_skips(): void + { + $this->assertTrue($this->shouldSkipOpenApi()); + $this->assertSame('', $this->findSkipOpenApiAttribute()->reason); + } + + #[Test] + #[SkipOpenApi(reason: 'experimental endpoint')] + public function method_level_reason_is_resolved(): void + { + $this->assertTrue($this->shouldSkipOpenApi()); + $this->assertSame('experimental endpoint', $this->findSkipOpenApiAttribute()->reason); + } +} diff --git a/tests/Unit/ValidatesOpenApiSchemaClassLevelSkipTest.php b/tests/Unit/ValidatesOpenApiSchemaClassLevelSkipTest.php new file mode 100644 index 0000000..e3995b8 --- /dev/null +++ b/tests/Unit/ValidatesOpenApiSchemaClassLevelSkipTest.php @@ -0,0 +1,89 @@ + */ + private array $capturedWarnings = []; + + protected function setUp(): void + { + parent::setUp(); + OpenApiSpecLoader::reset(); + OpenApiSpecLoader::configure(__DIR__ . '/../fixtures/specs'); + OpenApiCoverageTracker::reset(); + $GLOBALS['__openapi_testing_config'] = [ + 'openapi-contract-testing.default_spec' => 'petstore-3.0', + 'openapi-contract-testing.auto_assert' => true, + ]; + + $this->capturedWarnings = []; + self::$skipWarningHandler = function (string $message): void { + $this->capturedWarnings[] = $message; + }; + } + + protected function tearDown(): void + { + self::resetValidatorCache(); + self::$skipWarningHandler = null; + unset($GLOBALS['__openapi_testing_config']); + OpenApiSpecLoader::reset(); + OpenApiCoverageTracker::reset(); + parent::tearDown(); + } + + #[Test] + public function class_level_skip_opts_out_of_auto_assert(): void + { + $body = (string) json_encode(['wrong_key' => 'value'], JSON_THROW_ON_ERROR); + $response = $this->makeTestResponse($body, 200); + + $this->maybeAutoAssertOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + $this->assertArrayNotHasKey('petstore-3.0', OpenApiCoverageTracker::getCovered()); + // Auto-assert skip does not emit a warning (only explicit calls do). + $this->assertSame([], $this->capturedWarnings); + } + + #[Test] + public function class_level_skip_emits_warning_with_class_reason_on_explicit_assert(): void + { + $body = (string) json_encode( + ['data' => [['id' => 1, 'name' => 'Fido', 'tag' => null]]], + JSON_THROW_ON_ERROR, + ); + $response = $this->makeTestResponse($body, 200); + + $this->assertResponseMatchesOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + // Warning fires with the class-level reason (no method-level attribute overrides it). + $this->assertCount(1, $this->capturedWarnings); + $this->assertStringContainsString("reason: 'entire class is experimental'", $this->capturedWarnings[0]); + + // Explicit assert still ran — validation + coverage recorded. + $this->assertArrayHasKey('petstore-3.0', OpenApiCoverageTracker::getCovered()); + } +} diff --git a/tests/Unit/ValidatesOpenApiSchemaDefaultWarningHandlerTest.php b/tests/Unit/ValidatesOpenApiSchemaDefaultWarningHandlerTest.php new file mode 100644 index 0000000..0d4fe34 --- /dev/null +++ b/tests/Unit/ValidatesOpenApiSchemaDefaultWarningHandlerTest.php @@ -0,0 +1,93 @@ + 'petstore-3.0', + ]; + // Intentionally do NOT set $skipWarningHandler — exercise the default. + } + + protected function tearDown(): void + { + self::resetValidatorCache(); + unset($GLOBALS['__openapi_testing_config']); + OpenApiSpecLoader::reset(); + OpenApiCoverageTracker::reset(); + parent::tearDown(); + } + + #[Test] + #[SkipOpenApi(reason: 'testing default path')] + public function default_handler_emits_e_user_deprecated(): void + { + $captured = null; + set_error_handler( + static function (int $errno, string $msg) use (&$captured): bool { + $captured = ['errno' => $errno, 'msg' => $msg]; + + // Returning true suppresses the default PHP error handler so + // PHPUnit's own error converter doesn't turn the deprecation + // into a test failure. STDERR output still fires before this + // handler is invoked, which is fine — we only assert on the + // captured values. + return true; + }, + E_USER_DEPRECATED, + ); + + try { + $body = (string) json_encode( + ['data' => [['id' => 1, 'name' => 'Fido', 'tag' => null]]], + JSON_THROW_ON_ERROR, + ); + $response = $this->makeTestResponse($body, 200); + + $this->assertResponseMatchesOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + } finally { + restore_error_handler(); + } + + $this->assertNotNull($captured, 'Expected trigger_error to fire E_USER_DEPRECATED'); + $this->assertSame(E_USER_DEPRECATED, $captured['errno']); + $this->assertStringContainsString('#[SkipOpenApi', $captured['msg']); + $this->assertStringContainsString("reason: 'testing default path'", $captured['msg']); + } +} diff --git a/tests/Unit/ValidatesOpenApiSchemaSkipTest.php b/tests/Unit/ValidatesOpenApiSchemaSkipTest.php new file mode 100644 index 0000000..a601e9c --- /dev/null +++ b/tests/Unit/ValidatesOpenApiSchemaSkipTest.php @@ -0,0 +1,138 @@ + */ + private array $capturedWarnings = []; + + protected function setUp(): void + { + parent::setUp(); + OpenApiSpecLoader::reset(); + OpenApiSpecLoader::configure(__DIR__ . '/../fixtures/specs'); + OpenApiCoverageTracker::reset(); + $GLOBALS['__openapi_testing_config'] = [ + 'openapi-contract-testing.default_spec' => 'petstore-3.0', + 'openapi-contract-testing.auto_assert' => true, + ]; + + $this->capturedWarnings = []; + self::$skipWarningHandler = function (string $message): void { + $this->capturedWarnings[] = $message; + }; + } + + protected function tearDown(): void + { + self::resetValidatorCache(); + self::$skipWarningHandler = null; + unset($GLOBALS['__openapi_testing_config']); + OpenApiSpecLoader::reset(); + OpenApiCoverageTracker::reset(); + parent::tearDown(); + } + + #[Test] + #[SkipOpenApi] + public function method_level_skip_opts_out_of_auto_assert(): void + { + // Body intentionally violates the spec — if auto-assert ran, this + // would throw AssertionFailedError. + $body = (string) json_encode(['wrong_key' => 'value'], JSON_THROW_ON_ERROR); + $response = $this->makeTestResponse($body, 200); + + $this->maybeAutoAssertOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + // Coverage must not be recorded for skipped tests. + $this->assertArrayNotHasKey('petstore-3.0', OpenApiCoverageTracker::getCovered()); + } + + #[Test] + #[SkipOpenApi] + public function skipped_test_does_not_emit_warning_when_no_explicit_assert(): void + { + $body = (string) json_encode(['wrong_key' => 'value'], JSON_THROW_ON_ERROR); + $response = $this->makeTestResponse($body, 200); + + $this->maybeAutoAssertOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + $this->assertSame([], $this->capturedWarnings); + } + + #[Test] + #[SkipOpenApi] + public function explicit_assert_on_skipped_test_emits_warning_and_still_validates(): void + { + $body = (string) json_encode( + ['data' => [['id' => 1, 'name' => 'Fido', 'tag' => null]]], + JSON_THROW_ON_ERROR, + ); + $response = $this->makeTestResponse($body, 200); + + $this->assertResponseMatchesOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + // Warning was emitted once. + $this->assertCount(1, $this->capturedWarnings); + $this->assertStringContainsString('#[SkipOpenApi]', $this->capturedWarnings[0]); + $this->assertStringContainsString('assertResponseMatchesOpenApiSchema', $this->capturedWarnings[0]); + + // And validation actually ran — coverage was recorded despite the skip. + $covered = OpenApiCoverageTracker::getCovered(); + $this->assertArrayHasKey('petstore-3.0', $covered); + $this->assertArrayHasKey('GET /v1/pets', $covered['petstore-3.0']); + } + + #[Test] + public function non_skipped_explicit_assert_does_not_emit_warning(): void + { + $body = (string) json_encode( + ['data' => [['id' => 1, 'name' => 'Fido', 'tag' => null]]], + JSON_THROW_ON_ERROR, + ); + $response = $this->makeTestResponse($body, 200); + + $this->assertResponseMatchesOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + $this->assertSame([], $this->capturedWarnings); + } + + #[Test] + #[SkipOpenApi(reason: 'intentional violation test')] + public function warning_message_includes_reason_when_provided(): void + { + $body = (string) json_encode( + ['data' => [['id' => 1, 'name' => 'Fido', 'tag' => null]]], + JSON_THROW_ON_ERROR, + ); + $response = $this->makeTestResponse($body, 200); + + $this->assertResponseMatchesOpenApiSchema($response, HttpMethod::GET, '/v1/pets'); + + $this->assertCount(1, $this->capturedWarnings); + // Formatted via var_export so the reason value is quoted verbatim. + $this->assertStringContainsString("(reason: 'intentional violation test')", $this->capturedWarnings[0]); + } +}