diff --git a/acceptance/API_COVERAGE.md b/acceptance/API_COVERAGE.md index f53fab73a..f9f703df5 100644 --- a/acceptance/API_COVERAGE.md +++ b/acceptance/API_COVERAGE.md @@ -30,6 +30,21 @@ These run in the `wire` profile in addition to smoke coverage: | ---------- | ----------------------------------------------------------------------- | | Wire/SigV4 | raw `aws-chunked` PutObject and UploadPart, trailer-signature rejection | +## CDN Edge Coverage + +These assert real Cloudflare edge cache behavior (`cf-cache-status` MISS/HIT, ETag/304, purge +propagation) rather than the storage API's own request handling. There's no local equivalent - +`cf-cache-status` is only ever set on responses that pass through the deployed edge - so this +coverage only runs against a real remote target with CDN enabled (the `CDN Edge` capability below). + +| Area | Covered APIs / behavior | +| ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cache MISS/HIT | MISS then HIT across public, authenticated, and signed access; signed invalid-token and authenticated anon-key denial still bypass the cache correctly | +| Conditional GET | ETag `If-None-Match` returns 304 on a cache HIT | +| Object write ops | UPSERT, MOVE, and COPY create fresh cache entries without disturbing unrelated entries (old MOVE path, COPY source); DELETE invalidates the cache entry | +| Transformations | distinct transform dimensions cache independently of each other and of the base object | +| Cache Purge | object, bucket, and tenant purge only invalidate their own scope; object/bucket/tenant transformation purge invalidates the transform but leaves the base object cached | + ## Capability-Gated Coverage These require target-specific capabilities. They run when the capability is enabled or derived from @@ -38,7 +53,8 @@ the configured target, and the selected profile includes the spec: | Capability | Enable with | Covered APIs / behavior | | ---------- | --------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Admin | `ACCEPTANCE_ENABLE_ADMIN=true` plus admin URL/API key | admin status, API key protection, tenant read/create/patch/upsert/delete/health, tenant migration run/reset-current/jobs, queue migration enabled/disabled contract, queue/migration validation, JWKS validation/add/deactivate/status, orphan scan/sync validation, S3 credential create/authenticate/list/delete | -| CDN | `ACCEPTANCE_ENABLE_CDN=true` | `/cdn/:bucket/*` cache purge for existing objects and documented missing-object error | +| CDN | `ACCEPTANCE_ENABLE_CDN=true` | `/cdn/:bucket/*` and `/cdn` cache purge for objects, buckets, the tenant, and their transformations; purge succeeds even when the target object no longer exists | +| CDN Edge | Derived from `ACCEPTANCE_ENABLE_CDN=true` and `ACCEPTANCE_TARGET=remote` | Real Cloudflare edge cache behavior - see [CDN Edge Coverage](#cdn-edge-coverage) above | | Render | `ACCEPTANCE_ENABLE_RENDER=true` | public, authenticated, and signed image transformation routes, `webp` output format, non-image input errors, invalid transformation validation | | RLS | `ACCEPTANCE_ENABLE_RLS_SETUP=true` plus anon/authenticated keys and RLS resource config | authenticated allow and anon deny for read/write on configured policies | | Path edges | Derived from `ACCEPTANCE_TARGET` and `STORAGE_BACKEND` | list-v2 preservation for object names with empty path segments; local S3/MinIO backends skip this case directly | diff --git a/acceptance/README.md b/acceptance/README.md index d87aa286d..1e51d75fc 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -85,7 +85,7 @@ The file backend also works because imgproxy mounts the local `./data` directory ```bash mkdir -p data -STORAGE_BACKEND=file ACCEPTANCE_ENABLE_RENDER=true npm run acceptance -- --profile full acceptance/specs/cdn-render.test.ts +STORAGE_BACKEND=file ACCEPTANCE_ENABLE_RENDER=true npm run acceptance -- --profile full acceptance/specs/render.test.ts ``` Local CI enables render tests for both S3 and file backend runs. The S3 matrix sets @@ -121,30 +121,32 @@ secrets as environment secrets. ## Useful Configuration -| Variable | Meaning | -| --------------------------------- | ------------------------------------------------------------------------------------------------- | -| `ACCEPTANCE_BASE_URL` | REST base URL. Defaults to `http://127.0.0.1:5000`. | -| `ACCEPTANCE_PROFILE` | Acceptance profile to run. Sample env and local CI default to `full`. | -| `ACCEPTANCE_S3_ENDPOINT` | S3 endpoint. Defaults to `$ACCEPTANCE_BASE_URL/s3`. | -| `ACCEPTANCE_TUS_ENDPOINT` | TUS endpoint. Defaults to `$ACCEPTANCE_BASE_URL/upload/resumable`. | -| `ACCEPTANCE_X_FORWARDED_HOST` | Optional tenant-routing host header for multitenant targets. | -| `ACCEPTANCE_ADMIN_URL` | Admin API base URL for admin tests. | -| `ACCEPTANCE_SERVICE_KEY` | Service role JWT for REST tests. | -| `ACCEPTANCE_S3_ACCESS_KEY_ID` | S3 protocol access key. | -| `ACCEPTANCE_S3_SECRET_ACCESS_KEY` | S3 protocol secret. | -| `ACCEPTANCE_REGION` | S3 signing region. Defaults to `us-east-1`. | -| `ACCEPTANCE_RESOURCE_PREFIX` | Prefix for all resources created by this run. | -| `ACCEPTANCE_ENABLE_ADMIN` | Enables admin route tests. Requires admin URL and API key. | -| `ACCEPTANCE_ENABLE_CDN` | Enables CDN purge tests. Managed local runs provide a purge stub by default. | -| `ACCEPTANCE_ENABLE_RENDER` | Enables image transformation tests. | -| `ACCEPTANCE_ENABLE_RLS_SETUP` | Enables RLS tests; requires service, anon, authenticated keys and bucket/prefix policy resources. | -| `ACCEPTANCE_ENABLE_VECTOR` | Enables vector bucket API tests. Requires local pgvector or a configured S3 Vectors target. | -| `ACCEPTANCE_ENABLE_ICEBERG` | Enables Iceberg catalog API tests. | -| `ACCEPTANCE_ENABLE_WIRE` | Enables wire-level tests outside the `wire` / `full` profiles. | -| `ACCEPTANCE_RLS_BUCKET` | Bucket used by opt-in RLS tests. Defaults to local dummy `bucket2`. | -| `ACCEPTANCE_RLS_READ_OBJECT` | Existing object used for RLS read checks; unset self-provisions one under the write prefix. | -| `ACCEPTANCE_RLS_WRITE_PREFIX` | Prefix where authenticated role may upload and anon may not. | -| `ACCEPTANCE_ALLOW_DESTRUCTIVE` | Required for destructive tests when `ACCEPTANCE_TARGET=remote`. | +| Variable | Meaning | +| ----------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `ACCEPTANCE_BASE_URL` | REST base URL. Defaults to `http://127.0.0.1:5000`. | +| `ACCEPTANCE_PROFILE` | Acceptance profile to run. Sample env and local CI default to `full`. | +| `ACCEPTANCE_S3_ENDPOINT` | S3 endpoint. Defaults to `$ACCEPTANCE_BASE_URL/s3`. | +| `ACCEPTANCE_TUS_ENDPOINT` | TUS endpoint. Defaults to `$ACCEPTANCE_BASE_URL/upload/resumable`. | +| `ACCEPTANCE_X_FORWARDED_HOST` | Optional tenant-routing host header for multitenant targets. | +| `ACCEPTANCE_ADMIN_URL` | Admin API base URL for admin tests. | +| `ACCEPTANCE_SERVICE_KEY` | Service role JWT for REST tests. | +| `ACCEPTANCE_S3_ACCESS_KEY_ID` | S3 protocol access key. | +| `ACCEPTANCE_S3_SECRET_ACCESS_KEY` | S3 protocol secret. | +| `ACCEPTANCE_REGION` | S3 signing region. Defaults to `us-east-1`. | +| `ACCEPTANCE_RESOURCE_PREFIX` | Prefix for all resources created by this run. | +| `ACCEPTANCE_ENABLE_ADMIN` | Enables admin route tests. Requires admin URL and API key. | +| `ACCEPTANCE_ADMIN_RETURN_TENANT_SENSITIVE_DATA` | Whether the admin API returns sensitive tenant fields (keys, database URL). Defaults to `true`. | +| `ACCEPTANCE_ADMIN_DATABASE_URL_OVERRIDE` | Database URL to provision tenants with when sensitive tenant data isn't returned (see above). | +| `ACCEPTANCE_ENABLE_CDN` | Enables CDN purge tests. Managed local runs provide a purge stub by default. | +| `ACCEPTANCE_ENABLE_RENDER` | Enables image transformation tests. | +| `ACCEPTANCE_ENABLE_RLS_SETUP` | Enables RLS tests; requires service, anon, authenticated keys and bucket/prefix policy resources. | +| `ACCEPTANCE_ENABLE_VECTOR` | Enables vector bucket API tests. Requires local pgvector or a configured S3 Vectors target. | +| `ACCEPTANCE_ENABLE_ICEBERG` | Enables Iceberg catalog API tests. | +| `ACCEPTANCE_ENABLE_WIRE` | Enables wire-level tests outside the `wire` / `full` profiles. | +| `ACCEPTANCE_RLS_BUCKET` | Bucket used by opt-in RLS tests. Defaults to local dummy `bucket2`. | +| `ACCEPTANCE_RLS_READ_OBJECT` | Existing object used for RLS read checks; unset self-provisions one under the write prefix. | +| `ACCEPTANCE_RLS_WRITE_PREFIX` | Prefix where authenticated role may upload and anon may not. | +| `ACCEPTANCE_ALLOW_DESTRUCTIVE` | Required for destructive tests when `ACCEPTANCE_TARGET=remote`. | ## HTTPS And Wire Tests @@ -161,3 +163,19 @@ npm run acceptance:run -- --profile wire `ACCEPTANCE_TLS_REJECT_UNAUTHORIZED=false` sets `NODE_TLS_REJECT_UNAUTHORIZED=0` in the runner for local self-signed certificates. Do not use it for remote runs unless the target is explicitly provisioned for that. + +## Reset A Target Project + +Repeated destructive runs against a shared target can leave behind buckets from crashed or interrupted runs. +The script `acceptance/scripts/reset-project.ts` wipes every storage, analytics (Iceberg), and vector bucket +in the target project so the next run starts clean. It preserves `ACCEPTANCE_RLS_BUCKET` (emptying its contents +but not deleting the bucket, since it carries hand-configured RLS policies the suite can't recreate). + +The script reads the same `ACCEPTANCE_*` variables as the rest of the suite and does not load an +env file itself, so run it with `tsx --env-file`: + +```bash +tsx --env-file=.env.acceptance-staging acceptance/scripts/reset-project.ts --yes +``` + +Omit `--yes` to preview what would be deleted without changing anything. diff --git a/acceptance/scripts/reset-project.ts b/acceptance/scripts/reset-project.ts new file mode 100644 index 000000000..478bfcbac --- /dev/null +++ b/acceptance/scripts/reset-project.ts @@ -0,0 +1,276 @@ +/** + * Wipes every storage/analytics/vector bucket in the target project so acceptance tests + * can run against a clean slate. Admin API resources (tenants, etc.) are out of scope. + * + * Usage: + * tsx --env-file=.env.acceptance-staging acceptance/scripts/reset-project.ts [--yes] + * + * Reads the same ACCEPTANCE_* variables as the rest of the acceptance suite + * Requires ACCEPTANCE_SERVICE_KEY + * pass --yes to actually delete anything, otherwise runs in dry-run mode. + */ +import { setTimeout as delay } from 'node:timers/promises' +import { getAcceptanceConfig } from '../support/config' +import { AcceptanceHttpClient, createRestClient } from '../support/http' +import { requireServiceKey } from '../support/resources' + +const CONCURRENCY = 5 +const LIST_PAGE_SIZE = 1000 +const EMPTY_BUCKET_POLL_INTERVAL_MS = 2000 +const EMPTY_BUCKET_POLL_TIMEOUT_MS = 5 * 60 * 1000 + +const confirmed = process.argv.includes('--yes') && process.env.ACCEPTANCE_ALLOW_DESTRUCTIVE + +interface StorageBucket { + id: string +} + +interface AnalyticsBucket { + name: string +} + +interface ListObjectsV2Response { + folders?: unknown[] + objects?: unknown[] +} + +interface VectorListBucketsResponse { + nextToken?: string + vectorBuckets?: Array<{ vectorBucketName?: string }> +} + +interface VectorListIndexesResponse { + indexes?: Array<{ indexName?: string }> + nextToken?: string +} + +async function main() { + const config = getAcceptanceConfig() + const token = requireServiceKey(config) + const client = createRestClient() + + console.log(`Target: ${config.baseUrl}`) + console.log( + confirmed + ? 'Mode: LIVE (--yes passed, resources will be deleted)' + : 'Mode: DRY RUN (pass --yes and ensure ACCEPTANCE_ALLOW_DESTRUCTIVE is set to actually delete anything)' + ) + if (config.rlsBucket) { + console.log(`Preserving RLS fixture bucket: ${config.rlsBucket} (emptied, not deleted)`) + } + console.log() + + await resetStorageBuckets(client, token, config.rlsBucket) + await resetAnalyticsBuckets(client, token) + await resetVectorBuckets(client, token) + + console.log('\nDone.') +} + +async function resetStorageBuckets( + client: AcceptanceHttpClient, + token: string, + preserveBucketId: string | undefined +) { + const buckets = await listAllPages((limit, offset) => + client + .request('GET', `/bucket?limit=${limit}&offset=${offset}`, { + expectedStatus: 200, + token, + }) + .then((res) => res.json ?? []) + ) + console.log(`Storage buckets: ${buckets.length}`) + + await mapWithConcurrency(buckets, CONCURRENCY, async (bucket) => { + const preserve = bucket.id === preserveBucketId + console.log(` [storage] emptying ${bucket.id}${preserve ? ' (preserving bucket)' : ''}`) + + if (confirmed) { + await emptyStorageBucketAndWait(client, token, bucket.id) + } + + if (preserve) { + return + } + + console.log(` [storage] deleting ${bucket.id}`) + if (confirmed) { + await client.request('DELETE', `/bucket/${bucket.id}`, { + expectedStatus: [200, 400, 404], + token, + }) + } + }) +} + +async function emptyStorageBucketAndWait( + client: AcceptanceHttpClient, + token: string, + bucketId: string +) { + await client.request('POST', `/bucket/${bucketId}/empty`, { + expectedStatus: 200, + token, + }) + + const deadline = Date.now() + EMPTY_BUCKET_POLL_TIMEOUT_MS + for (;;) { + const listing = await client.request( + 'POST', + `/object/list-v2/${bucketId}`, + { + body: { limit: 1 }, + expectedStatus: 200, + token, + } + ) + const isEmpty = + (listing.json?.folders?.length ?? 0) === 0 && (listing.json?.objects?.length ?? 0) === 0 + if (isEmpty) { + return + } + if (Date.now() > deadline) { + throw new Error(`Timed out waiting for bucket ${bucketId} to empty`) + } + await delay(EMPTY_BUCKET_POLL_INTERVAL_MS) + } +} + +async function resetAnalyticsBuckets(client: AcceptanceHttpClient, token: string) { + const buckets = await listAllPages((limit, offset) => + client + .request('GET', `/iceberg/bucket?limit=${limit}&offset=${offset}`, { + expectedStatus: 200, + token, + }) + .then((res) => res.json ?? []) + ) + console.log(`Analytics (Iceberg) buckets: ${buckets.length}`) + + // Deleting an analytics bucket queues an async job that cascades the delete to its + // namespaces and tables, so there's nothing to empty/poll here - just delete the bucket. + await mapWithConcurrency(buckets, CONCURRENCY, async (bucket) => { + console.log(` [analytics] deleting ${bucket.name}`) + if (confirmed) { + await client.request('DELETE', `/iceberg/bucket/${bucket.name}`, { + expectedStatus: [200, 400, 404], + token, + }) + } + }) +} + +async function resetVectorBuckets(client: AcceptanceHttpClient, token: string) { + const buckets: string[] = [] + let nextToken: string | undefined + + do { + const page = await client.request( + 'POST', + '/vector/ListVectorBuckets', + { + body: { maxResults: 500, nextToken }, + expectedStatus: 200, + token, + } + ) + for (const bucket of page.json?.vectorBuckets ?? []) { + if (bucket.vectorBucketName) { + buckets.push(bucket.vectorBucketName) + } + } + nextToken = page.json?.nextToken + } while (nextToken) + + console.log(`Vector buckets: ${buckets.length}`) + + await mapWithConcurrency(buckets, CONCURRENCY, async (vectorBucketName) => { + let indexToken: string | undefined + + do { + const page = await client.request('POST', '/vector/ListIndexes', { + body: { maxResults: 500, nextToken: indexToken, vectorBucketName }, + expectedStatus: 200, + token, + }) + + for (const index of page.json?.indexes ?? []) { + if (!index.indexName) { + continue + } + console.log(` [vector] deleting index ${vectorBucketName}/${index.indexName}`) + if (confirmed) { + await client.request('POST', '/vector/DeleteIndex', { + body: { indexName: index.indexName, vectorBucketName }, + expectedStatus: [200, 400, 404], + token, + }) + } + } + + indexToken = page.json?.nextToken + } while (indexToken) + + console.log(` [vector] deleting bucket ${vectorBucketName}`) + if (confirmed) { + await client.request('POST', '/vector/DeleteVectorBucket', { + body: { vectorBucketName }, + expectedStatus: [200, 400, 404], + token, + }) + } + }) +} + +/** Fetches limit/offset pages via `fetchPage` until a short page signals the end. */ +async function listAllPages( + fetchPage: (limit: number, offset: number) => Promise +): Promise { + const items: T[] = [] + let offset = 0 + + for (;;) { + const page = await fetchPage(LIST_PAGE_SIZE, offset) + items.push(...page) + if (page.length < LIST_PAGE_SIZE) { + return items + } + offset += LIST_PAGE_SIZE + } +} + +async function mapWithConcurrency( + items: T[], + concurrency: number, + fn: (item: T) => Promise +): Promise { + const queue = [...items] + const errors: unknown[] = [] + + async function worker() { + for (;;) { + const item = queue.shift() + if (item === undefined) { + return + } + try { + await fn(item) + } catch (error) { + errors.push(error) + console.error(' error:', error) + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker())) + + if (errors.length > 0) { + throw new Error(`${errors.length} resource(s) failed to reset (see errors above)`) + } +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/acceptance/specs/admin.test.ts b/acceptance/specs/admin.test.ts index 8fbd246e9..eae4190d5 100644 --- a/acceptance/specs/admin.test.ts +++ b/acceptance/specs/admin.test.ts @@ -320,8 +320,23 @@ describeAcceptance( const tenantIdsToCleanup = new Set() try { + const createTenantOverrides = config.adminReturnSensitiveData + ? { + fileSizeLimit: 1_048_576, + } + : { + fileSizeLimit: 1_048_576, + anonKey: 'abc', + databaseUrl: config.adminDatabaseUrlOverride ?? 'def', + jwtSecret: 'ghi', + serviceKey: 'jkl', + } await client.request('POST', `/tenants/${createdTenantId}`, { - body: buildTenantProvisionBody(sourceTenant.json, 1_048_576), + body: buildTenantProvisionBody( + sourceTenant.json, + createTenantOverrides, + config.adminReturnSensitiveData + ), expectedStatus: 201, headers, }) @@ -397,15 +412,45 @@ describeAcceptance( expect(reset.json?.message).toBe('Migrations reset') } + const putTenantOverrides = config.adminReturnSensitiveData + ? { + fileSizeLimit: 3_145_728, + } + : { + fileSizeLimit: 3_145_728, + anonKey: 'abc', + databaseUrl: config.adminDatabaseUrlOverride ?? 'def', + jwtSecret: 'ghi', + serviceKey: 'jkl', + } await client.request('PUT', `/tenants/${upsertedTenantId}`, { - body: buildTenantProvisionBody(sourceTenant.json, 3_145_728), + body: buildTenantProvisionBody( + sourceTenant.json, + putTenantOverrides, + config.adminReturnSensitiveData + ), expectedStatus: 204, headers, }) tenantIdsToCleanup.add(upsertedTenantId) + const putTenantOverride2 = config.adminReturnSensitiveData + ? { + fileSizeLimit: 4_194_304, + } + : { + fileSizeLimit: 4_194_304, + anonKey: 'abc', + databaseUrl: config.adminDatabaseUrlOverride ?? 'def', + jwtSecret: 'ghi', + serviceKey: 'jkl', + } await client.request('PUT', `/tenants/${upsertedTenantId}`, { - body: buildTenantProvisionBody(sourceTenant.json, 4_194_304), + body: buildTenantProvisionBody( + sourceTenant.json, + putTenantOverride2, + config.adminReturnSensitiveData + ), expectedStatus: 204, headers, }) @@ -562,15 +607,17 @@ function uniqueTenantId(): string { function buildTenantProvisionBody( sourceTenant: TenantDetailResponse | undefined, - fileSizeLimit: number + overrides: Partial, + returnSensitiveData: boolean ): Record { + const checkExpectedSensitiveData = returnSensitiveData ? requireString : requireUndefined const body: Record = { - anonKey: requireString(sourceTenant?.anonKey, 'anonKey'), - databaseUrl: requireString(sourceTenant?.databaseUrl, 'databaseUrl'), + anonKey: checkExpectedSensitiveData(sourceTenant?.anonKey, 'anonKey'), + databaseUrl: checkExpectedSensitiveData(sourceTenant?.databaseUrl, 'databaseUrl'), features: normalizeTenantFeatures(sourceTenant?.features), - fileSizeLimit, - jwtSecret: requireString(sourceTenant?.jwtSecret, 'jwtSecret'), - serviceKey: requireString(sourceTenant?.serviceKey, 'serviceKey'), + jwtSecret: checkExpectedSensitiveData(sourceTenant?.jwtSecret, 'jwtSecret'), + serviceKey: checkExpectedSensitiveData(sourceTenant?.serviceKey, 'serviceKey'), + ...overrides, } if (typeof sourceTenant?.databasePoolUrl === 'string') { @@ -617,3 +664,10 @@ function requireString(value: unknown, name: string): string { return value } + +function requireUndefined(value: unknown, name: string): unknown { + if (typeof value !== 'undefined') { + throw new Error(`Admin tenant response unexpectedly included ${name}`) + } + return value +} diff --git a/acceptance/specs/cdn.test.ts b/acceptance/specs/cdn.test.ts new file mode 100644 index 000000000..f35416b0f --- /dev/null +++ b/acceptance/specs/cdn.test.ts @@ -0,0 +1,969 @@ +import { setTimeout as delay } from 'node:timers/promises' +import { + describeAcceptance, + encodePathSegments, + getAcceptanceConfig, + requireConfigValue, +} from '../support/config' +import { AcceptanceHttpClient, createRestClient, parseStorageError } from '../support/http' +import { + cleanupRestObjects, + cleanupRestResources, + createRestBucket, + requireServiceKey, + uniqueBucketName, + uniqueObjectKey, + uploadRestObject, +} from '../support/resources' + +interface CdnPurgeResponse { + message: string + statusCode?: string +} + +interface SignedUrlResponse { + signedURL: string +} + +type BucketType = 'public' | 'private' +type AccessMethod = 'public' | 'authenticated' | 'signed' +type CacheStatus = 'HIT' | 'MISS' | 'REVALIDATED' | 'BYPASS' | 'DYNAMIC' | 'EXPIRED' + +interface TestConfig { + bucketType: BucketType + accessMethods: AccessMethod[] +} + +interface TransformParams { + width: number + height: number +} + +interface GetObjectUrlOptions { + transform?: TransformParams + expiresIn?: number +} + +const onePixelPng = new Uint8Array( + Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p94AAAAASUVORK5CYII=', + 'base64' + ) +) + +const SIGNED_EXPIRES_IN_S = 7 +const CACHE_RETRIES = 10 +const TEST_CONFIGS: TestConfig[] = [ + { bucketType: 'public', accessMethods: ['public'] }, + { bucketType: 'private', accessMethods: ['authenticated', 'signed'] }, +] + +/** + * Adds a pause conditionally for signed requests + * + * This must be placed after write operations (upload, upsert, move, copy) for signed requests + * to ensure the webhook is processed before the first GET + * + * Without this we get 60s of EXPIRED or REVALIDATED headers due to KV cache + */ +async function pauseForWebhookIfNeeded(accessMethod: AccessMethod) { + if (accessMethod === 'signed') { + await delay(3000) + } +} + +/** + * Get a route/token for an object based on bucket type and access method. + */ +async function getObjectUrl( + bucketName: string, + objectKey: string, + bucketType: BucketType, + accessMethod: AccessMethod, + { transform, expiresIn = SIGNED_EXPIRES_IN_S }: GetObjectUrlOptions = {} +): Promise<{ route: string; token?: string }> { + const config = getAcceptanceConfig() + const encodedKey = encodePathSegments(objectKey) + const routePrefix = transform ? 'render/image' : 'object' + const query = transform ? `?width=${transform.width}&height=${transform.height}` : '' + + if (bucketType === 'public' && accessMethod === 'public') { + return { + route: `/${routePrefix}/public/${bucketName}/${encodedKey}${query}`, + } + } + + if (accessMethod === 'authenticated') { + return { + token: requireConfigValue(config.authenticatedKey, 'ACCEPTANCE_AUTHENTICATED_KEY'), + route: `/${routePrefix}/authenticated/${bucketName}/${encodedKey}${query}`, + } + } + + if (accessMethod === 'signed') { + const client = createRestClient() + const signed = await client.request( + 'POST', + `/object/sign/${bucketName}/${encodedKey}`, + { + body: { + expiresIn, + ...(transform + ? { transform: { height: transform.height, width: transform.width, resize: 'contain' } } + : {}), + }, + expectedStatus: 200, + token: requireServiceKey(config), + } + ) + return { + route: signed.json?.signedURL ?? '', + } + } + + throw new Error(`Invalid access method: ${accessMethod}`) +} + +async function warmCacheEndpoint(client: AcceptanceHttpClient, route: string, token?: string) { + const miss = await client.request('GET', route, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + token, + retries: CACHE_RETRIES, + }) + const hit = await client.request('GET', route, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + token, + retries: CACHE_RETRIES, + }) + return { hit, miss } +} + +/** + * Replaces the `token` query param on a signed route + */ +function withToken(route: string, token: string): string { + const parsed = new URL(route, 'http://abc') + parsed.searchParams.set('token', token) + return parsed.pathname + parsed.search +} + +/** + * Returns the `token` query param from a signed route + */ +function getToken(route: string): string { + return new URL(route, 'http://abc').searchParams.get('token') || '' +} + +describeAcceptance( + 'CDN cache purge operations', + { + destructive: true, + profiles: ['full'], + requires: ['cdn'], + }, + () => { + let client: AcceptanceHttpClient + let bucketName: string + const canCdnEdge = getAcceptanceConfig().capabilities.cdnEdge + const canRender = getAcceptanceConfig().capabilities.render + + beforeAll(async () => { + client = createRestClient() + bucketName = uniqueBucketName('cdn-purge') + await createRestBucket(bucketName, { isPublic: true }) + }) + + afterAll(async () => { + await cleanupRestResources(bucketName, [], client) + }) + + it('purges an object cache entry', async () => { + const purgedKey = uniqueObjectKey('cdn-purge-target') + const controlKey = uniqueObjectKey('cdn-purge-control') + let purgedRoute = '' + let controlRoute = '' + + try { + if (canCdnEdge) { + await uploadRestObject(bucketName, purgedKey, 'cdn-purge-target-content') + await uploadRestObject(bucketName, controlKey, 'cdn-purge-control-content') + + purgedRoute = (await getObjectUrl(bucketName, purgedKey, 'public', 'public')).route + controlRoute = (await getObjectUrl(bucketName, controlKey, 'public', 'public')).route + + await warmCacheEndpoint(client, purgedRoute) + await warmCacheEndpoint(client, controlRoute) + } + + const purge = await client.request( + 'DELETE', + `/cdn/${bucketName}/${encodePathSegments(purgedKey)}`, + { + expectedStatus: 200, + token: requireServiceKey(), + } + ) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge) { + await client.request('GET', purgedRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', controlRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + }) + } + } finally { + if (canCdnEdge) { + await cleanupRestObjects(bucketName, [purgedKey, controlKey], client) + } + } + }) + + it('purges object transformations cache', async () => { + const objectKey = uniqueObjectKey('cdn-purge', 'png') + let objectRoute = '' + let transformRoute = '' + + try { + if (canCdnEdge && canRender) { + await uploadRestObject(bucketName, objectKey, onePixelPng, { + contentType: 'image/png', + }) + + objectRoute = (await getObjectUrl(bucketName, objectKey, 'public', 'public')).route + transformRoute = ( + await getObjectUrl(bucketName, objectKey, 'public', 'public', { + transform: { + width: 10, + height: 10, + }, + }) + ).route + + await warmCacheEndpoint(client, objectRoute) + await warmCacheEndpoint(client, transformRoute) + } + + const purge = await client.request( + 'DELETE', + `/cdn/${bucketName}/${encodePathSegments(objectKey)}?transformations=true`, + { + expectedStatus: 200, + token: requireServiceKey(), + } + ) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge && canRender) { + await warmCacheEndpoint(client, transformRoute) + } + } finally { + if (canCdnEdge && canRender) { + await cleanupRestObjects(bucketName, [objectKey], client) + } + } + }) + + it('purges an entire bucket cache', async () => { + const objectKey = uniqueObjectKey('cdn-purge-bucket') + const controlBucketName = uniqueBucketName('cdn-purge-bucket-control') + const controlKey = uniqueObjectKey('cdn-purge-bucket-control') + let route = '' + let controlRoute = '' + + try { + if (canCdnEdge) { + await createRestBucket(controlBucketName, { isPublic: true }) + await uploadRestObject(bucketName, objectKey, 'cdn-purge-bucket-target') + await uploadRestObject(controlBucketName, controlKey, 'cdn-purge-bucket-control') + + route = (await getObjectUrl(bucketName, objectKey, 'public', 'public')).route + controlRoute = (await getObjectUrl(controlBucketName, controlKey, 'public', 'public')) + .route + + await warmCacheEndpoint(client, route) + await warmCacheEndpoint(client, controlRoute) + } + + const purge = await client.request('DELETE', `/cdn/${bucketName}`, { + expectedStatus: 200, + token: requireServiceKey(), + }) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge) { + await client.request('GET', route, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', controlRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + }) + } + } finally { + if (canCdnEdge) { + await cleanupRestResources(controlBucketName, [controlKey], client) + await cleanupRestObjects(bucketName, [objectKey], client) + } + } + }) + + it('purges bucket transformations cache', async () => { + const objectKey = uniqueObjectKey('cdn-purge-bucket', 'png') + let objectRoute = '' + let transformRoute = '' + + try { + if (canCdnEdge && canRender) { + await uploadRestObject(bucketName, objectKey, onePixelPng, { + contentType: 'image/png', + }) + + objectRoute = (await getObjectUrl(bucketName, objectKey, 'public', 'public')).route + transformRoute = ( + await getObjectUrl(bucketName, objectKey, 'public', 'public', { + transform: { + width: 10, + height: 10, + }, + }) + ).route + + await warmCacheEndpoint(client, objectRoute) + await warmCacheEndpoint(client, transformRoute) + } + + const purge = await client.request( + 'DELETE', + `/cdn/${bucketName}?transformations=true`, + { + expectedStatus: 200, + token: requireServiceKey(), + } + ) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge && canRender) { + await client.request('GET', transformRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', objectRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + }) + } + } finally { + if (canCdnEdge && canRender) { + await cleanupRestObjects(bucketName, [objectKey], client) + } + } + }) + + it('purges entire tenant cache', async () => { + const objectKey = uniqueObjectKey('cdn-purge-tenant') + const secondBucketName = uniqueBucketName('cdn-purge-tenant') + const secondKey = uniqueObjectKey('cdn-purge-tenant') + let route = '' + let secondRoute = '' + + try { + if (canCdnEdge) { + await createRestBucket(secondBucketName, { isPublic: true }) + await uploadRestObject(bucketName, objectKey, 'cdn-purge-tenant-target') + await uploadRestObject(secondBucketName, secondKey, 'cdn-purge-tenant-target-2') + + route = (await getObjectUrl(bucketName, objectKey, 'public', 'public')).route + secondRoute = (await getObjectUrl(secondBucketName, secondKey, 'public', 'public')).route + + await warmCacheEndpoint(client, route) + await warmCacheEndpoint(client, secondRoute) + } + + const purge = await client.request('DELETE', '/cdn/', { + expectedStatus: 200, + token: requireServiceKey(), + }) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge) { + await client.request('GET', route, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', secondRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + } + } finally { + if (canCdnEdge) { + await cleanupRestResources(secondBucketName, [secondKey], client) + await cleanupRestObjects(bucketName, [objectKey], client) + } + } + }) + + it('purges tenant transformations cache', async () => { + const objectKey = uniqueObjectKey('cdn-purge-tenant', 'png') + const secondBucketName = uniqueBucketName('cdn-purge-tenant') + const secondKey = uniqueObjectKey('cdn-purge-tenant', 'png') + let objectRoute = '' + let transformRoute = '' + let secondObjectRoute = '' + let secondTransformRoute = '' + + try { + if (canCdnEdge && canRender) { + await createRestBucket(secondBucketName, { isPublic: true }) + await uploadRestObject(bucketName, objectKey, onePixelPng, { contentType: 'image/png' }) + await uploadRestObject(secondBucketName, secondKey, onePixelPng, { + contentType: 'image/png', + }) + + objectRoute = (await getObjectUrl(bucketName, objectKey, 'public', 'public')).route + transformRoute = ( + await getObjectUrl(bucketName, objectKey, 'public', 'public', { + transform: { + width: 10, + height: 10, + }, + }) + ).route + secondObjectRoute = (await getObjectUrl(secondBucketName, secondKey, 'public', 'public')) + .route + secondTransformRoute = ( + await getObjectUrl(secondBucketName, secondKey, 'public', 'public', { + transform: { + width: 10, + height: 10, + }, + }) + ).route + + await warmCacheEndpoint(client, objectRoute) + await warmCacheEndpoint(client, transformRoute) + await warmCacheEndpoint(client, secondObjectRoute) + await warmCacheEndpoint(client, secondTransformRoute) + } + + const purge = await client.request( + 'DELETE', + '/cdn?transformations=true', + { + expectedStatus: 200, + token: requireServiceKey(), + } + ) + expect(purge.json).toMatchObject({ + message: 'success', + }) + + if (canCdnEdge && canRender) { + await client.request('GET', transformRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', secondTransformRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + }) + await client.request('GET', objectRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + }) + await client.request('GET', secondObjectRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + }) + } + } finally { + if (canCdnEdge && canRender) { + await cleanupRestResources(secondBucketName, [secondKey], client) + await cleanupRestObjects(bucketName, [objectKey], client) + } + } + }) + } +) + +TEST_CONFIGS.forEach(({ bucketType, accessMethods }) => { + accessMethods.forEach((accessMethod) => { + const isPublic = bucketType === 'public' + const isAuthenticated = accessMethod === 'authenticated' + const isSigned = accessMethod === 'signed' + const testLabel = `${bucketType} bucket with ${accessMethod} access` + + describeAcceptance( + `CDN cache behavior - ${testLabel}`, + { + destructive: true, + profiles: ['full'], + requires: isAuthenticated ? ['cdnEdge', 'rlsSetup'] : ['cdnEdge'], + }, + () => { + const canRender = getAcceptanceConfig().capabilities.render + let client: AcceptanceHttpClient + let bucketName: string + let writePrefix: string | undefined + + beforeAll(async () => { + client = createRestClient() + + if (isAuthenticated) { + // Authenticated access is governed by RLS, so reuse the shared RLS + // bucket/prefix instead of provisioning a dedicated bucket. + const config = getAcceptanceConfig() + bucketName = requireConfigValue(config.rlsBucket, 'ACCEPTANCE_RLS_BUCKET') + writePrefix = requireConfigValue( + config.rlsWritePrefix, + 'ACCEPTANCE_RLS_WRITE_PREFIX' + ).replace(/\/+$/, '') + } else { + bucketName = uniqueBucketName(`cdn-${bucketType}-${accessMethod}`) + await createRestBucket(bucketName, { isPublic }) + } + }) + + afterAll(async () => { + if (!isAuthenticated) { + await cleanupRestResources(bucketName, [], client) + } + }) + + function makeObjectKey(kind: string, extension = 'txt'): string { + const key = uniqueObjectKey(kind, extension) + return writePrefix ? `${writePrefix}/${key}` : key + } + + it('supports basic cache MISS / HIT for multiple files', async () => { + const files = [ + { key: makeObjectKey('file1'), content: 'Content of file 1' }, + { key: makeObjectKey('file2'), content: 'Content of file 2' }, + { key: makeObjectKey('file3'), content: 'Content of file 3' }, + ] + + try { + const items = await Promise.all( + files.map(async (file) => { + await uploadRestObject(bucketName, file.key, file.content) + const { route, token } = await getObjectUrl( + bucketName, + file.key, + bucketType, + accessMethod + ) + return { file, route, token } + }) + ) + + await pauseForWebhookIfNeeded(accessMethod) + + // check invalid token / access denied + if (isSigned) { + const invalidRoute = withToken(items[0].route, getToken(items[1].route)) + const denied = await client.request('GET', invalidRoute, { + expectedCacheStatus: 'BYPASS', + expectedStatus: 400, + }) + expect(parseStorageError(denied.json)).toMatchObject({ + statusCode: '400', + error: 'InvalidSignature', + }) + } else if (isAuthenticated) { + const anonKey = requireConfigValue( + getAcceptanceConfig().anonKey, + 'ACCEPTANCE_ANON_KEY' + ) + const denied = await client.request('GET', items[0].route, { + expectedCacheStatus: 'DYNAMIC', + expectedStatus: 400, + token: anonKey, + }) + expect(parseStorageError(denied.json)).toMatchObject({ + statusCode: '404', + error: 'not_found', + }) + } + + for (let i = 0; i < items.length; i++) { + const { file, route, token } = items[i] + const first = await client.request('GET', route, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + token, + }) + expect(first.body).toBe(file.content) + + const second = await client.request('GET', route, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + retries: CACHE_RETRIES, + token, + }) + expect(second.body).toBe(file.content) + } + } finally { + await cleanupRestObjects( + bucketName, + files.map((f) => f.key), + client + ) + } + }) + + it('supports ETag-based conditional requests with 304 responses', async () => { + const objectKey = makeObjectKey('etag') + const content = 'etag-test-content' + + try { + await uploadRestObject(bucketName, objectKey, content) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route, token } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod + ) + + const first = await client.request('GET', route, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + token, + }) + const etag = first.headers.get('etag') + expect(etag).toBeTruthy() + + const second = await client.request('GET', route, { + expectedCacheStatus: 'HIT', + expectedStatus: 304, + headers: { 'If-None-Match': etag! }, + token, + }) + + expect(second.body).toBe('') + } finally { + await cleanupRestObjects(bucketName, [objectKey], client) + } + }) + + it('invalidates cache after UPSERT operation', async () => { + const objectKey = makeObjectKey('upsert') + const initialContent = 'initial-content' + const updatedContent = 'updated-content' + + try { + await uploadRestObject(bucketName, objectKey, initialContent) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route, token } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod + ) + + await warmCacheEndpoint(client, route, token) + + await uploadRestObject(bucketName, objectKey, updatedContent) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: newRoute, token: newToken } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod + ) + + const afterUpsert = await client.request('GET', newRoute, { + expectedCacheStatus: 'MISS', + expectedStatus: 200, + retries: CACHE_RETRIES, + token: newToken, + }) + expect(afterUpsert.body).toBe(updatedContent) + + await client.request('GET', newRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + retries: CACHE_RETRIES, + token: newToken, + }) + } finally { + await cleanupRestObjects(bucketName, [objectKey], client) + } + }) + + it('invalidates cache at old path and creates new cache after MOVE', async () => { + const config = getAcceptanceConfig() + const oldKey = makeObjectKey('move-old') + const newKey = makeObjectKey('move-new') + const content = 'move-test-content' + + try { + await uploadRestObject(bucketName, oldKey, content) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: oldRoute, token: oldToken } = await getObjectUrl( + bucketName, + oldKey, + bucketType, + accessMethod + ) + + await warmCacheEndpoint(client, oldRoute, oldToken) + + let nonExpiringSignedRoute = '' + if (isSigned) { + nonExpiringSignedRoute = ( + await getObjectUrl(bucketName, oldKey, bucketType, accessMethod, { + expiresIn: 1000, + }) + ).route + await warmCacheEndpoint(client, nonExpiringSignedRoute) + } + + await client.request('POST', '/object/move', { + body: { + bucketId: bucketName, + destinationKey: newKey, + sourceKey: oldKey, + }, + expectedStatus: 200, + token: requireServiceKey(config), + }) + + if (isSigned) { + // Signed requests return HIT for 60s or until the token expires + await delay(60_000) + + const oldResult = await client.request('GET', nonExpiringSignedRoute, { + expectedCacheStatus: 'BYPASS', + expectedStatus: 400, + retries: CACHE_RETRIES, + }) + expect(parseStorageError(oldResult.json)).toMatchObject({ + statusCode: '404', + error: 'not_found', + }) + } + + const expectedError = isSigned + ? { statusCode: '400', error: 'InvalidJWT' } + : { statusCode: '404', error: 'not_found' } + const expectedCacheStatus: CacheStatus = isSigned ? 'BYPASS' : 'DYNAMIC' + const oldResult = await client.request('GET', oldRoute, { + expectedCacheStatus, + expectedStatus: 400, + retries: CACHE_RETRIES, + token: oldToken, + }) + expect(parseStorageError(oldResult.json)).toMatchObject(expectedError) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: newRoute, token: newToken } = await getObjectUrl( + bucketName, + newKey, + bucketType, + accessMethod + ) + + const { hit, miss } = await warmCacheEndpoint(client, newRoute, newToken) + expect(hit.body).toBe(content) + expect(miss.body).toBe(content) + } finally { + await cleanupRestObjects(bucketName, [newKey], client) + } + }, 90_000) // signed cache takes 60s to expire + + it('creates new cache entry for COPY destination without affecting source', async () => { + const config = getAcceptanceConfig() + const sourceKey = makeObjectKey('copy-src') + const destKey = makeObjectKey('copy-dest') + const content = 'copy-test-content' + + try { + await uploadRestObject(bucketName, sourceKey, content) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: sourceRoute, token: sourceToken } = await getObjectUrl( + bucketName, + sourceKey, + bucketType, + accessMethod + ) + await warmCacheEndpoint(client, sourceRoute, sourceToken) + + await client.request('POST', '/object/copy', { + body: { + bucketId: bucketName, + destinationKey: destKey, + sourceKey, + }, + expectedStatus: 200, + token: requireServiceKey(config), + }) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: destRoute, token: destToken } = await getObjectUrl( + bucketName, + destKey, + bucketType, + accessMethod + ) + + const { hit, miss } = await warmCacheEndpoint(client, destRoute, destToken) + expect(hit.body).toBe(content) + expect(miss.body).toBe(content) + + await client.request('GET', sourceRoute, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + token: sourceToken, + }) + } finally { + await cleanupRestObjects(bucketName, [sourceKey, destKey], client) + } + }) + + it('invalidates cache after DELETE operation', async () => { + const objectKey = makeObjectKey('delete') + const content = 'delete-test-content' + + try { + await uploadRestObject(bucketName, objectKey, content) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route, token } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod + ) + + await warmCacheEndpoint(client, route, token) + + let nonExpiringSignedRoute = '' + if (isSigned) { + nonExpiringSignedRoute = ( + await getObjectUrl(bucketName, objectKey, bucketType, accessMethod, { + expiresIn: 1000, + }) + ).route + await warmCacheEndpoint(client, nonExpiringSignedRoute) + } + + await client.request( + 'DELETE', + `/object/${bucketName}/${encodePathSegments(objectKey)}`, + { + expectedStatus: 200, + token: requireServiceKey(), + } + ) + + if (isSigned) { + // Signed requests return HIT for 60s or until the token expires + await delay(60_000) + + const oldResult = await client.request('GET', nonExpiringSignedRoute, { + expectedCacheStatus: 'BYPASS', + expectedStatus: 400, + retries: CACHE_RETRIES, + }) + expect(parseStorageError(oldResult.json)).toMatchObject({ + statusCode: '404', + error: 'not_found', + }) + } + + const expectedError = isSigned + ? { statusCode: '400', error: 'InvalidJWT' } + : { statusCode: '404', error: 'not_found' } + const expectedCacheStatus: CacheStatus = isSigned ? 'BYPASS' : 'DYNAMIC' + const result = await client.request('GET', route, { + expectedCacheStatus, + expectedStatus: 400, + retries: CACHE_RETRIES, + token, + }) + expect(parseStorageError(result.json)).toMatchObject(expectedError) + } finally { + await cleanupRestObjects(bucketName, [objectKey], client) + } + }, 90_000) // signed cache takes 60s to expire + + it.skipIf(!canRender)('caches different transformations independently', async () => { + const objectKey = makeObjectKey('transform', 'png') + + try { + await uploadRestObject(bucketName, objectKey, onePixelPng, { + contentType: 'image/png', + }) + + await pauseForWebhookIfNeeded(accessMethod) + + const { route: transform1Route, token: token1 } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod, + { transform: { width: 10, height: 10 } } + ) + const { route: transform2Route, token: token2 } = await getObjectUrl( + bucketName, + objectKey, + bucketType, + accessMethod, + { transform: { width: 20, height: 20 } } + ) + + await warmCacheEndpoint(client, transform1Route, token1) + await warmCacheEndpoint(client, transform2Route, token2) + + await client.request('GET', transform1Route, { + expectedCacheStatus: 'HIT', + expectedStatus: 200, + retries: CACHE_RETRIES, + token: token1, + }) + } finally { + await cleanupRestObjects(bucketName, [objectKey], client) + } + }) + } + ) + }) +}) diff --git a/acceptance/specs/cdn-render.test.ts b/acceptance/specs/render.test.ts similarity index 65% rename from acceptance/specs/cdn-render.test.ts rename to acceptance/specs/render.test.ts index ce785e76b..2e303dbb5 100644 --- a/acceptance/specs/cdn-render.test.ts +++ b/acceptance/specs/render.test.ts @@ -18,11 +18,6 @@ interface SignedUrlResponse { signedURL: string } -interface CdnPurgeResponse { - message: string - statusCode: string -} - const onePixelPng = new Uint8Array( Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p94AAAAASUVORK5CYII=', @@ -33,132 +28,6 @@ const onePixelPng = new Uint8Array( // v3.26 identifies it as HEIF-like, then rejects it as incompatible with heic/avif. const incompatibleMif1Heif = new Uint8Array(Buffer.from('00000010667479706d69663100000000', 'hex')) -describeAcceptance( - 'CDN cache contract', - { - destructive: true, - profiles: ['full'], - requires: ['cdn'], - }, - () => { - it('purges an object cache entry', async () => { - const client = createRestClient() - const bucketName = uniqueBucketName('cdn') - const objectKey = uniqueObjectKey('cdn') - - try { - await createRestBucket(bucketName, { isPublic: true }) - await uploadRestObject(bucketName, objectKey, 'cdn-purge-target') - - const purge = await client.request( - 'DELETE', - `/cdn/${bucketName}/${encodePathSegments(objectKey)}`, - { - expectedStatus: 200, - token: requireServiceKey(), - } - ) - expect(purge.json).toMatchObject({ - message: 'success', - }) - } finally { - await cleanupRestResources(bucketName, [objectKey], client) - } - }) - - it('purges object transformations cache', async () => { - const client = createRestClient() - const bucketName = uniqueBucketName('cdn') - const objectKey = uniqueObjectKey('cdn') - - try { - await createRestBucket(bucketName, { isPublic: true }) - await uploadRestObject(bucketName, objectKey, 'cdn-purge-transforms-target') - - const purge = await client.request( - 'DELETE', - `/cdn/${bucketName}/${encodePathSegments(objectKey)}?transformations=true`, - { - expectedStatus: 200, - token: requireServiceKey(), - } - ) - expect(purge.json).toMatchObject({ - message: 'success', - }) - } finally { - await cleanupRestResources(bucketName, [objectKey], client) - } - }) - - it('purges an entire bucket cache', async () => { - const client = createRestClient() - const bucketName = uniqueBucketName('cdn') - - try { - await createRestBucket(bucketName, { isPublic: true }) - - const purge = await client.request('DELETE', `/cdn/${bucketName}`, { - expectedStatus: 200, - token: requireServiceKey(), - }) - expect(purge.json).toMatchObject({ - message: 'success', - }) - } finally { - await cleanupRestResources(bucketName, [], client) - } - }) - - it('purges bucket transformations cache', async () => { - const client = createRestClient() - const bucketName = uniqueBucketName('cdn') - - try { - await createRestBucket(bucketName, { isPublic: true }) - - const purge = await client.request( - 'DELETE', - `/cdn/${bucketName}?transformations=true`, - { - expectedStatus: 200, - token: requireServiceKey(), - } - ) - expect(purge.json).toMatchObject({ - message: 'success', - }) - } finally { - await cleanupRestResources(bucketName, [], client) - } - }) - - it('purges entire tenant cache', async () => { - const client = createRestClient() - - const purge = await client.request('DELETE', '/cdn/', { - expectedStatus: 200, - token: requireServiceKey(), - }) - expect(purge.json).toMatchObject({ - message: 'success', - }) - }) - - it('purges tenant transformations cache', async () => { - const client = createRestClient() - - const purge = await client.request('DELETE', '/cdn?transformations=true', { - expectedStatus: 200, - token: requireServiceKey(), - }) - expect(purge.json).toMatchObject({ - message: 'success', - }) - }) - } -) - describeAcceptance( 'image rendering contract', { diff --git a/acceptance/specs/rest-extended.test.ts b/acceptance/specs/rest-extended.test.ts index 3e92ab76d..dd90fb633 100644 --- a/acceptance/specs/rest-extended.test.ts +++ b/acceptance/specs/rest-extended.test.ts @@ -241,6 +241,7 @@ describeAcceptance( const updatedPayload = `${payload}-updated` const signedPayload = `${payload}-signed` const trackedKeys = [objectKey, signedUploadKey, copyKey, movedKey] + const httpRetries = config.target === 'remote' ? 10 : 0 try { await createRestBucket(bucketName, { isPublic: false }) @@ -456,7 +457,7 @@ describeAcceptance( await client.request( 'GET', `/object/authenticated/${bucketName}/${encodePathSegments(copyKey)}`, - { expectedStatus: [400, 404], token } + { expectedStatus: [400, 404], token, retries: httpRetries } ) const deleted = await client.request('DELETE', `/object/${bucketName}`, { @@ -476,7 +477,7 @@ describeAcceptance( await client.request( 'GET', `/object/authenticated/${bucketName}/${encodePathSegments(objectKey)}`, - { expectedStatus: [400, 404], token } + { expectedStatus: [400, 404], token, retries: httpRetries } ) const missingHead = await client.request( @@ -510,6 +511,7 @@ describeAcceptance( runId: config.runId, nested: { value: 'alpha' }, } + const httpRetries = config.target === 'remote' ? 10 : 0 try { await createRestBucket(sourceBucket, { isPublic: false }) @@ -719,7 +721,7 @@ describeAcceptance( await client.request( 'GET', `/object/authenticated/${sourceBucket}/${encodePathSegments(bravoKey)}`, - { expectedStatus: [400, 404], token } + { expectedStatus: [400, 404], token, retries: httpRetries } ) const moved = await client.request( diff --git a/acceptance/specs/s3.test.ts b/acceptance/specs/s3.test.ts index 6ee2899a7..4e3b254e4 100644 --- a/acceptance/specs/s3.test.ts +++ b/acceptance/specs/s3.test.ts @@ -456,7 +456,7 @@ describeAcceptance( Key: keyA, ResponseCacheControl: 'no-cache, no-store', ResponseContentDisposition: 'attachment; filename="acceptance.txt"', - ResponseContentEncoding: 'gzip', + ResponseContentEncoding: 'identity', ResponseContentLanguage: 'en-US', ResponseContentType: 'application/octet-stream', ResponseExpires: expires, @@ -464,7 +464,7 @@ describeAcceptance( ) expect(overridden.CacheControl).toBe('no-cache, no-store') expect(overridden.ContentDisposition).toBe('attachment; filename="acceptance.txt"') - expect(overridden.ContentEncoding).toBe('gzip') + expect(overridden.ContentEncoding).toBe('identity') expect(overridden.ContentLanguage).toBe('en-US') expect(overridden.ContentType).toBe('application/octet-stream') expect(overridden.ExpiresString).toBe(expires.toUTCString()) diff --git a/acceptance/specs/tus.test.ts b/acceptance/specs/tus.test.ts index 6af0375f1..072a4d7a1 100644 --- a/acceptance/specs/tus.test.ts +++ b/acceptance/specs/tus.test.ts @@ -543,7 +543,13 @@ async function createTusUpload({ method: 'OPTIONS', }) expect([200, 204]).toContain(options.status) - expect(options.headers.get('tus-version')).toContain(tusVersion) + if (options.headers.has('cf-ray')) { + // api gateway does not proxy OPTIONS calls so the tus headers are not included + expect(options.headers.has('tus-version')).toBe(false) + } else { + // without gateway tus-version should be set + expect(options.headers.get('tus-version')).toContain(tusVersion) + } } finally { await options?.body?.cancel() } diff --git a/acceptance/support/config.ts b/acceptance/support/config.ts index d5bb12f54..399fe4fb4 100644 --- a/acceptance/support/config.ts +++ b/acceptance/support/config.ts @@ -5,6 +5,7 @@ export type AcceptanceProfile = (typeof acceptanceProfiles)[number] export type AcceptanceCapability = | 'admin' | 'cdn' + | 'cdnEdge' | 'iceberg' | 'render' | 'rlsSetup' @@ -15,6 +16,8 @@ export type AcceptanceCapability = export interface AcceptanceConfig { adminApiKey?: string adminUrl?: string + adminReturnSensitiveData: boolean + adminDatabaseUrlOverride?: string allowDestructive: boolean anonKey?: string authenticatedKey?: string @@ -128,6 +131,10 @@ function buildAcceptanceConfig(): AcceptanceConfig { 'enable-admin', envOption('ACCEPTANCE_ENABLE_ADMIN') ) + const adminReturnSensitiveData = boolOptionDefaultTrue( + 'enable-admin-sensitive-data', + envOption('ACCEPTANCE_ADMIN_RETURN_TENANT_SENSITIVE_DATA') + ) const runId = sanitizeRunId( readOption('run-id') ?? envOption('ACCEPTANCE_RUN_ID') ?? @@ -144,17 +151,21 @@ function buildAcceptanceConfig(): AcceptanceConfig { const adminCapabilityEnabled = adminCapabilityOption !== false && (adminCapabilityOption === true || hasAdminConfig) const storageBackend = optionalLower(envOption('STORAGE_BACKEND')) + const cdnEnabled = boolOption('enable-cdn', process.env.ACCEPTANCE_ENABLE_CDN) const config: AcceptanceConfig = { adminApiKey, adminUrl, + adminReturnSensitiveData, + adminDatabaseUrlOverride: envOption('ACCEPTANCE_ADMIN_DATABASE_URL_OVERRIDE'), allowDestructive: boolOption('allow-destructive', envOption('ACCEPTANCE_ALLOW_DESTRUCTIVE')), anonKey: envOption('ACCEPTANCE_ANON_KEY'), authenticatedKey: envOption('ACCEPTANCE_AUTHENTICATED_KEY'), baseUrl, capabilities: { admin: adminCapabilityEnabled && hasAdminConfig, - cdn: boolOption('enable-cdn', process.env.ACCEPTANCE_ENABLE_CDN), + cdn: cdnEnabled, + cdnEdge: cdnEnabled && target === 'remote', iceberg: boolOption('enable-iceberg', process.env.ACCEPTANCE_ENABLE_ICEBERG), render: boolOption('enable-render', process.env.ACCEPTANCE_ENABLE_RENDER), rlsSetup: boolOption('enable-rls-setup', process.env.ACCEPTANCE_ENABLE_RLS_SETUP), diff --git a/acceptance/support/http.ts b/acceptance/support/http.ts index bde537fbc..69403eb14 100644 --- a/acceptance/support/http.ts +++ b/acceptance/support/http.ts @@ -1,17 +1,23 @@ +import { setTimeout as delay } from 'node:timers/promises' import { getAcceptanceConfig, joinUrl } from './config' import { createAcceptanceHeaders } from './headers' export { createAcceptanceHeaders, withAcceptanceHeaders } from './headers' +const RETRY_DELAY_MS = 1000 + export interface HttpRequestOptions { body?: BodyInit | Record + expectedCacheStatus?: string | string[] expectedStatus?: number | number[] headers?: Record + retries?: number token?: string } export interface HttpResponse { body: string + cacheStatus?: string headers: Headers json: T | undefined status: number @@ -25,6 +31,30 @@ export class AcceptanceHttpClient { method: string, route: string, options: HttpRequestOptions = {} + ): Promise> { + const retries = options.retries ?? 0 + for (let attempt = 0; ; attempt++) { + try { + return await this.sendRequest(method, route, options) + } catch (error) { + if (attempt >= retries) { + console.log( + 'AcceptanceHttpClient.request try:', + attempt, + ' - error: ', + (error as Error).message + ) + throw error + } + await delay(RETRY_DELAY_MS) + } + } + } + + private async sendRequest( + method: string, + route: string, + options: HttpRequestOptions ): Promise> { const url = joinUrl(this.baseUrl, route) const headers = createAcceptanceHeaders(options.headers) @@ -44,27 +74,51 @@ export class AcceptanceHttpClient { headers, method, }) - const text = await response.text() - const parsed = parseJson(text) - const expected = options.expectedStatus - - if (expected !== undefined && !statusMatches(response.status, expected)) { - throw new Error( - [ - `Unexpected HTTP status for ${method} ${url}`, - `expected: ${Array.isArray(expected) ? expected.join(', ') : expected}`, - `received: ${response.status}`, - `body: ${text}`, - ].join('\n') - ) - } - return { - body: text, - headers: response.headers, - json: parsed, - status: response.status, - url, + try { + const cacheStatus = response.headers.get('cf-cache-status') ?? undefined + const text = response.status !== 304 ? await response.text() : '' + const parsed = parseJson(text) + + const expectedStatus = options.expectedStatus + if (expectedStatus !== undefined && !matches(response.status, expectedStatus)) { + throw new Error( + [ + `Unexpected HTTP status for ${method} ${url}`, + `expected: ${Array.isArray(expectedStatus) ? expectedStatus.join(', ') : expectedStatus}`, + `received: ${response.status}`, + `body: ${text}`, + ].join('\n') + ) + } + + const expectedCacheStatus = options.expectedCacheStatus + if (expectedCacheStatus !== undefined && !matches(cacheStatus, expectedCacheStatus)) { + throw new Error( + [ + `Unexpected cache status for ${method} ${url}`, + `expected: ${ + Array.isArray(expectedCacheStatus) + ? expectedCacheStatus.join(', ') + : expectedCacheStatus + }`, + `received: ${cacheStatus}`, + ].join('\n') + ) + } + + return { + body: text, + cacheStatus, + headers: response.headers, + json: parsed, + status: response.status, + url, + } + } finally { + if (!response.bodyUsed) { + await response.body?.cancel() + } } } } @@ -82,8 +136,27 @@ export function createAdminClient() { return new AcceptanceHttpClient(adminUrl) } -function statusMatches(status: number, expected: number | number[]) { - return Array.isArray(expected) ? expected.includes(status) : status === expected +function matches(value: T, expected: T | T[]): boolean { + return Array.isArray(expected) ? expected.includes(value) : value === expected +} + +interface StorageError { + statusCode: string + error: string + message: string +} + +export function parseStorageError(payload: unknown): StorageError { + const valid = + payload !== null && + typeof payload === 'object' && + 'statusCode' in payload && + 'error' in payload && + 'message' in payload + if (!valid) { + throw new Error('Invalid storage error payload') + } + return payload as StorageError } function parseJson(text: string): T | undefined {