diff --git a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts index deec304b7..dae937ba6 100644 --- a/graphile/graphile-presigned-url-plugin/src/download-url-field.ts +++ b/graphile/graphile-presigned-url-plugin/src/download-url-field.ts @@ -135,9 +135,9 @@ export function createDownloadUrlPlugin( : null; if (withPgClient) { const resolved = await withPgClient(null, async (pgClient: any) => { - const dbResult = await pgClient.query( - `SELECT jwt_private.current_database_id() AS id`, - ); + const dbResult = await pgClient.query({ + text: `SELECT jwt_private.current_database_id() AS id`, + }); const databaseId = dbResult.rows[0]?.id; if (!databaseId) return null; const config = await getStorageModuleConfig(pgClient, databaseId); diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index f668a4baa..c6460678e 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -59,9 +59,9 @@ function buildS3Key(contentHash: string): string { * metaschema query needed. */ async function resolveDatabaseId(pgClient: any): Promise { - const result = await pgClient.query( - `SELECT jwt_private.current_database_id() AS id`, - ); + const result = await pgClient.query({ + text: `SELECT jwt_private.current_database_id() AS id`, + }); return result.rows[0]?.id ?? null; } @@ -235,15 +235,14 @@ export function createPresignedUrlPlugin( } return withPgClient(pgSettings, async (pgClient: any) => { - await pgClient.query('BEGIN'); - try { + return pgClient.withTransaction(async (txClient: any) => { // --- Resolve storage module config (all limits come from here) --- - const databaseId = await resolveDatabaseId(pgClient); + const databaseId = await resolveDatabaseId(txClient); if (!databaseId) { throw new Error('DATABASE_NOT_FOUND'); } - const storageConfig = await getStorageModuleConfig(pgClient, databaseId); + const storageConfig = await getStorageModuleConfig(txClient, databaseId); if (!storageConfig) { throw new Error('STORAGE_MODULE_NOT_PROVISIONED'); } @@ -259,7 +258,7 @@ export function createPresignedUrlPlugin( } // --- Look up the bucket (cached; first miss queries via RLS) --- - const bucket = await getBucketConfig(pgClient, storageConfig, databaseId, bucketKey); + const bucket = await getBucketConfig(txClient, storageConfig, databaseId, bucketKey); if (!bucket) { throw new Error('BUCKET_NOT_FOUND'); } @@ -288,29 +287,28 @@ export function createPresignedUrlPlugin( const s3Key = buildS3Key(contentHash); // --- Dedup check: look for existing file with same content_hash in this bucket --- - const dedupResult = await pgClient.query( - `SELECT id, status + const dedupResult = await txClient.query({ + text: `SELECT id, status FROM ${storageConfig.filesQualifiedName} WHERE content_hash = $1 AND bucket_id = $2 AND status IN ('ready', 'processed') LIMIT 1`, - [contentHash, bucket.id], - ); + values: [contentHash, bucket.id], + }); if (dedupResult.rows.length > 0) { const existingFile = dedupResult.rows[0]; log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); // Track the dedup request - await pgClient.query( - `INSERT INTO ${storageConfig.uploadRequestsQualifiedName} + await txClient.query({ + text: `INSERT INTO ${storageConfig.uploadRequestsQualifiedName} (file_id, bucket_id, key, content_type, content_hash, size, status, expires_at) VALUES ($1, $2, $3, $4, $5, $6, 'confirmed', NOW())`, - [existingFile.id, bucket.id, s3Key, contentType, contentHash, size], - ); + values: [existingFile.id, bucket.id, s3Key, contentType, contentHash, size], + }); - await pgClient.query('COMMIT'); return { uploadUrl: null, fileId: existingFile.id, @@ -321,12 +319,12 @@ export function createPresignedUrlPlugin( } // --- Create file record (status=pending) --- - const fileResult = await pgClient.query( - `INSERT INTO ${storageConfig.filesQualifiedName} + const fileResult = await txClient.query({ + text: `INSERT INTO ${storageConfig.filesQualifiedName} (bucket_id, key, content_type, content_hash, size, filename, owner_id, is_public, status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending') RETURNING id`, - [ + values: [ bucket.id, s3Key, contentType, @@ -336,7 +334,7 @@ export function createPresignedUrlPlugin( bucket.owner_id, bucket.is_public, ], - ); + }); const fileId = fileResult.rows[0].id; @@ -356,14 +354,13 @@ export function createPresignedUrlPlugin( const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString(); // --- Track the upload request --- - await pgClient.query( - `INSERT INTO ${storageConfig.uploadRequestsQualifiedName} + await txClient.query({ + text: `INSERT INTO ${storageConfig.uploadRequestsQualifiedName} (file_id, bucket_id, key, content_type, content_hash, size, status, expires_at) VALUES ($1, $2, $3, $4, $5, $6, 'issued', $7)`, - [fileId, bucket.id, s3Key, contentType, contentHash, size, expiresAt], - ); + values: [fileId, bucket.id, s3Key, contentType, contentHash, size, expiresAt], + }); - await pgClient.query('COMMIT'); return { uploadUrl, fileId, @@ -371,10 +368,7 @@ export function createPresignedUrlPlugin( deduplicated: false, expiresAt, }; - } catch (err) { - await pgClient.query('ROLLBACK'); - throw err; - } + }); }); }); }, @@ -397,27 +391,26 @@ export function createPresignedUrlPlugin( } return withPgClient(pgSettings, async (pgClient: any) => { - await pgClient.query('BEGIN'); - try { + return pgClient.withTransaction(async (txClient: any) => { // --- Resolve storage module config --- - const databaseId = await resolveDatabaseId(pgClient); + const databaseId = await resolveDatabaseId(txClient); if (!databaseId) { throw new Error('DATABASE_NOT_FOUND'); } - const storageConfig = await getStorageModuleConfig(pgClient, databaseId); + const storageConfig = await getStorageModuleConfig(txClient, databaseId); if (!storageConfig) { throw new Error('STORAGE_MODULE_NOT_PROVISIONED'); } // --- Look up the file (RLS enforced) --- - const fileResult = await pgClient.query( - `SELECT id, key, content_type, status, bucket_id + const fileResult = await txClient.query({ + text: `SELECT id, key, content_type, status, bucket_id FROM ${storageConfig.filesQualifiedName} WHERE id = $1 LIMIT 1`, - [fileId], - ); + values: [fileId], + }); if (fileResult.rows.length === 0) { throw new Error('FILE_NOT_FOUND'); @@ -427,7 +420,6 @@ export function createPresignedUrlPlugin( if (file.status !== 'pending') { // File is already confirmed or processed — idempotent success - await pgClient.query('COMMIT'); return { fileId: file.id, status: file.status, @@ -446,45 +438,40 @@ export function createPresignedUrlPlugin( // --- Content-type verification --- if (s3Head.contentType && s3Head.contentType !== file.content_type) { // Mark upload_request as rejected - await pgClient.query( - `UPDATE ${storageConfig.uploadRequestsQualifiedName} + await txClient.query({ + text: `UPDATE ${storageConfig.uploadRequestsQualifiedName} SET status = 'rejected' WHERE file_id = $1 AND status = 'issued'`, - [fileId], - ); + values: [fileId], + }); - await pgClient.query('COMMIT'); throw new Error( `CONTENT_TYPE_MISMATCH: expected ${file.content_type}, got ${s3Head.contentType}`, ); } // --- Transition file to 'ready' --- - await pgClient.query( - `UPDATE ${storageConfig.filesQualifiedName} + await txClient.query({ + text: `UPDATE ${storageConfig.filesQualifiedName} SET status = 'ready' WHERE id = $1`, - [fileId], - ); + values: [fileId], + }); // --- Update upload_request to 'confirmed' --- - await pgClient.query( - `UPDATE ${storageConfig.uploadRequestsQualifiedName} + await txClient.query({ + text: `UPDATE ${storageConfig.uploadRequestsQualifiedName} SET status = 'confirmed', confirmed_at = NOW() WHERE file_id = $1 AND status = 'issued'`, - [fileId], - ); + values: [fileId], + }); - await pgClient.query('COMMIT'); return { fileId: file.id, status: 'ready', success: true, }; - } catch (err) { - await pgClient.query('ROLLBACK'); - throw err; - } + }); }); }); }, diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index dd1fd6221..e5ef211be 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -91,7 +91,7 @@ interface StorageModuleRow { * @returns StorageModuleConfig or null if no storage module is provisioned */ export async function getStorageModuleConfig( - pgClient: { query: (sql: string, params: unknown[]) => Promise<{ rows: unknown[] }> }, + pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, databaseId: string, ): Promise { const cacheKey = `storage:${databaseId}`; @@ -102,7 +102,7 @@ export async function getStorageModuleConfig( log.debug(`Cache miss for database ${databaseId}, querying metaschema...`); - const result = await pgClient.query(STORAGE_MODULE_QUERY, [databaseId]); + const result = await pgClient.query({ text: STORAGE_MODULE_QUERY, values: [databaseId] }); if (result.rows.length === 0) { log.warn(`No storage module found for database ${databaseId}`); return null; @@ -172,7 +172,7 @@ const bucketCache = new LRUCache({ * @returns BucketConfig or null if the bucket doesn't exist / isn't accessible */ export async function getBucketConfig( - pgClient: { query: (sql: string, params: unknown[]) => Promise<{ rows: unknown[] }> }, + pgClient: { query: (opts: { text: string; values?: unknown[] }) => Promise<{ rows: unknown[] }> }, storageConfig: StorageModuleConfig, databaseId: string, bucketKey: string, @@ -185,13 +185,13 @@ export async function getBucketConfig( log.debug(`Bucket cache miss for ${databaseId}:${bucketKey}, querying DB...`); - const result = await pgClient.query( - `SELECT id, key, type, is_public, owner_id, allowed_mime_types, max_file_size + const result = await pgClient.query({ + text: `SELECT id, key, type, is_public, owner_id, allowed_mime_types, max_file_size FROM ${storageConfig.bucketsQualifiedName} WHERE key = $1 LIMIT 1`, - [bucketKey], - ); + values: [bucketKey], + }); if (result.rows.length === 0) { return null; diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql new file mode 100644 index 000000000..ae9abd582 --- /dev/null +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/schema.sql @@ -0,0 +1,75 @@ +-- Schema creation for simple-seed-storage test scenario +-- Creates the app schema with storage tables (buckets, files, upload_requests) + +-- Create app schemas +CREATE SCHEMA IF NOT EXISTS "simple-storage-public"; + +-- Grant schema usage +GRANT USAGE ON SCHEMA "simple-storage-public" TO administrator, authenticated, anonymous; + +-- Set default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA "simple-storage-public" + GRANT ALL ON TABLES TO administrator; +ALTER DEFAULT PRIVILEGES IN SCHEMA "simple-storage-public" + GRANT USAGE ON SEQUENCES TO administrator, authenticated; +ALTER DEFAULT PRIVILEGES IN SCHEMA "simple-storage-public" + GRANT ALL ON FUNCTIONS TO administrator, authenticated, anonymous; + +-- ===================================================== +-- STORAGE TABLES (mirroring what the storage module generator creates) +-- ===================================================== + +-- Buckets table +CREATE TABLE IF NOT EXISTS "simple-storage-public".buckets ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + key text NOT NULL, + type text NOT NULL DEFAULT 'private', + is_public boolean NOT NULL DEFAULT false, + owner_id uuid, + allowed_mime_types text[] NULL, + max_file_size bigint NULL, + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now(), + UNIQUE (key) +); + +COMMENT ON TABLE "simple-storage-public".buckets IS E'@storageBuckets\nStorage buckets table'; + +-- Files table +CREATE TABLE IF NOT EXISTS "simple-storage-public".files ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + bucket_id uuid NOT NULL REFERENCES "simple-storage-public".buckets(id), + key text NOT NULL, + content_type text NOT NULL, + content_hash text, + size bigint, + filename text, + owner_id uuid, + is_public boolean NOT NULL DEFAULT false, + status text NOT NULL DEFAULT 'pending', + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now() +); + +COMMENT ON TABLE "simple-storage-public".files IS E'@storageFiles\nStorage files table'; + +-- Upload requests table +CREATE TABLE IF NOT EXISTS "simple-storage-public".upload_requests ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + file_id uuid NOT NULL REFERENCES "simple-storage-public".files(id), + bucket_id uuid NOT NULL REFERENCES "simple-storage-public".buckets(id), + key text NOT NULL, + content_type text NOT NULL, + content_hash text, + size bigint, + status text NOT NULL DEFAULT 'issued', + expires_at timestamptz, + confirmed_at timestamptz, + created_at timestamptz DEFAULT now(), + updated_at timestamptz DEFAULT now() +); + +-- Grant table permissions (allow anonymous to do CRUD for tests — no RLS) +GRANT SELECT, INSERT, UPDATE, DELETE ON "simple-storage-public".buckets TO administrator, authenticated, anonymous; +GRANT SELECT, INSERT, UPDATE, DELETE ON "simple-storage-public".files TO administrator, authenticated, anonymous; +GRANT SELECT, INSERT, UPDATE, DELETE ON "simple-storage-public".upload_requests TO administrator, authenticated, anonymous; diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql new file mode 100644 index 000000000..9d923ba67 --- /dev/null +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/setup.sql @@ -0,0 +1,70 @@ +-- Storage-module additions for upload testing +-- +-- This file is loaded AFTER simple-seed-services/setup.sql, which already +-- creates all metaschema / services tables, roles, extensions, etc. +-- We only add the bits that are specific to the storage module here. + +-- ===================================================== +-- JWT PRIVATE SCHEMA (required by presigned URL plugin) +-- ===================================================== + +CREATE SCHEMA IF NOT EXISTS jwt_private; + +GRANT USAGE ON SCHEMA jwt_private TO authenticated, anonymous; + +ALTER DEFAULT PRIVILEGES IN SCHEMA jwt_private + GRANT EXECUTE ON FUNCTIONS TO authenticated; + +-- current_database_id reads from pgSettings set by the middleware +CREATE OR REPLACE FUNCTION jwt_private.current_database_id() RETURNS uuid AS $EOFCODE$ +DECLARE + v_identifier_id uuid; +BEGIN + IF current_setting('jwt.claims.database_id', TRUE) + IS NOT NULL THEN + BEGIN + v_identifier_id = current_setting('jwt.claims.database_id', TRUE)::uuid; + EXCEPTION + WHEN OTHERS THEN + RAISE NOTICE 'Invalid UUID value'; + RETURN NULL; + END; + RETURN v_identifier_id; + ELSE + RETURN NULL; + END IF; +END; +$EOFCODE$ LANGUAGE plpgsql STABLE; + +-- ===================================================== +-- STORAGE MODULE TABLE (metaschema_modules_public) +-- ===================================================== + +CREATE TABLE IF NOT EXISTS metaschema_modules_public.storage_module ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + database_id uuid NOT NULL, + schema_id uuid NOT NULL DEFAULT uuid_nil(), + private_schema_id uuid NOT NULL DEFAULT uuid_nil(), + buckets_table_id uuid NOT NULL DEFAULT uuid_nil(), + files_table_id uuid NOT NULL DEFAULT uuid_nil(), + upload_requests_table_id uuid NOT NULL DEFAULT uuid_nil(), + buckets_table_name text NOT NULL DEFAULT 'buckets', + files_table_name text NOT NULL DEFAULT 'files', + upload_requests_table_name text NOT NULL DEFAULT 'upload_requests', + entity_table_id uuid NULL, + endpoint text NULL, + public_url_prefix text NULL, + provider text NULL, + allowed_origins text[] NULL, + upload_url_expiry_seconds integer NULL, + download_url_expiry_seconds integer NULL, + default_max_file_size bigint NULL, + max_filename_length integer NULL, + cache_ttl_seconds integer NULL, + CONSTRAINT sm_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS storage_module_database_id_idx + ON metaschema_modules_public.storage_module (database_id); + +GRANT SELECT, INSERT, UPDATE, DELETE ON metaschema_modules_public.storage_module TO administrator, authenticated, anonymous; diff --git a/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql b/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql new file mode 100644 index 000000000..800619a5a --- /dev/null +++ b/graphql/server-test/__fixtures__/seed/simple-seed-storage/test-data.sql @@ -0,0 +1,109 @@ +-- Test data for simple-seed-storage scenario +-- Seeds metaschema, services, storage_module, and bucket data + +SET session_replication_role TO replica; + +-- ===================================================== +-- METASCHEMA DATA +-- ===================================================== + +-- Database entry (ID matches servicesDatabaseId in test file) +INSERT INTO metaschema_public.database (id, owner_id, name, hash) +VALUES ( + '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + NULL, + 'simple-storage', + '425a0f10-0170-5760-85df-2a980c378224' +) ON CONFLICT (id) DO NOTHING; + +-- Schema entries +INSERT INTO metaschema_public.schema (id, database_id, name, schema_name, description, is_public) +VALUES + ('6dbae92a-5450-401b-1ed5-d69e7754940d', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', 'public', 'simple-storage-public', NULL, true) +ON CONFLICT (id) DO NOTHING; + +-- Table entries for storage tables +-- buckets +INSERT INTO metaschema_public.table (id, database_id, schema_id, name, description) +VALUES ( + 'b0000001-0000-0000-0000-000000000001', + '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + '6dbae92a-5450-401b-1ed5-d69e7754940d', + 'buckets', + NULL +) ON CONFLICT (id) DO NOTHING; + +-- files +INSERT INTO metaschema_public.table (id, database_id, schema_id, name, description) +VALUES ( + 'b0000001-0000-0000-0000-000000000002', + '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + '6dbae92a-5450-401b-1ed5-d69e7754940d', + 'files', + NULL +) ON CONFLICT (id) DO NOTHING; + +-- upload_requests +INSERT INTO metaschema_public.table (id, database_id, schema_id, name, description) +VALUES ( + 'b0000001-0000-0000-0000-000000000003', + '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + '6dbae92a-5450-401b-1ed5-d69e7754940d', + 'upload_requests', + NULL +) ON CONFLICT (id) DO NOTHING; + +-- ===================================================== +-- SERVICES DATA +-- ===================================================== + +INSERT INTO services_public.apis (id, database_id, name, dbname, is_public, role_name, anon_role) +VALUES + ('6c9997a4-591b-4cb3-9313-4ef45d6f134e', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', 'app', current_database(), true, 'authenticated', 'anonymous') +ON CONFLICT (id) DO NOTHING; + +INSERT INTO services_public.api_schemas (id, database_id, schema_id, api_id) +VALUES + ('71181146-890e-4991-9da7-3dddf87d9e01', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', '6dbae92a-5450-401b-1ed5-d69e7754940d', '6c9997a4-591b-4cb3-9313-4ef45d6f134e') +ON CONFLICT (id) DO NOTHING; + +-- ===================================================== +-- STORAGE MODULE CONFIG +-- ===================================================== + +INSERT INTO metaschema_modules_public.storage_module ( + id, + database_id, + schema_id, + buckets_table_id, + files_table_id, + upload_requests_table_id, + endpoint, + public_url_prefix, + provider, + allowed_origins +) +VALUES ( + 'c0000001-0000-0000-0000-000000000001', + '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + '6dbae92a-5450-401b-1ed5-d69e7754940d', + 'b0000001-0000-0000-0000-000000000001', + 'b0000001-0000-0000-0000-000000000002', + 'b0000001-0000-0000-0000-000000000003', + NULL, -- use global CDN_ENDPOINT + NULL, -- use global CDN_PUBLIC_URL_PREFIX + 'minio', + ARRAY['*'] +) ON CONFLICT (id) DO NOTHING; + +-- ===================================================== +-- BUCKET SEED DATA +-- ===================================================== + +INSERT INTO "simple-storage-public".buckets (id, key, type, is_public, owner_id) +VALUES + ('d0000001-0000-0000-0000-000000000001', 'public', 'public', true, NULL), + ('d0000001-0000-0000-0000-000000000002', 'private', 'private', false, NULL) +ON CONFLICT (id) DO NOTHING; + +SET session_replication_role TO DEFAULT; diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts new file mode 100644 index 000000000..8355816a1 --- /dev/null +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -0,0 +1,276 @@ +/** + * Upload Integration Tests — end-to-end presigned URL flow + * + * Exercises the full upload pipeline for both public and private files: + * requestUploadUrl → PUT to presigned URL → confirmUpload → downloadUrl + * + * Uses real MinIO (available in CI as minio_cdn service) and lazy bucket + * provisioning. No RLS — that will be tested in constructive-db. + * + * Run tests: + * pnpm test -- --testPathPattern=upload.integration + */ + +import crypto from 'crypto'; +import path from 'path'; +import { getConnections, seed } from '../src'; +import type supertest from 'supertest'; + +jest.setTimeout(60000); + +const seedRoot = path.join(__dirname, '..', '__fixtures__', 'seed'); +const sql = (seedDir: string, file: string) => + path.join(seedRoot, seedDir, file); + +const servicesDatabaseId = '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9'; +const metaSchemas = [ + 'services_public', + 'metaschema_public', + 'metaschema_modules_public', +]; +const schemas = ['simple-storage-public']; + +const seedFiles = [ + // Reuse the shared metaschema / services infrastructure + sql('simple-seed-services', 'setup.sql'), + // Storage-specific additions (jwt_private + storage_module table) + sql('simple-seed-storage', 'setup.sql'), + sql('simple-seed-storage', 'schema.sql'), + sql('simple-seed-storage', 'test-data.sql'), +]; + +// --- GraphQL operations --- + +const REQUEST_UPLOAD_URL = ` + mutation RequestUploadUrl($input: RequestUploadUrlInput!) { + requestUploadUrl(input: $input) { + uploadUrl + fileId + key + deduplicated + expiresAt + } + } +`; + +const CONFIRM_UPLOAD = ` + mutation ConfirmUpload($input: ConfirmUploadInput!) { + confirmUpload(input: $input) { + fileId + status + success + } + } +`; + +// --- Helpers --- + +/** + * Generate a deterministic SHA-256 hex hash for test content. + */ +function sha256(content: string): string { + return crypto.createHash('sha256').update(content).digest('hex'); +} + +/** + * PUT file content to a presigned URL using fetch. + */ +async function putToPresignedUrl( + url: string, + content: string, + contentType: string, +): Promise { + return fetch(url, { + method: 'PUT', + headers: { + 'Content-Type': contentType, + 'Content-Length': Buffer.byteLength(content).toString(), + }, + body: content, + }); +} + +// --- Tests --- + +describe('Upload integration (presigned URL flow)', () => { + let request: supertest.Agent; + let teardown: () => Promise; + + const postGraphQL = (payload: { + query: string; + variables?: Record; + }) => { + return request + .post('/graphql') + .set('X-Database-Id', servicesDatabaseId) + .set('X-Schemata', schemas.join(',')) + .send(payload); + }; + + beforeAll(async () => { + ({ request, teardown } = await getConnections( + { + schemas, + authRole: 'anonymous', + server: { + api: { + enableServicesApi: true, + isPublic: false, + metaSchemas, + }, + }, + }, + [seed.sqlfile(seedFiles)], + )); + }); + + afterAll(async () => { + if (teardown) await teardown(); + }); + + describe('Public file upload', () => { + const fileContent = 'Hello, public world!'; + const contentType = 'text/plain'; + const contentHash = sha256(fileContent); + let uploadUrl: string; + let fileId: string; + + it('should return a presigned PUT URL via requestUploadUrl', async () => { + const res = await postGraphQL({ + query: REQUEST_UPLOAD_URL, + variables: { + input: { + bucketKey: 'public', + contentHash, + contentType, + size: Buffer.byteLength(fileContent), + filename: 'hello-public.txt', + }, + }, + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + + const payload = res.body.data.requestUploadUrl; + expect(payload.uploadUrl).toBeTruthy(); + expect(payload.fileId).toBeTruthy(); + expect(payload.key).toBe(contentHash); + expect(payload.deduplicated).toBe(false); + expect(payload.expiresAt).toBeTruthy(); + + uploadUrl = payload.uploadUrl; + fileId = payload.fileId; + }); + + it('should accept a PUT to the presigned URL', async () => { + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType); + expect(putRes.ok).toBe(true); + }); + + it('should confirm the upload and transition file to ready', async () => { + const res = await postGraphQL({ + query: CONFIRM_UPLOAD, + variables: { + input: { fileId }, + }, + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + + const payload = res.body.data.confirmUpload; + expect(payload.fileId).toBe(fileId); + expect(payload.status).toBe('ready'); + expect(payload.success).toBe(true); + }); + }); + + describe('Private file upload', () => { + const fileContent = 'Hello, private world!'; + const contentType = 'text/plain'; + const contentHash = sha256(fileContent); + let uploadUrl: string; + let fileId: string; + + it('should return a presigned PUT URL via requestUploadUrl', async () => { + const res = await postGraphQL({ + query: REQUEST_UPLOAD_URL, + variables: { + input: { + bucketKey: 'private', + contentHash, + contentType, + size: Buffer.byteLength(fileContent), + filename: 'hello-private.txt', + }, + }, + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + + const payload = res.body.data.requestUploadUrl; + expect(payload.uploadUrl).toBeTruthy(); + expect(payload.fileId).toBeTruthy(); + expect(payload.key).toBe(contentHash); + expect(payload.deduplicated).toBe(false); + expect(payload.expiresAt).toBeTruthy(); + + uploadUrl = payload.uploadUrl; + fileId = payload.fileId; + }); + + it('should accept a PUT to the presigned URL', async () => { + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType); + expect(putRes.ok).toBe(true); + }); + + it('should confirm the upload and transition file to ready', async () => { + const res = await postGraphQL({ + query: CONFIRM_UPLOAD, + variables: { + input: { fileId }, + }, + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + + const payload = res.body.data.confirmUpload; + expect(payload.fileId).toBe(fileId); + expect(payload.status).toBe('ready'); + expect(payload.success).toBe(true); + }); + }); + + describe('Deduplication', () => { + it('should return deduplicated=true for a file with an existing content hash', async () => { + // Re-request the same public file content hash + const fileContent = 'Hello, public world!'; + const contentHash = sha256(fileContent); + + const res = await postGraphQL({ + query: REQUEST_UPLOAD_URL, + variables: { + input: { + bucketKey: 'public', + contentHash, + contentType: 'text/plain', + size: Buffer.byteLength(fileContent), + filename: 'hello-public-copy.txt', + }, + }, + }); + + expect(res.status).toBe(200); + expect(res.body.errors).toBeUndefined(); + + const payload = res.body.data.requestUploadUrl; + expect(payload.deduplicated).toBe(true); + expect(payload.uploadUrl).toBeNull(); + expect(payload.expiresAt).toBeNull(); + expect(payload.fileId).toBeTruthy(); + }); + }); +}); diff --git a/packages/bucket-provisioner/src/provisioner.ts b/packages/bucket-provisioner/src/provisioner.ts index dd1c7af9d..980d80f71 100644 --- a/packages/bucket-provisioner/src/provisioner.ts +++ b/packages/bucket-provisioner/src/provisioner.ts @@ -239,6 +239,10 @@ export class BucketProvisioner { /** * Configure S3 Block Public Access settings. + * + * Gracefully skips if the S3-compatible backend (e.g. MinIO) does not + * support PutPublicAccessBlock — the operation is best-effort since + * not all providers implement this AWS-specific API. */ async setPublicAccessBlock( bucketName: string, @@ -252,6 +256,20 @@ export class BucketProvisioner { }), ); } catch (err: any) { + // Some S3-compatible backends (e.g. older MinIO) don't support + // PutPublicAccessBlock. Treat XML parse errors or "not implemented" + // responses as non-fatal — the bucket is still usable. + if ( + err.Code === 'XmlParseException' || + err.name === 'XmlParseException' || + err.Code === 'NotImplemented' || + err.name === 'NotImplemented' || + err.message?.includes('not well-formed') || + err.message?.includes('not implemented') || + err.message?.includes('PublicAccessBlockConfiguration') + ) { + return; + } throw new ProvisionerError( 'POLICY_FAILED', `Failed to set public access block on '${bucketName}': ${err.message}`, @@ -285,6 +303,9 @@ export class BucketProvisioner { /** * Delete an S3 bucket policy (used to clear leftover public policies). + * + * Gracefully handles backends that don't support this operation or have + * no policy to delete. */ async deleteBucketPolicy(bucketName: string): Promise { try { @@ -293,7 +314,16 @@ export class BucketProvisioner { ); } catch (err: any) { // No policy to delete — that's fine - if (err.name === 'NoSuchBucketPolicy' || err.$metadata?.httpStatusCode === 404) { + if ( + err.name === 'NoSuchBucketPolicy' || + err.$metadata?.httpStatusCode === 404 || + err.Code === 'XmlParseException' || + err.name === 'XmlParseException' || + err.Code === 'NotImplemented' || + err.name === 'NotImplemented' || + err.message?.includes('not well-formed') || + err.message?.includes('not implemented') + ) { return; } throw new ProvisionerError( @@ -306,6 +336,10 @@ export class BucketProvisioner { /** * Set CORS configuration on an S3 bucket. + * + * Gracefully skips if the S3-compatible backend (e.g. older MinIO) does + * not support PutBucketCors — CORS is best-effort since not all providers + * implement this API via the same endpoint path. */ async setCors(bucketName: string, rules: CorsRule[]): Promise { try { @@ -324,6 +358,20 @@ export class BucketProvisioner { }), ); } catch (err: any) { + // Some S3-compatible backends (e.g. older MinIO) don't support + // PutBucketCors via the standard path. Treat XML parse errors or + // "not implemented" responses as non-fatal. + if ( + err.Code === 'XmlParseException' || + err.name === 'XmlParseException' || + err.Code === 'NotImplemented' || + err.name === 'NotImplemented' || + err.message?.includes('not well-formed') || + err.message?.includes('not implemented') || + err.message?.includes('CORSConfiguration') + ) { + return; + } throw new ProvisionerError( 'CORS_FAILED', `Failed to set CORS on '${bucketName}': ${err.message}`,