feat: lazy S3 bucket provisioning on first upload

When a presigned PUT URL is requested for a database whose S3 bucket
doesn't exist yet, the plugin now automatically provisions it with
the correct privacy policies, CORS rules, and lifecycle settings.

Changes:
- Add EnsureBucketProvisioned callback type to plugin options
- Add in-memory Set cache (provisionedBuckets) in storage-module-cache
  to track which S3 buckets are known to exist — no TTL needed since
  buckets are never deleted; resets on server restart (idempotent)
- Wire ensureS3BucketExists into requestUploadUrl before generating
  the presigned PUT URL — first request provisions, subsequent skip
- Add createEnsureBucketProvisioned factory in presigned-url-resolver
  that uses BucketProvisioner for full bucket setup
- Wire into ConstructivePreset with same allowedOrigins as bucket
  provisioner plugin

Author: pyramation

graphile/graphile-presigned-url-plugin/src/index.ts

@@ -30,7 +30,7 @@
 export { PresignedUrlPlugin, createPresignedUrlPlugin } from './plugin';
 export { createDownloadUrlPlugin } from './download-url-field';
 export { PresignedUrlPreset } from './preset';
-export { getStorageModuleConfig, getBucketConfig, clearStorageModuleCache, clearBucketCache } from './storage-module-cache';
+export { getStorageModuleConfig, getBucketConfig, clearStorageModuleCache, clearBucketCache, isS3BucketProvisioned, markS3BucketProvisioned } from './storage-module-cache';
 export { generatePresignedPutUrl, generatePresignedGetUrl, headObject } from './s3-signer';
 export type {
   BucketConfig,
@@ -43,4 +43,5 @@ export type {
   S3ConfigOrGetter,
   PresignedUrlPluginOptions,
   BucketNameResolver,
+  EnsureBucketProvisioned,
 } from './types';

graphile/graphile-presigned-url-plugin/src/plugin.ts

@@ -22,8 +22,8 @@ import type { GraphileConfig } from 'graphile-config';
 import { extendSchema, gql } from 'graphile-utils';
 import { Logger } from '@pgpmjs/logger';
 
-import type { PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types';
-import { getStorageModuleConfig, getBucketConfig } from './storage-module-cache';
+import type { PresignedUrlPluginOptions, S3Config, StorageModuleConfig, BucketConfig } from './types';
+import { getStorageModuleConfig, getBucketConfig, isS3BucketProvisioned, markS3BucketProvisioned } from './storage-module-cache';
 import { generatePresignedPutUrl, headObject } from './s3-signer';
 
 const log = new Logger('graphile-presigned-url:plugin');
@@ -112,6 +112,31 @@ function resolveS3ForDatabase(
   };
 }
 
+/**
+ * Ensure the S3 bucket for a database exists, provisioning it lazily if needed.
+ *
+ * Checks an in-memory Set of known-provisioned bucket names. On the first
+ * request for an unseen bucket, calls the `ensureBucketProvisioned` callback
+ * (which creates the bucket with correct CORS, policies, etc.), then marks
+ * it as provisioned so subsequent requests skip the check entirely.
+ *
+ * If no `ensureBucketProvisioned` callback is configured, this is a no-op.
+ */
+async function ensureS3BucketExists(
+  options: PresignedUrlPluginOptions,
+  s3BucketName: string,
+  bucket: BucketConfig,
+  databaseId: string,
+): Promise<void> {
+  if (!options.ensureBucketProvisioned) return;
+  if (isS3BucketProvisioned(s3BucketName)) return;
+
+  log.info(`Lazy-provisioning S3 bucket "${s3BucketName}" for database ${databaseId}`);
+  await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId);
+  markS3BucketProvisioned(s3BucketName);
+  log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`);
+}
+
 export function createPresignedUrlPlugin(
   options: PresignedUrlPluginOptions,
 ): GraphileConfig.Plugin {
@@ -314,8 +339,11 @@ export function createPresignedUrlPlugin(
 
                 const fileId = fileResult.rows[0].id;
 
-                // --- Generate presigned PUT URL (per-database bucket) ---
+                // --- Ensure the S3 bucket exists (lazy provisioning) ---
                 const s3ForDb = resolveS3ForDatabase(options, storageConfig, databaseId);
+                await ensureS3BucketExists(options, s3ForDb.bucket, bucket, databaseId);
+
+                // --- Generate presigned PUT URL (per-database bucket) ---
                 const uploadUrl = await generatePresignedPutUrl(
                   s3ForDb,
                   s3Key,

graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts

@@ -220,13 +220,46 @@ export async function getBucketConfig(
   return config;
 }
 
+// --- S3 bucket existence cache ---
+
+/**
+ * In-memory set of S3 bucket names that are known to exist.
+ *
+ * Used by the lazy provisioning logic in the presigned URL plugin:
+ * before generating a presigned PUT URL, the plugin checks this set.
+ * If the bucket name is absent, it calls `ensureBucketProvisioned`
+ * to create the S3 bucket, then adds the name here. Subsequent
+ * requests for the same bucket skip the provisioning entirely.
+ *
+ * No TTL needed — S3 buckets are never deleted during normal operation.
+ * The set resets on server restart, which is fine because the
+ * provisioner's createBucket is idempotent (handles "already exists").
+ */
+const provisionedBuckets = new Set<string>();
+
+/**
+ * Check whether an S3 bucket has already been provisioned (cached).
+ */
+export function isS3BucketProvisioned(s3BucketName: string): boolean {
+  return provisionedBuckets.has(s3BucketName);
+}
+
+/**
+ * Mark an S3 bucket as provisioned in the in-memory cache.
+ */
+export function markS3BucketProvisioned(s3BucketName: string): void {
+  provisionedBuckets.add(s3BucketName);
+  log.debug(`Marked S3 bucket "${s3BucketName}" as provisioned`);
+}
+
 /**
  * Clear the storage module cache AND bucket cache.
  * Useful for testing or schema changes.
  */
 export function clearStorageModuleCache(): void {
   storageModuleCache.clear();
   bucketCache.clear();
+  provisionedBuckets.clear();
 }
 
 /**

graphile/graphile-presigned-url-plugin/src/types.ts

@@ -147,6 +147,25 @@ export type S3ConfigOrGetter = S3Config | (() => S3Config);
  */
 export type BucketNameResolver = (databaseId: string) => string;
 
+/**
+ * Callback to lazily provision an S3 bucket on first use.
+ *
+ * Called by the presigned URL plugin before generating a presigned PUT URL
+ * when the bucket has not been seen before (tracked in an in-memory cache).
+ * The implementation should create and fully configure the S3 bucket
+ * (privacy policies, CORS, lifecycle rules, etc.) — or no-op if the
+ * bucket already exists.
+ *
+ * @param bucketName - The S3 bucket name to provision
+ * @param accessType - The logical bucket type ('public', 'private', 'temp')
+ * @param databaseId - The metaschema database UUID
+ */
+export type EnsureBucketProvisioned = (
+  bucketName: string,
+  accessType: 'public' | 'private' | 'temp',
+  databaseId: string,
+) => Promise<void>;
+
 /**
  * Plugin options for the presigned URL plugin.
  */
@@ -160,4 +179,13 @@ export interface PresignedUrlPluginOptions {
    * the global `s3Config.bucket`. The S3 credentials (client) remain shared.
    */
   resolveBucketName?: BucketNameResolver;
+
+  /**
+   * Optional callback to lazily provision an S3 bucket on first upload.
+   * When set, the plugin calls this before generating a presigned PUT URL
+   * for any S3 bucket it hasn't seen yet (tracked in an in-memory cache).
+   * This enables graceful bucket creation without requiring buckets to
+   * exist at database provisioning time.
+   */
+  ensureBucketProvisioned?: EnsureBucketProvisioned;
 }

graphile/graphile-settings/package.json

@@ -30,6 +30,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1009.0",
+    "@constructive-io/bucket-provisioner": "workspace:^",
     "@constructive-io/graphql-env": "workspace:^",
     "@constructive-io/graphql-types": "workspace:^",
     "@constructive-io/s3-streamer": "workspace:^",

graphile/graphile-settings/src/presets/constructive-preset.ts

@@ -19,7 +19,7 @@ import { PresignedUrlPreset } from 'graphile-presigned-url-plugin';
 import { BucketProvisionerPreset } from 'graphile-bucket-provisioner-plugin';
 import { SqlExpressionValidatorPreset } from 'graphile-sql-expression-validator';
 import { constructiveUploadFieldDefinitions } from '../upload-resolver';
-import { getPresignedUrlS3Config, createBucketNameResolver } from '../presigned-url-resolver';
+import { getPresignedUrlS3Config, createBucketNameResolver, createEnsureBucketProvisioned } from '../presigned-url-resolver';
 import { getBucketProvisionerConnection } from '../bucket-provisioner-resolver';
 
 /**
@@ -90,7 +90,11 @@ export const ConstructivePreset: GraphileConfig.Preset = {
       uploadFieldDefinitions: constructiveUploadFieldDefinitions,
       maxFileSize: 10 * 1024 * 1024, // 10MB
     }),
-    PresignedUrlPreset({ s3: getPresignedUrlS3Config, resolveBucketName: createBucketNameResolver() }),
+    PresignedUrlPreset({
+      s3: getPresignedUrlS3Config,
+      resolveBucketName: createBucketNameResolver(),
+      ensureBucketProvisioned: createEnsureBucketProvisioned(['http://localhost:3000']),
+    }),
     BucketProvisionerPreset({
       connection: getBucketProvisionerConnection,
       allowedOrigins: ['http://localhost:3000'],

graphile/graphile-settings/src/presigned-url-resolver.ts

@@ -14,7 +14,9 @@
 import { S3Client } from '@aws-sdk/client-s3';
 import { getEnvOptions } from '@constructive-io/graphql-env';
 import { Logger } from '@pgpmjs/logger';
-import type { S3Config, BucketNameResolver } from 'graphile-presigned-url-plugin';
+import type { S3Config, BucketNameResolver, EnsureBucketProvisioned } from 'graphile-presigned-url-plugin';
+import { BucketProvisioner } from '@constructive-io/bucket-provisioner';
+import { getBucketProvisionerConnection } from './bucket-provisioner-resolver';
 
 const log = new Logger('presigned-url-resolver');
 
@@ -81,3 +83,47 @@ export function createBucketNameResolver(): BucketNameResolver {
     return `${prefix}-${databaseId}`;
   };
 }
+
+/**
+ * Create a lazy bucket provisioner callback for the presigned URL plugin.
+ *
+ * On the first upload to an S3 bucket that doesn't exist yet, this callback
+ * uses the BucketProvisioner to create and fully configure the bucket
+ * (Block Public Access, CORS, policies, lifecycle rules for temp buckets).
+ *
+ * Uses the same S3 connection config as the bucket provisioner plugin
+ * (getBucketProvisionerConnection) and the same CORS origins.
+ *
+ * @param allowedOrigins - CORS origins for presigned URL uploads
+ */
+export function createEnsureBucketProvisioned(
+  allowedOrigins: string[],
+): EnsureBucketProvisioned {
+  let provisioner: BucketProvisioner | null = null;
+
+  return async (
+    bucketName: string,
+    accessType: 'public' | 'private' | 'temp',
+    databaseId: string,
+  ): Promise<void> => {
+    if (!provisioner) {
+      provisioner = new BucketProvisioner({
+        connection: getBucketProvisionerConnection(),
+        allowedOrigins,
+      });
+    }
+
+    log.info(
+      `[lazy-provision] Provisioning S3 bucket "${bucketName}" ` +
+      `(type=${accessType}) for database ${databaseId}`,
+    );
+
+    await provisioner.provision({
+      bucketName,
+      accessType,
+      versioning: false,
+    });
+
+    log.info(`[lazy-provision] S3 bucket "${bucketName}" provisioned successfully`);
+  };
+}