Merge pull request #969 from constructive-io/feat/lazy-s3-bucket-provisioning

feat: lazy S3 bucket provisioning on first upload

Author: pyramation

graphile/graphile-presigned-url-plugin/src/index.ts

@@ -30,7 +30,7 @@
 export { PresignedUrlPlugin, createPresignedUrlPlugin } from './plugin';
 export { createDownloadUrlPlugin } from './download-url-field';
 export { PresignedUrlPreset } from './preset';
-export { getStorageModuleConfig, getBucketConfig, clearStorageModuleCache, clearBucketCache } from './storage-module-cache';
+export { getStorageModuleConfig, getBucketConfig, clearStorageModuleCache, clearBucketCache, isS3BucketProvisioned, markS3BucketProvisioned } from './storage-module-cache';
 export { generatePresignedPutUrl, generatePresignedGetUrl, headObject } from './s3-signer';
 export type {
   BucketConfig,
@@ -43,4 +43,5 @@ export type {
   S3ConfigOrGetter,
   PresignedUrlPluginOptions,
   BucketNameResolver,
+  EnsureBucketProvisioned,
 } from './types';

graphile/graphile-presigned-url-plugin/src/plugin.ts

@@ -22,8 +22,8 @@ import type { GraphileConfig } from 'graphile-config';
 import { extendSchema, gql } from 'graphile-utils';
 import { Logger } from '@pgpmjs/logger';
 
-import type { PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types';
-import { getStorageModuleConfig, getBucketConfig } from './storage-module-cache';
+import type { PresignedUrlPluginOptions, S3Config, StorageModuleConfig, BucketConfig } from './types';
+import { getStorageModuleConfig, getBucketConfig, isS3BucketProvisioned, markS3BucketProvisioned } from './storage-module-cache';
 import { generatePresignedPutUrl, headObject } from './s3-signer';
 
 const log = new Logger('graphile-presigned-url:plugin');
@@ -112,6 +112,32 @@ function resolveS3ForDatabase(
   };
 }
 
+/**
+ * Ensure the S3 bucket for a database exists, provisioning it lazily if needed.
+ *
+ * Checks an in-memory Set of known-provisioned bucket names. On the first
+ * request for an unseen bucket, calls the `ensureBucketProvisioned` callback
+ * (which creates the bucket with correct CORS, policies, etc.), then marks
+ * it as provisioned so subsequent requests skip the check entirely.
+ *
+ * If no `ensureBucketProvisioned` callback is configured, this is a no-op.
+ */
+async function ensureS3BucketExists(
+  options: PresignedUrlPluginOptions,
+  s3BucketName: string,
+  bucket: BucketConfig,
+  databaseId: string,
+  allowedOrigins: string[] | null,
+): Promise<void> {
+  if (!options.ensureBucketProvisioned) return;
+  if (isS3BucketProvisioned(s3BucketName)) return;
+
+  log.info(`Lazy-provisioning S3 bucket "${s3BucketName}" for database ${databaseId}`);
+  await options.ensureBucketProvisioned(s3BucketName, bucket.type, databaseId, allowedOrigins);
+  markS3BucketProvisioned(s3BucketName);
+  log.info(`Lazy-provisioned S3 bucket "${s3BucketName}" successfully`);
+}
+
 export function createPresignedUrlPlugin(
   options: PresignedUrlPluginOptions,
 ): GraphileConfig.Plugin {
@@ -314,8 +340,11 @@ export function createPresignedUrlPlugin(
 
                 const fileId = fileResult.rows[0].id;
 
-                // --- Generate presigned PUT URL (per-database bucket) ---
+                // --- Ensure the S3 bucket exists (lazy provisioning) ---
                 const s3ForDb = resolveS3ForDatabase(options, storageConfig, databaseId);
+                await ensureS3BucketExists(options, s3ForDb.bucket, bucket, databaseId, storageConfig.allowedOrigins);
+
+                // --- Generate presigned PUT URL (per-database bucket) ---
                 const uploadUrl = await generatePresignedPutUrl(
                   s3ForDb,
                   s3Key,

graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts

@@ -47,6 +47,7 @@ const STORAGE_MODULE_QUERY = `
     sm.endpoint,
     sm.public_url_prefix,
     sm.provider,
+    sm.allowed_origins,
     sm.upload_url_expiry_seconds,
     sm.download_url_expiry_seconds,
     sm.default_max_file_size,
@@ -74,6 +75,7 @@ interface StorageModuleRow {
   endpoint: string | null;
   public_url_prefix: string | null;
   provider: string | null;
+  allowed_origins: string[] | null;
   upload_url_expiry_seconds: number | null;
   download_url_expiry_seconds: number | null;
   default_max_file_size: number | null;
@@ -121,6 +123,7 @@ export async function getStorageModuleConfig(
     endpoint: row.endpoint,
     publicUrlPrefix: row.public_url_prefix,
     provider: row.provider,
+    allowedOrigins: row.allowed_origins,
     uploadUrlExpirySeconds: row.upload_url_expiry_seconds ?? DEFAULT_UPLOAD_URL_EXPIRY_SECONDS,
     downloadUrlExpirySeconds: row.download_url_expiry_seconds ?? DEFAULT_DOWNLOAD_URL_EXPIRY_SECONDS,
     defaultMaxFileSize: row.default_max_file_size ?? DEFAULT_MAX_FILE_SIZE,
@@ -220,13 +223,46 @@ export async function getBucketConfig(
   return config;
 }
 
+// --- S3 bucket existence cache ---
+
+/**
+ * In-memory set of S3 bucket names that are known to exist.
+ *
+ * Used by the lazy provisioning logic in the presigned URL plugin:
+ * before generating a presigned PUT URL, the plugin checks this set.
+ * If the bucket name is absent, it calls `ensureBucketProvisioned`
+ * to create the S3 bucket, then adds the name here. Subsequent
+ * requests for the same bucket skip the provisioning entirely.
+ *
+ * No TTL needed — S3 buckets are never deleted during normal operation.
+ * The set resets on server restart, which is fine because the
+ * provisioner's createBucket is idempotent (handles "already exists").
+ */
+const provisionedBuckets = new Set<string>();
+
+/**
+ * Check whether an S3 bucket has already been provisioned (cached).
+ */
+export function isS3BucketProvisioned(s3BucketName: string): boolean {
+  return provisionedBuckets.has(s3BucketName);
+}
+
+/**
+ * Mark an S3 bucket as provisioned in the in-memory cache.
+ */
+export function markS3BucketProvisioned(s3BucketName: string): void {
+  provisionedBuckets.add(s3BucketName);
+  log.debug(`Marked S3 bucket "${s3BucketName}" as provisioned`);
+}
+
 /**
  * Clear the storage module cache AND bucket cache.
  * Useful for testing or schema changes.
  */
 export function clearStorageModuleCache(): void {
   storageModuleCache.clear();
   bucketCache.clear();
+  provisionedBuckets.clear();
 }
 
 /**

graphile/graphile-presigned-url-plugin/src/types.ts

@@ -42,6 +42,8 @@ export interface StorageModuleConfig {
   publicUrlPrefix: string | null;
   /** Storage provider type: 'minio', 's3', 'gcs', etc. (per-database override) */
   provider: string | null;
+  /** CORS allowed origins (per-database override, NULL = use global fallback) */
+  allowedOrigins: string[] | null;
 
   // --- Per-database configurable settings ---
 
@@ -147,6 +149,27 @@ export type S3ConfigOrGetter = S3Config | (() => S3Config);
  */
 export type BucketNameResolver = (databaseId: string) => string;
 
+/**
+ * Callback to lazily provision an S3 bucket on first use.
+ *
+ * Called by the presigned URL plugin before generating a presigned PUT URL
+ * when the bucket has not been seen before (tracked in an in-memory cache).
+ * The implementation should create and fully configure the S3 bucket
+ * (privacy policies, CORS, lifecycle rules, etc.) — or no-op if the
+ * bucket already exists.
+ *
+ * @param bucketName - The S3 bucket name to provision
+ * @param accessType - The logical bucket type ('public', 'private', 'temp')
+ * @param databaseId - The metaschema database UUID
+ * @param allowedOrigins - Per-database CORS origins (from storage_module), or null to use global fallback
+ */
+export type EnsureBucketProvisioned = (
+  bucketName: string,
+  accessType: 'public' | 'private' | 'temp',
+  databaseId: string,
+  allowedOrigins: string[] | null,
+) => Promise<void>;
+
 /**
  * Plugin options for the presigned URL plugin.
  */
@@ -160,4 +183,13 @@ export interface PresignedUrlPluginOptions {
    * the global `s3Config.bucket`. The S3 credentials (client) remain shared.
    */
   resolveBucketName?: BucketNameResolver;
+
+  /**
+   * Optional callback to lazily provision an S3 bucket on first upload.
+   * When set, the plugin calls this before generating a presigned PUT URL
+   * for any S3 bucket it hasn't seen yet (tracked in an in-memory cache).
+   * This enables graceful bucket creation without requiring buckets to
+   * exist at database provisioning time.
+   */
+  ensureBucketProvisioned?: EnsureBucketProvisioned;
 }

graphile/graphile-settings/package.json

@@ -30,6 +30,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.1009.0",
+    "@constructive-io/bucket-provisioner": "workspace:^",
     "@constructive-io/graphql-env": "workspace:^",
     "@constructive-io/graphql-types": "workspace:^",
     "@constructive-io/s3-streamer": "workspace:^",

graphile/graphile-settings/src/presets/constructive-preset.ts

@@ -19,7 +19,7 @@ import { PresignedUrlPreset } from 'graphile-presigned-url-plugin';
 import { BucketProvisionerPreset } from 'graphile-bucket-provisioner-plugin';
 import { SqlExpressionValidatorPreset } from 'graphile-sql-expression-validator';
 import { constructiveUploadFieldDefinitions } from '../upload-resolver';
-import { getPresignedUrlS3Config, createBucketNameResolver } from '../presigned-url-resolver';
+import { getPresignedUrlS3Config, createBucketNameResolver, createEnsureBucketProvisioned, getAllowedOrigins } from '../presigned-url-resolver';
 import { getBucketProvisionerConnection } from '../bucket-provisioner-resolver';
 
 /**
@@ -90,10 +90,14 @@ export const ConstructivePreset: GraphileConfig.Preset = {
       uploadFieldDefinitions: constructiveUploadFieldDefinitions,
       maxFileSize: 10 * 1024 * 1024, // 10MB
     }),
-    PresignedUrlPreset({ s3: getPresignedUrlS3Config, resolveBucketName: createBucketNameResolver() }),
+    PresignedUrlPreset({
+      s3: getPresignedUrlS3Config,
+      resolveBucketName: createBucketNameResolver(),
+      ensureBucketProvisioned: createEnsureBucketProvisioned(),
+    }),
     BucketProvisionerPreset({
       connection: getBucketProvisionerConnection,
-      allowedOrigins: ['http://localhost:3000'],
+      allowedOrigins: getAllowedOrigins(),
     }),
     SqlExpressionValidatorPreset(),
     PgTypeMappingsPreset,

graphile/graphile-settings/src/presigned-url-resolver.ts

@@ -14,7 +14,9 @@
 import { S3Client } from '@aws-sdk/client-s3';
 import { getEnvOptions } from '@constructive-io/graphql-env';
 import { Logger } from '@pgpmjs/logger';
-import type { S3Config, BucketNameResolver } from 'graphile-presigned-url-plugin';
+import type { S3Config, BucketNameResolver, EnsureBucketProvisioned } from 'graphile-presigned-url-plugin';
+import { BucketProvisioner } from '@constructive-io/bucket-provisioner';
+import { getBucketProvisionerConnection } from './bucket-provisioner-resolver';
 
 const log = new Logger('presigned-url-resolver');
 
@@ -81,3 +83,64 @@ export function createBucketNameResolver(): BucketNameResolver {
     return `${prefix}-${databaseId}`;
   };
 }
+
+/**
+ * Resolve CORS allowed origins from the env/config system.
+ *
+ * Reads SERVER_ORIGIN from the standard env hierarchy
+ * (pgpmDefaults → config file → env vars) and wraps it in an array.
+ * Falls back to ['http://localhost:3000'] for local development.
+ */
+export function getAllowedOrigins(): string[] {
+  const { server } = getEnvOptions();
+  if (server?.origin) return [server.origin];
+  return ['*'];
+}
+
+/**
+ * Create a lazy bucket provisioner callback for the presigned URL plugin.
+ *
+ * On the first upload to an S3 bucket that doesn't exist yet, this callback
+ * uses the BucketProvisioner to create and fully configure the bucket
+ * (Block Public Access, CORS, policies, lifecycle rules for temp buckets).
+ *
+ * Uses the same S3 connection config as the bucket provisioner plugin
+ * (getBucketProvisionerConnection) and reads CORS origins from
+ * SERVER_ORIGIN env var (falls back to localhost for local dev).
+ */
+export function createEnsureBucketProvisioned(): EnsureBucketProvisioned {
+  let provisioner: BucketProvisioner | null = null;
+
+  return async (
+    bucketName: string,
+    accessType: 'public' | 'private' | 'temp',
+    databaseId: string,
+    allowedOrigins: string[] | null,
+  ): Promise<void> => {
+    // Per-database origins from storage_module, falling back to global SERVER_ORIGIN
+    const effectiveOrigins = (allowedOrigins && allowedOrigins.length > 0)
+      ? allowedOrigins
+      : getAllowedOrigins();
+
+    if (!provisioner) {
+      provisioner = new BucketProvisioner({
+        connection: getBucketProvisionerConnection(),
+        allowedOrigins: effectiveOrigins,
+      });
+    }
+
+    log.info(
+      `[lazy-provision] Provisioning S3 bucket "${bucketName}" ` +
+      `(type=${accessType}) for database ${databaseId}`,
+    );
+
+    await provisioner.provision({
+      bucketName,
+      accessType,
+      versioning: false,
+      allowedOrigins: effectiveOrigins,
+    });
+
+    log.info(`[lazy-provision] S3 bucket "${bucketName}" provisioned successfully`);
+  };
+}