feat: switch to s3 storage (#926)

* feat: switch to s3 storage

* chore: make env var names consistent

* chore: make s3 loading lazy

* feat: migrations

* feat: fm migration

* chore: move migrations

* chore: s3 access issue

Author: coodos

platforms/esigner/api/package.json

@@ -13,6 +13,8 @@
         "migration:revert": "npm run typeorm migration:revert -- -d src/database/data-source.ts"
     },
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.1009.0",
+        "@aws-sdk/s3-request-presigner": "^3.1009.0",
         "axios": "^1.6.7",
         "cors": "^2.8.5",
         "dotenv": "^16.4.5",
@@ -23,9 +25,9 @@
         "multer": "^1.4.5-lts.1",
         "pg": "^8.11.3",
         "reflect-metadata": "^0.2.1",
+        "signature-validator": "workspace:*",
         "typeorm": "^0.3.24",
         "uuid": "^9.0.1",
-        "signature-validator": "workspace:*",
         "web3-adapter": "workspace:*"
     },
     "devDependencies": {
@@ -44,5 +46,3 @@
         "typescript": "^5.3.3"
     }
 }
-
-

platforms/esigner/api/src/controllers/FileController.ts

@@ -1,6 +1,7 @@
 import { Request, Response } from "express";
 import { FileService, ReservedFileNameError } from "../services/FileService";
 import multer from "multer";
+import { v4 as uuidv4 } from "uuid";
 
 export const MAX_FILE_SIZE = 20 * 1024 * 1024; // 20MB limit
 
@@ -16,6 +17,80 @@ export class FileController {
         this.fileService = new FileService();
     }
 
+    presignUpload = async (req: Request, res: Response) => {
+        try {
+            if (!req.user) {
+                return res.status(401).json({ error: "Authentication required" });
+            }
+
+            const { filename, mimeType, size } = req.body;
+
+            if (!filename || !mimeType || !size) {
+                return res.status(400).json({ error: "filename, mimeType, and size are required" });
+            }
+
+            const fileId = uuidv4();
+            const key = this.fileService.s3Service.generateKey(req.user.id, fileId, filename);
+            const uploadUrl = await this.fileService.s3Service.generateUploadUrl(key, mimeType);
+
+            res.json({ uploadUrl, key, fileId });
+        } catch (error) {
+            console.error("Error generating presigned URL:", error);
+            res.status(500).json({ error: "Failed to generate upload URL" });
+        }
+    };
+
+    confirmUpload = async (req: Request, res: Response) => {
+        try {
+            if (!req.user) {
+                return res.status(401).json({ error: "Authentication required" });
+            }
+
+            const { key, fileId, filename, mimeType, size, displayName, description } = req.body;
+
+            if (!key || !fileId || !filename || !mimeType || !size) {
+                return res.status(400).json({ error: "key, fileId, filename, mimeType, and size are required" });
+            }
+
+            const head = await this.fileService.s3Service.headObject(key);
+            const md5Hash = head.etag;
+            const url = this.fileService.s3Service.getPublicUrl(key);
+
+            const file = await this.fileService.createFileWithUrl(
+                fileId,
+                filename,
+                mimeType,
+                size,
+                md5Hash,
+                url,
+                req.user.id,
+                displayName,
+                description,
+            );
+
+            res.status(201).json({
+                id: file.id,
+                name: file.name,
+                displayName: file.displayName,
+                description: file.description,
+                mimeType: file.mimeType,
+                size: file.size,
+                md5Hash: file.md5Hash,
+                url: file.url,
+                createdAt: file.createdAt,
+            });
+        } catch (error) {
+            console.error("Error confirming upload:", error);
+            if (error instanceof ReservedFileNameError) {
+                return res.status(400).json({ error: error.message });
+            }
+            if (error instanceof Error) {
+                return res.status(400).json({ error: error.message });
+            }
+            res.status(500).json({ error: "Failed to confirm upload" });
+        }
+    };
+
     uploadFile = [
         upload.single('file'),
         async (req: Request, res: Response) => {
@@ -97,6 +172,7 @@ export class FileController {
                 mimeType: file.mimeType,
                 size: file.size,
                 md5Hash: file.md5Hash,
+                url: file.url,
                 ownerId: file.ownerId,
                 createdAt: file.createdAt,
                 updatedAt: file.updatedAt,
@@ -158,6 +234,11 @@ export class FileController {
                 return res.status(404).json({ error: "File not found" });
             }
 
+            if (file.url) {
+                return res.redirect(file.url);
+            }
+
+            // Legacy fallback for files still in DB
             res.setHeader('Content-Type', file.mimeType);
             res.setHeader('Content-Disposition', `attachment; filename="${file.name}"`);
             res.setHeader('Content-Length', file.size.toString());

platforms/esigner/api/src/controllers/WebhookController.ts

@@ -299,21 +299,18 @@ export class WebhookController {
                         file.md5Hash = local.data.md5Hash as string;
                     file.ownerId = owner.id;
 
-                    // Decode base64 data if provided
+                    // Store URL if provided
+                    if (local.data.url && typeof local.data.url === "string") {
+                        file.url = local.data.url;
+                    }
+                    // Legacy: decode base64 data if provided (backward compat)
                     if (local.data.data && typeof local.data.data === "string") {
                         file.data = Buffer.from(local.data.data, "base64");
                     }
 
                     this.adapter.addToLockedIds(localId);
                     await this.fileRepository.save(file);
                 } else {
-                    // Create new file with binary data
-                    // Decode base64 data if provided
-                    let fileData: Buffer = Buffer.alloc(0);
-                    if (local.data.data && typeof local.data.data === "string") {
-                        fileData = Buffer.from(local.data.data, "base64");
-                    }
-                    
                     const file = this.fileRepository.create({
                         name: local.data.name as string,
                         displayName: local.data.displayName as string | null,
@@ -322,7 +319,8 @@ export class WebhookController {
                         size: local.data.size as number,
                         md5Hash: local.data.md5Hash as string,
                         ownerId: owner.id,
-                        data: fileData,
+                        url: (local.data.url as string) || null,
+                        data: local.data.data ? Buffer.from(local.data.data as string, "base64") : null,
                     });
 
                     this.adapter.addToLockedIds(file.id);

platforms/esigner/api/src/database/entities/File.ts

@@ -35,8 +35,11 @@ export class File {
     @Column({ type: "text" })
     md5Hash!: string;
 
-    @Column({ type: "bytea" })
-    data!: Buffer;
+    @Column({ type: "bytea", nullable: true })
+    data!: Buffer | null;
+
+    @Column({ type: "text", nullable: true })
+    url!: string | null;
 
     @Column()
     ownerId!: string;

platforms/esigner/api/src/database/migrations/1773657072411-addurl.ts

@@ -0,0 +1,16 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Addurl1773657072411 implements MigrationInterface {
+    name = 'Addurl1773657072411'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "files" ADD "url" text`);
+        await queryRunner.query(`ALTER TABLE "files" ALTER COLUMN "data" DROP NOT NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "files" ALTER COLUMN "data" SET NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "files" DROP COLUMN "url"`);
+    }
+
+}

platforms/esigner/api/src/index.ts

@@ -83,6 +83,8 @@ app.post("/api/webhook", webhookController.handleWebhook);
 app.use(authMiddleware);
 
 // File routes
+app.post("/api/files/presign", authGuard, fileController.presignUpload);
+app.post("/api/files/confirm", authGuard, fileController.confirmUpload);
 app.post("/api/files", authGuard, fileController.uploadFile);
 app.get("/api/files", authGuard, fileController.getFiles);
 app.get("/api/files/:id", authGuard, fileController.getFile);

platforms/esigner/api/src/services/FileService.ts

@@ -2,6 +2,7 @@ import { AppDataSource } from "../database/data-source";
 import { File } from "../database/entities/File";
 import { FileSignee } from "../database/entities/FileSignee";
 import { SignatureContainer } from "../database/entities/SignatureContainer";
+import { S3Service } from "./S3Service";
 import crypto from "crypto";
 
 /** Soft-deleted marker from File Manager (no delete webhook); hide these in eSigner. */
@@ -19,6 +20,7 @@ export class FileService {
     private fileRepository = AppDataSource.getRepository(File);
     private fileSigneeRepository = AppDataSource.getRepository(FileSignee);
     private signatureRepository = AppDataSource.getRepository(SignatureContainer);
+    public s3Service = new S3Service();
 
     /**
      * Validates that the given filename is not the reserved soft-delete sentinel.
@@ -67,6 +69,39 @@ export class FileService {
         return savedFile;
     }
 
+    async createFileWithUrl(
+        id: string,
+        name: string,
+        mimeType: string,
+        size: number,
+        md5Hash: string,
+        url: string,
+        ownerId: string,
+        displayName?: string,
+        description?: string
+    ): Promise<File> {
+        this.validateFileName(name);
+
+        const fileData: Partial<File> = {
+            id,
+            name,
+            displayName: displayName || name,
+            mimeType,
+            size,
+            md5Hash,
+            url,
+            ownerId,
+        };
+
+        if (description !== undefined) {
+            fileData.description = description || null;
+        }
+
+        const file = this.fileRepository.create(fileData);
+        const savedFile = await this.fileRepository.save(file);
+        return savedFile;
+    }
+
     async getFileById(id: string, userId?: string): Promise<File | null> {
         const file = await this.fileRepository.findOne({
             where: { id },
@@ -174,6 +209,7 @@ export class FileService {
                 mimeType: file.mimeType,
                 size: file.size,
                 md5Hash: file.md5Hash,
+                url: file.url,
                 ownerId: file.ownerId,
                 owner: file.owner ? {
                     id: file.owner.id,

platforms/esigner/api/src/services/S3Service.ts

@@ -0,0 +1,114 @@
+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import { Readable } from "stream";
+
+export class S3Service {
+    private _client: S3Client | null = null;
+    private _bucket: string = "";
+    private _region: string = "";
+    private _cdnEndpoint: string = "";
+    private _originEndpoint: string = "";
+    private _initialized = false;
+
+    private init() {
+        if (this._initialized) return;
+        this._initialized = true;
+
+        this._originEndpoint = process.env.S3_ORIGIN || "";
+        this._cdnEndpoint = process.env.S3_CDN || "";
+
+        if (!this._originEndpoint) {
+            console.warn("S3_ORIGIN not set — S3 features will not work");
+            return;
+        }
+
+        const originUrl = new URL(this._originEndpoint);
+        const hostParts = originUrl.hostname.split(".");
+        this._bucket = hostParts[0];
+        this._region = hostParts[1];
+
+        this._client = new S3Client({
+            endpoint: `https://${this._region}.digitaloceanspaces.com`,
+            region: this._region,
+            credentials: {
+                accessKeyId: process.env.S3_ACCESS_KEY || "",
+                secretAccessKey: process.env.S3_SECRET_KEY || "",
+            },
+            forcePathStyle: false,
+        });
+    }
+
+    private get client(): S3Client {
+        this.init();
+        if (!this._client) throw new Error("S3 not configured — set S3_ORIGIN env var");
+        return this._client;
+    }
+
+    private get bucket(): string {
+        this.init();
+        return this._bucket;
+    }
+
+    generateKey(userId: string, fileId: string, filename: string): string {
+        return `files/${userId}/${fileId}/${filename}`;
+    }
+
+    async generateUploadUrl(key: string, contentType: string): Promise<string> {
+        const command = new PutObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+            ContentType: contentType,
+            ACL: "public-read",
+        });
+
+        return getSignedUrl(this.client, command, { expiresIn: 900 });
+    }
+
+    getPublicUrl(key: string): string {
+        this.init();
+        return `${this._cdnEndpoint}/${key}`;
+    }
+
+    extractKeyFromUrl(url: string): string {
+        this.init();
+        if (this._cdnEndpoint && url.startsWith(this._cdnEndpoint)) {
+            return url.slice(this._cdnEndpoint.length + 1);
+        }
+        if (this._originEndpoint && url.startsWith(this._originEndpoint)) {
+            return url.slice(this._originEndpoint.length + 1);
+        }
+        const urlObj = new URL(url);
+        return urlObj.pathname.slice(1);
+    }
+
+    async headObject(key: string): Promise<{ contentLength: number; etag: string }> {
+        const command = new HeadObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+        });
+
+        const response = await this.client.send(command);
+        return {
+            contentLength: response.ContentLength || 0,
+            etag: (response.ETag || "").replace(/"/g, ""),
+        };
+    }
+
+    async deleteObject(key: string): Promise<void> {
+        const command = new DeleteObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+        });
+        await this.client.send(command);
+    }
+
+    async getObjectStream(key: string): Promise<Readable> {
+        const command = new GetObjectCommand({
+            Bucket: this.bucket,
+            Key: key,
+        });
+
+        const response = await this.client.send(command);
+        return response.Body as Readable;
+    }
+}

platforms/esigner/api/src/web3adapter/mappings/file.mapping.json

@@ -11,6 +11,7 @@
         "size": "size",
         "md5Hash": "md5Hash",
         "data": "data",
+        "url": "url",
         "ownerId": "users(owner.id),ownerId",
         "createdAt": "__date(createdAt)",
         "updatedAt": "__date(updatedAt)"