diff --git a/.env.template b/.env.template index 706ee1a..18365ab 100644 --- a/.env.template +++ b/.env.template @@ -30,10 +30,6 @@ REDIS_URL="redis://default:password@localhost:6379" # VirusTotal API Configuration VIRUSTOTAL_API_KEY="your_virustotal_api_key" -# Vultr Object Storage — used for automated S3 bucket provisioning -# Generate at: https://my.vultr.com/settings/#settingsapi -VULTR_API_KEY="your_vultr_api_key" - # Sentry Configuration # DSN from: https://sentry.io → Project → Settings → Client Keys NEXT_PUBLIC_SENTRY_DSN="https://xxxx@oXXXX.ingest.sentry.io/XXXX" diff --git a/.github/README.md b/.github/README.md deleted file mode 100644 index 5cd40ac..0000000 --- a/.github/README.md +++ /dev/null @@ -1,188 +0,0 @@ -# Emberly - -Emberly is an open source platform for modern file storage, sharing, and identity verification. Build your digital presence with powerful tools for teams and individuals. - -[![Build Checks](https://github.com/EmberlyOSS/Emberly/actions/workflows/build.yml/badge.svg)](https://github.com/EmberlyOSS/Emberly/actions/workflows/build.yml) [![CodeQL Advanced](https://github.com/EmberlyOSS/Emberly/actions/workflows/codeql.yml/badge.svg)](https://github.com/EmberlyOSS/Emberly/actions/workflows/codeql.yml) ![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/EmberlyOSS/Emberly?utm_source=oss&utm_medium=github&utm_campaign=EmberlyOSS%2FEmberly&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FEmberlyOSS%2FEmberly.svg?type=shield&issueType=security)](https://app.fossa.com/projects/git%2Bgithub.com%2FEmberlyOSS%2FEmberly?ref=badge_shield&issueType=security) - -## Features - -**File Storage & Sharing** - -- S3 compatible object storage with configurable upload limits -- Secure file sharing with customizable access controls -- Real world file organization and management -- Bandwidth efficient delivery through global infrastructure - -**Domain & Branding** - -- Custom domain support with annual registration -- Personal or team branded file sharing pages -- Domain SSL certificate management -- DNS configuration assistance - -**Identity & Verification** - -- User verification badges with multiple tier options -- Verification queue with application review system -- Badge display on public profiles -- Organization verification for teams - -**Team & Collaboration** - -- Squad based team subscriptions with seat based pricing -- Granular permission management (roles: SUPPORT, DEVELOPER, MODERATOR, DESIGNER, STAFF) -- Team member invitations and management -- Shared storage pools with usage tracking - -**Applications & Trust** - -- Staff application system for organizational partnerships -- Partner program enrollment -- Verification badge applications -- Ban appeal process with review workflow -- Email notifications for all application updates - -**Administrative Tools** - -- Promo code management with configurable discounts -- User management dashboard -- Application review queue with multi stage triage -- Service status page link ([emberlystat.us](https://emberlystat.us)) -- Analytics and usage reporting - -## Quick Start - -For development setup, contribution guidelines, and detailed documentation, see [CONTRIBUTING.md](CONTRIBUTING.md). - -### Prerequisites - -- Node.js 18+ or later -- Bun package manager -- PostgreSQL 14+ database -- Redis 6+ (optional for caching) - -### Development Setup - -```bash -# Clone the repository -git clone https://github.com/EmberlyOSS/Emberly.git -cd Emberly - -# Install dependencies -bun install - -# Configure environment -cp .env.template .env -# Edit .env with your configuration - -# Initialize database -bun prisma generate -bun prisma migrate deploy - -# Start development server -bun dev -``` - -The application will be available at http://localhost:3000. - -## Tech Stack - -**Frontend & Framework** - -- [Next.js 15](https://nextjs.org/) - React framework with App Router -- [React 19](https://react.dev/) - UI library -- [TypeScript](https://www.typescriptlang.org/) - Type safety -- [Tailwind CSS](https://tailwindcss.com/) - Utility first styling -- [shadcn/ui](https://ui.shadcn.com/) - Component library - -**Backend & Database** - -- [PostgreSQL](https://www.postgresql.org/) - Relational database -- [Prisma ORM](https://www.prisma.io/) - Database toolkit -- [Stripe](https://stripe.com/) - Payment processing -- [Resend](https://resend.com/) - Email delivery - -**Infrastructure & Services** - -- [S3 compatible storage](https://aws.amazon.com/s3/) - File storage -- [Next.js Auth](https://next-auth.js.org/) - Authentication -- [Sentry](https://sentry.io/) - Error tracking - -**Development Tools** - -- [Bun](https://bun.sh/) - Runtime and package manager -- [ESLint](https://eslint.org/) - Code linting -- [Prisma Migrate](https://www.prisma.io/docs/orm/prisma-migrate) - Database migrations - -## Project Structure - -``` -app/ # Next.js App Router pages and routes - (main)/ # Public user pages - (raw)/ # Raw file serving - (shorturl)/ # Short URL redirects - api/ # API endpoints - -packages/ - components/ # React components - - admin/ # Admin dashboard components - - pricing/ # Pricing and plans - - auth/ # Authentication UI - - dashboard/ # User dashboard - - ui/ # Base UI building blocks - - hooks/ # Custom React hooks - - use file upload # File uploading - - use profile # User profile data - - use user content # User content queries - - lib/ # Utility functions - - api/ # API helpers - - auth/ # Authentication utilities - - cache/ # Caching logic - - stripe/ # Stripe integration - - types/ # TypeScript definitions - -prisma/ # Database schema and migrations - - schema.prisma # Prisma schema definition - - migrations/ # Migration files - -public/ # Static assets -scripts/ # Build and utility scripts -``` - -## Contributing - -We welcome contributions from the community. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines including: - -- Development environment setup -- Code standards and conventions -- Pull request process -- Commit message format -- How to report issues -- Community channels and support - -## Support - -Get help and connect with the community: - -- **Discord** - [Join our server](https://discord.gg/36spBmzZVB) for realtime discussions -- **GitHub Discussions** - Ask questions and share ideas -- **Email** - Contact [hey@embrly.ca](mailto:hey@embrly.ca) for support - -## License - -This project is licensed under the GNU Affero General Public License v3 (AGPL-3.0). See the [LICENSE](LICENSE) file for details. - -## Code of Conduct - -This project adheres to the Contributor Covenant Code of Conduct. By participating, you agree to uphold this code. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for the full text. - -## Acknowledgments - -Thank you to all [contributors](https://github.com/EmberlyOSS/Emberly/graphs/contributors) who have helped make Emberly possible. We also appreciate the [open source projects and communities](https://github.com/EmberlyOSS/Emberly/network/dependencies) that make Emberly possible. - - - Contributors - diff --git a/.gitignore b/.gitignore index 99204e9..fe36055 100644 --- a/.gitignore +++ b/.gitignore @@ -64,8 +64,6 @@ next-env.d.ts .idea/ .vscode/ .cursor/ -CLAUDE.md -AGENTS.md *.swp *.swo diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..7283800 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,323 @@ +# AGENTS.md — Emberly Codebase Guide for AI Agents + +This file is for AI coding assistants (Claude Code, GitHub Copilot, Cursor, etc.). It provides the practical context needed to work effectively in this codebase without re-deriving it every session. + +--- + +## Project at a glance + +**Emberly** is an open-source file storage, sharing, discovery, and identity-verification platform. +Stack: **Next.js 16 App Router · TypeScript (strict) · PostgreSQL (Prisma) · Redis · S3-compatible storage · Tailwind CSS · shadcn/ui** +License: AGPL-3.0-only + +--- + +## Essential commands + +```bash +# Install +bun install + +# Dev server (Turbopack) +bun dev # http://localhost:3000 + +# Type check (no emit) +bun typecheck # tsc --noEmit + +# Lint / format +bun lint +bun lint:fix +bun format # Prettier across all .ts/.tsx/.json/.md + +# Build +bun run build + +# Database +bun run db:generate # Regenerate Prisma client after schema changes +bun run db:migrate # Create + apply a dev migration +bun run db:deploy # Apply migrations (production/CI) +bun run db:push # Push schema without a migration (prototyping only) +bun run db:studio # GUI at http://localhost:5555 +bun run db:seed # Seed subscription plans +``` + +Run `bun typecheck` and `bun lint` before committing. Both run in CI. + +--- + +## Directory map + +``` +app/ Next.js App Router + (main)/ UI pages (auth, admin, [userUrlId] file serving) + (raw)/ Raw file responses + (shorturl)/ Short-URL redirect handler + api/ ~180 REST endpoints (admin/, auth/, files/, users/, urls/, ...) + +packages/ + components/ React component library + ui/ shadcn/ui base components + admin/ Admin UI + dashboard/ User dashboard + auth/ Auth UI + file/ File management UI + theme/ Snowfall / effects / dark mode + providers/ Context providers + shared/ Cross-cutting UI helpers + hooks/ Custom React hooks (use-file-upload, use-profile, ...) + lib/ Business logic + api/handler.ts Middleware wrapper for all API routes (auth, logging, error) + auth/ NextAuth config + helpers + events/ Event system (emitter · consumer · worker · handlers/) + database/ Prisma client singleton + cache/ Redis helpers + storage/ S3 / Vultr integration + emails/ Resend email templates + stripe/ Payment processing + permissions/ RBAC permission helpers + nexium/ Team/squad system + security/ Security utilities + logger/ Pino structured logging + startup/index.ts Server initialization (event system, Sentry, monitoring) + types/ Shared TypeScript definitions + events.ts Discriminated-union event types + dto/ Data-transfer object types + +prisma/ + schema.prisma Source of truth for DB models + migrations/ Migration history (do not edit manually) + generated/ Generated Prisma client (do not edit) + +scripts/ One-off utilities (seed, media-kit, migration helpers) +public/ Static assets +.github/ CI workflows, CONTRIBUTING.md +``` + +--- + +## API route conventions + +Every API handler must go through the wrapper in [packages/lib/api/handler.ts](packages/lib/api/handler.ts). It handles: + +- Request-ID tagging +- Structured logging (Pino) +- Session/auth validation +- Standardized error responses + +**Never** write raw `NextResponse` handlers — always use the handler wrapper. + +```typescript +// Correct pattern +import { createHandler } from '@/lib/api/handler' + +export const GET = createHandler(async (req, ctx) => { + // ctx.session is populated if the handler requires auth +}) +``` + +--- + +## Event system + +Background work goes through the event system in [packages/lib/events/](packages/lib/events/). + +- **Emit**: `packages/lib/events/emitter.ts` — publishes a typed event to the DB +- **Handle**: `packages/lib/events/handlers/` — one file per domain (auth, file, email, billing, discord, audit, …) +- **Worker**: runs when `EMBERLY_RUN_EVENT_WORKER=true`; processes queued events asynchronously + +Add new handlers in `handlers/` and register them in the consumer. Event types are in [packages/types/events.ts](packages/types/events.ts) — extend the discriminated union there first. + +--- + +## Path aliases + +Configured in `tsconfig.json`. Use these everywhere; never use `../../..` relative paths across package boundaries. + +| Alias | Resolves to | +| ---------------- | ----------------------- | +| `@/*` | project root | +| `@/components/*` | `packages/components/*` | +| `@/lib/*` | `packages/lib/*` | +| `@/hooks/*` | `packages/hooks/*` | +| `@/types/*` | `packages/types/*` | +| `@/database/*` | `prisma/*` | + +--- + +## Code style rules + +- **TypeScript strict mode** — no implicit `any`. Use `unknown` + type narrowing instead. +- **No `any`** — ESLint enforces this. The only allowed escape hatch is an explicit disable comment with a reason. +- **Server components by default** — only add `'use client'` when hooks, events, or browser APIs are required. +- **Functional components only** — hooks for state and side-effects. +- **Tailwind for styling** — no inline styles, no CSS modules. Dark mode is implemented via Tailwind's `dark:` variant. +- **shadcn/ui base components** — extend from `packages/components/ui/`, do not fork the components themselves. +- **No comments explaining what the code does** — only add a comment when the _why_ is non-obvious (hidden constraint, workaround, invariant). + +### Naming + +| Thing | Convention | +| --------------------- | ---------------------- | +| Files | `kebab-case.tsx` | +| React components | `PascalCase` | +| Functions / variables | `camelCase` | +| Constants | `SCREAMING_SNAKE_CASE` | +| Types / interfaces | `PascalCase` | + +### Import order (enforced by Prettier plugin) + +1. React +2. Next.js +3. Third-party packages +4. `@/components/…` +5. `@/lib/…` +6. `@/hooks/…` +7. `@/types/…` +8. Relative imports + +--- + +## Commit convention (enforced by commitlint + Husky) + +``` +(): +``` + +Types: `feat` · `fix` · `docs` · `style` · `refactor` · `perf` · `test` · `chore` · `ci` · `revert` + +Examples: + +``` +feat(auth): add magic-link sign-in flow +fix(upload): resolve off-by-one in chunked upload progress +refactor(api): consolidate 404 error path in handler wrapper +``` + +Commits that don't match the pattern are rejected by the pre-commit hook. + +--- + +## Database rules + +- **Never** edit `prisma/migrations/` by hand. +- **Never** call `db push` in production — use `db deploy`. +- After any change to `schema.prisma`, run `bun run db:generate` to regenerate the client. +- The Prisma client singleton lives in `packages/lib/database/` — always import from there, never instantiate `PrismaClient` directly. +- Keep the generated client out of git (`prisma/generated/` is gitignored). CI regenerates it. + +--- + +## Authentication + +- NextAuth v4 is configured in `packages/lib/auth/`. +- Providers: Discord OAuth, GitHub OAuth, credentials (email + password). +- Sessions are validated server-side via the API handler wrapper — do not roll your own session checks. +- 2FA (TOTP) and magic links are supported; see `packages/lib/auth/` for helpers. +- Password hashing: bcryptjs. Never store or log plaintext passwords. + +--- + +## Permissions / RBAC + +Roles: `USER` · `STAFF` · `SUPPORT` · `DEVELOPER` · `MODERATOR` · `DESIGNER` · `PARTNER` + +Permission checks live in `packages/lib/permissions/`. Always use the helpers there; do not hardcode role string comparisons in route handlers. + +--- + +## Storage + +Files are stored in S3-compatible object storage (AWS S3 or Vultr Object Storage). Metadata lives in the database; content lives in the bucket. + +- Client: `@aws-sdk/client-s3` — configured in `packages/lib/storage/` +- Use pre-signed URLs for uploads/downloads; never proxy raw file bytes through the Next.js process unless absolutely necessary +- Per-user bucket overrides are supported alongside the global default bucket + +--- + +## Caching (Redis) + +Redis is used for: + +- Rate limiting +- Session caching +- Event handler state + +Helpers are in `packages/lib/cache/`. Import from there; do not create raw Redis clients. + +Redis is optional for local development but required in production. + +--- + +## Environment variables + +Copy `.env.template` → `.env` to get started. Critical variables: + +| Variable | Purpose | +| --------------------------------------- | ---------------------------------- | +| `DATABASE_URL` | PostgreSQL connection string | +| `REDIS_URL` | Redis connection string | +| `NEXTAUTH_SECRET` | Session signing key (≥32 chars) | +| `NEXTAUTH_URL` / `NEXT_PUBLIC_BASE_URL` | App base URL | +| `DISCORD_OAUTH_CLIENT_ID/SECRET` | Discord OAuth | +| `GITHUB_OAUTH_CLIENT_ID/SECRET` | GitHub OAuth | +| `VIRUSTOTAL_API_KEY` | Malware scanning for uploads | +| `VULTR_API_KEY` | S3 bucket provisioning | +| `NEXT_PUBLIC_SENTRY_DSN` | Client-side error tracking | +| `EMBERLY_RUN_CLOUD` | Enable cloud/marketing features | +| `EMBERLY_RUN_EVENT_WORKER` | Enable background event processing | + +Never commit `.env`. Never log env vars. `SENTRY_AUTH_TOKEN` is build-time only (CI). + +--- + +## CI pipelines + +| Workflow | Trigger | What it checks | +| ------------- | -------------------------- | -------------------------------------------- | +| `build.yml` | push/PR → master, PR → dev | install → prisma generate → migrate → build | +| `quality.yml` | push/PR → master, PR → dev | install → prisma generate → typecheck → lint | +| `codeql.yml` | push → master + weekly | CodeQL advanced security scan | + +Both `build` and `quality` use Node.js 22 + Bun + a PostgreSQL 16 service container. + +--- + +## Security-sensitive areas + +- `packages/lib/security/` — rate limiting, IP checks, bot detection +- `packages/lib/middleware/` — `auth-checker.ts`, `bot-handler.ts` +- All file uploads go through VirusTotal scanning before acceptance +- User-generated HTML is sanitized with DomPurify before rendering +- Never introduce `dangerouslySetInnerHTML` without sanitization +- SQL queries always go through Prisma — no raw SQL string interpolation +- JWT/session tokens must never appear in logs or error responses + +--- + +## Key third-party integrations + +| Service | Package | Location | +| -------------- | ----------------------- | -------------------------------------- | +| PostgreSQL | `@prisma/client` | `packages/lib/database/` | +| Redis | `redis` | `packages/lib/cache/` | +| S3 / Vultr | `@aws-sdk/client-s3` | `packages/lib/storage/` | +| Email | `resend` | `packages/lib/emails/` | +| Payments | `stripe` | `packages/lib/stripe/` | +| Error tracking | `@sentry/nextjs` | `instrumentation.ts`, `next.config.ts` | +| Logging | `pino` | `packages/lib/logger/` | +| OCR | `tesseract.js` | `packages/lib/ocr/` | +| Code editor | `@uiw/react-codemirror` | used in file preview components | + +--- + +## What to avoid + +- Do not bypass the API handler wrapper — missing auth/logging is a security hole. +- Do not call `new PrismaClient()` — use the singleton. +- Do not use `any` without a disable comment and a reason. +- Do not add `'use client'` to components that don't need it — it expands the client bundle. +- Do not commit secrets, `.env`, or `prisma/generated/`. +- Do not write raw SQL strings — Prisma only. +- Do not add feature flags, backwards-compat shims, or half-finished stubs — implement fully or don't implement. +- Do not add error handling for scenarios that can't happen — trust Prisma/Next.js guarantees at internal boundaries. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d9e23f..6c766ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,35 @@ All notable changes to this project will be documented in this file. The format is based on "Keep a Changelog" and follows [Semantic Versioning](https://semver.org/). +## [2.4.7] - 2026-06-15 + +### Added + +- **Core Pool Storage Buckets** — `StorageBucket` now supports `isCore` (boolean) and `priority` (integer) fields. Core buckets are shared across all users via least-filled load balancing (`fileCount ASC, priority DESC`). Dedicated buckets are assigned per-user or per-squad. Priority breaks ties when two core buckets have equal file counts — higher number wins. Default is `0` (no preference). +- **`File → StorageBucket` relation** — Files now track which bucket they were stored in via `storageBucketId`. This is used at serve time to route reads, streams, deletes, OCR, and thumbnail generation to the correct bucket rather than the global default. +- **User bucket assignment UI** — New `UserBucketAssignment` component in the admin settings Storage tab. Paginated user list with per-user bucket dropdown and save button. Clears assignment (falls back to core pool) or assigns a dedicated bucket; triggers credentials email on assignment. +- **Admin storage bucket manager — `isCore` / `priority` UI** — Edit dialog now includes a Core Pool Bucket toggle and a Priority number input (visible when `isCore` is on). List rows show Core Pool badge (violet), Dedicated badge, file count, and priority for core buckets. +- **Bucket connection test — per-bucket provider** — `POST /api/admin/storage/buckets/[id]/test` now uses an explicit `select` in `findUnique` (fixes a `@prisma/adapter-pg` driver adapter issue that caused `findUnique` without a select to return `null` for existing records). Test error detail is now surfaced inline in the UI alongside the status message. +- **`PUT /api/admin/storage/buckets` — `priority` on create** — Create route now reads and persists `priority` from the request body. Previously new buckets always defaulted to `0` regardless of the value submitted. +- **User bucket page — secret key reveal** — The `/dashboard/bucket` page now shows the actual secret access key with a show/hide toggle (`BucketSecretReveal` client component). All other credentials are hidden behind a "coming soon" notice while direct S3 credential access is being developed. +- **`reconcileDeprovisionedBuckets` — atomic `Subscription.status` update** — Both cancellation paths (cancelled subscription, subscription not found in Stripe) now use `prisma.$transaction` to atomically update the bucket (`provisionStatus → deprovisioning`), subscription (`status → canceled`), and clear user assignments in a single operation. + +### Fixed + +- **File serving — wrong storage provider for bucket-routed files** — All file read/stream/delete paths used `getStorageProvider()` (global config bucket singleton) even when the file was stored in a core or dedicated bucket. Affected routes: `GET /api/files/[...path]`, `GET /api/files/[id]/thumbnail`, suggestion approval in `PUT /api/files/[id]/suggestions`, OCR processing, file expiry handler, and single/bulk delete in the file service. All now use `getProviderForStoredFile(file.storageBucketId)`, which routes to the correct bucket and falls back to global for legacy files. +- **S3 key normalization — Windows backslash paths** — `path.join()` produces backslash-separated paths on Windows (e.g. `uploads\userUrlId\filename.png`). The S3 provider's key derivation regex `/^uploads\//` never matched these, leaving the full backslash path as the S3 key and causing `NoSuchKey` on every read. Added `toS3Key()` helper that normalizes backslashes to forward slashes before stripping the `uploads/` prefix; applied to all five S3 provider methods (`uploadFile`, `getFile`, `getFileStream`, `deleteFile`, `getFileUrl`, `getFileSize`). +- **Upload path generation — forward slashes enforced** — `app/api/files/route.ts` and `app/api/files/chunks/route.ts` used `path.join()` to build `filePath`, producing backslash paths on Windows that were stored in the database and used as S3 keys. Both now use `path.posix.join()` so paths are always forward-slash regardless of platform. +- **OCR — wrong bucket** — `processImageOCRTask` called `getStorageProvider()` unconditionally. It now looks up `file.storageBucketId` and routes to `getProviderForStoredFile` so OCR reads from the same bucket the file was uploaded to. +- **Bucket list — `priority` missing from select** — `GET /api/admin/storage/buckets` did not include `priority` in its field select. The edit dialog received `priority: undefined`, causing the priority input to be uncontrolled (React warning) and saving to silently leave the existing value unchanged. +- **Edit dialog — uncontrolled `priority` input** — `openEdit` now uses `bucket.priority ?? 0` when populating the form, preventing the React "uncontrolled → controlled" warning when `priority` was absent from the list response. +- **Dev logger — error detail suppressed** — The development pino transport only printed `obj.msg`, silently dropping `obj.error`. Error message and stack trace are now appended to the console output. +- **Chunks route — Prisma XOR type error** — `tx.file.create` mixed `storageBucketId` (unchecked input) with `user: { connect: { id } }` (checked input), which Prisma's XOR type rejects. Replaced with `userId` scalar field. + +### Changed + +- **`getStorageProvider()` scope clarified** — The global singleton is now the fallback only for legacy files (`storageBucketId = null`). All new file operations route through `getProviderForStoredFile` or `getUploadBucketForUser`/`getUploadBucketForSquad`. +- **`selectCoreBucket` ordering** — Core bucket selection now orders by `fileCount ASC, priority DESC` so higher-priority buckets win when file counts are equal. + ## [2.4.6] - 2026-06-13 ### Security diff --git a/.github/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md similarity index 100% rename from .github/CODE_OF_CONDUCT.md rename to CODE_OF_CONDUCT.md diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 100% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..363b285 --- /dev/null +++ b/README.md @@ -0,0 +1,199 @@ +# Emberly + +Emberly is an open-source platform for modern file storage, sharing, discovery, and identity verification. Build your digital presence with powerful tools for teams and individuals. + +[![Build Checks](https://github.com/EmberlyOSS/Emberly/actions/workflows/build.yml/badge.svg)](https://github.com/EmberlyOSS/Emberly/actions/workflows/build.yml) [![CodeQL Advanced](https://github.com/EmberlyOSS/Emberly/actions/workflows/codeql.yml/badge.svg)](https://github.com/EmberlyOSS/Emberly/actions/workflows/codeql.yml) ![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/EmberlyOSS/Emberly?utm_source=oss&utm_medium=github&utm_campaign=EmberlyOSS%2FEmberly&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FEmberlyOSS%2FEmberly.svg?type=shield&issueType=security)](https://app.fossa.com/projects/git%2Bgithub.com%2FEmberlyOSS%2FEmberly?ref=badge_shield&issueType=security) + +## Features + +**File Storage & Sharing** + +- S3-compatible object storage with configurable upload limits +- Secure file sharing with customizable access controls +- File organization, tagging, and search +- OCR-powered text extraction from uploaded images and documents +- URL shortening with redirect tracking +- Bandwidth-efficient delivery through global infrastructure + +**Domain & Branding** + +- Custom domain support with annual registration +- Personal or team branded file-sharing pages +- Domain SSL certificate management +- DNS configuration assistance + +**Identity & Verification** + +- User verification badges with multiple tier options +- Verification queue with application review system +- Badge display on public profiles +- Organization verification for teams + +**Team & Collaboration** + +- Squad-based team subscriptions with seat-based pricing +- Granular permission management (roles: `SUPPORT`, `DEVELOPER`, `MODERATOR`, `DESIGNER`, `STAFF`) +- Team member invitations and management +- Shared storage pools with usage tracking + +**Applications & Trust** + +- Staff application system for organizational partnerships +- Partner program enrollment +- Verification badge applications +- Ban appeal process with review workflow +- Email notifications for all application updates + +**Administrative Tools** + +- Promo code management with configurable discounts +- User management dashboard +- Application review queue with multi-stage triage +- Service status page ([emberlystat.us](https://emberlystat.us)) +- Analytics and usage reporting + +## Quick Start + +For contribution guidelines and detailed documentation see [CONTRIBUTING.md](CONTRIBUTING.md). + +### Prerequisites + +- Node.js 18+ +- [Bun](https://bun.sh/) (recommended package manager) +- PostgreSQL 14+ +- Redis 6+ (optional for caching and rate limiting) + +### Development Setup + +```bash +# Clone the repository +git clone https://github.com/EmberlyOSS/Emberly.git +cd Emberly + +# Install dependencies +bun install + +# Configure environment +cp .env.template .env +# Edit .env with your local configuration + +# Initialize database +bun run db:generate +bun run db:migrate + +# (Optional) Seed subscription plans +bun run db:seed + +# Start development server +bun dev +``` + +The application will be available at `http://localhost:3000`. + +## Tech Stack + +**Frontend & Framework** + +- [Next.js 16](https://nextjs.org/) — React framework with App Router +- [React 19](https://react.dev/) — UI library +- [TypeScript](https://www.typescriptlang.org/) — strict-mode type safety +- [Tailwind CSS](https://tailwindcss.com/) — utility-first styling +- [shadcn/ui](https://ui.shadcn.com/) — accessible component library + +**Backend & Database** + +- [PostgreSQL](https://www.postgresql.org/) — relational database +- [Prisma ORM](https://www.prisma.io/) — database toolkit and migrations +- [Redis](https://redis.io/) — caching and rate limiting +- [Stripe](https://stripe.com/) — payment processing +- [Resend](https://resend.com/) — transactional email delivery + +**Infrastructure & Services** + +- S3-compatible object storage (AWS S3 / Vultr) — file storage +- [NextAuth](https://next-auth.js.org/) — authentication (Discord OAuth, GitHub OAuth, credentials) +- [Sentry](https://sentry.io/) — error tracking and monitoring +- [VirusTotal](https://www.virustotal.com/) — file scanning on upload + +**Development Tools** + +- [Bun](https://bun.sh/) — runtime and package manager +- [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) — linting and formatting +- [Husky](https://typicode.github.io/husky/) + [commitlint](https://commitlint.js.org/) — git hooks and commit linting + +## Project Structure + +``` +app/ Next.js App Router pages and routes + (main)/ Public user pages (auth, admin, user profiles) + (raw)/ Raw file serving + (shorturl)/ Short URL redirects + api/ ~180 REST API endpoints + +packages/ + components/ React component library + admin/ Admin dashboard components + dashboard/ User dashboard + auth/ Authentication UI + file/ File management UI + pricing/ Pricing and plans + ui/ shadcn/ui base components + hooks/ Custom React hooks + use-file-upload.tsx File uploading with progress + use-profile.ts User profile data + use-user-content.ts User content queries + use-file-actions.ts File action handlers + lib/ Business logic and integrations + api/ API handler wrapper (auth, logging, errors) + auth/ NextAuth config and helpers + events/ Event system (emitter, consumer, worker, handlers) + cache/ Redis helpers + storage/ S3 / Vultr integration + emails/ Email templates (Resend) + stripe/ Payment processing + permissions/ RBAC permission helpers + nexium/ Team/squad system + security/ Security utilities and rate limiting + logger/ Pino structured logging + types/ Shared TypeScript type definitions + +prisma/ + schema.prisma Database schema + migrations/ Migration history + +public/ Static assets +scripts/ Utility scripts (seed, media-kit) +``` + +## Contributing + +We welcome contributions from the community. Please see [CONTRIBUTING.md](CONTRIBUTING.md) for detailed guidelines including: + +- Development environment setup +- Code standards and conventions +- Pull request process +- Commit message format +- How to report issues +- Community channels and support + +## Support + +- **Discord** — [Join our server](https://discord.gg/36spBmzZVB) for real-time discussions +- **GitHub Discussions** — Ask questions and share ideas +- **Email** — [hey@embrly.ca](mailto:hey@embrly.ca) for support + +## License + +This project is licensed under the GNU Affero General Public License v3 (AGPL-3.0). See the [LICENSE](LICENSE) file for details. + +## Code of Conduct + +This project adheres to the Contributor Covenant Code of Conduct. By participating, you agree to uphold this code. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for the full text. + +## Acknowledgments + +Thank you to all [contributors](https://github.com/EmberlyOSS/Emberly/graphs/contributors) who have helped make Emberly possible, and to the [open-source projects](https://github.com/EmberlyOSS/Emberly/network/dependencies) that power it. + + + Contributors + diff --git a/.github/SECURITY.md b/SECURITY.md similarity index 82% rename from .github/SECURITY.md rename to SECURITY.md index 4ad5cc1..881dc0b 100644 --- a/.github/SECURITY.md +++ b/SECURITY.md @@ -6,13 +6,13 @@ Thanks for helping keep Emberly and its users safe. We provide security fixes for: -| Version | Supported | -| --------- | ------- | -| `main` branch | Yes | -| `develop` branch | Yes | -| 2.4.x | Yes | -| 2.3.x | Yes | -| < 2.2.x | No | +| Version | Supported | +| ---------------- | --------- | +| `main` branch | Yes | +| `develop` branch | Yes | +| 2.4.x | Yes | +| 2.3.x | Yes | +| < 2.2.x | No | ## Reporting a Vulnerability diff --git a/app/(main)/dashboard/bucket/page.tsx b/app/(main)/dashboard/bucket/page.tsx index 2bb6b20..efba8c6 100644 --- a/app/(main)/dashboard/bucket/page.tsx +++ b/app/(main)/dashboard/bucket/page.tsx @@ -1,43 +1,19 @@ import { redirect } from 'next/navigation' import { getServerSession } from 'next-auth' -import { AlertTriangle, Key, Server } from 'lucide-react' +import { AlertTriangle, Clock, Key, Server } from 'lucide-react' import { authOptions } from '@/packages/lib/auth' import { prisma } from '@/packages/lib/database/prisma' import { buildPageMetadata } from '@/packages/lib/embeds/metadata' import { provisionBucketForUserSubscription } from '@/packages/lib/storage/bucket-provisioning' import { DashboardShell } from '@/packages/components/dashboard/dashboard-shell' +import { BucketSecretReveal } from '@/packages/components/dashboard/bucket-secret-reveal' export const metadata = buildPageMetadata({ title: 'Storage Bucket', description: 'View your dedicated S3 storage bucket credentials and status.', }) -function CredentialRow({ - label, - value, - mono = true, -}: { - label: string - value: string - mono?: boolean -}) { - return ( -
- - {label} - -
- - {value} - -
-
- ) -} - export default async function BucketPage() { const session = await getServerSession(authOptions) if (!session?.user?.id) redirect('/auth/signin') @@ -56,6 +32,7 @@ export default async function BucketPage() { s3Bucket: true, s3Region: true, s3AccessKeyId: true, + s3SecretKey: true, s3Endpoint: true, s3ForcePathStyle: true, }, @@ -122,6 +99,7 @@ export default async function BucketPage() { s3Bucket: true, s3Region: true, s3AccessKeyId: true, + s3SecretKey: true, s3Endpoint: true, s3ForcePathStyle: true, }, @@ -192,12 +170,6 @@ export default async function BucketPage() { ) } - // Mask access key ID (show only first 4 + last 4 chars) - const maskedKeyId = - bucket.s3AccessKeyId.length > 8 - ? `${bucket.s3AccessKeyId.slice(0, 4)}••••••••${bucket.s3AccessKeyId.slice(-4)}` - : `${bucket.s3AccessKeyId.slice(0, 4)}••••` - return ( } > - {/* Security notice */} -
- + {/* Coming soon notice */} +
+ - Keep your credentials safe. Never expose your Secret Key publicly. If - you believe your credentials have been compromised, contact{' '} + Direct S3 credential access is coming soon. In the meantime, your + connection credentials were delivered by email when your bucket was + assigned. Contact{' '} support@embrly.ca {' '} - immediately. + if you need them resent.
- {/* Credentials */} -
-
- -

Connection Credentials

-
-
- - - - - {bucket.s3ForcePathStyle !== undefined && ( - - )} - + {/* Secret key reveal */} + {bucket.s3SecretKey && ( +
+
+ +

Secret Access Key

+
+
+
+ + + Never share your secret key. If you believe it has been + compromised, contact support immediately. + +
+ +
-
+ )} {/* Help */}

Need help?

- Check your email for the full credentials sent when your bucket was - assigned. For troubleshooting or to rotate your access key, contact{' '} + For troubleshooting or to rotate your access key, contact{' '} ({ ...b, - s3AccessKeyId: b.s3AccessKeyId ? `${b.s3AccessKeyId.slice(0, 4)}••••` : '', + s3AccessKeyId: b.s3AccessKeyId + ? `${b.s3AccessKeyId.slice(0, 4)}••••` + : '', s3SecretKey: '', })) ) @@ -52,22 +58,49 @@ export async function POST(req: Request) { if (response) return response const body = await req.json() - const { name, provider = 's3', s3Bucket, s3Region, s3AccessKeyId, s3SecretKey, s3Endpoint, s3ForcePathStyle, vultrObjectStorageId, vultrBucketName } = body + const { + name, + provider = 's3', + isCore = false, + priority = 0, + s3Bucket, + s3Region, + s3AccessKeyId, + s3SecretKey, + s3Endpoint, + s3ForcePathStyle, + vultrObjectStorageId, + vultrBucketName, + } = body - if (!name?.trim()) return apiError('Bucket name is required', HTTP_STATUS.BAD_REQUEST) + if (!name?.trim()) + return apiError('Bucket name is required', HTTP_STATUS.BAD_REQUEST) // For Vultr-backed buckets, populate S3 credentials from the existing VultrObjectStorage record if (provider === 'vultr') { - if (!vultrObjectStorageId) return apiError('Vultr instance is required', HTTP_STATUS.BAD_REQUEST) - if (!vultrBucketName?.trim()) return apiError('Bucket name within the Vultr instance is required', HTTP_STATUS.BAD_REQUEST) + if (!vultrObjectStorageId) + return apiError('Vultr instance is required', HTTP_STATUS.BAD_REQUEST) + if (!vultrBucketName?.trim()) + return apiError( + 'Bucket name within the Vultr instance is required', + HTTP_STATUS.BAD_REQUEST + ) - const vultr = await prisma.vultrObjectStorage.findUnique({ where: { id: vultrObjectStorageId } }) - if (!vultr) return apiError('Vultr Object Storage instance not found', HTTP_STATUS.NOT_FOUND) + const vultr = await prisma.vultrObjectStorage.findUnique({ + where: { id: vultrObjectStorageId }, + }) + if (!vultr) + return apiError( + 'Vultr Object Storage instance not found', + HTTP_STATUS.NOT_FOUND + ) const bucket = await prisma.storageBucket.create({ data: { name: name.trim(), provider: 's3', + isCore: Boolean(isCore), + priority: Number(priority) || 0, s3Bucket: vultrBucketName.trim(), s3Region: vultr.region, s3AccessKeyId: vultr.s3AccessKey, @@ -79,15 +112,21 @@ export async function POST(req: Request) { }, }) - return apiResponse( - { ...bucket, s3AccessKeyId: bucket.s3AccessKeyId ? `${bucket.s3AccessKeyId.slice(0, 4)}••••` : '', s3SecretKey: '' } - ) + return apiResponse({ + ...bucket, + s3AccessKeyId: bucket.s3AccessKeyId + ? `${bucket.s3AccessKeyId.slice(0, 4)}••••` + : '', + s3SecretKey: '', + }) } const bucket = await prisma.storageBucket.create({ data: { name: name.trim(), provider, + isCore: Boolean(isCore), + priority: Number(priority) || 0, s3Bucket: s3Bucket ?? '', s3Region: s3Region ?? '', s3AccessKeyId: s3AccessKeyId ?? '', @@ -97,12 +136,15 @@ export async function POST(req: Request) { }, }) - return apiResponse( - { ...bucket, s3AccessKeyId: bucket.s3AccessKeyId ? `${bucket.s3AccessKeyId.slice(0, 4)}••••` : '', s3SecretKey: '' } - ) + return apiResponse({ + ...bucket, + s3AccessKeyId: bucket.s3AccessKeyId + ? `${bucket.s3AccessKeyId.slice(0, 4)}••••` + : '', + s3SecretKey: '', + }) } catch (error) { logger.error('Failed to create storage bucket', error as Error) return apiError('Internal server error') } } - diff --git a/app/api/admin/storage/users/[id]/route.ts b/app/api/admin/storage/users/[id]/route.ts new file mode 100644 index 0000000..e7b0764 --- /dev/null +++ b/app/api/admin/storage/users/[id]/route.ts @@ -0,0 +1,100 @@ +import { HTTP_STATUS, apiError, apiResponse } from '@/packages/lib/api/response' +import { requireAdmin } from '@/packages/lib/auth/api-auth' +import { prisma } from '@/packages/lib/database/prisma' +import { loggers } from '@/packages/lib/logger' + +const logger = loggers.storage + +interface Params { + params: Promise<{ id: string }> +} + +/** Assign or clear a dedicated storage bucket for a user. Body: { bucketId: string | null } */ +export async function PUT(req: Request, { params }: Params) { + try { + const { response } = await requireAdmin(req) + if (response) return response + + const { id: userId } = await params + const body = await req.json() + const bucketId: string | null = body?.bucketId ?? null + + const user = await prisma.user.findUnique({ + where: { id: userId }, + select: { id: true, storageBucketId: true }, + }) + if (!user) return apiError('User not found', HTTP_STATUS.NOT_FOUND) + + if (bucketId) { + const bucket = await prisma.storageBucket.findUnique({ + where: { id: bucketId }, + select: { id: true, provisionStatus: true }, + }) + if (!bucket) + return apiError('Storage bucket not found', HTTP_STATUS.NOT_FOUND) + if (bucket.provisionStatus !== 'active') { + return apiError( + 'Cannot assign a bucket that is not active', + HTTP_STATUS.BAD_REQUEST + ) + } + } + + await prisma.user.update({ + where: { id: userId }, + data: { storageBucketId: bucketId }, + }) + + logger.info('Admin assigned storage bucket to user', { + userId, + bucketId, + previousBucketId: user.storageBucketId, + }) + + return apiResponse({ userId, storageBucketId: bucketId }) + } catch (error) { + logger.error('Failed to assign storage bucket to user', error as Error) + return apiError('Internal server error', HTTP_STATUS.INTERNAL_SERVER_ERROR) + } +} + +/** GET current bucket assignment for a user. */ +export async function GET(req: Request, { params }: Params) { + try { + const { response } = await requireAdmin(req) + if (response) return response + + const { id: userId } = await params + + const user = await prisma.user.findUnique({ + where: { id: userId }, + select: { + id: true, + storageBucketId: true, + storageBucket: { + select: { + id: true, + name: true, + provider: true, + isCore: true, + provisionStatus: true, + s3Region: true, + s3Bucket: true, + fileCount: true, + createdAt: true, + }, + }, + }, + }) + if (!user) return apiError('User not found', HTTP_STATUS.NOT_FOUND) + + return apiResponse({ + userId: user.id, + storageBucketId: user.storageBucketId, + bucket: user.storageBucket ?? null, + }) + } catch (error) { + logger.error('Failed to get user storage bucket assignment', error as Error) + return apiError('Internal server error', HTTP_STATUS.INTERNAL_SERVER_ERROR) + } +} diff --git a/app/api/files/[...path]/route.ts b/app/api/files/[...path]/route.ts index bcf14a5..17bf3f5 100644 --- a/app/api/files/[...path]/route.ts +++ b/app/api/files/[...path]/route.ts @@ -6,7 +6,7 @@ import { authOptions } from '@/packages/lib/auth' import { checkFileAccess } from '@/packages/lib/files/access' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { handleCORSPreflight, getCORSHeaders } from '@/packages/lib/api/cors' const logger = loggers.files @@ -40,13 +40,19 @@ export async function GET( // urlPath here is already assembled from the path segments let file = await prisma.file.findUnique({ where: { urlPath }, - include: { user: { select: { enableRichEmbeds: true } } }, + include: { + user: { select: { enableRichEmbeds: true } }, + storageBucket: { select: { id: true } }, + }, }) if (!file && urlPath.includes(' ')) { file = await prisma.file.findUnique({ where: { urlPath: urlPath.replace(/ /g, '-') }, - include: { user: { select: { enableRichEmbeds: true } } }, + include: { + user: { select: { enableRichEmbeds: true } }, + storageBucket: { select: { id: true } }, + }, }) } @@ -71,7 +77,9 @@ export async function GET( const enableRichEmbeds = file.user?.enableRichEmbeds ?? true const shouldForceDownload = isDownloadRequest || !enableRichEmbeds - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + file.storageBucket?.id + ) const isVideo = file.mimeType.startsWith('video/') const range = request.headers.get('range') const size = await storageProvider.getFileSize(file.path) diff --git a/app/api/files/[id]/content/route.ts b/app/api/files/[id]/content/route.ts index df77d88..6915027 100644 --- a/app/api/files/[id]/content/route.ts +++ b/app/api/files/[id]/content/route.ts @@ -5,170 +5,178 @@ import { getServerSession } from 'next-auth' import { authOptions } from '@/packages/lib/auth' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { bytesToMB } from '@/packages/lib/utils' const logger = loggers.files // Update file content (for text-based files like pastes) export async function PUT( - request: Request, - { params }: { params: Promise<{ id: string }> } + request: Request, + { params }: { params: Promise<{ id: string }> } ) { - try { - const { id } = await params - const session = await getServerSession(authOptions) - - if (!session?.user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const file = await prisma.file.findUnique({ - where: { id }, - include: { - collaborators: { - where: { userId: session.user.id }, - }, - }, - }) + try { + const { id } = await params + const session = await getServerSession(authOptions) - if (!file) { - return NextResponse.json({ error: 'File not found' }, { status: 404 }) - } - - // Check if user is owner or has EDITOR role - const isOwner = file.userId === session.user.id - const collaborator = file.collaborators[0] - const canEdit = isOwner || collaborator?.role === 'EDITOR' - - if (!canEdit) { - // Check if they can at least suggest - const canSuggest = collaborator?.role === 'SUGGESTER' || file.allowSuggestions - if (canSuggest) { - return NextResponse.json( - { error: 'You can only suggest edits, not edit directly. Use the suggestions endpoint.' }, - { status: 403 } - ) - } - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - // Only allow updating text-based files - const textMimeTypes = [ - 'text/plain', - 'text/markdown', - 'text/x-markdown', - 'text/javascript', - 'text/typescript', - 'text/jsx', - 'text/tsx', - 'text/x-python', - 'text/html', - 'text/css', - 'text/xml', - 'text/yaml', - 'text/x-yaml', - 'text/x-sql', - 'text/x-java', - 'text/x-c++src', - 'text/x-csrc', - 'text/x-rustsrc', - 'text/x-go', - 'text/x-php', - 'text/x-sass', - 'text/x-scss', - 'text/x-less', - 'application/json', - 'application/javascript', - 'application/typescript', - 'application/xml', - ] - - if (!textMimeTypes.includes(file.mimeType) && !file.mimeType.startsWith('text/')) { - return NextResponse.json( - { error: 'Only text-based files can be edited' }, - { status: 400 } - ) - } - - const body = await request.json() - const { content, allowSuggestions } = body - - if (typeof content !== 'string') { - return NextResponse.json( - { error: 'Content must be a string' }, - { status: 400 } - ) - } - - const storageProvider = await getStorageProvider() - - // Convert content to buffer and upload - const buffer = Buffer.from(content, 'utf-8') - const newSizeBytes = buffer.length - // Store size in MB (consistent with upload API) - const newSizeMB = bytesToMB(newSizeBytes) - - // Update storage with new content - await storageProvider.uploadFile(buffer, file.path, file.mimeType) - - // Calculate storage difference in MB (file.size is already in MB) - const sizeDifference = newSizeMB - file.size - - // Prepare update data - const updateData: { - size: number - updatedAt: Date - allowSuggestions?: boolean - } = { - size: newSizeMB, - updatedAt: new Date(), - } - - // Only allow owner to change allowSuggestions - if (isOwner && typeof allowSuggestions === 'boolean') { - updateData.allowSuggestions = allowSuggestions - } - - // Update file record - const updatedFile = await prisma.$transaction(async (tx) => { - const updated = await tx.file.update({ - where: { id }, - data: updateData, - }) - - // Update file owner's storage usage (not the editor's) - if (sizeDifference !== 0) { - await tx.user.update({ - where: { id: file.userId }, - data: { - storageUsed: { - increment: sizeDifference, - }, - }, - }) - } - - return updated - }) + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - logger.info('File content updated', { - fileId: id, - editorId: session.user.id, - ownerId: file.userId, - oldSizeMB: file.size, - newSizeMB, - }) + const file = await prisma.file.findUnique({ + where: { id }, + include: { + collaborators: { + where: { userId: session.user.id }, + }, + storageBucket: { select: { id: true } }, + }, + }) + + if (!file) { + return NextResponse.json({ error: 'File not found' }, { status: 404 }) + } - return NextResponse.json({ - success: true, - file: updatedFile, - }) - } catch (error) { - logger.error('File content update error', error as Error) + // Check if user is owner or has EDITOR role + const isOwner = file.userId === session.user.id + const collaborator = file.collaborators[0] + const canEdit = isOwner || collaborator?.role === 'EDITOR' + + if (!canEdit) { + // Check if they can at least suggest + const canSuggest = + collaborator?.role === 'SUGGESTER' || file.allowSuggestions + if (canSuggest) { return NextResponse.json( - { error: 'Failed to update file content' }, - { status: 500 } + { + error: + 'You can only suggest edits, not edit directly. Use the suggestions endpoint.', + }, + { status: 403 } ) + } + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + // Only allow updating text-based files + const textMimeTypes = [ + 'text/plain', + 'text/markdown', + 'text/x-markdown', + 'text/javascript', + 'text/typescript', + 'text/jsx', + 'text/tsx', + 'text/x-python', + 'text/html', + 'text/css', + 'text/xml', + 'text/yaml', + 'text/x-yaml', + 'text/x-sql', + 'text/x-java', + 'text/x-c++src', + 'text/x-csrc', + 'text/x-rustsrc', + 'text/x-go', + 'text/x-php', + 'text/x-sass', + 'text/x-scss', + 'text/x-less', + 'application/json', + 'application/javascript', + 'application/typescript', + 'application/xml', + ] + + if ( + !textMimeTypes.includes(file.mimeType) && + !file.mimeType.startsWith('text/') + ) { + return NextResponse.json( + { error: 'Only text-based files can be edited' }, + { status: 400 } + ) + } + + const body = await request.json() + const { content, allowSuggestions } = body + + if (typeof content !== 'string') { + return NextResponse.json( + { error: 'Content must be a string' }, + { status: 400 } + ) + } + + const storageProvider = await getProviderForStoredFile(file.storageBucketId) + + // Convert content to buffer and upload + const buffer = Buffer.from(content, 'utf-8') + const newSizeBytes = buffer.length + // Store size in MB (consistent with upload API) + const newSizeMB = bytesToMB(newSizeBytes) + + // Update storage with new content + await storageProvider.uploadFile(buffer, file.path, file.mimeType) + + // Calculate storage difference in MB (file.size is already in MB) + const sizeDifference = newSizeMB - file.size + + // Prepare update data + const updateData: { + size: number + updatedAt: Date + allowSuggestions?: boolean + } = { + size: newSizeMB, + updatedAt: new Date(), + } + + // Only allow owner to change allowSuggestions + if (isOwner && typeof allowSuggestions === 'boolean') { + updateData.allowSuggestions = allowSuggestions } + + // Update file record + const updatedFile = await prisma.$transaction(async (tx) => { + const updated = await tx.file.update({ + where: { id }, + data: updateData, + }) + + // Update file owner's storage usage (not the editor's) + if (sizeDifference !== 0) { + await tx.user.update({ + where: { id: file.userId }, + data: { + storageUsed: { + increment: sizeDifference, + }, + }, + }) + } + + return updated + }) + + logger.info('File content updated', { + fileId: id, + editorId: session.user.id, + ownerId: file.userId, + oldSizeMB: file.size, + newSizeMB, + }) + + return NextResponse.json({ + success: true, + file: updatedFile, + }) + } catch (error) { + logger.error('File content update error', error as Error) + return NextResponse.json( + { error: 'Failed to update file content' }, + { status: 500 } + ) + } } diff --git a/app/api/files/[id]/download/route.ts b/app/api/files/[id]/download/route.ts index 2a19b27..bc207d1 100644 --- a/app/api/files/[id]/download/route.ts +++ b/app/api/files/[id]/download/route.ts @@ -5,7 +5,7 @@ import { compare } from 'bcryptjs' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { handleCORSPreflight, getCORSHeaders } from '@/packages/lib/api/cors' const logger = loggers.files @@ -46,6 +46,7 @@ export async function GET( userId: true, visibility: true, password: true, + storageBucketId: true, }, }) @@ -76,7 +77,7 @@ export async function GET( data: { downloads: { increment: 1 } }, }) - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile(file.storageBucketId) const range = request.headers.get('range') const size = await storageProvider.getFileSize(file.path) @@ -138,7 +139,7 @@ export async function POST( try { const body = await request.json() providedPassword = body.password || null - } catch { } + } catch {} const file = await prisma.file.findUnique({ where: { id: fileId }, @@ -151,6 +152,7 @@ export async function POST( userId: true, visibility: true, password: true, + storageBucketId: true, }, }) @@ -181,7 +183,7 @@ export async function POST( data: { downloads: { increment: 1 } }, }) - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile(file.storageBucketId) const size = await storageProvider.getFileSize(file.path) diff --git a/app/api/files/[id]/route.ts b/app/api/files/[id]/route.ts index 4cbcf32..ce29de0 100644 --- a/app/api/files/[id]/route.ts +++ b/app/api/files/[id]/route.ts @@ -7,7 +7,7 @@ import { z } from 'zod' import { prisma } from '@/packages/lib/database/prisma' import { emitAuditEvent } from '@/packages/lib/events/audit-helper' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' const logger = loggers.files @@ -103,7 +103,9 @@ export async function DELETE( } try { - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + file.storageBucketId + ) await storageProvider.deleteFile(file.path) } catch (error) { logger.error('Error deleting file from storage', error as Error, { @@ -125,6 +127,13 @@ export async function DELETE( }, }, }) + + if (file.storageBucketId) { + await tx.storageBucket.update({ + where: { id: file.storageBucketId }, + data: { fileCount: { decrement: 1 } }, + }) + } }) void emitAuditEvent('file.deleted', { diff --git a/app/api/files/[id]/suggestions/route.ts b/app/api/files/[id]/suggestions/route.ts index 69bbe8b..00aa600 100644 --- a/app/api/files/[id]/suggestions/route.ts +++ b/app/api/files/[id]/suggestions/route.ts @@ -5,371 +5,372 @@ import { getServerSession } from 'next-auth' import { authOptions } from '@/packages/lib/auth' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { bytesToMB } from '@/packages/lib/utils' const logger = loggers.files // Get suggestions for a file export async function GET( - request: Request, - { params }: { params: Promise<{ id: string }> } + request: Request, + { params }: { params: Promise<{ id: string }> } ) { - try { - const { id } = await params - const session = await getServerSession(authOptions) + try { + const { id } = await params + const session = await getServerSession(authOptions) - if (!session?.user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const file = await prisma.file.findUnique({ - where: { id }, - select: { userId: true }, - }) - - if (!file) { - return NextResponse.json({ error: 'File not found' }, { status: 404 }) - } - - // Only owner can view suggestions - if (file.userId !== session.user.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - const { searchParams } = new URL(request.url) - const status = searchParams.get('status') || 'PENDING' + const file = await prisma.file.findUnique({ + where: { id }, + select: { userId: true }, + }) - const suggestions = await prisma.fileEditSuggestion.findMany({ - where: { - fileId: id, - status: status as 'PENDING' | 'APPROVED' | 'REJECTED', - }, - include: { - user: { - select: { - id: true, - name: true, - email: true, - image: true, - urlId: true, - }, - }, - }, - orderBy: { createdAt: 'desc' }, - }) + if (!file) { + return NextResponse.json({ error: 'File not found' }, { status: 404 }) + } - return NextResponse.json({ suggestions }) - } catch (error) { - logger.error('Failed to get suggestions', error as Error) - return NextResponse.json( - { error: 'Failed to get suggestions' }, - { status: 500 } - ) + // Only owner can view suggestions + if (file.userId !== session.user.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + + const { searchParams } = new URL(request.url) + const status = searchParams.get('status') || 'PENDING' + + const suggestions = await prisma.fileEditSuggestion.findMany({ + where: { + fileId: id, + status: status as 'PENDING' | 'APPROVED' | 'REJECTED', + }, + include: { + user: { + select: { + id: true, + name: true, + email: true, + image: true, + urlId: true, + }, + }, + }, + orderBy: { createdAt: 'desc' }, + }) + + return NextResponse.json({ suggestions }) + } catch (error) { + logger.error('Failed to get suggestions', error as Error) + return NextResponse.json( + { error: 'Failed to get suggestions' }, + { status: 500 } + ) + } } // Submit a suggestion export async function POST( - request: Request, - { params }: { params: Promise<{ id: string }> } + request: Request, + { params }: { params: Promise<{ id: string }> } ) { - try { - const { id } = await params - const session = await getServerSession(authOptions) + try { + const { id } = await params + const session = await getServerSession(authOptions) - if (!session?.user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } - - const file = await prisma.file.findUnique({ - where: { id }, - select: { - userId: true, - allowSuggestions: true, - mimeType: true, - collaborators: { - where: { userId: session.user.id }, - }, - }, - }) - - if (!file) { - return NextResponse.json({ error: 'File not found' }, { status: 404 }) - } - - // Check permission: owner, collaborator (SUGGESTER), or public suggestions enabled - const isOwner = file.userId === session.user.id - const collaborator = file.collaborators[0] - const canSuggest = - isOwner || - collaborator?.role === 'SUGGESTER' || - file.allowSuggestions - - if (!canSuggest) { - return NextResponse.json( - { error: 'You do not have permission to suggest edits' }, - { status: 403 } - ) - } - - // Owner shouldn't need to suggest edits to their own file - if (isOwner) { - return NextResponse.json( - { error: 'Owners should edit directly, not suggest' }, - { status: 400 } - ) - } + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - const body = await request.json() - const { content, message } = body + const file = await prisma.file.findUnique({ + where: { id }, + select: { + userId: true, + allowSuggestions: true, + mimeType: true, + collaborators: { + where: { userId: session.user.id }, + }, + }, + }) + + if (!file) { + return NextResponse.json({ error: 'File not found' }, { status: 404 }) + } - if (typeof content !== 'string' || !content.trim()) { - return NextResponse.json( - { error: 'Content is required' }, - { status: 400 } - ) - } + // Check permission: owner, collaborator (SUGGESTER), or public suggestions enabled + const isOwner = file.userId === session.user.id + const collaborator = file.collaborators[0] + const canSuggest = + isOwner || collaborator?.role === 'SUGGESTER' || file.allowSuggestions + + if (!canSuggest) { + return NextResponse.json( + { error: 'You do not have permission to suggest edits' }, + { status: 403 } + ) + } - const suggestion = await prisma.fileEditSuggestion.create({ - data: { - fileId: id, - userId: session.user.id, - content, - message: message || null, - }, - include: { - user: { - select: { - id: true, - name: true, - email: true, - image: true, - urlId: true, - }, - }, - }, - }) + // Owner shouldn't need to suggest edits to their own file + if (isOwner) { + return NextResponse.json( + { error: 'Owners should edit directly, not suggest' }, + { status: 400 } + ) + } - logger.info('Edit suggestion submitted', { - fileId: id, - suggestionId: suggestion.id, - userId: session.user.id, - }) + const body = await request.json() + const { content, message } = body - return NextResponse.json({ suggestion }) - } catch (error) { - logger.error('Failed to submit suggestion', error as Error) - return NextResponse.json( - { error: 'Failed to submit suggestion' }, - { status: 500 } - ) + if (typeof content !== 'string' || !content.trim()) { + return NextResponse.json( + { error: 'Content is required' }, + { status: 400 } + ) } + + const suggestion = await prisma.fileEditSuggestion.create({ + data: { + fileId: id, + userId: session.user.id, + content, + message: message || null, + }, + include: { + user: { + select: { + id: true, + name: true, + email: true, + image: true, + urlId: true, + }, + }, + }, + }) + + logger.info('Edit suggestion submitted', { + fileId: id, + suggestionId: suggestion.id, + userId: session.user.id, + }) + + return NextResponse.json({ suggestion }) + } catch (error) { + logger.error('Failed to submit suggestion', error as Error) + return NextResponse.json( + { error: 'Failed to submit suggestion' }, + { status: 500 } + ) + } } // Approve or reject a suggestion export async function PATCH( - request: Request, - { params }: { params: Promise<{ id: string }> } + request: Request, + { params }: { params: Promise<{ id: string }> } ) { - try { - const { id } = await params - const session = await getServerSession(authOptions) + try { + const { id } = await params + const session = await getServerSession(authOptions) - if (!session?.user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - const body = await request.json() - const { suggestionId, action } = body + const body = await request.json() + const { suggestionId, action } = body - if (!suggestionId || !['approve', 'reject'].includes(action)) { - return NextResponse.json( - { error: 'suggestionId and action (approve/reject) are required' }, - { status: 400 } - ) - } + if (!suggestionId || !['approve', 'reject'].includes(action)) { + return NextResponse.json( + { error: 'suggestionId and action (approve/reject) are required' }, + { status: 400 } + ) + } - const suggestion = await prisma.fileEditSuggestion.findUnique({ - where: { id: suggestionId }, - include: { - file: { - select: { - id: true, - userId: true, - path: true, - mimeType: true, - size: true, - }, - }, - }, - }) + const suggestion = await prisma.fileEditSuggestion.findUnique({ + where: { id: suggestionId }, + include: { + file: { + select: { + id: true, + userId: true, + path: true, + mimeType: true, + storageBucketId: true, + size: true, + }, + }, + }, + }) + + if (!suggestion) { + return NextResponse.json( + { error: 'Suggestion not found' }, + { status: 404 } + ) + } - if (!suggestion) { - return NextResponse.json( - { error: 'Suggestion not found' }, - { status: 404 } - ) - } + // Only owner can approve/reject + if (suggestion.file.userId !== session.user.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - // Only owner can approve/reject - if (suggestion.file.userId !== session.user.id) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + if (suggestion.status !== 'PENDING') { + return NextResponse.json( + { error: 'Suggestion has already been reviewed' }, + { status: 400 } + ) + } - if (suggestion.status !== 'PENDING') { - return NextResponse.json( - { error: 'Suggestion has already been reviewed' }, - { status: 400 } - ) - } + if (action === 'approve') { + // Apply the suggestion to the file + const storageProvider = await getProviderForStoredFile( + suggestion.file.storageBucketId + ) + const buffer = Buffer.from(suggestion.content, 'utf-8') + // Store size in MB (consistent with upload API) + const newSizeMB = bytesToMB(buffer.length) + + await storageProvider.uploadFile( + buffer, + suggestion.file.path, + suggestion.file.mimeType + ) + + // Both newSizeMB and suggestion.file.size are in MB + const sizeDifference = newSizeMB - suggestion.file.size + + await prisma.$transaction(async (tx) => { + // Update file size in MB + await tx.file.update({ + where: { id: suggestion.file.id }, + data: { + size: newSizeMB, + updatedAt: new Date(), + }, + }) - if (action === 'approve') { - // Apply the suggestion to the file - const storageProvider = await getStorageProvider() - const buffer = Buffer.from(suggestion.content, 'utf-8') - // Store size in MB (consistent with upload API) - const newSizeMB = bytesToMB(buffer.length) - - await storageProvider.uploadFile( - buffer, - suggestion.file.path, - suggestion.file.mimeType - ) - - // Both newSizeMB and suggestion.file.size are in MB - const sizeDifference = newSizeMB - suggestion.file.size - - await prisma.$transaction(async (tx) => { - // Update file size in MB - await tx.file.update({ - where: { id: suggestion.file.id }, - data: { - size: newSizeMB, - updatedAt: new Date(), - }, - }) - - // Update user storage - if (sizeDifference !== 0) { - await tx.user.update({ - where: { id: suggestion.file.userId }, - data: { - storageUsed: { increment: sizeDifference }, - }, - }) - } - - // Update suggestion status - await tx.fileEditSuggestion.update({ - where: { id: suggestionId }, - data: { - status: 'APPROVED', - reviewedAt: new Date(), - reviewedBy: session.user.id, - }, - }) - }) - - logger.info('Suggestion approved and applied', { - suggestionId, - fileId: suggestion.file.id, - reviewedBy: session.user.id, - }) - - return NextResponse.json({ - success: true, - status: 'APPROVED', - message: 'Suggestion approved and applied', - }) - } else { - // Reject the suggestion - await prisma.fileEditSuggestion.update({ - where: { id: suggestionId }, - data: { - status: 'REJECTED', - reviewedAt: new Date(), - reviewedBy: session.user.id, - }, - }) - - logger.info('Suggestion rejected', { - suggestionId, - fileId: suggestion.file.id, - reviewedBy: session.user.id, - }) - - return NextResponse.json({ - success: true, - status: 'REJECTED', - message: 'Suggestion rejected', - }) + // Update user storage + if (sizeDifference !== 0) { + await tx.user.update({ + where: { id: suggestion.file.userId }, + data: { + storageUsed: { increment: sizeDifference }, + }, + }) } - } catch (error) { - logger.error('Failed to review suggestion', error as Error) - return NextResponse.json( - { error: 'Failed to review suggestion' }, - { status: 500 } - ) + + // Update suggestion status + await tx.fileEditSuggestion.update({ + where: { id: suggestionId }, + data: { + status: 'APPROVED', + reviewedAt: new Date(), + reviewedBy: session.user.id, + }, + }) + }) + + logger.info('Suggestion approved and applied', { + suggestionId, + fileId: suggestion.file.id, + reviewedBy: session.user.id, + }) + + return NextResponse.json({ + success: true, + status: 'APPROVED', + message: 'Suggestion approved and applied', + }) + } else { + // Reject the suggestion + await prisma.fileEditSuggestion.update({ + where: { id: suggestionId }, + data: { + status: 'REJECTED', + reviewedAt: new Date(), + reviewedBy: session.user.id, + }, + }) + + logger.info('Suggestion rejected', { + suggestionId, + fileId: suggestion.file.id, + reviewedBy: session.user.id, + }) + + return NextResponse.json({ + success: true, + status: 'REJECTED', + message: 'Suggestion rejected', + }) } + } catch (error) { + logger.error('Failed to review suggestion', error as Error) + return NextResponse.json( + { error: 'Failed to review suggestion' }, + { status: 500 } + ) + } } // Delete a suggestion (by author or owner) export async function DELETE( - request: Request, - { params }: { params: Promise<{ id: string }> } + request: Request, + { params }: { params: Promise<{ id: string }> } ) { - try { - const { id } = await params - const session = await getServerSession(authOptions) - - if (!session?.user) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + try { + const { id } = await params + const session = await getServerSession(authOptions) - const { searchParams } = new URL(request.url) - const suggestionId = searchParams.get('suggestionId') - - if (!suggestionId) { - return NextResponse.json( - { error: 'suggestionId is required' }, - { status: 400 } - ) - } - - const suggestion = await prisma.fileEditSuggestion.findUnique({ - where: { id: suggestionId }, - include: { - file: { select: { userId: true } }, - }, - }) + if (!session?.user) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } - if (!suggestion) { - return NextResponse.json( - { error: 'Suggestion not found' }, - { status: 404 } - ) - } + const { searchParams } = new URL(request.url) + const suggestionId = searchParams.get('suggestionId') - // Only author or file owner can delete - const canDelete = - suggestion.userId === session.user.id || - suggestion.file.userId === session.user.id + if (!suggestionId) { + return NextResponse.json( + { error: 'suggestionId is required' }, + { status: 400 } + ) + } - if (!canDelete) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) - } + const suggestion = await prisma.fileEditSuggestion.findUnique({ + where: { id: suggestionId }, + include: { + file: { select: { userId: true } }, + }, + }) + + if (!suggestion) { + return NextResponse.json( + { error: 'Suggestion not found' }, + { status: 404 } + ) + } - await prisma.fileEditSuggestion.delete({ - where: { id: suggestionId }, - }) + // Only author or file owner can delete + const canDelete = + suggestion.userId === session.user.id || + suggestion.file.userId === session.user.id - return NextResponse.json({ success: true }) - } catch (error) { - logger.error('Failed to delete suggestion', error as Error) - return NextResponse.json( - { error: 'Failed to delete suggestion' }, - { status: 500 } - ) + if (!canDelete) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } + + await prisma.fileEditSuggestion.delete({ + where: { id: suggestionId }, + }) + + return NextResponse.json({ success: true }) + } catch (error) { + logger.error('Failed to delete suggestion', error as Error) + return NextResponse.json( + { error: 'Failed to delete suggestion' }, + { status: 500 } + ) + } } diff --git a/app/api/files/[id]/thumbnail/route.ts b/app/api/files/[id]/thumbnail/route.ts index dd92d1d..c3b23e5 100644 --- a/app/api/files/[id]/thumbnail/route.ts +++ b/app/api/files/[id]/thumbnail/route.ts @@ -1,10 +1,9 @@ import { NextRequest, NextResponse } from 'next/server' import { getAuthenticatedUser } from '@/packages/lib/auth/api-auth' - import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { handleCORSPreflight, getCORSHeaders } from '@/packages/lib/api/cors' const logger = loggers.files @@ -35,6 +34,7 @@ export async function GET( visibility: true, userId: true, password: true, + storageBucketId: true, }, }) @@ -71,7 +71,7 @@ export async function GET( } // Skip password check for thumbnails (thumbnails should be accessible if file is accessible) - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile(file.storageBucketId) const fileStream = await storageProvider.getFileStream(file.path) // Just serve the original image - let the browser/CSS handle sizing diff --git a/app/api/files/chunks/[uploadId]/complete/route.ts b/app/api/files/chunks/[uploadId]/complete/route.ts index 79b5286..d572b4b 100644 --- a/app/api/files/chunks/[uploadId]/complete/route.ts +++ b/app/api/files/chunks/[uploadId]/complete/route.ts @@ -13,7 +13,7 @@ import { scheduleFileExpiration } from '@/packages/lib/events/handlers/file-expi import { validateFileSecurityChecksWithVT } from '@/packages/lib/files/security-validation' import { loggers } from '@/packages/lib/logger' import { processImageOCR } from '@/packages/lib/ocr' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { bytesToMB } from '@/packages/lib/utils' import { getConfig } from '@/packages/lib/config' @@ -114,7 +114,9 @@ export async function POST( return NextResponse.json({ error: 'Invalid parts data' }, { status: 400 }) } - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + metadata.storageBucketId + ) await storageProvider.completeMultipartUpload( metadata.fileKey, metadata.s3UploadId, @@ -200,6 +202,7 @@ export async function POST( } } + const storageBucketId = metadata.storageBucketId ?? null const fileRecord = await prisma.$transaction(async (tx) => { const file = await tx.file.create({ data: { @@ -212,6 +215,7 @@ export async function POST( password: metadata.password ? await hash(metadata.password, 10) : null, + storageBucketId, user: { connect: { id: metadata.userId, @@ -229,6 +233,13 @@ export async function POST( }, }) + if (storageBucketId) { + await tx.storageBucket.update({ + where: { id: storageBucketId }, + data: { fileCount: { increment: 1 } }, + }) + } + return file }) diff --git a/app/api/files/chunks/[uploadId]/part/[partNumber]/route.ts b/app/api/files/chunks/[uploadId]/part/[partNumber]/route.ts index 883896a..d902452 100644 --- a/app/api/files/chunks/[uploadId]/part/[partNumber]/route.ts +++ b/app/api/files/chunks/[uploadId]/part/[partNumber]/route.ts @@ -5,7 +5,7 @@ import { join } from 'path' import { getAuthenticatedUser } from '@/packages/lib/auth/api-auth' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' const logger = loggers.files @@ -51,7 +51,9 @@ export async function GET( return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + metadata.storageBucketId + ) const url = await storageProvider.getPresignedPartUploadUrl( metadata.fileKey, metadata.s3UploadId, @@ -98,7 +100,9 @@ export async function PUT( const chunk = await req.arrayBuffer() - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + metadata.storageBucketId + ) const response = await storageProvider.uploadPart( metadata.fileKey, metadata.s3UploadId, diff --git a/app/api/files/chunks/route.ts b/app/api/files/chunks/route.ts index bceb8cc..637b602 100644 --- a/app/api/files/chunks/route.ts +++ b/app/api/files/chunks/route.ts @@ -3,10 +3,13 @@ import { NextResponse } from 'next/server' import { hash } from 'bcryptjs' import { existsSync } from 'fs' import { mkdir, readFile, unlink, writeFile } from 'fs/promises' -import { join } from 'path' +import { join, posix } from 'path' import { requireAuth } from '@/packages/lib/auth/api-auth' -import { uploadCache, type UploadMetadata } from '@/packages/lib/cache/upload-cache' +import { + uploadCache, + type UploadMetadata, +} from '@/packages/lib/cache/upload-cache' import { isRedisConnected } from '@/packages/lib/cache/redis' import { getConfig } from '@/packages/lib/config' import { prisma } from '@/packages/lib/database/prisma' @@ -15,7 +18,10 @@ import { validateUploadRequest } from '@/packages/lib/files/upload-validation' import { validateFileSecurityChecksWithVT } from '@/packages/lib/files/security-validation' import { loggers } from '@/packages/lib/logger' import { processImageOCR } from '@/packages/lib/ocr' -import { getStorageProvider } from '@/packages/lib/storage' +import { + getStorageProvider, + getUploadBucketForUser, +} from '@/packages/lib/storage' import { bytesToMB } from '@/packages/lib/utils' const logger = loggers.files @@ -139,10 +145,11 @@ export async function POST(req: Request) { // Check file size against plan upload cap and storage quota if (user.role !== 'ADMIN') { - const { getPlanLimits, canUploadSize } = await import('@/packages/lib/storage/quota') + const { getPlanLimits, canUploadSize } = + await import('@/packages/lib/storage/quota') const planLimits = await getPlanLimits(user.id) const fileSizeMB = bytesToMB(size) - + // Check plan upload size cap const uploadSizeCapMB = planLimits.uploadSizeCapMB ?? 0 const maxUploadBytes = uploadSizeCapMB * 1024 * 1024 @@ -156,14 +163,16 @@ export async function POST(req: Request) { { status: 413 } ) } - + // Check storage quota const uploadCheck = await canUploadSize(user.id, fileSizeMB) if (!uploadCheck.allowed) { return NextResponse.json( { error: 'Storage quota exceeded', - message: uploadCheck.reason || 'You have reached your storage quota. Purchase additional storage to continue uploading.', + message: + uploadCheck.reason || + 'You have reached your storage quota. Purchase additional storage to continue uploading.', action: 'upgrade', }, { status: 413 } @@ -217,7 +226,7 @@ export async function POST(req: Request) { } const { urlSafeName, displayName } = await getUniqueFilename( - join('uploads', user.urlId), + posix.join('uploads', user.urlId), filename, user.randomizeFileUrls ) @@ -225,8 +234,8 @@ export async function POST(req: Request) { let filePath: string let urlPath: string try { - filePath = join('uploads', user.urlId, urlSafeName) - if (!filePath.startsWith(join('uploads', user.urlId))) { + filePath = posix.join('uploads', user.urlId, urlSafeName) + if (!filePath.startsWith(posix.join('uploads', user.urlId))) { throw new Error('Invalid file path: Path traversal detected') } urlPath = `/${user.urlId}/${urlSafeName}` @@ -238,7 +247,11 @@ export async function POST(req: Request) { return NextResponse.json({ error: 'Invalid file path' }, { status: 400 }) } - const storageProvider = await getStorageProvider() + // Resolve upload destination: dedicated bucket → core bucket → global fallback + const uploadDest = await getUploadBucketForUser(user.id) + const storageProvider = uploadDest?.provider ?? (await getStorageProvider()) + const storageBucketId = uploadDest?.bucket.id ?? null + // capture host headers from the incoming request and attach as metadata const meta: Record = {} try { @@ -273,6 +286,7 @@ export async function POST(req: Request) { urlPath, s3UploadId, domain: typeof domain === 'string' && domain.length > 0 ? domain : null, + storageBucketId, } await saveUploadMetadata(localId, metadata) @@ -316,7 +330,10 @@ export async function GET(req: Request) { return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) } - const storageProvider = await getStorageProvider() + const { getProviderForStoredFile } = await import('@/packages/lib/storage') + const storageProvider = await getProviderForStoredFile( + metadata.storageBucketId + ) const presignedUrl = await storageProvider.getPresignedPartUploadUrl( metadata.fileKey, metadata.s3UploadId, @@ -362,13 +379,17 @@ export async function PUT(req: Request) { const body = await req.json() const { parts: uploadedParts } = body - const storageProvider = await getStorageProvider() + const { getProviderForStoredFile } = await import('@/packages/lib/storage') + const storageProvider = await getProviderForStoredFile( + metadata.storageBucketId + ) await storageProvider.completeMultipartUpload( metadata.fileKey, metadata.s3UploadId, uploadedParts ) + const storageBucketId = metadata.storageBucketId ?? null const fileRecord = await prisma.$transaction(async (tx) => { const file = await tx.file.create({ data: { @@ -381,11 +402,8 @@ export async function PUT(req: Request) { password: metadata.password ? await hash(metadata.password, 10) : null, - user: { - connect: { - id: metadata.userId, - }, - }, + storageBucketId, + userId: metadata.userId, }, }) @@ -398,6 +416,13 @@ export async function PUT(req: Request) { }, }) + if (storageBucketId) { + await tx.storageBucket.update({ + where: { id: storageBucketId }, + data: { fileCount: { increment: 1 } }, + }) + } + return file }) @@ -426,4 +451,3 @@ export async function PUT(req: Request) { ) } } - diff --git a/app/api/files/route.ts b/app/api/files/route.ts index 652872e..05155c1 100644 --- a/app/api/files/route.ts +++ b/app/api/files/route.ts @@ -5,7 +5,7 @@ import { } from '@/packages/types/dto/file' import { Prisma } from '@/prisma/generated/prisma/client' import { hash } from 'bcryptjs' -import { join } from 'path' +import { posix } from 'path' import { HTTP_STATUS, @@ -31,7 +31,11 @@ import { } from '@/packages/lib/files/security-validation' import { loggers } from '@/packages/lib/logger' import { processImageOCR } from '@/packages/lib/ocr' -import { getStorageProvider } from '@/packages/lib/storage' +import { + getStorageProvider, + getUploadBucketForUser, + getUploadBucketForSquad, +} from '@/packages/lib/storage' import { bytesToMB, urlForHost } from '@/packages/lib/utils' const logger = loggers.files @@ -176,20 +180,24 @@ export async function POST(req: Request) { return apiError(uploadValidation.error!, HTTP_STATUS.FORBIDDEN) } - // Buffer file + resolve provider and unique filename in parallel - const [buf, { urlSafeName, displayName }, storageProviderResolved] = - await Promise.all([ - uploadedFile.arrayBuffer().then((ab) => Buffer.from(ab)), - getUniqueFilename( - join('uploads', user.urlId), - uploadedFile.name, - user.randomizeFileUrls - ), - getStorageProvider(), - ]) - storageProvider = storageProviderResolved + // Resolve upload destination: dedicated bucket → core bucket → global fallback + const uploadDest = squadContext + ? await getUploadBucketForSquad(squadContext.squadId) + : await getUploadBucketForUser(user.id) + const storageBucketId = uploadDest?.bucket.id ?? null + + // Buffer file + resolve unique filename in parallel (provider already resolved above) + const [buf, { urlSafeName, displayName }] = await Promise.all([ + uploadedFile.arrayBuffer().then((ab) => Buffer.from(ab)), + getUniqueFilename( + posix.join('uploads', user.urlId), + uploadedFile.name, + user.randomizeFileUrls + ), + ]) + storageProvider = uploadDest?.provider ?? (await getStorageProvider()) - filePath = join('uploads', user.urlId, urlSafeName) + filePath = posix.join('uploads', user.urlId, urlSafeName) const urlPath = `/${user.urlId}/${urlSafeName}` // Fast local security checks only (extension, MIME, zip bomb) — no network calls @@ -242,6 +250,7 @@ export async function POST(req: Request) { password: passwordHash, userId: user.id, allowSuggestions, + storageBucketId, }, }) @@ -258,6 +267,14 @@ export async function POST(req: Request) { }) } + // Keep the bucket's file counter in sync for load-balancing accuracy + if (storageBucketId) { + await tx.storageBucket.update({ + where: { id: storageBucketId }, + data: { fileCount: { increment: 1 } }, + }) + } + return file }) diff --git a/instrumentation.ts b/instrumentation.ts index 0aafe73..20ed49e 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,8 +1,5 @@ export async function register() { if (process.env.NEXT_RUNTIME === 'nodejs') { - // Only initialize Sentry when a DSN is configured. - // Without this guard, @sentry/nextjs v10 sets up OpenTelemetry providers - // (even with `enabled: false`) which patches async_hooks and hangs requests. if (process.env.NEXT_PUBLIC_SENTRY_DSN) { await import('./sentry.server.config') } @@ -17,19 +14,17 @@ export async function register() { ) .catch((err) => logger.error('Startup tasks failed', err as Error)) - // Monitor memory usage in production if (process.env.NODE_ENV === 'production') { setInterval(() => { const memUsage = process.memoryUsage() if (memUsage.heapUsed > 1024 * 1024 * 1024) { - // 1GB threshold logger.warn('High memory usage detected', { heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024) + 'MB', heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024) + 'MB', external: Math.round(memUsage.external / 1024 / 1024) + 'MB', }) } - }, 60000) // Check every minute + }, 60000) } } diff --git a/package.json b/package.json index 6393f3c..1c65268 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@emberly/website", - "version": "2.4.6", + "version": "2.4.7", "license": "AGPL-3.0-only", "author": "CodeMeAPixel", "homepage": "https://github.com/EmberlyOSS/Emberly", diff --git a/packages/components/admin/settings/settings-manager.tsx b/packages/components/admin/settings/settings-manager.tsx index cf15c05..2e37a7b 100644 --- a/packages/components/admin/settings/settings-manager.tsx +++ b/packages/components/admin/settings/settings-manager.tsx @@ -79,6 +79,7 @@ import type { EmberlyConfig } from '@/lib/config' import type { BackgroundEffect, AnimationSpeed } from '@/lib/theme/theme-types' import { StorageBucketManager } from '@/packages/components/admin/settings/storage-bucket-manager' +import { UserBucketAssignment } from '@/packages/components/admin/settings/user-bucket-assignment' import { VultrInstanceManager } from '@/packages/components/admin/settings/vultr-instance-manager' import { useToast } from '@/hooks/use-toast' import { ToastAction } from '@/components/ui/toast' @@ -1762,10 +1763,19 @@ export function SettingsManager() { + + {/* User Bucket Assignment */} + + + {/* Appearance Tab */} diff --git a/packages/components/admin/settings/storage-bucket-manager.tsx b/packages/components/admin/settings/storage-bucket-manager.tsx index d465f2c..c70e160 100644 --- a/packages/components/admin/settings/storage-bucket-manager.tsx +++ b/packages/components/admin/settings/storage-bucket-manager.tsx @@ -1,4 +1,4 @@ -"use client" +'use client' import { useCallback, useEffect, useState } from 'react' @@ -7,6 +7,7 @@ import { CheckCircle2, Cloud, HardDrive, + Layers, Loader2, Pencil, Plus, @@ -45,17 +46,20 @@ interface StorageBucket { id: string name: string provider: string + isCore: boolean + priority: number + fileCount: number + provisionStatus: string s3Bucket: string s3Region: string s3AccessKeyId: string - s3SecretKey: string s3Endpoint: string | null s3ForcePathStyle: boolean vultrObjectStorageId: string | null vultrBucketName: string | null createdAt: string updatedAt: string - _count: { assignedUsers: number; assignedSquads: number } + _count: { assignedUsers: number; assignedSquads: number; files: number } } interface VultrInstance { @@ -70,7 +74,9 @@ interface VultrInstance { interface BucketForm { name: string - provider: string // 's3' | 'local' | 'vultr' (vultr is UI-only; stored as 's3' in DB) + provider: string + isCore: boolean + priority: number s3Bucket: string s3Region: string s3AccessKeyId: string @@ -84,6 +90,8 @@ interface BucketForm { const EMPTY_FORM: BucketForm = { name: '', provider: 'vultr', + isCore: false, + priority: 0, s3Bucket: '', s3Region: '', s3AccessKeyId: '', @@ -98,6 +106,22 @@ interface TestState { loading: boolean ok?: boolean message?: string + detail?: string +} + +function ProvisionStatusBadge({ status }: { status: string }) { + if (status === 'active') return null + const map: Record = { + provisioning: 'bg-yellow-500/15 text-yellow-400 border-yellow-400/30', + deprovisioning: 'bg-red-500/15 text-red-400 border-red-400/30', + } + return ( + + {status} + + ) } export function StorageBucketManager() { @@ -126,7 +150,9 @@ export function StorageBucketManager() { } }, [toast]) - useEffect(() => { refresh() }, [refresh]) + useEffect(() => { + refresh() + }, [refresh]) const fetchVultrInstances = useCallback(async () => { setVultrLoading(true) @@ -153,11 +179,12 @@ export function StorageBucketManager() { setEditingId(bucket.id) setForm({ name: bucket.name, - // If this bucket is Vultr-backed, show 'vultr' in the provider dropdown provider: bucket.vultrObjectStorageId ? 'vultr' : bucket.provider, + isCore: bucket.isCore, + priority: bucket.priority ?? 0, s3Bucket: bucket.s3Bucket, s3Region: bucket.s3Region, - s3AccessKeyId: '', // never pre-fill secrets + s3AccessKeyId: '', s3SecretKey: '', s3Endpoint: bucket.s3Endpoint ?? '', s3ForcePathStyle: bucket.s3ForcePathStyle, @@ -195,7 +222,8 @@ export function StorageBucketManager() { const handleDelete = (id: string, name: string) => { toast({ title: `Delete "${name}"?`, - description: 'Users and squads assigned to it will fall back to the default storage.', + description: + 'Users and squads assigned to it will fall back to the default storage.', variant: 'destructive', action: ( { setDeleting(id) try { - const res = await fetch(`/api/admin/storage/buckets/${id}`, { method: 'DELETE' }) + const res = await fetch(`/api/admin/storage/buckets/${id}`, { + method: 'DELETE', + }) if (!res.ok) throw new Error('Failed to delete') toast({ title: `Bucket "${name}" deleted` }) refresh() } catch { - toast({ title: 'Failed to delete bucket', variant: 'destructive' }) + toast({ + title: 'Failed to delete bucket', + variant: 'destructive', + }) } finally { setDeleting(null) } @@ -223,11 +256,24 @@ export function StorageBucketManager() { const handleTest = async (id: string) => { setTestStates((s) => ({ ...s, [id]: { loading: true } })) try { - const res = await fetch(`/api/admin/storage/buckets/${id}/test`, { method: 'POST' }) + const res = await fetch(`/api/admin/storage/buckets/${id}/test`, { + method: 'POST', + }) const data = await res.json() - setTestStates((s) => ({ ...s, [id]: { loading: false, ok: data?.data?.ok, message: data?.data?.message } })) + setTestStates((s) => ({ + ...s, + [id]: { + loading: false, + ok: data?.data?.ok, + message: data?.data?.message, + detail: data?.data?.detail, + }, + })) } catch { - setTestStates((s) => ({ ...s, [id]: { loading: false, ok: false, message: 'Request failed' } })) + setTestStates((s) => ({ + ...s, + [id]: { loading: false, ok: false, message: 'Request failed' }, + })) } } @@ -265,7 +311,9 @@ export function StorageBucketManager() { className="flex items-center gap-3 rounded-xl border border-border/50 bg-muted/20 px-4 py-3" >

-
-

{bucket.name}

- - {bucket.vultrObjectStorageId ? 'Vultr' : bucket.provider === 's3' ? 'S3' : 'Local'} +
+

+ {bucket.name} +

+ {bucket.isCore ? ( + + Core Pool + + ) : ( + + Dedicated + + )} + + {bucket.vultrObjectStorageId + ? 'Vultr' + : bucket.provider === 's3' + ? 'S3' + : 'Local'} + {bucket.s3Bucket && ( - {bucket.s3Bucket}{bucket.s3Region ? ` · ${bucket.s3Region}` : ''} + {bucket.s3Bucket} + {bucket.s3Region ? ` · ${bucket.s3Region}` : ''} )}
-
+
- {bucket._count.assignedUsers} user{bucket._count.assignedUsers !== 1 ? 's' : ''} - {bucket._count.assignedSquads > 0 && `, ${bucket._count.assignedSquads} squad${bucket._count.assignedSquads !== 1 ? 's' : ''}`} + {bucket._count.assignedUsers} user + {bucket._count.assignedUsers !== 1 ? 's' : ''} + {bucket._count.assignedSquads > 0 && + `, ${bucket._count.assignedSquads} squad${bucket._count.assignedSquads !== 1 ? 's' : ''}`} + {bucket.isCore && ( + <> + + {bucket.fileCount.toLocaleString()} file + {bucket.fileCount !== 1 ? 's' : ''} + + {bucket.priority !== 0 && ( + + priority {bucket.priority} + + )} + + )} {test && !test.loading && ( - - {test.ok ? : } + + {test.ok ? ( + + ) : ( + + )} {test.message} + {test.detail ? ` — ${test.detail}` : ''} )}
@@ -348,9 +442,12 @@ export function StorageBucketManager() { - {editingId ? 'Edit Storage Bucket' : 'Add Storage Bucket'} + + {editingId ? 'Edit Storage Bucket' : 'Add Storage Bucket'} + - Named S3 buckets can be assigned to specific users or squads as their dedicated storage. + Named S3 buckets can be assigned to specific users or squads as + their dedicated storage. @@ -360,30 +457,79 @@ export function StorageBucketManager() { setForm((f) => ({ ...f, name: e.target.value }))} + onChange={(e) => + setForm((f) => ({ ...f, name: e.target.value })) + } />
+ {/* Core pool toggle */} +
+
+

Core Pool Bucket

+

+ Core buckets are shared across all users via load balancing. + Dedicated buckets are assigned to specific users or squads. +

+
+ setForm((f) => ({ ...f, isCore: v }))} + /> +
+ + {/* Priority — only relevant for core buckets */} + {form.isCore && ( +
+ + + setForm((f) => ({ + ...f, + priority: parseInt(e.target.value) || 0, + })) + } + /> +

+ Higher number = more preferred. When two core buckets have the + same file count, the one with the higher priority wins. + Default is 0 (no preference). Example: set a low-latency + bucket to 10 and leave others at 0. +

+
+ )} +
- setForm((f) => ({ ...f, provider: v }))} + >
- Vultr Object Storage + + Vultr Object Storage
- S3 (manual) + + S3 (manual)
- Local + + Local
@@ -393,51 +539,78 @@ export function StorageBucketManager() { {form.provider === 'vultr' && (

- Vultr Instance + + Vultr Instance

{editingId && ( -

Instance cannot be changed after creation.

+

+ Instance cannot be changed after creation. +

)}
- + setForm((f) => ({ ...f, vultrBucketName: e.target.value }))} + onChange={(e) => + setForm((f) => ({ + ...f, + vultrBucketName: e.target.value, + })) + } disabled={!!editingId} /> {editingId ? ( -

Bucket name cannot be changed after creation.

+

+ Bucket name cannot be changed after creation. +

) : ( -

Must match an existing bucket in the selected Vultr instance. Credentials are filled automatically.

+

+ Must match an existing bucket in the selected Vultr + instance. Credentials are filled automatically. +

)}
@@ -446,7 +619,8 @@ export function StorageBucketManager() { {form.provider === 's3' && (

- S3 Configuration + + S3 Configuration

@@ -456,7 +630,9 @@ export function StorageBucketManager() { placeholder="my-bucket" className="h-8 text-sm" value={form.s3Bucket} - onChange={(e) => setForm((f) => ({ ...f, s3Bucket: e.target.value }))} + onChange={(e) => + setForm((f) => ({ ...f, s3Bucket: e.target.value })) + } />
@@ -465,7 +641,9 @@ export function StorageBucketManager() { placeholder="us-east-1" className="h-8 text-sm" value={form.s3Region} - onChange={(e) => setForm((f) => ({ ...f, s3Region: e.target.value }))} + onChange={(e) => + setForm((f) => ({ ...f, s3Region: e.target.value })) + } />
@@ -475,7 +653,12 @@ export function StorageBucketManager() { placeholder={editingId ? '(unchanged)' : 'AKIA...'} className="h-8 text-sm font-mono" value={form.s3AccessKeyId} - onChange={(e) => setForm((f) => ({ ...f, s3AccessKeyId: e.target.value }))} + onChange={(e) => + setForm((f) => ({ + ...f, + s3AccessKeyId: e.target.value, + })) + } />
@@ -485,7 +668,9 @@ export function StorageBucketManager() { placeholder={editingId ? '(unchanged)' : '••••••••'} className="h-8 text-sm font-mono" value={form.s3SecretKey} - onChange={(e) => setForm((f) => ({ ...f, s3SecretKey: e.target.value }))} + onChange={(e) => + setForm((f) => ({ ...f, s3SecretKey: e.target.value })) + } />
@@ -496,18 +681,24 @@ export function StorageBucketManager() { placeholder="https://s3.custom-domain.com" className="h-8 text-sm" value={form.s3Endpoint} - onChange={(e) => setForm((f) => ({ ...f, s3Endpoint: e.target.value }))} + onChange={(e) => + setForm((f) => ({ ...f, s3Endpoint: e.target.value })) + } />

Force Path Style

-

Required for MinIO, DigitalOcean Spaces, etc.

+

+ Required for MinIO, DigitalOcean Spaces, etc. +

setForm((f) => ({ ...f, s3ForcePathStyle: v }))} + onCheckedChange={(v) => + setForm((f) => ({ ...f, s3ForcePathStyle: v })) + } />
@@ -515,7 +706,11 @@ export function StorageBucketManager() {
- + )} +
+ + {/* User list */} +
+ {loading ? ( + Array.from({ length: 4 }).map((_, i) => ( +
+ +
+ + +
+ +
+ )) + ) : filtered.length === 0 ? ( +
+ No users found +
+ ) : ( + filtered.map((user) => { + const currentBucketId = user.storageBucketId ?? '' + const selectedBucket = pendingBucket[user.id] ?? currentBucketId + const isDirty = pendingBucket[user.id] !== undefined + + return ( +
+ {/* Avatar */} +
+ {user.image ? ( + + ) : ( + + )} +
+ + {/* User info */} +
+

+ {user.name ?? '(no name)'} +

+

+ {user.email} +

+
+ + {/* Current assignment badge */} +
+ {user.storageBucket ? ( + + {user.storageBucket.isCore ? ( + + ) : ( + + )} + {user.storageBucket.name} + + ) : ( + + default + + )} +
+ + {/* Bucket select + assign button */} +
+ + + +
+
+ ) + }) + )} +
+ + {/* Pagination */} + {!loading && totalPages > 1 && !search && ( +
+ + Page {page} of {totalPages} + +
+ + +
+
+ )} +
+ ) +} diff --git a/packages/components/dashboard/bucket-secret-reveal.tsx b/packages/components/dashboard/bucket-secret-reveal.tsx new file mode 100644 index 0000000..0c82199 --- /dev/null +++ b/packages/components/dashboard/bucket-secret-reveal.tsx @@ -0,0 +1,32 @@ +'use client' + +import { useState } from 'react' +import { Eye, EyeOff } from 'lucide-react' + +export function BucketSecretReveal({ secret }: { secret: string }) { + const [visible, setVisible] = useState(false) + + return ( +
+ + Secret Access Key + +
+ + {visible ? secret : '••••••••••••••••••••••••••••••••'} + + +
+
+ ) +} diff --git a/packages/lib/cache/upload-cache.ts b/packages/lib/cache/upload-cache.ts index c1ddb61..de2374b 100644 --- a/packages/lib/cache/upload-cache.ts +++ b/packages/lib/cache/upload-cache.ts @@ -5,17 +5,19 @@ import { getRedisClient, isRedisConnected, redisKeys } from './redis' const logger = loggers.events.getChildLogger('upload-cache') export interface UploadMetadata { - fileKey: string - filename: string - mimeType: string - totalSize: number - userId: string - visibility: 'PUBLIC' | 'PRIVATE' - password: string | null - lastActivity: number - urlPath: string - s3UploadId: string - domain?: string | null + fileKey: string + filename: string + mimeType: string + totalSize: number + userId: string + visibility: 'PUBLIC' | 'PRIVATE' + password: string | null + lastActivity: number + urlPath: string + s3UploadId: string + domain?: string | null + /** ID of the StorageBucket record the multipart upload was initiated against. Null = global config bucket. */ + storageBucketId?: string | null } // TTL for upload metadata (1 hour) @@ -25,144 +27,155 @@ const UPLOAD_TTL_SECONDS = 60 * 60 * Redis-based upload metadata cache for chunked uploads */ export const uploadCache = { - /** - * Save upload metadata - */ - async save(uploadId: string, metadata: UploadMetadata): Promise { - if (!isRedisConnected()) { - logger.warn('Redis not connected, cannot save upload metadata') - return false + /** + * Save upload metadata + */ + async save(uploadId: string, metadata: UploadMetadata): Promise { + if (!isRedisConnected()) { + logger.warn('Redis not connected, cannot save upload metadata') + return false + } + + try { + const redis = await getRedisClient() + const key = redisKeys.uploadMetadata(uploadId) + const data = JSON.stringify(metadata) + + // Set with TTL + await redis.setEx(key, UPLOAD_TTL_SECONDS, data) + + // Add to active uploads set for cleanup tracking + await redis.sAdd(redisKeys.uploadList(), uploadId) + + logger.debug('Upload metadata saved', { uploadId }) + return true + } catch (error) { + logger.error('Failed to save upload metadata', error as Error, { + uploadId, + }) + return false + } + }, + + /** + * Get upload metadata + */ + async get(uploadId: string): Promise { + if (!isRedisConnected()) { + return null + } + + try { + const redis = await getRedisClient() + const key = redisKeys.uploadMetadata(uploadId) + const data = await redis.get(key) + + if (!data) return null + return JSON.parse(data) as UploadMetadata + } catch (error) { + logger.error('Failed to get upload metadata', error as Error, { + uploadId, + }) + return null + } + }, + + /** + * Update upload metadata (refreshes TTL) + */ + async update( + uploadId: string, + updates: Partial + ): Promise { + if (!isRedisConnected()) { + return false + } + + try { + const existing = await this.get(uploadId) + if (!existing) return false + + const updated = { ...existing, ...updates, lastActivity: Date.now() } + return await this.save(uploadId, updated) + } catch (error) { + logger.error('Failed to update upload metadata', error as Error, { + uploadId, + }) + return false + } + }, + + /** + * Delete upload metadata + */ + async delete(uploadId: string): Promise { + if (!isRedisConnected()) { + return false + } + + try { + const redis = await getRedisClient() + const key = redisKeys.uploadMetadata(uploadId) + + await redis.del(key) + await redis.sRem(redisKeys.uploadList(), uploadId) + + logger.debug('Upload metadata deleted', { uploadId }) + return true + } catch (error) { + logger.error('Failed to delete upload metadata', error as Error, { + uploadId, + }) + return false + } + }, + + /** + * Clean up stale uploads (older than TTL) + */ + async cleanup(): Promise { + if (!isRedisConnected()) { + return 0 + } + + try { + const redis = await getRedisClient() + const uploadIds = await redis.sMembers(redisKeys.uploadList()) + let cleaned = 0 + + for (const uploadId of uploadIds) { + const exists = await redis.exists(redisKeys.uploadMetadata(uploadId)) + if (!exists) { + // Key expired, remove from set + await redis.sRem(redisKeys.uploadList(), uploadId) + cleaned++ } - - try { - const redis = await getRedisClient() - const key = redisKeys.uploadMetadata(uploadId) - const data = JSON.stringify(metadata) - - // Set with TTL - await redis.setEx(key, UPLOAD_TTL_SECONDS, data) - - // Add to active uploads set for cleanup tracking - await redis.sAdd(redisKeys.uploadList(), uploadId) - - logger.debug('Upload metadata saved', { uploadId }) - return true - } catch (error) { - logger.error('Failed to save upload metadata', error as Error, { uploadId }) - return false - } - }, - - /** - * Get upload metadata - */ - async get(uploadId: string): Promise { - if (!isRedisConnected()) { - return null - } - - try { - const redis = await getRedisClient() - const key = redisKeys.uploadMetadata(uploadId) - const data = await redis.get(key) - - if (!data) return null - return JSON.parse(data) as UploadMetadata - } catch (error) { - logger.error('Failed to get upload metadata', error as Error, { uploadId }) - return null - } - }, - - /** - * Update upload metadata (refreshes TTL) - */ - async update(uploadId: string, updates: Partial): Promise { - if (!isRedisConnected()) { - return false - } - - try { - const existing = await this.get(uploadId) - if (!existing) return false - - const updated = { ...existing, ...updates, lastActivity: Date.now() } - return await this.save(uploadId, updated) - } catch (error) { - logger.error('Failed to update upload metadata', error as Error, { uploadId }) - return false - } - }, - - /** - * Delete upload metadata - */ - async delete(uploadId: string): Promise { - if (!isRedisConnected()) { - return false - } - - try { - const redis = await getRedisClient() - const key = redisKeys.uploadMetadata(uploadId) - - await redis.del(key) - await redis.sRem(redisKeys.uploadList(), uploadId) - - logger.debug('Upload metadata deleted', { uploadId }) - return true - } catch (error) { - logger.error('Failed to delete upload metadata', error as Error, { uploadId }) - return false - } - }, - - /** - * Clean up stale uploads (older than TTL) - */ - async cleanup(): Promise { - if (!isRedisConnected()) { - return 0 - } - - try { - const redis = await getRedisClient() - const uploadIds = await redis.sMembers(redisKeys.uploadList()) - let cleaned = 0 - - for (const uploadId of uploadIds) { - const exists = await redis.exists(redisKeys.uploadMetadata(uploadId)) - if (!exists) { - // Key expired, remove from set - await redis.sRem(redisKeys.uploadList(), uploadId) - cleaned++ - } - } - - if (cleaned > 0) { - logger.info('Cleaned up stale upload entries', { cleaned }) - } - return cleaned - } catch (error) { - logger.error('Failed to cleanup uploads', error as Error) - return 0 - } - }, - - /** - * Get all active upload IDs - */ - async getActiveUploads(): Promise { - if (!isRedisConnected()) { - return [] - } - - try { - const redis = await getRedisClient() - return await redis.sMembers(redisKeys.uploadList()) - } catch (error) { - logger.error('Failed to get active uploads', error as Error) - return [] - } - }, + } + + if (cleaned > 0) { + logger.info('Cleaned up stale upload entries', { cleaned }) + } + return cleaned + } catch (error) { + logger.error('Failed to cleanup uploads', error as Error) + return 0 + } + }, + + /** + * Get all active upload IDs + */ + async getActiveUploads(): Promise { + if (!isRedisConnected()) { + return [] + } + + try { + const redis = await getRedisClient() + return await redis.sMembers(redisKeys.uploadList()) + } catch (error) { + logger.error('Failed to get active uploads', error as Error) + return [] + } + }, } diff --git a/packages/lib/events/handlers/file-expiry.ts b/packages/lib/events/handlers/file-expiry.ts index dea794c..c931261 100644 --- a/packages/lib/events/handlers/file-expiry.ts +++ b/packages/lib/events/handlers/file-expiry.ts @@ -3,7 +3,7 @@ import { EventStatus, ExpiryAction } from '@/packages/types/events' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { events } from '../index' @@ -59,7 +59,9 @@ export function registerFileExpiryHandlers() { } if (payload.action === ExpiryAction.DELETE) { - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile( + file.storageBucketId + ) await storageProvider.deleteFile(file.path) logger.info('Deleted file from storage', { path: file.path }) diff --git a/packages/lib/files/service.ts b/packages/lib/files/service.ts index f3e87b0..30b0cc5 100644 --- a/packages/lib/files/service.ts +++ b/packages/lib/files/service.ts @@ -14,7 +14,7 @@ */ import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import { File, Prisma, User } from '@/prisma/generated/prisma/client' import { compare, hash } from 'bcryptjs' @@ -167,7 +167,7 @@ export async function deleteFileWithCleanup( try { // Delete from storage provider - const storageProvider = await getStorageProvider() + const storageProvider = await getProviderForStoredFile(file.storageBucketId) await storageProvider.deleteFile(file.path) // Delete from database @@ -215,30 +215,11 @@ export async function deleteFilesWithCleanup( where: { id: { in: fileIds }, userId }, }) - let storageProvider: Awaited> - try { - storageProvider = await getStorageProvider() - } catch (error) { - const message = - error instanceof Error - ? error.message - : 'Failed to initialize storage provider' - logger.error( - 'Failed to initialize storage provider for bulk delete', - error as Error, - { userId } - ) - return { - success: false, - deletedCount: 0, - failedCount: files.length, - totalStorageBytes: 0, - error: message, - } - } - for (const file of files) { try { + const storageProvider = await getProviderForStoredFile( + file.storageBucketId + ) await storageProvider.deleteFile(file.path) await prisma.file.delete({ where: { id: file.id } }) deletedCount++ diff --git a/packages/lib/logger/index.ts b/packages/lib/logger/index.ts index 534114d..0e88d6b 100644 --- a/packages/lib/logger/index.ts +++ b/packages/lib/logger/index.ts @@ -26,7 +26,7 @@ interface ErrorLogContext { // Guard access to Node globals so this module can load in Edge runtime. const runtimeProcess = typeof globalThis !== 'undefined' && - typeof (globalThis as { process?: NodeJS.Process }).process !== 'undefined' + typeof (globalThis as { process?: NodeJS.Process }).process !== 'undefined' ? (globalThis as { process?: NodeJS.Process }).process : undefined @@ -59,38 +59,45 @@ const formatters = { // Fallback to console transport due to worker thread issues const transport = isDevelopment ? { - write: (msg: string) => { - try { - const obj = JSON.parse(msg) - const timestamp = new Date(obj.time) - .toISOString() - .replace('T', ' ') - .slice(0, -5) - // Handle both numeric levels (pino default) and string levels (from formatter) - const rawLevel = obj.level - const level = ( - rawLevel === 30 || rawLevel === 'INFO' - ? 'INFO' - : rawLevel === 40 || rawLevel === 'WARN' - ? 'WARN' - : rawLevel === 50 || rawLevel === 'ERROR' - ? 'ERROR' - : rawLevel === 60 || rawLevel === 'FATAL' - ? 'FATAL' - : rawLevel === 20 || rawLevel === 'DEBUG' - ? 'DEBUG' - : rawLevel === 10 || rawLevel === 'TRACE' - ? 'TRACE' - : String(rawLevel).toUpperCase() - ).padEnd(5) - const name = obj.name ? `[${obj.name}]`.padEnd(12) : '' - const message = obj.msg || '' - console.log(`${timestamp} ${level} ${name} ${message}`) - } catch { - console.log(msg) - } - }, - } + write: (msg: string) => { + try { + const obj = JSON.parse(msg) + const timestamp = new Date(obj.time) + .toISOString() + .replace('T', ' ') + .slice(0, -5) + // Handle both numeric levels (pino default) and string levels (from formatter) + const rawLevel = obj.level + const level = ( + rawLevel === 30 || rawLevel === 'INFO' + ? 'INFO' + : rawLevel === 40 || rawLevel === 'WARN' + ? 'WARN' + : rawLevel === 50 || rawLevel === 'ERROR' + ? 'ERROR' + : rawLevel === 60 || rawLevel === 'FATAL' + ? 'FATAL' + : rawLevel === 20 || rawLevel === 'DEBUG' + ? 'DEBUG' + : rawLevel === 10 || rawLevel === 'TRACE' + ? 'TRACE' + : String(rawLevel).toUpperCase() + ).padEnd(5) + const name = obj.name ? `[${obj.name}]`.padEnd(12) : '' + const message = obj.msg || '' + const err = obj.error + const errDetail = err + ? ` — ${err.message || JSON.stringify(err)}` + : '' + console.log(`${timestamp} ${level} ${name} ${message}${errDetail}`) + if (err?.stack) { + console.log(err.stack) + } + } catch { + console.log(msg) + } + }, + } : undefined const baseLogger = pino( @@ -140,7 +147,7 @@ const baseLogger = pino( }, ...(isProduction && { browser: { - write: () => { }, + write: () => {}, }, }), }, diff --git a/packages/lib/ocr/processor.ts b/packages/lib/ocr/processor.ts index 5e997eb..8ac35fd 100644 --- a/packages/lib/ocr/processor.ts +++ b/packages/lib/ocr/processor.ts @@ -2,7 +2,7 @@ import { createWorker } from 'tesseract.js' import { prisma } from '@/packages/lib/database/prisma' import { loggers } from '@/packages/lib/logger' -import { getStorageProvider } from '@/packages/lib/storage' +import { getProviderForStoredFile } from '@/packages/lib/storage' import type { OCRTask } from './queue' @@ -10,11 +10,20 @@ const logger = loggers.ocr export async function processImageOCRTask({ filePath, fileId }: OCRTask) { try { - const storageProvider = await getStorageProvider() + const fileRecord = await prisma.file.findUnique({ + where: { id: fileId }, + select: { storageBucketId: true }, + }) + const storageProvider = await getProviderForStoredFile( + fileRecord?.storageBucketId + ) const stream = await storageProvider.getFileStream(filePath) // Stream file into memory but enforce a hard size cap to avoid OOM - const MAX_OCR_FILE_SIZE = parseInt(process.env.MAX_OCR_FILE_SIZE || String(20 * 1024 * 1024), 10) // 20MB default + const MAX_OCR_FILE_SIZE = parseInt( + process.env.MAX_OCR_FILE_SIZE || String(20 * 1024 * 1024), + 10 + ) // 20MB default const chunks: Buffer[] = [] let totalLength = 0 @@ -22,7 +31,11 @@ export async function processImageOCRTask({ filePath, fileId }: OCRTask) { const buf = Buffer.from(chunk) totalLength += buf.length if (totalLength > MAX_OCR_FILE_SIZE) { - logger.warn('OCR file too large, skipping OCR', { filePath, fileId, size: totalLength }) + logger.warn('OCR file too large, skipping OCR', { + filePath, + fileId, + size: totalLength, + }) // Mark as processed without OCR to avoid requeueing await prisma.file.update({ where: { id: fileId }, diff --git a/packages/lib/storage/index.ts b/packages/lib/storage/index.ts index 4f02fbd..529b301 100644 --- a/packages/lib/storage/index.ts +++ b/packages/lib/storage/index.ts @@ -1,6 +1,7 @@ import { loggers } from '@/packages/lib/logger' import { prisma } from '@/packages/lib/database/prisma' +import type { StorageBucket } from '@/prisma/generated/prisma/client' import { getConfig } from '../config/index' import { LocalStorageProvider } from './providers/local' import { S3StorageProvider } from './providers/s3' @@ -11,13 +12,19 @@ const logger = loggers.storage export type { StorageProvider, RangeOptions } from './types' export { LocalStorageProvider, S3StorageProvider } +/** { bucket record, ready-to-use provider } returned by upload-destination helpers. */ +export type UploadDestination = { + bucket: StorageBucket + provider: StorageProvider +} + // Stored on globalThis so the singleton survives Next.js hot-reloads in dev mode const _g = globalThis as typeof globalThis & { __storageProvider?: StorageProvider } let storageProvider: StorageProvider | null = _g.__storageProvider ?? null -/** Build a provider from a StorageBucket DB record (no caching — used for per-user routing). */ +/** Build a provider from a StorageBucket DB record (no caching — used for per-bucket routing). */ function providerFromBucket(bucket: { provider: string s3Bucket: string @@ -40,6 +47,7 @@ function providerFromBucket(bucket: { return new LocalStorageProvider() } +/** The global config-driven provider (singleton). Used as the fallback for legacy files (storageBucketId = null). */ export async function getStorageProvider(): Promise { if (storageProvider) return storageProvider @@ -78,14 +86,46 @@ export async function getStorageProvider(): Promise { return storageProvider } +export function invalidateStorageProvider(): void { + storageProvider = null + _g.__storageProvider = undefined +} + +// --------------------------------------------------------------------------- +// Core bucket selection — least-filled load balancing across internal buckets +// --------------------------------------------------------------------------- + /** - * Get a storage provider scoped to a specific user. - * If the user has an assigned `storageBucketId`, returns a provider for that bucket. - * Otherwise falls back to the global default provider. + * Pick the active core bucket with the lowest fileCount. + * Returns null when no active core buckets are configured; callers fall back + * to the global config provider in that case. */ -export async function getStorageProviderForUser( +async function selectCoreBucket(): Promise { + const bucket = await prisma.storageBucket.findFirst({ + where: { isCore: true, provisionStatus: 'active' }, + orderBy: [{ fileCount: 'asc' }, { priority: 'desc' }], + }) + + if (!bucket) return null + + return { bucket, provider: providerFromBucket(bucket) } +} + +// --------------------------------------------------------------------------- +// Upload-destination helpers — call these from upload routes +// --------------------------------------------------------------------------- + +/** + * Resolve where a new file for this user should be stored. + * + * Priority: + * 1. User's dedicated bucket (storageBucketId on the User record) + * 2. Least-filled active core bucket + * 3. null → caller falls back to getStorageProvider() + */ +export async function getUploadBucketForUser( userId: string -): Promise { +): Promise { const user = await prisma.user.findUnique({ where: { id: userId }, select: { storageBucketId: true }, @@ -93,29 +133,32 @@ export async function getStorageProviderForUser( if (user?.storageBucketId) { const bucket = await prisma.storageBucket.findUnique({ - where: { id: user.storageBucketId }, + where: { id: user.storageBucketId, provisionStatus: 'active' }, }) if (bucket) { - logger.info('Using custom storage bucket for user', { + logger.info('Routing upload to user dedicated bucket', { userId, bucketId: bucket.id, bucketName: bucket.name, }) - return providerFromBucket(bucket) + return { bucket, provider: providerFromBucket(bucket) } } } - return getStorageProvider() + return selectCoreBucket() } /** - * Get a storage provider scoped to a specific squad. - * If the squad has an assigned `storageBucketId`, returns a provider for that bucket. - * Otherwise falls back to the global default provider. + * Resolve where a new file for this squad should be stored. + * + * Priority: + * 1. Squad's dedicated bucket + * 2. Least-filled active core bucket + * 3. null → caller falls back to getStorageProvider() */ -export async function getStorageProviderForSquad( +export async function getUploadBucketForSquad( squadId: string -): Promise { +): Promise { const squad = await prisma.nexiumSquad.findUnique({ where: { id: squadId }, select: { storageBucketId: true }, @@ -123,34 +166,91 @@ export async function getStorageProviderForSquad( if (squad?.storageBucketId) { const bucket = await prisma.storageBucket.findUnique({ - where: { id: squad.storageBucketId }, + where: { id: squad.storageBucketId, provisionStatus: 'active' }, }) if (bucket) { - logger.info('Using custom storage bucket for squad', { + logger.info('Routing upload to squad dedicated bucket', { squadId, bucketId: bucket.id, bucketName: bucket.name, }) - return providerFromBucket(bucket) + return { bucket, provider: providerFromBucket(bucket) } } } - return getStorageProvider() + return selectCoreBucket() } -export function invalidateStorageProvider(): void { - storageProvider = null - _g.__storageProvider = undefined +// --------------------------------------------------------------------------- +// Serve-time helper — call this when reading/streaming a stored file +// --------------------------------------------------------------------------- + +/** + * Return the storage provider for an already-stored file. + * + * Pass `file.storageBucketId` from the DB record. A null value means the file + * was stored before bucket routing was introduced and lives in the global + * config bucket — getStorageProvider() is used as the fallback. + */ +export async function getProviderForStoredFile( + storageBucketId: string | null | undefined +): Promise { + if (!storageBucketId) return getStorageProvider() + + const bucket = await prisma.storageBucket.findUnique({ + where: { id: storageBucketId }, + }) + + if (!bucket) { + logger.warn( + 'storageBucketId on file does not match any bucket record — falling back to global provider', + { + storageBucketId, + } + ) + return getStorageProvider() + } + + return providerFromBucket(bucket) } // --------------------------------------------------------------------------- -// Convenience URL helpers — avoids repeating getStorageProvider() + getFileUrl() -// in every route that only needs a URL. +// Convenience URL helpers — bucket-aware // --------------------------------------------------------------------------- /** - * Build a public (or presigned) URL for a stored file. - * Delegates to the active provider's `getFileUrl` implementation. + * Build a public (or presigned) URL for a stored file using its bucket. + * Pass `storageBucketId` from the File record so the correct provider is used. + */ +export async function getFileUrlForFile( + storageBucketId: string | null | undefined, + path: string, + expiresIn?: number, + hostOverride?: string +): Promise { + const provider = await getProviderForStoredFile(storageBucketId) + return provider.getFileUrl(path, expiresIn, hostOverride) +} + +/** + * Build a download URL for a stored file using its bucket. + */ +export async function getDownloadUrlForFile( + storageBucketId: string | null | undefined, + path: string, + filename?: string, + hostOverride?: string +): Promise { + const provider = await getProviderForStoredFile(storageBucketId) + if (provider.getDownloadUrl) { + return provider.getDownloadUrl(path, filename, hostOverride) + } + return provider.getFileUrl(path, undefined, hostOverride) +} + +/** + * @deprecated Use getFileUrlForFile() so the correct bucket provider is used. + * Kept for call sites that only deal with the global config bucket. */ export async function getFileUrl( path: string, @@ -162,8 +262,8 @@ export async function getFileUrl( } /** - * Build a download URL (with `Content-Disposition: attachment`) for a stored file. - * Falls back to `getFileUrl` when the provider doesn't implement `getDownloadUrl`. + * @deprecated Use getDownloadUrlForFile() so the correct bucket provider is used. + * Kept for call sites that only deal with the global config bucket. */ export async function getDownloadUrl( path: string, @@ -176,3 +276,31 @@ export async function getDownloadUrl( } return provider.getFileUrl(path, undefined, hostOverride) } + +// --------------------------------------------------------------------------- +// Legacy per-entity helpers (kept for any future direct use) +// --------------------------------------------------------------------------- + +/** + * Get a storage provider scoped to a specific user's dedicated bucket. + * Falls back to the global default provider if no dedicated bucket is assigned. + * Prefer getUploadBucketForUser() for upload routing (includes core bucket selection). + */ +export async function getStorageProviderForUser( + userId: string +): Promise { + const dest = await getUploadBucketForUser(userId) + return dest?.provider ?? getStorageProvider() +} + +/** + * Get a storage provider scoped to a specific squad's dedicated bucket. + * Falls back to the global default provider if no dedicated bucket is assigned. + * Prefer getUploadBucketForSquad() for upload routing (includes core bucket selection). + */ +export async function getStorageProviderForSquad( + squadId: string +): Promise { + const dest = await getUploadBucketForSquad(squadId) + return dest?.provider ?? getStorageProvider() +} diff --git a/packages/lib/storage/providers/s3.ts b/packages/lib/storage/providers/s3.ts index 0765fd9..53f4a4b 100644 --- a/packages/lib/storage/providers/s3.ts +++ b/packages/lib/storage/providers/s3.ts @@ -20,6 +20,13 @@ import type { RangeOptions, S3Config, StorageProvider } from '../types' const logger = loggers.storage.getChildLogger('s3') +function toS3Key(path: string): string { + return path + .replace(/\\/g, '/') + .replace(/^\/+/, '') + .replace(/^uploads\//, '') +} + export class S3StorageProvider implements StorageProvider { private client: S3Client private bucket: string @@ -59,7 +66,7 @@ export class S3StorageProvider implements StorageProvider { path: string, mimeType: string ): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) await this.client.send( new PutObjectCommand({ @@ -73,7 +80,7 @@ export class S3StorageProvider implements StorageProvider { } async deleteFile(path: string): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) await this.client.send( new DeleteObjectCommand({ @@ -84,7 +91,7 @@ export class S3StorageProvider implements StorageProvider { } async getFile(path: string): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const response = await this.client.send( new GetObjectCommand({ @@ -106,7 +113,7 @@ export class S3StorageProvider implements StorageProvider { } async getFileStream(path: string, range?: RangeOptions): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const options: { Range?: string } = {} if (range) { @@ -160,7 +167,7 @@ export class S3StorageProvider implements StorageProvider { } async getFileUrl(path: string, expiresIn: number = 3600): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) if (key.startsWith('avatars/')) { if (this.endpoint) { @@ -182,7 +189,7 @@ export class S3StorageProvider implements StorageProvider { } async getDownloadUrl(path: string, filename?: string): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const command = new GetObjectCommand({ Bucket: this.bucket, @@ -196,7 +203,7 @@ export class S3StorageProvider implements StorageProvider { } async getFileSize(path: string): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const command = new HeadObjectCommand({ Bucket: this.bucket, @@ -288,7 +295,7 @@ export class S3StorageProvider implements StorageProvider { path: string, mimeType: string ): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const { PassThrough } = await import('stream') const passThrough = new PassThrough() @@ -539,7 +546,7 @@ export class S3StorageProvider implements StorageProvider { path: string, mimeType: string ): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const response = await this.client.send( new CreateMultipartUploadCommand({ @@ -561,7 +568,7 @@ export class S3StorageProvider implements StorageProvider { uploadId: string, partNumber: number ): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const command = new UploadPartCommand({ Bucket: this.bucket, @@ -578,7 +585,7 @@ export class S3StorageProvider implements StorageProvider { uploadId: string, parts: { ETag: string; PartNumber: number }[] ): Promise { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) await this.client.send( new CompleteMultipartUploadCommand({ @@ -598,7 +605,7 @@ export class S3StorageProvider implements StorageProvider { partNumber: number, data: Buffer ): Promise<{ ETag: string }> { - const key = path.replace(/^\/+/, '').replace(/^uploads\//, '') + const key = toS3Key(path) const response = await this.client.send( new UploadPartCommand({ @@ -616,4 +623,4 @@ export class S3StorageProvider implements StorageProvider { return { ETag: response.ETag } } -} \ No newline at end of file +} diff --git a/packages/lib/storage/sync-buckets.ts b/packages/lib/storage/sync-buckets.ts index ea940ef..72249bc 100644 --- a/packages/lib/storage/sync-buckets.ts +++ b/packages/lib/storage/sync-buckets.ts @@ -231,17 +231,26 @@ async function reconcileDeprovisionedBuckets(): Promise { } } - // Mark as deprovisioned - await prisma.storageBucket.update({ - where: { id: bucket.id }, - data: { provisionStatus: 'deprovisioning' }, - }) - - // Unassign from users - await prisma.user.updateMany({ - where: { storageBucketId: bucket.id }, - data: { storageBucketId: null }, - }) + // Mark as deprovisioned and update the Subscription record so + // getPlanLimits stops granting unlimited access. This is the recovery + // path for cases where the webhook failed to fire or was not processed. + await prisma.$transaction([ + prisma.storageBucket.update({ + where: { id: bucket.id }, + data: { provisionStatus: 'deprovisioning' }, + }), + prisma.subscription.updateMany({ + where: { + stripeSubscriptionId: bucket.stripeSubscriptionId, + status: 'active', + }, + data: { status: 'canceled', cancelAtPeriodEnd: false }, + }), + prisma.user.updateMany({ + where: { storageBucketId: bucket.id }, + data: { storageBucketId: null }, + }), + ]) } } catch (err: any) { // 404 means subscription doesn't exist @@ -271,15 +280,23 @@ async function reconcileDeprovisionedBuckets(): Promise { } } - await prisma.storageBucket.update({ - where: { id: bucket.id }, - data: { provisionStatus: 'deprovisioning' }, - }) - - await prisma.user.updateMany({ - where: { storageBucketId: bucket.id }, - data: { storageBucketId: null }, - }) + await prisma.$transaction([ + prisma.storageBucket.update({ + where: { id: bucket.id }, + data: { provisionStatus: 'deprovisioning' }, + }), + prisma.subscription.updateMany({ + where: { + stripeSubscriptionId: bucket.stripeSubscriptionId, + status: 'active', + }, + data: { status: 'canceled', cancelAtPeriodEnd: false }, + }), + prisma.user.updateMany({ + where: { storageBucketId: bucket.id }, + data: { storageBucketId: null }, + }), + ]) } else { logger.error( `[Sync] Error checking subscription ${bucket.stripeSubscriptionId}`, diff --git a/packages/lib/users/service.ts b/packages/lib/users/service.ts index 416b17a..b5dcf3d 100644 --- a/packages/lib/users/service.ts +++ b/packages/lib/users/service.ts @@ -51,6 +51,10 @@ export const USER_ADMIN_SELECT = { }, }, }, + storageBucketId: true, + storageBucket: { + select: { id: true, name: true, isCore: true, provisionStatus: true }, + }, _count: { select: { files: true, diff --git a/packages/lib/vultr/index.ts b/packages/lib/vultr/index.ts index 755a9bf..8e28b03 100644 --- a/packages/lib/vultr/index.ts +++ b/packages/lib/vultr/index.ts @@ -18,66 +18,71 @@ const VULTR_API_BASE = 'https://api.vultr.com/v2' const VULTR_API_BASE_URL = new URL(VULTR_API_BASE) function buildVultrApiUrl(path: string): string { - if (!path.startsWith('/')) { - throw new Error(`Invalid Vultr API path: "${path}"`) - } - if (path.startsWith('//') || path.includes('..') || path.includes('://')) { - throw new Error(`Unsafe Vultr API path: "${path}"`) - } - - const url = new URL(path, VULTR_API_BASE_URL) - - if (url.origin !== VULTR_API_BASE_URL.origin) { - throw new Error(`Unsafe Vultr API URL origin: "${url.origin}"`) - } - - if (!url.pathname.startsWith('/v2/')) { - throw new Error(`Unsafe Vultr API URL path: "${url.pathname}"`) - } - - const lowerPath = url.pathname.toLowerCase() - if ( - lowerPath.includes('%2e') || - lowerPath.includes('%2f') || - lowerPath.includes('%5c') - ) { - throw new Error(`Unsafe encoded Vultr API path: "${url.pathname}"`) - } - - return url.toString() + if (!path.startsWith('/')) { + throw new Error(`Invalid Vultr API path: "${path}"`) + } + if (path.startsWith('//') || path.includes('..') || path.includes('://')) { + throw new Error(`Unsafe Vultr API path: "${path}"`) + } + + const url = new URL('/v2' + path, VULTR_API_BASE_URL) + + if (url.origin !== VULTR_API_BASE_URL.origin) { + throw new Error(`Unsafe Vultr API URL origin: "${url.origin}"`) + } + + if (!url.pathname.startsWith('/v2/')) { + throw new Error(`Unsafe Vultr API URL path: "${url.pathname}"`) + } + + const lowerPath = url.pathname.toLowerCase() + if ( + lowerPath.includes('%2e') || + lowerPath.includes('%2f') || + lowerPath.includes('%5c') + ) { + throw new Error(`Unsafe encoded Vultr API path: "${url.pathname}"`) + } + + return url.toString() } async function getApiKey(): Promise { - const integrations = await getIntegrations() - const key = integrations?.vultr?.apiKey || process.env.VULTR_API_KEY - if (!key) throw new Error('Vultr API key is not configured. Set it in Admin → Integrations or via the VULTR_API_KEY environment variable.') - return key + const integrations = await getIntegrations() + const key = integrations?.vultr?.apiKey || process.env.VULTR_API_KEY + if (!key) + throw new Error( + 'Vultr API key is not configured. Set it in Admin → Integrations or via the VULTR_API_KEY environment variable.' + ) + return key } async function vultrRequest( - method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH', - path: string, - body?: unknown, + method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH', + path: string, + body?: unknown ): Promise { - const response = await fetch(buildVultrApiUrl(path), { - method, - headers: { - Authorization: `Bearer ${await getApiKey()}`, - 'Content-Type': 'application/json', - }, - body: body !== undefined ? JSON.stringify(body) : undefined, - }) - - // 204 No Content — success with no body - if (response.status === 204) return {} as T - - const text = await response.text() - - if (!response.ok) { - throw new Error(`Vultr API ${method} ${path} failed (${response.status}): ${text}`) - } + const response = await fetch(buildVultrApiUrl(path), { + method, + headers: { + Authorization: `Bearer ${await getApiKey()}`, + 'Content-Type': 'application/json', + }, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + + // 204 No Content — success with no body + if (response.status === 204) return {} as T + + const text = await response.text() + + if (!response.ok) { + throw new Error( + `Vultr API ${method} ${path} failed (${response.status}): ${text}` + ) + } - return JSON.parse(text) as T + return JSON.parse(text) as T } // ───────────────────────────────────────────────────────────────────────────── @@ -85,41 +90,41 @@ async function vultrRequest( // ───────────────────────────────────────────────────────────────────────────── export interface VultrObjectStorageInstance { - id: string - date_created: string - cluster_id: number - region: string - label: string - status: 'pending' | 'active' - s3_hostname: string - s3_access_key: string - s3_secret_key: string + id: string + date_created: string + cluster_id: number + region: string + label: string + status: 'pending' | 'active' + s3_hostname: string + s3_access_key: string + s3_secret_key: string } export interface VultrCluster { - id: number - region: string - hostname: string - deploy: 'yes' | 'no' + id: number + region: string + hostname: string + deploy: 'yes' | 'no' } /** Tier as returned by GET /object-storage/tiers or GET /object-storage/clusters/{id}/tiers */ export interface VultrTier { - id: number - sales_name: string - sales_desc: string - price: number - disk_gb_price: number - bw_gb_price: number - is_default: 'yes' | 'no' - slug: string + id: number + sales_name: string + sales_desc: string + price: number + disk_gb_price: number + bw_gb_price: number + is_default: 'yes' | 'no' + slug: string } export interface VultrBucket { - name: string - date_created: string - size: number - object_count: number + name: string + date_created: string + size: number + object_count: number } // ───────────────────────────────────────────────────────────────────────────── @@ -127,12 +132,13 @@ export interface VultrBucket { // ───────────────────────────────────────────────────────────────────────────── /** List all provisioned Object Storage instances on the Vultr account. */ -export async function listObjectStorages(): Promise { - const data = await vultrRequest<{ object_storages: VultrObjectStorageInstance[] }>( - 'GET', - '/object-storage?per_page=500', - ) - return data.object_storages ?? [] +export async function listObjectStorages(): Promise< + VultrObjectStorageInstance[] +> { + const data = await vultrRequest<{ + object_storages: VultrObjectStorageInstance[] + }>('GET', '/object-storage?per_page=500') + return data.object_storages ?? [] } /** @@ -142,30 +148,36 @@ export async function listObjectStorages(): Promise { - const data = await vultrRequest<{ object_storage: VultrObjectStorageInstance }>( - 'POST', - '/object-storage', - { cluster_id: clusterId, label, tier_id: tierId }, - ) - return data.object_storage + const data = await vultrRequest<{ + object_storage: VultrObjectStorageInstance + }>('POST', '/object-storage', { + cluster_id: clusterId, + label, + tier_id: tierId, + }) + return data.object_storage } /** Get a single provisioned instance by Vultr resource ID. */ -export async function getObjectStorage(vultrId: string): Promise { - const data = await vultrRequest<{ object_storage: VultrObjectStorageInstance }>( - 'GET', - `/object-storage/${vultrId}`, - ) - return data.object_storage +export async function getObjectStorage( + vultrId: string +): Promise { + const data = await vultrRequest<{ + object_storage: VultrObjectStorageInstance + }>('GET', `/object-storage/${vultrId}`) + return data.object_storage } /** Update the label of a provisioned instance. */ -export async function updateObjectStorage(vultrId: string, label: string): Promise { - await vultrRequest('PUT', `/object-storage/${vultrId}`, { label }) +export async function updateObjectStorage( + vultrId: string, + label: string +): Promise { + await vultrRequest('PUT', `/object-storage/${vultrId}`, { label }) } /** @@ -173,7 +185,7 @@ export async function updateObjectStorage(vultrId: string, label: string): Promi * WARNING: Destroys all buckets and objects inside it permanently. */ export async function deleteObjectStorage(vultrId: string): Promise { - await vultrRequest('DELETE', `/object-storage/${vultrId}`) + await vultrRequest('DELETE', `/object-storage/${vultrId}`) } /** @@ -181,13 +193,12 @@ export async function deleteObjectStorage(vultrId: string): Promise { * All existing keys are immediately invalidated. */ export async function regenerateObjectStorageKeys( - vultrId: string, + vultrId: string ): Promise<{ s3_access_key: string; s3_secret_key: string }> { - const data = await vultrRequest<{ s3_credentials: { s3_access_key: string; s3_secret_key: string } }>( - 'POST', - `/object-storage/${vultrId}/regenerate-keys`, - ) - return data.s3_credentials + const data = await vultrRequest<{ + s3_credentials: { s3_access_key: string; s3_secret_key: string } + }>('POST', `/object-storage/${vultrId}/regenerate-keys`) + return data.s3_credentials } // ───────────────────────────────────────────────────────────────────────────── @@ -195,15 +206,24 @@ export async function regenerateObjectStorageKeys( // ───────────────────────────────────────────────────────────────────────────── /** Create a bucket inside a provisioned Object Storage instance. */ -export async function createObjectStorageBucket(vultrId: string, bucketName: string): Promise { - await vultrRequest('POST', `/object-storage/${vultrId}/buckets`, { - bucket_name: bucketName, - }) +export async function createObjectStorageBucket( + vultrId: string, + bucketName: string +): Promise { + await vultrRequest('POST', `/object-storage/${vultrId}/buckets`, { + bucket_name: bucketName, + }) } /** Delete a bucket and all of its objects inside a provisioned instance. */ -export async function deleteObjectStorageBucket(vultrId: string, bucketName: string): Promise { - await vultrRequest('DELETE', `/object-storage/${vultrId}/buckets/${bucketName}`) +export async function deleteObjectStorageBucket( + vultrId: string, + bucketName: string +): Promise { + await vultrRequest( + 'DELETE', + `/object-storage/${vultrId}/buckets/${bucketName}` + ) } // ───────────────────────────────────────────────────────────────────────────── @@ -212,36 +232,53 @@ export async function deleteObjectStorageBucket(vultrId: string, bucketName: str /** List all available clusters (regions) that support Object Storage. */ export async function listClusters(): Promise { - const data = await vultrRequest<{ clusters: VultrCluster[] }>('GET', '/object-storage/clusters') - return data.clusters ?? [] + const data = await vultrRequest<{ clusters: VultrCluster[] }>( + 'GET', + '/object-storage/clusters' + ) + return data.clusters ?? [] } /** List tiers available for a specific cluster. */ -export async function listClusterTiers(clusterId: number): Promise { - if (!Number.isFinite(clusterId) || !Number.isSafeInteger(clusterId)) { - throw new Error(`Invalid clusterId: ${clusterId}`) - } - - const safeClusterId = Math.trunc(clusterId) - if (safeClusterId !== clusterId || safeClusterId < 0) { - throw new Error(`Invalid clusterId: ${clusterId}`) - } - - const data = await vultrRequest<{ tiers: VultrTier[] }>( - 'GET', - `/object-storage/clusters/${safeClusterId}/tiers` - ) - return data.tiers ?? [] +export async function listClusterTiers( + clusterId: number +): Promise { + if (!Number.isFinite(clusterId) || !Number.isSafeInteger(clusterId)) { + throw new Error(`Invalid clusterId: ${clusterId}`) + } + + const safeClusterId = Math.trunc(clusterId) + if (safeClusterId !== clusterId || safeClusterId < 0) { + throw new Error(`Invalid clusterId: ${clusterId}`) + } + + const data = await vultrRequest<{ tiers: VultrTier[] }>( + 'GET', + `/object-storage/clusters/${safeClusterId}/tiers` + ) + return data.tiers ?? [] } /** List all Vultr regions (superset — useful for label enrichment). */ -export async function listRegions(): Promise> { - const data = await vultrRequest<{ regions: Array<{ id: string; city: string; country: string; continent: string }> }>('GET', '/regions') - return data.regions ?? [] +export async function listRegions(): Promise< + Array<{ id: string; city: string; country: string; continent: string }> +> { + const data = await vultrRequest<{ + regions: Array<{ + id: string + city: string + country: string + continent: string + }> + }>('GET', '/regions') + return data.regions ?? [] } /** List all available storage tiers and their pricing. */ export async function listTiers(): Promise { - const data = await vultrRequest<{ tiers: VultrTier[] }>('GET', '/object-storage/tiers') - return data.tiers ?? [] + const data = await vultrRequest<{ tiers: VultrTier[] }>( + 'GET', + '/object-storage/tiers' + ) + return data.tiers ?? [] } diff --git a/prisma/migrations/20260615181957_better_bucket_handling/migration.sql b/prisma/migrations/20260615181957_better_bucket_handling/migration.sql new file mode 100644 index 0000000..e3fed4a --- /dev/null +++ b/prisma/migrations/20260615181957_better_bucket_handling/migration.sql @@ -0,0 +1,18 @@ +-- DropIndex +DROP INDEX "StorageBucket_stripeSubscriptionId_idx"; + +-- AlterTable +ALTER TABLE "File" ADD COLUMN "storageBucketId" TEXT; + +-- AlterTable +ALTER TABLE "StorageBucket" ADD COLUMN "fileCount" INTEGER NOT NULL DEFAULT 0, +ADD COLUMN "isCore" BOOLEAN NOT NULL DEFAULT false; + +-- CreateIndex +CREATE INDEX "File_storageBucketId_idx" ON "File"("storageBucketId"); + +-- CreateIndex +CREATE INDEX "StorageBucket_isCore_idx" ON "StorageBucket"("isCore"); + +-- AddForeignKey +ALTER TABLE "File" ADD CONSTRAINT "File_storageBucketId_fkey" FOREIGN KEY ("storageBucketId") REFERENCES "StorageBucket"("id") ON DELETE SET NULL ON UPDATE CASCADE; diff --git a/prisma/migrations/20260615184339_bucket_priority_and_more/migration.sql b/prisma/migrations/20260615184339_bucket_priority_and_more/migration.sql new file mode 100644 index 0000000..7c5cf76 --- /dev/null +++ b/prisma/migrations/20260615184339_bucket_priority_and_more/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "StorageBucket" ADD COLUMN "priority" INTEGER NOT NULL DEFAULT 0; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index db45bed..200f394 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -167,6 +167,9 @@ model File { // Squad association — null for personal files squadId String? squad NexiumSquad? @relation(fields: [squadId], references: [id], onDelete: SetNull) + // Which storage bucket this file lives in. Null = global config provider (legacy / fallback). + storageBucketId String? + storageBucket StorageBucket? @relation("FileBucket", fields: [storageBucketId], references: [id], onDelete: SetNull) editSuggestions FileEditSuggestion[] collaborators FileCollaborator[] contentReports ContentReport[] @@ -175,6 +178,7 @@ model File { @@index([isOcrProcessed]) // Worker queue filtering @@index([flagged]) @@index([squadId]) + @@index([storageBucketId]) } // Collaborators who can edit a file @@ -961,6 +965,14 @@ model StorageBucket { s3SecretKey String @default("") s3Endpoint String? s3ForcePathStyle Boolean @default(false) + // Internal shared pool buckets (isCore = true) vs user-dedicated buckets (isCore = false). + // Core buckets are selected by least-filled load balancing for users without a dedicated bucket. + isCore Boolean @default(false) + // Running count of files stored in this bucket — used for core bucket load balancing. + fileCount Int @default(0) + // Higher priority core buckets are preferred when file counts are equal. + // Useful for routing uploads to higher-performance or lower-latency buckets. + priority Int @default(0) // Vultr-backed buckets: tracks which pooled instance this bucket lives in // and the Stripe subscription that triggered provisioning. vultrObjectStorageId String? @@ -973,8 +985,10 @@ model StorageBucket { vultrObjectStorage VultrObjectStorage? @relation(fields: [vultrObjectStorageId], references: [id], onDelete: SetNull) assignedUsers User[] @relation("UserStorageBucket") assignedSquads NexiumSquad[] @relation("SquadStorageBucket") + files File[] @relation("FileBucket") @@index([provider]) + @@index([isCore]) @@index([vultrObjectStorageId]) @@unique([vultrBucketName]) @@unique([stripeSubscriptionId]) diff --git a/proxy.ts b/proxy.ts index 59307d9..38d62b2 100644 --- a/proxy.ts +++ b/proxy.ts @@ -133,9 +133,6 @@ export async function proxy(request: NextRequest) { const isNextAuthRoute = pathname.startsWith('/api/auth/') const isApiRoute = pathname.startsWith('/api/') - // ── File URL handling — single unified check before auth ────────────────── - // Covers both trailing-slash and non-trailing-slash variants via normalizedPathname. - // Must run before getToken() so media range requests skip JWT verification entirely. const isFileUrl = FILE_URL_PATTERN.test(normalizedPathname) && !normalizedPathname.endsWith('/raw') && @@ -149,18 +146,12 @@ export async function proxy(request: NextRequest) { rangeHeader != null || (acceptHeader !== '' && !acceptHeader.includes('text/html')) - // Video/audio range or non-HTML requests → raw bytes. - // Must run before the bot handler: Discord's media proxy UA contains "discord" - // and would otherwise be caught by handleBotRequest. if (fileExt && VIDEO_EXTENSIONS_SET.has(fileExt) && isMediaRequest) { const url = new URL(request.url) url.pathname = `${normalizedPathname}/raw` return NextResponse.rewrite(url) } - // Non-trailing-slash, non-bot requests → rewrite to trailing slash so the - // file page renders correctly. Bots fall through so handleBotRequest can - // respect the uploader's rich-embed setting. if (pathname === normalizedPathname) { const userAgent = request.headers.get('user-agent') || '' if (!isBotRequest(userAgent)) {