diff --git a/.changeset/README.md b/.changeset/README.md index e5b6d8d6..5654e898 100644 --- a/.changeset/README.md +++ b/.changeset/README.md @@ -1,8 +1,5 @@ # Changesets -Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works -with multi-package repos, or single-package repos to help you version and publish your code. You can -find the full documentation for it [in our repository](https://github.com/changesets/changesets) +Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works with multi-package repos, or single-package repos to help you version and publish your code. You can find the full documentation for it [in our repository](https://github.com/changesets/changesets) -We have a quick list of common questions to get you started engaging with this project in -[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) +We have a quick list of common questions to get you started engaging with this project in [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) diff --git a/.changeset/adapter-drizzle-1.0.md b/.changeset/adapter-drizzle-1.0.md new file mode 100644 index 00000000..ac0b1102 --- /dev/null +++ b/.changeset/adapter-drizzle-1.0.md @@ -0,0 +1,30 @@ +--- +"@vorsteh-queue/adapter-drizzle": major +--- + +## @vorsteh-queue/adapter-drizzle 1.0 + +### Breaking: QueueAdapter Interface Rewritten + +Implements the new `QueueAdapter` interface with additional methods: + +- `getJobById(id)` — retrieve a single job +- `getNextJob(options)` — now requires `handlerNames` and `activeGroups` for FIFO support +- `getNextJobsForHandler(name, count, groupConstraints)` — batch picking with group constraints +- `updateJobStatus(id, update)` — new signature with `JobStatusUpdate` object +- `cancelJob(id, reason?)` / `cancelJobs(filter)` — cancellation support +- `getDeadJobs()` / `redriveJob(id)` / `redriveJobs(filter?)` — DLQ support +- `findJobByUniqueKey(uniqueKey)` — unique job lookup +- `updateJobSteps(id, steps)` — step state persistence + +### Breaking: Schema Changes (migration required) + +New columns: `group_key`, `unique_key`, `cancelled_at`, `cancellation_reason`, `steps` + +Changed: `timeout` from JSONB to INT (nullable, milliseconds) + +Changed: `progress` from nullable to non-nullable INT (default 0) + +Changed: `repeat_count` from nullable to non-nullable (default 0) + +New indexes: `idx_queue_jobs_polling`, `idx_queue_jobs_delayed`, `idx_queue_jobs_active_groups`, `idx_queue_jobs_stats` diff --git a/.changeset/adapter-kysely-1.0.md b/.changeset/adapter-kysely-1.0.md new file mode 100644 index 00000000..fc286d1a --- /dev/null +++ b/.changeset/adapter-kysely-1.0.md @@ -0,0 +1,32 @@ +--- +"@vorsteh-queue/adapter-kysely": major +--- + +## @vorsteh-queue/adapter-kysely 1.0 + +### Breaking: QueueAdapter Interface Rewritten + +Implements the new `QueueAdapter` interface with additional methods: + +- `getJobById(id)` — retrieve a single job +- `getNextJob(options)` — now requires `handlerNames` and `activeGroups` for FIFO support +- `getNextJobsForHandler(name, count, groupConstraints)` — batch picking with group constraints +- `updateJobStatus(id, update)` — new signature with `JobStatusUpdate` object +- `cancelJob(id, reason?)` / `cancelJobs(filter)` — cancellation support +- `getDeadJobs()` / `redriveJob(id)` / `redriveJobs(filter?)` — DLQ support +- `findJobByUniqueKey(uniqueKey)` — unique job lookup +- `updateJobSteps(id, steps)` — step state persistence + +### Breaking: Schema Changes (migration required) + +New columns: `group_key`, `unique_key`, `cancelled_at`, `cancellation_reason`, `steps` + +Changed: `timeout` from JSONB to INT (nullable, milliseconds) + +Changed: `progress` from nullable to non-nullable INT (default 0) + +Changed: `repeat_count` from nullable to non-nullable (default 0) + +New indexes: `idx_queue_jobs_polling`, `idx_queue_jobs_delayed`, `idx_queue_jobs_active_groups`, `idx_queue_jobs_stats` + +Updated `createQueueJobsTable()` helper and migration files with new schema. diff --git a/.changeset/adapter-mikroorm-initial.md b/.changeset/adapter-mikroorm-initial.md new file mode 100644 index 00000000..43e28158 --- /dev/null +++ b/.changeset/adapter-mikroorm-initial.md @@ -0,0 +1,22 @@ +--- +"@vorsteh-queue/adapter-mikroorm": major +"@vorsteh-queue/core": patch +"@vorsteh-queue/query-builder": patch +--- + +## @vorsteh-queue/adapter-mikroorm (new package) + +Initial release of the MikroORM adapter for Vorsteh Queue. + +### Features + +- `PostgresMikroormQueueAdapter` — full implementation of the `QueueAdapter` interface +- PostgreSQL support via MikroORM v6 EntityManager with unit of work pattern +- Raw SQL job picking with `FOR UPDATE SKIP LOCKED` for reliable concurrent processing +- Pre-built `QueueJobSchema` (EntitySchema) for quick database setup +- Configurable table name and schema name + +### Related changes + +- `@vorsteh-queue/core`: Added `MikroormAdapterProps` interface and extended `AdapterKind` union +- `@vorsteh-queue/query-builder`: Added `./mikroorm` export with MikroORM-compatible `buildWhere` diff --git a/.changeset/adapter-prisma-1.0.md b/.changeset/adapter-prisma-1.0.md new file mode 100644 index 00000000..5fb668e7 --- /dev/null +++ b/.changeset/adapter-prisma-1.0.md @@ -0,0 +1,32 @@ +--- +"@vorsteh-queue/adapter-prisma": major +--- + +## @vorsteh-queue/adapter-prisma 1.0 + +### Breaking: QueueAdapter Interface Rewritten + +Implements the new `QueueAdapter` interface with additional methods: + +- `getJobById(id)` — retrieve a single job +- `getNextJob(options)` — now requires `handlerNames` and `activeGroups` for FIFO support +- `getNextJobsForHandler(name, count, groupConstraints)` — batch picking with group constraints +- `updateJobStatus(id, update)` — new signature with `JobStatusUpdate` object +- `cancelJob(id, reason?)` / `cancelJobs(filter)` — cancellation support +- `getDeadJobs()` / `redriveJob(id)` / `redriveJobs(filter?)` — DLQ support +- `findJobByUniqueKey(uniqueKey)` — unique job lookup +- `updateJobSteps(id, steps)` — step state persistence + +### Breaking: Schema Changes (migration required) + +New columns: `group_key`, `unique_key`, `cancelled_at`, `cancellation_reason`, `steps` + +Changed: `timeout` from JSONB to INT (nullable, milliseconds) + +Changed: `progress` from nullable to non-nullable INT (default 0) + +Changed: `repeat_count` from nullable to non-nullable (default 0) + +New indexes: `idx_queue_jobs_polling`, `idx_queue_jobs_delayed`, `idx_queue_jobs_active_groups`, `idx_queue_jobs_stats` + +Updated `schema.prisma` with new model fields. diff --git a/.changeset/adapter-sequelize-initial.md b/.changeset/adapter-sequelize-initial.md new file mode 100644 index 00000000..9f986f3e --- /dev/null +++ b/.changeset/adapter-sequelize-initial.md @@ -0,0 +1,22 @@ +--- +"@vorsteh-queue/adapter-sequelize": major +"@vorsteh-queue/core": patch +"@vorsteh-queue/query-builder": patch +--- + +## @vorsteh-queue/adapter-sequelize (new package) + +Initial release of the Sequelize adapter for Vorsteh Queue. + +### Features + +- `PostgresSequelizeQueueAdapter` — full implementation of the `QueueAdapter` interface +- PostgreSQL support via Sequelize Model pattern with raw SQL for job picking +- Raw SQL job picking with `FOR UPDATE SKIP LOCKED` for reliable concurrent processing +- Pre-built `QueueJobModel` with `initQueueJobModel()` for quick model registration +- Configurable table name and schema name + +### Related changes + +- `@vorsteh-queue/core`: Added `SequelizeAdapterProps` interface and extended `AdapterKind` union +- `@vorsteh-queue/query-builder`: Added `./sequelize` export with Sequelize-compatible `buildWhere` diff --git a/.changeset/adapter-typeorm-initial.md b/.changeset/adapter-typeorm-initial.md new file mode 100644 index 00000000..c4b2562a --- /dev/null +++ b/.changeset/adapter-typeorm-initial.md @@ -0,0 +1,22 @@ +--- +"@vorsteh-queue/adapter-typeorm": major +"@vorsteh-queue/core": patch +"@vorsteh-queue/query-builder": patch +--- + +## @vorsteh-queue/adapter-typeorm (new package) + +Initial release of the TypeORM adapter for Vorsteh Queue. + +### Features + +- `PostgresTypeormQueueAdapter` — full implementation of the `QueueAdapter` interface +- PostgreSQL support via TypeORM's DataSource and Repository pattern +- Raw SQL job picking with `FOR UPDATE SKIP LOCKED` for reliable concurrent processing +- Pre-built `QueueJobEntity` with all required columns, types, and indexes +- Configurable table name and schema name + +### Related changes + +- `@vorsteh-queue/core`: Added `TypeormAdapterProps` interface and extended `AdapterKind` union +- `@vorsteh-queue/query-builder`: Added `./typeorm` export with TypeORM-compatible `buildWhere` diff --git a/.changeset/adapter-zenstack-initial.md b/.changeset/adapter-zenstack-initial.md new file mode 100644 index 00000000..626178e0 --- /dev/null +++ b/.changeset/adapter-zenstack-initial.md @@ -0,0 +1,22 @@ +--- +"@vorsteh-queue/adapter-zenstack": major +"@vorsteh-queue/core": patch +"@vorsteh-queue/query-builder": patch +--- + +## @vorsteh-queue/adapter-zenstack (new package) + +Initial release of the ZenStack ORM adapter for Vorsteh Queue. + +### Features + +- `PostgresZenstackQueueAdapter` — full implementation of the `QueueAdapter` interface +- PostgreSQL support via ZenStack ORM v3 (Prisma-compatible API built on Kysely) +- Raw SQL job picking with `FOR UPDATE SKIP LOCKED` for reliable concurrent processing +- Configurable model name, table name, and schema name +- ZModel schema provided for quick database setup + +### Related changes + +- `@vorsteh-queue/core`: Added `ZenstackAdapterProps` interface and extended `AdapterKind` union +- `@vorsteh-queue/query-builder`: Added `./zenstack` export (re-exports Prisma-compatible `buildWhere`) diff --git a/.changeset/cli-initial.md b/.changeset/cli-initial.md new file mode 100644 index 00000000..ada67e3e --- /dev/null +++ b/.changeset/cli-initial.md @@ -0,0 +1,14 @@ +--- +"@vorsteh-queue/cli": minor +--- + +## @vorsteh-queue/cli — Initial Release + +CLI tool for monitoring and managing vorsteh-queue jobs. + +- Commands: `status`, `inspect`, `cancel`, `redrive`, `clear` +- Transports: direct adapter connection or remote GraphQL server +- `--json` flag on all commands for machine-readable output +- Configuration via environment variables (`VORSTEH_QUEUE_URL`, `VORSTEH_QUEUE_TOKEN`) +- `defineCliConfig()` helper for typed configuration +- Built with citty (unjs CLI framework) diff --git a/.changeset/core-1.0.md b/.changeset/core-1.0.md new file mode 100644 index 00000000..3939065e --- /dev/null +++ b/.changeset/core-1.0.md @@ -0,0 +1,59 @@ +--- +"@vorsteh-queue/core": major +--- + +## @vorsteh-queue/core 1.0 + +### Breaking: Queue/Worker Architecture Split + +The `Queue` class is now a producer-only. Job processing is handled by the new `Worker` class. + +Before: + +```typescript +const queue = new Queue(adapter, { name: "my-queue" }) +queue.register("job", handler) +queue.start() +``` + +After: + +```typescript +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue", concurrency: 5 }) +worker.register("job", handler) +worker.start() + +// Queue is now producer-only — use it to add jobs +await queue.add("job", { data: "payload" }) +``` + +### Breaking: Handler Signature + +Handlers now receive a `JobContext` with an `AbortSignal` for timeout/cancellation support. + +- Before: `(job) => Promise` +- After: `(job, { signal, step }) => Promise` + +### Breaking: New Statuses + +Added `cancelled` and `dead` statuses. `QueueStats` now includes all 7 statuses. + +### Breaking: Removed APIs + +`queue.register()`, `queue.start()`, `queue.stop()`, `queue.pause()`, `queue.resume()`, `queue.enqueue()`, `queue.dequeue()` + +### New Features + +- Job cancellation (`queue.cancel()`, `worker.cancelJob()`) +- Dead-letter queue (`queue.getDeadJobs()`, `queue.redrive()`, `queue.redriveAll()`) +- Group FIFO ordering (`{ group: "tenant-1" }`) +- Unique job deduplication (`{ unique: { key: "...", action: "reject" | "replace" } }`) +- `enqueueAndWait()` — enqueue and wait for result (hybrid event + polling) +- Typed event emitter on both Queue and Worker +- Configurable retry strategies (exponential, linear, fixed, custom function) +- State machine validation for job status transitions +- Per-handler concurrency limits +- Job Steps (`step.run()`, `step.sleep()`, `step.all()`) for durable multi-step execution +- Job Dependencies (`dependsOn`, circular detection) +- Rate Limiting (token-bucket per handler) diff --git a/.changeset/drizzle-v1-migration.md b/.changeset/drizzle-v1-migration.md new file mode 100644 index 00000000..33bf176a --- /dev/null +++ b/.changeset/drizzle-v1-migration.md @@ -0,0 +1,33 @@ +--- +"@vorsteh-queue/adapter-drizzle": major +"@vorsteh-queue/query-builder": major +--- + +## Breaking: drizzle-orm peer dependency bumped to >=1.0.0-rc.4 + +The minimum `drizzle-orm` peer dependency has been raised from `>=0.45.x` to `>=1.0.0-rc.4` for both the adapter and query builder packages. + +### Breaking: Drizzle client initialization + +The `drizzle()` call now requires a `relations` object: + +**Before:** + +```typescript +import * as schema from "./schema" +const db = drizzle(pool, { schema }) +``` + +**After:** + +```typescript +import { relations } from "./schema" +const db = drizzle({ client: pool, relations }) +``` + +### Migration steps + +1. Upgrade `drizzle-orm` to `>=1.0.0-rc.4` and `drizzle-kit` to `>=1.0.0-rc.4` +2. Define your own relations with `defineRelations` including `queueJobs` +3. Pass `relations` (not `schema`) to your `drizzle()` call +4. The query builder's `buildWhere` now returns a plain RQB v2 where object instead of SQL conditions diff --git a/.changeset/flow-producer.md b/.changeset/flow-producer.md new file mode 100644 index 00000000..0b8050dd --- /dev/null +++ b/.changeset/flow-producer.md @@ -0,0 +1,38 @@ +--- +"@vorsteh-queue/core": major +"@vorsteh-queue/adapter-drizzle": major +"@vorsteh-queue/adapter-prisma": major +"@vorsteh-queue/adapter-kysely": major +"@vorsteh-queue/adapter-zenstack": major +"@vorsteh-queue/adapter-typeorm": major +"@vorsteh-queue/adapter-mikroorm": major +"@vorsteh-queue/adapter-sequelize": major +"@vorsteh-queue/shared-tests": major +--- + +## FlowProducer V1 + +Redesigned the flow system with a clean separation between flow orchestration and job processing. + +### Breaking Changes + +- Removed `waiting-children` from `JobStatus` and `STATE_TRANSITIONS` +- Removed flow fields from `Job` interface: `parentId`, `flowId`, `childrenCount`, `childrenCompleted`, `failParentOnFailure` +- Removed `dependsOn` and `onDependencyFailure` from `Job` and `JobOptions` +- Removed `dependencies.ts` module (`detectCircularDependencies`, `areDependenciesMet`, etc.) +- Removed flow methods from `QueueAdapter`: `getFlowTree`, `deleteFlow`, `incrementChildrenCompleted`, `getChildrenJobs`, `getFlows` +- Removed flow methods from `Queue` class: `addFlow`, `getFlowTree`, `createFlowNode` +- Removed `getChildrenResults` from `JobContext` + +### New Features + +- `FlowProducer` class for creating and managing multi-step job flows +- `FlowAdapter` interface for flow persistence (separate from `QueueAdapter`) +- 2-table architecture: `queue_flows` table for orchestration, `queue_jobs` for work +- Parent jobs only created after all direct children reach terminal state +- Three failure strategies: `default`, `fail-parent`, `continue-parent` +- `FlowJobContext` in parent handlers: `getChildrenValues`, `getFailedChildrenValues`, `getChildrenValuesBy`, `removeUnprocessedChildren` +- Flow events: `flow:created`, `flow:completed`, `flow:failed`, `flow:node:promoted` +- Convenience methods: `addChain` (sequential), `addBulkThen` (fan-in) +- Configurable flow cleanup via `removeOnComplete` +- Cross-queue flow support diff --git a/.changeset/job-where-filter.md b/.changeset/job-where-filter.md new file mode 100644 index 00000000..8a4aeee0 --- /dev/null +++ b/.changeset/job-where-filter.md @@ -0,0 +1,51 @@ +--- +"@vorsteh-queue/core": major +"@vorsteh-queue/query-builder": minor +"@vorsteh-queue/adapter-drizzle": major +"@vorsteh-queue/adapter-prisma": major +"@vorsteh-queue/adapter-kysely": major +"@vorsteh-queue/server": major +"@vorsteh-queue/cli": major +--- + +## Job Where Filter + +### Breaking: `getJobs` and `size` Signatures + +Replaced flat `status`/`name` filter parameters with a composable `where` input supporting multiple filter conditions with comparison operators. + +**Before:** + +```typescript +adapter.getJobs({ status: "failed", name: "send-email", limit: 10 }) +adapter.size() +``` + +**After:** + +```typescript +adapter.getJobs({ where: { status: "failed", name: "send-email" }, limit: 10 }) +adapter.size({ status: "failed" }) +``` + +### Breaking: GraphQL Schema + +- `jobs(queue, status, name, limit, offset)` → `jobs(queue, where: JobWhereInput, limit, offset)` +- `size(queue)` → `size(queue, where: JobWhereInput)` + +### New: `@vorsteh-queue/query-builder` + +New package providing filter normalization, in-memory matching, and ORM-specific where-clause builders: + +- `normalizeWhere()` — expands shorthand filters +- `matchesWhere()` — in-memory job matching +- `buildWhere()` — translates to Drizzle/Kysely/Prisma conditions + +### Supported Operators + +- **String:** eq, neq, contains, startsWith, like, in, isNull +- **Int:** eq, neq, lt, lte, gt, gte, isNull +- **DateTime:** lt, lte, gt, gte, isNull +- **Status:** eq, neq, in +- **Null:** isNull +- **Logical:** AND, OR (recursive) diff --git a/.changeset/node-22-minimum.md b/.changeset/node-22-minimum.md new file mode 100644 index 00000000..e8f1e4fa --- /dev/null +++ b/.changeset/node-22-minimum.md @@ -0,0 +1,13 @@ +--- +"@vorsteh-queue/core": major +"@vorsteh-queue/adapter-drizzle": major +"@vorsteh-queue/adapter-prisma": major +"@vorsteh-queue/adapter-kysely": major +"@vorsteh-queue/server": major +"@vorsteh-queue/cli": major +"create-vorsteh-queue": major +--- + +**Breaking:** Minimum Node.js version raised from 20 to 22. + +Node.js 22 is the current active LTS. This allows the use of stable native `fetch`, improved `AbortSignal` support, and aligns with the TypeScript 6.x target. diff --git a/.changeset/retry-runnow-delete.md b/.changeset/retry-runnow-delete.md new file mode 100644 index 00000000..4625e74e --- /dev/null +++ b/.changeset/retry-runnow-delete.md @@ -0,0 +1,23 @@ +--- +"@vorsteh-queue/core": minor +"@vorsteh-queue/adapter-drizzle": minor +"@vorsteh-queue/adapter-kysely": minor +"@vorsteh-queue/adapter-prisma": minor +"@vorsteh-queue/server": minor +"@vorsteh-queue/cli": minor +--- + +## New operations: retry, runNow, deleteJob + +Added three new job management operations across the full stack: + +- **`retry(jobId)`** — resets a `failed` job to `pending` (clears error, resets attempts to 0) +- **`runNow(jobId)`** — promotes a `delayed` job to `pending` immediately (sets processAt to now) +- **`deleteJob(jobId)`** — permanently removes a single job by ID + +Available on: + +- `Queue` class: `queue.retry()`, `queue.runNow()`, `queue.deleteJob()` +- `QueueAdapter` interface: `retryJob()`, `runJobNow()`, `deleteJob()` +- GraphQL API: mutations `retryJob`, `runJobNow`, `deleteJob` +- CLI: commands `retry`, `run-now`, `delete` diff --git a/.changeset/server-1.0.md b/.changeset/server-1.0.md new file mode 100644 index 00000000..4e650ec8 --- /dev/null +++ b/.changeset/server-1.0.md @@ -0,0 +1,33 @@ +--- +"@vorsteh-queue/server": major +--- + +## @vorsteh-queue/server 1.0 — Initial Release + +GraphQL server, web dashboard, and monitoring API for Vorsteh Queue. + +### GraphQL API + +- `createQueueServer(config)` — standalone server (Hono + Node.js) +- `createQueueMiddleware(config)` — composable Hono middleware for existing apps +- Queries: `stats`, `job`, `jobs` (with `where: JobWhereInput`), `deadJobs`, `size`, `flows` +- Mutations: `cancelJob`, `redriveJob`, `redriveAll`, `retryJob`, `runJobNow`, `deleteJob`, `clearJobs` +- Subscriptions: `jobStatusChanged`, `statsUpdated` (SSE-based) +- PubSub for real-time event streaming + +### Web Dashboard + +- Embedded dashboard served as static assets when `dashboard: true` (default) +- Overview stats, Jobs list (filtered/paginated), Job detail, DLQ, Flows, Flow DAG visualization +- Communicates with the API via `gql.tada` + `graphql-request` + +### Authentication + +- Token-based auth: `tokens: string[]` +- Custom middleware support for advanced auth scenarios +- Disable with `false` for development + +### Configuration + +- `loadConfig()` to load `queue.config.ts` via c12 +- `defineConfig()` helper for typed configuration diff --git a/.changeset/workflow-engine.md b/.changeset/workflow-engine.md new file mode 100644 index 00000000..a1838406 --- /dev/null +++ b/.changeset/workflow-engine.md @@ -0,0 +1,39 @@ +--- +"@vorsteh-queue/core": minor +--- + +## Workflow Engine Features + +### Saga Compensation + +Steps can declare `compensate` functions that run in reverse order when a later step fails: + +```typescript +await step.run("charge", () => payments.charge(amount), { + compensate: (result) => payments.refund(result.txId), +}) +``` + +### Signals / Human-in-the-Loop + +Jobs can pause and wait for an external signal: + +```typescript +const approval = await step.waitFor("approval", "manager-decision", { + timeout: "24h", +}) +// External: await queue.signal(jobId, "manager-decision", { approved: true }) +``` + +### Event Triggers + +Automatically create follow-up jobs on completion: + +```typescript +worker.trigger({ + on: "order", + create: "send-receipt", + data: (result, job) => ({ email: result.email }), + condition: (result) => result.total > 0, +}) +``` diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9d9a8cc9..a3ef108c 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,13 +1,15 @@ # GitHub Copilot Instructions ## Development Philosophy + - **Quality over quantity**: Write clean, maintainable, and well-structured code - **Senior-level TypeScript**: Leverage advanced TypeScript features for type safety - **Minimal and focused**: Every line of code should serve a clear purpose - **Performance-conscious**: Consider efficiency and avoid unnecessary complexity -## TypeScript & ESLint Rules -- **No console.log**: Use proper logging or remove debug statements +## TypeScript & Linting Rules (oxlint via ultracite) + +- **No console.log**: Use proper logging or remove debug statements (`no-console: error`) - **Consistent type imports**: Use `import type` for type-only imports - **No unused vars**: Prefix with `_` if intentionally unused - **No unnecessary conditions**: Avoid redundant null checks @@ -20,22 +22,46 @@ - **No useless path segments**: Avoid `/index` in import paths when possible - **Prefer directory imports**: Use `../src` instead of `../src/index` for cleaner imports -## Prettier Configuration -- **Print width**: 100 characters max +## Formatting Configuration (oxfmt via ultracite) + +- **Line width**: 100 characters max - **No semicolons**: Use semicolon-free style -- **Import order**: Follow the specified import grouping: +- **Trailing commas**: ES5 style +- **Import sorting**: Handled automatically by oxfmt (`sortImports: true`) +- **Import order** (enforced by oxfmt): 1. Types first 2. React/Next.js/Expo (if applicable) 3. Third-party modules 4. @vorsteh-queue packages 5. Relative imports (~/,../, ./) +## Tooling & Verification + +After making code changes, verify with these commands (in this order): + +1. `pnpm format:fix` — Apply formatting +2. `pnpm lint:fix` — Auto-fix lint issues +3. `pnpm typecheck` — Verify types +4. `vitest --run` — Run tests (single run, no watch mode) + +Always use the pnpm scripts from the root `package.json`. Never call tools directly via `npx` or `pnpm dlx`. + +| Tool | Check | Fix | +| ------------------ | ---------------- | ----------------- | +| Formatter (oxfmt) | `pnpm format` | `pnpm format:fix` | +| Linter (oxlint) | `pnpm lint` | `pnpm lint:fix` | +| Types (TypeScript) | `pnpm typecheck` | — | +| Tests (Vitest) | `vitest --run` | — | +| Build (Turborepo) | `pnpm build` | — | +| Workspace (sherif) | `pnpm lint:ws` | — | + ## Code Generation Guidelines + - Remove all `console.log` statements from generated code - Use proper TypeScript types instead of `any` when possible - **Use type-fest when available** - Prefer battle-tested utility types from type-fest over custom implementations -- Add ESLint disable comments only when absolutely necessary -- Follow the import order specified in prettier config +- Add lint disable comments only when absolutely necessary +- Follow the import order enforced by oxfmt - Use consistent naming conventions (camelCase for variables, PascalCase for types) - **Generic type parameters**: Always prefix with `T` (e.g., `TJobPayload`, `TResult`, `TEventData`) - Prefer explicit return types for functions @@ -46,8 +72,9 @@ - Optimize for readability and maintainability ## File Structure -- Keep imports organized according to prettier rules + +- Keep imports organized according to oxfmt rules - Use meaningful variable and function names - Add proper JSDoc comments for public APIs - Prefer composition over inheritance -- Use readonly arrays and objects where appropriate \ No newline at end of file +- Use readonly arrays and objects where appropriate diff --git a/.github/setup/action.yml b/.github/setup/action.yml index 3fb7ce12..204bf50e 100644 --- a/.github/setup/action.yml +++ b/.github/setup/action.yml @@ -4,7 +4,7 @@ description: "Common setup steps for Actions" runs: using: composite steps: - - uses: pnpm/action-setup@v4 + - uses: pnpm/action-setup@v6 - uses: actions/setup-node@v6 with: node-version-file: ".nvmrc" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee3a2c31..dc9e060b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,10 +74,26 @@ jobs: - name: Typecheck run: pnpm typecheck - pkg-new-release: - needs: [lint, format, typecheck] + detect-package-changes: if: github.event_name == 'pull_request' runs-on: ubuntu-latest + outputs: + packages_changed: ${{ steps.filter.outputs.packages }} + steps: + - uses: actions/checkout@v6 + + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + packages: + - 'packages/**' + - 'pnpm-lock.yaml' + + pkg-new-release: + needs: [lint, format, typecheck, detect-package-changes] + if: github.event_name == 'pull_request' && needs.detect-package-changes.outputs.packages_changed == 'true' + runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -90,4 +106,4 @@ jobs: - name: Build run: pnpm build:pkg - - run: pnpx pkg-pr-new publish ./packages/* + - run: pnpx pkg-pr-new publish --commentWithSha --packageManager=pnpm ./packages/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff368e97..21f53b9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,8 @@ jobs: steps: - uses: actions/checkout@v6 with: - persist-credentials: false + token: ${{ secrets.CHANGESETS_TOKEN }} + persist-credentials: true - name: Setup uses: ./.github/setup @@ -32,4 +33,3 @@ jobs: publish: pnpm ci:publish env: GITHUB_TOKEN: ${{ secrets.CHANGESETS_TOKEN }} - NPM_TOKEN: "" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e2579bfa..f16c6b9e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,11 +29,31 @@ env: FORCE_COLOR: 3 jobs: + detect-package-changes: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + outputs: + packages_changed: ${{ steps.filter.outputs.packages }} + steps: + - uses: actions/checkout@v6 + + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + packages: + - 'packages/**' + - 'pnpm-lock.yaml' + node_versions: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') runs-on: ubuntu-latest strategy: matrix: - node-version: [20, 22, 24] + node-version: [22, 24, 26] name: Test (Node.js ${{ matrix.node-version }}) steps: - uses: actions/checkout@v6 @@ -44,7 +64,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6 with: run_install: false @@ -79,7 +99,31 @@ jobs: # Ensure Docker is available for Testcontainers TESTCONTAINERS_RYUK_DISABLED: true + - name: Test ZenStack + run: pnpm test:zenstack + env: + TESTCONTAINERS_RYUK_DISABLED: true + + - name: Test TypeORM + run: pnpm test:typeorm + env: + TESTCONTAINERS_RYUK_DISABLED: true + + - name: Test MikroORM + run: pnpm test:mikroorm + env: + TESTCONTAINERS_RYUK_DISABLED: true + + - name: Test Sequelize + run: pnpm test:sequelize + env: + TESTCONTAINERS_RYUK_DISABLED: true + postgres-drizzle: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') runs-on: ubuntu-latest strategy: matrix: @@ -92,7 +136,7 @@ jobs: uses: ./.github/setup - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6 with: run_install: false @@ -111,6 +155,10 @@ jobs: PG_VERSION: ${{ matrix.pg-version }} postgres-prisma: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') runs-on: ubuntu-latest strategy: matrix: @@ -123,7 +171,7 @@ jobs: uses: ./.github/setup - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6 with: run_install: false @@ -144,6 +192,10 @@ jobs: PG_VERSION: ${{ matrix.pg-version }} postgres-kysely: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') runs-on: ubuntu-latest strategy: matrix: @@ -156,7 +208,7 @@ jobs: uses: ./.github/setup - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@v6 with: run_install: false @@ -173,3 +225,139 @@ jobs: # Ensure Docker is available for Testcontainers TESTCONTAINERS_RYUK_DISABLED: true PG_VERSION: ${{ matrix.pg-version }} + + postgres-zenstack: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') + runs-on: ubuntu-latest + strategy: + matrix: + pg-version: [12, 13, 14, 15, 16, 17] + name: Test ZenStack Adapter (Postgres ${{ matrix.pg-version }}) + steps: + - uses: actions/checkout@v6 + + - name: Setup + uses: ./.github/setup + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + run_install: false + + - shell: bash + run: pnpm install + + - name: Build package + run: | + pnpm --filter "./packages/*" build + + - name: Test PostgreSQL (Postgres ${{ matrix.pg-version }}) + run: pnpm test:zenstack + env: + TESTCONTAINERS_RYUK_DISABLED: true + PG_VERSION: ${{ matrix.pg-version }} + + postgres-typeorm: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') + runs-on: ubuntu-latest + strategy: + matrix: + pg-version: [12, 13, 14, 15, 16, 17] + name: Test TypeORM Adapter (Postgres ${{ matrix.pg-version }}) + steps: + - uses: actions/checkout@v6 + + - name: Setup + uses: ./.github/setup + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + run_install: false + + - shell: bash + run: pnpm install + + - name: Build package + run: | + pnpm --filter "./packages/*" build + + - name: Test PostgreSQL (Postgres ${{ matrix.pg-version }}) + run: pnpm test:typeorm + env: + TESTCONTAINERS_RYUK_DISABLED: true + PG_VERSION: ${{ matrix.pg-version }} + + postgres-mikroorm: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') + runs-on: ubuntu-latest + strategy: + matrix: + pg-version: [12, 13, 14, 15, 16, 17] + name: Test MikroORM Adapter (Postgres ${{ matrix.pg-version }}) + steps: + - uses: actions/checkout@v6 + + - name: Setup + uses: ./.github/setup + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + run_install: false + + - shell: bash + run: pnpm install + + - name: Build package + run: | + pnpm --filter "./packages/*" build + + - name: Test PostgreSQL (Postgres ${{ matrix.pg-version }}) + run: pnpm test:mikroorm + env: + TESTCONTAINERS_RYUK_DISABLED: true + PG_VERSION: ${{ matrix.pg-version }} + + postgres-sequelize: + needs: [detect-package-changes] + if: | + always() && + (needs.detect-package-changes.result == 'skipped' || needs.detect-package-changes.outputs.packages_changed == 'true') + runs-on: ubuntu-latest + strategy: + matrix: + pg-version: [12, 13, 14, 15, 16, 17] + name: Test Sequelize Adapter (Postgres ${{ matrix.pg-version }}) + steps: + - uses: actions/checkout@v6 + + - name: Setup + uses: ./.github/setup + + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + run_install: false + + - shell: bash + run: pnpm install + + - name: Build package + run: | + pnpm --filter "./packages/*" build + + - name: Test PostgreSQL (Postgres ${{ matrix.pg-version }}) + run: pnpm test:sequelize + env: + TESTCONTAINERS_RYUK_DISABLED: true + PG_VERSION: ${{ matrix.pg-version }} diff --git a/.gitignore b/.gitignore index dc496141..dde66633 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,4 @@ yarn-error.log* # turbo .turbo -apps/docs/public/pagefind/ \ No newline at end of file +.renoun \ No newline at end of file diff --git a/.kiro/settings/mcp.json b/.kiro/settings/mcp.json new file mode 100644 index 00000000..85839b65 --- /dev/null +++ b/.kiro/settings/mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "shadcn": { + "command": "npx", + "args": ["shadcn@latest", "mcp"], + "disabled": false + } + } +} diff --git a/.kiro/specs/flow-producer-v1/design.md b/.kiro/specs/flow-producer-v1/design.md new file mode 100644 index 00000000..e9da91f0 --- /dev/null +++ b/.kiro/specs/flow-producer-v1/design.md @@ -0,0 +1,540 @@ +# FlowProducer V1 — Design + +## Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ User Code │ +├─────────────────────────────────────────────────────────────┤ +│ FlowProducer Queue Worker │ +│ ───────────── ───── ────── │ +│ add() add() register() │ +│ addChain() cancel() start() / stop() │ +│ addBulkThen() getStats() promotes flow nodes │ +│ getFlow() after job complete │ +│ getFlows() │ +├─────────────────────────────────────────────────────────────┤ +│ FlowAdapter + QueueAdapter │ +├─────────────────────────────────────────────────────────────┤ +│ ┌──────────────┐ ┌──────────────┐ │ +│ │ queue_flows │ │ queue_jobs │ (same DB/schema) │ +│ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Database Schema + +### `queue_flows` table + +| Column | Type | Default | Description | +| --- | --- | --- | --- | +| `id` | UUID | gen_random_uuid() | Primary key | +| `flow_id` | UUID | NOT NULL | Groups all nodes in a flow | +| `parent_node_id` | UUID / NULL | NULL | FK to `queue_flows.id` — tree structure | +| `job_id` | UUID / NULL | NULL | FK to `queue_jobs.id` — set when node is promoted | +| `queue_name` | VARCHAR(255) | NOT NULL | Target queue for this node's job | +| `name` | VARCHAR(255) | NOT NULL | Handler name | +| `payload` | JSONB | NOT NULL | Job payload (stored until job creation) | +| `options` | JSONB / NULL | NULL | JobOptions (priority, timeout, etc.) | +| `status` | VARCHAR(50) | NOT NULL | `waiting`, `ready`, `completed`, `failed`, `cancelled` | +| `failure_strategy` | VARCHAR(20) | `'default'` | `default`, `fail-parent`, `continue-parent` | +| `children_count` | INT | 0 | Number of direct children | +| `children_completed` | INT | 0 | Direct children that reached terminal state | +| `result` | JSONB / NULL | NULL | Job result after completion | +| `error` | JSONB / NULL | NULL | Error info when failed | +| `created_at` | TIMESTAMPTZ | now() | Creation timestamp | +| `completed_at` | TIMESTAMPTZ / NULL | NULL | When node reached terminal state | + +**Indexes:** + +- `idx_queue_flows_flow_id` ON (`flow_id`) +- `idx_queue_flows_parent_node_id` ON (`parent_node_id`) +- `idx_queue_flows_job_id` ON (`job_id`) — for Worker lookup after job completes +- `idx_queue_flows_status` ON (`flow_id`, `status`) + +### `queue_jobs` table changes + +**Removed columns:** + +- `parent_id` +- `flow_id` +- `children_count` +- `children_completed` +- `fail_parent_on_failure` +- `depends_on` +- `on_dependency_failure` + +**Added column:** + +- `flow_node_id` UUID / NULL — backlink to `queue_flows.id` + +## Flow Node Status Lifecycle + +``` +┌──────────┐ All children terminal ┌───────┐ Job created ┌───────────┐ +│ waiting │ ────────────────────────────── │ ready │ ────────────────── │ completed │ +└──────────┘ └───────┘ └───────────┘ + │ │ + │ fail-parent cascade │ Job handler fails + │ OR all children terminal with error │ (terminal failure) + ▼ ▼ +┌──────────┐ ┌──────────┐ +│ failed │ │ failed │ +└──────────┘ └──────────┘ + + │ removeUnprocessedChildren + ▼ +┌───────────┐ +│ cancelled │ +└───────────┘ +``` + +**Status definitions:** + +- `waiting` — Node is waiting for children to complete (parent nodes start here) +- `ready` — Node has been promoted; a job exists in `queue_jobs` (leaf nodes start here) +- `completed` — Node's job completed successfully +- `failed` — Node failed (handler failure, or `fail-parent` cascade with no job created) +- `cancelled` — Node was cancelled via `removeUnprocessedChildren()` + +## Core Interfaces + +### FlowAdapter + +```typescript +/** Persistence interface for flow orchestration data. */ +interface FlowAdapter { + /** Create all flow nodes (and leaf jobs) in a single transaction. */ + createFlow( + nodes: readonly NewFlowNode[], + leafJobs: readonly NewJob[] + ): Promise + + /** Get a flow node by ID. */ + getFlowNode(nodeId: string): Promise + + /** Get the full flow tree by flow ID. */ + getFlowTree(flowId: string): Promise + + /** List flows with pagination and optional status filter. */ + getFlows(options?: FlowListOptions): Promise + + /** Update a flow node's status and optional metadata. */ + updateFlowNode(nodeId: string, update: FlowNodeUpdate): Promise + + /** Increment children_completed on a parent node. Returns updated counts. */ + incrementNodeChildrenCompleted( + nodeId: string + ): Promise<{ completed: number; total: number }> + + /** Get direct children of a node. */ + getNodeChildren(nodeId: string): Promise + + /** Get results of all completed child nodes. */ + getChildrenResults(nodeId: string): Promise> + + /** Get errors of all failed child nodes. */ + getFailedChildrenResults( + nodeId: string + ): Promise> + + /** Cancel all unprocessed children (pending/delayed jobs) and their subtrees. */ + cancelUnprocessedChildren(nodeId: string): Promise + + /** Delete all nodes in a flow. */ + deleteFlow(flowId: string): Promise + + /** Cleanup completed flows, keeping only N most recent. */ + cleanupFlows(keepCount: number): Promise +} +``` + +### Flow Types + +```typescript +/** Status of a flow node. */ +type FlowNodeStatus = "waiting" | "ready" | "completed" | "failed" | "cancelled" + +/** Failure strategy for a child node. */ +type FlowFailureStrategy = "default" | "fail-parent" | "continue-parent" + +/** A flow node record as stored in the database. */ +interface FlowNode { + readonly id: string + readonly flowId: string + readonly parentNodeId?: string + readonly jobId?: string + readonly queueName: string + readonly name: string + readonly payload: unknown + readonly options?: JobOptions + readonly status: FlowNodeStatus + readonly failureStrategy: FlowFailureStrategy + readonly childrenCount: number + readonly childrenCompleted: number + readonly result?: unknown + readonly error?: SerializedError + readonly createdAt: Date + readonly completedAt?: Date +} + +/** Input for creating a flow node (no id/createdAt). */ +type NewFlowNode = Omit< + FlowNode, + "id" | "createdAt" | "completedAt" | "result" | "error" +> + +/** Tree representation returned by getFlowTree. */ +interface FlowTree { + readonly node: FlowNode + readonly children: readonly FlowTree[] +} + +/** Summary for flow listing. */ +interface FlowSummary { + readonly flowId: string + readonly rootNode: FlowNode + readonly status: FlowNodeStatus + readonly createdAt: Date + readonly completedAt?: Date +} + +/** Options for listing flows. */ +interface FlowListOptions { + readonly status?: FlowNodeStatus + readonly limit?: number + readonly offset?: number +} +``` + +### FlowProducer Input Types + +```typescript +/** Input for flowProducer.add() — defines a node in the flow tree. */ +interface FlowJobDefinition { + readonly name: string + readonly queueName: string + readonly payload: unknown + readonly options?: JobOptions + readonly failureStrategy?: FlowFailureStrategy + readonly children?: readonly FlowJobDefinition[] +} + +/** Input for flowProducer.addChain() — a sequential step. */ +interface FlowChainStep { + readonly name: string + readonly queueName: string + readonly payload: unknown + readonly options?: JobOptions +} + +/** Result of flow creation. */ +interface FlowResult { + readonly flowId: string + readonly rootNode: FlowNode +} + +/** Result of addChain. */ +interface FlowChainResult { + readonly flowId: string + readonly nodes: readonly FlowNode[] +} + +/** Result of addBulkThen. */ +interface FlowBulkThenResult { + readonly flowId: string + readonly parallelNodes: readonly FlowNode[] + readonly finalNode: FlowNode +} +``` + +## FlowProducer Class + +```typescript +class FlowProducer extends TypedEventEmitter { + constructor(adapter: FlowAdapter & QueueAdapter, config?: FlowProducerConfig) + + /** Create a parent-child tree. Children complete before parent. */ + async add(definition: FlowJobDefinition): Promise + + /** Create a sequential chain. A → B → C. */ + async addChain(steps: readonly FlowChainStep[]): Promise + + /** Create a fan-in flow. [A, B, C] → D. */ + async addBulkThen( + parallel: readonly FlowChainStep[], + final: FlowChainStep + ): Promise + + /** Get a flow tree by ID. */ + async getFlow(flowId: string): Promise + + /** List flows with pagination and status filter. */ + async getFlows(options?: FlowListOptions): Promise +} +``` + +**Config:** + +```typescript +interface FlowProducerConfig { + /** Remove completed flows: true = immediate, number = keep N. + * @default 100 + */ + readonly removeOnComplete?: boolean | number +} +``` + +**Events:** + +```typescript +interface FlowProducerEvents { + "flow:created": FlowResult + "flow:completed": { flowId: string; result: unknown } + "flow:failed": { flowId: string; error: SerializedError } + "flow:node:promoted": { flowId: string; nodeId: string; jobId: string } +} +``` + +## Worker Integration + +### How the Worker promotes flow nodes + +After a job completes or fails terminally, the Worker checks if the job belongs to a flow: + +``` +Job completed/failed + → Has flowNodeId? No → done + → Yes: update flow_node (status, result/error) + → Get parent_node_id from this flow_node + → No parent? This is root → emit flow:completed/flow:failed, cleanup + → Has parent: increment parent.children_completed + → Check: is parent promotable? + → All children terminal (completed/failed)? → promote parent + → Child has "continue-parent" and just failed? → promote parent immediately + → Child has "fail-parent" and just failed? → fail parent (no job), cascade up +``` + +### Promotion logic + +When a parent node is promoted: + +1. Create a real job in `queue_jobs` with `flowNodeId` set +2. Set flow_node.status = `"ready"`, flow_node.job_id = new job ID +3. Emit `flow:node:promoted` + +### JobContext additions (FlowJobContext) + +```typescript +/** Filter criteria for querying children of a flow node. */ +interface ChildrenFilter { + /** Filter by child node name(s). */ + readonly name?: string | readonly string[] + /** Filter by child node status(es). */ + readonly status?: FlowNodeStatus | readonly FlowNodeStatus[] +} + +/** Value returned per child node from getChildrenValuesBy. */ +interface ChildNodeValue { + readonly nodeId: string + readonly name: string + readonly status: "completed" | "failed" | "cancelled" + readonly result?: unknown + readonly error?: SerializedError +} + +interface FlowJobContext { + /** Results of completed children. Shortcut for getChildrenValuesBy({ status: "completed" }). */ + getChildrenValues: () => Promise> + + /** Errors from failed children. Shortcut for getChildrenValuesBy({ status: "failed" }). */ + getFailedChildrenValues: () => Promise> + + /** Flexible query for children by filter criteria. */ + getChildrenValuesBy: ( + filter: ChildrenFilter + ) => Promise> + + /** Cancel all unprocessed children and their subtrees. */ + removeUnprocessedChildren: () => Promise +} +``` + +Internally, `getChildrenValues()` delegates to `getChildrenValuesBy({ status: "completed" })` and extracts only the `result` field. `getFailedChildrenValues()` delegates to `getChildrenValuesBy({ status: "failed" })` and extracts only the `error` field. + +The `ChildrenFilter` object is extensible — additional filter fields (e.g. `queueName`) can be added in the future without breaking changes. + +These methods are only populated when the job has a `flowNodeId`. For non-flow jobs they return empty maps / 0. + +## addChain and addBulkThen internals + +### addChain([A, B, C]) + +Transforms into a nested parent-child tree (deepest = first to run): + +``` +C (root — runs last) +└── B + └── A (leaf — runs first) +``` + +This is the same pattern BullMQ uses for sequential chains. + +### addBulkThen([A, B, C], D) + +Transforms into: + +``` +D (root — runs after all parallel jobs complete) +├── A (leaf) +├── B (leaf) +└── C (leaf) +``` + +This is a simple parent with multiple children. + +## Changes to Existing Code + +### Job interface (removals) + +```diff +interface Job { + // ... kept fields ... +- readonly dependsOn?: readonly string[] +- readonly onDependencyFailure?: "fail" | "cancel" +- readonly parentId?: string +- readonly flowId?: string +- readonly childrenCount?: number +- readonly childrenCompleted?: number +- readonly failParentOnFailure?: boolean ++ readonly flowNodeId?: string +} +``` + +### JobStatus (removal) + +```diff +type JobStatus = + | "pending" + | "delayed" + | "processing" + | "completed" + | "failed" + | "cancelled" + | "dead" +- | "waiting-children" +``` + +### STATE_TRANSITIONS (removal) + +```diff +const STATE_TRANSITIONS = { + // ... +- "waiting-children": ["pending", "cancelled", "failed"], +} +``` + +### QueueStats (removal) + +```diff +interface QueueStats { + // ... +- readonly "waiting-children": number +} +``` + +### QueueAdapter (removals) + +```diff +interface QueueAdapter { + // ... kept methods ... +- getFlowTree: (flowId: string) => Promise +- deleteFlow: (flowId: string) => Promise +- incrementChildrenCompleted: (parentId: string) => Promise<{ completed: number; total: number }> +- getChildrenJobs: (parentId: string) => Promise +- getFlows: (options?: PaginationOptions) => Promise +} +``` + +### Files to delete + +- `packages/core/src/dependencies.ts` — entire module (circular detection, areDependenciesMet, cascadeDependencyFailure) + +### Files to create + +- `packages/core/src/flow-producer.ts` — FlowProducer class +- `packages/core/src/flow-types.ts` — Flow-specific type definitions (to keep types.ts manageable) + +### Worker changes + +- Remove `promoteParentIfReady` (replaced with new flow-aware promotion) +- Remove `failParentOnChildFailure` +- Remove `filterByDependencies` logic +- Add `promoteFlowNode` — new method that works against FlowAdapter +- Add `handleFlowNodeFailure` — handles fail-parent cascade + +### Queue changes + +- Remove `addFlow`, `getFlowTree`, `createFlowNode` methods +- Remove `detectCircularDependencies` import and usage + +## Adapter Configuration + +### Extended adapter props + +```typescript +interface PrismaAdapterProps { + modelName?: string // default "QueueJob" + tableName?: string // default "queue_jobs" + schemaName?: string // default undefined (public) + flowModelName?: string // default "QueueFlow" + flowTableName?: string // default "queue_flows" +} + +interface KyselyAdapterProps { + tableName?: string // default "queue_jobs" + schemaName?: string // default "public" + flowTableName?: string // default "queue_flows" +} + +interface DrizzleAdapterProps { + modelName?: string // default "queueJobs" + flowModelName?: string // default "queueFlows" +} + +interface ZenstackAdapterProps { + modelName?: string // default "queueJob" + tableName?: string // default "queue_jobs" + schemaName?: string // default undefined (public) + flowModelName?: string // default "queueFlow" + flowTableName?: string // default "queue_flows" +} + +interface TypeormAdapterProps { + tableName?: string // default "queue_jobs" + schemaName?: string // default undefined (public) + flowTableName?: string // default "queue_flows" +} + +interface MikroormAdapterProps { + tableName?: string // default "queue_jobs" + schemaName?: string // default undefined (public) + flowTableName?: string // default "queue_flows" +} + +interface SequelizeAdapterProps { + tableName?: string // default "queue_jobs" + schemaName?: string // default undefined (public) + flowTableName?: string // default "queue_flows" +} +``` + +## Migration Path (for documentation) + +Since Flows, `dependsOn`, and `waiting-children` never existed in a published V0 release, there is no user-facing migration from old flow APIs. + +The V1 migration guide for users only needs: + +1. Add column to `queue_jobs`: `flow_node_id UUID NULL` +2. Create the `queue_flows` table (schema provided by each adapter package) +3. Remove any columns that were added during the refactor branch but never shipped: `parent_id`, `flow_id`, `children_count`, `children_completed`, `fail_parent_on_failure`, `depends_on`, `on_dependency_failure` diff --git a/.kiro/specs/flow-producer-v1/requirements.md b/.kiro/specs/flow-producer-v1/requirements.md new file mode 100644 index 00000000..55b0c844 --- /dev/null +++ b/.kiro/specs/flow-producer-v1/requirements.md @@ -0,0 +1,168 @@ +# FlowProducer V1 — Requirements + +## Context + +The current flow implementation in vorsteh-queue treats flow parents as regular jobs with a `waiting-children` status. This leads to problems: + +- Parent jobs inherit default retry/timeout settings (maxAttempts: 3, timeout: 30000) which make no sense for a job that only waits for children. +- Flow state is mixed into the job table, creating entries that no worker ever picks up. +- The `dependsOn` / `onDependencyFailure` fields on jobs overlap with flow functionality. + +This spec redesigns the flow system to cleanly separate flow orchestration from job processing. + +## Functional Requirements + +### FR-1: FlowProducer as a separate class + +The `FlowProducer` must be a standalone class exported from `@vorsteh-queue/core`. It must not be part of the `Queue` class. + +### FR-2: 2-table architecture + +Flow orchestration data must live in a separate `queue_flows` table (default name). The `jobs` table must only contain actual work units. The `waiting-children` status must be removed from the job state machine. + +### FR-3: Parent jobs are not created until direct children complete + +A parent flow node must not create a real job entry until all its **direct** children (not recursive) have reached a terminal state (completed/failed). Until then, it exists only as a `queue_flows` record. Grandchildren are tracked by their own parent nodes — promotion is always based on direct children only. + +### FR-4: Leaf nodes create jobs immediately + +Leaf nodes (no children) must create a real job in the `queue_jobs` table immediately when the flow is added. This happens within the same transaction as the flow node creation (see FR-6). + +### FR-5: Cross-queue support + +Each flow node must have a `queueName` property, allowing nodes in the same flow to target different queues. + +### FR-6: Atomic flow creation + +All flow nodes and their corresponding leaf jobs must be created in a single database transaction. If any part fails, the entire flow (both `queue_flows` entries and `queue_jobs` entries) is rolled back. This requires both tables to reside in the same database schema (see FR-16). + +### FR-7: BullMQ-compatible API + +- `flowProducer.add(tree)` — parent-child tree (children complete before parent) +- `flowProducer.addChain(steps)` — sequential execution (A → B → C) +- `flowProducer.addBulkThen(parallel, final)` — fan-in (parallel jobs → merge job) +- `flowProducer.getFlow(flowId)` — retrieve flow tree with status +- `flowProducer.getFlows(options)` — list flows with pagination and status filter + +### FR-8: Failure strategies per child node + +Each child node must support a `failureStrategy` option: + +- `"default"` — Parent is promoted when ALL direct children reach a terminal state (completed or failed). Parent handler receives information about failed children via `ctx.getFailedChildrenValues()`. +- `"fail-parent"` — Parent flow node is immediately set to `failed` status. No job is created for the parent. The failure reason references the child that triggered it. Cascades recursively upward if ancestor nodes also have children with `"fail-parent"`. +- `"continue-parent"` — Parent is immediately promoted (job created), even while other children are still running. Parent handler decides what to do. + +### FR-9: Parent handler context + +When a parent job is created and processed, the handler context must provide: + +- `ctx.getChildrenValues()` — results of completed children (convenience for `getChildrenValuesBy({ status: "completed" })`) +- `ctx.getFailedChildrenValues()` — errors from failed children (convenience for `getChildrenValuesBy({ status: "failed" })`) +- `ctx.getChildrenValuesBy(filter)` — flexible query accepting a filter object with optional `name` (string or array) and `status` (single or array of `"completed"`, `"failed"`, `"cancelled"`). Returns a map with nodeId, name, status, result, and error per child. The filter object is extensible for future fields. +- `ctx.removeUnprocessedChildren()` — cancel all children (and their subtrees) whose jobs are still in `pending`/`delayed` state. Active, completed, and failed children remain unaffected. + +### FR-10: Remove `dependsOn` and `onDependencyFailure` from Job + +These fields and their associated logic (the `dependencies.ts` module) must be removed. Job dependencies are expressed exclusively via flows or triggers. + +### FR-11: Remove flow fields from Job interface + +The following fields must be removed from `Job`: + +- `parentId` +- `flowId` +- `childrenCount` +- `childrenCompleted` +- `failParentOnFailure` +- `waiting-children` status + +A single `flowNodeId?: string` backlink remains for the worker to identify flow membership. + +### FR-12: Remove `waiting-children` from state machine + +The `JobStatus` type must only contain: `pending`, `delayed`, `processing`, `completed`, `failed`, `cancelled`, `dead`. + +### FR-13: Separate `FlowAdapter` interface + +The flow persistence methods must be defined in a dedicated `FlowAdapter` interface, separate from `QueueAdapter`. Concrete adapter classes implement both interfaces. Users who do not need flows only need to satisfy `QueueAdapter`. + +### FR-14: Configurable table and model names + +The `queue_flows` table name must be configurable per adapter: + +- Drizzle: `flowTableName` option (default: `"queue_flows"`) +- Kysely: `flowTableName` option (default: `"queue_flows"`) +- Prisma: `flowTableName` + `flowModelName` options (defaults: `"queue_flows"` / `"QueueFlow"`) +- TypeORM: `flowTableName` option (default: `"queue_flows"`) +- MikroORM: `flowTableName` option (default: `"queue_flows"`) +- Sequelize: `flowTableName` option (default: `"queue_flows"`) +- ZenStack: `flowTableName` + `flowModelName` options (defaults: `"queue_flows"` / `"QueueFlow"`) + +### FR-15: Default failure behavior promotes parent + +When no explicit `failureStrategy` is set and a child fails terminally (no retries left), the parent is NOT blocked indefinitely. The parent is promoted once ALL direct children have reached a terminal state (completed or failed). The parent handler can then inspect `ctx.getFailedChildrenValues()` to determine partial success. + +### FR-16: Same database, schema, and connection pool + +The `queue_flows` table must reside in the same database AND the same schema as `queue_jobs`. They must share the same connection pool / client instance. This is a hard requirement to enable cross-table transactions (FR-6). This constraint must be documented clearly for users. + +### FR-17: Flow lifecycle cleanup + +Flow nodes must support configurable cleanup after the root node completes: + +- `removeOnComplete: true` — delete all flow nodes immediately after root completion +- `removeOnComplete: false` — keep flow nodes indefinitely (for observability/audit) +- `removeOnComplete: number` — keep the N most recent completed flows, delete older ones + +This mirrors the existing job cleanup behavior. + +### FR-18: Flow events + +The `FlowProducer` must emit events for observability: + +- `flow:created` — when a new flow is created +- `flow:completed` — when the root node's job completes successfully +- `flow:failed` — when the root node is marked as failed (either via handler failure or `fail-parent` cascade) +- `flow:node:promoted` — when a parent node transitions from waiting to ready (job created) + +## Non-Functional Requirements + +### NFR-1: No backward compatibility required + +Since this is a V1 breaking change, no backward compatibility with V0 flow APIs is required. Old code can be deleted directly. + +### NFR-2: All 7 adapters must implement FlowAdapter + +All adapter packages must implement the `FlowAdapter` interface: + +- `@vorsteh-queue/adapter-drizzle` +- `@vorsteh-queue/adapter-prisma` +- `@vorsteh-queue/adapter-kysely` +- `@vorsteh-queue/adapter-zenstack` +- `@vorsteh-queue/adapter-typeorm` +- `@vorsteh-queue/adapter-mikroorm` +- `@vorsteh-queue/adapter-sequelize` + +### NFR-3: Shared test coverage + +The `shared-tests` package must contain flow tests that all adapters run. + +### NFR-4: Documentation completeness + +All public APIs must have JSDoc documentation following project conventions. The same-schema constraint (FR-16) must be prominently documented. + +### NFR-5: Telemetry extension for Flows + +The `Telemetry` interface (or a separate `FlowTelemetry` interface) must be extended to support flow-level observability: + +- Metrics: flows created, completed, failed, active (gauge), duration (histogram), nodes promoted +- Tracing: a parent span per flow that links all child-job spans for end-to-end visibility + +This can be implemented after the core flow logic is stable, but the design must not preclude it. + +## Out of Scope + +- DAGs (directed acyclic graphs) with arbitrary edges — only tree structures +- Dynamic child spawning after flow creation +- Flow-level retry (re-running entire flow on failure) +- The step engine (saga, sleep, waitFor) — stays as-is within individual jobs diff --git a/.kiro/specs/flow-producer-v1/tasks.md b/.kiro/specs/flow-producer-v1/tasks.md new file mode 100644 index 00000000..4e247c07 --- /dev/null +++ b/.kiro/specs/flow-producer-v1/tasks.md @@ -0,0 +1,126 @@ +# FlowProducer V1 — Implementation Tasks + +## Phase 1: Core Cleanup (Remove old flow/dependency code) + +- [x] 1. Remove `waiting-children` from `JobStatus` type and `STATE_TRANSITIONS` in `types.ts` +- [x] 2. Remove flow fields from `Job` interface: `parentId`, `flowId`, `childrenCount`, `childrenCompleted`, `failParentOnFailure` +- [x] 3. Remove `dependsOn` and `onDependencyFailure` from `Job` interface and `JobOptions` +- [x] 4. Remove `QueueStats["waiting-children"]` +- [x] 5. Delete `packages/core/src/dependencies.ts` and remove its exports from `index.ts` +- [x] 6. Remove flow methods from `QueueAdapter` interface: `getFlowTree`, `deleteFlow`, `incrementChildrenCompleted`, `getChildrenJobs`, `getFlows` +- [x] 7. Remove `addFlow`, `getFlowTree`, `createFlowNode` from `Queue` class +- [x] 8. Remove `promoteParentIfReady`, `failParentOnChildFailure`, and `filterByDependencies` from `Worker` class +- [x] 9. Remove old `FlowJobDefinition`, `FlowNode`, `FlowResult` types from `types.ts` +- [x] 10. Add `flowNodeId?: string` to `Job` interface +- [x] 11. Update `NewJob` type to reflect removed/added fields +- [x] 12. Fix all TypeScript compilation errors in core package after removals + +## Phase 2: New Flow Types and Interfaces + +- [x] 13. Create `packages/core/src/flow-types.ts` with: `FlowNodeStatus`, `FlowFailureStrategy`, `FlowNode`, `NewFlowNode`, `FlowTree`, `FlowSummary`, `FlowListOptions`, `FlowJobDefinition`, `FlowChainStep`, `FlowResult`, `FlowChainResult`, `FlowBulkThenResult`, `FlowProducerConfig`, `FlowProducerEvents`, `ChildrenFilter`, `ChildNodeValue`, `FlowJobContext` +- [x] 14. Create `FlowAdapter` interface in `packages/core/src/flow-types.ts` +- [x] 15. Update adapter props types (`PrismaAdapterProps`, `DrizzleAdapterProps`, etc.) with `flowTableName`/`flowModelName` fields +- [x] 16. Export new flow types from `packages/core/src/index.ts` + +## Phase 3: FlowProducer Class + +- [x] 17. Create `packages/core/src/flow-producer.ts` with `FlowProducer` class extending `TypedEventEmitter` +- [x] 18. Implement `FlowProducer.add(definition)` — creates flow tree, leaf jobs in transaction +- [x] 19. Implement `FlowProducer.addChain(steps)` — transforms to nested parent-child tree +- [x] 20. Implement `FlowProducer.addBulkThen(parallel, final)` — transforms to parent with N children +- [x] 21. Implement `FlowProducer.getFlow(flowId)` — retrieves full tree +- [x] 22. Implement `FlowProducer.getFlows(options)` — list with pagination/status filter +- [x] 23. Implement flow cleanup logic (removeOnComplete configuration) +- [x] 24. Export `FlowProducer` from `packages/core/src/index.ts` + +## Phase 4: Worker Integration + +- [x] 25. Add flow-node promotion logic to Worker: after job completes, check `flowNodeId`, update node, check parent promotability +- [x] 26. Implement `fail-parent` cascade: when child with `fail-parent` strategy fails terminally, fail parent node recursively +- [x] 27. Implement `continue-parent` promotion: when child with `continue-parent` fails, promote parent immediately +- [x] 28. Implement default strategy: promote parent when all direct children are terminal +- [x] 29. Implement `FlowJobContext` methods in Worker's job context: `getChildrenValues`, `getFailedChildrenValues`, `getChildrenValuesBy`, `removeUnprocessedChildren` +- [x] 30. Emit flow events from Worker: `flow:completed`, `flow:failed`, `flow:node:promoted` + +## Phase 5: Memory Adapter (reference implementation) + +- [x] 31. Remove old flow methods from `MemoryQueueAdapter`: `getFlowTree`, `deleteFlow`, `incrementChildrenCompleted`, `getChildrenJobs`, `getFlows` +- [x] 32. Remove old flow fields from memory adapter's job storage/mapping +- [x] 33. Add `flowNodeId` support to memory adapter's `addJob`/`getNextJob` +- [x] 34. Implement `FlowAdapter` interface in `MemoryQueueAdapter` with in-memory `flow_nodes` Map +- [x] 35. Implement all `FlowAdapter` methods: `createFlow`, `getFlowNode`, `getFlowTree`, `getFlows`, `updateFlowNode`, `incrementNodeChildrenCompleted`, `getNodeChildren`, `getChildrenResults`, `getFailedChildrenResults`, `cancelUnprocessedChildren`, `deleteFlow`, `cleanupFlows` + +## Phase 6: Drizzle Adapter + +- [x] 36. Update `postgres-schema.ts`: remove old flow columns (`parent_id`, `flow_id`, `children_count`, `children_completed`, `fail_parent_on_failure`, `depends_on`, `on_dependency_failure`), add `flow_node_id` +- [x] 37. Create `flow-schema.ts` with `queueFlows` table definition +- [x] 38. Update adapter class: remove old flow methods, remove old field mappings in `transformJob`/`addJob`/`addJobs` +- [x] 39. Implement `FlowAdapter` interface in Drizzle adapter with transactional `createFlow` +- [x] 40. Update adapter config to accept `flowModelName` option +- [x] 41. Export new flow schema from package + +## Phase 7: Prisma Adapter + +- [x] 42. Update `schema.prisma`: remove old flow columns, add `flowNodeId`, add `QueueFlow` model with `@@map("queue_flows")` +- [x] 43. Regenerate Prisma client +- [x] 44. Update adapter class: remove old flow methods, remove old field mappings in `transformPrismaJob`/`addJob` +- [x] 45. Implement `FlowAdapter` interface in Prisma adapter with transactional `createFlow` +- [x] 46. Update adapter config to accept `flowModelName` and `flowTableName` options + +## Phase 8: Kysely Adapter + +- [x] 47. Update table type definition: remove old flow columns, add `flow_node_id` +- [x] 48. Create `queue_flows` table type definition +- [x] 49. Update adapter class: remove old flow methods, remove old field mappings in `transformJob`/`addJob` +- [x] 50. Implement `FlowAdapter` interface in Kysely adapter with transactional `createFlow` +- [x] 51. Update adapter config to accept `flowTableName` option + +## Phase 9: ZenStack Adapter + +- [x] 52. Update ZenStack schema: remove old flow columns, add `flowNodeId`, add `QueueFlow` model +- [x] 53. Update adapter class: remove old flow methods, implement `FlowAdapter` +- [x] 54. Update adapter config to accept `flowModelName` and `flowTableName` options + +## Phase 10: TypeORM Adapter + +- [x] 55. Update entity definition: remove old flow columns, add `flowNodeId` +- [x] 56. Create `QueueFlow` entity +- [x] 57. Update adapter class: remove old flow methods, implement `FlowAdapter` +- [x] 58. Update adapter config to accept `flowTableName` option + +## Phase 11: MikroORM Adapter + +- [x] 59. Update entity definition: remove old flow columns, add `flowNodeId` +- [x] 60. Create `QueueFlow` entity +- [x] 61. Update adapter class: remove old flow methods, implement `FlowAdapter` +- [x] 62. Update adapter config to accept `flowTableName` option + +## Phase 12: Sequelize Adapter + +- [x] 63. Update model definition: remove old flow columns, add `flowNodeId` +- [x] 64. Create `QueueFlow` model +- [x] 65. Update adapter class: remove old flow methods, implement `FlowAdapter` +- [x] 66. Update adapter config to accept `flowTableName` option + +## Phase 13: Shared Tests + +- [x] 67. Remove old flow field persistence tests from `shared-tests` +- [x] 68. Remove old `deleteFlow` / `incrementChildrenCompleted` / `getFlowTree` tests +- [x] 69. Add new `FlowAdapter` shared tests: `createFlow` atomicity, `getFlowTree`, `getFlows`, `updateFlowNode`, `incrementNodeChildrenCompleted`, `getNodeChildren`, `cancelUnprocessedChildren`, `deleteFlow`, `cleanupFlows` +- [x] 70. Add flow integration tests: full lifecycle (create → children complete → parent promoted → parent completes), failure strategies, nested flows, cross-queue flows + +## Phase 14: Core Flow E2E Tests + +- [x] 71. Rewrite `packages/core/tests/flow.test.ts`: FlowProducer.add with parent/children, FlowProducer.addChain, FlowProducer.addBulkThen +- [x] 72. Test failure strategies: default (promote on all terminal), fail-parent (cascade), continue-parent (immediate promote) +- [x] 73. Test FlowJobContext: getChildrenValues, getFailedChildrenValues, getChildrenValuesBy, removeUnprocessedChildren +- [x] 74. Test flow cleanup (removeOnComplete) +- [x] 75. Test flow events emission + +## Phase 15: Final Cleanup + +- [x] 76. Remove `packages/core/tests/dependency.test.ts` (if exists) +- [x] 77. Update changeset: replace `workflow-engine.md` with new flow-producer changeset +- [x] 78. Run `pnpm format:fix && pnpm lint:fix && pnpm typecheck` — fix all issues +- [x] 79. Run all adapter tests: `pnpm test:drizzle`, `pnpm test:prisma`, `pnpm test:kysely`, `pnpm test:zenstack`, `pnpm test:typeorm`, `pnpm test:mikroorm`, `pnpm test:sequelize` +- [x] 80. Run core tests: verify all pass diff --git a/.kiro/specs/flow-producer-v1/tasks.meta.json b/.kiro/specs/flow-producer-v1/tasks.meta.json new file mode 100644 index 00000000..f1b867d4 --- /dev/null +++ b/.kiro/specs/flow-producer-v1/tasks.meta.json @@ -0,0 +1,1365 @@ +{ + "pbtResults": {}, + "executionHistory": { + "1. Remove `waiting-children` from `JobStatus` type and `STATE_TRANSITIONS` in `types.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074890908 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074903180 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074950182 + } + ], + "2. Remove flow fields from `Job` interface: `parentId`, `flowId`, `childrenCount`, `childrenCompleted`, `failParentOnFailure`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074890988 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074953448 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074991307 + } + ], + "3. Remove `dependsOn` and `onDependencyFailure` from `Job` interface and `JobOptions`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891162 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074994057 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075032128 + } + ], + "4. Remove `QueueStats[\"waiting-children\"]`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891229 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075034950 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075059557 + } + ], + "5. Delete `packages/core/src/dependencies.ts` and remove its exports from `index.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891298 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075062366 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075115518 + } + ], + "6. Remove flow methods from `QueueAdapter` interface: `getFlowTree`, `deleteFlow`, `incrementChildrenCompleted`, `getChildrenJobs`, `getFlows`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891362 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075118800 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075163702 + } + ], + "7. Remove `addFlow`, `getFlowTree`, `createFlowNode` from `Queue` class": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891423 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075167418 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075248612 + } + ], + "8. Remove `promoteParentIfReady`, `failParentOnChildFailure`, and `filterByDependencies` from `Worker` class": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891498 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075251565 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075395272 + } + ], + "9. Remove old `FlowJobDefinition`, `FlowNode`, `FlowResult` types from `types.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891563 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075398348 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075447033 + } + ], + "10. Add `flowNodeId?: string` to `Job` interface": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891629 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075449860 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075495593 + } + ], + "11. Update `NewJob` type to reflect removed/added fields": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891690 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075498452 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075525885 + } + ], + "12. Fix all TypeScript compilation errors in core package after removals": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891748 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075528719 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075721693 + } + ], + "13. Create `packages/core/src/flow-types.ts` with: `FlowNodeStatus`, `FlowFailureStrategy`, `FlowNode`, `NewFlowNode`, `FlowTree`, `FlowSummary`, `FlowListOptions`, `FlowJobDefinition`, `FlowChainStep`, `FlowResult`, `FlowChainResult`, `FlowBulkThenResult`, `FlowProducerConfig`, `FlowProducerEvents`, `ChildrenFilter`, `ChildNodeValue`, `FlowJobContext`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891806 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075726092 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075827573 + } + ], + "14. Create `FlowAdapter` interface in `packages/core/src/flow-types.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891862 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075830764 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075917416 + } + ], + "15. Update adapter props types (`PrismaAdapterProps`, `DrizzleAdapterProps`, etc.) with `flowTableName`/`flowModelName` fields": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891921 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075920549 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785075999830 + } + ], + "16. Export new flow types from `packages/core/src/index.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074891973 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076002771 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076043849 + } + ], + "17. Create `packages/core/src/flow-producer.ts` with `FlowProducer` class extending `TypedEventEmitter`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892034 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076047612 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076113057 + } + ], + "18. Implement `FlowProducer.add(definition)` — creates flow tree, leaf jobs in transaction": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892098 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076116065 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076326370 + } + ], + "19. Implement `FlowProducer.addChain(steps)` — transforms to nested parent-child tree": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892154 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076329554 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076510687 + } + ], + "20. Implement `FlowProducer.addBulkThen(parallel, final)` — transforms to parent with N children": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892212 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076513835 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076592083 + } + ], + "21. Implement `FlowProducer.getFlow(flowId)` — retrieves full tree": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892267 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076595757 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076598815 + } + ], + "22. Implement `FlowProducer.getFlows(options)` — list with pagination/status filter": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892323 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076601767 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076604873 + } + ], + "23. Implement flow cleanup logic (removeOnComplete configuration)": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892387 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076607858 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076698597 + } + ], + "24. Export `FlowProducer` from `packages/core/src/index.ts`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892442 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076701979 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076742961 + } + ], + "25. Add flow-node promotion logic to Worker: after job completes, check `flowNodeId`, update node, check parent promotability": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892497 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076746766 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076880745 + } + ], + "26. Implement `fail-parent` cascade: when child with `fail-parent` strategy fails terminally, fail parent node recursively": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892557 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785076883922 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077023993 + } + ], + "27. Implement `continue-parent` promotion: when child with `continue-parent` fails, promote parent immediately": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892615 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077026903 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077080196 + } + ], + "28. Implement default strategy: promote parent when all direct children are terminal": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892671 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077084163 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077087345 + } + ], + "29. Implement `FlowJobContext` methods in Worker's job context: `getChildrenValues`, `getFailedChildrenValues`, `getChildrenValuesBy`, `removeUnprocessedChildren`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892725 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077090761 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077235974 + } + ], + "30. Emit flow events from Worker: `flow:completed`, `flow:failed`, `flow:node:promoted`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892798 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077239169 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077386166 + } + ], + "31. Remove old flow methods from `MemoryQueueAdapter`: `getFlowTree`, `deleteFlow`, `incrementChildrenCompleted`, `getChildrenJobs`, `getFlows`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892855 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077390358 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077411250 + } + ], + "32. Remove old flow fields from memory adapter's job storage/mapping": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892913 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077414466 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077446069 + } + ], + "33. Add `flowNodeId` support to memory adapter's `addJob`/`getNextJob`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074892967 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077449284 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077478901 + } + ], + "34. Implement `FlowAdapter` interface in `MemoryQueueAdapter` with in-memory `flow_nodes` Map": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893022 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077482478 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077642819 + } + ], + "35. Implement all `FlowAdapter` methods: `createFlow`, `getFlowNode`, `getFlowTree`, `getFlows`, `updateFlowNode`, `incrementNodeChildrenCompleted`, `getNodeChildren`, `getChildrenResults`, `getFailedChildrenResults`, `cancelUnprocessedChildren`, `deleteFlow`, `cleanupFlows`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893076 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077646522 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077650556 + } + ], + "36. Update `postgres-schema.ts`: remove old flow columns (`parent_id`, `flow_id`, `children_count`, `children_completed`, `fail_parent_on_failure`, `depends_on`, `on_dependency_failure`), add `flow_node_id`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893132 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077655253 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077709280 + } + ], + "37. Create `flow-schema.ts` with `queueFlows` table definition": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893188 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077712486 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077777870 + } + ], + "38. Update adapter class: remove old flow methods, remove old field mappings in `transformJob`/`addJob`/`addJobs`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893256 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785077782053 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078725979 + } + ], + "39. Implement `FlowAdapter` interface in Drizzle adapter with transactional `createFlow`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893316 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078729344 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078732969 + } + ], + "40. Update adapter config to accept `flowModelName` option": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893374 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078736226 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078739287 + } + ], + "41. Export new flow schema from package": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893432 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078742445 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078745820 + } + ], + "42. Update `schema.prisma`: remove old flow columns, add `flowNodeId`, add `QueueFlow` model with `@@map(\"queue_flows\")`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893493 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785078750137 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079362427 + } + ], + "43. Regenerate Prisma client": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893557 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079365945 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079369217 + } + ], + "44. Update adapter class: remove old flow methods, remove old field mappings in `transformPrismaJob`/`addJob`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893615 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079372484 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079375915 + } + ], + "45. Implement `FlowAdapter` interface in Prisma adapter with transactional `createFlow`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893672 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079379367 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079382659 + } + ], + "46. Update adapter config to accept `flowModelName` and `flowTableName` options": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893729 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079386109 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079389361 + } + ], + "47. Update table type definition: remove old flow columns, add `flow_node_id`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893787 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079393694 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079890523 + } + ], + "48. Create `queue_flows` table type definition": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893845 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079894068 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079897981 + } + ], + "49. Update adapter class: remove old flow methods, remove old field mappings in `transformJob`/`addJob`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893904 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079902228 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079905555 + } + ], + "50. Implement `FlowAdapter` interface in Kysely adapter with transactional `createFlow`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074893956 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079908938 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079912308 + } + ], + "51. Update adapter config to accept `flowTableName` option": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894008 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079915850 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079919630 + } + ], + "52. Update ZenStack schema: remove old flow columns, add `flowNodeId`, add `QueueFlow` model": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894079 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785079929176 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080298204 + } + ], + "53. Update adapter class: remove old flow methods, implement `FlowAdapter`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894137 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080301716 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080305133 + } + ], + "54. Update adapter config to accept `flowModelName` and `flowTableName` options": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894213 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080308527 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080312168 + } + ], + "55. Update entity definition: remove old flow columns, add `flowNodeId`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894271 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080316085 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080778792 + } + ], + "56. Create `QueueFlow` entity": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894330 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080782369 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080786767 + } + ], + "57. Update adapter class: remove old flow methods, implement `FlowAdapter`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894392 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080790192 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080793743 + } + ], + "58. Update adapter config to accept `flowTableName` option": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894450 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080797263 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080800588 + } + ], + "59. Update entity definition: remove old flow columns, add `flowNodeId`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894508 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785080804574 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081194822 + } + ], + "60. Create `QueueFlow` entity": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894571 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081198322 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081201962 + } + ], + "61. Update adapter class: remove old flow methods, implement `FlowAdapter`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894623 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081205421 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081209001 + } + ], + "62. Update adapter config to accept `flowTableName` option": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894679 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081212457 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081215892 + } + ], + "63. Update model definition: remove old flow columns, add `flowNodeId`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894734 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081219589 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081686662 + } + ], + "64. Create `QueueFlow` model": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894790 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081690187 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081693780 + } + ], + "65. Update adapter class: remove old flow methods, implement `FlowAdapter`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894847 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081697489 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081701034 + } + ], + "66. Update adapter config to accept `flowTableName` option": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894907 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081704643 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081708334 + } + ], + "67. Remove old flow field persistence tests from `shared-tests`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074894963 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081713124 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081841180 + } + ], + "68. Remove old `deleteFlow` / `incrementChildrenCompleted` / `getFlowTree` tests": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895021 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081844646 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081848488 + } + ], + "69. Add new `FlowAdapter` shared tests: `createFlow` atomicity, `getFlowTree`, `getFlows`, `updateFlowNode`, `incrementNodeChildrenCompleted`, `getNodeChildren`, `cancelUnprocessedChildren`, `deleteFlow`, `cleanupFlows`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895073 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785081853620 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082279755 + } + ], + "70. Add flow integration tests: full lifecycle (create → children complete → parent promoted → parent completes), failure strategies, nested flows, cross-queue flows": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895131 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082289048 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082852169 + } + ], + "71. Rewrite `packages/core/tests/flow.test.ts`: FlowProducer.add with parent/children, FlowProducer.addChain, FlowProducer.addBulkThen": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895189 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082857252 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082861347 + } + ], + "72. Test failure strategies: default (promote on all terminal), fail-parent (cascade), continue-parent (immediate promote)": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895246 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082865043 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082868617 + } + ], + "73. Test FlowJobContext: getChildrenValues, getFailedChildrenValues, getChildrenValuesBy, removeUnprocessedChildren": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895300 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082872710 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082876510 + } + ], + "74. Test flow cleanup (removeOnComplete)": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895362 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082880162 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082884070 + } + ], + "75. Test flow events emission": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895428 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082887759 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082891447 + } + ], + "76. Remove `packages/core/tests/dependency.test.ts` (if exists)": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895503 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082896355 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082899910 + } + ], + "77. Update changeset: replace `workflow-engine.md` with new flow-producer changeset": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895588 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082903616 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082953167 + } + ], + "78. Run `pnpm format:fix && pnpm lint:fix && pnpm typecheck` — fix all issues": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895650 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785082957131 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785083252904 + } + ], + "79. Run all adapter tests: `pnpm test:drizzle`, `pnpm test:prisma`, `pnpm test:kysely`, `pnpm test:zenstack`, `pnpm test:typeorm`, `pnpm test:mikroorm`, `pnpm test:sequelize`": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895712 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785083257223 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785083578354 + } + ], + "80. Run core tests: verify all pass": [ + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785074895771 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785083582690 + }, + { + "executionId": "5a389003-d128-4ac2-9a8f-f69e777b7038", + "chatSessionId": "sess_00924a4f-e7ec-4e8f-971d-348250ebd33b", + "timestamp": 1785083610850 + } + ] + } +} diff --git a/.kiro/steering/project.md b/.kiro/steering/project.md new file mode 100644 index 00000000..14e58e9e --- /dev/null +++ b/.kiro/steering/project.md @@ -0,0 +1,170 @@ +# vorsteh-queue - Project Steering + +## Project Overview + +vorsteh-queue is a TypeScript-based job queue library published as a set of packages: + +- `packages/core` - Core queue logic +- `packages/adapter-drizzle` - Drizzle ORM adapter +- `packages/adapter-prisma` - Prisma adapter +- `packages/adapter-kysely` - Kysely adapter +- `packages/create-vorsteh-queue` - CLI scaffolding tool +- `packages/shared-tests` - Shared test utilities +- `tooling/typescript` - Shared TypeScript configurations + +## Monorepo Structure + +- **Package manager**: pnpm (workspace monorepo) +- **Build orchestration**: Turborepo +- **Workspace packages**: `apps/*`, `packages/*`, `tooling/*`, `examples/*` +- **Workspace constraints**: sherif (runs on `postinstall`) +- **Versioning**: changesets + +## Tooling + +### Linting (oxlint via ultracite) + +- Command: `pnpm lint` (check) / `pnpm lint:fix` (auto-fix) +- Config: `oxlint.config.ts` at workspace root +- Presets: `ultracite/oxlint/core`, `ultracite/oxlint/react`, `ultracite/oxlint/next`, `ultracite/oxlint/vitest` + +### Formatting (oxfmt via ultracite) + +- Command: `pnpm format` (check) / `pnpm format:fix` (auto-fix) +- Config: `oxfmt.config.ts` at workspace root +- Settings: line width 100, no semicolons, ES5 trailing commas, auto import sorting + +### Type Checking + +- Command: `pnpm typecheck` +- TypeScript 6.x with shared configs from `tooling/typescript` + +### Testing + +- Framework: Vitest +- Command: `pnpm test` (watch) / `vitest --run` (single run) +- Per-package: `pnpm test:core`, `pnpm test:drizzle`, `pnpm test:prisma`, `pnpm test:kysely` +- Coverage: `@vitest/coverage-v8` + +### Building + +- Command: `pnpm build` (all) / `pnpm build:pkg` (packages only) +- Turborepo handles dependency ordering + +## Code Style & Conventions + +### TypeScript + +- Use `import type` for type-only imports +- Prefix unused variables with `_` +- Use proper type guards instead of non-null assertions (`!`) +- Prefer `??` over `||` for null/undefined checks +- Prefer `?.` for optional property access +- No `.js` / `.ts` extensions in imports +- Prefer directory imports over explicit `/index` paths +- Use `readonly T[]` over `ReadonlyArray` +- Generic type parameters: always prefix with `T` (e.g., `TJobPayload`, `TResult`, `TEventData`) +- Prefer explicit return types for exported functions +- No `any` - use proper types or `unknown` +- Use type-fest utility types when available over custom implementations + +### Formatting + +- No semicolons +- 100 character line width +- ES5 trailing commas +- Import order (handled by oxfmt): + 1. Type imports + 2. React/Next.js/Expo + 3. Third-party modules + 4. `@vorsteh-queue` packages + 5. Relative imports (`~/`, `../`, `./`) + +### Documentation (JSDoc) + +All public-facing code must have JSDoc headers: + +- Classes: describe purpose, include `@example` where useful +- Functions/Methods: describe what it does, `@param`, `@returns`, `@throws`, `@example` +- Interface properties: inline `/** description */` with `@default` where applicable +- Type aliases: describe what the type represents +- Enums/Constants: describe each member + +````typescript +/** + * Register a job handler for a specific job type. + * + * @param name - The job type name (must be unique per queue) + * @param handler - Function to process jobs of this type + * @throws {Error} If a handler with this name is already registered + * + * @example + * ```typescript + * queue.register("send-email", async (job) => { + * await sendEmail(job.payload.to) + * return { sent: true } + * }) + * ``` + */ +```` + +```typescript +interface JobOptions { + /** Job priority (lower number = higher priority) + * @default 2 + */ + readonly priority?: number + + /** Delay in milliseconds before job becomes available + * @default undefined (no delay) + */ + readonly delay?: number +} +``` + +Do NOT add JSDoc to: + +- Private/internal helpers (unless complex logic) +- Test files +- Obvious one-liner utility functions + +### General + +- No `console.log` - remove debug statements +- Prefer functional programming patterns +- Prefer composition over inheritance +- Use readonly arrays and objects where appropriate +- Write self-documenting code; minimize inline comments +- Use proper error handling (no generic `throw new Error`) +- camelCase for variables/functions, PascalCase for types/interfaces/classes + +## Verification Workflow + +After making code changes, always verify with these commands (in this order): + +1. `pnpm format:fix` — Apply formatting +2. `pnpm lint:fix` — Auto-fix lint issues +3. `pnpm typecheck` — Verify types +4. `vitest --run` — Run tests (single run, no watch mode) + +Never use `npx`, `pnpm dlx`, or call tools directly (e.g. `oxlint`, `oxfmt`). Always use the pnpm scripts defined in the root `package.json`. + +## Commands Reference + +| Task | Command | +| ------------------ | ------------------ | +| Install | `pnpm install` | +| Dev (packages) | `pnpm dev` | +| Dev (docs) | `pnpm dev:docs` | +| Build all | `pnpm build` | +| Build packages | `pnpm build:pkg` | +| Lint | `pnpm lint` | +| Lint fix | `pnpm lint:fix` | +| Format check | `pnpm format` | +| Format fix | `pnpm format:fix` | +| Typecheck | `pnpm typecheck` | +| Test (watch) | `pnpm test` | +| Test (single run) | `vitest --run` | +| Changeset | `pnpm cs` | +| Clean node_modules | `pnpm clean` | +| Clean turbo cache | `pnpm clean:cache` | diff --git a/.nvmrc b/.nvmrc index 8fdd954d..cabf43b5 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -22 \ No newline at end of file +24 \ No newline at end of file diff --git a/.prettierignore b/.prettierignore deleted file mode 100644 index 9211984a..00000000 --- a/.prettierignore +++ /dev/null @@ -1,4 +0,0 @@ -turbo/generators/scaffold/templates/**/*.hbs -**/.cache -CHANGELOG.md -**/CHANGELOG.md \ No newline at end of file diff --git a/.vscode/mcp.json b/.vscode/mcp.json new file mode 100644 index 00000000..8c764699 --- /dev/null +++ b/.vscode/mcp.json @@ -0,0 +1,8 @@ +{ + "servers": { + "shadcn": { + "command": "npx", + "args": ["shadcn@latest", "mcp"] + } + } +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 450c6699..9006c68f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,51 +1,74 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": "explicit" + "source.fixAll": "always", + "source.fixAll.oxc": "always", + "source.format.oxc": "always" }, - "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.defaultFormatter": "oxc.oxc-vscode", "editor.formatOnSave": true, - "eslint.rules.customizations": [{ "rule": "*", "severity": "warn" }], - "eslint.workingDirectories": [ - { "pattern": "apps/*/" }, - { "pattern": "packages/*/" }, - { "pattern": "tooling/*/" } - ], - "tailwindCSS.experimental.configFile": { - "apps/docs/src/app/globals.css": ["apps/docs/**"] - }, - "tailwindCSS.emmetCompletions": true, - "tailwindCSS.files.exclude": [ - "**/.git/**", - "**/node_modules/**", - "**/.hg/**", - "**/.svn/**", - "**/.vscode/**" - ], - "tailwindCSS.experimental.classRegex": [ - ["cva\\(([^)]*)\\)", "[\"'`]([^\"'`]*).*?[\"'`]"], - ["cx\\(([^)]*)\\)", "(?:'|\"|`)([^']*)(?:'|\"|`)"] - ], - "prettier.ignorePath": ".gitignore", - "typescript.enablePromptUseWorkspaceTsdk": true, - "typescript.preferences.autoImportFileExcludePatterns": [ - "next/router.d.ts", - "next/dist/client/router.d.ts" - ], - "typescript.tsdk": "node_modules/typescript/lib", "vitest.nodeEnv": { "npm_lifecycle_event": "test" }, - "[shellscript]": { - "editor.defaultFormatter": "foxundermoon.shell-format" + + "files.associations": { + "*.css": "tailwindcss", + "*.scss": "tailwindcss" }, - "[dotenv]": { - "editor.defaultFormatter": "foxundermoon.shell-format" + "[xml]": { + "editor.defaultFormatter": "redhat.vscode-xml" }, - "[sql]": { - "editor.defaultFormatter": "mtxr.sqltools" + "editor.formatOnPaste": true, + "emmet.showExpandedAbbreviation": "never", + "[css]": { + "editor.defaultFormatter": "oxc.oxc-vscode" }, - "files.associations": { - "*.css": "tailwindcss" - } + "[graphql]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[handlebars]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[html]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[javascript]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[javascriptreact]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[json]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[jsonc]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[less]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[markdown]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[scss]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[typescript]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[typescriptreact]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[vue-html]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[vue]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "[yaml]": { + "editor.defaultFormatter": "oxc.oxc-vscode" + }, + "oxc.enable.oxlint": true, + "oxc.fixKind": "safe_fix_or_suggestion", + "oxc.requireConfig": true } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f58ea3e7..05fd2744 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,9 @@ Thank you for your interest in contributing to Vorsteh Queue! This document outl payload: TJobPayload } - function process(job: BaseJob): Promise + function process( + job: BaseJob + ): Promise // ❌ Avoid interface BaseJob { @@ -48,6 +50,7 @@ Follow this import order: 5. Relative imports (`~/`, `../`, `./`) **Import Path Conventions**: + - No file extensions: Never use `.js`, `.ts` extensions - Prefer directory imports: Use `../src` instead of `../src/index` - Avoid useless path segments: Skip `/index` when possible diff --git a/README.md b/README.md index f0640644..2d654eab 100644 --- a/README.md +++ b/README.md @@ -1,362 +1,95 @@
- Vorsteh Queue + Vorsteh Queue

Vorsteh Queue

-

A powerful, ORM-agnostic queue engine for PostgreSQL 12+. Handle background jobs, scheduled tasks, and recurring processes with ease.

+

A type-safe PostgreSQL job queue for Node.js with durable execution, flow orchestration, and first-class ORM adapters.

## Features - **Type-safe**: Full TypeScript support with generic job payloads -- **Multiple adapters**: Drizzle ORM (PostgreSQL), Prisma ORM (PostgreSQL), Kysely (PostgreSQL) and in-memory implementations +- **Multiple adapters**: Drizzle ORM, Prisma ORM, Kysely, ZenStack, TypeORM, MikroORM, Sequelize (all PostgreSQL) - **Priority queues**: Numeric priority system (lower = higher priority) - **Delayed jobs**: Schedule jobs for future execution - **Recurring jobs**: Cron expressions and interval-based repetition -- **Batch processing**: Process multiple jobs concurrently for higher efficiency and better performance by supporting parallel execution and grouping of jobs. +- **Batch processing**: Process multiple jobs concurrently with grouping support +- **Flow producer**: Define parent/child job dependencies - **UTC-first timezone support**: Reliable timezone handling with UTC storage - **Progress tracking**: Real-time job progress updates - **Event system**: Listen to job lifecycle events - **Graceful shutdown**: Clean job processing termination - -## Repository Structure - -``` -. -├── packages/ -│ ├── core/ # Core queue logic and interfaces -│ ├── adapter-drizzle/ # Drizzle ORM adapter (PostgreSQL) -│ └── adapter-prisma/ # Prisma ORM adapter (PostgreSQL) -│ └── adapter-kysely/ # Kysely adapter (PostgreSQL) -├── examples/ # Standalone usage examples -└── tooling/ # Shared development tools -``` +- **GraphQL API**: Built-in GraphQL server for queue management and monitoring + +## Packages + +| Package | Description | +| --- | --- | +| [`@vorsteh-queue/core`](./packages/core) | Core queue engine and interfaces | +| [`@vorsteh-queue/adapter-drizzle`](./packages/adapter-drizzle) | Drizzle ORM adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-prisma`](./packages/adapter-prisma) | Prisma ORM adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-kysely`](./packages/adapter-kysely) | Kysely adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-zenstack`](./packages/adapter-zenstack) | ZenStack ORM adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-typeorm`](./packages/adapter-typeorm) | TypeORM adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-mikroorm`](./packages/adapter-mikroorm) | MikroORM adapter (PostgreSQL) | +| [`@vorsteh-queue/adapter-sequelize`](./packages/adapter-sequelize) | Sequelize adapter (PostgreSQL) | +| [`@vorsteh-queue/server`](./packages/server) | GraphQL API server for queue management | +| [`create-vorsteh-queue`](./packages/create-vorsteh-queue) | CLI scaffolding tool | ## Quick Start -### Requirements - -- **Node.js 20+** -- **ESM only** - This package is ESM-only and cannot be imported with `require()` +```bash +pnpm dlx create-vorsteh-queue my-queue-app +``` -### Installation +Or install manually: ```bash -npm install @vorsteh-queue/core @vorsteh-queue/adapter-drizzle -# or pnpm add @vorsteh-queue/core @vorsteh-queue/adapter-drizzle ``` -> **Note**: Make sure your project has `"type": "module"` in package.json or use `.mjs` file extensions. - -### Basic Usage - ```typescript -// Drizzle ORM with PostgreSQL - -// Prisma ORM with PostgreSQL -import { PrismaClient } from "@prisma/client" import { drizzle } from "drizzle-orm/node-postgres" import { Pool } from "pg" import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" -import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" import { Queue } from "@vorsteh-queue/core" -interface EmailPayload { - to: string - subject: string - body: string -} - -interface EmailResult { - messageId: string - sent: boolean -} - const pool = new Pool({ connectionString: "postgresql://..." }) const db = drizzle(pool) const queue = new Queue(new PostgresQueueAdapter(db), { name: "my-queue" }) -const prisma = new PrismaClient() -const queue = new Queue(new PostgresPrismaQueueAdapter(prisma), { name: "my-queue" }) - -// Register job handlers -queue.register("send-email", async (job) => { - console.log(`Sending email to ${job.payload.to}`) - - // Send email logic here +queue.register("send-email", async (job) => { await sendEmail(job.payload) - - // Return result - will be stored in job.result field - return { - messageId: "msg_123", - sent: true, - } + return { sent: true } }) -// Add jobs -await queue.add("send-email", { - to: "user@example.com", - subject: "Welcome!", - body: "Welcome to our service!", -}) -await queue.add( - "send-email", - { to: "admin@example.com", subject: "Report" }, - { - priority: 1, // Higher priority - delay: 5000, // Delay 5 seconds - }, -) +await queue.add("send-email", { to: "user@example.com", subject: "Welcome!" }) -// Start processing queue.start() ``` -## Examples - -Check out the [examples directory](./examples/) for complete, runnable examples. - -> **Note**: All examples demonstrate the UTC-first timezone approach and automatic job cleanup features. - -## Priority System - -Jobs are processed by priority (lower number = higher priority): - -```typescript -await queue.add("urgent-task", payload, { priority: 1 }) // Processed first -await queue.add("normal-task", payload, { priority: 2 }) // Default priority -await queue.add("low-task", payload, { priority: 3 }) // Processed last -``` - -## Recurring Jobs - -```typescript -// Cron expression -await queue.add("daily-report", payload, { - cron: "0 9 * * *", // Every day at 9 AM -}) - -// Interval with limit -await queue.add("health-check", payload, { - repeat: { every: 30000, limit: 10 }, // Every 30s, 10 times -}) -``` - -## Batch Processing - -Process multiple jobs in a single batch for higher throughput and efficiency. - -```typescript -queue.registerBatch<{ file: string }, { ok: boolean }>("process-files", async (jobs) => { - console.log(`Processing batch of ${jobs.length} files...`) - return jobs.map(() => ({ ok: true })) -}) - -await queue.addJobs("process-files", [{ file: "a.csv" }, { file: "b.csv" }, { file: "c.csv" }]) - -queue.on("batch:processing", (jobs) => { - console.log(`Batch started: ${jobs.length} jobs`) -}) -queue.on("batch:completed", (jobs) => { - console.log(`Batch completed: ${jobs.length} jobs`) -}) -``` - -## Job Cleanup - -```typescript -// Automatic cleanup configuration -const queue = new Queue(adapter, { - name: "my-queue", - removeOnComplete: true, // Remove completed jobs immediately - removeOnFail: false, // Keep failed jobs for debugging - // Or use numbers to keep N jobs - removeOnComplete: 100, // Keep last 100 completed jobs - removeOnFail: 50, // Keep last 50 failed jobs -}) -``` - -## Timezone Support - -Vorsteh Queue uses a **UTC-first approach** for reliable timezone handling: - -```typescript -// Schedule job for 9 AM New York time - converted to UTC immediately -await queue.add("daily-report", payload, { - cron: "0 9 * * *", - timezone: "America/New_York", // Timezone used for conversion only -}) - -// Schedule job for specific time in Tokyo - stored as UTC -await queue.add("notification", payload, { - runAt: new Date("2024-01-15T10:00:00"), - timezone: "Asia/Tokyo", // Interprets runAt in Tokyo time -}) - -// Complex cron with timezone - result always UTC -await queue.add("business-task", payload, { - cron: "*/15 9-17 * * 1-5", - timezone: "Europe/London", // Business hours in London time -}) -``` - -### How It Works - -1. **Timezone conversion happens at job creation** - not at execution -2. **All timestamps stored in database are UTC** - no timezone ambiguity -3. **Recurring jobs recalculate using original timezone** - maintains accuracy -4. **Simple and predictable** - no runtime timezone complexity -5. **Server timezone independent** - works consistently across environments - -## Job Results - -Job handlers can return results that are automatically stored and made available: - -```typescript -interface ProcessResult { - processed: number - errors: string[] - duration: number -} +## Requirements -queue.register<{ items: string[] }, ProcessResult>("process-data", async (job) => { - const startTime = Date.now() - const errors: string[] = [] - let processed = 0 +- Node.js 22+ +- PostgreSQL 12+ +- ESM only (`"type": "module"` in package.json) - for (const item of job.payload.items) { - try { - await processItem(item) - processed++ - } catch (error) { - errors.push(`Failed to process ${item}: ${error.message}`) - } - } +## Documentation - // Return result - automatically stored in job.result field - return { - processed, - errors, - duration: Date.now() - startTime, - } -}) - -// Access results in events -queue.on("job:completed", (job) => { - const result = job.result as ProcessResult - console.log(`Processed ${result.processed} items in ${result.duration}ms`) - if (result.errors.length > 0) { - console.warn(`Errors: ${result.errors.join(", ")}`) - } -}) -``` - -## Progress Tracking - -```typescript -// Register a job that reports progress -queue.register("process-data", async (job) => { - const items = job.payload.items - - for (let i = 0; i < items.length; i++) { - // Process item - await processItem(items[i]) - - // Update progress (0-100) - const progress = Math.round(((i + 1) / items.length) * 100) - await job.updateProgress(progress) - } - - return { processed: items.length } -}) - -// Listen to progress updates -queue.on("job:progress", (job) => { - console.log(`Job ${job.name}: ${job.progress}% complete`) -}) -``` - -## Event System - -```typescript -// Listen to all job lifecycle events -queue.on("job:added", (job) => { - console.log(`✅ Job ${job.name} added to queue`) -}) - -queue.on("job:processing", (job) => { - console.log(`⚡ Job ${job.name} started processing`) -}) - -queue.on("job:completed", (job) => { - console.log(`🎉 Job ${job.name} completed successfully`) - console.log(`📊 Result:`, job.result) // Access job result -}) - -queue.on("job:failed", (job) => { - console.error(`❌ Job ${job.name} failed: ${job.error}`) -}) - -queue.on("job:retried", (job) => { - console.log(`🔄 Job ${job.name} retrying (attempt ${job.attempts})`) -}) - -// Queue-level events -queue.on("queue:paused", () => { - console.log("⏸️ Queue paused") -}) - -queue.on("queue:resumed", () => { - console.log("▶️ Queue resumed") -}) -``` - -## Architecture - -### UTC-First Design - -- **Database storage**: All timestamps stored as UTC -- **Timezone conversion**: Happens at job creation time -- **No timezone fields**: Database schema contains no timezone columns -- **Predictable behavior**: Same results across different server timezones - -### Adapter Pattern - -- **Pluggable storage**: Easy to add new database adapters -- **Consistent interface**: Same API across all adapters -- **Type safety**: Full TypeScript support throughout +See the [Vorsteh Queue Documentation](https://vorsteh-queue.dev/docs) for detailed guides, API reference, and examples. ## Development -This project was developed with AI assistance, combining human expertise with AI-powered code generation to accelerate development while maintaining high code quality standards. - ```bash -# Install dependencies -pnpm install - -# Autofix format issues -pnpm format:fix - -# Fix issues in the `package.json` ( order, version mismatch ) -pnpm lint:ws -f - -# Lint -pnpm lint - -# Type check -pnpm typecheck - -# Run all tests -pnpm test - -# Build packages -pnpm build +pnpm install # Install dependencies +pnpm dev # Start dev mode +pnpm build # Build all packages +pnpm typecheck # Type check +pnpm lint:fix # Lint and auto-fix +pnpm format:fix # Format code +pnpm test # Run tests (watch) +vitest --run # Run tests (single run) ``` -## Contributing - -Contributions are welcome! Please read our [contributing guidelines](./CONTRIBUTING.md) and submit pull requests to our repository. - ## License -MIT License - see [LICENSE](./LICENSE) file for details. +[MIT](./LICENSE) diff --git a/_oxlint.config.ts b/_oxlint.config.ts new file mode 100644 index 00000000..1db06e3d --- /dev/null +++ b/_oxlint.config.ts @@ -0,0 +1,704 @@ +import { defineConfig } from "oxlint" +import next from "ultracite/oxlint/next" +import react from "ultracite/oxlint/react" +import vitest from "ultracite/oxlint/vitest" + +export default defineConfig({ + env: { + browser: true, + }, + extends: [vitest, react, next], + ignorePatterns: [ + "**/node_modules", + "**/.git", + "**/dist", + "**/build", + "**/out", + "**/.next", + "**/.nuxt", + "**/.output", + "**/.turbo", + "**/.vercel", + "**/.cache", + "**/.vite", + "**/storybook-static", + "**/_generated", + "**/*.gen.*", + "**/*.generated.*", + "**/*.auto.*", + "**/generated", + "**/__generated__", + "**/*.d.ts.map", + "**/.yarn", + "**/coverage", + "**/bun.lock", + "**/bun.lockb", + "**/package-lock.json", + "**/yarn.lock", + "**/pnpm-lock.yaml", + "**/next-env.d.ts", + // Project-specific + "apps/docs/src/components/beautiful-mermaid/**", + "packages/server/src/ui/routeTree.gen.ts", + "packages/server/src/ui/graphql-env.d.ts", + ], + overrides: [ + { + files: [ + "**/*.{test,spec}.{ts,tsx,js,jsx}", + "**/__tests__/**/*.{ts,tsx,js,jsx}", + ], + rules: { + "no-empty-function": "off", + "promise/prefer-await-to-then": "off", + }, + }, + { + files: ["examples/**/*.ts"], + rules: { + "no-console": "off", + "no-restricted-properties": "off", + "no-await-in-loop": "off", + "no-plusplus": "off", + "no-promise-executor-return": "off", + "promise/avoid-new": "off", + "promise/prefer-await-to-then": "off", + "promise/prefer-await-to-callbacks": "off", + "react-doctor/async-parallel": "off", + "react-doctor/async-await-in-loop": "off", + "react-doctor/async-defer-await": "off", + "react-doctor/server-sequential-independent-await": "off", + }, + }, + { + files: [ + "apps/docs/src/app/**/*.{ts,tsx}", + "packages/cli/src/**/*.ts", + "packages/query-builder/src/**/*.ts", + "packages/trino-client/src/**/*.ts", + ], + rules: { + "no-use-before-define": [ + "error", + { + allowNamedExports: true, + functions: false, + ignoreTypeReferences: true, + }, + ], + }, + }, + ], + plugins: [ + "eslint", + "typescript", + "unicorn", + "oxc", + "import", + "jsdoc", + "node", + "promise", + "react", + ], + rules: { + // ── eslint ────────────────────────────────────────────────────────────── + "accessor-pairs": "error", + "array-callback-return": "error", + "arrow-body-style": ["error", "as-needed"], + "block-scoped-var": "error", + "capitalized-comments": "off", + "class-methods-use-this": "error", + complexity: "error", + "constructor-super": "error", + curly: "error", + "default-case": "error", + "default-case-last": "error", + "default-param-last": "error", + eqeqeq: "error", + "for-direction": "error", + "func-name-matching": "error", + "func-names": "error", + "func-style": "off", + "getter-return": "error", + "grouped-accessor-pairs": "error", + "guard-for-in": "error", + "id-length": "off", + "id-match": "error", + "init-declarations": "off", + "logical-assignment-operators": "error", + "max-classes-per-file": "error", + "max-depth": "off", + "max-lines": "off", + "max-lines-per-function": "off", + "max-nested-callbacks": "error", + "max-params": "off", + "max-statements": "off", + "new-cap": "off", + "no-alert": "error", + "no-array-constructor": "error", + "no-async-promise-executor": "error", + "no-await-in-loop": "error", + "no-bitwise": "error", + "no-caller": "error", + "no-case-declarations": "error", + "no-class-assign": "error", + "no-compare-neg-zero": "error", + "no-cond-assign": "error", + "no-const-assign": "error", + "no-constant-binary-expression": "error", + "no-constant-condition": "error", + "no-console": "error", + "no-constructor-return": "error", + "no-continue": "off", + "no-control-regex": "error", + "no-debugger": "error", + "no-delete-var": "error", + "no-div-regex": "error", + "no-dupe-class-members": "error", + "no-dupe-else-if": "error", + "no-dupe-keys": "error", + "no-duplicate-case": "error", + "no-duplicate-imports": ["error", { allowSeparateTypeImports: true }], + "no-else-return": "error", + "no-empty": "error", + "no-empty-character-class": "error", + "no-empty-function": "error", + "no-empty-pattern": "error", + "no-empty-static-block": "error", + "no-eq-null": "error", + "no-eval": "error", + "no-ex-assign": "error", + "no-extend-native": "error", + "no-extra-bind": "error", + "no-extra-boolean-cast": "error", + "no-extra-label": "error", + "no-fallthrough": "error", + "no-func-assign": "error", + "no-global-assign": "error", + "no-implicit-coercion": "off", + "no-implicit-globals": "error", + "no-implied-eval": "error", + "no-import-assign": "error", + "no-inline-comments": "off", + "no-inner-declarations": "error", + "no-invalid-regexp": "error", + "no-irregular-whitespace": "error", + "no-iterator": "error", + "no-label-var": "error", + "no-labels": "error", + "no-lone-blocks": "error", + "no-lonely-if": "error", + "no-loop-func": "error", + "no-loss-of-precision": "error", + "no-magic-numbers": "off", + "no-misleading-character-class": "error", + "no-multi-assign": "error", + "no-multi-str": "error", + "no-negated-condition": "error", + "no-nested-ternary": "off", + "no-new": "error", + "no-new-func": "error", + "no-new-native-nonconstructor": "error", + "no-new-wrappers": "error", + "no-nonoctal-decimal-escape": "error", + "no-obj-calls": "error", + "no-object-constructor": "error", + "no-param-reassign": "error", + "no-plusplus": "error", + "no-promise-executor-return": "error", + "no-proto": "error", + "no-prototype-builtins": "error", + "no-redeclare": "error", + "no-regex-spaces": "error", + "no-restricted-globals": "error", + "no-restricted-imports": [ + "error", + { + importNames: ["env"], + message: + "Use `import { env } from '~/env'` instead to ensure validated types.", + name: "process", + }, + ], + "no-restricted-properties": [ + "error", + { + message: + "Use `import { env } from '~/env'` instead to ensure validated types.", + object: "process", + property: "env", + }, + ], + "no-return-assign": "error", + "no-script-url": "error", + "no-self-assign": "error", + "no-self-compare": "error", + "no-sequences": "error", + "no-setter-return": "error", + "no-shadow": "error", + "no-shadow-restricted-names": "error", + "no-sparse-arrays": "error", + "no-template-curly-in-string": "error", + "no-ternary": "off", + "no-this-before-super": "error", + "no-throw-literal": "error", + "no-unassigned-vars": "error", + "no-undefined": "off", + "no-underscore-dangle": "off", + "no-unexpected-multiline": "error", + "no-unmodified-loop-condition": "error", + "no-unneeded-ternary": "error", + "no-unreachable": "error", + "no-unsafe-finally": "error", + "no-unsafe-negation": "error", + "no-unsafe-optional-chaining": "error", + "no-unused-expressions": "error", + "no-unused-labels": "error", + "no-unused-private-class-members": "error", + "no-unused-vars": "error", + "no-use-before-define": "off", + "no-useless-backreference": "error", + "no-useless-call": "error", + "no-useless-catch": "error", + "no-useless-computed-key": "error", + "no-useless-concat": "error", + "no-useless-constructor": "error", + "no-useless-escape": "error", + "no-useless-rename": "error", + "no-useless-return": "error", + "no-var": "error", + "no-void": ["error", { allowAsStatement: true }], + "no-warning-comments": "error", + "no-with": "error", + "object-shorthand": "error", + "operator-assignment": "error", + "prefer-arrow-callback": "error", + "prefer-const": "error", + "prefer-destructuring": "error", + "prefer-exponentiation-operator": "error", + "prefer-named-capture-group": "error", + "prefer-numeric-literals": "error", + "prefer-object-has-own": "error", + "prefer-object-spread": "error", + "prefer-promise-reject-errors": "error", + "prefer-regex-literals": "error", + "prefer-rest-params": "error", + "prefer-spread": "error", + "prefer-template": "error", + "preserve-caught-error": "error", + radix: "error", + "require-await": "off", + "require-unicode-regexp": "error", + "require-yield": "error", + "sort-imports": "off", + "sort-keys": "off", + "sort-vars": "error", + "symbol-description": "error", + "unicode-bom": "error", + "use-isnan": "error", + "valid-typeof": "error", + "vars-on-top": "error", + yoda: "error", + + // ── import ────────────────────────────────────────────────────────────── + "import/consistent-type-specifier-style": ["error", "prefer-top-level"], + "import/default": "error", + "import/exports-last": "off", + "import/extensions": "off", + "import/first": "error", + "import/group-exports": "off", + "import/max-dependencies": "off", + "import/namespace": "error", + "import/newline-after-import": "error", + "import/no-absolute-path": "error", + "import/no-amd": "error", + "import/no-anonymous-default-export": "off", + "import/no-commonjs": "off", + "import/no-cycle": "error", + "import/no-default-export": "off", + "import/no-duplicates": "error", + "import/no-dynamic-require": "off", + "import/no-empty-named-blocks": "error", + "import/no-mutable-exports": "error", + "import/no-named-as-default": "error", + "import/no-named-as-default-member": "error", + "import/no-named-default": "error", + "import/no-named-export": "off", + "import/no-namespace": "off", + "import/no-nodejs-modules": "off", + "import/no-relative-parent-imports": "off", + "import/no-self-import": "error", + "import/no-unassigned-import": "off", + "import/no-webpack-loader-syntax": "error", + "import/prefer-default-export": "off", + "import/unambiguous": "off", + + // ── jsdoc ─────────────────────────────────────────────────────────────── + "jsdoc/check-access": "error", + "jsdoc/check-property-names": "error", + "jsdoc/check-tag-names": "error", + "jsdoc/empty-tags": "error", + "jsdoc/implements-on-classes": "error", + "jsdoc/no-defaults": "error", + "jsdoc/require-param": "off", + "jsdoc/require-param-description": "error", + "jsdoc/require-param-name": "error", + "jsdoc/require-param-type": "off", + "jsdoc/require-property": "error", + "jsdoc/require-property-description": "error", + "jsdoc/require-property-name": "error", + "jsdoc/require-property-type": "error", + "jsdoc/require-returns": "off", + "jsdoc/require-returns-description": "error", + "jsdoc/require-returns-type": "off", + "jsdoc/require-throws-description": "error", + "jsdoc/require-throws-type": "error", + "jsdoc/require-yields": "error", + "jsdoc/require-yields-description": "error", + "jsdoc/require-yields-type": "error", + + // ── node ──────────────────────────────────────────────────────────────── + "node/callback-return": "error", + "node/global-require": "error", + "node/handle-callback-err": "error", + "node/no-exports-assign": "error", + "node/no-mixed-requires": "error", + "node/no-new-require": "error", + "node/no-path-concat": "error", + "node/no-process-env": "off", + "node/no-sync": "off", + + // ── oxc ───────────────────────────────────────────────────────────────── + "oxc/approx-constant": "error", + "oxc/bad-array-method-on-arguments": "error", + "oxc/bad-bitwise-operator": "error", + "oxc/bad-char-at-comparison": "error", + "oxc/bad-comparison-sequence": "error", + "oxc/bad-min-max-func": "error", + "oxc/bad-object-literal-comparison": "error", + "oxc/bad-replace-all-arg": "error", + "oxc/branches-sharing-code": "error", + "oxc/const-comparisons": "error", + "oxc/double-comparisons": "error", + "oxc/erasing-op": "error", + "oxc/misrefactored-assign-op": "error", + "oxc/missing-throw": "error", + "oxc/no-accumulating-spread": "error", + "oxc/no-async-await": "off", + "oxc/no-async-endpoint-handlers": "error", + "oxc/no-barrel-file": "error", + "oxc/no-const-enum": "error", + "oxc/no-map-spread": "off", + "oxc/no-optional-chaining": "off", + "oxc/no-rest-spread-properties": "off", + "oxc/no-this-in-exported-function": "error", + "oxc/number-arg-out-of-range": "error", + "oxc/only-used-in-recursion": "error", + "oxc/uninvoked-array-callback": "error", + + // ── promise ───────────────────────────────────────────────────────────── + "promise/always-return": "off", + "promise/avoid-new": "error", + "promise/catch-or-return": "off", + "promise/no-callback-in-promise": "error", + "promise/no-multiple-resolved": "error", + "promise/no-nesting": "error", + "promise/no-new-statics": "error", + "promise/no-promise-in-callback": "error", + "promise/no-return-wrap": "error", + "promise/param-names": "error", + "promise/prefer-await-to-callbacks": "error", + "promise/prefer-await-to-then": "error", + "promise/prefer-catch": "error", + "promise/spec-only": "error", + "promise/valid-params": "error", + + // ── typescript ────────────────────────────────────────────────────────── + "typescript/adjacent-overload-signatures": "error", + "typescript/array-type": "error", + "typescript/await-thenable": "error", + "typescript/ban-ts-comment": "error", + "typescript/ban-tslint-comment": "error", + "typescript/ban-types": "error", + "typescript/class-literal-property-style": "error", + "typescript/consistent-generic-constructors": "error", + "typescript/consistent-indexed-object-style": "error", + "typescript/consistent-return": "error", + "typescript/consistent-type-assertions": "error", + "typescript/consistent-type-definitions": "off", + "typescript/consistent-type-exports": "error", + "typescript/consistent-type-imports": "error", + "typescript/default-param-last": "error", + "typescript/explicit-function-return-type": "off", + "typescript/explicit-member-accessibility": "off", + "typescript/explicit-module-boundary-types": "off", + "typescript/init-declarations": "off", + "typescript/max-params": "off", + "typescript/method-signature-style": "error", + + "typescript/no-confusing-non-null-assertion": "error", + "typescript/no-confusing-void-expression": "error", + "typescript/no-deprecated": "error", + "typescript/no-duplicate-enum-values": "error", + "typescript/no-duplicate-type-constituents": "error", + "typescript/no-dynamic-delete": "error", + "typescript/no-empty-function": "error", + "typescript/no-empty-interface": "error", + "typescript/no-empty-object-type": "error", + "typescript/no-explicit-any": "error", + "typescript/no-extra-non-null-assertion": "error", + "typescript/no-extraneous-class": "error", + "typescript/no-floating-promises": "error", + "typescript/no-for-in-array": "error", + "typescript/no-import-type-side-effects": "error", + "typescript/no-inferrable-types": "error", + "typescript/no-invalid-void-type": "error", + "typescript/no-meaningless-void-operator": "error", + "typescript/no-misused-new": "error", + "typescript/no-misused-promises": "error", + "typescript/no-mixed-enums": "error", + "typescript/no-namespace": "error", + "typescript/no-non-null-asserted-nullish-coalescing": "error", + "typescript/no-non-null-asserted-optional-chain": "error", + "typescript/no-non-null-assertion": "error", + "typescript/no-redeclare": "error", + "typescript/no-redundant-type-constituents": "error", + "typescript/no-require-imports": "error", + "typescript/no-restricted-types": "error", + "typescript/no-shadow": "error", + "typescript/no-this-alias": "error", + "typescript/no-unnecessary-boolean-literal-compare": "error", + "typescript/no-unnecessary-condition": "error", + "typescript/no-unnecessary-qualifier": "error", + "typescript/no-unnecessary-template-expression": "error", + "typescript/no-unnecessary-type-arguments": "error", + "typescript/no-unnecessary-type-assertion": "error", + "typescript/no-unnecessary-type-constraint": "error", + "typescript/no-unnecessary-type-parameters": "error", + "typescript/no-unsafe-argument": "error", + "typescript/no-unsafe-assignment": "error", + "typescript/no-unsafe-call": "error", + "typescript/no-unsafe-declaration-merging": "error", + "typescript/no-unsafe-enum-comparison": "error", + "typescript/no-unsafe-function-type": "error", + "typescript/no-unsafe-member-access": "error", + "typescript/no-unsafe-return": "error", + "typescript/no-unsafe-unary-minus": "error", + "typescript/no-unused-expressions": "error", + "typescript/no-unused-vars": "error", + "typescript/no-use-before-define": "off", + "typescript/no-useless-constructor": "error", + "typescript/no-useless-empty-export": "error", + "typescript/no-wrapper-object-types": "error", + "typescript/non-nullable-type-assertion-style": "error", + "typescript/only-throw-error": "error", + "typescript/parameter-properties": "off", + "typescript/prefer-as-const": "error", + "typescript/prefer-enum-initializers": "error", + "typescript/prefer-find": "error", + "typescript/prefer-for-of": "error", + "typescript/prefer-function-type": "error", + "typescript/prefer-includes": "error", + "typescript/prefer-literal-enum-member": "error", + "typescript/prefer-namespace-keyword": "error", + "typescript/prefer-nullish-coalescing": "error", + "typescript/prefer-optional-chain": "error", + "typescript/prefer-readonly": "error", + "typescript/prefer-reduce-type-parameter": "error", + "typescript/prefer-regexp-exec": "error", + "typescript/prefer-return-this-type": "error", + "typescript/prefer-string-starts-ends-with": "error", + "typescript/prefer-ts-expect-error": "error", + "typescript/promise-function-async": "off", + "typescript/require-array-sort-compare": "error", + "typescript/restrict-plus-operands": "error", + "typescript/restrict-template-expressions": "error", + "typescript/return-await": "error", + "typescript/strict-boolean-expressions": "off", + "typescript/switch-exhaustiveness-check": "error", + "typescript/triple-slash-reference": "error", + + "typescript/unified-signatures": "error", + "typescript/use-unknown-in-catch-callback-variable": "error", + + // ── unicorn ───────────────────────────────────────────────────────────── + + "unicorn/catch-error-name": "error", + "unicorn/consistent-date-clone": "error", + + "unicorn/consistent-empty-array-spread": "error", + "unicorn/consistent-existence-index-check": "error", + "unicorn/consistent-function-scoping": "error", + "unicorn/custom-error-definition": "error", + "unicorn/error-message": "error", + "unicorn/escape-case": "error", + + "unicorn/explicit-length-check": "off", + "unicorn/filename-case": "error", + "unicorn/import-style": "error", + "unicorn/max-nested-calls": "off", + "unicorn/new-for-builtins": "error", + "unicorn/no-abusive-eslint-disable": "error", + "unicorn/no-accessor-recursion": "error", + "unicorn/no-anonymous-default-export": "error", + "unicorn/no-array-callback-reference": "off", + "unicorn/no-array-fill-with-reference-type": "error", + "unicorn/no-array-for-each": "error", + "unicorn/no-array-method-this-argument": "error", + "unicorn/no-array-reduce": "error", + "unicorn/no-array-reverse": "error", + "unicorn/no-array-sort": "error", + "unicorn/no-await-expression-member": "error", + "unicorn/no-await-in-promise-methods": "error", + "unicorn/no-console-spaces": "error", + "unicorn/no-document-cookie": "error", + "unicorn/no-empty-file": "error", + "unicorn/no-hex-escape": "error", + "unicorn/no-immediate-mutation": "error", + "unicorn/no-instanceof-array": "error", + "unicorn/no-instanceof-builtins": "error", + "unicorn/no-invalid-fetch-options": "error", + "unicorn/no-invalid-remove-event-listener": "error", + "unicorn/no-length-as-slice-end": "error", + "unicorn/no-lonely-if": "error", + "unicorn/no-magic-array-flat-depth": "error", + "unicorn/no-negated-condition": "error", + "unicorn/no-negation-in-equality-check": "error", + "unicorn/no-nested-ternary": "off", + "unicorn/no-new-array": "error", + "unicorn/no-new-buffer": "error", + "unicorn/no-null": "off", + "unicorn/no-object-as-default-parameter": "error", + "unicorn/no-process-exit": "off", + "unicorn/no-single-promise-in-promise-methods": "error", + "unicorn/no-static-only-class": "error", + "unicorn/no-thenable": "error", + "unicorn/no-this-assignment": "error", + "unicorn/no-typeof-undefined": "error", + "unicorn/no-unnecessary-array-flat-depth": "error", + "unicorn/no-unnecessary-array-splice-count": "error", + "unicorn/no-unnecessary-await": "error", + "unicorn/no-unnecessary-slice-end": "error", + "unicorn/no-unreadable-array-destructuring": "error", + "unicorn/no-unreadable-iife": "error", + "unicorn/no-useless-collection-argument": "error", + "unicorn/no-useless-error-capture-stack-trace": "error", + "unicorn/no-useless-fallback-in-spread": "error", + "unicorn/no-useless-length-check": "error", + "unicorn/no-useless-promise-resolve-reject": "error", + "unicorn/no-useless-spread": "error", + "unicorn/no-useless-switch-case": "error", + "unicorn/no-useless-undefined": "error", + "unicorn/no-zero-fractions": "error", + "unicorn/number-literal-case": "off", + "unicorn/numeric-separators-style": "error", + "unicorn/prefer-add-event-listener": "error", + "unicorn/prefer-array-find": "error", + "unicorn/prefer-array-flat": "error", + "unicorn/prefer-array-flat-map": "error", + "unicorn/prefer-array-index-of": "error", + "unicorn/prefer-array-some": "error", + "unicorn/prefer-at": "error", + "unicorn/prefer-bigint-literals": "error", + "unicorn/prefer-blob-reading-methods": "error", + "unicorn/prefer-class-fields": "error", + "unicorn/prefer-classlist-toggle": "error", + "unicorn/prefer-code-point": "error", + "unicorn/prefer-date-now": "error", + "unicorn/prefer-default-parameters": "error", + "unicorn/prefer-dom-node-append": "error", + "unicorn/prefer-dom-node-dataset": "error", + "unicorn/prefer-dom-node-remove": "error", + "unicorn/prefer-dom-node-text-content": "error", + "unicorn/prefer-event-target": "error", + "unicorn/prefer-export-from": "warn", + "unicorn/prefer-global-this": "off", + "unicorn/prefer-import-meta-properties": "error", + "unicorn/prefer-includes": "error", + "unicorn/prefer-keyboard-event-key": "error", + "unicorn/prefer-logical-operator-over-ternary": "error", + "unicorn/prefer-math-min-max": "error", + "unicorn/prefer-math-trunc": "error", + "unicorn/prefer-modern-dom-apis": "error", + "unicorn/prefer-modern-math-apis": "error", + "unicorn/prefer-module": "error", + "unicorn/prefer-native-coercion-functions": "error", + "unicorn/prefer-negative-index": "error", + "unicorn/prefer-node-protocol": "error", + "unicorn/prefer-number-coercion": "warn", + "unicorn/prefer-number-properties": "error", + "unicorn/prefer-object-from-entries": "error", + "unicorn/prefer-optional-catch-binding": "error", + "unicorn/prefer-prototype-methods": "error", + "unicorn/prefer-query-selector": "error", + "unicorn/prefer-reflect-apply": "error", + "unicorn/prefer-regexp-test": "error", + "unicorn/prefer-response-static-json": "error", + "unicorn/prefer-set-has": "error", + "unicorn/prefer-set-size": "error", + "unicorn/prefer-single-call": "error", + "unicorn/prefer-spread": "error", + "unicorn/prefer-string-raw": "off", + "unicorn/prefer-string-replace-all": "error", + "unicorn/prefer-string-slice": "error", + "unicorn/prefer-string-starts-ends-with": "error", + "unicorn/prefer-string-trim-start-end": "error", + "unicorn/prefer-structured-clone": "error", + "unicorn/prefer-ternary": "error", + "unicorn/prefer-top-level-await": "off", + "unicorn/prefer-type-error": "error", + "unicorn/relative-url-style": "error", + "unicorn/require-array-join-separator": "error", + "unicorn/require-module-attributes": "error", + "unicorn/require-module-specifiers": "error", + "unicorn/require-number-to-fixed-digits-argument": "error", + "unicorn/require-post-message-target-origin": "error", + "unicorn/switch-case-braces": "error", + "unicorn/switch-case-break-position": "error", + "unicorn/text-encoding-identifier-case": ["error", { withDash: true }], + "unicorn/throw-new-error": "error", + + // ── react (project-level overrides) ───────────────────────────────────── + "react/react-compiler": "off", + "react/no-danger": "warn", + "react/no-clone-element": "warn", + "react/no-react-children": "warn", + "react/jsx-pascal-case": "off", + "react/state-in-constructor": "off", + "react/jsx-no-constructed-context-values": "warn", + "react/hook-use-state": "warn", + "react/button-has-type": "warn", + "react/jsx-no-useless-fragment": "warn", + "jsx-a11y/prefer-tag-over-role": "off", + "jsx-a11y/control-has-associated-label": "off", + + // ── react-doctor (project-level overrides) ────────────────────────────── + "react-doctor/react-compiler-no-manual-memoization": "off", + "react-doctor/no-react19-deprecated-apis": "off", + "react-doctor/async-parallel": "warn", + "react-doctor/async-await-in-loop": "warn", + "react-doctor/async-defer-await": "warn", + "react-doctor/server-sequential-independent-await": "warn", + "react-doctor/server-hoist-static-io": "warn", + "react-doctor/js-set-map-lookups": "warn", + "react-doctor/no-giant-component": "warn", + "react-doctor/only-export-components": "warn", + "react-doctor/no-barrel-import": "warn", + "react-doctor/no-polymorphic-children": "warn", + "react-doctor/prefer-module-scope-pure-function": "warn", + "react-doctor/prefer-module-scope-static-value": "warn", + "react-doctor/rerender-state-only-in-handlers": "warn", + "react-doctor/rerender-memo-with-default-value": "warn", + "react-doctor/no-render-in-render": "warn", + "react-doctor/no-cascading-set-state": "warn", + "react-doctor/no-array-index-as-key": "warn", + "react-doctor/no-derived-useState": "warn", + "react-doctor/no-derived-state": "warn", + "react-doctor/no-initialize-state": "warn", + "react-doctor/rendering-hydration-no-flicker": "warn", + "react-doctor/rendering-hydration-mismatch-time": "warn", + "react-doctor/auth-token-in-web-storage": "warn", + "react-doctor/js-flatmap-filter": "warn", + "react-doctor/js-combine-iterations": "warn", + }, +}) diff --git a/apps/docs/.gitignore b/apps/docs/.gitignore index f650315f..e6ce5a4c 100644 --- a/apps/docs/.gitignore +++ b/apps/docs/.gitignore @@ -1,7 +1,15 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - # dependencies /node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage # next.js /.next/ @@ -10,18 +18,19 @@ # production /build +# misc +.DS_Store +*.pem + # debug npm-debug.log* yarn-debug.log* yarn-error.log* -.pnpm-debug.log* +pnpm-debug.log* # env files -.env* - -# vercel -.vercel +.env*.local # typescript *.tsbuildinfo -next-env.d.ts \ No newline at end of file +next-env.d.ts diff --git a/apps/docs/README.md b/apps/docs/README.md new file mode 100644 index 00000000..29a87698 --- /dev/null +++ b/apps/docs/README.md @@ -0,0 +1,21 @@ +# Next.js template + +This is a Next.js template with shadcn/ui. + +## Adding components + +To add components to your app, run the following command: + +```bash +npx shadcn@latest add button +``` + +This will place the ui components in the `components` directory. + +## Using components + +To use the components in your app, import them as follows: + +```tsx +import { Button } from "@/components/ui/button" +``` diff --git a/apps/docs/components.json b/apps/docs/components.json index 540fc6e9..028d1561 100644 --- a/apps/docs/components.json +++ b/apps/docs/components.json @@ -1,21 +1,27 @@ { "$schema": "https://ui.shadcn.com/schema.json", - "style": "default", + "style": "base-nova", "rsc": true, "tsx": true, "tailwind": { "config": "", - "css": "src/app/globals.css", + "css": "app/globals.css", "baseColor": "neutral", "cssVariables": true, "prefix": "" }, + "iconLibrary": "lucide", + "rtl": false, + "menuColor": "default", + "menuAccent": "subtle", "aliases": { - "components": "~/components", - "utils": "~/lib/utils", - "ui": "~/components/ui", - "lib": "~/lib", - "hooks": "~/hooks" + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" }, - "iconLibrary": "lucide" + "registries": { + "@reui": "https://reui.io/r/{style}/{name}.json" + } } diff --git a/apps/docs/content/api-reference/01.core/01.queue.mdx b/apps/docs/content/api-reference/01.core/01.queue.mdx new file mode 100644 index 00000000..adb676e5 --- /dev/null +++ b/apps/docs/content/api-reference/01.core/01.queue.mdx @@ -0,0 +1,9 @@ +--- +title: Queue +description: The main Queue class for managing jobs, handlers, and processing lifecycle. +apiReference: + - name: Queue + file: core/src/queue +--- + +The `Queue` class is the primary interface for adding jobs, registering handlers, and controlling processing. diff --git a/apps/docs/content/api-reference/01.core/02.worker.mdx b/apps/docs/content/api-reference/01.core/02.worker.mdx new file mode 100644 index 00000000..6546d9a4 --- /dev/null +++ b/apps/docs/content/api-reference/01.core/02.worker.mdx @@ -0,0 +1,9 @@ +--- +title: Worker +description: The Worker class handles job execution, retries, timeouts, and batch processing. +apiReference: + - name: Worker + file: core/src/worker +--- + +The `Worker` class is responsible for polling, executing, and managing the lifecycle of jobs. diff --git a/apps/docs/content/api-reference/01.core/03.types.mdx b/apps/docs/content/api-reference/01.core/03.types.mdx new file mode 100644 index 00000000..ff171150 --- /dev/null +++ b/apps/docs/content/api-reference/01.core/03.types.mdx @@ -0,0 +1,9 @@ +--- +title: Types +description: Core type definitions — Job, JobOptions, QueueConfig, and adapter interfaces. +apiReference: + - name: Types + file: core/src/types +--- + +Type definitions for jobs, queue configuration, adapter contracts, and handler signatures. diff --git a/apps/docs/content/api-reference/01.core/04.events.mdx b/apps/docs/content/api-reference/01.core/04.events.mdx new file mode 100644 index 00000000..ec20c64d --- /dev/null +++ b/apps/docs/content/api-reference/01.core/04.events.mdx @@ -0,0 +1,9 @@ +--- +title: Events +description: Typed event emitter and event type definitions for job lifecycle. +apiReference: + - name: TypedEventEmitter + file: core/src/events +--- + +The typed event system for subscribing to job and queue lifecycle events. diff --git a/apps/docs/content/api-reference/01.core/05.errors.mdx b/apps/docs/content/api-reference/01.core/05.errors.mdx new file mode 100644 index 00000000..236cdb0a --- /dev/null +++ b/apps/docs/content/api-reference/01.core/05.errors.mdx @@ -0,0 +1,9 @@ +--- +title: Errors +description: Custom error classes for timeout, duplicate jobs, cancellation, and invalid transitions. +apiReference: + - name: Errors + file: core/src/errors +--- + +Error classes thrown by the queue engine during job processing. diff --git a/apps/docs/content/api-reference/01.core/06.adapters.mdx b/apps/docs/content/api-reference/01.core/06.adapters.mdx new file mode 100644 index 00000000..786294ee --- /dev/null +++ b/apps/docs/content/api-reference/01.core/06.adapters.mdx @@ -0,0 +1,11 @@ +--- +title: Base Adapters +description: Abstract base class and memory adapter for implementing queue storage backends. +apiReference: + - name: BaseQueueAdapter + file: core/src/adapters/base + - name: MemoryQueueAdapter + file: core/src/adapters/memory +--- + +The base adapter interface that all storage backends must implement, plus the built-in memory adapter for testing. diff --git a/apps/docs/content/api-reference/01.core/index.mdx b/apps/docs/content/api-reference/01.core/index.mdx new file mode 100644 index 00000000..09cba2e5 --- /dev/null +++ b/apps/docs/content/api-reference/01.core/index.mdx @@ -0,0 +1,5 @@ +--- +title: "@vorsteh-queue/core" +navTitle: Core +description: Core queue engine — Queue, Worker, adapters, events, errors, and type definitions. +--- diff --git a/apps/docs/content/api-reference/index.mdx b/apps/docs/content/api-reference/index.mdx new file mode 100644 index 00000000..b1ef8b90 --- /dev/null +++ b/apps/docs/content/api-reference/index.mdx @@ -0,0 +1,6 @@ +--- +title: API Reference +navTitle: API Reference +description: Auto-generated API reference for all Vorsteh Queue packages. +entrypoint: /docs/api-reference/core/queue +--- diff --git a/apps/docs/content/cli/01.overview/01.installation.mdx b/apps/docs/content/cli/01.overview/01.installation.mdx new file mode 100644 index 00000000..f7a01aab --- /dev/null +++ b/apps/docs/content/cli/01.overview/01.installation.mdx @@ -0,0 +1,174 @@ +--- +title: Installation +description: How to install and configure the Vorsteh Queue CLI for job management and monitoring. +--- + +The Vorsteh Queue CLI (`@vorsteh-queue/cli`) provides commands for inspecting, managing, and monitoring queue jobs from the terminal. + +## Install as a project dependency + +@vorsteh-queue/cli + +## Run without installing + +vorsteh-queue + +## Transport Modes + +The CLI supports two transport modes for communicating with your queue: + +### Direct (default) + +In direct mode, the CLI loads your `queue.config.ts` and uses the configured adapter to interact with the database directly. No running server is required. + +This is the default behavior when no `--url` flag or `VORSTEH_QUEUE_URL` environment variable is provided. + +### Remote + +In remote mode, the CLI connects to a running Vorsteh Queue server via GraphQL. This mode is activated by providing a URL through the `--url` flag or the `VORSTEH_QUEUE_URL` environment variable. + +```bash +# Via flag +vorsteh-queue status --url https://my-server.example.com/graphql + +# Via environment variable +export VORSTEH_QUEUE_URL=https://my-server.example.com/graphql +vorsteh-queue status +``` + +## Configuration + +For direct mode, create a `queue.config.ts` in your project root: + + + + ```typescript + // queue.config.ts + import { defineRelations } from "drizzle-orm" + import { drizzle } from "drizzle-orm/node-postgres" + import { Pool } from "pg" + + import { + PostgresQueueAdapter, + queueFlows, + queueJobs, + } from "@vorsteh-queue/adapter-drizzle" + import { Queue } from "@vorsteh-queue/core" + + const relations = defineRelations({ queueFlows, queueJobs }) + + const pool = new Pool({ connectionString: process.env.DATABASE_URL }) + const db = drizzle({ client: pool, relations }) + const adapter = new PostgresQueueAdapter(db) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + } + ``` + + + + ```typescript + // queue.config.ts + import { PrismaClient } from "@prisma/client" + + import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" + import { Queue } from "@vorsteh-queue/core" + + const prisma = new PrismaClient() + const adapter = new PostgresPrismaQueueAdapter(prisma) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + } + ``` + + + + ```typescript + // queue.config.ts + import { Kysely, PostgresDialect } from "kysely" + import { Pool } from "pg" + + import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" + import { Queue } from "@vorsteh-queue/core" + + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + const adapter = new PostgresQueueAdapter(db) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + } + ``` + + + + +The `adapter` and `queues` fields are required for direct mode. If either is missing, the CLI will exit with an error. When multiple queues are configured, `defaultQueue` is also required. + +For a single-queue setup, `defaultQueue` is optional (the single queue is used automatically): + +```typescript +export default { + adapter, + queues: [emailQueue], +} +``` + +## Global Options + +These options can be passed to any command: + +| Option | Description | +| --- | --- | +| `--url ` | Remote server GraphQL endpoint. Enables remote mode. | +| `--token ` | Bearer token for authentication with the remote server. | +| `--queue ` | Target queue name. Overrides the default queue from configuration. | + +The `--url` flag takes precedence over the `VORSTEH_QUEUE_URL` environment variable. Likewise, `--token` takes precedence over `VORSTEH_QUEUE_TOKEN`. + +### Queue Resolution + +The `--queue` flag determines which queue a command targets. Resolution priority: + +1. `--queue` flag (highest priority) +2. `defaultQueue` from config +3. Single queue in array (when only one queue is configured) + +In remote mode (`--url` without a local config file), the `--queue` flag is required. + +## Environment Variables + +| Variable | Description | +| --- | --- | +| `VORSTEH_QUEUE_URL` | Remote server GraphQL endpoint. Enables remote mode when set. | +| `VORSTEH_QUEUE_TOKEN` | Bearer token for authentication with the remote server. | + +## Troubleshooting + +Use the `doctor` command to diagnose configuration and connectivity issues: + +```bash +vorsteh-queue doctor +``` + +The doctor command displays your current transport mode, adapter or endpoint details, authentication status, and verifies connectivity. It exits with code 0 on success or 1 if there is a problem. diff --git a/apps/docs/content/cli/01.overview/index.mdx b/apps/docs/content/cli/01.overview/index.mdx new file mode 100644 index 00000000..87ec9b8b --- /dev/null +++ b/apps/docs/content/cli/01.overview/index.mdx @@ -0,0 +1,4 @@ +--- +title: Overview +description: Overview of the Vorsteh Queue CLI tool. +--- diff --git a/apps/docs/content/cli/02.commands/00.global-options.mdx b/apps/docs/content/cli/02.commands/00.global-options.mdx new file mode 100644 index 00000000..cae3f645 --- /dev/null +++ b/apps/docs/content/cli/02.commands/00.global-options.mdx @@ -0,0 +1,23 @@ +--- +title: Global CLI Options +description: Reference for all available global CLI options with syntax, options, and usage examples. +--- + +All commands support the same global transport and targeting options: + + + +Queue resolution priority: + +1. `--queue` flag +2. `defaultQueue` from `queue.config.ts` +3. Single configured queue (when only one queue exists) + +In remote mode without a local config file, `--queue` is required. + +Examples: + +```bash +vorsteh-queue status --queue email-queue +vorsteh-queue inspect --url https://my-server.example.com/graphql --queue report-queue +``` diff --git a/apps/docs/content/cli/02.commands/01.status.mdx b/apps/docs/content/cli/02.commands/01.status.mdx new file mode 100644 index 00000000..2e382ee7 --- /dev/null +++ b/apps/docs/content/cli/02.commands/01.status.mdx @@ -0,0 +1,36 @@ +--- +title: "vorsteh-queue status" +navTitle: "status" +description: Show queue status overview with job counts per state. +cliCommand: "status" +--- + +Displays a summary of job counts grouped by status. + +## Usage + +```bash +vorsteh-queue status +``` + +``` +ℹ Queue Status: + Pending: 12 + Delayed: 3 + Processing: 2 + Completed: 847 + Failed: 5 + Cancelled: 1 + Dead: 0 + Size: 870 +``` + +## JSON Output + +```bash +vorsteh-queue status --json +``` + +Returns structured JSON for scripting and CI pipelines. + + diff --git a/apps/docs/content/cli/02.commands/02.inspect.mdx b/apps/docs/content/cli/02.commands/02.inspect.mdx new file mode 100644 index 00000000..93008db4 --- /dev/null +++ b/apps/docs/content/cli/02.commands/02.inspect.mdx @@ -0,0 +1,29 @@ +--- +title: "vorsteh-queue inspect" +navTitle: "inspect" +description: Show details of a specific job. +cliCommand: "inspect" +--- + +Displays detailed information about a single job including status, payload, result, and error data. + +## Usage + +```bash +vorsteh-queue inspect +``` + +``` +ℹ Job: abc-123 + Name: send-email + Status: completed + Priority: 2 + Attempts: 1/3 + Progress: 100% + Created: 2025-01-15T10:30:00Z + Process At: 2025-01-15T10:30:00Z + Result: {"sent":true} + Payload: {"to":"user@example.com"} +``` + + diff --git a/apps/docs/content/cli/02.commands/03.cancel.mdx b/apps/docs/content/cli/02.commands/03.cancel.mdx new file mode 100644 index 00000000..04aac173 --- /dev/null +++ b/apps/docs/content/cli/02.commands/03.cancel.mdx @@ -0,0 +1,20 @@ +--- +title: "vorsteh-queue cancel" +navTitle: "cancel" +description: Cancel a pending or delayed job. +cliCommand: "cancel" +--- + +Cancels a job, moving it to the `cancelled` status. Only jobs that are not yet in a terminal state can be cancelled. + +## Usage + +```bash +vorsteh-queue cancel +``` + +```bash +vorsteh-queue cancel --reason "no longer needed" +``` + + diff --git a/apps/docs/content/cli/02.commands/04.retry.mdx b/apps/docs/content/cli/02.commands/04.retry.mdx new file mode 100644 index 00000000..e1f43ab7 --- /dev/null +++ b/apps/docs/content/cli/02.commands/04.retry.mdx @@ -0,0 +1,16 @@ +--- +title: "vorsteh-queue retry" +navTitle: "retry" +description: Retry a failed job by resetting it to pending. +cliCommand: "retry" +--- + +Resets a failed job to `pending` status so it can be processed again. The job must be in `failed` status. + +## Usage + +```bash +vorsteh-queue retry +``` + + diff --git a/apps/docs/content/cli/02.commands/05.redrive.mdx b/apps/docs/content/cli/02.commands/05.redrive.mdx new file mode 100644 index 00000000..0ebbe376 --- /dev/null +++ b/apps/docs/content/cli/02.commands/05.redrive.mdx @@ -0,0 +1,23 @@ +--- +title: "vorsteh-queue redrive" +navTitle: "redrive" +description: Redrive dead jobs back to pending for reprocessing. +cliCommand: "redrive" +--- + +Moves dead jobs (jobs that exhausted all retry attempts) back to `pending` status for another attempt. + +## Usage + +```bash +# Redrive a single job +vorsteh-queue redrive + +# Redrive all dead jobs +vorsteh-queue redrive --all + +# Redrive dead jobs filtered by name +vorsteh-queue redrive --all --name send-email +``` + + diff --git a/apps/docs/content/cli/02.commands/06.run-now.mdx b/apps/docs/content/cli/02.commands/06.run-now.mdx new file mode 100644 index 00000000..b562e695 --- /dev/null +++ b/apps/docs/content/cli/02.commands/06.run-now.mdx @@ -0,0 +1,16 @@ +--- +title: "vorsteh-queue run-now" +navTitle: "run-now" +description: Promote a delayed job to run immediately. +cliCommand: "run-now" +--- + +Promotes a delayed job to `pending` status, making it eligible for immediate processing. The job must be in `delayed` status. + +## Usage + +```bash +vorsteh-queue run-now +``` + + diff --git a/apps/docs/content/cli/02.commands/07.delete.mdx b/apps/docs/content/cli/02.commands/07.delete.mdx new file mode 100644 index 00000000..28c9cae4 --- /dev/null +++ b/apps/docs/content/cli/02.commands/07.delete.mdx @@ -0,0 +1,16 @@ +--- +title: "vorsteh-queue delete" +navTitle: "delete" +description: Delete a single job from the queue. +cliCommand: "delete" +--- + +Permanently removes a job from the queue. This operation cannot be undone. + +## Usage + +```bash +vorsteh-queue delete +``` + + diff --git a/apps/docs/content/cli/02.commands/08.clear.mdx b/apps/docs/content/cli/02.commands/08.clear.mdx new file mode 100644 index 00000000..b5e03dad --- /dev/null +++ b/apps/docs/content/cli/02.commands/08.clear.mdx @@ -0,0 +1,24 @@ +--- +title: "vorsteh-queue clear" +navTitle: "clear" +description: Clear jobs from the queue by status. +cliCommand: "clear" +--- + +Removes multiple jobs from the queue. Use `--status` to target a specific state or `--all` to clear everything. + +## Usage + +```bash +# Clear completed jobs +vorsteh-queue clear --status completed + +# Clear all jobs +vorsteh-queue clear --all +``` + +## Valid Statuses + +`pending`, `delayed`, `processing`, `completed`, `failed`, `cancelled`, `dead` + + diff --git a/apps/docs/content/cli/02.commands/09.flow.mdx b/apps/docs/content/cli/02.commands/09.flow.mdx new file mode 100644 index 00000000..4e658e44 --- /dev/null +++ b/apps/docs/content/cli/02.commands/09.flow.mdx @@ -0,0 +1,25 @@ +--- +title: "vorsteh-queue flow" +navTitle: "flow" +description: Show a flow tree (parent-child job hierarchy). +cliCommand: "flow" +--- + +Displays the parent-child job hierarchy as a tree, useful for debugging flow-based job pipelines. + +## Usage + +```bash +vorsteh-queue flow +``` + +``` +ℹ Flow: process-order [flow-id: flow_abc123] + +process-order (completed) ✓ +├── validate-payment (completed) ✓ +├── reserve-stock (completed) ✓ +└── send-confirmation (processing) ⟳ +``` + + diff --git a/apps/docs/content/cli/02.commands/10.serve.mdx b/apps/docs/content/cli/02.commands/10.serve.mdx new file mode 100644 index 00000000..753cd80f --- /dev/null +++ b/apps/docs/content/cli/02.commands/10.serve.mdx @@ -0,0 +1,110 @@ +--- +title: "vorsteh-queue serve" +navTitle: "serve" +description: Start the graphql server +cliCommand: "serve" +--- + +Starts the GraphQL API server. Requires a `queue.config.ts` with a direct adapter configuration. + +## Usage + +```bash +vorsteh-queue serve +``` + +```bash +vorsteh-queue serve --port 4000 +``` + +## Configuration + +The serve command reads from `queue.config.ts`: + + + + ```typescript + // queue.config.ts + import { defineRelations } from "drizzle-orm" + import { drizzle } from "drizzle-orm/node-postgres" + import { Pool } from "pg" + + import { + PostgresQueueAdapter, + queueFlows, + queueJobs, + } from "@vorsteh-queue/adapter-drizzle" + import { Queue } from "@vorsteh-queue/core" + + const relations = defineRelations({ queueFlows, queueJobs }) + + const pool = new Pool({ connectionString: process.env.DATABASE_URL }) + const db = drizzle({ client: pool, relations }) + const adapter = new PostgresQueueAdapter(db) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + port: 3000, + } + ``` + + + + ```typescript + // queue.config.ts + import { PrismaClient } from "@prisma/client" + + import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" + import { Queue } from "@vorsteh-queue/core" + + const prisma = new PrismaClient() + const adapter = new PostgresPrismaQueueAdapter(prisma) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + port: 3000, + } + ``` + + + + ```typescript + // queue.config.ts + import { Kysely, PostgresDialect } from "kysely" + import { Pool } from "pg" + + import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" + import { Queue } from "@vorsteh-queue/core" + + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + const adapter = new PostgresQueueAdapter(db) + + const emailQueue = new Queue(adapter, { name: "email-queue" }) + const reportQueue = new Queue(adapter, { name: "report-queue" }) + + export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", + port: 3000, + } + ``` + + + + + diff --git a/apps/docs/content/cli/02.commands/11.doctor.mdx b/apps/docs/content/cli/02.commands/11.doctor.mdx new file mode 100644 index 00000000..c04d6dff --- /dev/null +++ b/apps/docs/content/cli/02.commands/11.doctor.mdx @@ -0,0 +1,54 @@ +--- +title: "vorsteh-queue doctor" +navTitle: "doctor" +description: Check configuration and connectivity. +cliCommand: "doctor" +--- + +Verifies your CLI configuration and tests connectivity to the queue backend. Useful for troubleshooting connection issues. + +## Usage + +```bash +vorsteh-queue doctor +``` + +### Direct Mode Output + +``` +ℹ Mode: Direct +ℹ Adapter: configured via queue.config.ts +✔ Connection: OK +``` + +### Remote Mode Output + +```bash +vorsteh-queue doctor --url http://localhost:3000/graphql --token my-token +``` + +``` +ℹ Mode: Remote +ℹ Endpoint: http://localhost:3000/graphql +ℹ Authentication: bearer token +✔ Connection: OK +``` + +### Connection Failure + +``` +ℹ Mode: Remote +ℹ Endpoint: http://localhost:3000/graphql +ℹ Authentication: none +✗ Connection: FAILED +✗ Reason: Could not connect to http://localhost:3000/health. Verify the server is running and the URL is correct. +``` + +## Exit Codes + +| Code | Meaning | +| ---- | --------------------------------------------- | +| `0` | Configuration valid and connection successful | +| `1` | Connection failed or configuration error | + + diff --git a/apps/docs/content/cli/02.commands/12.queues.mdx b/apps/docs/content/cli/02.commands/12.queues.mdx new file mode 100644 index 00000000..a4248fc3 --- /dev/null +++ b/apps/docs/content/cli/02.commands/12.queues.mdx @@ -0,0 +1,39 @@ +--- +title: "vorsteh-queue queues" +navTitle: "queues" +description: List all configured queues. +cliCommand: "queues" +--- + +Lists all queues defined in your `queue.config.ts`. Shows each queue name and indicates which one is the default. + +## Usage + +```bash +vorsteh-queue queues +``` + +``` +ℹ Available queues: + email-queue (default) + report-queue +``` + +## JSON Output + +```bash +vorsteh-queue queues --json +``` + +```json +[ + { "name": "email-queue", "isDefault": true }, + { "name": "report-queue", "isDefault": false } +] +``` + +## Notes + +This command reads directly from the config file and does not require a database connection or running server. + + diff --git a/apps/docs/content/cli/02.commands/13.dashboard.mdx b/apps/docs/content/cli/02.commands/13.dashboard.mdx new file mode 100644 index 00000000..c93ce793 --- /dev/null +++ b/apps/docs/content/cli/02.commands/13.dashboard.mdx @@ -0,0 +1,81 @@ +--- +title: "vorsteh-queue dashboard" +navTitle: "dashboard" +description: Interactive TUI dashboard for monitoring and managing jobs +cliCommand: "dashboard" +--- + +Opens an interactive terminal dashboard for real-time queue monitoring and job management. + +## Usage + +```bash +vorsteh-queue dashboard +``` + +```bash +vorsteh-queue dashboard --url http://localhost:3000/graphql --token secret +``` + +```bash +vorsteh-queue dashboard --refresh 5000 +``` + +## Views + +The dashboard provides three views, navigable via keyboard: + +### Stats View + +Displays queue statistics with visual bar chart: + +- Pending, Delayed, Processing, Completed, Failed, Cancelled, Dead counts +- Auto-refreshes at the configured interval + +### Jobs View + +Paginated job list with: + +- Status filter (all, pending, processing, completed, failed, dead, delayed, cancelled) +- Priority and attempt count per job +- Keyboard navigation and pagination + +### Detail View + +Full job inspection with: + +- All job metadata (ID, status, priority, timestamps, progress) +- Payload and result display +- Error details for failed jobs +- Inline actions: Cancel, Retry, Run Now, Delete (with confirmation) + +## Keyboard Navigation + +| View | Key | Action | +| ------ | ------------- | ---------------------- | +| Stats | `j` / `Enter` | Go to Jobs view | +| Jobs | `↑` / `k` | Move cursor up | +| Jobs | `↓` / `j` | Move cursor down | +| Jobs | `←` / `h` | Previous status filter | +| Jobs | `→` / `l` | Next status filter | +| Jobs | `Enter` | Open job detail | +| Jobs | `Ctrl+D` | Next page | +| Jobs | `Ctrl+U` | Previous page | +| Jobs | `g` | Jump to first page | +| Jobs | `G` | Jump to last item | +| Detail | `c` | Cancel job | +| Detail | `r` | Retry job | +| Detail | `n` | Run job now | +| Detail | `d` | Delete job | +| Any | `Esc` | Go back | +| Any | `q` | Quit | + +## Transport + +The dashboard uses the same transport resolution as all other commands: + +1. `--url` flag → connects to remote GraphQL server +2. `VORSTEH_QUEUE_URL` env var → remote mode +3. `queue.config.ts` → direct adapter connection + + diff --git a/apps/docs/content/cli/02.commands/index.mdx b/apps/docs/content/cli/02.commands/index.mdx new file mode 100644 index 00000000..41bf98c0 --- /dev/null +++ b/apps/docs/content/cli/02.commands/index.mdx @@ -0,0 +1,23 @@ +--- +title: Commands +description: Reference for all available CLI commands with syntax, options, and usage examples. +--- + +All commands support the same global transport and targeting options: + + + +Queue resolution priority: + +1. `--queue` flag +2. `defaultQueue` from `queue.config.ts` +3. Single configured queue (when only one queue exists) + +In remote mode without a local config file, `--queue` is required. + +Examples: + +```bash +vorsteh-queue status --queue email-queue +vorsteh-queue inspect --url https://my-server.example.com/graphql --queue report-queue +``` diff --git a/apps/docs/content/cli/index.mdx b/apps/docs/content/cli/index.mdx new file mode 100644 index 00000000..ee00abbf --- /dev/null +++ b/apps/docs/content/cli/index.mdx @@ -0,0 +1,6 @@ +--- +title: Vorsteh Queue CLI +navTitle: CLI +description: Command-line tool for monitoring and managing Vorsteh Queue jobs. +entrypoint: /docs/cli/overview/installation +--- diff --git a/apps/docs/content/docs/01.getting-started/01.introduction.mdx b/apps/docs/content/docs/01.getting-started/01.introduction.mdx deleted file mode 100644 index 0fffb6fe..00000000 --- a/apps/docs/content/docs/01.getting-started/01.introduction.mdx +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Introduction -description: Get started with Vorsteh Queue, a powerful TypeScript-first job queue for PostgreSQL. ---- - -Vorsteh Queue is a powerful, type-safe queue engine designed for PostgreSQL 12+. It provides a robust solution for handling background jobs, scheduled tasks, and recurring processes in your Node.js applications. - -## Key Features - -- **Type-safe**: Full TypeScript support with generic job payloads -- **Multiple adapters**: Drizzle ORM (PostgreSQL), Prisma ORM (PostgreSQL), Kysely (PostgreSQL) and in-memory implementations -- **Priority queues**: Numeric priority system (lower = higher priority) -- **Delayed jobs**: Schedule jobs for future execution -- **Recurring jobs**: Cron expressions and interval-based repetition -- **UTC-first timezone support**: Reliable timezone handling with UTC storage -- **Progress tracking**: Real-time job progress updates -- **Event system**: Listen to job lifecycle events -- **Graceful shutdown**: Clean job processing termination - -## Requirements - -- **Node.js 20+** -- **ESM only** - This package is ESM-only -- **PostgreSQL 12+** for production use - -## Architecture - -Vorsteh Queue follows a modular architecture with: - -- **Core package** - Contains the main queue logic and interfaces -- **Adapter packages** - ORM-specific implementations -- **UTC-first design** - All timestamps stored as UTC for consistency -- **Event-driven** - Comprehensive event system for monitoring and debugging diff --git a/apps/docs/content/docs/01.getting-started/02.installation.mdx b/apps/docs/content/docs/01.getting-started/02.installation.mdx deleted file mode 100644 index 5e36346b..00000000 --- a/apps/docs/content/docs/01.getting-started/02.installation.mdx +++ /dev/null @@ -1,81 +0,0 @@ ---- -title: Installation -description: Learn how to install Vorsteh Queue packages for your preferred ORM adapter. ---- - -## Prerequisites - -Before installing Vorsteh Queue, ensure you have: - -- **Node.js 20+** -- **PostgreSQL 12+** database -- **ESM project** - Add `"type": "module"` to your `package.json` or use `.mjs` file extensions - -## Quick Start with CLI (Recommended) - -The fastest way to get started is using the `create-vorsteh-queue` CLI tool: - -```bash -# Interactive setup -npx create-vorsteh-queue my-queue-app - -# Or with specific options -npx create-vorsteh-queue worker-service \ - --template=drizzle-postgres \ - --package-manager=pnpm \ - --quiet -``` - -### Available Templates - -- **drizzle-pg** - Basic Drizzle ORM with node-postgres -- **drizzle-pglite** - Zero-setup with embedded PostgreSQL -- **drizzle-postgres** - Advanced with recurring jobs -- **event-system** - Comprehensive event monitoring -- **progress-tracking** - Real-time progress updates -- **pm2-workers** - Multi-process workers with PM2 - -## Manual Installation - -If you prefer to add Vorsteh Queue to an existing project: - -### Drizzle ORM (PostgreSQL) - - - @vorsteh-queue/core @vorsteh-queue/adapter-drizzle - - -### Prisma ORM (PostgreSQL) - - - @vorsteh-queue/core @vorsteh-queue/adapter-prisma - - -### Kysely (PostgreSQL) - - - @vorsteh-queue/core @vorsteh-queue/adapter-kysely - - -## Database Setup - -After installation, you'll need to set up the required database tables. Each adapter provides migration scripts or schema definitions: - -- **Drizzle**: Use the provided schema and run migrations -- **Prisma**: Add the schema to your `schema.prisma` and run `prisma migrate` -- **Kysely**: Use the provided schema and run migrations - -Refer to the specific adapter documentation for detailed setup instructions. - -## Verify Installation - -Create a simple test file to verify your installation: - -```typescript -import { Queue } from "@vorsteh-queue/core" - -// Your adapter import will vary based on your choice -console.log("Vorsteh Queue installed successfully!") -``` - -Run the file with Node.js to confirm everything is working correctly. diff --git a/apps/docs/content/docs/02.packages/01.core.mdx b/apps/docs/content/docs/02.packages/01.core.mdx deleted file mode 100644 index f062737b..00000000 --- a/apps/docs/content/docs/02.packages/01.core.mdx +++ /dev/null @@ -1,325 +0,0 @@ ---- -title: Core Package -navTitle: "@vorsteh-queue/core" -description: The main queue engine containing all core functionality for job processing, scheduling, and management. ---- - -The core package contains the main queue engine and all essential functionality for job processing, scheduling, and event management. - -## Installation - - - @vorsteh-queue/core - - -## Key Features - -- **Queue Management** - Create and manage multiple queues -- **Job Processing** - Register handlers and process jobs -- **Event System** - Comprehensive event lifecycle -- **Progress Tracking** - Real-time job progress updates -- **Scheduling** - Delayed and recurring jobs -- **Priority System** - Numeric priority-based processing -- **Graceful Shutdown** - Clean termination handling - -## Core Methods - -### Registering Job Handlers - -Define how jobs should be processed: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -interface Payload { - to: string - subject: string -} - -interface Result { - messageId: string - sent: true -} - -queue.register("send-email", async (job) => { - //await sendEmail(job.payload) - return { messageId: "msg_123", sent: true } -}) -``` - -### Adding Jobs - -Add jobs to the queue for processing: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -interface EmailPayload { - to: string - subject: string -} - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -// Basic job -await queue.add("send-email", { - to: "user@example.com", - subject: "Welcome!", -}) - -// Job with options -const payload: EmailPayload = { - to: "user@example.com", - subject: "Welcome!", -} - -await queue.add("send-email", payload, { - priority: 1, - delay: 5000, - maxAttempts: 5, -}) -``` - -### Queue Control - -Manage queue processing: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -// Start processing jobs -queue.start() - -// Pause processing -queue.pause() - -// Resume processing -queue.resume() - -// Graceful shutdown -await queue.stop() -``` - -### Progress Tracking - -Update job progress in real-time: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -interface ProcessPayload { - items: string[] -} - -interface ProcessResult { - processed: number -} - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -queue.register("process-data", async (job) => { - const items = job.payload.items - - for (let i = 0; i < items.length; i++) { - //await processItem(items[i]) - - // Update progress (0-100) - const progress = Math.round(((i + 1) / items.length) * 100) - await job.updateProgress(progress) - } - - return { processed: items.length } -}) -``` - -### Scheduling Jobs - -Schedule jobs for future execution: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -interface CleanupPayload { - type: string -} - -interface ReportPayload { - date: string -} - -interface HealthCheckPayload { - endpoint: string -} - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -// Delayed job -await queue.add( - "cleanup", - { type: "temp-files" }, - { - delay: 60000, // 1 minute - }, -) - -// Recurring job with cron -await queue.add( - "daily-report", - { date: new Date().toISOString() }, - { - cron: "0 9 * * *", // Every day at 9 AM - timezone: "America/New_York", - }, -) - -// Recurring job with interval -await queue.add( - "health-check", - { endpoint: "/api/health" }, - { - repeat: { every: 30000, limit: 10 }, // Every 30s, 10 times - }, -) -``` - -### Event Handling - -Listen to job and queue events: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -// Job lifecycle events -queue.on("job:added", (job) => { - console.log(`Job ${job.name} added to queue`) -}) - -queue.on("job:completed", (job) => { - console.log(`Job ${job.name} completed`) - console.log("Result:", job.result) -}) - -queue.on("job:failed", (job) => { - console.error(`Job ${job.name} failed:`, job.error) -}) - -queue.on("job:progress", (job) => { - console.log(`Job ${job.name}: ${job.progress}% complete`) -}) - -// Queue control events -queue.on("queue:paused", () => { - console.log("Queue paused") -}) - -queue.on("queue:resumed", () => { - console.log("Queue resumed") -}) -``` - -### Queue Statistics - -Get queue metrics and status: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -// Get current stats -const stats = await queue.getStats() -console.log(stats) // { pending: 5, processing: 2, completed: 100, failed: 3 } - -// Clear jobs -await queue.clear() // Clear all jobs -await queue.clear("failed") // Clear only failed jobs -``` - -### Error Handling - -Handle job failures and timeouts: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -interface RiskyJobPayload { - operation: string - data: unknown -} - -interface RiskyJobResult { - success: boolean - error?: string -} - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -queue.register("risky-job", async (job) => { - try { - //await riskyOperation(job.payload) - return { success: true } - } catch (error) { - throw error // Re-throw to mark job as failed - } -}) - -// Set job timeout -const payload: RiskyJobPayload = { - operation: "data-processing", - data: { items: [1, 2, 3] }, -} - -await queue.add("risky-job", payload, { - timeout: 30000, // 30 seconds -}) -``` - -## Memory Adapter - -Includes a built-in memory adapter for testing and development: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) -``` - -## TypeScript Support - -Full TypeScript support with generic job payloads: - -```ts -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -const queue = new Queue(new MemoryQueueAdapter(), { name: "example" }) - -interface EmailPayload { - to: string - subject: string - body: string -} - -interface EmailResult { - success: boolean -} - -queue.register("send-email", async (job) => { - // job.payload is typed as EmailPayload - //await sendEmail(job.payload) - - return { - success: true, - } -}) -``` - -## References - -- Sources: https://github.com/noxify/vorsteh-queue/tree/main/packages/core -- NPM: https://www.npmjs.com/package/@vorsteh-queue/core diff --git a/apps/docs/content/docs/02.packages/02.create-vorsteh-queue.mdx b/apps/docs/content/docs/02.packages/02.create-vorsteh-queue.mdx deleted file mode 100644 index 0020fa3c..00000000 --- a/apps/docs/content/docs/02.packages/02.create-vorsteh-queue.mdx +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: "create-vorsteh-queue CLI" -navTitle: create-vorsteh-queue -description: CLI tool for quickly creating new Vorsteh Queue projects with pre-configured templates. ---- - -The `create-vorsteh-queue` CLI tool provides the fastest way to get started with Vorsteh Queue by creating new projects from pre-built templates. - -## Installation - -No installation required - use with `npx`: - -```bash -npx create-vorsteh-queue -``` - -## Usage - -### Interactive Mode - -```bash -# Full interactive experience -npx create-vorsteh-queue -``` - -The CLI will prompt you for: - -- Project name -- Template selection -- Package manager preference -- Dependency installation - -### Direct Mode - -```bash -# With project name -npx create-vorsteh-queue my-queue-app - -# With template selection -npx create-vorsteh-queue my-app --template=drizzle-postgres - -# With package manager -npx create-vorsteh-queue my-app --package-manager=pnpm - -# Fully automated -npx create-vorsteh-queue my-app \ - --template=drizzle-postgres \ - --package-manager=pnpm \ - --quiet -``` - -## CLI Options - -| Option | Short | Description | Example | -| ------------------------ | ----------- | ----------------------- | --------------------- | -| `--template=` | `-t=` | Choose template | `-t=drizzle-postgres` | -| `--package-manager=` | `-pm=` | Package manager | `-pm=pnpm` | -| `--no-install` | - | Skip dependency install | `--no-install` | -| `--quiet` | `-q` | Minimal output | `--quiet` | - -## Available Templates - -Templates are dynamically discovered from the repository: - -- **drizzle-pg** - Basic Drizzle ORM with node-postgres -- **drizzle-pglite** - Zero-setup with embedded PostgreSQL -- **drizzle-postgres** - Advanced with recurring jobs -- **event-system** - Comprehensive event monitoring -- **progress-tracking** - Real-time progress updates -- **pm2-workers** - Multi-process workers with PM2 -- **prisma-client** - Example with the new `prisma-client` -- **prisma-client-js** - Example with the old `prisma-client-js` - -## Package Managers - -Supported package managers: - -- **npm** - Default Node.js package manager -- **pnpm** - Fast, disk space efficient -- **yarn** - Popular alternative -- **bun** - Ultra-fast (experimental) - -## Template Structure - -Each template includes: - -- **Complete source code** - Ready-to-run example -- **Database schema** - Pre-configured tables -- **Environment setup** - Example `.env` files -- **Documentation** - README with setup instructions -- **TypeScript configuration** - Optimized settings - -## Examples - -### Quick Start - -```bash -npx create-vorsteh-queue my-app --template=drizzle-pglite -cd my-app -npm run dev -``` - -### Production Setup - -```bash -npx create-vorsteh-queue worker-service \ - --template=drizzle-postgres \ - --package-manager=pnpm -cd worker-service -pnpm db:push -pnpm dev -``` - -### CI/CD Friendly - -```bash -npx create-vorsteh-queue my-app \ - --template=event-system \ - --package-manager=pnpm \ - --no-install \ - --quiet -``` - -## Template Development - -Templates are automatically discovered from the `examples/` directory in the repository. To add a new template: - -1. Create a new example in `examples/` -2. Include a `package.json` with proper metadata -3. Add a `README.md` with setup instructions -4. The template becomes available automatically diff --git a/apps/docs/content/docs/02.packages/adapter-drizzle.mdx b/apps/docs/content/docs/02.packages/adapter-drizzle.mdx deleted file mode 100644 index 12b22c4d..00000000 --- a/apps/docs/content/docs/02.packages/adapter-drizzle.mdx +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: Drizzle Adapter -navTitle: "@vorsteh-queue/adapter-drizzle" -description: Drizzle ORM adapter for PostgreSQL providing type-safe database operations with excellent performance. ---- - -The Drizzle adapter provides PostgreSQL support using Drizzle ORM, offering excellent TypeScript integration and performance. - -## Installation - -@vorsteh-queue/core @vorsteh-queue/adapter-drizzle - -## Quick Start - -```typescript -import { drizzle } from "drizzle-orm/node-postgres" -import { Pool } from "pg" - -import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" -import { Queue } from "@vorsteh-queue/core" - -const pool = new Pool({ connectionString: "postgresql://..." }) -const db = drizzle(pool) -const adapter = new PostgresQueueAdapter(db) -const queue = new Queue(adapter) -``` - -## Supported providers - -- PGlite - https://orm.drizzle.team/docs/connect-pglite -- Postgres.JS - https://orm.drizzle.team/docs/get-started-postgresql#postgresjs -- Node Progress - https://orm.drizzle.team/docs/get-started-postgresql#node-postgres - -## Database Setup - -### Schema - -The adapter includes a pre-defined schema: - -```typescript -import { queueJobsTable } from "@vorsteh-queue/adapter-drizzle" - -// Use in your schema file -export { queueJobsTable } -``` - -If you don't want to use the pre-defined schema, you can create your own schema definition. - - - -### Custom Table & Schema Names - -To create a table with custom name and schema, you can use the `createQueueJobsTable` helper, which will create the table without taking care about the field definitions: - -```typescript path="drizzle-schema.ts" -import { createQueueJobsTable } from "@vorsteh-queue/adapter-drizzle" - -export const { table: customQueueJobs, schema: customSchema } = createQueueJobsTable( - "custom_queue_jobs", // your custom table name - "custom_schema", // your custom schema name -) -``` - -To use the custom table and schema in the adapter, pass the adapter configuration to the `PostgresQueueAdapter`: - -```typescript -import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" - -const adapter = new PostgresQueueAdapter(db, { - // Your custom model name ( based on the example above - `table: customQueueJobs`) - modelName: "customQueueJobs", -}) -``` - - - Checkout our drizzle example with [custom schema and - table](/docs/examples/drizzle-custom-schema-table/). - - -### Migration - -To run the migrations, we recommend to use [`drizzle-kit`](https://orm.drizzle.team/docs/kit-overview). - -> Drizzle Kit is a CLI tool for managing SQL database migrations with Drizzle. - -## References - -- Sources: https://github.com/noxify/vorsteh-queue/tree/main/packages/adapter-drizzle -- NPM: https://www.npmjs.com/package/@vorsteh-queue/adapter-drizzle diff --git a/apps/docs/content/docs/02.packages/adapter-kysely.mdx b/apps/docs/content/docs/02.packages/adapter-kysely.mdx deleted file mode 100644 index 7e07d1d8..00000000 --- a/apps/docs/content/docs/02.packages/adapter-kysely.mdx +++ /dev/null @@ -1,125 +0,0 @@ ---- -title: Kysely Adapter -navTitle: "@vorsteh-queue/adapter-kysely" -description: Kysely adapter for PostgreSQL providing type-safe database operations with excellent performance. ---- - -The Kysely adapter provides PostgreSQL support using Kysely, offering excellent TypeScript integration and performance. - -## Installation - -@vorsteh-queue/core @vorsteh-queue/adapter-kysely - -## Quick Start - -```typescript -import { Kysely } from "kysely" -import { PostgresJSDialect } from "kysely-postgres-js" -import postgres from "postgres" - -import type { QueueJobTableDefinition } from "@vorsteh-queue/adapter-kysely/types" -import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" -import { Queue } from "@vorsteh-queue/core" - -interface DB { - queue_jobs: QueueJobTableDefinition - other_table: { - name: string - } -} - -// Shared database connection -const client = postgres( - process.env.DATABASE_URL || "postgresql://postgres:password@localhost:5432/queue_db", - { max: 10 }, // Connection pool -) - -const db = new Kysely({ - dialect: new PostgresJSDialect({ - postgres: client, - }), -}) - -const adapter = new PostgresQueueAdapter(db) -const queue = new Queue(adapter) -``` - -## Supported providers - -- PGlite - https://github.com/czeidler/kysely-pglite-dialect -- Postgres.JS - https://github.com/kysely-org/kysely-postgres-js -- Node Progress - https://kysely-org.github.io/kysely-apidoc/classes/PostgresDialect.html - -## Database Setup - -### Schema - -The adapter includes a pre-defined migration: - -```typescript path="migrations/queue-jobs.ts" -export { down, up } from "@vorsteh-queue/adapter-kysely/migrations" -``` - -If you don't want to use the pre-defined schema, you can create your own migration. - - - -### Custom Table & Schema Names - -To create a table with custom name and schema, you can use the `createQueueJobsTable` helper, which will create the table without taking care about the field definitions: - -```typescript path="migrations/queue-jobs.ts" -import { createQueueJobsTable } from "@vorsteh-queue/adapter-kysely" - -export const { up, down } = createQueueJobsTable( - // your custom table name - "custom_queue_jobs", - // your custom schema name - "custom_schema", -) -``` - -To use the custom table and schema in the adapter, pass the adapter configuration to the `PostgresQueueAdapter`: - -```typescript -import { Kysely } from "kysely" - -import type { QueueJobTableDefinition } from "@vorsteh-queue/adapter-kysely/types" -import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" - -interface DB { - custom_queue_jobs: QueueJobTableDefinition -} - -const db = new Kysely({ - //... your Kysely configuration -}) - -const adapter = new PostgresQueueAdapter(db, { - // Your custom schema name - schemaName: "custom_schema", - // Your custom table name - tableName: "custom_queue_jobs", -}) -``` - - - Checkout our kysely example with [custom schema and - table](/docs/examples/kysely-custom-schema-table/). - - -### Migration - -To run the migrations, we recommend to use [`kysely-ctl`](https://github.com/kysely-org/kysely-ctl). - -> `kysely-ctl` is the command-line tool for Kysely. - -## References - -- Sources: https://github.com/noxify/vorsteh-queue/tree/main/packages/adapter-kysely -- NPM: https://www.npmjs.com/package/@vorsteh-queue/adapter-kysely diff --git a/apps/docs/content/docs/02.packages/adapter-prisma.mdx b/apps/docs/content/docs/02.packages/adapter-prisma.mdx deleted file mode 100644 index 0ed47178..00000000 --- a/apps/docs/content/docs/02.packages/adapter-prisma.mdx +++ /dev/null @@ -1,109 +0,0 @@ ---- -title: Prisma Adapter -navTitle: "@vorsteh-queue/adapter-prisma" -description: Prisma ORM adapter for PostgreSQL with generated types and excellent developer tooling. ---- - -The Prisma adapter provides PostgreSQL support using Prisma ORM, offering generated types, migrations, and excellent developer tooling. - -## Installation - -@vorsteh-queue/core @vorsteh-queue/adapter-prisma - -## Quick Start - -### With prisma-client-js - -```typescript -import { PrismaClient } from "@prisma/client" - -import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" -import { Queue } from "@vorsteh-queue/core" - -const prisma = new PrismaClient() -const adapter = new PostgresPrismaQueueAdapter(prisma) -const queue = new Queue(adapter) -``` - -### With prisma-client - -```typescript -import { PrismaPg } from "@prisma/adapter-pg" -import { PrismaClient } from "src/generated/prisma/client" - -import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" -import { Queue } from "@vorsteh-queue/core" - -const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL }) -const prisma = new PrismaClient({ adapter }) -const adapter = new PostgresPrismaQueueAdapter(prisma) -const queue = new Queue(adapter) -``` - -## Database Setup - -### Schema - -Add the queue job model to your `schema.prisma`: - - - -### Custom Table & Schema Names - -To use the custom table and schema with prisma, you have to customize the prisma schema. -Here an example with custom schema and table name ( the relevant parts are highlighted ): - - - -To use the custom table and schema in the adapter, pass the adapter configuration to the `PostgresQueueAdapter`: - -```typescript -import { PrismaPg } from "@prisma/adapter-pg" -import { PrismaClient } from "src/generated/prisma/client" - -import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" -import { Queue } from "@vorsteh-queue/core" - -const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL }) -const prisma = new PrismaClient({ adapter }) - -const adapter = new PostgresQueueAdapter(prisma, { - // Your custom model name ( based on the example above ) - modelName: "CustomQueueJobs", - // Your custom schema name - schemaName: "custom_schema", - // Your custom table name - tableName: "custom_queue_jobs", -}) - -const queue = new Queue(adapter) -``` - - - Checkout our prisma example with [custom schema and - table](/docs/examples/prisma-custom-schema-table/). - - -### Migration - -To run the migrations, we recommend to use the [`prisma cli`](https://www.prisma.io/docs/orm/tools/prisma-cli). - -> The Prisma command line interface (CLI) is the primary way to interact with your Prisma project from the command line. - -## References - -- Sources: https://github.com/noxify/vorsteh-queue/tree/main/packages/adapter-prisma -- NPM: https://www.npmjs.com/package/@vorsteh-queue/adapter-prisma diff --git a/apps/docs/content/docs/02.packages/index.mdx b/apps/docs/content/docs/02.packages/index.mdx deleted file mode 100644 index d6d378ea..00000000 --- a/apps/docs/content/docs/02.packages/index.mdx +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: Packages -description: Overview of all Vorsteh Queue packages and their purposes. ---- - -Vorsteh Queue is built as a modular system with separate packages for different functionality. This allows you to install only what you need and keeps the core lightweight. - -## Core Package - -- **[@vorsteh-queue/core](/docs/packages/core)** - The main queue engine with all core functionality - -## Adapter Packages - -- **[@vorsteh-queue/adapter-drizzle](/docs/packages/adapter-drizzle)** - Drizzle ORM adapter for PostgreSQL -- **[@vorsteh-queue/adapter-prisma](/docs/packages/adapter-prisma)** - Prisma ORM adapter for PostgreSQL -- **[@vorsteh-queue/adapter-kysely](/docs/packages/adapter-kysely)** - Kysely adapter for PostgreSQL - -## CLI Tools - -- **[create-vorsteh-queue](/docs/packages/create-vorsteh-queue)** - CLI tool for creating new projects - -## Package Architecture - -The modular design allows for: - -- **Minimal dependencies** - Only install what you need -- **Flexible adapters** - Easy to add new database adapters -- **Type safety** - Full TypeScript support across all packages -- **Independent versioning** - Packages can be updated independently - -## Installation Patterns - -### Basic Setup - -```bash -npm install @vorsteh-queue/core @vorsteh-queue/adapter-drizzle -``` - -### With CLI - -```bash -npx create-vorsteh-queue my-app --template=drizzle-postgres -``` - -Choose the packages that match your project's needs and ORM preference. diff --git a/apps/docs/content/docs/index.mdx b/apps/docs/content/docs/index.mdx deleted file mode 100644 index 1595f571..00000000 --- a/apps/docs/content/docs/index.mdx +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Documentation -ignoreSearch: true ---- diff --git a/apps/docs/content/examples/01.getting-started/01.drizzle-pglite.mdx b/apps/docs/content/examples/01.getting-started/01.drizzle-pglite.mdx new file mode 100644 index 00000000..760572d4 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/01.drizzle-pglite.mdx @@ -0,0 +1,39 @@ +--- +title: Drizzle + PGlite +description: Zero-setup example using Drizzle ORM with PGlite (embedded PostgreSQL). +tags: ["drizzle", "pglite", "beginner"] +--- + +A zero-dependency-on-external-services example that uses PGlite as an embedded PostgreSQL database. Great for quick experimentation without running a Postgres server. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=drizzle-pglite + + +Then install dependencies and start the dev server: + +```bash +cd my-app +npm install +npm run dev +``` + +## What it demonstrates + +- Basic queue setup with Drizzle ORM +- Embedded PGlite database (no external PostgreSQL needed) +- Job registration and processing +- Simple job handler returning results + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Drizzle Adapter](/docs/vorsteh-queue/adapters/drizzle) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/drizzle-pglite) diff --git a/apps/docs/content/examples/01.getting-started/02.drizzle-pg.mdx b/apps/docs/content/examples/01.getting-started/02.drizzle-pg.mdx new file mode 100644 index 00000000..0563e8c0 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/02.drizzle-pg.mdx @@ -0,0 +1,40 @@ +--- +title: Drizzle + node-postgres +navTitle: Drizzle + pg +description: Basic example using Drizzle ORM with node-postgres (pg). +tags: ["drizzle", "node-postgres", "beginner"] +--- + +A minimal example using Drizzle ORM with the `node-postgres` (pg) driver connecting to a real PostgreSQL database. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=drizzle-pg + + +Then install dependencies, push the schema and start the dev server: + +```bash +cd my-app +npm install +npm run db:push +npm run dev +``` + +## What it demonstrates + +- Drizzle ORM with node-postgres driver +- Schema migration with drizzle-kit +- Queue creation, handler registration, and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Drizzle Adapter](/docs/vorsteh-queue/adapters/drizzle) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/drizzle-pg) diff --git a/apps/docs/content/examples/01.getting-started/03.drizzle-postgres.mdx b/apps/docs/content/examples/01.getting-started/03.drizzle-postgres.mdx new file mode 100644 index 00000000..e4f59e4b --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/03.drizzle-postgres.mdx @@ -0,0 +1,42 @@ +--- +title: Drizzle + postgres.js +navTitle: Drizzle + postgres.js +description: Advanced example using Drizzle ORM with postgres.js and recurring jobs. +tags: ["drizzle", "postgres.js", "recurring"] +--- + +An advanced example using Drizzle ORM with the `postgres.js` driver, demonstrating recurring jobs with cron expressions. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=drizzle-postgres + + +Then install dependencies, push the schema and start the dev server: + +```bash +cd my-app +pnpm install +pnpm db:push +pnpm dev +``` + +## What it demonstrates + +- Drizzle ORM with postgres.js driver +- Recurring jobs using cron expressions +- Timezone-aware scheduling +- Multiple job types in one queue + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Drizzle Adapter](/docs/vorsteh-queue/adapters/drizzle) +- [Job Types](/docs/vorsteh-queue/core/job-types) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/drizzle-postgres) diff --git a/apps/docs/content/examples/01.getting-started/04.kysely-postgres.mdx b/apps/docs/content/examples/01.getting-started/04.kysely-postgres.mdx new file mode 100644 index 00000000..650d90a6 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/04.kysely-postgres.mdx @@ -0,0 +1,41 @@ +--- +title: Kysely + PostgreSQL +navTitle: Kysely +description: Example using Kysely with postgres.js and recurring jobs. +tags: ["kysely", "postgres.js", "recurring"] +--- + +An example using Kysely as the query builder with a PostgreSQL database, demonstrating recurring job scheduling. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=kysely-postgres + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Kysely adapter setup +- Migration-based schema creation +- Recurring job scheduling +- Job handler with typed payloads + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Kysely Adapter](/docs/vorsteh-queue/adapters/kysely) +- [Job Types](/docs/vorsteh-queue/core/job-types) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/kysely-postgres) diff --git a/apps/docs/content/examples/01.getting-started/05.prisma-client.mdx b/apps/docs/content/examples/01.getting-started/05.prisma-client.mdx new file mode 100644 index 00000000..315d6cdb --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/05.prisma-client.mdx @@ -0,0 +1,39 @@ +--- +title: Prisma Client +description: Example using Prisma with prisma-client provider and PostgreSQL. +tags: ["prisma", "postgresql"] +--- + +An example using the modern `prisma-client` provider with PostgreSQL and the `@prisma/adapter-pg` driver adapter. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=prisma-client + + +Then install dependencies, generate the Prisma client and start the dev server: + +```bash +cd my-app +pnpm install +pnpm prisma:generate +pnpm dev +``` + +## What it demonstrates + +- Prisma adapter with the `prisma-client` provider +- TypeScript-generated Prisma client +- Queue setup and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Prisma Adapter](/docs/vorsteh-queue/adapters/prisma) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/prisma-client) diff --git a/apps/docs/content/examples/01.getting-started/06.zenstack-postgres.mdx b/apps/docs/content/examples/01.getting-started/06.zenstack-postgres.mdx new file mode 100644 index 00000000..4c621f66 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/06.zenstack-postgres.mdx @@ -0,0 +1,39 @@ +--- +title: ZenStack PostgreSQL +description: Example using ZenStack ORM with PostgreSQL. +tags: ["zenstack", "postgresql"] +--- + +An example using ZenStack ORM v3 with PostgreSQL and the `pg` driver. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=zenstack-postgres + + +Then install dependencies, generate the ZenStack schema and start the dev server: + +```bash +cd my-app +pnpm install +pnpm zen:generate +pnpm dev +``` + +## What it demonstrates + +- ZenStack adapter with PostgreSQL +- ZenStackClient initialization with PostgresDialect +- Queue setup and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [ZenStack Adapter](/docs/vorsteh-queue/adapters/zenstack) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/zenstack-postgres) diff --git a/apps/docs/content/examples/01.getting-started/07.typeorm-postgres.mdx b/apps/docs/content/examples/01.getting-started/07.typeorm-postgres.mdx new file mode 100644 index 00000000..4078b62a --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/07.typeorm-postgres.mdx @@ -0,0 +1,40 @@ +--- +title: TypeORM PostgreSQL +description: Example using TypeORM with PostgreSQL. +tags: ["typeorm", "postgresql"] +--- + +An example using TypeORM with PostgreSQL and the `pg` driver. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=typeorm-postgres + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +The `QueueJobEntity` and `QueueFlowEntity` with `synchronize: true` will automatically create the tables on first run. + +## What it demonstrates + +- TypeORM adapter with PostgreSQL +- DataSource initialization with entity registration +- Queue setup and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [TypeORM Adapter](/docs/vorsteh-queue/adapters/typeorm) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/typeorm-postgres) diff --git a/apps/docs/content/examples/01.getting-started/08.mikroorm-postgres.mdx b/apps/docs/content/examples/01.getting-started/08.mikroorm-postgres.mdx new file mode 100644 index 00000000..c58c1156 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/08.mikroorm-postgres.mdx @@ -0,0 +1,38 @@ +--- +title: MikroORM PostgreSQL +description: Example using MikroORM with PostgreSQL. +tags: ["mikroorm", "postgresql"] +--- + +An example using MikroORM v6 with PostgreSQL. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=mikroorm-postgres + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- MikroORM adapter with PostgreSQL +- MikroORM initialization with EntitySchema +- Queue setup and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [MikroORM Adapter](/docs/vorsteh-queue/adapters/mikroorm) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/mikroorm-postgres) diff --git a/apps/docs/content/examples/01.getting-started/09.sequelize-postgres.mdx b/apps/docs/content/examples/01.getting-started/09.sequelize-postgres.mdx new file mode 100644 index 00000000..67fe2b2a --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/09.sequelize-postgres.mdx @@ -0,0 +1,38 @@ +--- +title: Sequelize PostgreSQL +description: Example using Sequelize with PostgreSQL. +tags: ["sequelize", "postgresql"] +--- + +An example using Sequelize v6 with PostgreSQL. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=sequelize-postgres + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Sequelize adapter with PostgreSQL +- Sequelize initialization with model registration +- Queue setup and job processing + +## Related documentation + +- [Quickstart](/docs/vorsteh-queue/getting-started/quickstart) +- [Sequelize Adapter](/docs/vorsteh-queue/adapters/sequelize) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/sequelize-postgres) diff --git a/apps/docs/content/examples/01.getting-started/index.mdx b/apps/docs/content/examples/01.getting-started/index.mdx new file mode 100644 index 00000000..1eae5d56 --- /dev/null +++ b/apps/docs/content/examples/01.getting-started/index.mdx @@ -0,0 +1,4 @@ +--- +title: Getting Started +description: Basic examples for each supported adapter. +--- diff --git a/apps/docs/content/examples/02.features/01.event-system.mdx b/apps/docs/content/examples/02.features/01.event-system.mdx new file mode 100644 index 00000000..08b99a51 --- /dev/null +++ b/apps/docs/content/examples/02.features/01.event-system.mdx @@ -0,0 +1,40 @@ +--- +title: Event System +description: Comprehensive event monitoring and statistics using the queue event system. +tags: ["events", "monitoring"] +--- + +Demonstrates the full event lifecycle — listen to job additions, completions, failures, progress updates, and queue control events. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=event-system + + +Then install dependencies, push the schema and start the dev server: + +```bash +cd my-app +pnpm install +pnpm db:push +pnpm dev +``` + +## What it demonstrates + +- All job lifecycle events (`job:added`, `job:completed`, `job:failed`, `job:progress`) +- Queue control events (`queue:paused`, `queue:resumed`) +- Real-time statistics tracking via events +- Event-driven logging patterns + +## Related documentation + +- [Events](/docs/vorsteh-queue/core/events) +- [Queue](/docs/vorsteh-queue/core/queue) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/event-system) diff --git a/apps/docs/content/examples/02.features/02.progress-tracking.mdx b/apps/docs/content/examples/02.features/02.progress-tracking.mdx new file mode 100644 index 00000000..9f1fe25a --- /dev/null +++ b/apps/docs/content/examples/02.features/02.progress-tracking.mdx @@ -0,0 +1,39 @@ +--- +title: Progress Tracking +description: Real-time job progress tracking with percentage updates. +tags: ["progress", "monitoring"] +--- + +Shows how to report and monitor real-time progress during long-running job execution. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=progress-tracking + + +Then install dependencies, push the schema and start the dev server: + +```bash +cd my-app +pnpm install +pnpm db:push +pnpm dev +``` + +## What it demonstrates + +- `job.updateProgress()` calls during processing +- Progress event listening +- Batch processing with progress reporting + +## Related documentation + +- [Progress Tracking](/docs/vorsteh-queue/core/progress-tracking) +- [Events](/docs/vorsteh-queue/core/events) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/progress-tracking) diff --git a/apps/docs/content/examples/02.features/03.flow-producer.mdx b/apps/docs/content/examples/02.features/03.flow-producer.mdx new file mode 100644 index 00000000..f1986028 --- /dev/null +++ b/apps/docs/content/examples/02.features/03.flow-producer.mdx @@ -0,0 +1,39 @@ +--- +title: Flow Producer +description: Parent-child job flows for multi-step pipelines. +tags: ["flows", "pipeline", "parent-child"] +--- + +Demonstrates parent-child job relationships where a parent job spawns child jobs and waits for their completion before finishing. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=flow-producer + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Flow creation with parent and child jobs via `FlowProducer` +- Parent node promotion when children complete +- Flow tree visualization +- Multi-step deploy pipeline pattern + +## Related documentation + +- [Flows Overview](/docs/vorsteh-queue/core/flows/overview) +- [Flow Trees](/docs/vorsteh-queue/core/flows/flow-trees) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/flow-producer) diff --git a/apps/docs/content/examples/02.features/04.batch-processing.mdx b/apps/docs/content/examples/02.features/04.batch-processing.mdx new file mode 100644 index 00000000..14e5d193 --- /dev/null +++ b/apps/docs/content/examples/02.features/04.batch-processing.mdx @@ -0,0 +1,38 @@ +--- +title: Batch Processing +description: Process large datasets in batches with progress tracking. +tags: ["batch", "progress"] +--- + +Shows how to split large workloads into batches and process them with progress reporting. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=batch-processing + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Splitting large payloads into manageable batches +- Progress reporting across batch iterations +- Job result aggregation + +## Related documentation + +- [Queue](/docs/vorsteh-queue/core/queue) +- [Job Types](/docs/vorsteh-queue/core/job-types) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/batch-processing) diff --git a/apps/docs/content/examples/02.features/05.rate-limiting.mdx b/apps/docs/content/examples/02.features/05.rate-limiting.mdx new file mode 100644 index 00000000..1e5eecff --- /dev/null +++ b/apps/docs/content/examples/02.features/05.rate-limiting.mdx @@ -0,0 +1,38 @@ +--- +title: Rate Limiting +description: Per-handler rate limiting to control processing throughput. +tags: ["rate-limiting", "throttling"] +--- + +Demonstrates rate-limited job processing to respect external API limits or control resource usage. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=rate-limiting + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Per-handler rate limiting configuration +- Controlling concurrent execution +- Respecting external API rate limits + +## Related documentation + +- [Job Types](/docs/vorsteh-queue/core/job-types) +- [Queue](/docs/vorsteh-queue/core/queue) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/rate-limiting) diff --git a/apps/docs/content/examples/02.features/06.unique-jobs.mdx b/apps/docs/content/examples/02.features/06.unique-jobs.mdx new file mode 100644 index 00000000..5ecfcd98 --- /dev/null +++ b/apps/docs/content/examples/02.features/06.unique-jobs.mdx @@ -0,0 +1,38 @@ +--- +title: Unique Jobs +description: Job deduplication with unique keys to prevent duplicate processing. +tags: ["unique", "deduplication"] +--- + +Shows how to use unique keys to prevent duplicate jobs from being added to the queue. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=unique-jobs + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Unique key assignment to prevent duplicates +- Handling duplicate job attempts gracefully +- Use cases like email deduplication or idempotent operations + +## Related documentation + +- [Job Types](/docs/vorsteh-queue/core/job-types) +- [Queue](/docs/vorsteh-queue/core/queue) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/unique-jobs) diff --git a/apps/docs/content/examples/02.features/07.group-fifo.mdx b/apps/docs/content/examples/02.features/07.group-fifo.mdx new file mode 100644 index 00000000..ead216b2 --- /dev/null +++ b/apps/docs/content/examples/02.features/07.group-fifo.mdx @@ -0,0 +1,39 @@ +--- +title: Group FIFO +description: Per-group FIFO ordering for multi-tenant job processing. +tags: ["groups", "fifo", "multi-tenant"] +--- + +Demonstrates grouped job processing where jobs within the same group are processed in strict FIFO order, while different groups can be processed concurrently. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=group-fifo + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Group key assignment for job ordering +- FIFO processing within a group +- Concurrent processing across groups +- Multi-tenant isolation patterns + +## Related documentation + +- [Job Types](/docs/vorsteh-queue/core/job-types) +- [Queue](/docs/vorsteh-queue/core/queue) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/group-fifo) diff --git a/apps/docs/content/examples/02.features/08.result-storage.mdx b/apps/docs/content/examples/02.features/08.result-storage.mdx new file mode 100644 index 00000000..1940123f --- /dev/null +++ b/apps/docs/content/examples/02.features/08.result-storage.mdx @@ -0,0 +1,38 @@ +--- +title: Result Storage +description: Storing and retrieving job results after completion. +tags: ["results", "storage"] +--- + +Shows how to store structured results from job handlers and retrieve them after completion. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=result-storage + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Returning typed results from job handlers +- Retrieving results by job ID +- Result inspection via CLI and API + +## Related documentation + +- [Job Types](/docs/vorsteh-queue/core/job-types) +- [Queue](/docs/vorsteh-queue/core/queue) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/result-storage) diff --git a/apps/docs/content/examples/02.features/index.mdx b/apps/docs/content/examples/02.features/index.mdx new file mode 100644 index 00000000..4ce0676d --- /dev/null +++ b/apps/docs/content/examples/02.features/index.mdx @@ -0,0 +1,4 @@ +--- +title: Features +description: Examples demonstrating specific Vorsteh Queue features and patterns. +--- diff --git a/apps/docs/content/examples/03.advanced/01.pm2-workers.mdx b/apps/docs/content/examples/03.advanced/01.pm2-workers.mdx new file mode 100644 index 00000000..7c7b0bd7 --- /dev/null +++ b/apps/docs/content/examples/03.advanced/01.pm2-workers.mdx @@ -0,0 +1,40 @@ +--- +title: PM2 Workers +description: Multi-process workers managed by PM2 for production deployments. +tags: ["pm2", "workers", "production"] +--- + +Shows how to run multiple queue worker processes managed by PM2 for horizontal scaling. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=pm2-workers + + +Then install dependencies, push the schema and start the workers: + +```bash +cd my-app +pnpm install +pnpm db:push +pnpm start +``` + +## What it demonstrates + +- PM2 ecosystem configuration +- Multiple worker processes sharing a queue +- Graceful shutdown handling +- Production-ready process management + +## Related documentation + +- [Queue](/docs/vorsteh-queue/core/queue) +- [Job Types](/docs/vorsteh-queue/core/job-types) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/pm2-workers) diff --git a/apps/docs/content/examples/03.advanced/02.graphql-server.mdx b/apps/docs/content/examples/03.advanced/02.graphql-server.mdx new file mode 100644 index 00000000..2b5721c0 --- /dev/null +++ b/apps/docs/content/examples/03.advanced/02.graphql-server.mdx @@ -0,0 +1,39 @@ +--- +title: GraphQL Server +description: Standalone GraphQL server for queue monitoring and management. +tags: ["graphql", "server", "monitoring"] +--- + +A standalone GraphQL server exposing queue operations — inspect jobs, get stats, cancel, retry, and redrive from any GraphQL client. + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=graphql-server + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- `@vorsteh-queue/server` setup +- GraphQL API with Yoga and Hono +- Query queue stats and job details +- Mutation operations (cancel, retry, redrive) + +## Related documentation + +- [Queue](/docs/vorsteh-queue/core/queue) +- [Events](/docs/vorsteh-queue/core/events) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/graphql-server) diff --git a/apps/docs/content/examples/03.advanced/03.workflow-saga.mdx b/apps/docs/content/examples/03.advanced/03.workflow-saga.mdx new file mode 100644 index 00000000..6dd091fc --- /dev/null +++ b/apps/docs/content/examples/03.advanced/03.workflow-saga.mdx @@ -0,0 +1,57 @@ +--- +title: Workflow Saga +description: Multi-step workflow with Saga compensation pattern for distributed transactions. +tags: ["workflow", "saga", "compensation"] +--- + +Implements the Saga pattern for multi-step workflows where each step has a compensation action that runs on failure. + +## Flow + +```mermaid +flowchart TD + Start([Job starts]) --> S1[Step 1: Reserve Stock] + S1 -->|success| S2[Step 2: Charge Payment] + S2 -->|success| S3[Step 3: Ship Order] + S3 -->|success| Done([Job completed]) + + S3 -->|failure| C2[Compensate: Refund Payment] + C2 --> C1[Compensate: Release Reservation] + C1 --> Failed([Job failed]) + + S2 -->|failure| C1b[Compensate: Release Reservation] + C1b --> Failed +``` + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=workflow-saga + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- Multi-step workflow orchestration +- Compensation actions for rollback +- Step-by-step execution with error boundaries +- Distributed transaction patterns without 2PC + +## Related documentation + +- [Flows Overview](/docs/vorsteh-queue/core/flows/overview) +- [Steps](/docs/vorsteh-queue/core/flows/steps) +- [Error Handling](/docs/vorsteh-queue/core/error-handling) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/workflow-saga) diff --git a/apps/docs/content/examples/03.advanced/04.workflow-signals.mdx b/apps/docs/content/examples/03.advanced/04.workflow-signals.mdx new file mode 100644 index 00000000..deeff70a --- /dev/null +++ b/apps/docs/content/examples/03.advanced/04.workflow-signals.mdx @@ -0,0 +1,61 @@ +--- +title: Workflow Signals +description: Human-in-the-loop workflow with waitFor signals for external approvals. +tags: ["workflow", "signals", "human-in-the-loop"] +--- + +Demonstrates workflows that pause and wait for external signals (e.g. human approval, webhook callback) before continuing. + +## Flow + +```mermaid +sequenceDiagram + participant E as Employee + participant Q as Queue + participant W as Worker + participant M as Manager + + E->>Q: Add expense-request job + Q->>W: Pick up job + W->>W: Step 1: Submit expense + W-->>Q: Pause (waitFor "manager-decision") + + Note over W,M: Job is suspended, waiting for signal + + M->>Q: queue.signal(jobId, "manager-decision", { approved }) + Q->>W: Resume job with signal data + W->>W: Step 2: Process decision + W-->>Q: Job completed +``` + +## Setup + +First, generate the project: + + + create-vorsteh-queue my-app --template=workflow-signals + + +Then install dependencies and start the dev server: + +```bash +cd my-app +pnpm install +pnpm dev +``` + +## What it demonstrates + +- `waitFor` signal mechanism +- Pausing a workflow until external input arrives +- Human-in-the-loop approval patterns +- Signal timeout handling + +## Related documentation + +- [Flows Overview](/docs/vorsteh-queue/core/flows/overview) +- [Steps](/docs/vorsteh-queue/core/flows/steps) + +## Source + +[View on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/workflow-signals) diff --git a/apps/docs/content/examples/03.advanced/05.tui-dashboard.mdx b/apps/docs/content/examples/03.advanced/05.tui-dashboard.mdx new file mode 100644 index 00000000..5942f88f --- /dev/null +++ b/apps/docs/content/examples/03.advanced/05.tui-dashboard.mdx @@ -0,0 +1,153 @@ +--- +title: TUI Dashboard +description: Interactive terminal dashboard with PGlite for exploring multi-queue monitoring. +tags: ["tui", "dashboard", "cli", "pglite", "monitoring"] +--- + +A zero-setup example that spins up multiple queues with diverse job types using PGlite (embedded PostgreSQL), then launches the interactive TUI dashboard for real-time monitoring. + +Two variants are available: + +- **tui-api** — Dashboard connects via GraphQL API (remote mode) +- **tui-direct** — Dashboard connects directly to the adapter via PGlite socket server + +## API Mode (tui-api) + +The API variant starts a GraphQL server that the dashboard connects to remotely. This is the typical setup for production monitoring where the dashboard runs on a different machine than the workers. + +### Setup + +create-vorsteh-queue my-app --template=tui-api + +```bash +cd my-app +pnpm install +pnpm dev # Starts GraphQL server + workers + seeds +``` + +In a second terminal: + +```bash +pnpm dashboard # Connects via http://localhost:4000/graphql +``` + +### Architecture + +``` +┌─────────────┐ GraphQL ┌─────────────────┐ +│ Dashboard │ ──────────────▶ │ Server + Queue │ +│ (TUI CLI) │ │ (Workers + DB) │ +└─────────────┘ └─────────────────┘ +``` + +## Direct Mode (tui-direct) + +The direct variant uses a PGlite socket server for multi-process access. No GraphQL server needed. + +### Quick Start (All-in-One) + +For a quick demo, workers and dashboard run in a single process with in-memory PGlite: + + + create-vorsteh-queue my-app --template=tui-direct + + +```bash +cd my-app +pnpm install +pnpm dev # Workers + seeds + dashboard (all-in-one, in-memory) +``` + +### Multi-Process Setup (PGlite Socket Server) + +For a more realistic setup, workers and dashboard run as separate processes. A PGlite socket server provides shared database access: + +```bash +# Terminal 1: Workers + PGlite socket server +pnpm dev:prod + +# Terminal 2: Dashboard (connects via queue.config.ts) +pnpm dashboard +``` + +### Architecture (Multi-Process) + +``` +┌─────────────┐ PostgreSQL ┌─────────────────────────┐ +│ Dashboard │ Protocol │ Worker Process │ +│ (TUI CLI) │ ◀──────────────────▶│ (PGlite Socket Server) │ +└─────────────┘ port 5488 │ + Seed + Workers │ + └─────────────────────────┘ +``` + +### queue.config.ts + +The direct mode uses `queue.config.ts` to connect to the PGlite socket server via standard `node-postgres`: + +```typescript +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { drizzle } from "drizzle-orm/node-postgres" + +const db = drizzle({ + connection: "postgresql://postgres:postgres@127.0.0.1:5488/postgres", +}) + +export default { + adapter: new PostgresQueueAdapter(db), + queues: [emailQueue, dataQueue, notificationQueue, deploymentQueue], + defaultQueue: "email", +} +``` + +## What both examples demonstrate + +- PGlite embedded PostgreSQL (zero external dependencies) +- Multiple queues: email, data-processing, notifications, deployments +- Diverse job types with different priorities, delays, and failure rates +- Flow trees with parent-child job relationships +- Multi-step workflows (step.run, step.sleep, step.waitFor) +- Real-time TUI dashboard with queue switching, job filtering, and keyboard navigation +- Flow tree visualization overlay (press `f` in job detail) + +## Dashboard commands + +Both examples provide per-queue dashboard shortcuts: + +```bash +pnpm dashboard # Default queue +pnpm dashboard:email # Email queue +pnpm dashboard:data # Data-processing queue +pnpm dashboard:notifications # Notifications queue +pnpm dashboard:deployments # Deployments queue (flows + steps) +``` + +## Related documentation + +- [CLI Overview](/docs/cli/overview/installation) +- [Queue](/docs/vorsteh-queue/core/queue) +- [Flow Trees](/docs/vorsteh-queue/core/flows/flow-trees) +- [Drizzle Adapter](/docs/vorsteh-queue/adapters/drizzle) +- [Next.js Dashboard](/docs/examples/advanced/dashboard-nextjs) + +## Troubleshooting + +### Direct mode: "Connection refused" on port 5488 + +Make sure the worker process is running first (`pnpm dev:prod`). It starts the PGlite socket server that the dashboard connects to. + +### Direct mode: "No adapter configured" + +The dashboard command looks for `queue.config.ts` in the current working directory. Make sure you run `pnpm dashboard` from the project root. + +### API mode: "Connection refused" or "Unauthorized" + +Make sure the server is running (`pnpm dev`) and the URL/token match. The default example uses: + +```bash +--url http://localhost:4000/graphql --token dev-token +``` + +## Source + +- [tui-api on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/tui-api) +- [tui-direct on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/tui-direct) diff --git a/apps/docs/content/examples/03.advanced/06.dashboard-nextjs.mdx b/apps/docs/content/examples/03.advanced/06.dashboard-nextjs.mdx new file mode 100644 index 00000000..29fc88a0 --- /dev/null +++ b/apps/docs/content/examples/03.advanced/06.dashboard-nextjs.mdx @@ -0,0 +1,186 @@ +--- +title: Next.js Dashboard +description: Full-featured web dashboard for queue monitoring built with Next.js and shadcn/ui. +tags: ["dashboard", "nextjs", "react", "monitoring", "shadcn"] +--- + +A production-ready web dashboard for monitoring and managing your job queues. Built with Next.js App Router, TanStack Query, and shadcn/ui components. + +Supports two connection modes: + +- **Direct** — Reads `queue.config.ts` and talks to the adapter directly (no separate server needed) +- **API** — Connects to a remote GraphQL server + +## Quick Start + + + create-vorsteh-queue my-app --template=dashboard-nextjs + + +```bash +cd my-app +pnpm install + +# Terminal 1: Start PGlite socket server + workers +pnpm dev:queue + +# Terminal 2: Start the Next.js dashboard +pnpm dev +``` + +Open [http://localhost:3000](http://localhost:3000). + +## Architecture + +``` +┌─────────────────┐ PostgreSQL ┌─────────────────────────┐ +│ Next.js │ Protocol │ Worker Process │ +│ (Dashboard UI) │ ◀──────────────────▶│ (PGlite Socket Server) │ +│ localhost:3000 │ port 5488 │ + Seed + Workers │ +└─────────────────┘ └─────────────────────────┘ +``` + +The worker process starts a PGlite instance with a socket server. Next.js connects via standard `node-postgres` — the same connection that works with any real PostgreSQL database. + +## Connection Modes + +### Direct Mode (default) + +The dashboard reads `queue.config.ts` via c12 and instantiates the adapter directly within the Next.js server process. No separate API server needed. + +```bash +QUEUE_MODE=direct +DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5488/postgres +``` + +```typescript +// queue.config.ts +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { drizzle } from "drizzle-orm/node-postgres" + +const db = drizzle({ connection: process.env.DATABASE_URL }) + +export default { + adapter: new PostgresQueueAdapter(db), +} +``` + +### API Mode + +The dashboard connects to a running `@vorsteh-queue/server` GraphQL endpoint. Use this when the dashboard and workers run on different machines. + +```bash +QUEUE_MODE=api +QUEUE_API_URL=http://localhost:4000/graphql +QUEUE_API_TOKEN=your-token-here +``` + +No `queue.config.ts` needed in API mode. + +## Key Features + +### Server-Side Prefetching + +Pages render with data on the server (no loading spinners). TanStack Query picks up the initial data on the client and starts polling: + +```typescript +// app/page.tsx (Server Component) +export default async function OverviewPage() { + const client = await getQueueClient() + const stats = await client.getStats() + return +} + +// app/_components/stats-cards.tsx (Client Component) +export function StatsCards({ initialData }) { + const { data } = useStats(initialData) // polls every 5s + return
...
+} +``` + +### Type-Safe URL State + +Filters and pagination are managed via URL search params with nuqs: + +```typescript +// lib/search-params.ts +export const jobsSearchParams = { + status: parseAsStringLiteral(jobStatuses), + name: parseAsString, + page: parseAsInteger.withDefault(1), +} +``` + +Works on both server (for prefetching) and client (for navigation). + +### Type-Safe Environment + +All environment variables are validated at startup via t3-env: + +```typescript +// lib/env.ts +export const env = createEnv({ + server: { + QUEUE_MODE: z.enum(["direct", "api"]).default("direct"), + DATABASE_URL: z.string().url().optional(), + QUEUE_API_URL: z.string().url().optional(), + QUEUE_API_TOKEN: z.string().optional(), + }, +}) +``` + +### Queue Client Abstraction + +A single interface switches between direct and API mode at runtime: + +```typescript +// lib/queue-client.ts +export async function getQueueClient(): Promise { + if (env.QUEUE_MODE === "api") { + return createApiClient() // GraphQL fetch + } + return createDirectClient() // c12 + adapter +} +``` + +## Pages + +| Route | Refresh | Description | +| ------------- | ------- | ----------------------------------- | +| `/` | 5s | Overview with stat cards | +| `/jobs` | 10s | Filterable, paginated job table | +| `/jobs/[id]` | 5s | Job detail (payload, error, result) | +| `/dlq` | 10s | Dead letter queue with redrive | +| `/flows` | 10s | Flow card grid | +| `/flows/[id]` | 5s | Flow tree visualization | + +## Stack + +- Next.js 16 (App Router, Server Components, Server Actions) +- TanStack Query (server prefetch + client polling) +- nuqs (type-safe URL search params) +- t3-env (type-safe environment variables) +- shadcn/ui (base-nova style) +- PGlite + pglite-socket (zero-setup PostgreSQL) +- Drizzle ORM +- c12 (config file loading) + +## Using as a Starting Point + +This example is designed to be forked and adapted: + +1. **Auth** — Add your preferred auth solution (NextAuth, Clerk, custom middleware) +2. **Database** — Replace PGlite with your production PostgreSQL URL +3. **Styling** — Customize the shadcn/ui theme in `globals.css` +4. **Features** — Add queue-specific views, charts, or alerting + +## Related Documentation + +- [TUI Dashboard](/docs/examples/advanced/tui-dashboard) +- [GraphQL Server](/docs/examples/advanced/graphql-server) +- [Queue](/docs/vorsteh-queue/core/queue) +- [Drizzle Adapter](/docs/vorsteh-queue/adapters/drizzle) + +## Source + +- [dashboard-nextjs on GitHub](https://github.com/noxify/vorsteh-queue/tree/main/examples/dashboard-nextjs) diff --git a/apps/docs/content/examples/03.advanced/index.mdx b/apps/docs/content/examples/03.advanced/index.mdx new file mode 100644 index 00000000..c862f41f --- /dev/null +++ b/apps/docs/content/examples/03.advanced/index.mdx @@ -0,0 +1,4 @@ +--- +title: Advanced +description: Advanced patterns and production deployment examples. +--- diff --git a/apps/docs/content/examples/index.mdx b/apps/docs/content/examples/index.mdx new file mode 100644 index 00000000..fab69415 --- /dev/null +++ b/apps/docs/content/examples/index.mdx @@ -0,0 +1,6 @@ +--- +title: Examples +navTitle: Examples +description: Ready-to-run example projects demonstrating Vorsteh Queue features and patterns. +entrypoint: /docs/examples/getting-started/drizzle-pglite +--- diff --git a/apps/docs/content/features/10.excellent-dx.mdx b/apps/docs/content/features/10.excellent-dx.mdx deleted file mode 100644 index f930b532..00000000 --- a/apps/docs/content/features/10.excellent-dx.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: Excellent DX -icon: Zap ---- - -Intuitive API design with TypeScript support, comprehensive documentation, and helpful error messages that make development a breeze. diff --git a/apps/docs/content/features/11.orm-agnostic.mdx b/apps/docs/content/features/11.orm-agnostic.mdx deleted file mode 100644 index 29f7d2ba..00000000 --- a/apps/docs/content/features/11.orm-agnostic.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: ORM Agnostic -icon: Database ---- - -Works with Drizzle ORM, Kysely and Prisma ORM for PostgreSQL. Adapter pattern allows easy integration with your existing database setup. diff --git a/apps/docs/content/features/12.production-ready.mdx b/apps/docs/content/features/12.production-ready.mdx deleted file mode 100644 index 8d8af46e..00000000 --- a/apps/docs/content/features/12.production-ready.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: Production Ready -icon: Shield ---- - -Battle-tested with built-in retry logic, job cleanup, progress tracking, and graceful shutdown handling for mission-critical applications. diff --git a/apps/docs/content/features/13.event-system.mdx b/apps/docs/content/features/13.event-system.mdx deleted file mode 100644 index 2d0664d1..00000000 --- a/apps/docs/content/features/13.event-system.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: Event System -icon: Zap ---- - -Comprehensive event system for monitoring job lifecycle. Listen to job progress, completion, failures, and queue state changes. diff --git a/apps/docs/content/features/14.timezone.mdx b/apps/docs/content/features/14.timezone.mdx deleted file mode 100644 index af011523..00000000 --- a/apps/docs/content/features/14.timezone.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: UTC-First Timezone -icon: Globe ---- - -Reliable timezone handling with UTC-first approach. All timestamps stored as UTC with timezone conversion at job creation time. diff --git a/apps/docs/content/features/15.type-safety.mdx b/apps/docs/content/features/15.type-safety.mdx deleted file mode 100644 index 87a605c8..00000000 --- a/apps/docs/content/features/15.type-safety.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: key_feature -title: Type Safety -icon: Shield ---- - -Full TypeScript support with generic job payloads and results. Compile-time type checking ensures your jobs are properly typed and safe. diff --git a/apps/docs/content/features/20.one-time-jobs.mdx b/apps/docs/content/features/20.one-time-jobs.mdx deleted file mode 100644 index 9d256397..00000000 --- a/apps/docs/content/features/20.one-time-jobs.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: One-time Jobs -icon: Play ---- - -Execute tasks once with optional delays and priority levels. diff --git a/apps/docs/content/features/21.recurring-jobs.mdx b/apps/docs/content/features/21.recurring-jobs.mdx deleted file mode 100644 index 25cd4c5b..00000000 --- a/apps/docs/content/features/21.recurring-jobs.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Recurring Jobs -icon: RotateCcw ---- - -Set up repeating tasks with flexible intervals and cron expressions. diff --git a/apps/docs/content/features/22.scheduled-jobs.mdx b/apps/docs/content/features/22.scheduled-jobs.mdx deleted file mode 100644 index e7647f09..00000000 --- a/apps/docs/content/features/22.scheduled-jobs.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Scheduled Jobs -icon: Calendar ---- - -Schedule jobs for specific dates and times with UTC-first timezone conversion for reliable execution. diff --git a/apps/docs/content/features/23.retry-logic.mdx b/apps/docs/content/features/23.retry-logic.mdx deleted file mode 100644 index e8780a94..00000000 --- a/apps/docs/content/features/23.retry-logic.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Retry Logic -icon: RotateCcw ---- - -Configurable retry strategies with exponential backoff and maximum attempt limits for failed jobs. diff --git a/apps/docs/content/features/24.job-delays.mdx b/apps/docs/content/features/24.job-delays.mdx deleted file mode 100644 index ebeb80a4..00000000 --- a/apps/docs/content/features/24.job-delays.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Job Delays -icon: Clock ---- - -Delay job execution with precise timing control. diff --git a/apps/docs/content/features/25.priority-queues.mdx b/apps/docs/content/features/25.priority-queues.mdx deleted file mode 100644 index c18a9d91..00000000 --- a/apps/docs/content/features/25.priority-queues.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Priority Queues -icon: ArrowUp ---- - -Process high-priority jobs first with numeric priority system (lower number = higher priority). diff --git a/apps/docs/content/features/26.job-results.mdx b/apps/docs/content/features/26.job-results.mdx deleted file mode 100644 index a51238be..00000000 --- a/apps/docs/content/features/26.job-results.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Job Results -icon: CheckCircle ---- - -Store and access job results returned by handlers. Results are automatically persisted and available through events. diff --git a/apps/docs/content/features/27.progress-tracking.mdx b/apps/docs/content/features/27.progress-tracking.mdx deleted file mode 100644 index 5cfeab1e..00000000 --- a/apps/docs/content/features/27.progress-tracking.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Progress Tracking -icon: TrendingUp ---- - -Real-time job progress updates with percentage tracking. Monitor long-running tasks with built-in progress reporting. diff --git a/apps/docs/content/features/28.job-cleanup.mdx b/apps/docs/content/features/28.job-cleanup.mdx deleted file mode 100644 index 30703581..00000000 --- a/apps/docs/content/features/28.job-cleanup.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Job Cleanup -icon: Trash2 ---- - -Automatic cleanup of completed and failed jobs with configurable retention policies to manage database size. diff --git a/apps/docs/content/features/29.job-timeouts.mdx b/apps/docs/content/features/29.job-timeouts.mdx deleted file mode 100644 index e7d3a37e..00000000 --- a/apps/docs/content/features/29.job-timeouts.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Job Timeouts -icon: Clock ---- - -Configurable job timeouts to prevent long-running jobs from blocking the queue. Set per-job or global timeout limits. diff --git a/apps/docs/content/features/30.queue-stats.mdx b/apps/docs/content/features/30.queue-stats.mdx deleted file mode 100644 index 44bdbb43..00000000 --- a/apps/docs/content/features/30.queue-stats.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Queue Statistics -icon: BarChart3 ---- - -Get real-time queue statistics including pending, processing, completed, failed, and delayed job counts. diff --git a/apps/docs/content/features/31.graceful-shutdown.mdx b/apps/docs/content/features/31.graceful-shutdown.mdx deleted file mode 100644 index 6b9d9059..00000000 --- a/apps/docs/content/features/31.graceful-shutdown.mdx +++ /dev/null @@ -1,7 +0,0 @@ ---- -type: feature -title: Graceful Shutdown -icon: Power ---- - -Clean job processing termination that waits for active jobs to complete before stopping the queue. diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/01.quickstart.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/01.quickstart.mdx new file mode 100644 index 00000000..e9b10fff --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/01.quickstart.mdx @@ -0,0 +1,93 @@ +--- +title: Quickstart +description: Get started with Vorsteh Queue in under 5 minutes. +--- + +Vorsteh Queue is a type-safe job queue engine for PostgreSQL 12+. It supports multiple ORM adapters, priority queues, delayed and recurring jobs, progress tracking, and a comprehensive event system. + +## Prerequisites + +- **Node.js 22+** +- **PostgreSQL 12+** database +- **ESM only** + +## Create a new project (recommended) + +The fastest way to get started is using the scaffolding CLI: + +create-vorsteh-queue my-queue-app + +This walks you through template selection, package manager choice, and dependency installation. + +## Manual installation + +Pick the adapter that matches your ORM: + +### Drizzle ORM + + + @vorsteh-queue/core @vorsteh-queue/adapter-drizzle + + +### Prisma ORM + + + @vorsteh-queue/core @vorsteh-queue/adapter-prisma + + +### Kysely + + + @vorsteh-queue/core @vorsteh-queue/adapter-kysely + + +### ZenStack ORM + + + @vorsteh-queue/core @vorsteh-queue/adapter-zenstack @zenstackhq/orm + + +### TypeORM + + + @vorsteh-queue/core @vorsteh-queue/adapter-typeorm typeorm pg + + +### MikroORM + + + @vorsteh-queue/core @vorsteh-queue/adapter-mikroorm @mikro-orm/core + @mikro-orm/postgresql + + +### Sequelize + + + @vorsteh-queue/core @vorsteh-queue/adapter-sequelize sequelize pg + + +## Minimal example + +```typescript +import { Queue, MemoryQueueAdapter } from "@vorsteh-queue/core" + +const queue = new Queue(new MemoryQueueAdapter(), { name: "my-queue" }) + +// Register a handler +queue.register("send-email", async (job) => { + console.log(`Sending email to ${job.payload.to}`) + return { sent: true } +}) + +// Add a job +await queue.add("send-email", { to: "user@example.com", subject: "Hello!" }) + +// Start processing +queue.start() +``` + +Replace `MemoryQueueAdapter` with a PostgreSQL adapter for production use. See the adapter docs for setup instructions. + +## Recommended Setup + +For real projects, we recommend centralizing your queue definitions in a shared file. This avoids repetition and makes configuration changes easy. See [Project Structure](/docs/vorsteh-queue/getting-started/project-structure) for the recommended approach. diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/02.installation.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/02.installation.mdx new file mode 100644 index 00000000..992a4894 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/02.installation.mdx @@ -0,0 +1,79 @@ +--- +title: Installation +description: Install Vorsteh Queue packages for your preferred ORM adapter. +--- + +## Quick Start with CLI + +create-vorsteh-queue my-queue-app + +The scaffolding CLI provides interactive prompts for template selection, package manager, and dependency installation. + +See the [create-vorsteh-queue docs](/docs/vorsteh-queue/packages/create-vorsteh-queue) for details. + +## Manual Installation + +### Drizzle ORM (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-drizzle + + +### Prisma ORM (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-prisma + + +### Kysely (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-kysely + + +### ZenStack ORM (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-zenstack @zenstackhq/orm + + +### TypeORM (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-typeorm typeorm pg + + +### MikroORM (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-mikroorm @mikro-orm/core + @mikro-orm/postgresql + + +### Sequelize (PostgreSQL) + + + @vorsteh-queue/core @vorsteh-queue/adapter-sequelize sequelize pg + + +## Database Setup + +Each adapter provides schema definitions and migration tooling: + +- **Drizzle** — use the included schema and run `drizzle-kit` migrations +- **Prisma** — add the model to `schema.prisma` and run `prisma migrate` +- **Kysely** — use the provided migration and run via `kysely-ctl` +- **ZenStack** — add the model to `schema.zmodel` and run `zen db push` +- **TypeORM** — use the included `QueueJobEntity` and `QueueFlowEntity` with `synchronize: true` or generate migrations +- **MikroORM** — use the included EntitySchemas and run `mikro-orm migration:create` +- **Sequelize** — use the included models with `sequelize.sync()` or CLI migrations + +Refer to the specific adapter documentation for detailed instructions. + +## Requirements + +| Requirement | Version | +| ------------- | ------- | +| Node.js | 22+ | +| PostgreSQL | 12+ | +| Module system | ESM | diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/03.project-structure.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/03.project-structure.mdx new file mode 100644 index 00000000..e4d39e31 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/03.project-structure.mdx @@ -0,0 +1,409 @@ +--- +title: Project Structure +description: Recommended project structure for Vorsteh Queue applications. +--- + +## Why centralize? + +Defining your adapter and queue instances in a single shared file keeps your project DRY. Instead of recreating the adapter in every file that needs it, you export once and import everywhere — your config file, workers, and application code all reference the same instances. This makes refactoring painless and ensures configuration changes propagate automatically. + +## Recommended structure + +``` +my-app/ +├── src/ +│ ├── schema.ts # Database schema (Drizzle tables + relations) +│ ├── queues.ts # Queue definitions (adapter + Queue instances) +│ ├── handlers/ +│ │ ├── email.ts # Job handlers +│ │ └── reports.ts +│ └── worker.ts # Worker setup +├── queue.config.ts # CLI/Server config (imports from src/queues.ts) +└── package.json +``` + +## The shared queue file + +`src/queues.ts` is the single source of truth for your adapter and queue instances: + + + + ```typescript + // src/queues.ts + import { defineRelations } from "drizzle-orm" + import { drizzle } from "drizzle-orm/node-postgres" + import { Pool } from "pg" + + import { + PostgresQueueAdapter, + queueFlows, + queueJobs, + } from "@vorsteh-queue/adapter-drizzle" + import { Queue } from "@vorsteh-queue/core" + + const relations = defineRelations({ queueFlows, queueJobs }) + + const pool = new Pool({ connectionString: process.env.DATABASE_URL }) + const db = drizzle({ client: pool, relations }) + + export const adapter = new PostgresQueueAdapter(db) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { PrismaClient } from "@prisma/client" + + import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" + import { Queue } from "@vorsteh-queue/core" + + const prisma = new PrismaClient() + + export const adapter = new PostgresPrismaQueueAdapter(prisma) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { Kysely, PostgresDialect } from "kysely" + import { Pool } from "pg" + + import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" + import { Queue } from "@vorsteh-queue/core" + + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + + export const adapter = new PostgresQueueAdapter(db) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { ZenStackClient } from "@zenstackhq/orm" + import { PostgresDialect } from "@zenstackhq/orm/dialects/postgres" + import { Pool } from "pg" + + import { PostgresZenstackQueueAdapter } from "@vorsteh-queue/adapter-zenstack" + import { Queue } from "@vorsteh-queue/core" + + import { schema } from "../zenstack/schema" + + const db = new ZenStackClient(schema, { + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + + export const adapter = new PostgresZenstackQueueAdapter(db) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { DataSource } from "typeorm" + + import { + PostgresTypeormQueueAdapter, + QueueFlowEntity, + QueueJobEntity, + } from "@vorsteh-queue/adapter-typeorm" + import { Queue } from "@vorsteh-queue/core" + + const dataSource = new DataSource({ + type: "postgres", + url: process.env.DATABASE_URL, + entities: [QueueJobEntity, QueueFlowEntity], + synchronize: true, + }) + + export const adapter = new PostgresTypeormQueueAdapter(dataSource) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { MikroORM } from "@mikro-orm/postgresql" + + import { + PostgresMikroormQueueAdapter, + QueueFlowSchema, + QueueJobSchema, + } from "@vorsteh-queue/adapter-mikroorm" + import { Queue } from "@vorsteh-queue/core" + + const orm = await MikroORM.init({ + entities: [QueueJobSchema, QueueFlowSchema], + clientUrl: process.env.DATABASE_URL, + }) + + export const adapter = new PostgresMikroormQueueAdapter(orm) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { Sequelize } from "sequelize" + + import { + PostgresSequelizeQueueAdapter, + initQueueFlowModel, + initQueueJobModel, + } from "@vorsteh-queue/adapter-sequelize" + import { Queue } from "@vorsteh-queue/core" + + const sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: "postgres", + logging: false, + }) + initQueueJobModel(sequelize) + initQueueFlowModel(sequelize) + + export const adapter = new PostgresSequelizeQueueAdapter(sequelize) + + export const emailQueue = new Queue(adapter, { name: "email-queue" }) + export const reportQueue = new Queue(adapter, { name: "report-queue" }) + ``` + + + + +## CLI and server configuration + +Your `queue.config.ts` simply imports from the shared file: + +```typescript +// queue.config.ts +import { adapter, emailQueue, reportQueue } from "./src/queues" + +export default { + adapter, + queues: [emailQueue, reportQueue], + defaultQueue: "email-queue", +} +``` + +## Worker setup + +Workers import the adapter and use the `Worker` class: + +```typescript +// src/worker.ts +import { Worker } from "@vorsteh-queue/core" + +import { adapter } from "./queues" + +const worker = new Worker(adapter, { name: "email-queue", concurrency: 5 }) + +worker.register("send-email", async (job) => { + // handle job +}) + +worker.start() +``` + +## Adding jobs from your app + +Application code imports the queue instance to add jobs: + +```typescript +import { emailQueue } from "./queues" + +await emailQueue.add("send-email", { to: "user@example.com", subject: "Hello" }) +``` + +## Single queue simplification + +For projects with a single queue, the pattern is the same — just simpler: + + + + ```typescript + // src/queues.ts + import { defineRelations } from "drizzle-orm" + import { drizzle } from "drizzle-orm/node-postgres" + import { Pool } from "pg" + + import { + PostgresQueueAdapter, + queueFlows, + queueJobs, + } from "@vorsteh-queue/adapter-drizzle" + import { Queue } from "@vorsteh-queue/core" + + const relations = defineRelations({ queueFlows, queueJobs }) + + const pool = new Pool({ connectionString: process.env.DATABASE_URL }) + const db = drizzle({ client: pool, relations }) + + export const adapter = new PostgresQueueAdapter(db) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { PrismaClient } from "@prisma/client" + + import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" + import { Queue } from "@vorsteh-queue/core" + + const prisma = new PrismaClient() + + export const adapter = new PostgresPrismaQueueAdapter(prisma) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { Kysely, PostgresDialect } from "kysely" + import { Pool } from "pg" + + import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely" + import { Queue } from "@vorsteh-queue/core" + + const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + + export const adapter = new PostgresQueueAdapter(db) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { ZenStackClient } from "@zenstackhq/orm" + import { PostgresDialect } from "@zenstackhq/orm/dialects/postgres" + import { Pool } from "pg" + + import { PostgresZenstackQueueAdapter } from "@vorsteh-queue/adapter-zenstack" + import { Queue } from "@vorsteh-queue/core" + + import { schema } from "../zenstack/schema" + + const db = new ZenStackClient(schema, { + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), + }) + + export const adapter = new PostgresZenstackQueueAdapter(db) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { DataSource } from "typeorm" + + import { + PostgresTypeormQueueAdapter, + QueueFlowEntity, + QueueJobEntity, + } from "@vorsteh-queue/adapter-typeorm" + import { Queue } from "@vorsteh-queue/core" + + const dataSource = new DataSource({ + type: "postgres", + url: process.env.DATABASE_URL, + entities: [QueueJobEntity, QueueFlowEntity], + synchronize: true, + }) + + export const adapter = new PostgresTypeormQueueAdapter(dataSource) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { MikroORM } from "@mikro-orm/postgresql" + + import { + PostgresMikroormQueueAdapter, + QueueFlowSchema, + QueueJobSchema, + } from "@vorsteh-queue/adapter-mikroorm" + import { Queue } from "@vorsteh-queue/core" + + const orm = await MikroORM.init({ + entities: [QueueJobSchema, QueueFlowSchema], + clientUrl: process.env.DATABASE_URL, + }) + + export const adapter = new PostgresMikroormQueueAdapter(orm) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + ```typescript + // src/queues.ts + import { Sequelize } from "sequelize" + + import { + PostgresSequelizeQueueAdapter, + initQueueFlowModel, + initQueueJobModel, + } from "@vorsteh-queue/adapter-sequelize" + import { Queue } from "@vorsteh-queue/core" + + const sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: "postgres", + logging: false, + }) + initQueueJobModel(sequelize) + initQueueFlowModel(sequelize) + + export const adapter = new PostgresSequelizeQueueAdapter(sequelize) + export const queue = new Queue(adapter, { name: "my-queue" }) + ``` + + + + +```typescript +// queue.config.ts +import { adapter, queue } from "./src/queues" + +export default { + adapter, + queues: [queue], +} +``` diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/04.alternatives.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/04.alternatives.mdx new file mode 100644 index 00000000..fd859f39 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/04.alternatives.mdx @@ -0,0 +1,247 @@ +--- +title: Alternatives +description: Other job queue libraries for Node.js and when they might be a better fit. +--- + +Vorsteh Queue isn't the only option — and depending on your stack, it may not even be the best one. + +This page compares other established queue engines and where they shine. Vorsteh Queue is inspired by multiple projects in this space: we've learned a lot from Redis-first queues, Postgres-native workers, and scheduler-focused libraries. + +The goal is to bring those patterns together into one cohesive package — PostgreSQL durability, TypeScript-first DX, ORM adapters, and modern ops tooling — while staying honest about when a dedicated alternative is the better fit. + + + If you think something is wrong here, please feel free to raise an issue. + + +## BullMQ + +[BullMQ](https://bullmq.io/) is the most popular Node.js job queue, backed by Redis. + +- **Backend:** Redis +- **TypeScript:** First-class +- **Flows (DAG):** Yes (FlowProducer) +- **Web Dashboard:** community / paid + - community: [Bull Board](https://github.com/felixmosh/bull-board) + - paid: [Taskforce](https://taskforce.sh/) +- **Cron:** Yes (repeatable jobs) +- **Rate Limiting:** Yes + +**Choose BullMQ when:** + +- You already run Redis in production +- You need extremely high throughput (10k+ jobs/sec) +- You want the largest community and ecosystem + +**Choose Vorsteh Queue instead when:** + +- You want a **PostgreSQL-only stack** (no Redis) for queueing +- You want **enqueue + app data writes in the same PostgreSQL transaction** +- You want **first-class ORM adapters** (Drizzle, Prisma, Kysely, ZenStack, TypeORM, MikroORM, Sequelize) and typed DX around your schema + +## pg-boss + +[pg-boss](https://github.com/timgit/pg-boss) is a PostgreSQL-based job queue built on `SKIP LOCKED`. + +- **Backend:** PostgreSQL +- **TypeScript:** First-class +- **Flows (DAG):** No (supports dependency/workflow orchestration, but not a FlowProducer-style tree model) +- **Web Dashboard:** official (add-on) + - [@pg-boss/dashboard](https://github.com/timgit/pg-boss/tree/master/packages/dashboard) +- **CLI:** Yes +- **Cron:** Yes +- **Rate Limiting:** Yes (throttling) +- **DLQ:** Yes +- **ORM integration:** Transaction adapters (Drizzle, Prisma, Kysely, Knex) + +**Choose pg-boss when:** + +- You want a battle-tested, mature PostgreSQL queue +- You want strong operational fundamentals (DLQ, throttling, retries) +- You don't need FlowProducer-style DAG flows + +**Choose Vorsteh Queue instead when:** + +- You need **FlowProducer-style parent-child DAG trees** (children-first, parent waits, tree introspection) +- You want **end-to-end typed payload + typed result contracts** (not just "types exist") +- You want **built-in** monitoring/ops UX (dashboard/CLI) rather than relying on add-ons + +## Graphile Worker + +[Graphile Worker](https://github.com/graphile/worker) is a high-performance PostgreSQL job queue focused on simplicity and low latency. + +- **Backend:** PostgreSQL +- **TypeScript:** Types available +- **Flows (DAG):** No +- **Web Dashboard:** none +- **Cron:** Yes +- **Rate Limiting:** No (community package exists) + +**Choose Graphile Worker when:** + +- You want minimal dependencies and proven Postgres fundamentals +- You like a SQL-first approach +- You care about low latency (LISTEN/NOTIFY) + +**Choose Vorsteh Queue instead when:** + +- You want **built-in** progress tracking and richer job state APIs (not DIY via tables/logs) +- You need **FlowProducer-style DAG flows** +- You want **ORM adapters + schema-managed setup** rather than hand-managed SQL + +## Agenda + +[Agenda](https://github.com/agenda/agenda) is a job scheduler with pluggable persistence backends. + +- **Backend:** Pluggable (MongoDB, PostgreSQL, Redis) +- **TypeScript:** Types available +- **Flows (DAG):** No +- **Web Dashboard:** community + - [Agendash](https://github.com/agenda/agendash) +- **Cron:** Yes +- **Rate Limiting:** No (not a core feature) + +**Choose Agenda when:** + +- You want scheduler-first recurring jobs +- You're already in an ecosystem where Agenda is a good fit + +**Choose Vorsteh Queue instead when:** + +- You want a **queue-first system** (workers, retries/DLQ, visibility) rather than mainly a scheduler +- You want **PostgreSQL-backed durability** aligned with application data +- You need **flows / group FIFO / progress tracking** as first-class features + +## GroupMQ + +[GroupMQ](https://github.com/Openpanel-dev/groupmq) is a Redis-backed queue with first-class **per-group FIFO** ordering. + +- **Backend:** Redis +- **TypeScript:** First-class +- **Flows (DAG):** No +- **Web Dashboard:** community / paid + - community: [Bull Board integration](https://github.com/Openpanel-dev/groupmq) + - paid: [QueueDash integration](https://www.queuedash.com/docs/integrations/groupmq) +- **Cron:** Yes (repeatable jobs / cron patterns) +- **Group FIFO:** Yes +- **Rate Limiting:** No (not a core feature) + +**Choose GroupMQ when:** + +- Per-group FIFO ordering is the main requirement +- You already run Redis and want strict sequential processing per group + +**Choose Vorsteh Queue instead when:** + +- You want **PostgreSQL** instead of Redis for queue storage +- You want **one system** that combines **flows + cron + progress + monitoring** on Postgres +- You want **ORM adapters** and schema management integrated with your app + +## Platformatic Job Queue + +[Platformatic Job Queue](https://github.com/platformatic/job-queue) is a TypeScript queue with pluggable storage and a request/response pattern. + +- **Backend:** Pluggable (Memory, Redis/Valkey, Filesystem) +- **TypeScript:** First-class +- **Flows (DAG):** No +- **Web Dashboard:** none +- **CLI:** none +- **Cron:** No +- **Deduplication:** Yes +- **Request/Response:** Yes (enqueue and await result) + +**Choose Platformatic Job Queue when:** + +- You want enqueue-and-wait request/response semantics +- You need deduplication built-in +- You're in the Platformatic ecosystem + +**Choose Vorsteh Queue instead when:** + +- You want **PostgreSQL-backed durability** and multi-instance workers +- You need **cron scheduling** and **flow orchestration** (DAG flows) +- You want **monitoring UX** (dashboard/CLI) designed for ops, not just an API + +## prisma-queue + +[prisma-queue](https://github.com/mgcrea/prisma-queue) is a lightweight Prisma-based PostgreSQL queue. + +- **Backend:** PostgreSQL +- **TypeScript:** First-class +- **ORM Integration:** Prisma +- **Flows (DAG):** No +- **Web Dashboard:** none +- **CLI:** none +- **Cron:** Yes (cron syntax scheduling) +- **Rate Limiting:** No + +**Choose prisma-queue when:** + +- You want a small Prisma-only queue with minimal surface area +- You're happy to bring your own monitoring/ops + +**Choose Vorsteh Queue instead when:** + +- You want **multiple ORM adapters** (not Prisma-only) +- You need **FlowProducer-style DAG flows** and built-in progress reporting +- You want **dashboard + CLI** for monitoring instead of building your own + +## BunQueue + +[BunQueue](https://bunqueue.dev/) is a high-performance queue for the Bun runtime using SQLite persistence. + +- **Backend:** SQLite (Bun-native) +- **TypeScript:** First-class +- **Flows (DAG):** Yes (workflows / parent-child flows) +- **Web Dashboard:** official (beta / early access) +- **CLI:** Yes +- **Cron:** Yes +- **DLQ:** Yes +- **Rate Limiting:** Yes + +**Choose BunQueue when:** + +- You run Bun in production and want a native Bun queue +- You like SQLite persistence (single-file deployments) +- You want workflows in the Bun ecosystem + +**Choose Vorsteh Queue instead when:** + +- You need **Node.js** compatibility (not Bun-only) +- You want **PostgreSQL** for multi-instance / HA worker setups +- You want **ORM adapters** (Drizzle, Prisma, Kysely, ZenStack, TypeORM, MikroORM, Sequelize) in a Postgres-first design + +## Comparison + +**Terminology** + +- **Flows (DAG):** FlowProducer-style parent-child job trees (children run first; parent waits; flow tree can be inspected). +- **Web Dashboard:** browser-based monitoring UI (not logs/metrics). +- **Progress:** built-in API for reporting and querying job progress. + +### Backend & Developer Experience + +| Library | Backend | TypeScript | ORM Integration | +| --- | --- | --- | --- | +| Vorsteh Queue | PostgreSQL | First-class | Drizzle, Prisma, Kysely, ZenStack, TypeORM, MikroORM, Sequelize | +| BullMQ | Redis | First-class | — | +| pg-boss | PostgreSQL | First-class | Tx adapters (Drizzle, Prisma, Kysely, Knex) | +| Graphile Worker | PostgreSQL | Types avail. | — | +| Agenda | Pluggable (MongoDB, PostgreSQL, Redis) | Types avail. | — | +| GroupMQ | Redis | First-class | — | +| Platformatic | Pluggable (Memory, Redis/Valkey, FS) | First-class | — | +| prisma-queue | PostgreSQL | First-class | Prisma | +| BunQueue | SQLite | First-class | — | + +### Features + +| Library | Flows (DAG) | Progress | Web Dashboard | CLI | Cron | Rate Limit | Group FIFO | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Vorsteh Queue | Yes | Yes | built-in | Yes | Yes | Yes | Yes | +| BullMQ | Yes | Yes | community / paid | No | Yes | Yes | No | +| pg-boss | No | No | official (add-on) | Yes | Yes | Yes | No | +| Graphile Worker | No | No | none | No | Yes | No | No | +| Agenda | No | No | community | No | Yes | No | No | +| GroupMQ | No | No | community / paid | No | Yes | No | Yes | +| Platformatic | No | No | none | No | No | No | No | +| prisma-queue | No | No | none | No | Yes | No | No | +| BunQueue | Yes | No | official (beta) | Yes | Yes | Yes | No | diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/05.about.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/05.about.mdx new file mode 100644 index 00000000..2c622023 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/05.about.mdx @@ -0,0 +1,21 @@ +--- +title: About Vorsteh Queue +navTitle: About +description: The story behind the name and the project. +--- + +The name "Vorsteh Queue" is a tribute to our beloved German Spaniel. This breed is closely related to the Münsterländer, a type of pointing dog — known in German as a "Vorstehhund". + +Just as a pointing dog steadfastly indicates its target, Vorsteh Queue aims to reliably point your application towards efficient and robust background job processing. + +The inspiration for naming a tech project after a dog comes from the delightful story of **Bruno**, the API client. It's a nod to the personal touch and passion that drives open-source development, much like the loyalty and dedication of our canine companions. + +## Meet Raja + +Raja, the German Spaniel who inspired the Vorsteh Queue logo + +This is Raja — our German Spaniel and the direct inspiration behind the Vorsteh Queue logo. diff --git a/apps/docs/content/vorsteh-queue/01.getting-started/index.mdx b/apps/docs/content/vorsteh-queue/01.getting-started/index.mdx new file mode 100644 index 00000000..15aa0b87 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/01.getting-started/index.mdx @@ -0,0 +1,4 @@ +--- +title: Getting Started +description: Learn how to set up and use Vorsteh Queue in your Node.js project. +--- diff --git a/apps/docs/content/vorsteh-queue/02.core/01.queue.mdx b/apps/docs/content/vorsteh-queue/02.core/01.queue.mdx new file mode 100644 index 00000000..22b4a148 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/01.queue.mdx @@ -0,0 +1,138 @@ +--- +title: Queue +description: Create and manage job queues — add jobs, control processing, and monitor statistics. +--- + +The `Queue` class is the producer-side API. It manages adding jobs, cancellation, statistics, and flows. It does **not** process jobs — that responsibility belongs to the [Worker](/docs/vorsteh-queue/core/worker). + +## Creating a Queue + +A Queue requires an adapter (database connection) and a configuration with a name: + +```typescript +import { Queue, MemoryQueueAdapter } from "@vorsteh-queue/core" + +const adapter = new MemoryQueueAdapter() +const queue = new Queue(adapter, { name: "my-queue" }) + +await queue.connect() +``` + +For production, replace `MemoryQueueAdapter` with a PostgreSQL adapter: + +```typescript +import { drizzle } from "drizzle-orm/node-postgres" +import { Pool } from "pg" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { Queue } from "@vorsteh-queue/core" + +const pool = new Pool({ connectionString: process.env.DATABASE_URL }) +const db = drizzle(pool) +const adapter = new PostgresQueueAdapter(db) +const queue = new Queue(adapter, { name: "email-queue" }) + +await queue.connect() +``` + +## Public Getters + +```typescript +queue.name // The queue name ("email-queue") +queue.adapter // The adapter instance +``` + +## Adding Jobs + +```typescript +// Basic job +await queue.add("send-email", { to: "user@example.com", subject: "Welcome!" }) + +// Job with options +await queue.add( + "send-email", + { to: "user@example.com", subject: "Urgent" }, + { + priority: 1, // lower number = higher priority (default: 2) + delay: 5000, // delay 5 seconds before processing + maxAttempts: 5, // retry up to 5 times on failure + } +) + +// Batch add (all jobs get same options) +await queue.addJobs("send-email", [ + { to: "a@example.com", subject: "Hello" }, + { to: "b@example.com", subject: "Hello" }, +]) +``` + +See [Job Types](/docs/vorsteh-queue/core/job-types) for all scheduling patterns (delayed, cron, repeating, unique, grouped). + +## Cancellation + +```typescript +// Cancel a single job +await queue.cancel(jobId, "No longer needed") + +// Cancel multiple jobs by filter +const count = await queue.cancelAll({ name: "send-email", status: "pending" }) +``` + +## Statistics + +```typescript +const stats = await queue.getStats() +// { pending, delayed, processing, completed, failed, cancelled, dead } + +// Clear jobs +await queue.clear() // Clear all jobs +await queue.clear("completed") // Clear only completed jobs +``` + +## Dead Letter Queue + +```typescript +// Get dead jobs +const deadJobs = await queue.getDeadJobs({ limit: 20 }) + +// Redrive a single dead job back to pending +await queue.redrive(jobId) + +// Redrive all dead jobs (optionally filtered by handler name) +await queue.redriveAll({ name: "send-email" }) +``` + +## Job Lookup + +```typescript +const job = await queue.getJob(jobId) +// Returns Job | null +``` + +## Signals (for step.waitFor) + +Send a signal to a job that is waiting via `step.waitFor`: + +```typescript +await queue.signal(jobId, "manager-approved", { approved: true }) +``` + +## Lifecycle + +```typescript +await queue.connect() // Connect adapter to database +await queue.disconnect() // Disconnect adapter +``` + +## Queue Configuration + +| Option | Type | Default | Description | +| --- | --- | --- | --- | +| `name` | `string` | required | Queue name for job isolation | +| `defaultJobOptions` | `Partial` | `{}` | Default options applied to all jobs | +| `telemetry` | `Telemetry` | `noopTelemetry` | Optional telemetry instance for observability | +| `removeOnComplete` | `boolean \| number` | `100` | Auto-remove completed jobs (true = immediate, number = keep N) | +| `removeOnFail` | `boolean \| number` | `50` | Auto-remove failed jobs | + +## Project Organization + +For multi-queue setups, see [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). diff --git a/apps/docs/content/vorsteh-queue/02.core/02.job-types.mdx b/apps/docs/content/vorsteh-queue/02.core/02.job-types.mdx new file mode 100644 index 00000000..9515cbf9 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/02.job-types.mdx @@ -0,0 +1,144 @@ +--- +title: Job Types +description: Delayed jobs, recurring jobs with cron expressions, interval-based repetition, and unique jobs. +--- + +Vorsteh Queue supports multiple job patterns beyond simple fire-and-forget. This page covers all the ways you can configure a job when adding it to the queue. + +## Methods + +Three methods are available for adding jobs: + +| Method | Description | +| --- | --- | +| `queue.add(name, payload, options?)` | Add a single job | +| `queue.addJobs(name, payloads, options?)` | Add multiple jobs of the same type in a batch | +| `queue.enqueueAndWait(name, payload, options?)` | Add a job and await its result | +| `flowProducer.add(definition)` | Create a parent-child flow tree — see [Flows](/docs/vorsteh-queue/core/flows/overview) | + +## Immediate Jobs + +The simplest form — a job that is processed as soon as a worker picks it up: + +```typescript +await queue.add("send-email", { to: "user@example.com", subject: "Welcome!" }) +``` + +## Delayed Jobs + +Schedule a job to run after a specified delay: + +```typescript +await queue.add( + "cleanup", + { type: "temp-files" }, + { + delay: 60000, // 1 minute from now + } +) +``` + +Or schedule for a specific date/time: + +```typescript +await queue.add( + "reminder", + { userId: "123" }, + { + runAt: new Date("2025-03-01T09:00:00Z"), + } +) +``` + +## Recurring Jobs (Cron) + +Use standard cron expressions for recurring jobs: + +```typescript +await queue.add( + "daily-report", + { date: new Date().toISOString() }, + { + cron: "0 9 * * *", // Every day at 9 AM + timezone: "America/New_York", + } +) +``` + +Cron expressions follow the standard 5-field format. Timezone support uses the IANA timezone database with UTC as the storage format. + + + Need help writing cron expressions? Try + [natural-cron](https://github.com/satyajitnayk/natural-cron) to generate them + from human-readable descriptions. + + +## Interval-based Repetition + +Repeat a job at fixed intervals: + +```typescript +await queue.add( + "health-check", + { endpoint: "/api/health" }, + { + repeat: { + every: 30000, // Every 30 seconds + limit: 10, // Maximum 10 executions + }, + } +) +``` + +## Unique Jobs + +Prevent duplicate jobs using a deduplication key: + +```typescript +await queue.add( + "sync-user", + { userId: "123" }, + { + unique: { + key: "sync-user-123", + action: "reject", // or "replace" + }, + } +) +``` + +## Group FIFO + +Ensure jobs within the same group are processed sequentially: + +```typescript +await queue.add( + "process-order", + { orderId: "abc" }, + { + group: "customer-42", + } +) +``` + +## Enqueue and Wait + +Add a job and block until it completes, fails, or times out. Useful for request/response patterns where the caller needs the result: + +```typescript +const result = await queue.enqueueAndWait( + "process-image", + { url: "https://example.com/img.png" }, + { waitTimeout: 60000 } +) + +console.log(result) // { thumbnail: "..." } +``` + +The method uses event listeners (fast, in-process) with database polling as a cross-process fallback. It throws typed errors for failures, cancellations, timeouts, and dead-letter moves. + +## Options + + + +All timestamps are stored as UTC internally, regardless of the timezone specified in scheduling options. diff --git a/apps/docs/content/vorsteh-queue/02.core/03.flows/01.overview.mdx b/apps/docs/content/vorsteh-queue/02.core/03.flows/01.overview.mdx new file mode 100644 index 00000000..5aa67504 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/03.flows/01.overview.mdx @@ -0,0 +1,54 @@ +--- +title: Overview +description: Orchestrate complex job workflows using flow trees and multi-step execution. +--- + +Vorsteh Queue provides two complementary patterns for orchestrating work beyond simple independent jobs: + +## Flow Trees + +Declarative parent-child job hierarchies managed by the `FlowProducer` class. Children are processed first; the parent only runs when all children complete. Ideal for fan-out/fan-in patterns like build pipelines. + +```typescript +import { FlowProducer } from "@vorsteh-queue/core" + +const flowProducer = new FlowProducer(adapter) + +const flow = await flowProducer.add({ + name: "deploy", + queueName: "ops", + payload: { version: "1.0" }, + children: [ + { name: "build", queueName: "ci", payload: { target: "linux" } }, + { name: "build", queueName: "ci", payload: { target: "macos" } }, + ], +}) +``` + +[Learn more about Flow Trees →](/docs/vorsteh-queue/core/flows/flow-trees) + +## Steps + +Multi-step job execution with durable state, sleep/wait primitives, and saga-style compensation. Each step is idempotent — on retry, completed steps replay from cache. + +```typescript +worker.register("onboarding", async (job, { step }) => { + const user = await step.run("create-user", () => createUser(job.payload), { + compensate: (result) => deleteUser(result.id), + }) + await step.sleep("cooldown", "1h") + await step.run("send-welcome", () => sendEmail(user.email)) + return { userId: user.id } +}) +``` + +[Learn more about Steps →](/docs/vorsteh-queue/core/flows/steps) + +## When to Use What + +| Pattern | Use Case | Coordination | +| --- | --- | --- | +| **Flow Trees** | Fan-out/fan-in, pipelines, sequential chains | FlowProducer creates parent-child trees; parent waits for children | +| **Steps** | Multi-stage processing within a single job | Sequential steps with retry/compensation within one handler | + +Use flow trees when you need multiple independent jobs to coordinate (e.g., parallel processing, then aggregation). Use steps when the work is logically one job but has multiple durable stages. diff --git a/apps/docs/content/vorsteh-queue/02.core/03.flows/02.flow-trees.mdx b/apps/docs/content/vorsteh-queue/02.core/03.flows/02.flow-trees.mdx new file mode 100644 index 00000000..1987889b --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/03.flows/02.flow-trees.mdx @@ -0,0 +1,250 @@ +--- +title: Flow Trees +description: Parent-child job trees where children run first and the parent waits until all are complete. +--- + +The `FlowProducer` class creates declarative job trees with parent-child relationships. Children are processed first, and parent jobs are only created once all their children reach a terminal state. This enables DAG-style orchestration without external coordinators. + +## Setup + +Create a `FlowProducer` instance with an adapter that implements both `QueueAdapter` and `FlowAdapter`: + +```typescript +import { FlowProducer, Worker } from "@vorsteh-queue/core" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" + +const adapter = new PostgresQueueAdapter(db) +const flowProducer = new FlowProducer(adapter) +const worker = new Worker(adapter, { name: "my-queue" }) +``` + +## Creating a Flow + +Use `flowProducer.add()` with a declarative tree definition: + +```typescript +const flow = await flowProducer.add({ + name: "deploy", + queueName: "ops", + payload: { version: "1.2.0" }, + children: [ + { name: "build", queueName: "ci", payload: { target: "linux" } }, + { name: "build", queueName: "ci", payload: { target: "macos" } }, + { + name: "test", + queueName: "ci", + payload: {}, + children: [{ name: "lint", queueName: "ci", payload: {} }], + }, + ], +}) + +console.log(flow.flowId) // shared flow ID +console.log(flow.rootNode.status) // "waiting" +``` + +## How It Works + +1. `flowProducer.add()` creates all flow nodes in the `queue_flows` table atomically. +2. Leaf nodes (no children) immediately create jobs in `queue_jobs` with status `"pending"`. +3. Parent nodes stay in `"waiting"` status with no job created yet. +4. When a leaf job completes, the Worker updates the flow node and checks if the parent is promotable. +5. Once all children reach a terminal state, the parent node is promoted — a real job is created for it. +6. This cascades up the tree until the root job runs. + +``` +deploy (waiting — no job yet) +├── build:linux (ready — job pending) +├── build:macos (ready — job pending) +└── test (waiting — no job yet) + └── lint (ready — job pending) +``` + +## Convenience Methods + +### Sequential Chain + +`addChain` creates a sequence where each step runs after the previous one completes: + +```typescript +const chain = await flowProducer.addChain([ + { name: "fetch-data", queueName: "etl", payload: { source: "s3" } }, + { name: "transform", queueName: "etl", payload: { format: "parquet" } }, + { name: "upload", queueName: "etl", payload: { dest: "warehouse" } }, +]) +// fetch-data → transform → upload (sequential) +``` + +### Fan-In (Bulk Then) + +`addBulkThen` runs parallel jobs, then a final aggregation job: + +```typescript +const result = await flowProducer.addBulkThen( + [ + { name: "fetch-users", queueName: "data", payload: {} }, + { name: "fetch-orders", queueName: "data", payload: {} }, + { name: "fetch-products", queueName: "data", payload: {} }, + ], + { name: "generate-report", queueName: "reports", payload: { format: "pdf" } } +) +// All three fetch jobs run in parallel +// generate-report runs after all three complete +``` + +## Accessing Children Results + +Inside a parent handler, use `ctx.flow` to access children results: + +```typescript +worker.register("deploy", async (job, ctx) => { + const childResults = await ctx.flow?.getChildrenValues() + // ReadonlyMap + + const failedChildren = await ctx.flow?.getFailedChildrenValues() + // ReadonlyMap + + // Filter by name or status + const buildResults = await ctx.flow?.getChildrenValuesBy({ + name: "build", + status: "completed", + }) + + return { deployed: true, builds: childResults?.size } +}) +``` + +## Failure Strategies + +Each child node can specify a failure strategy that controls how the parent reacts when the child fails terminally (moves to dead): + +```typescript +const flow = await flowProducer.add({ + name: "pipeline", + queueName: "ops", + payload: {}, + children: [ + { + name: "critical-step", + queueName: "ops", + payload: {}, + failureStrategy: "fail-parent", + }, + { + name: "optional-step", + queueName: "ops", + payload: {}, + failureStrategy: "continue-parent", + }, + { + name: "normal-step", + queueName: "ops", + payload: {}, + // default: parent promoted when all children are terminal + }, + ], +}) +``` + +| Strategy | Behavior | +| --- | --- | +| `"default"` | Parent is promoted once ALL children reach a terminal state (completed, failed, or cancelled) | +| `"fail-parent"` | Parent is immediately marked as failed — no job is created for it. Cascades recursively upward. | +| `"continue-parent"` | Parent is immediately promoted (job created) even while other siblings may still be running. | + +## Cancelling Unprocessed Children + +A promoted parent handler can cancel remaining unprocessed children: + +```typescript +worker.register("eager-parent", async (job, ctx) => { + // Cancel children that haven't completed yet + const cancelled = await ctx.flow?.removeUnprocessedChildren() + console.log(`Cancelled ${cancelled} unprocessed children`) + + return { early: true } +}) +``` + +## Inspecting a Flow + +Retrieve the full tree structure for visualization or debugging: + +```typescript +const tree = await flowProducer.getFlow(flow.flowId) + +if (tree) { + console.log(tree.node.name, tree.node.status) + for (const child of tree.children) { + console.log(` ${child.node.name}: ${child.node.status}`) + } +} +``` + +List all flows with pagination and filtering: + +```typescript +const flows = await flowProducer.getFlows({ + status: "completed", + limit: 10, + offset: 0, +}) +``` + +## Cleanup + +Configure automatic cleanup of completed flows: + +```typescript +const flowProducer = new FlowProducer(adapter, { + removeOnComplete: 100, // keep 100 most recent completed flows (default) + // removeOnComplete: true // delete immediately on completion + // removeOnComplete: false // keep forever +}) +``` + +Call `cleanupCompletedFlows()` to trigger cleanup (typically called after a flow completes): + +```typescript +worker.on("flow:completed", async () => { + await flowProducer.cleanupCompletedFlows() +}) +``` + +## Cross-Queue Flows + +Nodes in a flow can target different queues. Each queue needs its own Worker: + +```typescript +const flow = await flowProducer.add({ + name: "process-order", + queueName: "orders", + payload: { orderId: "123" }, + children: [ + { name: "charge-payment", queueName: "payments", payload: { amount: 99 } }, + { name: "reserve-stock", queueName: "inventory", payload: { sku: "ABC" } }, + ], +}) + +// Each queue needs its own worker +const orderWorker = new Worker(adapter, { name: "orders" }) +const paymentWorker = new Worker(adapter, { name: "payments" }) +const inventoryWorker = new Worker(adapter, { name: "inventory" }) +``` + +> **Note:** All flow nodes must reside in the same database schema. Cross-database flows are not supported. + +## Database Schema + +The FlowProducer uses a separate `queue_flows` table for orchestration data. This table is created alongside `queue_jobs` when you set up your adapter schema. + +| Column | Description | +| -------------------- | -------------------------------------------- | +| `id` | Flow node UUID | +| `flow_id` | Groups all nodes in a flow | +| `parent_node_id` | FK to parent node (null for root) | +| `job_id` | FK to queue_jobs (set when promoted) | +| `status` | waiting, ready, completed, failed, cancelled | +| `failure_strategy` | default, fail-parent, continue-parent | +| `children_count` | Number of direct children | +| `children_completed` | Children at terminal state | diff --git a/apps/docs/content/vorsteh-queue/02.core/03.flows/03.steps.mdx b/apps/docs/content/vorsteh-queue/02.core/03.flows/03.steps.mdx new file mode 100644 index 00000000..35be64f3 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/03.flows/03.steps.mdx @@ -0,0 +1,163 @@ +--- +title: Steps +description: Multi-step job execution with durable state, sleep/wait primitives, and saga-style compensation. +--- + +Steps let you break a job handler into named, durable stages. Each step is idempotent — if a job is retried, completed steps replay their cached results without re-executing. This makes long-running workflows safe to retry. + +## Basic Usage + +Access the step API via the second argument in your handler: + +```typescript +worker.register("process-order", async (job, { step }) => { + const payment = await step.run("charge", () => + chargeCard(job.payload.cardId, job.payload.amount) + ) + + await step.run("fulfill", () => + shipOrder(job.payload.orderId, payment.transactionId) + ) + + await step.run("notify", () => + sendConfirmation(job.payload.email, payment.transactionId) + ) + + return { transactionId: payment.transactionId } +}) +``` + +On retry, if `"charge"` already completed, its result is returned from cache and the function is not called again. + +## step.run + +Execute a named step. Returns the cached result on replay. + +```typescript +const result = await step.run("step-name", async () => { + // your logic here + return { data: "value" } +}) +``` + +### With Compensation (Saga Pattern) + +Provide a `compensate` function to undo the step if a later step fails: + +```typescript +worker.register("onboarding", async (job, { step }) => { + const user = await step.run("create-user", () => createUser(job.payload), { + compensate: (result) => deleteUser(result.id), + }) + + const subscription = await step.run( + "create-subscription", + () => createSubscription(user.id), + { + compensate: (result) => cancelSubscription(result.id), + } + ) + + // If this step throws, compensations run in reverse order: + // cancelSubscription → deleteUser + await step.run("send-welcome", () => sendEmail(user.email)) + + return { userId: user.id, subscriptionId: subscription.id } +}) +``` + +Compensations run in reverse order (last registered first), following the saga pattern. If a compensation itself fails, it logs the error and continues running remaining compensations. + +## step.sleep + +Pause execution for a duration. The job is moved to `"delayed"` status and resumes after the specified time. + +```typescript +worker.register("drip-campaign", async (job, { step }) => { + await step.run("send-day-1", () => sendEmail(job.payload.email, "welcome")) + + await step.sleep("wait-3-days", "3d") + + await step.run("send-day-4", () => sendEmail(job.payload.email, "tips")) + + await step.sleep("wait-7-days", "7d") + + await step.run("send-day-11", () => sendEmail(job.payload.email, "offer")) +}) +``` + +Supported duration formats: + +| Format | Example | Description | +| -------- | --------- | ------------ | +| `number` | `5000` | Milliseconds | +| `"Nms"` | `"500ms"` | Milliseconds | +| `"Ns"` | `"30s"` | Seconds | +| `"Nm"` | `"5m"` | Minutes | +| `"Nh"` | `"1h"` | Hours | +| `"Nd"` | `"3d"` | Days | + +## step.waitFor + +Pause execution until an external signal is received. Useful for human-in-the-loop workflows or external system callbacks. + +```typescript +worker.register("approval-flow", async (job, { step }) => { + await step.run("submit", () => createApprovalRequest(job.payload)) + + const approval = await step.waitFor("wait-approval", "manager-approved", { + timeout: "24h", + }) + + if (approval.approved) { + await step.run("provision", () => provisionAccess(job.payload.userId)) + } + + return { approved: approval.approved } +}) +``` + +Send a signal to a waiting job using `queue.sendSignal()`: + +```typescript +await queue.sendSignal(jobId, "manager-approved", { + approved: true, + approvedBy: "manager@company.com", +}) +``` + +If the timeout expires before the signal arrives, the step fails. + +## step.all + +Run multiple async operations in parallel within a step context: + +```typescript +worker.register("parallel-fetch", async (job, { step }) => { + const [users, orders] = await step.all([ + step.run("fetch-users", () => fetchUsers()), + step.run("fetch-orders", () => fetchOrders()), + ]) + + return { users, orders } +}) +``` + +## How Retries Work with Steps + +When a job with steps is retried: + +1. The step context loads previously completed steps from the database +2. Each `step.run` checks if it already has a cached result +3. Completed steps return their cached result instantly (no re-execution) +4. Execution resumes from the first incomplete step + +This makes steps safe for operations with side effects (payments, emails, API calls) — they won't be duplicated on retry. + +## Step Context API + + + +## Step State + + diff --git a/apps/docs/content/vorsteh-queue/02.core/03.flows/index.mdx b/apps/docs/content/vorsteh-queue/02.core/03.flows/index.mdx new file mode 100644 index 00000000..a63d83f1 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/03.flows/index.mdx @@ -0,0 +1,8 @@ +--- +title: Flows +description: Workflow orchestration with flow trees and durable steps. +--- + +Flows help you orchestrate jobs across multiple stages. + +Use flow trees (via `FlowProducer`) for structured parent-child pipelines with fan-out/fan-in patterns, and steps for durable resumable workflows within a single job. diff --git a/apps/docs/content/vorsteh-queue/02.core/04.events.mdx b/apps/docs/content/vorsteh-queue/02.core/04.events.mdx new file mode 100644 index 00000000..83e1457b --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/04.events.mdx @@ -0,0 +1,131 @@ +--- +title: Events +description: Listen to job lifecycle and queue control events for monitoring and debugging. +--- + +The event system provides real-time notifications for job state changes and queue operations. Both `Queue` (producer) and `Worker` (consumer) emit typed events you can subscribe to with `.on()`, `.once()`, and `.off()`. + +## Usage + +### Queue Events + +The `Queue` instance emits events when jobs are added or their state changes through the public API (e.g. cancellation). + +```typescript +import { Queue, MemoryQueueAdapter } from "@vorsteh-queue/core" + +const adapter = new MemoryQueueAdapter() +const queue = new Queue(adapter, { name: "my-queue" }) + +queue.on("job:added", (job) => { + console.log(`[${job.name}] added with priority ${job.priority}`) +}) + +queue.on("job:cancelled", (job) => { + console.log(`[${job.id}] cancelled: ${job.cancellationReason}`) +}) +``` + +### Worker Events + +The `Worker` instance emits events during job processing — lifecycle transitions, errors, and batch operations. + +```typescript +import { Worker } from "@vorsteh-queue/core" + +const worker = new Worker(adapter, { name: "my-queue", concurrency: 5 }) + +worker.on("job:completed", (job) => { + console.log(`[${job.name}] completed in queue`) +}) + +worker.on("job:failed", (job) => { + console.error(`[${job.name}] failed:`, job.error.message) +}) + +worker.on("job:dead", (job) => { + // Job exhausted all retry attempts + console.error(`[${job.name}] moved to DLQ after ${job.attempts} attempts`) +}) + +worker.on("worker:error", (error) => { + // Unexpected error in polling loop + console.error("Unexpected worker error:", error) +}) +``` + +### One-Time Listeners + +Use `.once()` to listen for a single occurrence: + +```typescript +worker.once("worker:started", () => { + console.log("Worker is ready") +}) +``` + +### Removing Listeners + +```typescript +const handler = (job) => console.log(job.id) +worker.on("job:completed", handler) +worker.off("job:completed", handler) +``` + +## Available Events + +### Queue Events + + + +### Worker Events + + + +### Flow Events + +The Worker emits flow lifecycle events when jobs participate in a flow: + +```typescript +worker.on("flow:completed", ({ flowId, result }) => { + console.log(`Flow ${flowId} completed with:`, result) +}) + +worker.on("flow:failed", ({ flowId, error }) => { + console.error(`Flow ${flowId} failed:`, error.message) +}) + +worker.on("flow:node:promoted", ({ flowId, nodeId, jobId }) => { + console.log(`Flow ${flowId}: node ${nodeId} promoted, job ${jobId} created`) +}) +``` + +The `FlowProducer` also emits events: + +```typescript +flowProducer.on("flow:created", ({ flowId, rootNode }) => { + console.log(`New flow ${flowId} created: ${rootNode.name}`) +}) +``` + +## Use Cases + +- Logging and audit trails +- Custom metrics collection +- Chaining dependent operations +- Notifications on failures +- Real-time dashboards (via the server subscription bridge) diff --git a/apps/docs/content/vorsteh-queue/02.core/05.progress-tracking.mdx b/apps/docs/content/vorsteh-queue/02.core/05.progress-tracking.mdx new file mode 100644 index 00000000..54380193 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/05.progress-tracking.mdx @@ -0,0 +1,47 @@ +--- +title: Progress Tracking +description: Report and monitor real-time job progress within handlers. +--- + +Jobs can report progress (0–100) during execution, useful for long-running operations like batch processing or file uploads. + +## Reporting Progress + +Use `job.updateProgress()` inside your handler: + +```typescript +interface BatchPayload { + items: string[] +} + +interface BatchResult { + processed: number +} + +queue.register("process-batch", async (job) => { + const { items } = job.payload + + for (let i = 0; i < items.length; i++) { + await processItem(items[i]) + await job.updateProgress(Math.round(((i + 1) / items.length) * 100)) + } + + return { processed: items.length } +}) +``` + +## Listening to Progress + +Subscribe to progress updates via the event system: + +```typescript +queue.on("job:progress", (job) => { + console.log(`[${job.name}] ${job.progress}%`) +}) +``` + +## Notes + +- Progress is stored as an integer between 0 and 100 +- Updates are persisted to the database, surviving restarts +- Calling `updateProgress` with the same value is a no-op (no extra writes) diff --git a/apps/docs/content/vorsteh-queue/02.core/05.worker.mdx b/apps/docs/content/vorsteh-queue/02.core/05.worker.mdx new file mode 100644 index 00000000..6e0916d2 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/05.worker.mdx @@ -0,0 +1,170 @@ +--- +title: Worker +description: Process jobs with handlers, concurrency, retries, timeouts, and batch processing. +--- + +The `Worker` class is the consumer-side component. It polls for jobs, executes handlers, manages retries, timeouts, and emits lifecycle events. A Worker connects to the same adapter as the Queue and processes jobs from the same named queue. + +## Creating a Worker + +```typescript +import { Worker, MemoryQueueAdapter } from "@vorsteh-queue/core" + +const adapter = new MemoryQueueAdapter() +const worker = new Worker(adapter, { + name: "email-queue", // must match the Queue name + concurrency: 5, // process up to 5 jobs simultaneously + pollInterval: 100, // poll every 100ms (default) +}) +``` + +In production, the adapter is shared with the Queue: + +```typescript +import { drizzle } from "drizzle-orm/node-postgres" +import { Pool } from "pg" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { Queue, Worker } from "@vorsteh-queue/core" + +const pool = new Pool({ connectionString: process.env.DATABASE_URL }) +const db = drizzle(pool) +const adapter = new PostgresQueueAdapter(db) + +const queue = new Queue(adapter, { name: "email-queue" }) +const worker = new Worker(adapter, { name: "email-queue", concurrency: 5 }) +``` + +## Registering Handlers + +Each job type needs a handler registered on the Worker: + +```typescript +interface EmailPayload { + to: string + subject: string + body: string +} + +interface EmailResult { + messageId: string +} + +worker.register( + "send-email", + async (job, { signal }) => { + const id = await sendEmail(job.payload, { signal }) + return { messageId: id } + } +) +``` + +The handler receives: + +- `job` — the full Job object with payload, metadata, and a `updateProgress()` method +- `ctx` — context with `signal` (AbortSignal for timeout/cancellation), `step` (multi-step API), and flow context methods (`getChildrenValues`, `getFailedChildrenValues`, `getChildrenValuesBy`, `removeUnprocessedChildren`) for flow parents + +## Starting and Stopping + +```typescript +worker.start() // Begin polling and processing +worker.pause() // Stop picking new jobs (in-flight jobs continue) +worker.resume() // Resume polling +await worker.stop() // Graceful shutdown (waits for active jobs) +``` + +## Batch Processing + +Process multiple jobs of the same type in a single handler call: + +```typescript +worker.registerBatch( + "index-documents", + async (jobs, { signal }) => { + const results = await bulkIndex( + jobs.map((j) => j.payload), + { signal } + ) + return results // must return one result per job + }, + { + minSize: 5, // wait for at least 5 jobs before processing + maxSize: 50, // process up to 50 at once + } +) +``` + +## Per-Handler Options + +```typescript +worker.register("send-email", handler, { + concurrency: 3, // max 3 concurrent jobs for this handler + rateLimit: { + max: 100, // max 100 jobs + duration: 60_000, // per 60 seconds + }, +}) +``` + +## Event Triggers + +Automatically create follow-up jobs when a job completes: + +```typescript +worker.trigger({ + on: "process-image", // when this job completes... + create: "send-notification", // ...create this job + data: (result, job) => ({ + // ...with this payload + userId: job.payload.userId, + imageUrl: result.url, + }), + condition: (result) => result.success, // only when condition is met +}) +``` + +## Worker Properties + +```typescript +worker.isRunning // boolean — is the worker polling? +worker.activeJobCount // number — currently processing jobs +worker.name // string — the queue name +``` + +## Cancellation + +Cancel a specific job being processed by this worker: + +```typescript +await worker.cancelJob(jobId, "User requested cancellation") +``` + +## Worker Configuration + +| Option | Type | Default | Description | +| --- | --- | --- | --- | +| `name` | `string` | required | Queue name to consume from (must match Queue) | +| `concurrency` | `number` | `1` | Max concurrent jobs | +| `pollInterval` | `number` | `100` | Polling interval in ms | +| `telemetry` | `Telemetry` | `noopTelemetry` | Optional telemetry instance | +| `retryStrategy` | `RetryStrategyConfig` | exponential backoff | Retry delay strategy | +| `removeOnComplete` | `boolean \| number` | `100` | Auto-remove completed jobs | +| `removeOnFail` | `boolean \| number` | `50` | Auto-remove failed jobs | + +## Retry Strategy + +```typescript +const worker = new Worker(adapter, { + name: "email-queue", + retryStrategy: { + type: "exponential", // "exponential" | "linear" | "fixed" + delay: 1000, // base delay in ms + maxDelay: 30000, // cap for exponential/linear + }, +}) + +// Or a custom function: +const worker = new Worker(adapter, { + name: "email-queue", + retryStrategy: (attempt) => Math.min(1000 * 2 ** attempt, 60000), +}) +``` diff --git a/apps/docs/content/vorsteh-queue/02.core/06.error-handling.mdx b/apps/docs/content/vorsteh-queue/02.core/06.error-handling.mdx new file mode 100644 index 00000000..b205c572 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/06.error-handling.mdx @@ -0,0 +1,63 @@ +--- +title: Error Handling +description: Retry logic, timeouts, dead letter queue, and job failure management. +--- + +Vorsteh Queue provides multiple layers of error handling to ensure reliable job processing. + +## Retry Logic + +Jobs are retried automatically when they fail, up to the configured maximum: + +```typescript +await queue.add("risky-job", payload, { + maxAttempts: 5, // Retry up to 5 times (default: 3) +}) +``` + +After all attempts are exhausted, the job moves to the **dead letter queue** (status: `dead`). + +## Job Timeouts + +Prevent stuck jobs with a timeout: + +```typescript +await queue.add("long-task", payload, { + timeout: 30000, // 30 seconds +}) +``` + +If a job exceeds the timeout, it is marked as failed and retried according to `maxAttempts`. + +## Dead Letter Queue + +Jobs that exceed their retry limit land in the DLQ with status `dead`. You can inspect and redrive them: + +```typescript +// Get dead jobs +const deadJobs = await adapter.getDeadJobs({ limit: 50 }) + +// Redrive a single job (resets to pending) +await adapter.redriveJob(jobId) + +// Redrive all dead jobs +await adapter.redriveJobs() +``` + +## Error Events + +```typescript +queue.on("job:failed", (job) => { + console.error( + `Job ${job.name} failed on attempt ${job.attempts}/${job.maxAttempts}` + ) + console.error(job.error?.message) +}) +``` + +## Best Practices + +- Set `maxAttempts` based on whether the operation is idempotent +- Use timeouts for external API calls and network operations +- Monitor the DLQ and set up alerts for accumulated dead jobs +- Use the API or CLI to inspect and redrive dead jobs diff --git a/apps/docs/content/vorsteh-queue/02.core/07.observability.mdx b/apps/docs/content/vorsteh-queue/02.core/07.observability.mdx new file mode 100644 index 00000000..f561a901 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/07.observability.mdx @@ -0,0 +1,185 @@ +--- +title: Observability +description: Optional OpenTelemetry metrics and distributed tracing for monitoring your queue in production. +--- + +vorsteh-queue supports OpenTelemetry instrumentation via an opt-in telemetry instance. By default, no OTel code is loaded — the core runs with zero observability overhead. To enable metrics and tracing, create a telemetry instance and pass it to your Queue and Worker. + +## Setup + +Install the OpenTelemetry packages: + + + {`@opentelemetry/api @opentelemetry/sdk-node @opentelemetry/exporter-metrics-otlp-http @opentelemetry/exporter-trace-otlp-http`} + + +Configure your application: + +```typescript +import { NodeSDK } from "@opentelemetry/sdk-node" +import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http" +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http" +import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics" + +// 1. Start the OTel SDK +const sdk = new NodeSDK({ + traceExporter: new OTLPTraceExporter(), + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter(), + }), +}) +sdk.start() + +// 2. Create vorsteh-queue telemetry instance +import { Queue, Worker } from "@vorsteh-queue/core" +import { createOtelTelemetry } from "@vorsteh-queue/core" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" + +const telemetry = createOtelTelemetry({ queueName: "email-queue" }) + +const adapter = new PostgresQueueAdapter(db) +const queue = new Queue(adapter, { name: "email-queue", telemetry }) +const worker = new Worker(adapter, { name: "email-queue", telemetry }) +``` + +The `telemetry` parameter is optional on both Queue and Worker. When omitted, a no-op implementation is used with zero overhead. + +The same telemetry instance can be passed to `FlowProducer` for flow-level observability: + +```typescript +const flowProducer = new FlowProducer(adapter, { telemetry }) +``` + +## How It Works + +- `createOtelTelemetry(config)` — creates a telemetry instance backed by the global OTel providers. Requires `@opentelemetry/api` to be installed. +- `noopTelemetry` — the default. All methods are empty stubs. Used when no telemetry is passed. + +The core package never imports `@opentelemetry/api` at module load time. The import happens only inside `createOtelTelemetry` via a deferred require, so your bundle and startup are unaffected if you don't use OTel. + +## Metrics + +All metrics use the `vorsteh_queue` namespace and include queue name and job name attributes. + +### Counters + +| Metric | Description | +| ------------------------------ | ------------------------------------- | +| `vorsteh_queue.jobs.added` | Total jobs added to the queue | +| `vorsteh_queue.jobs.processed` | Total jobs successfully completed | +| `vorsteh_queue.jobs.failed` | Total job failures (includes retries) | +| `vorsteh_queue.jobs.retried` | Total retry attempts | +| `vorsteh_queue.jobs.dead` | Jobs moved to the dead letter queue | +| `vorsteh_queue.jobs.cancelled` | Jobs cancelled | + +### Gauges + +| Metric | Description | +| --------------------------- | ------------------------------ | +| `vorsteh_queue.jobs.active` | Jobs currently being processed | + +### Histograms + +| Metric | Unit | Description | +| --- | --- | --- | +| `vorsteh_queue.jobs.duration` | ms | Time spent processing a job | +| `vorsteh_queue.jobs.wait_time` | ms | Time from enqueue to processing start | + +### Flow Counters + +| Metric | Description | +| --- | --- | +| `vorsteh_queue.flows.created` | Total flows created | +| `vorsteh_queue.flows.completed` | Total flows completed successfully | +| `vorsteh_queue.flows.failed` | Total flows that failed | +| `vorsteh_queue.flows.nodes_promoted` | Total flow nodes promoted (parent job created) | + +### Flow Histograms + +| Metric | Unit | Description | +| --- | --- | --- | +| `vorsteh_queue.flows.duration` | ms | End-to-end flow duration (creation to root completion) | + +## Distributed Tracing + +Every job execution creates a span with these attributes: + +| Attribute | Description | +| -------------------------------- | --------------------------- | +| `messaging.system` | Always `vorsteh-queue` | +| `messaging.operation` | Always `process` | +| `messaging.destination.name` | Queue name | +| `vorsteh_queue.job.id` | Job ID | +| `vorsteh_queue.job.name` | Handler name | +| `vorsteh_queue.job.priority` | Job priority | +| `vorsteh_queue.job.attempt` | Current attempt number | +| `vorsteh_queue.job.max_attempts` | Maximum attempts configured | + +Spans follow the [OTel Messaging Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/messaging/). + +### Success and Failure + +- Successful jobs end with `SpanStatusCode.OK` +- Failed jobs end with `SpanStatusCode.ERROR`, include the error message, and record an exception event + +## Examples + +### Prometheus Export + +```typescript +import { NodeSDK } from "@opentelemetry/sdk-node" +import { PrometheusExporter } from "@opentelemetry/exporter-prometheus" + +const sdk = new NodeSDK({ + metricReader: new PrometheusExporter({ port: 9464 }), +}) + +sdk.start() +// Metrics available at http://localhost:9464/metrics +``` + +### Grafana / OTLP + +```typescript +import { NodeSDK } from "@opentelemetry/sdk-node" +import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http" +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http" +import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics" + +const sdk = new NodeSDK({ + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4318/v1/traces", + }), + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: "http://localhost:4318/v1/metrics", + }), + }), +}) + +sdk.start() +``` + +### Custom Alerting with Metrics + +Combine OTel metrics with your alerting system to get notified on anomalies: + +```typescript +// Set up alerts in your monitoring platform: +// +// - Alert when vorsteh_queue.jobs.dead > 0 (DLQ is filling up) +// - Alert when vorsteh_queue.jobs.duration p99 > 30s (jobs are slow) +// - Alert when vorsteh_queue.jobs.active stays at max concurrency (backpressure) +``` + +## Without OTel + +If you don't pass a `telemetry` instance, the Queue and Worker use `noopTelemetry` — all operations are no-ops with zero performance penalty. You don't need `@opentelemetry/api` installed at all. + +```typescript +import { Queue, Worker } from "@vorsteh-queue/core" + +// No telemetry parameter → noopTelemetry is used automatically +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) +``` diff --git a/apps/docs/content/vorsteh-queue/02.core/index.mdx b/apps/docs/content/vorsteh-queue/02.core/index.mdx new file mode 100644 index 00000000..46090fda --- /dev/null +++ b/apps/docs/content/vorsteh-queue/02.core/index.mdx @@ -0,0 +1,4 @@ +--- +title: Core +description: The main queue engine containing all core functionality for job processing, scheduling, and management. +--- diff --git a/apps/docs/content/vorsteh-queue/03.adapters/01.drizzle.mdx b/apps/docs/content/vorsteh-queue/03.adapters/01.drizzle.mdx new file mode 100644 index 00000000..4b1f3f49 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/01.drizzle.mdx @@ -0,0 +1,109 @@ +--- +title: Drizzle Adapter +navTitle: Drizzle +description: Drizzle ORM adapter for PostgreSQL with type-safe database operations. +--- + +The Drizzle adapter provides PostgreSQL support using Drizzle ORM, offering excellent TypeScript integration and performance. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-drizzle + + +## Quick Start + +```typescript +import { defineRelations } from "drizzle-orm" +import { drizzle } from "drizzle-orm/node-postgres" +import { Pool } from "pg" + +import { + PostgresQueueAdapter, + queueFlows, + queueJobs, +} from "@vorsteh-queue/adapter-drizzle" +import { Queue } from "@vorsteh-queue/core" + +// 1. Define relations (include your own tables here too) +const relations = defineRelations({ queueFlows, queueJobs }) + +// 2. Create drizzle client with relations +const pool = new Pool({ connectionString: "postgresql://..." }) +const db = drizzle({ client: pool, relations }) + +// 3. Create adapter and queue +const adapter = new PostgresQueueAdapter(db) +const queue = new Queue(adapter, { name: "my-queue" }) +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Supported Drivers + +- **node-postgres (pg)** — `drizzle-orm/node-postgres` ( [Docs](https://orm.drizzle.team/docs/get-started-postgresql#node-postgres) ) +- **Postgres.js** — `drizzle-orm/postgres-js` ( [Docs](https://orm.drizzle.team/docs/get-started-postgresql#postgresjs) ) +- **PGlite** — `drizzle-orm/pglite` (embedded, great for development) ( [Docs](https://orm.drizzle.team/docs/connect-pglite) ) + +## Database Schema + +The adapter ships with a pre-defined schema: + +```typescript +import { defineRelations } from "drizzle-orm" + +import { queueFlows, queueJobs } from "@vorsteh-queue/adapter-drizzle" + +// Export the queue tables for drizzle-kit migrations +export { queueFlows, queueJobs } + +// Define relations — add your own tables here alongside queue tables +export const relations = defineRelations({ queueFlows, queueJobs }) +``` + +## Custom Table and Schema Names + +Use `createQueueJobsTable` to customize the table name and database schema: + +```typescript +import { defineRelations } from "drizzle-orm" + +import { + createQueueJobsTable, + queueFlows, +} from "@vorsteh-queue/adapter-drizzle" + +export const { table: customQueueJobs, schema: customSchema } = + createQueueJobsTable("custom_queue_jobs", "custom_schema") + +export const relations = defineRelations({ customQueueJobs, queueFlows }) +``` + +Pass the custom model name to the adapter: + +```typescript +const adapter = new PostgresQueueAdapter(db, { + modelName: "customQueueJobs", +}) +``` + +## Migrations + +Use [drizzle-kit](https://orm.drizzle.team/docs/kit-overview) to generate and run migrations: + +```bash +npx drizzle-kit generate +npx drizzle-kit migrate +``` + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/02.prisma.mdx b/apps/docs/content/vorsteh-queue/03.adapters/02.prisma.mdx new file mode 100644 index 00000000..39dc6dc2 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/02.prisma.mdx @@ -0,0 +1,124 @@ +--- +title: Prisma Adapter +navTitle: Prisma +description: Prisma ORM adapter for PostgreSQL with generated types and migrations. +--- + +The Prisma adapter provides PostgreSQL support using Prisma ORM, offering generated types, migrations, and developer tooling. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-prisma + + +## Quick Start + +```typescript +import { PrismaClient } from "@prisma/client" +import { PostgresPrismaQueueAdapter } from "@vorsteh-queue/adapter-prisma" +import { Queue, Worker } from "@vorsteh-queue/core" + +const prisma = new PrismaClient() +const adapter = new PostgresPrismaQueueAdapter(prisma) + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +await queue.connect() +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +Add these models to your `schema.prisma`: + +```prisma +model QueueJob { + id String @id @default(uuid()) + queueName String @map("queue_name") @db.VarChar(255) + name String @db.VarChar(255) + payload Json @db.JsonB + status String @db.VarChar(50) + priority Int + attempts Int @default(0) + maxAttempts Int @map("max_attempts") + timeout Int? + progress Int @default(0) + groupKey String? @map("group_key") @db.VarChar(255) + uniqueKey String? @map("unique_key") @db.VarChar(255) + cron String? @db.VarChar(255) + repeatEvery Int? @map("repeat_every") + repeatLimit Int? @map("repeat_limit") + repeatCount Int @default(0) @map("repeat_count") + cancellationReason String? @map("cancellation_reason") @db.Text + error Json? @db.JsonB + result Json? @db.JsonB + steps Json? @db.JsonB + signals Json? @db.JsonB + flowNodeId String? @map("flow_node_id") + createdAt DateTime @default(dbgenerated("timezone('utc', now())")) @map("created_at") @db.Timestamptz(6) + processAt DateTime @map("process_at") @db.Timestamptz(6) + processedAt DateTime? @map("processed_at") @db.Timestamptz(6) + completedAt DateTime? @map("completed_at") @db.Timestamptz(6) + failedAt DateTime? @map("failed_at") @db.Timestamptz(6) + cancelledAt DateTime? @map("cancelled_at") @db.Timestamptz(6) + + @@index([queueName, status, priority, createdAt], map: "idx_queue_jobs_polling") + @@index([queueName, processAt], map: "idx_queue_jobs_delayed") + @@index([queueName, groupKey], map: "idx_queue_jobs_active_groups") + @@index([queueName, status], map: "idx_queue_jobs_stats") + @@map("queue_jobs") +} + +model QueueFlow { + id String @id @default(uuid()) + flowId String @map("flow_id") + parentNodeId String? @map("parent_node_id") + jobId String? @map("job_id") + queueName String @map("queue_name") @db.VarChar(255) + name String @db.VarChar(255) + payload Json @db.JsonB + options Json? @db.JsonB + status String @db.VarChar(50) + failureStrategy String @default("default") @map("failure_strategy") @db.VarChar(20) + childrenCount Int @default(0) @map("children_count") + childrenCompleted Int @default(0) @map("children_completed") + result Json? @db.JsonB + error Json? @db.JsonB + createdAt DateTime @default(dbgenerated("timezone('utc', now())")) @map("created_at") @db.Timestamptz(6) + completedAt DateTime? @map("completed_at") @db.Timestamptz(6) + + @@index([flowId], map: "idx_queue_flows_flow_id") + @@index([parentNodeId], map: "idx_queue_flows_parent_node_id") + @@index([jobId], map: "idx_queue_flows_job_id") + @@index([flowId, status], map: "idx_queue_flows_status") + @@map("queue_flows") +} +``` + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresPrismaQueueAdapter(prisma, { + modelName: "CustomQueueJob", // model name in schema.prisma + tableName: "custom_queue_jobs", // @@map value + schemaName: "custom_schema", // PostgreSQL schema +}) +``` + +## Migrations + +Use the [Prisma CLI](https://www.prisma.io/docs/orm/tools/prisma-cli) for migrations: + +```bash +npx prisma migrate dev --name add-queue-table +``` + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/03.kysely.mdx b/apps/docs/content/vorsteh-queue/03.adapters/03.kysely.mdx new file mode 100644 index 00000000..82289778 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/03.kysely.mdx @@ -0,0 +1,71 @@ +--- +title: Kysely Adapter +navTitle: Kysely +description: Kysely adapter for PostgreSQL with migration support. +--- + +The Kysely adapter provides PostgreSQL support using Kysely, a type-safe SQL query builder. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-kysely + + +## Quick Start + +```typescript +import { Kysely, PostgresDialect } from "kysely" +import { Pool } from "pg" + +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-kysely/postgres-adapter" +import { Queue } from "@vorsteh-queue/core" + +const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: "postgresql://..." }), + }), +}) + +const adapter = new PostgresQueueAdapter(db) +const queue = new Queue(adapter, { name: "my-queue" }) +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +The adapter provides a migration for creating the queue table: + +```typescript +import { createQueueTable } from "@vorsteh-queue/adapter-kysely/migrations" +``` + +## Migrations + +Use [kysely-ctl](https://github.com/kysely-org/kysely-ctl) or the provided migration function: + +```typescript +import { Kysely } from "kysely" +import { createQueueTable } from "@vorsteh-queue/adapter-kysely/migrations" + +export async function up(db: Kysely): Promise { + await createQueueTable(db) +} +``` + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresQueueAdapter(db, { + tableName: "custom_queue_jobs", + schemaName: "custom_schema", +}) +``` + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/04.zenstack.mdx b/apps/docs/content/vorsteh-queue/03.adapters/04.zenstack.mdx new file mode 100644 index 00000000..0a09d24d --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/04.zenstack.mdx @@ -0,0 +1,177 @@ +--- +title: ZenStack Adapter +navTitle: ZenStack +description: ZenStack ORM adapter for PostgreSQL with built-in access control and Prisma-compatible API. +--- + +The ZenStack adapter provides PostgreSQL support using ZenStack ORM v3, offering a Prisma-compatible query API built on Kysely with additional features like built-in access control, polymorphic models, and a plugin system. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-zenstack @zenstackhq/orm + + +You also need a PostgreSQL database driver: + +pg + +And the ZenStack CLI as a dev dependency: + + + @zenstackhq/cli + + +## Quick Start + +```typescript +import { ZenStackClient } from "@zenstackhq/orm" +import { PostgresDialect } from "@zenstackhq/orm/dialects/postgres" +import { PostgresZenstackQueueAdapter } from "@vorsteh-queue/adapter-zenstack" +import { Queue, Worker } from "@vorsteh-queue/core" +import { Pool } from "pg" + +import { schema } from "./zenstack/schema" + +const db = new ZenStackClient(schema, { + dialect: new PostgresDialect({ + pool: new Pool({ connectionString: process.env.DATABASE_URL }), + }), +}) + +const adapter = new PostgresZenstackQueueAdapter(db) + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +await queue.connect() +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +Add these models to your `zenstack/schema.zmodel`: + +```prisma +datasource db { + provider = "postgresql" +} + +model QueueJob { + id String @id @default(uuid()) + queueName String @map("queue_name") @db.VarChar(255) + name String @db.VarChar(255) + payload Json @db.JsonB + status String @db.VarChar(50) + priority Int + attempts Int @default(0) + maxAttempts Int @map("max_attempts") + timeout Int? + progress Int @default(0) + groupKey String? @map("group_key") @db.VarChar(255) + uniqueKey String? @map("unique_key") @db.VarChar(255) + cron String? @db.VarChar(255) + repeatEvery Int? @map("repeat_every") + repeatLimit Int? @map("repeat_limit") + repeatCount Int @default(0) @map("repeat_count") + cancellationReason String? @map("cancellation_reason") @db.Text + error Json? @db.JsonB + result Json? @db.JsonB + steps Json? @db.JsonB + signals Json? @db.JsonB + flowNodeId String? @map("flow_node_id") + createdAt DateTime @default(dbgenerated("timezone('utc', now())")) @map("created_at") @db.Timestamptz(6) + processAt DateTime @map("process_at") @db.Timestamptz(6) + processedAt DateTime? @map("processed_at") @db.Timestamptz(6) + completedAt DateTime? @map("completed_at") @db.Timestamptz(6) + failedAt DateTime? @map("failed_at") @db.Timestamptz(6) + cancelledAt DateTime? @map("cancelled_at") @db.Timestamptz(6) + + @@index([queueName, status, priority, createdAt], map: "idx_queue_jobs_polling") + @@index([queueName, processAt], map: "idx_queue_jobs_delayed") + @@index([queueName, groupKey], map: "idx_queue_jobs_active_groups") + @@index([queueName, status], map: "idx_queue_jobs_stats") + @@map("queue_jobs") +} + +model QueueFlow { + id String @id @default(uuid()) + flowId String @map("flow_id") + parentNodeId String? @map("parent_node_id") + jobId String? @map("job_id") + queueName String @map("queue_name") @db.VarChar(255) + name String @db.VarChar(255) + payload Json @db.JsonB + options Json? @db.JsonB + status String @db.VarChar(50) + failureStrategy String @default("default") @map("failure_strategy") @db.VarChar(20) + childrenCount Int @default(0) @map("children_count") + childrenCompleted Int @default(0) @map("children_completed") + result Json? @db.JsonB + error Json? @db.JsonB + createdAt DateTime @default(dbgenerated("timezone('utc', now())")) @map("created_at") @db.Timestamptz(6) + completedAt DateTime? @map("completed_at") @db.Timestamptz(6) + + @@index([flowId], map: "idx_queue_flows_flow_id") + @@index([parentNodeId], map: "idx_queue_flows_parent_node_id") + @@index([jobId], map: "idx_queue_flows_job_id") + @@index([flowId, status], map: "idx_queue_flows_status") + @@map("queue_flows") +} +``` + +Then generate the TypeScript schema and push the database: + +```bash +npx zen generate +npx zen db push +``` + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresZenstackQueueAdapter(db, { + modelName: "customQueueJob", // model accessor name (camelCase) + tableName: "custom_queue_jobs", // @@map value + schemaName: "custom_schema", // PostgreSQL schema +}) +``` + +## Migrations + +Use the [ZenStack CLI](https://zenstack.dev/docs/orm/cli) for migrations: + +```bash +npx zen migrate dev --name add-queue-table +npx zen migrate deploy +``` + +## Why ZenStack? + +ZenStack v3 is a standalone ORM that provides: + +- **Intuitive schema language** — model data, relations, access control, and more in one place +- **Powerful ORM** — awesomely-typed API, built-in access control, and unmatched flexibility +- **Query-as-a-Service** — provides a full-fledged data API without the need to code it up +- **Prisma-compatible query API** — familiar `findMany`, `create`, `update` syntax +- **Built-in access control** — define policies directly in the schema with `@@allow` / `@@deny` +- **Low-level Kysely query builder** — escape hatch for complex queries via `db.$qb` +- **Polymorphic models** — model inheritance with `extends` and `@@delegate` +- **No Prisma runtime dependency** — lighter footprint, no Rust/WASM engines + + + ZenStack v3 does not depend on Prisma at runtime. The schema language is a + superset of Prisma Schema Language, making migration straightforward. + + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/05.typeorm.mdx b/apps/docs/content/vorsteh-queue/03.adapters/05.typeorm.mdx new file mode 100644 index 00000000..705f7401 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/05.typeorm.mdx @@ -0,0 +1,104 @@ +--- +title: TypeORM Adapter +navTitle: TypeORM +description: TypeORM adapter for PostgreSQL with entity-based schema and decorator-driven models. +--- + +The TypeORM adapter provides PostgreSQL support using TypeORM, offering a mature ORM with decorator-based entity definitions, migrations, and broad database support. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-typeorm typeorm pg + + +## Quick Start + +```typescript +import { DataSource } from "typeorm" +import { + PostgresTypeormQueueAdapter, + QueueFlowEntity, + QueueJobEntity, +} from "@vorsteh-queue/adapter-typeorm" +import { Queue, Worker } from "@vorsteh-queue/core" + +const dataSource = new DataSource({ + type: "postgres", + url: process.env.DATABASE_URL, + entities: [QueueJobEntity, QueueFlowEntity], + synchronize: true, +}) + +const adapter = new PostgresTypeormQueueAdapter(dataSource) + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +await queue.connect() +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +The adapter ships with pre-built `QueueJobEntity` and `QueueFlowEntity` that you can use directly with TypeORM's `synchronize: true` or migrations. The entities map to the `queue_jobs` and `queue_flows` tables respectively. + +If you prefer manual control, you can import the entities and use TypeORM migrations: + +```typescript +import { + QueueFlowEntity, + QueueJobEntity, +} from "@vorsteh-queue/adapter-typeorm/entity" +``` + +The entity defines all required columns with proper types, indexes, and column mappings. + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresTypeormQueueAdapter(dataSource, { + tableName: "custom_queue_jobs", // database table name + schemaName: "custom_schema", // PostgreSQL schema +}) +``` + + + When using custom table names, you need to create your own entity class with + the matching `@Entity("custom_queue_jobs")` decorator, or use raw SQL + migrations. + + +## Migrations + +Use the [TypeORM CLI](https://typeorm.io/docs/migrations/) for migrations: + +```bash +npx typeorm migration:generate -d src/data-source.ts src/migrations/AddQueueTable +npx typeorm migration:run -d src/data-source.ts +``` + +Alternatively, set `synchronize: true` in development for automatic schema creation. + +## Why TypeORM? + +TypeORM is one of the most established ORMs in the Node.js ecosystem: + +- **Decorator-based entities** — define schemas with familiar TypeScript decorators +- **Active Record and Data Mapper** — choose the pattern that fits your architecture +- **Broad database support** — PostgreSQL, MySQL, SQLite, MSSQL, Oracle, and more +- **Mature migration system** — automatic generation and manual migration support +- **QueryBuilder** — powerful and flexible SQL query construction +- **Large ecosystem** — extensive community and plugin support + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/06.mikroorm.mdx b/apps/docs/content/vorsteh-queue/03.adapters/06.mikroorm.mdx new file mode 100644 index 00000000..5731d5a8 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/06.mikroorm.mdx @@ -0,0 +1,88 @@ +--- +title: MikroORM Adapter +navTitle: MikroORM +description: MikroORM adapter for PostgreSQL with schema-first entity definitions and identity map. +--- + +The MikroORM adapter provides PostgreSQL support using MikroORM v6, offering a powerful ORM with unit of work pattern, identity map, and schema-first entity definitions. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-mikroorm @mikro-orm/core + @mikro-orm/postgresql + + +## Quick Start + +```typescript +import { MikroORM } from "@mikro-orm/postgresql" +import { + PostgresMikroormQueueAdapter, + QueueFlowSchema, + QueueJobSchema, +} from "@vorsteh-queue/adapter-mikroorm" +import { Queue, Worker } from "@vorsteh-queue/core" + +const orm = await MikroORM.init({ + entities: [QueueJobSchema, QueueFlowSchema], + clientUrl: process.env.DATABASE_URL, +}) + +const adapter = new PostgresMikroormQueueAdapter(orm) + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +await queue.connect() +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +The adapter ships with pre-built `QueueJobSchema` and `QueueFlowSchema` (EntitySchema) that you can use directly with MikroORM's schema generator or migrations. + +```bash +npx mikro-orm schema:create +``` + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresMikroormQueueAdapter(orm, { + tableName: "custom_queue_jobs", + schemaName: "custom_schema", +}) +``` + +## Migrations + +Use the [MikroORM CLI](https://mikro-orm.io/docs/migrations) for migrations: + +```bash +npx mikro-orm migration:create +npx mikro-orm migration:up +``` + +## Why MikroORM? + +MikroORM v6 is a modern TypeScript ORM with: + +- **Unit of Work pattern** — automatic change tracking and optimized flushes +- **Identity Map** — ensures entity identity across queries +- **Schema-first entities** — define schemas without decorators using `EntitySchema` +- **Powerful QueryBuilder** — Knex-powered query construction +- **Database-agnostic** — supports PostgreSQL, MySQL, SQLite, MongoDB, and more +- **Strict typing** — full TypeScript support with generic entities + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/07.sequelize.mdx b/apps/docs/content/vorsteh-queue/03.adapters/07.sequelize.mdx new file mode 100644 index 00000000..81b45594 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/07.sequelize.mdx @@ -0,0 +1,92 @@ +--- +title: Sequelize Adapter +navTitle: Sequelize +description: Sequelize adapter for PostgreSQL with model-based schema and battle-tested ORM. +--- + +The Sequelize adapter provides PostgreSQL support using Sequelize, one of the most established Node.js ORMs with model-based schema definitions, migrations, and broad database support. + +## Installation + + + @vorsteh-queue/core @vorsteh-queue/adapter-sequelize sequelize pg + + +## Quick Start + +```typescript +import { Sequelize } from "sequelize" +import { + PostgresSequelizeQueueAdapter, + initQueueFlowModel, + initQueueJobModel, +} from "@vorsteh-queue/adapter-sequelize" +import { Queue, Worker } from "@vorsteh-queue/core" + +const sequelize = new Sequelize(process.env.DATABASE_URL, { + dialect: "postgres", + logging: false, +}) + +initQueueJobModel(sequelize) +initQueueFlowModel(sequelize) + +const adapter = new PostgresSequelizeQueueAdapter(sequelize) + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +await queue.connect() +``` + + + For real projects, extract this setup into a shared `src/queues.ts` file. See + [Project Structure](/docs/vorsteh-queue/getting-started/project-structure). + + +## Database Schema + +The adapter ships with a pre-built model that can be synced with: + +```typescript +await sequelize.sync() +``` + +Or use Sequelize migrations for production. + +## Custom Table and Schema Names + +```typescript +const adapter = new PostgresSequelizeQueueAdapter(sequelize, { + tableName: "custom_queue_jobs", + schemaName: "custom_schema", +}) +``` + +## Migrations + +Use the [Sequelize CLI](https://sequelize.org/docs/v6/other-topics/migrations/) for migrations: + +```bash +npx sequelize-cli migration:generate --name add-queue-table +npx sequelize-cli db:migrate +``` + +## Why Sequelize? + +Sequelize is one of the most battle-tested ORMs for Node.js: + +- **Broad database support** — PostgreSQL, MySQL, MariaDB, SQLite, MSSQL +- **Model-based schemas** — define schemas with familiar `Model.init()` pattern +- **Mature migration system** — CLI-driven migration workflow +- **Transaction support** — managed and unmanaged transactions +- **Hooks and validators** — lifecycle hooks and built-in validation +- **Large community** — extensive ecosystem and documentation + +## Adapter Properties + + diff --git a/apps/docs/content/vorsteh-queue/03.adapters/index.mdx b/apps/docs/content/vorsteh-queue/03.adapters/index.mdx new file mode 100644 index 00000000..7c39ba0c --- /dev/null +++ b/apps/docs/content/vorsteh-queue/03.adapters/index.mdx @@ -0,0 +1,4 @@ +--- +title: Adapters +description: ORM-specific database adapters for PostgreSQL. +--- diff --git a/apps/docs/content/vorsteh-queue/04.packages/01.create-vorsteh-queue.mdx b/apps/docs/content/vorsteh-queue/04.packages/01.create-vorsteh-queue.mdx new file mode 100644 index 00000000..1d0b92c0 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/04.packages/01.create-vorsteh-queue.mdx @@ -0,0 +1,46 @@ +--- +title: create-vorsteh-queue +description: CLI scaffolding tool for quickly creating new Vorsteh Queue projects from templates. +--- + +The `create-vorsteh-queue` CLI provides the fastest way to get started by creating new projects from pre-built templates. + +## Usage + +### Interactive Mode + +```bash +npx create-vorsteh-queue +``` + +The CLI prompts for project name, template, package manager, and dependency installation. + +### Direct Mode + +```bash +npx create-vorsteh-queue my-app --template=drizzle-postgres --package-manager=pnpm --quiet +``` + +## Options + +| Option | Short | Description | +| ------------------------ | ----------- | ----------------------- | +| `--template=` | `-t=` | Choose template | +| `--package-manager=` | `-p=` | Package manager | +| `--no-install` | n/a | Skip dependency install | +| `--quiet` | `-q` | Minimal output | + +## Available Templates + + + +## Example + +```bash +npx create-vorsteh-queue worker-service \ + --template=drizzle-postgres \ + --package-manager=pnpm +cd worker-service +pnpm db:push +pnpm dev +``` diff --git a/apps/docs/content/vorsteh-queue/04.packages/02.server.mdx b/apps/docs/content/vorsteh-queue/04.packages/02.server.mdx new file mode 100644 index 00000000..835d929e --- /dev/null +++ b/apps/docs/content/vorsteh-queue/04.packages/02.server.mdx @@ -0,0 +1,214 @@ +--- +title: Server API +navTitle: Server +description: GraphQL API server for monitoring and managing queues. +--- + +The `@vorsteh-queue/server` package provides a standalone GraphQL API for managing Vorsteh Queue jobs. It can run as a standalone server or be mounted as middleware in an existing Hono app. + +## Installation + +@vorsteh-queue/server + +## Quick Start + +```typescript +import { Queue, Worker, MemoryQueueAdapter } from "@vorsteh-queue/core" +import { createQueueServer } from "@vorsteh-queue/server" + +const adapter = new MemoryQueueAdapter() +const queue = new Queue(adapter, { name: "email-queue" }) +const worker = new Worker(adapter, { name: "email-queue" }) + +worker.register("send-email", async (job) => { + // process job... + return { sent: true } +}) + +const server = createQueueServer({ + queues: [queue], + workers: [worker], // enables real-time subscription events + port: 3000, + auth: { tokens: [process.env.QUEUE_TOKEN] }, +}) + +worker.start() +await server.start() +``` + +## Features + +- **GraphQL API** — query queue stats, inspect jobs, cancel, retry, redrive +- **Multi-queue support** — manage multiple queues from a single server +- **Authentication** — optional token-based auth with custom middleware support +- **Subscriptions** — real-time job lifecycle events via SSE +- **Hono middleware** — mount on existing apps via `createQueueMiddleware` + +## Configuration + +### Standalone Server + +```typescript +import { Queue, Worker } from "@vorsteh-queue/core" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { createQueueServer } from "@vorsteh-queue/server" + +// 1. Create adapter (shared between queues and workers) +const adapter = new PostgresQueueAdapter(db) + +// 2. Create Queue instances (one per logical queue) +const emailQueue = new Queue(adapter, { name: "email-queue" }) +const reportQueue = new Queue(adapter, { name: "report-queue" }) + +// 3. Create Worker instances (must use same name as their queue) +const emailWorker = new Worker(adapter, { name: "email-queue", concurrency: 5 }) +const reportWorker = new Worker(adapter, { name: "report-queue" }) + +// 4. Register handlers +emailWorker.register("send-email", async (job) => { + await sendEmail(job.payload) + return { sent: true } +}) + +reportWorker.register("generate-report", async (job) => { + return generateReport(job.payload) +}) + +// 5. Create server — pass both queues and workers +const server = createQueueServer({ + queues: [emailQueue, reportQueue], + workers: [emailWorker, reportWorker], + port: 3000, + auth: { tokens: [process.env.QUEUE_TOKEN] }, +}) + +// 6. Start workers and server +emailWorker.start() +reportWorker.start() +await server.start() +``` + +### As Hono Middleware + +```typescript +import { Hono } from "hono" +import { Queue, Worker } from "@vorsteh-queue/core" +import { PostgresQueueAdapter } from "@vorsteh-queue/adapter-drizzle" +import { createQueueMiddleware } from "@vorsteh-queue/server" + +const adapter = new PostgresQueueAdapter(db) +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue" }) + +worker.register("process-task", async (job) => { + // handle job... + return { ok: true } +}) + +const app = new Hono() +app.route( + "/queue", + createQueueMiddleware({ + queues: [queue], + workers: [worker], + auth: { tokens: ["secret"] }, + }) +) + +worker.start() +export default app +``` + +### Via CLI + +Create a `queue.config.ts` in your project root and use the CLI: + +```bash +vorsteh-queue serve +vorsteh-queue serve --port 4000 +``` + +## Server Config + +| Property | Type | Default | Description | +| --- | --- | --- | --- | +| `queues` | `readonly Queue[]` | required | Queue instances to manage | +| `workers?` | `readonly Worker[]` | `[]` | Workers to observe for subscription events | +| `auth?` | `AuthConfig` | `false` | Authentication configuration | +| `port?` | `number` | `3000` | Server port (standalone mode) | + +### Authentication + +```typescript +// Token-based +auth: { + tokens: ["my-secret-token"] +} + +// Custom middleware +auth: { + middleware: myAuthMiddleware +} + +// Disabled +auth: false +``` + +## GraphQL API + +The server exposes a GraphQL endpoint at `/graphql`. + +### Queries + +| Query | Description | +| --- | --- | +| `queues` | List all configured queues | +| `stats(queue)` | Queue statistics by status | +| `size(queue)` | Count of pending + delayed jobs | +| `job(id, queue?)` | Get a single job by ID | +| `jobs(queue, status?, name?, limit?, offset?)` | Paginated job list | +| `deadJobs(queue, limit?, offset?)` | Dead letter queue jobs | +| `flows(queue, limit?, offset?)` | Flow trees | +| `flowTree(queue, flowId)` | Full flow tree structure | + +### Mutations + +| Mutation | Description | +| ------------------------------- | --------------------- | +| `cancelJob(id, queue, reason?)` | Cancel a job | +| `retryJob(id, queue)` | Retry a failed job | +| `runJobNow(id, queue)` | Promote a delayed job | +| `deleteJob(id, queue)` | Delete a job | +| `redriveAll(queue, name?)` | Redrive dead jobs | +| `clearJobs(queue, status?)` | Clear jobs by status | + +### Subscriptions + +| Subscription | Payload | Description | +| --- | --- | --- | +| `jobStatusChanged` | `JobLifecycleEvent` | Fires on any job status transition | +| `statsUpdated` | `QueueStats` | Fires when queue stats change | + +The `JobLifecycleEvent` envelope contains: + +```graphql +type JobLifecycleEvent { + jobId: ID! + jobName: String! + queueName: String! + previousStatus: JobStatus + currentStatus: JobStatus! + timestamp: String! + progress: Int +} +``` + +Subscriptions are delivered via Server-Sent Events (SSE). They require `workers` to be passed in the server config for processing lifecycle events. + +## Health Check + +The server exposes a `/health` endpoint (behind auth) that returns: + +```json +{ "status": "ok" } +``` diff --git a/apps/docs/content/vorsteh-queue/04.packages/index.mdx b/apps/docs/content/vorsteh-queue/04.packages/index.mdx new file mode 100644 index 00000000..b56d0e38 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/04.packages/index.mdx @@ -0,0 +1,4 @@ +--- +title: Packages +description: Additional packages in the Vorsteh Queue ecosystem. +--- diff --git a/apps/docs/content/vorsteh-queue/05.migration/01.v0-to-v1.mdx b/apps/docs/content/vorsteh-queue/05.migration/01.v0-to-v1.mdx new file mode 100644 index 00000000..4e66f99d --- /dev/null +++ b/apps/docs/content/vorsteh-queue/05.migration/01.v0-to-v1.mdx @@ -0,0 +1,190 @@ +--- +title: Migrating to v1.0 +navTitle: v0 → v1 +description: Complete migration guide from Vorsteh Queue v0.x to v1.0. +--- + +This guide covers all breaking changes when upgrading from v0.x to v1.0 across all packages. + +## Requirements + +- **Node.js 22+** — minimum version raised from 20 to 22 + +## Core (`@vorsteh-queue/core`) + +### Queue/Worker Split + +The `Queue` class is now producer-only. Job processing is handled by the new `Worker` class. + +**Before:** + +```typescript +const queue = new Queue(adapter, { name: "my-queue" }) +queue.register("send-email", handler) +queue.start() +``` + +**After:** + +```typescript +import { Queue, Worker } from "@vorsteh-queue/core" + +const queue = new Queue(adapter, { name: "my-queue" }) +const worker = new Worker(adapter, { name: "my-queue", concurrency: 5 }) + +worker.register("send-email", handler) +worker.start() + +await queue.add("send-email", { to: "user@example.com" }) +``` + +### Handler Signature + +Handlers now receive a `JobContext` as second argument with `signal` and `step` for timeout/cancellation and durable steps. + +**Before:** + +```typescript +queue.register("job", async (job) => { + return { done: true } +}) +``` + +**After:** + +```typescript +worker.register("job", async (job, { signal, step }) => { + // signal: AbortSignal for timeout/cancellation + // step: durable step execution (step.run, step.sleep, step.waitFor) + return { done: true } +}) +``` + +### New Statuses + +Added `cancelled` and `dead` statuses. Update any code that checks job statuses. + +### Removed APIs + +The following methods no longer exist on `Queue`: + +- `queue.register()` → use `worker.register()` +- `queue.start()` / `queue.stop()` → use `worker.start()` / `worker.stop()` +- `queue.pause()` / `queue.resume()` → use `worker.pause()` / `worker.resume()` +- `queue.enqueue()` / `queue.dequeue()` → use `queue.add()` / removed + +### Job Filter Changes + +`getJobs` and `size` now use a `where` input instead of flat parameters. + +**Before:** + +```typescript +await queue.getJobs({ status: "failed", name: "send-email", limit: 10 }) +await queue.size() +``` + +**After:** + +```typescript +await queue.getJobs({ + where: { status: "failed", name: "send-email" }, + limit: 10, +}) +await queue.size({ status: { eq: "failed" } }) +``` + +## Drizzle Adapter (`@vorsteh-queue/adapter-drizzle`) + +### Peer Dependency + +Upgrade `drizzle-orm` to `>=1.0.0-rc.4` and `drizzle-kit` to `>=1.0.0-rc.4` (was `>=0.30.0`). + + + For a complete list of changes in Drizzle ORM v1, see the official [Drizzle v0 + → v1 migration guide](https://orm.drizzle.team/docs/v0-v1-changes). + + +### Drizzle Client Initialization + +Drizzle ORM v1 requires a `relations` object instead of a `schema` object: + +**Before:** + +```typescript +import * as schema from "./schema" + +const db = drizzle(pool, { schema }) +``` + +**After:** + +```typescript +import { defineRelations } from "drizzle-orm" +import { queueFlows, queueJobs } from "@vorsteh-queue/adapter-drizzle" + +export { queueFlows, queueJobs } +export const relations = defineRelations({ queueFlows, queueJobs }) + +// In your database setup: +import { relations } from "./schema" + +const db = drizzle({ client: pool, relations }) +``` + +### Schema Migration + +New columns and indexes have been added. Run migrations: + +```bash +npx drizzle-kit generate +npx drizzle-kit migrate +``` + +New columns: `group_key`, `unique_key`, `cancelled_at`, `cancellation_reason`, `steps` + +Changed columns: + +- `timeout`: JSONB → INT (nullable, milliseconds) +- `progress`: nullable → non-nullable INT (default 0) +- `repeat_count`: nullable → non-nullable (default 0) + +New indexes: `idx_queue_jobs_polling`, `idx_queue_jobs_delayed`, `idx_queue_jobs_active_groups`, `idx_queue_jobs_stats` + +## Prisma Adapter (`@vorsteh-queue/adapter-prisma`) + +### Peer Dependency + +Upgrade `@prisma/client` to `>=7.0.0` (was `>=5.0.0`). + + + For a complete list of changes in Prisma 7, see the official [Prisma v7 + upgrade guide](https://www.prisma.io/docs/guides/upgrade-prisma-orm/v7). + + +### Schema Migration + +Update your `schema.prisma` with the new model fields and run migrations: + +```bash +npx prisma migrate dev +``` + +Same column changes as Drizzle (see above). + +## Kysely Adapter (`@vorsteh-queue/adapter-kysely`) + +Run the updated migration (same schema changes as Drizzle/Prisma above). The `createQueueJobsTable()` helper has been updated. + +## New Features in v1.0 + +These are not breaking changes but notable additions: + +- **Job Steps** — durable multi-step execution (`step.run()`, `step.sleep()`, `step.waitFor()`) +- **Flow Producer** — parent-child job trees with dependency tracking +- **Group FIFO** — ordered processing per group key +- **Unique Jobs** — deduplication via unique keys +- **Rate Limiting** — token-bucket per handler +- **Saga Compensation** — automatic rollback on step failure +- **Signals** — human-in-the-loop pause/resume +- **Event Triggers** — automatic follow-up job creation diff --git a/apps/docs/content/vorsteh-queue/05.migration/index.mdx b/apps/docs/content/vorsteh-queue/05.migration/index.mdx new file mode 100644 index 00000000..46c2da50 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/05.migration/index.mdx @@ -0,0 +1,6 @@ +--- +title: Migration Guides +description: Step-by-step guides for upgrading between versions of Vorsteh Queue. +--- + +Guides for migrating between versions of Vorsteh Queue packages. diff --git a/apps/docs/content/vorsteh-queue/index.mdx b/apps/docs/content/vorsteh-queue/index.mdx new file mode 100644 index 00000000..19af70d7 --- /dev/null +++ b/apps/docs/content/vorsteh-queue/index.mdx @@ -0,0 +1,6 @@ +--- +title: Vorsteh Queue +navTitle: Vorsteh Queue +description: A type-safe PostgreSQL job queue for Node.js with durable execution, flow orchestration, and first-class ORM adapters. +entrypoint: /docs/vorsteh-queue/getting-started/quickstart +--- diff --git a/apps/docs/eslint.config.js b/apps/docs/eslint.config.js deleted file mode 100644 index 966eba3d..00000000 --- a/apps/docs/eslint.config.js +++ /dev/null @@ -1,14 +0,0 @@ -import baseConfig, { restrictEnvAccess } from "@vorsteh-queue/eslint-config/base" -import nextjsConfig from "@vorsteh-queue/eslint-config/nextjs" -import reactConfig from "@vorsteh-queue/eslint-config/react" - -/** @type {import('typescript-eslint').Config} */ -export default [ - { - ignores: [".next/**"], - }, - ...baseConfig, - ...reactConfig, - ...nextjsConfig, - ...restrictEnvAccess, -] diff --git a/apps/docs/global.d.ts b/apps/docs/global.d.ts new file mode 100644 index 00000000..d7e961ee --- /dev/null +++ b/apps/docs/global.d.ts @@ -0,0 +1 @@ +declare module "*.css" diff --git a/apps/docs/next.config.mjs b/apps/docs/next.config.ts similarity index 60% rename from apps/docs/next.config.mjs rename to apps/docs/next.config.ts index 478622b7..7715e19f 100644 --- a/apps/docs/next.config.mjs +++ b/apps/docs/next.config.ts @@ -1,4 +1,22 @@ import createMDXPlugin from "@next/mdx" +import type { NextConfig } from "next" + +function normalizeBasePath(basePath: string | undefined): string { + if (!basePath) { + return "" + } + + const withLeadingSlash = basePath.startsWith("/") ? basePath : `/${basePath}` + + if (withLeadingSlash === "/") { + return "" + } + + return withLeadingSlash.replace(/\/+$/u, "") +} + +// oxlint-disable-next-line no-restricted-properties +const basePath = normalizeBasePath(process.env.BASE_PATH) const withMDX = createMDXPlugin({ options: { @@ -20,13 +38,23 @@ const withMDX = createMDXPlugin({ }, }) -/** @type {import('next').NextConfig} */ -const nextConfig = { +const nextConfig: NextConfig = { + ...(basePath + ? { + assetPrefix: basePath, + basePath, + } + : {}), + images: { + unoptimized: true, + }, output: "export", + pageExtensions: ["js", "jsx", "ts", "tsx", "md", "mdx"], + poweredByHeader: false, reactStrictMode: true, + // staticPageGenerationTimeout: 180, + trailingSlash: true, - poweredByHeader: false, - pageExtensions: ["js", "jsx", "ts", "tsx", "md", "mdx"], /** Enables hot reloading for local packages without a build step */ transpilePackages: [ "@vorsteh-queue/core", @@ -35,12 +63,14 @@ const nextConfig = { "@vorsteh-queue/adapter-kysely", "create-vorsteh-queue", ], - typescript: { ignoreBuildErrors: true, }, - images: { - unoptimized: true, + productionBrowserSourceMaps: false, + enablePrerenderSourceMaps: false, + experimental: { + cpus: 1, + serverSourceMaps: false, }, } diff --git a/apps/docs/package.json b/apps/docs/package.json index 47e6bc1f..6ac0333a 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -1,92 +1,94 @@ { "name": "docs", - "version": "0.0.0", + "version": "0.1.0", "private": true, "type": "module", "scripts": { - "build": "next typegen && renoun next build && pnpm generate-pagefind", - "clean": "git clean -xdf .cache .turbo dist node_modules .next", - "clean:cache": "git clean -xdf .cache", + "build": "next typegen && renoun next build", + "clean": "git clean -xdf .cache .turbo dist node_modules dist", + "clean:cache": "git clean -xdf .cache .turbo", + "clean:dist": "git clean -xdf dist", "dev": "next typegen && renoun next dev", - "format": "prettier --check . --ignore-path ../../.gitignore --ignore-path ../../.prettierignore", - "generate-pagefind": "pagefind --site out --output-path out/pagefind", - "lint": "eslint .", - "lint:links": "node --import ./src/register.js --import tsx/esm src/link-check.ts", - "matchtest": "tsx src/match.ts", + "generate:search-index": "tsx src/zbsearch/generate.ts", "preview": "tsx src/localserver.ts", + "start": "next start", "typecheck": "tsc --noEmit", "typegen": "next typegen" }, - "prettier": "@vorsteh-queue/prettier-config", "dependencies": { - "@icons-pack/react-simple-icons": "13.8.0", + "@base-ui/react": "1.6.0", + "@choo-choo/core": "0.2.1", + "@choo-choo/parser-ebnf": "0.1.3", + "@choo-choo/parser-utils": "0.1.3", + "@choo-choo/react": "0.1.3", + "@dnd-kit/core": "6.3.1", + "@dnd-kit/modifiers": "9.0.0", + "@dnd-kit/sortable": "10.0.0", + "@dnd-kit/utilities": "3.2.2", + "@icons-pack/react-simple-icons": "13.13.0", "@mdx-js/loader": "3.1.1", "@mdx-js/node-loader": "3.1.1", "@mdx-js/react": "3.1.1", - "@next/mdx": "16.1.6", - "@radix-ui/react-collapsible": "^1.1.12", - "@radix-ui/react-compose-refs": "1.1.2", - "@radix-ui/react-dialog": "^1.1.15", - "@radix-ui/react-dropdown-menu": "2.1.16", - "@radix-ui/react-id": "1.1.1", - "@radix-ui/react-primitive": "2.1.4", - "@radix-ui/react-separator": "^1.1.8", - "@radix-ui/react-slot": "^1.2.4", - "@radix-ui/react-tabs": "1.1.13", - "@radix-ui/react-tooltip": "^1.2.8", - "@vercel/og": "0.8.6", + "@next/mdx": "16.2.12", + "@renoun/screenshot": "0.3.3", + "@tailwindcss/typography": "0.5.20", + "@vorsteh-queue/cli": "workspace:*", + "@vorsteh-queue/server": "workspace:*", + "@zbsearch/plugin-data-persistence": "3.3.3", + "agentic-mermaid": "0.3.0", + "better-themes": "1.1.1", "class-variance-authority": "0.7.1", "clsx": "2.1.1", - "date-fns": "^4.1.0", - "globby": "16.1.0", - "interweave": "13.1.1", - "lucide-react": "0.563.0", - "multimatch": "7.0.0", - "next": "16.1.6", - "next-themes": "latest", - "p-map": "7.0.4", - "react": "19.2.4", - "react-dom": "19.2.4", - "read-pkg": "^10.0.0", - "rehype-mdx-import-media": "1.2.0", + "dedent": "1.7.2", + "elkjs": "0.12.0", + "entities": "8.0.0", + "globby": "16.2.2", + "lucide-react": "1.27.0", + "multimatch": "8.0.0", + "next": "16.2.12", + "onedollarstats": "0.0.23", + "ora": "9.4.1", + "p-map": "7.0.6", + "react": "19.2.8", + "react-dom": "19.2.8", + "read-pkg": "10.1.0", + "rehype-mdx-import-media": "1.4.0", "remark-frontmatter": "5.0.0", "remark-gfm": "4.0.1", "remark-mdx-frontmatter": "5.2.0", "remark-squeeze-paragraphs": "6.0.0", "remark-strip-badges": "7.0.0", - "renoun": "11.1.0", - "tm-grammars": "1.30.0", - "tm-themes": "1.11.0", - "ts-morph": "27.0.2", - "tw-animate-css": "^1.4.0", - "use-debounce": "10.1.0", - "zod": "4.3.6" + "renoun": "11.7.1", + "server-only": "0.0.1", + "shadcn": "4.15.0", + "tailwind-merge": "3.6.0", + "tm-grammars": "1.31.15", + "tm-themes": "1.12.2", + "ts-morph": "28.0.0", + "tw-animate-css": "1.4.0", + "uuid": "14.0.1", + "vaul": "1.1.2", + "yaml": "2.9.0", + "zbsearch": "3.3.3", + "zod": "4.4.3" }, "devDependencies": { - "@tailwindcss/postcss": "4.1.18", - "@tailwindcss/typography": "0.5.19", - "@types/mdx": "2.0.13", - "@types/node": "22.19.0", - "@types/react": "19.2.10", + "@tailwindcss/postcss": "4.3.3", + "@types/mdx": "2.0.14", + "@types/node": "24.13.3", + "@types/react": "19.2.17", "@types/react-dom": "19.2.3", "@types/serve-handler": "6.1.4", "@vorsteh-queue/adapter-drizzle": "workspace:*", "@vorsteh-queue/adapter-kysely": "workspace:*", "@vorsteh-queue/adapter-prisma": "workspace:*", "@vorsteh-queue/core": "workspace:*", - "@vorsteh-queue/eslint-config": "workspace:*", - "@vorsteh-queue/prettier-config": "workspace:*", "@vorsteh-queue/tsconfig": "workspace:*", - "eslint": "^9.39.2", - "next-validate-link": "1.6.4", - "pagefind": "1.4.0", - "postcss": "8.5.6", - "prettier": "^3.8.1", - "serve-handler": "6.1.6", - "tailwind-merge": "3.4.0", - "tailwindcss": "4.1.18", - "tailwindcss-animate": "1.0.7", - "tsx": "4.21.0", - "typescript": "^5.9.3" + "commander": "15.0.0", + "postcss": "8.5.23", + "serve-handler": "6.1.7", + "tailwindcss": "4.3.3", + "tsx": "4.23.1", + "typescript": "6.0.3" } } diff --git a/apps/docs/postcss.config.mjs b/apps/docs/postcss.config.mjs index 2f8795a9..f6c75ff9 100644 --- a/apps/docs/postcss.config.mjs +++ b/apps/docs/postcss.config.mjs @@ -1,3 +1,4 @@ +/** @type {import('postcss-load-config').Config} */ const config = { plugins: { "@tailwindcss/postcss": {}, diff --git a/apps/docs/public/.gitkeep b/apps/docs/public/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/apps/docs/public/apple-touch-icon.png b/apps/docs/public/apple-touch-icon.png new file mode 100644 index 00000000..09089c0f Binary files /dev/null and b/apps/docs/public/apple-touch-icon.png differ diff --git a/apps/docs/public/favicon.ico b/apps/docs/public/favicon.ico index 9910ac25..e6b66713 100644 Binary files a/apps/docs/public/favicon.ico and b/apps/docs/public/favicon.ico differ diff --git a/apps/docs/public/icon.svg b/apps/docs/public/icon.svg new file mode 100644 index 00000000..fd93c537 --- /dev/null +++ b/apps/docs/public/icon.svg @@ -0,0 +1,119 @@ + + + + + + + + + + + diff --git a/apps/docs/public/icons/icon-128x128.png b/apps/docs/public/icons/icon-128x128.png deleted file mode 100644 index d36d2e6c..00000000 Binary files a/apps/docs/public/icons/icon-128x128.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-144x144.png b/apps/docs/public/icons/icon-144x144.png deleted file mode 100644 index 22e65d02..00000000 Binary files a/apps/docs/public/icons/icon-144x144.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-152x152.png b/apps/docs/public/icons/icon-152x152.png deleted file mode 100644 index 6e09fbfa..00000000 Binary files a/apps/docs/public/icons/icon-152x152.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-192x192.png b/apps/docs/public/icons/icon-192x192.png deleted file mode 100644 index b9496865..00000000 Binary files a/apps/docs/public/icons/icon-192x192.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-256x256.png b/apps/docs/public/icons/icon-256x256.png deleted file mode 100644 index c5728b43..00000000 Binary files a/apps/docs/public/icons/icon-256x256.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-384x384.png b/apps/docs/public/icons/icon-384x384.png deleted file mode 100644 index d7dbc327..00000000 Binary files a/apps/docs/public/icons/icon-384x384.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-48x48.png b/apps/docs/public/icons/icon-48x48.png deleted file mode 100644 index 0b1dc070..00000000 Binary files a/apps/docs/public/icons/icon-48x48.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-512x512.png b/apps/docs/public/icons/icon-512x512.png deleted file mode 100644 index 8ca4f44d..00000000 Binary files a/apps/docs/public/icons/icon-512x512.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-72x72.png b/apps/docs/public/icons/icon-72x72.png deleted file mode 100644 index 9f5eb1d2..00000000 Binary files a/apps/docs/public/icons/icon-72x72.png and /dev/null differ diff --git a/apps/docs/public/icons/icon-96x96.png b/apps/docs/public/icons/icon-96x96.png deleted file mode 100644 index 633e4182..00000000 Binary files a/apps/docs/public/icons/icon-96x96.png and /dev/null differ diff --git a/apps/docs/public/raja.jpg b/apps/docs/public/raja.jpg new file mode 100644 index 00000000..51f5f4b8 Binary files /dev/null and b/apps/docs/public/raja.jpg differ diff --git a/apps/docs/public/vorsteh-queue-logo-nobg.png b/apps/docs/public/vorsteh-queue-logo-nobg.png deleted file mode 100644 index 54c1d56a..00000000 Binary files a/apps/docs/public/vorsteh-queue-logo-nobg.png and /dev/null differ diff --git a/apps/docs/public/vorsteh-queue-logo.svg b/apps/docs/public/vorsteh-queue-logo.svg deleted file mode 100644 index 4afa9ef3..00000000 --- a/apps/docs/public/vorsteh-queue-logo.svg +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/apps/docs/public/web-app-manifest-192x192.png b/apps/docs/public/web-app-manifest-192x192.png new file mode 100644 index 00000000..21596f16 Binary files /dev/null and b/apps/docs/public/web-app-manifest-192x192.png differ diff --git a/apps/docs/public/web-app-manifest-512x512.png b/apps/docs/public/web-app-manifest-512x512.png new file mode 100644 index 00000000..37242edd Binary files /dev/null and b/apps/docs/public/web-app-manifest-512x512.png differ diff --git a/apps/docs/src/app/(homepage)/_sections/dashboard.tsx b/apps/docs/src/app/(homepage)/_sections/dashboard.tsx new file mode 100644 index 00000000..dc6b515b --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/dashboard.tsx @@ -0,0 +1,156 @@ +import { Eye, Filter, Keyboard, Monitor } from "lucide-react" +import { Space_Grotesk } from "next/font/google" +import Link from "next/link" + +import { PageContainer } from "@/components/page-container" +import { buttonVariants } from "@/components/ui/button" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const bulletPoints = [ + { icon: Monitor, label: "Real-time queue stats with auto-refresh" }, + { icon: Filter, label: "Filter jobs by status, name, and time range" }, + { icon: Eye, label: "Inspect job details, payload, and errors" }, + { icon: Keyboard, label: "Full keyboard navigation with shortcuts" }, +] + +function TerminalMockup() { + return ( +
+ {/* Terminal title bar */} +
+
+
+
+
+
+ + vorsteh-queue dashboard + +
+ + {/* Terminal content */} +
+ {/* Sidebar + Content layout */} +
+ {/* Sidebar */} +
+ Queues + ▸ email + data-processing + notifications + Views + ▸ Overview + Jobs + Dead Letter +
+ + {/* Main content - Bar chart */} +
+ Job Status +
+ Pending + ████████░░░░░░░░ + 247 +
+
+ Processing + ███░░░░░░░░░░░░░ + 12 +
+
+ Completed + ████████████████ + 1,893 +
+
+ Failed + ██░░░░░░░░░░░░░░ + 8 +
+
+ Dead + █░░░░░░░░░░░░░░░ + 3 +
+
+ Total: 2,163 {" "}[j/↵] View jobs +
+
+
+ + {/* Footer - key hints */} +
+ Tab switch pane {" "} + o overview {" "} + j jobs {" "} + d dead letter {" "} + ? help {" "} + q quit +
+
+
+ ) +} + +export function DashboardSection() { + return ( +
+ +
+ {/* Left column - terminal mockup */} +
+ +
+ + {/* Right column - text */} +
+ + Terminal Dashboard + +

+ Monitor your queues without leaving the terminal. +

+

+ A full-featured interactive TUI dashboard for real-time queue + monitoring. Navigate between queues, inspect jobs, retry failures, + and manage dead letters — all from your terminal. +

+ +
    + {bulletPoints.map((point) => ( +
  • + + {point.label} +
    • + ))} +
+ +
+ + CLI Documentation + + + Try the Example + +
+
+
+ +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/_sections/features.tsx b/apps/docs/src/app/(homepage)/_sections/features.tsx new file mode 100644 index 00000000..42eff12e --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/features.tsx @@ -0,0 +1,118 @@ +import { + Activity, + ArrowUpDown, + Calendar, + Clock, + Code2, + Gauge, + GitBranch, + Inbox, + Pause, + RefreshCw, + Repeat, + TrendingUp, +} from "lucide-react" +import { Space_Grotesk } from "next/font/google" + +import { PageContainer } from "@/components/page-container" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const features = [ + { + description: "Fire and forget or run exactly once jobs.", + icon: Clock, + title: "One-time Jobs", + }, + { + description: "Schedule jobs to run in the future with precision.", + icon: Pause, + title: "Delayed Jobs", + }, + { + description: "Cron-based scheduling with timezone support.", + icon: Calendar, + title: "Scheduled Jobs", + }, + { + description: "Run jobs on fixed intervals with ease.", + icon: Repeat, + title: "Recurring Jobs", + }, + { + description: "Prioritize what matters most. High to low.", + icon: ArrowUpDown, + title: "Job Priorities", + }, + { + description: "Automatic retries with customizable backoff strategies.", + icon: RefreshCw, + title: "Retries & Backoff", + }, + { + description: "Failed jobs don't disappear. Inspect and retry them safely.", + icon: Inbox, + title: "Dead Letter Queue", + }, + { + description: "Parent-child job hierarchies with dependency tracking.", + icon: GitBranch, + title: "Flow Pipelines", + }, + { + description: "Real-time percentage updates during job execution.", + icon: TrendingUp, + title: "Progress Tracking", + }, + { + description: "Opt-in metrics and distributed tracing via OpenTelemetry.", + icon: Activity, + title: "OpenTelemetry Built-in", + }, + { + description: "Pause, resume, and manage workers with graceful shutdown.", + icon: Gauge, + title: "Worker Controls", + }, + { + description: "Full TypeScript support with excellent DX.", + icon: Code2, + title: "TypeScript First", + }, +] + +export function FeaturesSection() { + return ( +
+ +
+ + Powerful Features + +

+ Everything you need to build robust background systems. +

+
+ +
+ {features.map((feature) => ( +
+
+ +
+

+ {feature.title} +

+

+ {feature.description} +

+
+ ))} +
+ +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/_sections/hero.tsx b/apps/docs/src/app/(homepage)/_sections/hero.tsx new file mode 100644 index 00000000..c0fa779c --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/hero.tsx @@ -0,0 +1,124 @@ +import { + ArrowRight, + CheckCircle2, + Database, + Layers, + Shield, +} from "lucide-react" +import { Space_Grotesk } from "next/font/google" +import Link from "next/link" + +import { PageContainer } from "@/components/page-container" +import { buttonVariants } from "@/components/ui/button" +import { useMDXComponents } from "@/mdx-components" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const codeSnippet = `import { Queue, MemoryQueueAdapter } from "@vorsteh-queue/core" + +const queue = new Queue(new MemoryQueueAdapter(), { name: "my-queue" }) + +// Register a handler +queue.register("send-email", async (job) => { + await sendEmail(job.payload.to, job.payload.template) + return { sent: true } +}) + +// Add a job +await queue.add("send-email", { + to: "user@example.com", + template: "welcome", +}) + +// Start processing +queue.start()` + +const features = [ + { icon: Database, label: "PostgreSQL 12+" }, + { icon: Layers, label: "ORM-Agnostic" }, + { icon: Shield, label: "Production-Ready" }, + { icon: CheckCircle2, label: "Open Source" }, +] + +export function HeroSection() { + return ( +
+ {/* Dot pattern background — fades to transparent at bottom */} +
+ {/* Orange gradient glow - top center */} +
+ + +
+ {/* Left column - text content */} +
+

+ Reliable Job Queue for Modern Applications +

+ +

+ A type-safe PostgreSQL job queue for Node.js with durable + execution, flow orchestration, and first-class ORM adapters. +

+ +
+ + Get Started + + + + View on GitHub + +
+ +
+ {features.map((feature) => ( + + + {feature.label} + + ))} +
+
+ + {/* Right column - code snippet */} +
+ {useMDXComponents().CodeBlock({ + children: codeSnippet, + language: "ts", + shouldAnalyze: false, + })} +
+
+ +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/_sections/index.ts b/apps/docs/src/app/(homepage)/_sections/index.ts new file mode 100644 index 00000000..84572ffc --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/index.ts @@ -0,0 +1,5 @@ +export { HeroSection } from "./hero" +export { WhyChooseSection } from "./why-choose" +export { FeaturesSection } from "./features" +export { ObservabilitySection } from "./observability" +export { OpenSourceSection } from "./open-source" diff --git a/apps/docs/src/app/(homepage)/_sections/observability.tsx b/apps/docs/src/app/(homepage)/_sections/observability.tsx new file mode 100644 index 00000000..23f430a4 --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/observability.tsx @@ -0,0 +1,143 @@ +import { Activity, BarChart3, GitBranch } from "lucide-react" +import { Space_Grotesk } from "next/font/google" +import Link from "next/link" + +import { PageContainer } from "@/components/page-container" +import { buttonVariants } from "@/components/ui/button" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const bulletPoints = [ + { icon: Activity, label: "Automatic metrics: counters, histograms, gauges" }, + { icon: GitBranch, label: "Distributed tracing with job-level spans" }, + { + icon: BarChart3, + label: "Export to Prometheus, Grafana, Datadog, and more", + }, +] + +export function ObservabilitySection() { + return ( +
+ +
+ {/* Left column - text */} +
+ + OpenTelemetry Native + +

+ Observability without the glue code. +

+

+ vorsteh-queue instruments itself automatically via OpenTelemetry. + Metrics, traces, and job spans flow into your existing + observability stack with zero configuration. +

+ +
    + {bulletPoints.map((point) => ( +
  • + + {point.label} +
    • + ))} +
+ +
+ + Observability Guide + +
+
+ + {/* Right column - code example */} +
+
+ {/* Tab bar */} +
+
+ instrumentation.ts +
+
+ + Zero overhead without SDK + +
+
+ + {/* Code content */} +
+ 
+                  
+                    
+                      {"// Your existing OTel setup — that's it\n"}
+                    
+                    import
+                    {" { NodeSDK } "}
+                    from
+                    
+                      {" '@opentelemetry/sdk-node'\n"}
+                    
+                    import
+                    
+                      {" { OTLPMetricExporter } "}
+                    
+                    from
+                    
+                      {" '@opentelemetry/exporter-metrics-otlp-http'\n\n"}
+                    
+                    const
+                     sdk = 
+                    new
+                     NodeSDK
+                    {"({\n"}
+                    {"  metricReader: "}
+                    new
+                    
+                      {" OTLPMetricExporter"}
+                    
+                    {"(),\n"})
+                    {"\n\n"}
+                    sdk.
+                    start
+                    ()
+                    {"\n\n"}
+                    
+                      {"// vorsteh-queue automatically emits:\n"}
+                    
+                    
+                      {"// ✓ vorsteh_queue.jobs.processed\n"}
+                    
+                    
+                      {"// ✓ vorsteh_queue.jobs.failed\n"}
+                    
+                    
+                      {"// ✓ vorsteh_queue.jobs.duration\n"}
+                    
+                    
+                      {"// ✓ vorsteh_queue.jobs.wait_time\n"}
+                    
+                    
+                      {"// ✓ Distributed traces per job\n"}
+                    
+                  
+
+
+
+
+
+ +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/_sections/open-source.tsx b/apps/docs/src/app/(homepage)/_sections/open-source.tsx new file mode 100644 index 00000000..59e46539 --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/open-source.tsx @@ -0,0 +1,70 @@ +import { GitFork, Heart, Scale, Star, Users } from "lucide-react" +import { Space_Grotesk } from "next/font/google" + +import { PageContainer } from "@/components/page-container" +import { buttonVariants } from "@/components/ui/button" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const badges = [ + { icon: Scale, label: "MIT License" }, + { icon: Users, label: "Community Driven" }, + { icon: GitFork, label: "No vendor lock-in" }, +] + +export function OpenSourceSection() { + return ( +
+ +
+ +
+ +

+ Free & Open Source +

+ +

+ Vorsteh Queue is completely free and open source. Built by developers, + for developers. +

+ +
+ {badges.map((badge) => ( + + + {badge.label} + + ))} +
+ + + +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/_sections/why-choose.tsx b/apps/docs/src/app/(homepage)/_sections/why-choose.tsx new file mode 100644 index 00000000..964b2ad2 --- /dev/null +++ b/apps/docs/src/app/(homepage)/_sections/why-choose.tsx @@ -0,0 +1,92 @@ +import { Blocks, Clock, Gauge, Layers, Shield, Zap } from "lucide-react" +import { Space_Grotesk } from "next/font/google" + +import { PageContainer } from "@/components/page-container" +import { + Card, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) + +const reasons = [ + { + description: + "Optimized for high throughput with minimal overhead. Built on PostgreSQL, tuned for performance.", + icon: Zap, + title: "Blazing Fast", + }, + { + description: + "Backed by PostgreSQL's ACID guarantees and advanced locking. Your jobs are safe and consistent.", + icon: Shield, + title: "Reliable by Design", + }, + { + description: + "Works with any ORM or query builder. Use it your way, without being locked in.", + icon: Layers, + title: "ORM Agnostic", + }, + { + description: + "Cron jobs, delayed jobs, and recurring tasks. Powerful scheduling without added complexity.", + icon: Clock, + title: "Flexible Scheduling", + }, + { + description: + "Monitoring, retries, backoff strategies, dead letter queues, and more. Everything you need in production.", + icon: Gauge, + title: "Production Ready", + }, + { + description: + "100% open source. Clean architecture, extensible APIs, and a welcoming community.", + icon: Blocks, + title: "Open & Extensible", + }, +] + +export function WhyChooseSection() { + return ( +
+ +
+ + Why Choose Vorsteh Queue? + +

+ Built for developers who value reliability, +
+ simplicity, and control. +

+
+ +
+ {reasons.map((reason) => ( + + +
+ +
+ {reason.title} + + {reason.description} + + + + ))} +
+ +
+ ) +} diff --git a/apps/docs/src/app/(homepage)/page.tsx b/apps/docs/src/app/(homepage)/page.tsx new file mode 100644 index 00000000..b64f26da --- /dev/null +++ b/apps/docs/src/app/(homepage)/page.tsx @@ -0,0 +1,26 @@ +import { SiteFooter } from "@/components/site-footer" +import { SiteHeader } from "@/components/site-header" + +import { DashboardSection } from "./_sections/dashboard" +import { FeaturesSection } from "./_sections/features" +import { HeroSection } from "./_sections/hero" +import { ObservabilitySection } from "./_sections/observability" +import { OpenSourceSection } from "./_sections/open-source" +import { WhyChooseSection } from "./_sections/why-choose" + +export default function Page() { + return ( +
+ + + + + + + + + + +
+ ) +} diff --git a/apps/docs/src/app/(site)/(docs)/[...slug]/page.tsx b/apps/docs/src/app/(site)/(docs)/[...slug]/page.tsx deleted file mode 100644 index df69416e..00000000 --- a/apps/docs/src/app/(site)/(docs)/[...slug]/page.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import type { Metadata } from "next" -import { notFound } from "next/navigation" - -import { getBreadcrumbItems, transformedEntries } from "~/collections" -import { DirectoryContent } from "~/components/directory-content" -import { FileContent } from "~/components/file-content" - -export async function generateStaticParams() { - const slugs = (await transformedEntries()).map((entry) => ({ - slug: entry.segments, - })) - - return slugs -} - -export async function generateMetadata(props: PageProps<"/[...slug]">): Promise { - const { slug } = await props.params - const breadcrumbItems = await getBreadcrumbItems(slug) - - const titles = breadcrumbItems.map((ele) => ele.title) - - return { - title: `${titles.join(" - ")}`, - } -} - -export default async function DocsPage(props: PageProps<"/[...slug]">) { - const { slug } = await props.params - - const searchParam = `/${slug.join("/")}` - - const transformedEntry = (await transformedEntries()).find( - (ele) => ele.raw_pathname == searchParam, - ) - - if (!transformedEntry) { - return notFound() - } - - // if we can't find an index file, but we have a valid directory - // use the directory component for rendering - if (!transformedEntry.file && transformedEntry.isDirectory) { - return ( - <> - - - ) - } - - // if we have a valid file ( including the index file ) - // use the file component for rendering - if (transformedEntry.file) { - return ( - <> - - - ) - } - - // seems to be an invalid path - return notFound() -} diff --git a/apps/docs/src/app/(site)/(docs)/layout.tsx b/apps/docs/src/app/(site)/(docs)/layout.tsx deleted file mode 100644 index aa227b3b..00000000 --- a/apps/docs/src/app/(site)/(docs)/layout.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import type { TreeItem } from "~/lib/navigation" -import { AllDocumentation } from "~/collections" -import { DocsSidebar } from "~/components/docs-sidebar" -import { Sidebar, SidebarContent, SidebarInset, SidebarProvider } from "~/components/ui/sidebar" -import { getTree } from "~/lib/navigation" - -export function AppSidebar({ items }: { items: TreeItem[] }) { - return ( - - - - - - ) -} - -export default async function DocsLayout(props: LayoutProps<"/">) { - const recursiveCollections = await AllDocumentation.getEntries({ - recursive: true, - }) - - // here we're generating the items for the dropdown menu in the sidebar - // it's used to provide a short link for the user to switch easily between the different collections - // it expects an `index.mdx` file in each collection at the root level ( e.g. `aria-docs/index.mdx`) - - const tree = recursiveCollections.filter((ele) => ele.depth === 0) - - const sidebarItems = await getTree(tree) - - return ( -
- - - -
-
- {props.children} -
-
- - -
- ) -} diff --git a/apps/docs/src/app/(site)/(home)/page.tsx b/apps/docs/src/app/(site)/(home)/page.tsx deleted file mode 100644 index 7494cc0e..00000000 --- a/apps/docs/src/app/(site)/(home)/page.tsx +++ /dev/null @@ -1,280 +0,0 @@ -import Link from "next/link" -import { CheckCircle, Code, Heart, Info } from "lucide-react" -import pMap from "p-map" -import { CodeBlock, Link as GitLink, Logo as GitLogo } from "renoun/components" - -import type { AllowedIcon } from "~/lib/icon" -import { features } from "~/collections" -import { Button } from "~/components/ui/button" -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "~/components/ui/card" -import { asyncFilter } from "~/lib/array-helper" -import { getIcon } from "~/lib/icon" - -const example_snippet = /* typescript */ ` -import { MemoryQueueAdapter, Queue } from "@vorsteh-queue/core" - -interface TEmailPayload { - to: string - subject: string -} - -interface TEmailResult { - sent: boolean -} - -const adapter = new MemoryQueueAdapter() -const queue = new Queue(adapter, { name: "email-queue" }) - -queue.register("send-email", async ({ payload }) => { - // Send email logic here - return { sent: true } -}) - -await queue.add("send-email", { to: "user@example.com", subject: "Welcome!" }) -queue.start() -` - -export default async function Home() { - const featuresCollection = await features.getEntries() - - const filterKeyFeatures = await asyncFilter(featuresCollection, async (ele) => { - const frontmatter = await ele.getExportValue("frontmatter") - - return frontmatter.type === "key_feature" - }) - - const filterOtherFeatures = await asyncFilter(featuresCollection, async (ele) => { - const frontmatter = await ele.getExportValue("frontmatter") - - return frontmatter.type === "feature" - }) - - const keyFeatures = await pMap(filterKeyFeatures, async (ele) => { - const frontmatter = await ele.getExportValue("frontmatter") - const description = await ele.getExportValue("default") - return { - title: frontmatter.title, - description, - icon: frontmatter.icon as AllowedIcon, - } - }) - - const otherFeatures = await pMap(filterOtherFeatures, async (ele) => { - const frontmatter = await ele.getExportValue("frontmatter") - const description = await ele.getExportValue("default") - return { - title: frontmatter.title, - description, - icon: frontmatter.icon as AllowedIcon, - } - }) - - return ( - <> - {/* Hero Section */} -
-
-
-
-

- Reliable Job Queue for Modern Applications -

-

- A powerful, ORM-agnostic queue engine for PostgreSQL 12+. Handle background jobs, - scheduled tasks, and recurring processes with ease. -

-
- - - -
-
-
- {/* Code Example */} - - {example_snippet} - -
-
-
-
- - {/* Why Section */} -
-
-
-

- Why Choose Vorsteh Queue? -

-

- Built for developers who need reliability, flexibility, and excellent developer - experience -

-
- -
- {keyFeatures.map((ele, eleIdx) => ( - - -
- {getIcon(ele.icon, { className: "text-orange-primary h-6 w-6" })} -
- {ele.title} - - - - - - - - ))} -
-
-
- - {/* Features Section */} -
-
-
-

- Powerful Features -

-

- Everything you need to handle background processing in your applications -

-
- -
- {otherFeatures.map((ele, eleIdx) => ( -
-
- {getIcon(ele.icon, { className: "text-orange-primary h-6 w-6" })} -
-

{ele.title}

-
- -
-
- ))} -
-
-
- - {/* About Section */} -
-
-
-
- -
-

- About the Name: Vorsteh Queue -

-

- The name "Vorsteh Queue" is a tribute to our beloved German Spaniel. This breed is - closely related to the Münsterländer, a type of pointing dog, known in German as a - "Vorstehhund". Just as a pointing dog steadfastly indicates its target, Vorsteh Queue - aims to reliably point your application towards efficient and robust background job - processing. -

-

- The inspiration for naming a tech project after a dog comes from the delightful story - of{" "} - - Bruno - - , the API client. It's a nod to the personal touch and passion that drives open-source - development, much like the loyalty and dedication of our canine companions. -

-
-
-
- - {/* Open Source Section */} -
-
-
-
- -
-

- Free & Open Source -

-

- Vorsteh Queue is completely free and open source. Built by developers, for developers. - No hidden costs, no vendor lock-in, no limitations. Use it in your personal projects, - startups, or enterprise applications. -

-
-
- - MIT License -
-
- - Community Driven -
-
- - No Vendor Lock-in -
-
-
- - - -
-
-
-
- - ) -} diff --git a/apps/docs/src/app/(site)/layout.tsx b/apps/docs/src/app/(site)/layout.tsx deleted file mode 100644 index b29b2002..00000000 --- a/apps/docs/src/app/(site)/layout.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import Footer from "~/components/footer" -import MainHeader from "~/components/main-header" - -export default function SiteLayout({ children }: LayoutProps<"/">) { - return ( -
- {/* Header */} - - {children} -
-
- ) -} diff --git a/apps/docs/src/app/_opengraph-image.js b/apps/docs/src/app/_opengraph-image.js deleted file mode 100644 index 5440aca0..00000000 --- a/apps/docs/src/app/_opengraph-image.js +++ /dev/null @@ -1,99 +0,0 @@ -import { ImageResponse } from "next/og" - -export const dynamic = "force-static" -export const contentType = "image/png" - -// eslint-disable-next-line @typescript-eslint/require-await -export default async function Image() { - return new ImageResponse( -
-
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- Vorsteh Queue -
-
-
- A powerful, ORM-agnostic queue engine for PostgreSQL 12+. -
-
-
-
, - { - width: 1200, - height: 630, - }, - ) -} diff --git a/apps/docs/src/app/docs.snapshot.jsonl/route.ts b/apps/docs/src/app/docs.snapshot.jsonl/route.ts new file mode 100644 index 00000000..4d272c1c --- /dev/null +++ b/apps/docs/src/app/docs.snapshot.jsonl/route.ts @@ -0,0 +1,14 @@ +import { buildDocsSnapshotJsonl } from "@/lib/docs-snapshot" + +export const dynamic = "force-static" +export const runtime = "nodejs" + +export async function GET(): Promise { + const jsonl = await buildDocsSnapshotJsonl() + + return new Response(jsonl, { + headers: { + "Content-Type": "application/x-ndjson; charset=utf-8", + }, + }) +} diff --git a/apps/docs/src/app/docs.snapshot/[collectionName]/route.ts b/apps/docs/src/app/docs.snapshot/[collectionName]/route.ts new file mode 100644 index 00000000..9f5708cc --- /dev/null +++ b/apps/docs/src/app/docs.snapshot/[collectionName]/route.ts @@ -0,0 +1,54 @@ +import { notFound } from "next/navigation" + +import type { AvailableCollection } from "@/collections" +import { availableCollections } from "@/collections" +import { buildDocsSnapshotJsonl } from "@/lib/docs-snapshot" + +export const dynamic = "force-static" +export const runtime = "nodejs" + +function normalizeCollectionName( + collectionName: string +): AvailableCollection | undefined { + if (!collectionName.endsWith(".jsonl")) { + return undefined + } + + const collection = collectionName.slice(0, -6) + + if (!collection) { + return undefined + } + + return availableCollections.includes(collection as AvailableCollection) + ? (collection as AvailableCollection) + : undefined +} + +export async function generateStaticParams(): Promise< + { collectionName: string }[] +> { + return availableCollections.map((collection) => ({ + collectionName: `${collection}.jsonl`, + })) +} + +export async function GET( + _request: Request, + ctx: { params: Promise<{ collectionName: string }> } +): Promise { + const { collectionName } = await ctx.params + const collection = normalizeCollectionName(collectionName) + + if (!collection) { + notFound() + } + + const jsonl = await buildDocsSnapshotJsonl({ collection }) + + return new Response(jsonl, { + headers: { + "Content-Type": "application/x-ndjson; charset=utf-8", + }, + }) +} diff --git a/apps/docs/src/app/docs/[...slug]/_loading.tsx b/apps/docs/src/app/docs/[...slug]/_loading.tsx new file mode 100644 index 00000000..c5adb126 --- /dev/null +++ b/apps/docs/src/app/docs/[...slug]/_loading.tsx @@ -0,0 +1,28 @@ +import { Skeleton } from "@/components/ui/skeleton" + +export default function Loading() { + // The docs shell (sidebar, header, TOC rail) is rendered by the segment layout. + // Keep the loading UI limited to the page content to avoid duplicate sidebars. + return ( +
+
+ + +
+ +
+ + + + +
+ +
+ + + + +
+
+ ) +} diff --git a/apps/docs/src/app/docs/[...slug]/layout.tsx b/apps/docs/src/app/docs/[...slug]/layout.tsx new file mode 100644 index 00000000..d1b2694f --- /dev/null +++ b/apps/docs/src/app/docs/[...slug]/layout.tsx @@ -0,0 +1,276 @@ +import { createSlug } from "renoun" +import type { ContentSection } from "renoun/file-system" +import { isFile } from "renoun/file-system" + +import { + getFileContent, + getDocumentationEntryBySlug, + rootCollections, + getBreadcrumbItems, + getMetadata, +} from "@/collection-helpers" +import { SiteBreadcrumb } from "@/components/breadcrumb" +import { CollectionChooser } from "@/components/collection-chooser" +import { DocsLeftRailBackground } from "@/components/docs-left-rail-background" +import { DocsSidebar } from "@/components/docs-sidebar" +import { + kindToLabel, + resolvePackagePath, +} from "@/components/mdx/api-reference/utils" +import { MobileDocsHeader } from "@/components/mobile-docs-header" +import { SidebarToggle } from "@/components/sidebar-toggle" +import { + MobileTableOfContents, + TableOfContents, +} from "@/components/table-of-contents" +import { TableOfContentsScript } from "@/components/toc" +import { SidebarInset } from "@/components/ui/sidebar" +import { SidebarGridProvider } from "@/components/ui/sidebar-grid-provider" +import { getCliCommandTocSections } from "@/lib/cli-commands" +import { + getCollectionNavigation, + getFavoriteNavigationItems, +} from "@/lib/navigation" +import { analyzeSyntacticExports } from "@/lib/ts-morph-analysis" +import type { SyntacticExportInfo } from "@/lib/ts-morph-analysis" +import { cn } from "@/lib/utils" + +function createDocsLayoutConfig(input: { + layoutWidth: string + sidebarMinWidth: string + sidebarPreferredWidth: string + sidebarMaxWidth: string + tocContentWidth: string + tocPaddingX?: string + tocBorderWidth?: string +}) { + const { + layoutWidth, + sidebarMinWidth, + sidebarPreferredWidth, + sidebarMaxWidth, + tocContentWidth, + tocPaddingX = "1.5rem", + tocBorderWidth = "1px", + } = input + + return { + cssVars: { + "--docs-content-track-width": + "minmax(0, calc(var(--docs-layout-width) - var(--docs-sidebar-track-width) - var(--docs-toc-width)))", + "--docs-gutter-width": `clamp(0px, calc((100vw - var(--docs-layout-width)) / 2), calc(var(--docs-layout-width)))`, + "--docs-layout-width": layoutWidth, + "--docs-left-rail-width": + "calc(var(--docs-gutter-width) + var(--docs-sidebar-track-width))", + "--docs-sidebar-track-width": `clamp(${sidebarMinWidth}, ${sidebarPreferredWidth}, ${sidebarMaxWidth})`, + "--docs-toc-border-width": tocBorderWidth, + "--docs-toc-content-width": tocContentWidth, + "--docs-toc-padding-x": tocPaddingX, + "--docs-toc-width": `calc(var(--docs-toc-content-width) + (2 * var(--docs-toc-padding-x)) + var(--docs-toc-border-width))`, + } as React.CSSProperties, + layoutWidth, + sidebarMaxWidth, + sidebarMinWidth, + sidebarPreferredWidth, + tocBorderWidth, + tocContentWidth, + tocPaddingX, + } +} + +// Keep the previous 3-column docs shell semantics, just without parallel routes. +const DOCS_LAYOUT = createDocsLayoutConfig({ + layoutWidth: "100rem", + sidebarMaxWidth: "300px", + sidebarMinWidth: "250px", + sidebarPreferredWidth: "260px", + tocBorderWidth: "0px", + tocContentWidth: "250px", +}) + +async function fetchApiReferenceTocItems( + references: { name: string; file: string }[] +) { + const results = await Promise.all( + references.map(async (ref) => { + try { + const filePath = resolvePackagePath(ref.file) + const exports = await analyzeSyntacticExports(filePath) + return { name: ref.name, exports } + } catch { + return { name: ref.name, exports: [] as SyntacticExportInfo[] } + } + }) + ) + return results +} + +function syntacticExportsToTocItems( + exports: SyntacticExportInfo[], + baseDepth: number +) { + return exports.map((exp) => { + const slug = createSlug(exp.name) + const kindLabel = exp.kind ? kindToLabel(exp.kind) : null + + return { + depth: baseDepth, + id: slug, + title: exp.name, + ...(kindLabel + ? { + jsx: ( + + + {kindLabel.charAt(0)} + + {exp.name} + + ), + } + : {}), + ...(exp.methods?.length + ? { + children: exp.methods.map((m) => ({ + depth: baseDepth, + id: `${slug}-${createSlug(m)}`, + jsx: {m}, + title: m, + })), + } + : {}), + } + }) +} + +export default async function DocsSlugLayout({ + children, + params, +}: { + children: React.ReactNode + params: Promise<{ slug: string[] }> +}) { + const { slug } = await params + const docsLayoutVars = DOCS_LAYOUT.cssVars + const docsGridClasses = cn( + "relative z-10 overflow-x-clip", + // Default desktop grid: left gutter, sidebar, content, toc, right gutter. + "xl:grid-cols-[minmax(min-content,1fr)_var(--docs-sidebar-track-width)_var(--docs-content-track-width)_var(--docs-toc-width)_minmax(min-content,1fr)]", + // Offcanvas desktop grid: keep content + TOC centered when the sidebar collapses. + "has-data-[collapsible=offcanvas]:xl:grid-cols-[minmax(min-content,1fr)_0_var(--docs-content-track-width)_var(--docs-toc-width)_minmax(min-content,1fr)]" + ) + + const [collection] = slug + + if (!collection) { + return null + } + + const [availableCollections, entry, navigationItems, breadcrumbItems] = + await Promise.all([ + rootCollections(), + getDocumentationEntryBySlug(slug), + getCollectionNavigation(collection), + getBreadcrumbItems(slug), + ]) + + const currentCollection = availableCollections.find( + (item) => item.group === collection + ) + const favoriteItems = getFavoriteNavigationItems(navigationItems) + + let headings: ContentSection[] = [] + + if (isFile(entry)) { + const fileContent = await getFileContent(entry) + const frontmatter = await getMetadata(fileContent) + headings = (await fileContent?.getSections()) ?? [] + + if (frontmatter?.apiReference && frontmatter.apiReference.length > 0) { + const results = await fetchApiReferenceTocItems(frontmatter.apiReference) + const hasMultipleSources = results.length > 1 + const referenceSections = hasMultipleSources + ? results.map((result) => ({ + children: syntacticExportsToTocItems(result.exports, 4), + depth: 3, + id: result.name.replaceAll(/[^a-z0-9-]/gu, "-"), + title: result.name, + })) + : syntacticExportsToTocItems( + results.flatMap((r) => r.exports), + 3 + ) + + headings = [ + ...headings, + + ...(referenceSections.length + ? [ + { + children: referenceSections, + depth: 2, + id: "api-reference", + title: "API Reference", + }, + ] + : []), + ] + } + + if (frontmatter?.cliCommand) { + const cliSections = getCliCommandTocSections(frontmatter.cliCommand) + // Replace any static "Options"/"Arguments" headings from the MDX + // with the dynamic ones that have individual items as children + headings = [ + ...headings.filter((h) => h.id !== "options" && h.id !== "arguments"), + ...cliSections, + ] + } + } + + return ( +
+ + + + + } + /> + + + + {headings.length > 0 && } +
+
+ + {children} +
+
+ + + +
+ ) +} diff --git a/apps/docs/src/app/docs/[...slug]/page.tsx b/apps/docs/src/app/docs/[...slug]/page.tsx new file mode 100644 index 00000000..c243ffab --- /dev/null +++ b/apps/docs/src/app/docs/[...slug]/page.tsx @@ -0,0 +1,329 @@ +import type { Metadata } from "next" +import { Space_Grotesk } from "next/font/google" +import { notFound } from "next/navigation" +import { isDirectory, isFile, MDX } from "renoun" + +import { + getFileContent, + getDocumentationEntryBySlug, + getMetadata, + getSections, + getTitle, + staticRoutes, + getBreadcrumbItems, + getEntryFrontmatter, +} from "@/collection-helpers" +import type { EntryType } from "@/collection-helpers" +import { DocsPageActions } from "@/components/docs-page-actions" +import { ApiReference } from "@/components/mdx/api-reference" +import SectionGrid from "@/components/section-grid" +import Siblings from "@/components/siblings" +import { cn } from "@/lib/utils" +import { toRawHref } from "@/shared/doc-paths" + +const spaceGrotesk = Space_Grotesk({ subsets: ["latin"] }) +const SITE_URL = "https://vorsteh-queue.dev" + +function toJsonLd(value: unknown) { + return JSON.stringify(value).replaceAll(" ({ slug })) +} +export async function generateMetadata( + params: PageProps<"/docs/[...slug]"> +): Promise { + const { slug } = await params.params + const [collection] = slug + + let entry: EntryType | null = null + + try { + entry = await getDocumentationEntryBySlug(slug) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch { + entry = null + } + const breadcrumbItems = await getBreadcrumbItems(slug) + const metadata = entry ? await getEntryFrontmatter(entry) : null + + const titles = breadcrumbItems.map((ele) => ele.title) + + const alternateTypes: NonNullable["types"] = { + "application/x-ndjson": "/docs.snapshot.jsonl", + } + + if (collection) { + alternateTypes["application/x-ndjson+collection"] = + `/docs.snapshot/${collection}.jsonl` + } + + return { + alternates: { + types: alternateTypes, + }, + description: metadata?.description ?? "", + title: `${titles.join(" - ")}`, + } +} +export default async function DocsPage({ + params, +}: PageProps<"/docs/[...slug]">) { + const { slug } = await params + const normalizedSlug = slug.filter((segment) => segment !== "index") + const docsPath = + normalizedSlug.length > 0 ? `/docs/${normalizedSlug.join("/")}` : "/docs" + const pageUrl = `${SITE_URL}${docsPath}` + + let entry: EntryType | null = null + try { + entry = await getDocumentationEntryBySlug(slug) + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch { + notFound() + } + + const sections = await getSections(entry) + // oxlint-disable-next-line react-doctor/server-sequential-independent-await -- acceptable for readability + const breadcrumbItems = await getBreadcrumbItems(slug) + const breadcrumbJsonLd = { + "@context": "https://schema.org", + "@type": "BreadcrumbList", + itemListElement: [ + { + "@type": "ListItem", + item: `${SITE_URL}/docs`, + name: "Docs", + position: 1, + }, + ...breadcrumbItems.map((item, index) => ({ + "@type": "ListItem", + item: `${SITE_URL}/docs/${item.path.join("/")}`, + name: item.title, + position: index + 2, + })), + ], + } + + if (!isFile(entry) && isDirectory(entry)) { + const pageJsonLd = { + "@context": "https://schema.org", + "@type": "WebPage", + inLanguage: "en", + isPartOf: { + "@id": SITE_URL, + }, + name: getTitle(entry), + url: pageUrl, + } + + return ( +
+ {/* oxlint-disable react/no-danger -- rendered HTML content from trusted source */} +