From 1e73cb89d870d9841e68800c902f191ff1cc5ea9 Mon Sep 17 00:00:00 2001 From: Pete Koomen Date: Tue, 21 Jul 2026 21:19:28 -0700 Subject: [PATCH 1/2] Add create-scratchwork-server: npm create scaffolding for self-hosted servers New create/ workspace publishing the create-scratchwork-server package, so `npm create scratchwork-server my-server -- --platform cloudflare` scaffolds a standalone self-hosted server project (platforms: cloudflare, aws, local). - Templates are generated from the real deploy/* projects at build/pack time (create/generate-templates.ts) and never committed, so template content structurally cannot drift from the deploy sources; @scratchwork/* deps are pinned to the lockstep version read at staging time (set-version keeps it stamped via the root manifest). - The bin is non-interactive with args alone (agents first); it prompts for the platform only on a TTY. - Joins the publish flows: PUBLISHABLE staging order, tarball verification, lockstep version check, publish order, RELEASING.md. - check-npm-pack now also runs the packed bin under plain Node to scaffold every template into the hermetic consumer (staged tarballs, no network) and typechecks each scaffolded project; it also no longer symlinks packed package names into the consumer, which would overwrite workspace sources through the repo node_modules symlink. - Docs: scratchwork.dev/www/index.md self-hosting section now shows the real commands, server/README.md leads with the scaffolder, AGENTS.md workspace map gains create/. The regenerated default-renderer.generated.js hash follows the bun.lock change (new workspace), as expected. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 10 +- CHANGELOG.md | 7 + RELEASING.md | 6 +- bun.lock | 39 +++-- create/README.md | 32 ++++ create/generate-templates.ts | 116 +++++++++++++ create/package.json | 31 ++++ create/src/index.ts | 128 +++++++++++++++ create/src/scaffold.ts | 75 +++++++++ create/templates-src/aws/README.md | 42 +++++ create/templates-src/aws/_gitignore | 3 + create/templates-src/cloudflare/README.md | 43 +++++ create/templates-src/cloudflare/_gitignore | 3 + create/templates-src/local/README.md | 43 +++++ create/templates-src/local/_gitignore | 3 + create/test/scaffold.test.ts | 153 ++++++++++++++++++ create/tsconfig.build.json | 15 ++ create/tsconfig.json | 20 +++ package.json | 1 + scratchwork.dev/www/index.md | 14 +- scripts/build-packages.ts | 45 ++++-- scripts/check-npm-pack.ts | 83 +++++++++- scripts/publish-packages.ts | 4 +- server/README.md | 35 ++-- shared/src/site/default-renderer.generated.js | 2 +- 25 files changed, 899 insertions(+), 54 deletions(-) create mode 100644 create/README.md create mode 100644 create/generate-templates.ts create mode 100644 create/package.json create mode 100755 create/src/index.ts create mode 100644 create/src/scaffold.ts create mode 100644 create/templates-src/aws/README.md create mode 100644 create/templates-src/aws/_gitignore create mode 100644 create/templates-src/cloudflare/README.md create mode 100644 create/templates-src/cloudflare/_gitignore create mode 100644 create/templates-src/local/README.md create mode 100644 create/templates-src/local/_gitignore create mode 100644 create/test/scaffold.test.ts create mode 100644 create/tsconfig.build.json create mode 100644 create/tsconfig.json diff --git a/AGENTS.md b/AGENTS.md index 3a97cd3..af1f1ab 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,8 +5,8 @@ Scratchwork is tool for publishing static HTML and Markdown artifacts publicly a ## Workspace map Package names are `@scratchwork/` for everything under `server/` and `deploy/` -(plus `@scratchwork/shared` and `@scratchwork/e2e`); the two odd ones out are -`scratchwork-renderer` and `scratchwork-cli`. +(plus `@scratchwork/shared` and `@scratchwork/e2e`); the odd ones out are +`scratchwork-renderer`, `scratchwork-cli`, and `create-scratchwork-server`. - `renderer/` — build-once pipeline producing the single-file universal renderer (`index.html`). **Deliberately plain browser JS — the one exception to invariant 1.** @@ -23,6 +23,12 @@ Package names are `@scratchwork/` for everything under `server/` and `deplo - `server/deploy-local/` — local Bun deploy target. - `deploy/*` — one deployment project per domain (generic-aws, cloudflare-vanilla, cloudflare-access, local-dev), each deployable with one command. +- `create/` — the `create-scratchwork-server` package behind + `npm create scratchwork-server`: scaffolds a standalone self-hosted server + project. Its platform templates are generated from the `deploy/*` sources at + pack time (`create/generate-templates.ts`), never committed. Plain + Effect-free TypeScript, like `scripts/` — the shipped bin must run under + npm's Node. - `scratchwork.dev/` — the scratchwork.dev site: `server/` is its Cloudflare deployment workspace (same shape as `deploy/*`), `www/` the homepage project published to it (content, not a workspace). diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa2a65..cf12842 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ top released section into the GitHub Release notes (scripts/release-notes.ts). ## Unreleased +- `npm create scratchwork-server` — new `create-scratchwork-server` package + that scaffolds a standalone self-hosted server project + (`--platform cloudflare | aws | local`). Templates are generated from the + repo's `deploy/*` projects at pack time with `@scratchwork/*` dependencies + pinned to the lockstep version, and every template is scaffolded and + typechecked hermetically in ci. + ## v0.2.0 - Distribution: cross-platform CLI binaries on GitHub Releases, `install.sh` / diff --git a/RELEASING.md b/RELEASING.md index 3d90728..a0b4993 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -18,8 +18,10 @@ with confirmation gates at the irreversible steps: 5. **npm:** from a clean checkout of the tag, with an authenticated npm CLI: `bun scripts/publish-packages.ts` (add `--dry-run` first if in doubt; with npm 2FA, add `--otp `). - Publishes `@scratchwork/shared`, `@scratchwork/server-core`, and the three - `@scratchwork/server-deploy-*` packages in dependency order. + Publishes `@scratchwork/shared`, `@scratchwork/server-core`, the three + `@scratchwork/server-deploy-*` packages, and `create-scratchwork-server` + (the `npm create scratchwork-server` scaffolder, with its platform + templates generated from `deploy/*` at pack time) in dependency order. 6. **Homepage:** republish the install entry points so `https://scratchwork.dev/install.sh` serves the current content: `scratchwork publish scratchwork.dev/www --project www --public` (with the diff --git a/bun.lock b/bun.lock index 3580827..4881b24 100644 --- a/bun.lock +++ b/bun.lock @@ -11,7 +11,7 @@ }, "cli": { "name": "scratchwork-cli", - "version": "0.1.0", + "version": "0.2.0", "bin": { "scratchwork": "./src/index.ts", }, @@ -30,9 +30,20 @@ "typescript": "^6.0.3", }, }, + "create": { + "name": "create-scratchwork-server", + "version": "0.2.0", + "bin": { + "create-scratchwork-server": "./src/index.ts", + }, + "devDependencies": { + "@types/bun": "^1.3.14", + "typescript": "^6.0.3", + }, + }, "deploy/cloudflare-access": { "name": "@scratchwork/deploy-cloudflare-access", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@scratchwork/server-deploy-cloudflare": "workspace:*", }, @@ -43,7 +54,7 @@ }, "deploy/cloudflare-vanilla": { "name": "@scratchwork/deploy-cloudflare-vanilla", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@scratchwork/server-deploy-cloudflare": "workspace:*", }, @@ -54,7 +65,7 @@ }, "deploy/generic-aws": { "name": "@scratchwork/deploy-generic-aws", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@scratchwork/server-deploy-aws": "workspace:*", "@scratchwork/server-deploy-local": "workspace:*", @@ -66,7 +77,7 @@ }, "deploy/local-dev": { "name": "@scratchwork/deploy-local-dev", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@scratchwork/server-deploy-local": "workspace:*", }, @@ -77,7 +88,7 @@ }, "e2e": { "name": "@scratchwork/e2e", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@aws-sdk/client-dynamodb": "3.1075.0", "@aws-sdk/client-s3": "3.1075.0", @@ -95,7 +106,7 @@ }, "renderer": { "name": "scratchwork-renderer", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "htm": "^3.1.1", "prismjs": "^1.29.0", @@ -108,7 +119,7 @@ }, "scratchwork.dev/server": { "name": "@scratchwork/deploy-scratchwork-dev", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@scratchwork/server-deploy-cloudflare": "workspace:*", }, @@ -119,7 +130,7 @@ }, "server/core": { "name": "@scratchwork/server-core", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@effect/platform": "0.96.2", "@scratchwork/shared": "workspace:*", @@ -132,7 +143,7 @@ }, "server/deploy-aws": { "name": "@scratchwork/server-deploy-aws", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@aws-sdk/client-dynamodb": "3.1075.0", "@aws-sdk/client-s3": "3.1075.0", @@ -148,7 +159,7 @@ }, "server/deploy-cloudflare": { "name": "@scratchwork/server-deploy-cloudflare", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@effect/platform": "0.96.2", "@scratchwork/server-core": "workspace:*", @@ -164,7 +175,7 @@ }, "server/deploy-local": { "name": "@scratchwork/server-deploy-local", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@effect/platform": "0.96.2", "@effect/platform-bun": "0.90.0", @@ -178,7 +189,7 @@ }, "shared": { "name": "@scratchwork/shared", - "version": "0.1.0", + "version": "0.2.0", "dependencies": { "@effect/platform": "0.96.2", "effect": "3.21.4", @@ -522,6 +533,8 @@ "cookie": ["cookie@1.1.1", "", {}, "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ=="], + "create-scratchwork-server": ["create-scratchwork-server@workspace:create"], + "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], diff --git a/create/README.md b/create/README.md new file mode 100644 index 0000000..f2e27bf --- /dev/null +++ b/create/README.md @@ -0,0 +1,32 @@ +# create-scratchwork-server + +Scaffolds a standalone, self-hosted +[Scratchwork](https://github.com/scratch/scratchwork) publishing server +project: + +```sh +npm create scratchwork-server my-server -- --platform cloudflare +``` + +Platforms: + +- `cloudflare` — a Cloudflare Worker backed by R2 and D1 +- `aws` — an AWS Lambda Function URL backed by S3 and DynamoDB +- `local` — a single-machine Bun server with local file storage + +The scaffolded project depends on the matching published +`@scratchwork/server-deploy-*` package (pinned to this package's version) and +contains the same files as the Scratchwork repository's own deploy projects: +a `server-config.ts` for domains and auth policy, platform configuration, +`deploy.ts` / `local.ts` entrypoints, a `.env.example` for secrets, and a +README with setup instructions. + +```sh +cd my-server +bun install +bun run local # run the server locally +bun run deploy # deploy it (cloudflare and aws) +``` + +The scaffolder is non-interactive when given a directory and `--platform` +(it prompts for the platform only on a TTY). Run with `--help` for usage. diff --git a/create/generate-templates.ts b/create/generate-templates.ts new file mode 100644 index 0000000..cd9d744 --- /dev/null +++ b/create/generate-templates.ts @@ -0,0 +1,116 @@ +/* + * Generates the create-scratchwork-server platform templates from the real + * deploy/* projects. Templates are never committed or hand-maintained: they + * are regenerated from the deploy sources on every build/pack + * (scripts/build-packages.ts) and in this workspace's tests, so template + * content structurally cannot drift from the deploy projects (AGENTS.md + * invariant 2's spirit). + * + * Per platform, the transform is mechanical: + * - every committed file of the deploy project is copied with the + * sndbx.sh-specific values replaced by example.com placeholders; + * - package.json becomes a standalone project manifest: workspace:* deps + * pinned to the exact lockstep version passed in, repo-only scripts + * (ci/test) dropped, name set to a placeholder the scaffolder overwrites; + * - tsconfig.json drops the ../../shared includes that only make sense + * inside this repository; + * - README.md is replaced (the deploy READMEs describe this repo's own + * deployments) and a gitignore is added, both from templates-src/; + * - dotfiles are stored with a leading underscore (npm strips or repurposes + * some dotfiles in packed packages); scaffold() renames them back. + * + * Dev/build tooling only — this file is not shipped in the npm package. + */ +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))); +const templatesSrc = join(repoRoot, "create", "templates-src"); + +/** Template platform → the deploy project it is generated from (repo-relative). */ +export const TEMPLATE_SOURCES = { + cloudflare: "deploy/cloudflare-vanilla", + aws: "deploy/generic-aws", + local: "deploy/local-dev", +} as const; + +export type TemplatePlatform = keyof typeof TEMPLATE_SOURCES; + +/** Placeholder name in template package.json; scaffold() overwrites it. */ +export const TEMPLATE_PROJECT_NAME = "scratchwork-server"; + +/** Replaces the sndbx.sh-specific values in deploy sources with placeholders. */ +function substitutePlaceholders(text: string): string { + return text.replaceAll("sndbx.sh", "example.com").replaceAll("sndbx-sh", "example-com"); +} + +/** The standalone-project package.json derived from a deploy project's. */ +function templateManifest(source: Record, version: string): Record { + const scripts = Object.fromEntries( + Object.entries((source.scripts as Record) ?? {}).filter(([name]) => name !== "ci" && name !== "test"), + ); + const dependencies = Object.fromEntries( + Object.entries((source.dependencies as Record) ?? {}).map(([name, range]) => [ + name, + range === "workspace:*" ? version : range, + ]), + ); + return { + name: TEMPLATE_PROJECT_NAME, + version: "0.0.0", + private: true, + type: source.type, + scripts, + dependencies, + devDependencies: source.devDependencies, + }; +} + +/** The standalone tsconfig: same options, minus includes reaching into this repo. */ +function templateTsconfig(source: { compilerOptions: unknown; include: string[] }): Record { + return { + compilerOptions: source.compilerOptions, + include: source.include.filter((pattern) => !pattern.startsWith("..")), + }; +} + +/** + * Writes the templates for every platform under outDir, pinning @scratchwork + * dependencies to `version` (the repo's lockstep version at build/pack time). + */ +export function generateTemplates(outDir: string, version: string): void { + for (const [platform, sourceDir] of Object.entries(TEMPLATE_SOURCES)) { + const source = join(repoRoot, sourceDir); + const out = join(outDir, platform); + rmSync(out, { recursive: true, force: true }); + mkdirSync(out, { recursive: true }); + + for (const entry of readdirSync(source, { withFileTypes: true })) { + if (!entry.isFile()) continue; // deploy projects are flat; local state dirs are skipped + const name = entry.name; + if (name === "README.md") continue; // replaced by templates-src (repo-specific prose) + if (name.startsWith(".") && name !== ".env.example") continue; // .env, editor droppings + const text = readFileSync(join(source, name), "utf8"); + if (name === "package.json") { + writeFileSync(join(out, name), JSON.stringify(templateManifest(JSON.parse(text), version), null, 2) + "\n"); + } else if (name === "tsconfig.json") { + writeFileSync(join(out, name), JSON.stringify(templateTsconfig(JSON.parse(text)), null, 2) + "\n"); + } else { + const outName = name.startsWith(".") ? "_" + name.slice(1) : name; + writeFileSync(join(out, outName), substitutePlaceholders(text)); + } + } + + // Hand-written standalone extras: README.md and _gitignore per platform. + const extras = join(templatesSrc, platform); + for (const name of readdirSync(extras)) { + writeFileSync(join(out, name), readFileSync(join(extras, name), "utf8")); + } + for (const required of ["README.md", "_gitignore", "package.json", "tsconfig.json"]) { + if (!existsSync(join(out, required))) { + throw new Error(`generate-templates: ${platform} template is missing ${required}`); + } + } + } +} diff --git a/create/package.json b/create/package.json new file mode 100644 index 0000000..2bde4f4 --- /dev/null +++ b/create/package.json @@ -0,0 +1,31 @@ +{ + "name": "create-scratchwork-server", + "version": "0.2.0", + "type": "module", + "description": "Scaffolds a self-hosted Scratchwork publishing server project (Cloudflare, AWS, or local).", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/scratch/scratchwork.git", + "directory": "create" + }, + "engines": { + "node": ">=22" + }, + "files": [ + "dist", + "templates" + ], + "bin": { + "create-scratchwork-server": "./src/index.ts" + }, + "scripts": { + "ci": "bun run typecheck && bun run test", + "test": "bun test", + "typecheck": "tsc -p tsconfig.json" + }, + "devDependencies": { + "@types/bun": "^1.3.14", + "typescript": "^6.0.3" + } +} diff --git a/create/src/index.ts b/create/src/index.ts new file mode 100755 index 0000000..d5252b0 --- /dev/null +++ b/create/src/index.ts @@ -0,0 +1,128 @@ +#!/usr/bin/env node +/* + * create-scratchwork-server — the bin behind `npm create scratchwork-server`. + * Scaffolds a standalone self-hosted Scratchwork server project from a + * platform template (cloudflare | aws | local): + * + * npm create scratchwork-server my-server -- --platform cloudflare + * bun create scratchwork-server my-server --platform cloudflare + * + * Non-interactive by design (agents are a primary audience): with a directory + * and --platform it never prompts; without --platform it prompts only on a + * TTY and errors otherwise. Runs under plain Node — node: builtins only. + */ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { createInterface } from "node:readline/promises"; +import { fileURLToPath } from "node:url"; +import { listPlatforms, scaffold } from "./scaffold.ts"; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const templatesDir = join(packageRoot, "templates"); +const ownVersion = (JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")) as { version: string }).version; + +const DEFAULT_DIR = "scratchwork-server"; + +function usage(platforms: readonly string[]): string { + return [ + "Usage: npm create scratchwork-server [directory] -- --platform ", + "", + ` directory target directory (default: ${DEFAULT_DIR})`, + ` --platform, -p one of: ${platforms.join(", ")}`, + " --help, -h show this help", + " --version, -v print the version", + "", + "Example:", + " npm create scratchwork-server my-server -- --platform cloudflare", + ].join("\n"); +} + +interface ParsedArgs { + readonly directory: string | undefined; + readonly platform: string | undefined; + readonly help: boolean; + readonly version: boolean; +} + +function parseArgs(argv: readonly string[]): ParsedArgs { + let directory: string | undefined; + let platform: string | undefined; + let help = false; + let version = false; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--") continue; + if (arg === "--help" || arg === "-h") help = true; + else if (arg === "--version" || arg === "-v") version = true; + else if (arg === "--platform" || arg === "-p") platform = argv[++i]; + else if (arg.startsWith("--platform=")) platform = arg.slice("--platform=".length); + else if (arg.startsWith("-")) throw new Error(`unknown option "${arg}"`); + else if (directory === undefined) directory = arg; + else throw new Error(`unexpected argument "${arg}"`); + } + return { directory, platform, help, version }; +} + +async function promptPlatform(platforms: readonly string[]): Promise { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer = await rl.question(`Platform (${platforms.join(", ")}): `); + return answer.trim(); + } finally { + rl.close(); + } +} + +async function main(): Promise { + const platforms = listPlatforms(templatesDir); + let args: ParsedArgs; + try { + args = parseArgs(process.argv.slice(2)); + } catch (error) { + console.error(`create-scratchwork-server: ${(error as Error).message}\n`); + console.error(usage(platforms)); + return 1; + } + if (args.help) { + console.log(usage(platforms)); + return 0; + } + if (args.version) { + console.log(ownVersion); + return 0; + } + let platform = args.platform; + if (platform === undefined || platform === "") { + if (process.stdin.isTTY && process.stdout.isTTY) { + platform = await promptPlatform(platforms); + } else { + console.error("create-scratchwork-server: --platform is required when not running interactively\n"); + console.error(usage(platforms)); + return 1; + } + } + + const directory = args.directory ?? DEFAULT_DIR; + let result; + try { + result = scaffold(templatesDir, platform, directory); + } catch (error) { + console.error(`create-scratchwork-server: ${(error as Error).message}`); + return 1; + } + + const steps = [ + `cd ${directory}`, + "bun install", + ...(result.hasEnvExample ? ["cp .env.example .env # then fill in the secrets"] : []), + ...(result.scripts.includes("local") ? ["bun run local # run the server locally"] : []), + ...(result.scripts.includes("deploy") ? ["bun run deploy # deploy it"] : []), + ]; + console.log(`Scaffolded a Scratchwork ${platform} server project in ${directory}/`); + console.log("\nNext steps:\n"); + for (const step of steps) console.log(` ${step}`); + console.log("\nSee the project's README.md for configuration details."); + return 0; +} + +process.exit(await main()); diff --git a/create/src/scaffold.ts b/create/src/scaffold.ts new file mode 100644 index 0000000..279bedb --- /dev/null +++ b/create/src/scaffold.ts @@ -0,0 +1,75 @@ +/* + * Core scaffolding logic for create-scratchwork-server: copies one platform + * template into a target directory and personalizes it. Kept separate from + * the bin entrypoint so tests can drive it directly against generated + * templates. Must run under plain Node — node: builtins only, no Bun APIs. + * + * Template convention: files that must be dotfiles in the scaffolded project + * are stored with a leading underscore (`_env.example`, `_gitignore`) because + * npm strips or repurposes some dotfiles when packing/installing a package. + * scaffold() renames a leading `_` back to `.`. + */ +import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, statSync, writeFileSync } from "node:fs"; +import { basename, join, resolve } from "node:path"; + +/** What was scaffolded, for the bin's next-steps output. */ +export interface ScaffoldResult { + /** Absolute path of the scaffolded project. */ + readonly dir: string; + /** The npm package name written into the project's package.json. */ + readonly projectName: string; + /** Script names available in the scaffolded package.json (e.g. local, deploy). */ + readonly scripts: readonly string[]; + /** Whether the template ships a .env.example to copy to .env. */ + readonly hasEnvExample: boolean; +} + +/** The platforms available in a templates directory (one subdirectory each). */ +export function listPlatforms(templatesDir: string): string[] { + return readdirSync(templatesDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); +} + +/** A valid npm package name derived from the target directory's basename. */ +export function projectNameFor(targetDir: string): string { + const name = basename(resolve(targetDir)) + .toLowerCase() + .replace(/[^a-z0-9-._~]+/g, "-") + .replace(/^[-._]+/, ""); + return name === "" ? "scratchwork-server" : name; +} + +/** Copies the platform template into targetDir and personalizes package.json. */ +export function scaffold(templatesDir: string, platform: string, targetDir: string): ScaffoldResult { + const platforms = listPlatforms(templatesDir); + if (!platforms.includes(platform)) { + throw new Error(`unknown platform "${platform}" — expected one of: ${platforms.join(", ")}`); + } + const target = resolve(targetDir); + if (existsSync(target)) { + if (!statSync(target).isDirectory()) throw new Error(`${target} exists and is not a directory`); + if (readdirSync(target).length > 0) throw new Error(`${target} already exists and is not empty`); + } + mkdirSync(target, { recursive: true }); + cpSync(join(templatesDir, platform), target, { recursive: true }); + + // Restore dotfiles stored under the leading-underscore convention. + for (const entry of readdirSync(target)) { + if (entry.startsWith("_")) renameSync(join(target, entry), join(target, "." + entry.slice(1))); + } + + const manifestPath = join(target, "package.json"); + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + const projectName = projectNameFor(target); + manifest.name = projectName; + writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n"); + + return { + dir: target, + projectName, + scripts: Object.keys((manifest.scripts as Record) ?? {}), + hasEnvExample: existsSync(join(target, ".env.example")), + }; +} diff --git a/create/templates-src/aws/README.md b/create/templates-src/aws/README.md new file mode 100644 index 0000000..a0ecc47 --- /dev/null +++ b/create/templates-src/aws/README.md @@ -0,0 +1,42 @@ +# Scratchwork server on AWS + +A self-hosted [Scratchwork](https://github.com/scratch/scratchwork) publishing +server, deployed as an AWS Lambda Function URL backed by S3 and DynamoDB. + +## Configure + +1. Edit `server-config.ts` — auth policy and any fixed settings. Settings not + listed there come from `SCRATCHWORK_*` environment variables. +2. Copy `.env.example` to `.env` and fill in the public URLs, the Google OAuth + credentials, and the session secret (the comments in `.env.example` explain + each value). + +Configure your Google OAuth app with the redirect URI +`https:///auth/callback/google`. + +## Run locally + +```sh +bun install +bun run local +``` + +This runs the same server settings locally with local file storage and an +in-memory database (no AWS resources). For local browser logins, add +`http://localhost:43118/auth/callback/google` as a second redirect URI on the +Google OAuth client. + +## Deploy + +```sh +bun run deploy +``` + +The deploy uses the AWS credentials in your environment and creates or +updates the S3 bucket, an IAM role, the Lambda function, and a public Lambda +Function URL. Then publish to your server with the +[Scratchwork CLI](https://github.com/scratch/scratchwork): + +```sh +scratchwork publish --server +``` diff --git a/create/templates-src/aws/_gitignore b/create/templates-src/aws/_gitignore new file mode 100644 index 0000000..d51a045 --- /dev/null +++ b/create/templates-src/aws/_gitignore @@ -0,0 +1,3 @@ +node_modules/ +.env +.scratchwork-local-data/ diff --git a/create/templates-src/cloudflare/README.md b/create/templates-src/cloudflare/README.md new file mode 100644 index 0000000..c34c934 --- /dev/null +++ b/create/templates-src/cloudflare/README.md @@ -0,0 +1,43 @@ +# Scratchwork server on Cloudflare + +A self-hosted [Scratchwork](https://github.com/scratch/scratchwork) publishing +server, deployed as a Cloudflare Worker backed by R2 (published content) and +D1 (project metadata). + +## Configure + +1. Edit `server-config.ts` — your domains and auth policy. The scaffolded + values are `example.com` placeholders. +2. Edit `cloudflare-config.ts` — Worker name, R2 bucket, D1 database, zone, + and routes. +3. Copy `.env.example` to `.env` and fill in the Cloudflare API token and the + auth secrets (the comments in `.env.example` explain each value). + +Configure your Google OAuth app with the redirect URI +`https:///auth/callback/google`. + +## Run locally + +```sh +bun install +bun run local +``` + +This runs the same Worker locally with Wrangler's persistent R2 and D1 +simulations (state lives under `.scratchwork-cloudflare-data/`). To complete a +browser login locally, add `http://localhost:8787/auth/callback/google` as a +second redirect URI on the Google OAuth client. Set `PORT` to change the port. + +## Deploy + +```sh +bun run deploy +``` + +This creates or updates the R2 bucket, D1 database, Worker, bindings, and +routes through the Cloudflare API. Then publish to your server with the +[Scratchwork CLI](https://github.com/scratch/scratchwork): + +```sh +scratchwork publish --server https:// +``` diff --git a/create/templates-src/cloudflare/_gitignore b/create/templates-src/cloudflare/_gitignore new file mode 100644 index 0000000..db52bfa --- /dev/null +++ b/create/templates-src/cloudflare/_gitignore @@ -0,0 +1,3 @@ +node_modules/ +.env +.scratchwork-cloudflare-data/ diff --git a/create/templates-src/local/README.md b/create/templates-src/local/README.md new file mode 100644 index 0000000..aabbdd9 --- /dev/null +++ b/create/templates-src/local/README.md @@ -0,0 +1,43 @@ +# Scratchwork server (local) + +A self-hosted [Scratchwork](https://github.com/scratch/scratchwork) publishing +server that runs on a single machine under Bun, with local file storage and an +in-memory database. There is no cloud counterpart and no deploy step. + +## Configure + +Auth is required and cannot be disabled. Provide Google OAuth credentials and +a session secret in the shell environment: + +```sh +SCRATCHWORK_GOOGLE_CLIENT_ID=... +SCRATCHWORK_GOOGLE_CLIENT_SECRET=... +SCRATCHWORK_SESSION_SECRET=... # at least 32 bytes; openssl rand -base64 48 +``` + +Configure your Google OAuth app with the redirect URI +`http://localhost:43118/auth/callback/google`. + +Useful environment variables: + +```sh +PORT=43118 +SCRATCHWORK_STORAGE_DIR=.scratchwork-local-data +``` + +Edit `local.ts` to change fixed server settings (allowed users, project +naming, and so on). + +## Run + +```sh +bun install +bun run local +``` + +Then publish to it with the +[Scratchwork CLI](https://github.com/scratch/scratchwork): + +```sh +scratchwork publish --server http://localhost:43118 +``` diff --git a/create/templates-src/local/_gitignore b/create/templates-src/local/_gitignore new file mode 100644 index 0000000..d51a045 --- /dev/null +++ b/create/templates-src/local/_gitignore @@ -0,0 +1,3 @@ +node_modules/ +.env +.scratchwork-local-data/ diff --git a/create/test/scaffold.test.ts b/create/test/scaffold.test.ts new file mode 100644 index 0000000..0725802 --- /dev/null +++ b/create/test/scaffold.test.ts @@ -0,0 +1,153 @@ +/* + * Tests the template generation + scaffolding loop against the real deploy/* + * sources: templates are generated exactly as build-packages.ts stages them, + * then scaffolded the way the published bin does it. The hermetic + * pack-install-typecheck of every scaffolded template runs in the root gate + * (scripts/check-npm-pack.ts); these tests cover the scaffolder's own + * behavior and the template transform's guarantees. + */ +import { afterAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { generateTemplates, TEMPLATE_SOURCES } from "../generate-templates.ts"; +import { listPlatforms, projectNameFor, scaffold } from "../src/scaffold.ts"; + +const workDir = mkdtempSync(join(tmpdir(), "create-scratchwork-test-")); +const templatesDir = join(workDir, "templates"); +const version = "9.9.9-test"; +generateTemplates(templatesDir, version); + +afterAll(() => rmSync(workDir, { recursive: true, force: true })); + +const platforms = Object.keys(TEMPLATE_SOURCES).sort(); + +describe("generateTemplates", () => { + test("produces one template per platform", () => { + expect(listPlatforms(templatesDir)).toEqual(platforms); + }); + + test.each(platforms)("%s: pins @scratchwork deps to the exact version, no workspace ranges", (platform) => { + const manifest = JSON.parse(readFileSync(join(templatesDir, platform, "package.json"), "utf8")); + const dependencies = Object.entries(manifest.dependencies as Record); + expect(dependencies.length).toBeGreaterThan(0); + for (const [name, range] of dependencies) { + if (name.startsWith("@scratchwork/")) expect(range).toBe(version); + expect(range).not.toContain("workspace:"); + } + expect(manifest.private).toBe(true); + expect(manifest.scripts.ci).toBeUndefined(); + expect(manifest.scripts.test).toBeUndefined(); + expect(manifest.scripts.typecheck).toBeDefined(); + expect(manifest.scripts.local).toBeDefined(); + }); + + test.each(platforms)("%s: tsconfig does not reach into the repository", (platform) => { + const tsconfig = JSON.parse(readFileSync(join(templatesDir, platform, "tsconfig.json"), "utf8")); + expect(tsconfig.include.length).toBeGreaterThan(0); + for (const pattern of tsconfig.include as string[]) { + expect(pattern.startsWith("..")).toBe(false); + } + }); + + test.each(platforms)("%s: no sndbx.sh values or bare dotfiles remain", (platform) => { + for (const name of readdirSync(join(templatesDir, platform))) { + expect(name.startsWith(".")).toBe(false); + const text = readFileSync(join(templatesDir, platform, name), "utf8"); + expect(text).not.toContain("sndbx"); + } + }); + + test.each(platforms)("%s: template carries every code file of its deploy project", (platform) => { + const sourceDir = join(import.meta.dir, "..", "..", TEMPLATE_SOURCES[platform as keyof typeof TEMPLATE_SOURCES]); + const codeFiles = readdirSync(sourceDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".ts")) + .map((entry) => entry.name); + expect(codeFiles.length).toBeGreaterThan(0); + for (const name of codeFiles) { + expect(existsSync(join(templatesDir, platform, name))).toBe(true); + } + }); +}); + +describe("scaffold", () => { + test("materializes dotfiles and personalizes package.json", () => { + const target = join(workDir, "My Server!"); + const result = scaffold(templatesDir, "cloudflare", target); + expect(result.projectName).toBe("my-server-"); + expect(existsSync(join(target, ".gitignore"))).toBe(true); + expect(existsSync(join(target, ".env.example"))).toBe(true); + expect(readdirSync(target).some((name) => name.startsWith("_"))).toBe(false); + const manifest = JSON.parse(readFileSync(join(target, "package.json"), "utf8")); + expect(manifest.name).toBe("my-server-"); + expect(result.scripts).toContain("deploy"); + expect(result.scripts).toContain("local"); + }); + + test("local platform scaffolds without a deploy script or env example", () => { + const result = scaffold(templatesDir, "local", join(workDir, "local-server")); + expect(result.scripts).not.toContain("deploy"); + expect(result.hasEnvExample).toBe(false); + expect(result.scripts).toContain("local"); + }); + + test("refuses a non-empty target directory", () => { + const target = join(workDir, "occupied"); + scaffold(templatesDir, "aws", target); + expect(() => scaffold(templatesDir, "aws", target)).toThrow(/not empty/); + }); + + test("rejects an unknown platform", () => { + expect(() => scaffold(templatesDir, "vercel", join(workDir, "nope"))).toThrow(/unknown platform/); + }); + + test("refuses a target that is an existing file", () => { + const file = join(workDir, "a-file"); + writeFileSync(file, "hi"); + expect(() => scaffold(templatesDir, "aws", file)).toThrow(/not a directory/); + }); +}); + +describe("projectNameFor", () => { + test("derives a valid npm name from the directory basename", () => { + expect(projectNameFor("/tmp/x/Cool_Server")).toBe("cool_server"); + expect(projectNameFor("...")).toBe("scratchwork-server"); + }); +}); + +describe("bin", () => { + // The templates dir the bin resolves (../templates from src) does not exist + // in the repo — templates are generated at pack time — so bin tests run + // against a copy of src laid out next to the generated templates, mirroring + // the published package (pkg/{src,templates,package.json}). + Bun.spawnSync(["mkdir", "-p", join(workDir, "pkg")]); + Bun.spawnSync(["cp", "-R", join(import.meta.dir, "..", "src"), join(workDir, "pkg", "src")]); + Bun.spawnSync(["cp", join(import.meta.dir, "..", "package.json"), join(workDir, "pkg", "package.json")]); + Bun.spawnSync(["cp", "-R", templatesDir, join(workDir, "pkg", "templates")]); + const run = (args: string[], cwd: string) => + Bun.spawnSync(["bun", join(workDir, "pkg", "src", "index.ts"), ...args], { cwd, stdout: "pipe", stderr: "pipe" }); + + test("scaffolds non-interactively with args alone", () => { + const cwd = join(workDir, "bin-run"); + Bun.spawnSync(["mkdir", "-p", cwd]); + const result = run(["agent-server", "--platform", "aws"], cwd); + expect(result.exitCode).toBe(0); + expect(existsSync(join(cwd, "agent-server", "deploy.ts"))).toBe(true); + expect(result.stdout.toString()).toContain("bun run deploy"); + }); + + test("fails with usage when --platform is missing and stdin is not a TTY", () => { + const cwd = join(workDir, "bin-fail"); + Bun.spawnSync(["mkdir", "-p", cwd]); + const result = run(["another-server"], cwd); + expect(result.exitCode).toBe(1); + expect(result.stderr.toString()).toContain("--platform is required"); + expect(existsSync(join(cwd, "another-server"))).toBe(false); + }); + + test("--help prints usage", () => { + const result = run(["--help"], workDir); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("npm create scratchwork-server"); + }); +}); diff --git a/create/tsconfig.build.json b/create/tsconfig.build.json new file mode 100644 index 0000000..c6902ba --- /dev/null +++ b/create/tsconfig.build.json @@ -0,0 +1,15 @@ +// Publish build (scripts/build-packages.ts): tsc emit into dist/. The emitted +// bin must run under plain Node (npm create executes it with the user's npm), +// so src/ sticks to node: builtins — no Bun APIs. +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "rewriteRelativeImportExtensions": true + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/create/tsconfig.json b/create/tsconfig.json new file mode 100644 index 0000000..6e3074b --- /dev/null +++ b/create/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "allowArbitraryExtensions": true, + "allowImportingTsExtensions": true, + "allowJs": true, + "baseUrl": ".", + "ignoreDeprecations": "6.0", + "lib": ["ESNext", "DOM"], + "module": "Preserve", + "moduleDetection": "force", + "moduleResolution": "bundler", + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "target": "ESNext", + "types": ["bun"], + "verbatimModuleSyntax": true + }, + "include": ["src/**/*.ts", "test/**/*.ts", "generate-templates.ts"] +} diff --git a/package.json b/package.json index 9022e5e..c5471d1 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "server/deploy-cloudflare", "server/deploy-local", "deploy/*", + "create", "scratchwork.dev/server", "e2e" ], diff --git a/scratchwork.dev/www/index.md b/scratchwork.dev/www/index.md index 1d687c8..e529360 100644 --- a/scratchwork.dev/www/index.md +++ b/scratchwork.dev/www/index.md @@ -77,25 +77,29 @@ If you'd like to be able to pay for a hosted option, upvote [this Github issue]( ## Hosting your own server -Scratchwork is open source and can be hosted anywhere. To configure your own server, use: +Scratchwork is open source and can be hosted anywhere. To scaffold your own server project, use: ```sh # Configure for Cloudflare (a worker using R2 and D1) -npm create scratchwork-server-cloudflare +npm create scratchwork-server my-server -- --platform cloudflare ``` -swap `cloudflare` for your favorite serverless platform: `aws`, `vercel`, `railway`, or `smolmachines`. +swap `cloudflare` for `aws` (Lambda + S3 + DynamoDB) or `local` (a single-machine Bun server). The scaffolded README covers the credentials and settings each platform needs. Run your server locally with ```sh -node local.ts +cd my-server +bun install +bun run local ``` and deploy it with ```sh -node deploy.ts +bun run deploy ``` +(the `local` platform runs locally only, so it has no deploy step). + diff --git a/scripts/build-packages.ts b/scripts/build-packages.ts index 2299374..defb936 100644 --- a/scripts/build-packages.ts +++ b/scripts/build-packages.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun /* - * Builds the five publishable npm packages (notes/distribution-plan.md + * Builds the six publishable npm packages (notes/distribution-plan.md * Phase 4) and stages publishable directories under release/packages/: * * 1. tsc emit (ESM JS + .d.ts) into each package's dist/, in dependency @@ -11,17 +11,23 @@ * output, so the same rewrite is applied here — a shipped d.ts referencing * "./x.ts" would fail consumers' typechecking. * 3. Stage release/packages// with dist/, README.md, the root LICENSE, - * and a publish-shaped package.json: exports/files pointed at dist/, + * and a publish-shaped package.json: exports/bin/files pointed at dist/, * workspace:* pinned to the lockstep version, dev-only fields dropped. * (Neither bun nor npm applies publishConfig.exports, and `bun publish` * only rewrites workspace:* when run from the workspace itself, so the * staging transform owns both.) + * 4. For create-scratchwork-server, generate the platform templates from + * the deploy/* projects into the staged templates/ dir + * (create/generate-templates.ts), pinning @scratchwork deps to the + * lockstep version — templates are never committed, so they cannot + * drift from the deploy sources. * * scripts/check-npm-pack.ts packs and verifies the staged output in ci; * scripts/publish-packages.ts publishes it. */ import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { generateTemplates } from "../create/generate-templates"; import { repoRoot } from "./workspaces"; /** Publishable packages in dependency order (dir is repo-relative). */ @@ -31,6 +37,7 @@ export const PUBLISHABLE = [ "server/deploy-local", "server/deploy-aws", "server/deploy-cloudflare", + "create", ] as const; const rootVersion = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")).version as string; @@ -52,14 +59,24 @@ function publishedExportTarget(target: string): { types: string; default: string throw new Error(`exports target ${target} has an unexpected extension — extend the staging transform`); } -/** The publish-shaped package.json: dist exports, pinned deps, dev fields dropped. */ +/** The publish-shaped package.json: dist exports/bin, pinned deps, dev fields dropped. */ function publishManifest(pkg: Record): Record { - const exports = Object.fromEntries( - Object.entries(pkg.exports as Record).map(([key, target]) => [ - key, - publishedExportTarget(target), - ]), - ); + const exports = pkg.exports + ? Object.fromEntries( + Object.entries(pkg.exports as Record).map(([key, target]) => [ + key, + publishedExportTarget(target), + ]), + ) + : undefined; + const bin = pkg.bin + ? Object.fromEntries( + Object.entries(pkg.bin as Record).map(([name, target]) => [ + name, + publishedExportTarget(target).default, + ]), + ) + : undefined; const dependencies = Object.fromEntries( Object.entries((pkg.dependencies as Record) ?? {}).map(([name, range]) => [ name, @@ -76,9 +93,10 @@ function publishManifest(pkg: Record): Record repository, engines, ...(keywords ? { keywords } : {}), - exports, - files: ["dist"], - dependencies, + ...(exports ? { exports } : {}), + ...(bin ? { bin } : {}), + files: (pkg.files as string[] | undefined) ?? ["dist"], + ...(Object.keys(dependencies).length > 0 ? { dependencies } : {}), }; } @@ -121,6 +139,9 @@ export function buildAndStage(): string[] { process.exit(1); } cpSync(readme, join(staging, "README.md")); + // The scaffolder's templates are generated from the deploy/* sources at + // staging time (never committed), so template content cannot drift. + if (dir === "create") generateTemplates(join(staging, "templates"), rootVersion); staged.push(staging); console.log(`staged ${staging.slice(repoRoot.length + 1)} (${pkg.name}@${rootVersion})`); } diff --git a/scripts/check-npm-pack.ts b/scripts/check-npm-pack.ts index 1fdcfc7..4642f32 100644 --- a/scripts/check-npm-pack.ts +++ b/scripts/check-npm-pack.ts @@ -15,10 +15,16 @@ * imported under Bun instead. * 4. Typecheck a tiny consumer under NodeNext resolution against the * shipped declarations — the strictest consumer tsc configuration. + * 5. Run the packed create-scratchwork-server bin under plain Node (what + * `npm create` uses) to scaffold every platform template into the + * consumer, then typecheck each scaffolded project — its pinned + * @scratchwork/* deps resolve to the packed tarballs, so the templates + * are verified against exactly what ships, without the network. */ -import { cpSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { TEMPLATE_SOURCES } from "../create/generate-templates"; import { buildAndStage } from "./build-packages"; import { repoRoot } from "./workspaces"; @@ -65,7 +71,11 @@ function checkTarball(pkg: PackedPackage): void { const declaration = js.slice(0, -3) + ".d.ts"; if (!entries.includes(declaration)) failures.push(`${name}: ${js} has no matching ${declaration}`); } - const offenders = entries.filter((entry) => /\.test\.|\/test\/|\.ts$/.test(entry) && !entry.endsWith(".d.ts")); + // templates/ deliberately ships .ts sources: they are the scaffolded + // project's own files, not this package's code. + const offenders = entries.filter( + (entry) => /\.test\.|\/test\/|\.ts$/.test(entry) && !entry.endsWith(".d.ts") && !entry.startsWith("package/templates/"), + ); if (offenders.length > 0) failures.push(`${name}: tarball ships test or source files: ${offenders.join(", ")}`); const raw = JSON.stringify(manifest); @@ -75,7 +85,8 @@ function checkTarball(pkg: PackedPackage): void { } if (manifest.version !== rootVersion) failures.push(`${name}: version ${manifest.version} != lockstep ${rootVersion}`); if (manifest.license !== "MIT" || manifest.repository == null) failures.push(`${name}: missing license/repository metadata`); - for (const [subpath, target] of Object.entries(manifest.exports as Record)) { + if (manifest.exports == null && manifest.bin == null) failures.push(`${name}: manifest has neither exports nor bin`); + for (const [subpath, target] of Object.entries((manifest.exports ?? {}) as Record)) { for (const kind of ["types", "default"] as const) { const file = target[kind]; // Pattern exports are spot-checked by the import smoke below instead. @@ -85,16 +96,26 @@ function checkTarball(pkg: PackedPackage): void { } } } + for (const [binName, file] of Object.entries((manifest.bin ?? {}) as Record)) { + if (!entries.includes(`package/${file.slice(2)}`)) { + failures.push(`${name}: bin["${binName}"] points at ${file}, which is not in the tarball`); + } + } } -/** Lays out a consumer install: tarballs extracted into node_modules/@scratchwork, +/** Lays out a consumer install: tarballs extracted into node_modules, * every other dependency symlinked from the repo root's node_modules. */ function assembleConsumer(packages: readonly PackedPackage[]): string { const consumer = mkdtempSync(join(tmpdir(), "scratchwork-consumer-")); const nodeModules = join(consumer, "node_modules"); mkdirSync(join(nodeModules, "@scratchwork"), { recursive: true }); + // Never symlink a package that the tarballs provide: the repo's + // node_modules entry for it is a workspace symlink, and copying the + // extracted tarball through that symlink would overwrite the workspace + // source (this bit create-scratchwork-server once). + const packedNames = new Set(packages.map((pkg) => pkg.name)); for (const entry of readdirSync(join(repoRoot, "node_modules"))) { - if (entry === "@scratchwork" || entry.startsWith(".")) continue; + if (entry === "@scratchwork" || entry.startsWith(".") || packedNames.has(entry)) continue; symlinkSync(join(repoRoot, "node_modules", entry), join(nodeModules, entry)); } for (const pkg of packages) { @@ -160,6 +181,55 @@ if (failures.length === 0) { if (!tsc.success) { failures.push(`consumer typecheck under NodeNext failed:\n${tsc.stdout.toString().slice(0, 2000)}`); } + + // Scaffold every create-scratchwork-server template with the packed bin + // under plain Node (npm create's runtime) and typecheck the result — the + // scaffolded @scratchwork/* deps resolve to the packed tarballs already + // installed in the consumer's node_modules. + const createPackage = join(consumer, "node_modules", "create-scratchwork-server"); + const packedPlatforms = existsSync(join(createPackage, "templates")) + ? readdirSync(join(createPackage, "templates"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + : []; + const expectedPlatforms = Object.keys(TEMPLATE_SOURCES).sort(); + if (packedPlatforms.join(",") !== expectedPlatforms.join(",")) { + failures.push( + `create-scratchwork-server: packed templates [${packedPlatforms.join(", ")}] != expected [${expectedPlatforms.join(", ")}]`, + ); + } + for (const platform of packedPlatforms) { + const label = `create-scratchwork-server (${platform})`; + const targetName = `scaffold-${platform}`; + const target = join(consumer, targetName); + const scaffoldRun = Bun.spawnSync( + ["node", join(createPackage, "dist", "index.js"), targetName, "--platform", platform], + { cwd: consumer, stdout: "pipe", stderr: "pipe" }, + ); + if (!scaffoldRun.success) { + failures.push(`${label}: scaffold under node failed:\n${scaffoldRun.stderr.toString().slice(0, 2000)}`); + continue; + } + for (const required of ["package.json", "tsconfig.json", "README.md", ".gitignore"]) { + if (!existsSync(join(target, required))) failures.push(`${label}: scaffold is missing ${required}`); + } + const scaffolded = JSON.parse(readFileSync(join(target, "package.json"), "utf8")); + for (const [dep, range] of Object.entries((scaffolded.dependencies ?? {}) as Record)) { + if (dep.startsWith("@scratchwork/") && range !== rootVersion) { + failures.push(`${label}: scaffold pins ${dep}@${range}, expected the lockstep ${rootVersion}`); + } + } + // The scaffolded template pins typescript ^6 (baseUrl etc.), so run the + // consumer-resolved typescript rather than whatever `bunx tsc` picks up. + const scaffoldTsc = Bun.spawnSync( + ["node", join(consumer, "node_modules", "typescript", "bin", "tsc"), "-p", "tsconfig.json"], + { cwd: target, stdout: "pipe", stderr: "pipe" }, + ); + if (!scaffoldTsc.success) { + failures.push(`${label}: scaffolded project typecheck failed:\n${scaffoldTsc.stdout.toString().slice(0, 2000)}`); + } + } rmSync(consumer, { recursive: true, force: true }); } @@ -169,5 +239,6 @@ if (failures.length > 0) { process.exit(1); } console.log( - `check-npm-pack: ${packages.length} tarballs verified (shape, manifest, Node/Bun import smoke, NodeNext consumer typecheck)`, + `check-npm-pack: ${packages.length} tarballs verified (shape, manifest, Node/Bun import smoke, NodeNext consumer typecheck, ` + + `scaffolded ${Object.keys(TEMPLATE_SOURCES).length} create-scratchwork-server templates)`, ); diff --git a/scripts/publish-packages.ts b/scripts/publish-packages.ts index 35ea4f2..b691c31 100644 --- a/scripts/publish-packages.ts +++ b/scripts/publish-packages.ts @@ -1,13 +1,13 @@ #!/usr/bin/env bun /* - * Publishes the five packages to npm from their staged directories, in + * Publishes the six packages to npm from their staged directories, in * dependency order (notes/distribution-plan.md Phase 4). Run locally with an * authenticated npm CLI after the GitHub Release exists (see RELEASING.md): * * bun scripts/publish-packages.ts [--dry-run] [--otp ] * * With npm 2FA enabled, pass a fresh authenticator code via --otp (forwarded - * to every npm publish; all five run within one code's validity window). + * to every npm publish; all six run within one code's validity window). * * Refuses to run on a dirty tree or when HEAD isn't the tag matching the * lockstep version, so what's published is exactly what's tagged. Uses diff --git a/server/README.md b/server/README.md index 67cf24c..ad8ab6a 100644 --- a/server/README.md +++ b/server/README.md @@ -193,18 +193,34 @@ Set `SCRATCHWORK_APP_URL` and `SCRATCHWORK_CONTENT_URL` (or the `appDomain`/`con ## Deploy your own (from npm) -The server packages are published to npm as built JavaScript with type -declarations, so you can deploy a Scratchwork server from a fresh directory -without cloning this repository. Example for Cloudflare: +Scaffold a standalone server project with `create-scratchwork-server` — no +clone of this repository needed: + +```sh +npm create scratchwork-server my-server -- --platform cloudflare # or: aws, local +cd my-server +bun install +bun run local # run the server locally +bun run deploy # deploy it (cloudflare and aws) +``` + +The scaffolded project is generated from this repo's `deploy/*` projects +(cloudflare ← `deploy/cloudflare-vanilla`, aws ← `deploy/generic-aws`, +local ← `deploy/local-dev`): a `server-config.ts` describing your domains and +auth, platform configuration (`cloudflare-config.ts` on Cloudflare), +`deploy.ts` / `local.ts` entrypoints, a `.env.example` for secrets, and a +README with setup instructions. Its `@scratchwork/server-deploy-*` +dependencies are pinned to the release the scaffolder shipped with. + +The server packages are also plain libraries — published as built JavaScript +with type declarations — so you can assemble the same project by hand: ```sh mkdir my-scratchwork && cd my-scratchwork bun add @scratchwork/server-deploy-cloudflare # or: npm install @scratchwork/server-deploy-cloudflare ``` -Copy the config shape from this repo's `deploy/cloudflare-vanilla` project — -a `server-config.ts` describing your domains and auth, a `cloudflare-config.ts` -naming the Worker/bucket/database, and a two-line `deploy.ts`: +with a two-line `deploy.ts`: ```ts // deploy.ts @@ -222,8 +238,5 @@ bun deploy.ts # Bun node deploy.ts # Node 24+ runs your own TypeScript files directly ``` -The AWS equivalent starts from `deploy/generic-aws` and -`@scratchwork/server-deploy-aws`; a local single-machine server uses -`@scratchwork/server-deploy-local` under Bun. A `scratchwork server init` -scaffolder is future work — for now the `deploy/*` projects are the templates -you copy. +The AWS equivalent uses `@scratchwork/server-deploy-aws`; a local +single-machine server uses `@scratchwork/server-deploy-local` under Bun. diff --git a/shared/src/site/default-renderer.generated.js b/shared/src/site/default-renderer.generated.js index 918f9f3..66e28a1 100644 --- a/shared/src/site/default-renderer.generated.js +++ b/shared/src/site/default-renderer.generated.js @@ -1,6 +1,6 @@ // AUTO-GENERATED by renderer/build.js — do not edit. // The JSDoc casts keep tsc declaration emit at `string` instead of a // megabyte-scale literal type when shared is built for npm publishing. -export const defaultRendererSourceHash = /** @type {string} */ ("226ae14709f4141819dd8dcf6ab5c27727dad81b4f5ba847a3256bb9dd9fc2cf"); +export const defaultRendererSourceHash = /** @type {string} */ ("1e1472881e0287b5ececfeadeac7a9be26f872a5d6c78f05619f3e49da463c90"); export const defaultRendererHtml = /** @type {string} */ ("\n\n\n \n \n \n \n\n \n \n \n \n \n \n\n
\n\n \n \n \n\n"); export default defaultRendererHtml; From 76a86cb5747a012e56cb6221b0fb93478888f642 Mon Sep 17 00:00:00 2001 From: Pete Koomen Date: Wed, 22 Jul 2026 00:14:22 -0700 Subject: [PATCH 2/2] Default scaffolded templates to a fail-closed access policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Pete's decision: scaffolded servers should not inherit the deploy sources' allowedUsers (deploy/cloudflare-vanilla ships "public"). The template transform now forces allowedUsers: "@example.com" in every platform template — replacing an existing value, or inserting one after the auth: line where the source sets none (aws, local) — so a scaffolded server fails closed and the policy rides the same example.com placeholder edit as the domains. Generation now asserts the placeholder lands exactly once per template, the workspace tests pin the "@example.com" default (and its README mention) per platform, and the scaffolded READMEs explain replacing the placeholder policy. Co-Authored-By: Claude Fable 5 --- create/generate-templates.ts | 49 ++++++++++++++++++++++- create/templates-src/aws/README.md | 8 +++- create/templates-src/cloudflare/README.md | 5 ++- create/templates-src/local/README.md | 6 ++- create/test/scaffold.test.ts | 19 ++++++++- 5 files changed, 80 insertions(+), 7 deletions(-) diff --git a/create/generate-templates.ts b/create/generate-templates.ts index cd9d744..ba73650 100644 --- a/create/generate-templates.ts +++ b/create/generate-templates.ts @@ -9,6 +9,10 @@ * Per platform, the transform is mechanical: * - every committed file of the deploy project is copied with the * sndbx.sh-specific values replaced by example.com placeholders; + * - the access policy fails closed: whatever allowedUsers value the deploy + * source carries (including none) becomes allowedUsers: "@example.com", + * riding the same placeholder edit as the domains — a scaffolded server + * never starts out world-signupable by accident; * - package.json becomes a standalone project manifest: workspace:* deps * pinned to the exact lockstep version passed in, repo-only scripts * (ci/test) dropped, name set to a placeholder the scaffolder overwrites; @@ -40,11 +44,33 @@ export type TemplatePlatform = keyof typeof TEMPLATE_SOURCES; /** Placeholder name in template package.json; scaffold() overwrites it. */ export const TEMPLATE_PROJECT_NAME = "scratchwork-server"; +/** The fail-closed access-policy placeholder every template ships with. */ +export const TEMPLATE_ALLOWED_USERS = '@example.com'; + +const ALLOWED_USERS_LINE = `allowedUsers: "${TEMPLATE_ALLOWED_USERS}", // emails and @domains (comma-separated), or "public"`; + /** Replaces the sndbx.sh-specific values in deploy sources with placeholders. */ function substitutePlaceholders(text: string): string { return text.replaceAll("sndbx.sh", "example.com").replaceAll("sndbx-sh", "example-com"); } +/** + * Forces the scaffolded access policy to the fail-closed placeholder: an + * existing allowedUsers value (e.g. "public" in deploy/cloudflare-vanilla) is + * replaced, and a server-settings object without one (the aws and local + * sources) gets it inserted after its `auth:` line. Files that do not define + * server settings pass through unchanged. + */ +function substituteAccessPolicy(text: string): string { + if (/allowedUsers:/.test(text)) { + return text.replace(/allowedUsers:\s*"[^"]*",?[^\n]*/g, ALLOWED_USERS_LINE); + } + return text.replace( + /^([ \t]*)auth: "[^"]+",?[^\n]*\n/m, + (line, indent: string) => line + `${indent}${ALLOWED_USERS_LINE}\n`, + ); +} + /** The standalone-project package.json derived from a deploy project's. */ function templateManifest(source: Record, version: string): Record { const scripts = Object.fromEntries( @@ -98,10 +124,31 @@ export function generateTemplates(outDir: string, version: string): void { writeFileSync(join(out, name), JSON.stringify(templateTsconfig(JSON.parse(text)), null, 2) + "\n"); } else { const outName = name.startsWith(".") ? "_" + name.slice(1) : name; - writeFileSync(join(out, outName), substitutePlaceholders(text)); + const transformed = name.endsWith(".ts") + ? substituteAccessPolicy(substitutePlaceholders(text)) + : substitutePlaceholders(text); + writeFileSync(join(out, outName), transformed); } } + // The fail-closed access-policy placeholder must land exactly once, and + // nothing else may set allowedUsers — fail the build if the deploy + // sources change shape in a way the transform no longer covers. + const codeTexts = readdirSync(out) + .filter((name) => name.endsWith(".ts")) + .map((name) => readFileSync(join(out, name), "utf8")); + const placeholderCount = codeTexts.reduce( + (count, text) => count + (text.match(new RegExp(`allowedUsers: "${TEMPLATE_ALLOWED_USERS}"`, "g"))?.length ?? 0), + 0, + ); + const totalCount = codeTexts.reduce((count, text) => count + (text.match(/allowedUsers:/g)?.length ?? 0), 0); + if (placeholderCount !== 1 || totalCount !== 1) { + throw new Error( + `generate-templates: ${platform} template must set allowedUsers to "${TEMPLATE_ALLOWED_USERS}" exactly once ` + + `(found ${placeholderCount} placeholder / ${totalCount} total) — the deploy source's shape no longer matches the transform`, + ); + } + // Hand-written standalone extras: README.md and _gitignore per platform. const extras = join(templatesSrc, platform); for (const name of readdirSync(extras)) { diff --git a/create/templates-src/aws/README.md b/create/templates-src/aws/README.md index a0ecc47..7643013 100644 --- a/create/templates-src/aws/README.md +++ b/create/templates-src/aws/README.md @@ -5,8 +5,12 @@ server, deployed as an AWS Lambda Function URL backed by S3 and DynamoDB. ## Configure -1. Edit `server-config.ts` — auth policy and any fixed settings. Settings not - listed there come from `SCRATCHWORK_*` environment variables. +1. Edit `server-config.ts` — auth policy and any fixed settings. The + scaffolded access policy is an `example.com` placeholder: + `allowedUsers: "@example.com"` only admits Google accounts on that domain, + so replace it with your own domain(s) and emails (comma-separated), or + `"public"` to let anyone sign in. Settings not listed there come from + `SCRATCHWORK_*` environment variables. 2. Copy `.env.example` to `.env` and fill in the public URLs, the Google OAuth credentials, and the session secret (the comments in `.env.example` explain each value). diff --git a/create/templates-src/cloudflare/README.md b/create/templates-src/cloudflare/README.md index c34c934..5c16324 100644 --- a/create/templates-src/cloudflare/README.md +++ b/create/templates-src/cloudflare/README.md @@ -7,7 +7,10 @@ D1 (project metadata). ## Configure 1. Edit `server-config.ts` — your domains and auth policy. The scaffolded - values are `example.com` placeholders. + values are `example.com` placeholders, including the access policy: + `allowedUsers: "@example.com"` only admits Google accounts on that domain, + so replace it with your own domain(s) and emails (comma-separated), or + `"public"` to let anyone sign in. 2. Edit `cloudflare-config.ts` — Worker name, R2 bucket, D1 database, zone, and routes. 3. Copy `.env.example` to `.env` and fill in the Cloudflare API token and the diff --git a/create/templates-src/local/README.md b/create/templates-src/local/README.md index aabbdd9..793e3b6 100644 --- a/create/templates-src/local/README.md +++ b/create/templates-src/local/README.md @@ -25,8 +25,10 @@ PORT=43118 SCRATCHWORK_STORAGE_DIR=.scratchwork-local-data ``` -Edit `local.ts` to change fixed server settings (allowed users, project -naming, and so on). +Edit `local.ts` to change fixed server settings. The scaffolded access policy +is an `example.com` placeholder: `allowedUsers: "@example.com"` only admits +Google accounts on that domain, so replace it with your own domain(s) and +emails (comma-separated), or `"public"` to let anyone sign in. ## Run diff --git a/create/test/scaffold.test.ts b/create/test/scaffold.test.ts index 0725802..1fbee1d 100644 --- a/create/test/scaffold.test.ts +++ b/create/test/scaffold.test.ts @@ -10,7 +10,7 @@ import { afterAll, describe, expect, test } from "bun:test"; import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { generateTemplates, TEMPLATE_SOURCES } from "../generate-templates.ts"; +import { generateTemplates, TEMPLATE_ALLOWED_USERS, TEMPLATE_SOURCES } from "../generate-templates.ts"; import { listPlatforms, projectNameFor, scaffold } from "../src/scaffold.ts"; const workDir = mkdtempSync(join(tmpdir(), "create-scratchwork-test-")); @@ -58,6 +58,23 @@ describe("generateTemplates", () => { } }); + test.each(platforms)("%s: access policy fails closed to the @example.com placeholder", (platform) => { + const dir = join(templatesDir, platform); + const codeTexts = readdirSync(dir) + .filter((name) => name.endsWith(".ts")) + .map((name) => readFileSync(join(dir, name), "utf8")); + const combined = codeTexts.join("\n"); + // Exactly one allowedUsers, and it is the fail-closed placeholder — never + // the deploy source's value (deploy/cloudflare-vanilla ships "public"). + expect(TEMPLATE_ALLOWED_USERS).toBe("@example.com"); + expect(combined.match(/allowedUsers:/g)?.length).toBe(1); + expect(combined).toContain(`allowedUsers: "${TEMPLATE_ALLOWED_USERS}"`); + expect(combined).not.toContain('allowedUsers: "public"'); + // The README tells the user to replace the placeholder policy. + const readme = readFileSync(join(dir, "README.md"), "utf8"); + expect(readme).toContain(`allowedUsers: "${TEMPLATE_ALLOWED_USERS}"`); + }); + test.each(platforms)("%s: template carries every code file of its deploy project", (platform) => { const sourceDir = join(import.meta.dir, "..", "..", TEMPLATE_SOURCES[platform as keyof typeof TEMPLATE_SOURCES]); const codeFiles = readdirSync(sourceDir, { withFileTypes: true })