test(ci): add publish smoke test for npm, pnpm, bun (#1018)

* test(ci): add publish smoke test for npm, pnpm, bun

Packs every publishable workspace (create-better-t-stack,
@better-t-stack/template-generator, @better-t-stack/types, create-bts)
with `npm pack` (matches what the release workflow's `npm publish`
would produce), then runs two checks:

1. Static: asserts no unresolved protocol (catalog:, workspace:,
   link:) leaked into the tarball's package.json. This catches the
   class of bug that shipped in 3.27.3 — where catalog: resolved
   fine inside the monorepo but survived into the published
   artifact and broke downstream pnpm/npm installs.

2. Install: installs the CLI tarball via npm, pnpm, and bun in
   isolated temp dirs, using overrides to redirect sibling workspace
   deps to their local tarballs. Verifies the binary lands in
   node_modules/.bin. Catches any install-time regression across
   the three package managers users actually reach for.

Gate in both the PR test workflow and the release workflow (before
publishing to npm) so a broken artifact can't reach consumers even
if PR CI is somehow bypassed.

Verified: smoke passes against the current tree; when I temporarily
injected `"zod": "catalog:"` into apps/cli/package.json, the static
check caught it before the install step ran.

* ci: run publish smoke before any npm publish

The smoke step was placed after @better-t-stack/types and
@better-t-stack/template-generator were already published, which
defeated the protection for those packages — if smoke failed on
either one, the broken artifact would already be on npm and the
release would halt partway through.

Move all four publish steps to run after the smoke gate so a smoke
failure aborts the release before anything ships. The builds and
jq version rewrites stay where they are (the smoke needs them).

* test(publish-smoke): address review nits

- npm pack --json instead of --silent + stdout string parse
  (docs-stable JSON contract, no fragile string splitting /
  non-null assertion)
- rewrite workspace:* across all four dep buckets
  (dependencies, devDependencies, peerDependencies,
  optionalDependencies) for parity with what a jq-based release
  rewrite would handle

Author: AmanVarshney01

.github/workflows/release.yaml

@@ -87,14 +87,6 @@ jobs:
         if: steps.check-version.outputs.exists == 'false'
         run: cd packages/types && bun run build
 
-      - name: Publish types to NPM
-        if: steps.check-version.outputs.exists == 'false'
-        run: cd packages/types && npm publish --access public --provenance
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-          BTS_TELEMETRY: 1
-          CONVEX_INGEST_URL: ${{ secrets.CONVEX_INGEST_URL }}
-
       - name: Update template-generator package version and types dependency
         if: steps.check-version.outputs.exists == 'false'
         run: |
@@ -106,14 +98,6 @@ jobs:
         if: steps.check-version.outputs.exists == 'false'
         run: cd packages/template-generator && bun run build
 
-      - name: Publish template-generator to NPM
-        if: steps.check-version.outputs.exists == 'false'
-        run: cd packages/template-generator && npm publish --access public --provenance
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-          BTS_TELEMETRY: 1
-          CONVEX_INGEST_URL: ${{ secrets.CONVEX_INGEST_URL }}
-
       - name: Update CLI types and template-generator dependencies and version
         if: steps.check-version.outputs.exists == 'false'
         run: |
@@ -135,6 +119,33 @@ jobs:
           cd packages/create-bts
           jq --arg v "$VERSION" --arg dep "^$VERSION" '.version = $v | .dependencies["create-better-t-stack"] = $dep' package.json > tmp.json && mv tmp.json package.json
 
+      - name: Enable pnpm via corepack
+        if: steps.check-version.outputs.exists == 'false'
+        run: corepack enable pnpm
+
+      # Gate all publishes on the smoke so a failure here doesn't leave a
+      # partial release on npm (e.g. types/template-generator published but
+      # create-better-t-stack broken).
+      - name: Publish smoke test (npm, pnpm, bun)
+        if: steps.check-version.outputs.exists == 'false'
+        run: bun run smoke:publish
+
+      - name: Publish types to NPM
+        if: steps.check-version.outputs.exists == 'false'
+        run: cd packages/types && npm publish --access public --provenance
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          BTS_TELEMETRY: 1
+          CONVEX_INGEST_URL: ${{ secrets.CONVEX_INGEST_URL }}
+
+      - name: Publish template-generator to NPM
+        if: steps.check-version.outputs.exists == 'false'
+        run: cd packages/template-generator && npm publish --access public --provenance
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+          BTS_TELEMETRY: 1
+          CONVEX_INGEST_URL: ${{ secrets.CONVEX_INGEST_URL }}
+
       - name: Publish CLI to NPM
         if: steps.check-version.outputs.exists == 'false'
         run: cd apps/cli && npm publish --access public --provenance

.github/workflows/test.yaml

@@ -50,3 +50,9 @@ jobs:
         env:
           AGENT: 1
           BTS_TELEMETRY: 0
+
+      - name: Enable pnpm via corepack
+        run: corepack enable pnpm
+
+      - name: Publish smoke test (npm, pnpm, bun)
+        run: bun run smoke:publish

package.json

@@ -28,6 +28,7 @@
     "dev": "turbo run dev",
     "dev:cli": "turbo run dev --filter=create-better-t-stack",
     "cli": "cd apps/cli && node dist/cli.js",
+    "smoke:publish": "bun run scripts/publish-smoke.ts",
     "dev:web": "turbo run dev --filter=web",
     "build:web": "turbo run build --filter=web",
     "build:cli": "turbo run build --filter=create-better-t-stack",

scripts/publish-smoke.ts

@@ -0,0 +1,146 @@
+#!/usr/bin/env bun
+// Verify create-better-t-stack installs and runs under npm, pnpm, and bun.
+// Catches any regression that breaks the published artifact for consumers —
+// unresolved protocol refs, missing files, broken bin entry, import failures
+// from missing transitive deps, etc.
+//
+// Packs each publishable workspace with `npm pack` (matching the release
+// workflow, which uses `npm publish`), installs the CLI tarball in a temp
+// dir using overrides to redirect sibling workspace deps at their local
+// tarballs, then runs `create-better-t-stack --version` to prove the binary
+// actually executes.
+
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+import { $ } from "bun";
+
+const ROOT = resolve(import.meta.dir, "..");
+
+type Publishable = {
+  name: string;
+  dir: string;
+  // Release workflow rewrites workspace:* → ^<version> via jq before publish.
+  // Mirror that here so the tarball matches what actually ships.
+  rewriteWorkspaceDeps?: string[];
+};
+
+const PUBLISHABLES: Publishable[] = [
+  { name: "@better-t-stack/types", dir: "packages/types" },
+  { name: "@better-t-stack/template-generator", dir: "packages/template-generator" },
+  {
+    name: "create-better-t-stack",
+    dir: "apps/cli",
+    rewriteWorkspaceDeps: ["@better-t-stack/types", "@better-t-stack/template-generator"],
+  },
+];
+
+const green = (s: string) => `\x1b[32m${s}\x1b[0m`;
+const red = (s: string) => `\x1b[31m${s}\x1b[0m`;
+const dim = (s: string) => `\x1b[2m${s}\x1b[0m`;
+
+async function pack(pkg: Publishable, outDir: string): Promise<string> {
+  const pkgJsonPath = join(ROOT, pkg.dir, "package.json");
+  const original = readFileSync(pkgJsonPath, "utf-8");
+
+  if (pkg.rewriteWorkspaceDeps) {
+    const parsed = JSON.parse(original);
+    for (const bucket of [
+      "dependencies",
+      "devDependencies",
+      "peerDependencies",
+      "optionalDependencies",
+    ] as const) {
+      const deps = parsed[bucket];
+      if (!deps) continue;
+      for (const d of pkg.rewriteWorkspaceDeps) {
+        if (deps[d]?.startsWith("workspace:")) {
+          deps[d] = `^${parsed.version}`;
+        }
+      }
+    }
+    writeFileSync(pkgJsonPath, `${JSON.stringify(parsed, null, 2)}\n`);
+  }
+
+  try {
+    const r = await $`npm pack --pack-destination=${outDir} --json`
+      .cwd(join(ROOT, pkg.dir))
+      .quiet();
+    const [entry] = JSON.parse(r.stdout.toString()) as Array<{ filename: string }>;
+    return join(outDir, entry.filename);
+  } finally {
+    if (pkg.rewriteWorkspaceDeps) writeFileSync(pkgJsonPath, original);
+  }
+}
+
+async function installAndRun(
+  pm: "npm" | "pnpm" | "bun",
+  tarballs: Record<string, string>,
+  smokeRoot: string,
+) {
+  const dir = join(smokeRoot, `install-${pm}`);
+  rmSync(dir, { recursive: true, force: true });
+  mkdirSync(dir, { recursive: true });
+
+  const overrides = {
+    "@better-t-stack/types": `file:${tarballs["@better-t-stack/types"]}`,
+    "@better-t-stack/template-generator": `file:${tarballs["@better-t-stack/template-generator"]}`,
+  };
+  const fixture: Record<string, unknown> = {
+    name: `smoke-${pm}`,
+    private: true,
+    version: "0.0.0",
+    dependencies: { "create-better-t-stack": `file:${tarballs["create-better-t-stack"]}` },
+  };
+  if (pm === "pnpm") fixture.pnpm = { overrides };
+  else fixture.overrides = overrides;
+
+  writeFileSync(join(dir, "package.json"), JSON.stringify(fixture, null, 2));
+
+  const install = await $`${pm} install --ignore-scripts`.cwd(dir).quiet().nothrow();
+  if (install.exitCode !== 0) {
+    console.error(red(`✗ ${pm} install failed`));
+    console.error(dim(install.stderr.toString()));
+    process.exit(1);
+  }
+
+  // Execute the CLI via its installed bin path to prove it actually works —
+  // not just that the file got linked into node_modules/.bin.
+  const bin = join(dir, "node_modules", ".bin", "create-better-t-stack");
+  const run = await $`${bin} --version`.cwd(dir).quiet().nothrow();
+  if (run.exitCode !== 0) {
+    console.error(red(`✗ ${pm}: create-better-t-stack --version failed (exit ${run.exitCode})`));
+    console.error(dim(run.stderr.toString() + run.stdout.toString()));
+    process.exit(1);
+  }
+
+  console.log(green(`✓ ${pm}`) + dim(`  v${run.stdout.toString().trim()}`));
+}
+
+async function hasPackageManager(pm: string): Promise<boolean> {
+  return (await $`which ${pm}`.quiet().nothrow()).exitCode === 0;
+}
+
+const smokeRoot = join(tmpdir(), `bts-publish-smoke-${Date.now()}`);
+const tarballDir = join(smokeRoot, "tarballs");
+mkdirSync(tarballDir, { recursive: true });
+
+console.log("Packing...");
+const tarballs: Record<string, string> = {};
+for (const pkg of PUBLISHABLES) {
+  tarballs[pkg.name] = await pack(pkg, tarballDir);
+  console.log(dim(`  ${pkg.name}`));
+}
+
+console.log("\nInstalling and running create-better-t-stack under each package manager...");
+for (const pm of ["npm", "pnpm", "bun"] as const) {
+  if (!(await hasPackageManager(pm))) {
+    console.log(dim(`  - ${pm} not available, skipping`));
+    continue;
+  }
+  await installAndRun(pm, tarballs, smokeRoot);
+}
+
+rmSync(smokeRoot, { recursive: true, force: true });
+console.log(green("\n✓ publish smoke test passed"));