MILAB-7081: trusted publishing for node-matrix, node-matrix-pnpm, node-docker-simple-fast-pnpm, node-simple, java-gradle

.github/workflows/node-docker-simple-fast-pnpm.yaml

@@ -506,20 +506,26 @@ jobs:
         with:
           version: ${{ inputs.gcp-sdk-version }}
 
-      - name: Create Release Pull Request and Publish npm package (Public)
-        if: steps.context.outputs.is-release == 'true' && inputs.publish-to-public == 'true'
+      # changesets/action opens the "chore: update versions" PR while
+      # changesets are pending; with none left, the packages are published
+      # with npm trusted publishing (OIDC): pack, then upload, in this
+      # GitHub-hosted job (id-token: write). Each package needs a trusted
+      # publisher naming the caller's repository and workflow file.
+      - name: Create Release Pull Request
+        id: changesets
+        if: steps.context.outputs.is-release == 'true'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN }}
         uses: changesets/action@v1
         with:
           commit: "chore: update versions"
           title: "chore: update versions"
-          publish: pnpm -r publish --no-git-checks --access public
 
-      - name: Create Release Pull Request and Publish npm package
-        if: steps.context.outputs.is-release == 'true' && inputs.publish-to-public == 'false'
+      - name: Pack unpublished npm packages
+        id: npm-pack
+        if: steps.context.outputs.is-release == 'true'
+          && steps.changesets.outputs.hasChangesets == 'false'
+        uses: milaboratory/github-ci/actions/node/pack-unpublished@v4-beta
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -529,11 +535,14 @@ jobs:
           GCP_KMS_KEYRING: ${{ secrets.GCP_KMS_KEYRING }}
           GCP_KMS_LOCATION: ${{ secrets.GCP_KMS_LOCATION }}
           GCP_KMS_DIGEST_ALGORITHM: ${{ inputs.gcp-kms-digest-algorithm }}
-        uses: changesets/action@v1
+
+      - name: Publish npm packages
+        if: steps.npm-pack.outputs.count != '' && steps.npm-pack.outputs.count != '0'
+        uses: milaboratory/github-ci/actions/node/publish-tarballs@v4-beta
         with:
-          commit: "chore: update versions"
-          title: "chore: update versions"
-          publish: pnpm -r publish --no-git-checks
+          dir: ${{ steps.npm-pack.outputs.dir }}
+          access-public: ${{ inputs.publish-to-public }}
+          npm-auth-token: ${{ secrets.GITHUB_TOKEN }}
 
     outputs:
       # Even constant outputs become initialized only after job start.

.github/workflows/node-matrix-pnpm.yaml

@@ -927,31 +927,28 @@ jobs:
             git commit -m "Auto-generated changes"
             git push
 
-      - name: Publish npm package
-        id: publish-to-private
+      # npm trusted publishing (OIDC). This job runs on a GitHub-hosted runner
+      # with id-token: write, so pack and upload run back to back: pack runs
+      # prepublishOnly/prepublish and `pnpm pack` for each unpublished
+      # package, upload publishes the tarballs. Each package needs a trusted
+      # publisher naming the caller's repository and workflow file.
+      - name: Pack unpublished npm packages
+        id: npm-pack
         if: github.ref_name == 'main'
           && steps.check-changes.outputs.has-changes == '0'
-          && inputs.publish-to-public == 'false'
+        uses: milaboratory/github-ci/actions/node/pack-unpublished@v4-beta
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPMJS_TOKEN: ${{ env.NPMJS_TOKEN }}
-        uses: changesets/action@v1
-        with:
-          publish: pnpm -r publish --no-git-checks
 
-      - name: Publish npm package (public)
-        id: publish-to-public
-        if: github.ref_name == 'main'
-          && steps.check-changes.outputs.has-changes == '0'
-          && inputs.publish-to-public == 'true'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPMJS_TOKEN: ${{ env.NPMJS_TOKEN }}
-        uses: changesets/action@v1
+      - name: Publish npm packages
+        if: steps.npm-pack.outputs.count != '' && steps.npm-pack.outputs.count != '0'
+        uses: milaboratory/github-ci/actions/node/publish-tarballs@v4-beta
         with:
-          publish: pnpm -r publish --no-git-checks --access public
+          dir: ${{ steps.npm-pack.outputs.dir }}
+          access-public: ${{ inputs.publish-to-public }}
+          npm-auth-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create release with tag
         if: github.ref_name == 'main'

.github/workflows/node-matrix.yaml

@@ -801,6 +801,15 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
           aws-region: ${{ secrets.AWS_ASSUME_REGION }}
 
+      # npm trusted publishing (OIDC): npm 11.5.1+ and, in public repositories,
+      # a package.json `repository` naming this repository (npm adds
+      # provenance and the registry checks it). NPMJS_TOKEN is used only if
+      # the OIDC exchange fails.
+      - name: Prepare npm trusted publishing
+        uses: milaboratory/github-ci/actions/node/prepare-trusted-publishing@v4-beta
+        with:
+          registry-url: ${{ inputs.registry-url }}
+
       - name: Publish npm package
         uses: milaboratory/github-ci/actions/shell@v4-beta
         env:

.github/workflows/node-simple.yaml

@@ -688,6 +688,18 @@ jobs:
         if: inputs.gcp-login-enable
         uses: milaboratory/github-ci/actions/milab-shell-utils@v4-beta
 
+      # npm trusted publishing (OIDC): npm 11.5.1+ and, in public repositories,
+      # a package.json `repository` naming this repository (npm adds
+      # provenance and the registry checks it). NPMJS_TOKEN is used only if
+      # the OIDC exchange fails.
+      - name: Prepare npm trusted publishing
+        if: steps.npm-pkg-status.outputs.exist == '0'
+          && steps.context.outputs.is-release == 'true'
+        uses: milaboratory/github-ci/actions/node/prepare-trusted-publishing@v4-beta
+        with:
+          package-path: ${{ inputs.node-execution-path }}
+          registry-url: ${{ inputs.registry-url }}
+
       - name: Publish npm package
         if: steps.npm-pkg-status.outputs.exist == '0'
           && steps.context.outputs.is-release == 'true'

actions/node/npm-split-publish/README.md

@@ -6,6 +6,7 @@ Implementation of the split npm publish behind two composite actions:
 |---|---|---|
 | `actions/node/pack-unpublished` | `lib/pack` (`src/bin/pack.ts` → `src/pack.ts`) | the build runner, often self-hosted |
 | `actions/node/publish-tarballs` | `lib/publish` (`src/bin/publish.ts` → `src/publish.ts`) | GitHub-hosted, `id-token: write` |
+| `actions/node/prepare-trusted-publishing` | `lib/prepare` (`src/bin/prepare.ts` → `src/repository.ts`) | GitHub-hosted jobs that run `npm publish` themselves |
 
 npm trusted publishing (OIDC) does not accept self-hosted runners, so
 `pack` does everything `pnpm -r publish` does except the upload: for every
@@ -15,6 +16,29 @@ unpublished non-private package, in dependency order, it runs
 `npm publish <tarball> --ignore-scripts`. The two halves share
 `manifest.jsonl`, typed as `ManifestEntry` in `src/manifest.ts`.
 
+## Manifest Format (Public)
+
+`publish-tarballs` reads `<dir>/manifest.jsonl` plus the tarballs next to it.
+`pack-unpublished` writes it, and repositories that pack themselves (e.g.
+milaboratory/pframes-rs) write it by hand, so this format is a contract:
+change it only compatibly. One JSON object per line, uploaded in file order:
+
+```json
+{"name": "@scope/pkg", "version": "1.2.3", "file": "scope-pkg-1.2.3.tgz",
+ "registry": "https://registry.npmjs.org/", "tag": "", "access": "public"}
+```
+
+| Field | Rule |
+|---|---|
+| `name` | npm package name, lowercase |
+| `version` | semver |
+| `file` | tarball basename in the same directory, `.tgz` |
+| `registry` | https URL; registry.npmjs.org (OIDC) or npm.pkg.github.com |
+| `tag` | dist-tag, or `""` for `latest` |
+| `access` | `""`, `public` or `restricted` |
+
+Validation: `src/manifest.ts` (`validateEntry`).
+
 ## Behaviour worth knowing
 
 - **`repository` for provenance.** Under trusted publishing npm turns
@@ -46,6 +70,10 @@ unpublished non-private package, in dependency order, it runs
 - **Fallback token.** With `npm-token-fallback`, the upload job takes
   `NPMJS_TOKEN` out of the caller's `env` secret and masks it before any step
   receives it; no other caller secret reaches that runner.
+- **New packages.** npm accepts a trusted publisher only for a package that
+  exists, and no token can publish any more, so `pack` refuses a package that
+  registry.npmjs.org does not have yet, before any script runs: publish its
+  first version by hand, add its trusted publisher, re-run the job.
 - **Partial releases.** `prepublishOnly` (`block-tools publish` to the block
   registry, `pl-pkg prepublish`) runs in the build job, before the upload job
   starts. If the upload fails, those side effects have happened and npm lacks

actions/node/npm-split-publish/package.json

@@ -3,7 +3,7 @@
   "private": true,
   "description": "Split npm publish: pack on the build runner, upload from a GitHub-hosted runner with npm trusted publishing.",
   "scripts": {
-    "build": "rm -rf lib dist && tsc && ncc build dist/bin/pack.js --minify --out lib/pack --source-map && ncc build dist/bin/publish.js --minify --out lib/publish --source-map",
+    "build": "rm -rf lib dist && tsc && ncc build dist/bin/pack.js --minify --out lib/pack --source-map && ncc build dist/bin/publish.js --minify --out lib/publish --source-map && ncc build dist/bin/prepare.js --minify --out lib/prepare --source-map",
     "test": "vitest run"
   },
   "dependencies": {

actions/node/npm-split-publish/src/bin/pack.ts

@@ -7,9 +7,9 @@
 // root is the current directory. Outputs: dir, count.
 
 import * as core from '@actions/core';
-import * as fs from 'fs';
 import * as path from 'path';
 
+import { publicSourceRepository } from '../github';
 import { packWorkspace } from '../pack';
 import { spawnRunner } from '../run';
 
@@ -31,20 +31,6 @@ async function main(): Promise<void> {
   core.setOutput('count', String(entries.length));
 }
 
-/**
- * https://github.com/<owner>/<repo> when the repository being built is public
- * on github.com, the only case where npm adds provenance on its own. The
- * event payload says whether the repository is private.
- */
-function publicSourceRepository(): string | undefined {
-  const { GITHUB_SERVER_URL: server, GITHUB_REPOSITORY: repository, GITHUB_EVENT_PATH: eventPath } = process.env;
-  if (server !== 'https://github.com' || !repository || !eventPath || !fs.existsSync(eventPath)) {
-    return undefined;
-  }
-  const event = JSON.parse(fs.readFileSync(eventPath, 'utf8')) as { repository?: { private?: boolean } };
-  return event.repository?.private === false ? `${server}/${repository}` : undefined;
-}
-
 function requireEnv(name: string): string {
   const value = process.env[name];
   if (!value) throw new Error(`${name} must be set`);

actions/node/npm-split-publish/src/bin/prepare.ts

@@ -0,0 +1,33 @@
+// Entry point of actions/node/prepare-trusted-publishing, for workflows that
+// run `npm publish` themselves on a GitHub-hosted runner.
+//
+// Env: PACKAGE_DIR (default: the current directory), REGISTRY (the registry
+// the job publishes to; package.json publishConfig.registry wins; default
+// npmjs), GITHUB_WORKSPACE (the repository root, for `repository.directory`),
+// GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_EVENT_PATH.
+
+import * as core from '@actions/core';
+import * as path from 'path';
+
+import { publicSourceRepository } from '../github';
+import { prepareRepository } from '../repository';
+
+const MESSAGES = {
+  'other-registry': 'Not publishing to registry.npmjs.org: npm adds no provenance there, package.json left as is.',
+  'private-repository': 'Private repository: npm adds no provenance, package.json left as is.',
+  added: 'Added `repository` to package.json for npm provenance.',
+  present: 'package.json `repository` already names this repository.',
+};
+
+try {
+  const pkgDir = path.resolve(process.env.PACKAGE_DIR || '.');
+  const result = prepareRepository({
+    pkgDir,
+    workspaceRoot: process.env.GITHUB_WORKSPACE || process.cwd(),
+    registry: process.env.REGISTRY,
+    sourceRepository: publicSourceRepository(),
+  });
+  core.info(`${path.join(pkgDir, 'package.json')}: ${MESSAGES[result]}`);
+} catch (error) {
+  core.setFailed((error as Error).message);
+}

actions/node/npm-split-publish/src/github.ts

@@ -0,0 +1,15 @@
+import * as fs from 'fs';
+
+/**
+ * https://github.com/<owner>/<repo> when the repository being built is public
+ * on github.com, the only case where npm adds provenance on its own. The
+ * event payload says whether the repository is private.
+ */
+export function publicSourceRepository(env: NodeJS.ProcessEnv = process.env): string | undefined {
+  const { GITHUB_SERVER_URL: server, GITHUB_REPOSITORY: repository, GITHUB_EVENT_PATH: eventPath } = env;
+  if (server !== 'https://github.com' || !repository || !eventPath || !fs.existsSync(eventPath)) {
+    return undefined;
+  }
+  const event = JSON.parse(fs.readFileSync(eventPath, 'utf8')) as { repository?: { private?: boolean } };
+  return event.repository?.private === false ? `${server}/${repository}` : undefined;
+}

actions/node/npm-split-publish/src/pack.ts

@@ -6,7 +6,7 @@ import * as fs from 'fs';
 import * as path from 'path';
 
 import { ManifestEntry, writeManifest } from './manifest';
-import { PackageJson, isPublished, resolveRegistry } from './registry';
+import { PackageJson, resolveRegistry, versionState } from './registry';
 import { repositoryState, withRepository } from './repository';
 import { Runner, runChecked } from './run';
 import { readPackedManifest } from './tarball';
@@ -79,23 +79,37 @@ export async function packWorkspace(options: PackOptions, deps: PackDeps): Promi
   // their registry yet. Only these are checked; already-published packages
   // keep releasing as before.
   const planned: Planned[] = [];
+  const newOnNpmjs: string[] = [];
   for (const pkg of packages) {
     const { name, version } = pkg.manifest;
     if (pkg.manifest.private) {
       continue;
     }
     const registry = await resolveRegistry(pkg.manifest, npmrcDir, run);
-    if (await isPublished(name, version, registry, npmrcDir, run)) {
+    const state = await versionState(name, version, registry, npmrcDir, run);
+    if (state === 'published') {
       log(`${name}@${version}: already in ${registry}, skipping`);
       continue;
     }
+    if (state === 'new-package' && isNpmjs(registry)) {
+      newOnNpmjs.push(name);
+    }
     // Provenance applies to npmjs uploads from a public repository only.
     const provenanceFrom = sourceRepository && isNpmjs(registry) ? sourceRepository : undefined;
     planned.push({ pkg, registry, provenanceFrom });
   }
 
   // Every check before the first prepublishOnly, so a bad package fails the
   // run without side effects.
+  if (newOnNpmjs.length > 0) {
+    // npm accepts a trusted publisher only for a package that exists, and no
+    // token can publish any more, so the first version needs a person.
+    throw new Error(
+      `not on npmjs yet: ${newOnNpmjs.join(', ')}. Publish the first version of each by hand ` +
+        `(npm publish with 2FA), add its trusted publisher (repository + workflow file, "npm publish" allowed), ` +
+        `then re-run this job. Nothing was published or uploaded.`,
+    );
+  }
   for (const { pkg, provenanceFrom } of planned) {
     const { name } = pkg.manifest;
     for (const script of UNSUPPORTED_SCRIPTS) {

actions/node/npm-split-publish/src/registry.ts

@@ -35,26 +35,34 @@ export async function resolveRegistry(pkg: PackageJson & { name: string }, npmrc
   return configGet('registry');
 }
 
+export type VersionState = 'published' | 'new-version' | 'new-package';
+
 /**
- * Whether name@version is in the registry. Throws on any lookup failure other
- * than "package not found".
+ * Where name@version stands in the registry: published, a new version of an
+ * existing package, or a package the registry does not have at all. Throws on
+ * any lookup failure other than "not found".
  */
-export async function isPublished(
+export async function versionState(
   name: string,
   version: string,
   registry: string,
   cwd: string,
   run: Runner,
-): Promise<boolean> {
+): Promise<VersionState> {
   // npm view: exit 0 + the version when it exists; exit 0 + no output when the
   // package exists without that version; E404 when the package does not exist.
   const args = ['view', `${name}@${version}`, 'version', `--registry=${registry}`];
   const result = await run('npm', args, { cwd });
   if (result.code === 0) {
-    return result.stdout.trim() !== '';
+    return result.stdout.trim() !== '' ? 'published' : 'new-version';
   }
   if (/\bE404\b/.test(result.stderr + result.stdout)) {
-    return false;
+    return 'new-package';
   }
   throw new CommandError(['npm', ...args].join(' '), result);
 }
+
+/** Whether name@version is in the registry (see versionState). */
+export async function isPublished(name: string, version: string, registry: string, cwd: string, run: Runner): Promise<boolean> {
+  return (await versionState(name, version, registry, cwd, run)) === 'published';
+}