diff --git a/src/commands/manifest/bazel/bazel-cquery.mts b/src/commands/manifest/bazel/bazel-cquery.mts index f86c6ad49..ae701da04 100644 --- a/src/commands/manifest/bazel/bazel-cquery.mts +++ b/src/commands/manifest/bazel/bazel-cquery.mts @@ -115,7 +115,10 @@ function buildMetadataCqueryExpr(repoName: string): string { } // Build the full cquery argv for a per-repo metadata cquery. Exposed for -// argv-shape unit tests without touching `spawn`. +// argv-shape unit tests without touching `spawn`. The startup-flag +// composition mirrors `bazel-query-runner`'s `buildStartupFlags` so +// customer `--bazel-startup-flag` values land before the subcommand and +// `--bazel-flag` values land after the standard cquery flags. export function buildMetadataCqueryArgv( repoName: string, opts: BazelQueryOptions, @@ -130,7 +133,13 @@ export function buildMetadataCqueryArgv( if (opts.bazelOutputBase) { startup.push(`--output_base=${opts.bazelOutputBase}`) } - const userFlags = splitBazelFlags(opts.bazelFlags) + if (opts.extraBazelStartupFlags?.length) { + startup.push(...opts.extraBazelStartupFlags) + } + const userFlags = [ + ...splitBazelFlags(opts.bazelFlags), + ...(opts.extraBazelFlags ?? []), + ] return [ ...startup, 'cquery', diff --git a/src/commands/manifest/bazel/bazel-cquery.test.mts b/src/commands/manifest/bazel/bazel-cquery.test.mts index f26d1e7d4..22e741af5 100644 --- a/src/commands/manifest/bazel/bazel-cquery.test.mts +++ b/src/commands/manifest/bazel/bazel-cquery.test.mts @@ -192,6 +192,21 @@ describe('buildMetadataCqueryArgv', () => { expect(argv).toContain('--repo_env=SCALA_VERSION=2.13.18') }) + it('threads extraBazelStartupFlags ahead of cquery and extraBazelFlags after the standard flags', () => { + const argv = buildMetadataCqueryArgv('maven', { + bin: 'bazel', + cwd: '/r', + invocationFlags: [], + extraBazelStartupFlags: ['--host_jvm_args=-Xmx2g'], + extraBazelFlags: ['--config=ci'], + }) + const cqueryIdx = argv.indexOf('cquery') + const jvmIdx = argv.indexOf('--host_jvm_args=-Xmx2g') + const configIdx = argv.indexOf('--config=ci') + expect(jvmIdx).toBeLessThan(cqueryIdx) + expect(configIdx).toBeGreaterThan(cqueryIdx) + }) + it('includes invocationFlags between subcommand and target expression', () => { const argv = buildMetadataCqueryArgv('maven', { bin: 'bazel', diff --git a/src/commands/manifest/bazel/bazel-query-runner.mts b/src/commands/manifest/bazel/bazel-query-runner.mts index c9ef84f1a..7e52faf2f 100644 --- a/src/commands/manifest/bazel/bazel-query-runner.mts +++ b/src/commands/manifest/bazel/bazel-query-runner.mts @@ -19,6 +19,17 @@ export type BazelQueryOptions = { // orchestrator mkdtemp's a fresh path per invocation; the legacy PyPI // path may leave it unset for now. outputUserRoot?: string + // Customer-supplied `--bazel-flag=` values (repeatable on the CLI). + // Each entry is appended verbatim to every subcommand invocation, after + // the orchestrator's own flags and after `bazelFlags`. Used to thread + // matrix-cell selectors like `--config=ci-scala-2-13` or + // `--repo_env=SCALA_VERSION=2.13.18` per CI shard. + extraBazelFlags?: string[] + // Customer-supplied `--bazel-startup-flag=` values (repeatable). + // Each entry is appended to the startup-flag prefix, after the + // orchestrator's own startup flags (--bazelrc, --output_user_root, + // --output_base) and before the subcommand. + extraBazelStartupFlags?: string[] env?: NodeJS.ProcessEnv verbose?: boolean } @@ -49,7 +60,8 @@ export function splitBazelFlags(flags: string | undefined): string[] { // Build the shared startup-flag prefix for any bazel invocation. Centralised // so `--output_user_root` propagates to every spawn — principle 7 of the // Maven design requires per-invocation server isolation across query, -// cquery, and `bazel mod` commands alike. +// cquery, and `bazel mod` commands alike. Customer `--bazel-startup-flag` +// values are appended last so they appear before the subcommand. function buildStartupFlags(opts: BazelQueryOptions): string[] { const startup: string[] = [] if (opts.bazelRc) { @@ -61,11 +73,23 @@ function buildStartupFlags(opts: BazelQueryOptions): string[] { if (opts.bazelOutputBase) { startup.push(`--output_base=${opts.bazelOutputBase}`) } + if (opts.extraBazelStartupFlags?.length) { + startup.push(...opts.extraBazelStartupFlags) + } return startup } +// Compose the user-flag suffix appended after every subcommand's standard +// flags. The legacy whitespace-split `bazelFlags` string and the new +// repeatable `extraBazelFlags` array are concatenated in that order so a +// socket.json default plus a CLI override applies both. No deduplication +// — Bazel's last-wins semantics handle conflicts. +function userBazelFlags(opts: BazelQueryOptions): string[] { + return [...splitBazelFlags(opts.bazelFlags), ...(opts.extraBazelFlags ?? [])] +} + function buildBazelModShowVisibleReposArgv(opts: BazelQueryOptions): string[] { - const userFlags = splitBazelFlags(opts.bazelFlags) + const userFlags = userBazelFlags(opts) return [ ...buildStartupFlags(opts), 'mod', @@ -79,7 +103,7 @@ function buildBazelModShowVisibleReposArgv(opts: BazelQueryOptions): string[] { function buildBazelModShowMavenExtensionArgv( opts: BazelQueryOptions, ): string[] { - const userFlags = splitBazelFlags(opts.bazelFlags) + const userFlags = userBazelFlags(opts) return [ ...buildStartupFlags(opts), 'mod', @@ -99,7 +123,7 @@ function buildBazelModShowMavenExtensionArgv( } function buildBazelModShowPipExtensionArgv(opts: BazelQueryOptions): string[] { - const userFlags = splitBazelFlags(opts.bazelFlags) + const userFlags = userBazelFlags(opts) return [ ...buildStartupFlags(opts), 'mod', @@ -123,7 +147,7 @@ function buildBazelArgv( // Bazel argv shape: query --output= // Keep query output stable and avoid updating Bazel lockfiles while extracting. const queryFlags = ['--lockfile_mode=off', '--noshow_progress'] - const userFlags = splitBazelFlags(opts.bazelFlags) + const userFlags = userBazelFlags(opts) return [ ...buildStartupFlags(opts), 'query', @@ -144,7 +168,7 @@ function buildBazelProbeCqueryArgv( repoName: string, opts: BazelQueryOptions, ): string[] { - const userFlags = splitBazelFlags(opts.bazelFlags) + const userFlags = userBazelFlags(opts) return [ ...buildStartupFlags(opts), 'cquery', diff --git a/src/commands/manifest/bazel/bazel-query-runner.test.mts b/src/commands/manifest/bazel/bazel-query-runner.test.mts index c4e526f40..ad1cc62fb 100644 --- a/src/commands/manifest/bazel/bazel-query-runner.test.mts +++ b/src/commands/manifest/bazel/bazel-query-runner.test.mts @@ -121,6 +121,39 @@ describe('runBazelQuery', () => { expect(argv).toContain('--keep_going') }) + it('appends extraBazelFlags after bazelFlags (CLI overrides socket.json default)', async () => { + await runBazelQuery('q', { + bin: 'bazel', + cwd: '/r', + invocationFlags: [], + bazelFlags: '--config=default', + extraBazelFlags: ['--config=override', '--repo_env=K=V'], + }) + const argv = mocked.mock.calls[0]![1] as string[] + const def = argv.indexOf('--config=default') + const override = argv.indexOf('--config=override') + const envFlag = argv.indexOf('--repo_env=K=V') + expect(def).toBeGreaterThanOrEqual(0) + expect(override).toBeGreaterThan(def) + expect(envFlag).toBeGreaterThan(override) + }) + + it('threads extraBazelStartupFlags ahead of the subcommand but after the orchestrator startup flags', async () => { + await runBazelQuery('q', { + bin: 'bazel', + cwd: '/r', + invocationFlags: [], + outputUserRoot: '/tmp/x', + extraBazelStartupFlags: ['--host_jvm_args=-Xmx2g'], + }) + const argv = mocked.mock.calls[0]![1] as string[] + const root = argv.indexOf('--output_user_root=/tmp/x') + const jvm = argv.indexOf('--host_jvm_args=-Xmx2g') + const subcmd = argv.indexOf('query') + expect(root).toBeLessThan(jvm) + expect(jvm).toBeLessThan(subcmd) + }) + it('forwards env to spawn when provided', async () => { const env = { ...process.env, BAZEL_BENCH: 'yes' } await runBazelQuery('q', { @@ -433,6 +466,17 @@ describe('buildPypiProbeFor', () => { }) }) + it('does nothing when extra flags are absent', async () => { + const probe = buildPypiProbeFor({ + bin: 'bazel', + cwd: '/r', + invocationFlags: [], + }) + await probe('pypi') + const argv = mocked.mock.calls[0]![1] as string[] + expect(argv.some(a => a.startsWith('--config='))).toBe(false) + }) + it('returns the full triple when the hub has no :pkg targets', async () => { mocked.mockReset() // @ts-ignore — narrow return shape for the test's purposes. diff --git a/src/commands/manifest/bazel/bazel-repo-discovery.test.mts b/src/commands/manifest/bazel/bazel-repo-discovery.test.mts index 0be94f9c2..c5fd90746 100644 --- a/src/commands/manifest/bazel/bazel-repo-discovery.test.mts +++ b/src/commands/manifest/bazel/bazel-repo-discovery.test.mts @@ -254,7 +254,7 @@ describe('bazel-repo-discovery', () => { probeResult({ code: 2, stderr: - 'ERROR: In extension argument @rules_jvm_external//:extensions.bzl%maven: No module with the apparent repo name @rules_jvm_external exists in the dependency graph. Type \'bazel help mod\' for syntax and help.\n', + "ERROR: In extension argument @rules_jvm_external//:extensions.bzl%maven: No module with the apparent repo name @rules_jvm_external exists in the dependency graph. Type 'bazel help mod' for syntax and help.\n", }), 0, ), diff --git a/src/commands/manifest/bazel/cmd-manifest-bazel.mts b/src/commands/manifest/bazel/cmd-manifest-bazel.mts index 0ab10e861..cf5836d55 100644 --- a/src/commands/manifest/bazel/cmd-manifest-bazel.mts +++ b/src/commands/manifest/bazel/cmd-manifest-bazel.mts @@ -32,11 +32,26 @@ const config: CliCommandConfig = { description: 'Path to bazel/bazelisk binary; default: $(which bazelisk) || $(which bazel)', }, + bazelFlag: { + type: 'string', + isMultiple: true, + description: + 'Bazel flag forwarded to every subcommand (repeatable). E.g. ' + + '`--bazel-flag=--config=ci-scala-2-13` to scan a matrix cell.', + }, bazelFlags: { type: 'string', description: 'Flags forwarded to every bazel invocation (single quoted string)', }, + bazelMavenRepo: { + type: 'string', + isMultiple: true, + description: + 'Maven hub repo name to extract in addition to the auto-discovered ' + + 'set (repeatable). For legacy WORKSPACE workspaces with hubs that ' + + 'use non-conventional names. E.g. `--bazel-maven-repo=my_jars`.', + }, bazelOutputBase: { type: 'string', description: 'Bazel --output_base for read-only-cache CI environments', @@ -45,6 +60,13 @@ const config: CliCommandConfig = { type: 'string', description: 'Path to additional .bazelrc fragments forwarded to bazel', }, + bazelStartupFlag: { + type: 'string', + isMultiple: true, + description: + 'Bazel startup flag inserted before the subcommand (repeatable). ' + + 'E.g. `--bazel-startup-flag=--host_jvm_args=-Xmx2g`.', + }, ecosystem: { type: 'string', isMultiple: true, @@ -90,8 +112,15 @@ const config: CliCommandConfig = { \`bazel mod show_extension\` and filtered to the root module's own hubs. Under legacy WORKSPACE mode (no \`show_extension\`), only conventionally named hubs are probed (\`maven\`, \`maven_install\`, \`maven_dev\`, …). A hub - with a non-conventional name that \`show_extension\` does not enumerate is - not discovered yet; a flag to name extra hubs is planned. + with a non-conventional name that \`show_extension\` does not enumerate + can be named explicitly with \`--bazel-maven-repo\`. + + \`--bazel-flag\` forwards a flag to every subcommand (cquery, query, mod + show_extension, mod dump_repo_mapping), after the orchestrator's own + flags. Use it for matrix-cell selectors like + \`--repo_env=SCALA_VERSION=2.13.18\` or \`--platforms=//tools:linux_x86_64\`. + \`--bazel-startup-flag\` is inserted before the subcommand instead, for + host-side knobs like \`--host_jvm_args=-Xmx2g\`. To generate AND upload in one step, use \`socket scan create --auto-manifest\` instead — it detects Bazel workspaces, generates Maven manifests by @@ -102,6 +131,9 @@ const config: CliCommandConfig = { $ ${command} --ecosystem pypi . $ ${command} --ecosystem maven --ecosystem pypi . $ ${command} --bazel=/usr/local/bin/bazelisk . + $ ${command} --bazel-flag=--config=ci-scala-2-13 . + $ ${command} --bazel-startup-flag=--host_jvm_args=-Xmx2g . + $ ${command} --bazel-maven-repo=my_jars --bazel-maven-repo=test_maven . `, } @@ -261,6 +293,11 @@ async function run( ) const { ecosystem } = cli.flags + const bazelFlag = (cli.flags['bazelFlag'] as string[] | undefined) ?? [] + const bazelStartupFlag = + (cli.flags['bazelStartupFlag'] as string[] | undefined) ?? [] + const bazelMavenRepo = + (cli.flags['bazelMavenRepo'] as string[] | undefined) ?? [] let { bazel, bazelFlags, bazelOutputBase, bazelRc, out, verbose } = cli.flags let perRepoTimeout = cli.flags['perRepoTimeout'] as number | undefined @@ -390,6 +427,13 @@ async function run( bazelRc: bazelRc as string | undefined, bin: bazel as string | undefined, cwd, + ...(bazelFlag.length ? { extraBazelFlags: bazelFlag } : {}), + ...(bazelStartupFlag.length + ? { extraBazelStartupFlags: bazelStartupFlag } + : {}), + ...(bazelMavenRepo.length + ? { extraMavenRepoNames: bazelMavenRepo } + : {}), out: out as string, perRepoTimeoutMs: perRepoTimeout, verbose: Boolean(verbose), diff --git a/src/commands/manifest/bazel/cmd-manifest-bazel.test.mts b/src/commands/manifest/bazel/cmd-manifest-bazel.test.mts index 8cc8ea254..4936f484f 100644 --- a/src/commands/manifest/bazel/cmd-manifest-bazel.test.mts +++ b/src/commands/manifest/bazel/cmd-manifest-bazel.test.mts @@ -243,18 +243,18 @@ describe('evaluateEcosystemOutcomes (explicit mode)', () => { }) describe('perRepoTimeout flag wiring', () => { - const importMeta = { url: 'file:///cmd-manifest-bazel.test.mts' } as ImportMeta + const importMeta = { + url: 'file:///cmd-manifest-bazel.test.mts', + } as ImportMeta beforeEach(() => { vi.mocked(extractBazelToMaven).mockClear() }) it('defaults the explicit command to a 120s per-repo timeout', async () => { - await cmdManifestBazel.run( - [FLAG_CONFIG, '{}', '.'], - importMeta, - { parentName: 'manifest' } as CliCommandContext, - ) + await cmdManifestBazel.run([FLAG_CONFIG, '{}', '.'], importMeta, { + parentName: 'manifest', + } as CliCommandContext) expect(extractBazelToMaven).toHaveBeenCalledTimes(1) expect(extractBazelToMaven).toHaveBeenCalledWith( expect.objectContaining({ perRepoTimeoutMs: 120_000 }), diff --git a/src/commands/manifest/bazel/extract_bazel_to_maven.mts b/src/commands/manifest/bazel/extract_bazel_to_maven.mts index 6248d1d21..f8007e043 100644 --- a/src/commands/manifest/bazel/extract_bazel_to_maven.mts +++ b/src/commands/manifest/bazel/extract_bazel_to_maven.mts @@ -48,6 +48,18 @@ export type ExtractBazelOptions = { cwd: string // Optional env override used for python-shim PATH augmentation. env?: NodeJS.ProcessEnv + // Customer-supplied `--bazel-flag=` values; threaded into every + // bazel subcommand after the orchestrator's own flags. Repeatable on + // the CLI; entries are forwarded verbatim. + extraBazelFlags?: string[] | undefined + // Customer-supplied `--bazel-startup-flag=` values; injected into + // the startup-flag prefix before the subcommand. + extraBazelStartupFlags?: string[] | undefined + // Customer-supplied Maven hub names augmenting the auto-discovery + // candidate set. Wired in by the `--bazel-maven-repo=` flag for + // legacy WORKSPACE workspaces whose hubs use non-conventional names + // (or custom Bzlmod extensions `mod show_extension` doesn't enumerate). + extraMavenRepoNames?: string[] | undefined // Directory basenames the workspace walker must not descend into. // Caller-supplied so the orchestrator stays generic; the CLI command // composes the codebase-wide `IGNORED_DIRS` with Bazel-specific dirs @@ -517,6 +529,7 @@ async function discoverCandidatesForWorkspace( workspaceRoot: string, mode: WorkspaceMode, queryOpts: BazelQueryOptions, + extras: readonly string[], verbose: boolean, ): Promise { const candidates: string[] = [] @@ -582,13 +595,17 @@ async function discoverCandidatesForWorkspace( } } // Probe candidates the show_extension path could not authoritatively - // enumerate: when it produced root hubs, probe nothing extra; otherwise - // (WORKSPACE mode, a failed show_extension, or a parse with zero root - // hubs) probe the conventional hub names. + // enumerate: when it produced root hubs, probe nothing extra beyond the + // customer-supplied names; otherwise (WORKSPACE mode, a failed + // show_extension, or a parse with zero root hubs) probe the conventional + // hub names too. Customer `--bazel-maven-repo` extras are always probed — + // they exist precisely to cover non-conventional WORKSPACE hub names and + // custom Bzlmod extensions `mod show_extension` doesn't enumerate. const seen = new Set(candidates) - const toProbe = ( - showExtensionSucceeded ? [] : [...CONVENTIONAL_MAVEN_REPO_NAMES] - ).filter(name => !seen.has(name)) + const toProbe = [ + ...(showExtensionSucceeded ? [] : CONVENTIONAL_MAVEN_REPO_NAMES), + ...extras, + ].filter(name => !seen.has(name)) if (!toProbe.length) { return { candidates, discoveryIndeterminate, indeterminateProbes } } @@ -679,6 +696,12 @@ function buildQueryOpts(args: { ...(opts.bazelRc ? { bazelRc: opts.bazelRc } : {}), ...(opts.bazelFlags ? { bazelFlags: opts.bazelFlags } : {}), ...(opts.bazelOutputBase ? { bazelOutputBase: opts.bazelOutputBase } : {}), + ...(opts.extraBazelFlags?.length + ? { extraBazelFlags: opts.extraBazelFlags } + : {}), + ...(opts.extraBazelStartupFlags?.length + ? { extraBazelStartupFlags: opts.extraBazelStartupFlags } + : {}), ...(baseEnv ? { env: baseEnv } : {}), verbose, } @@ -697,6 +720,7 @@ export async function extractBazelToMaven( logger.groupEnd() const perRepoTimeoutMs = opts.perRepoTimeoutMs ?? DEFAULT_PER_REPO_TIMEOUT_MS + const extraMavenRepoNames = opts.extraMavenRepoNames ?? [] // Validate config + ensure toolchains BEFORE we mint a tempdir. let bin: string @@ -843,6 +867,7 @@ export async function extractBazelToMaven( workspaceRoot, mode, queryOptsFor(outputUserRoot), + extraMavenRepoNames, verbose, ) // Authoritative hub enumeration failed to execute (e.g. `bazel mod diff --git a/src/commands/manifest/bazel/extract_bazel_to_maven.test.mts b/src/commands/manifest/bazel/extract_bazel_to_maven.test.mts index 6b50ac7df..1d52d5a95 100644 --- a/src/commands/manifest/bazel/extract_bazel_to_maven.test.mts +++ b/src/commands/manifest/bazel/extract_bazel_to_maven.test.mts @@ -671,6 +671,49 @@ Fetched repositories: expect(runBazelModShowMavenExtension).not.toHaveBeenCalled() }) + it('threads extraMavenRepoNames into the candidate list (WORKSPACE mode)', async () => { + vi.mocked(detectWorkspaceMode).mockReturnValue({ + bzlmod: false, + workspace: true, + }) + // Probe accepts only `my_jars`; conventional names all return not-defined. + vi.mocked(buildMavenProbeFor).mockReturnValue(async (name: string) => { + if (name === 'my_jars') { + return { code: 0, stdout: '@my_jars//:foo\n', stderr: '' } + } + return { + code: 1, + stdout: '', + stderr: "ERROR: No repository visible as '@x' from main repository\n", + } + }) + vi.mocked(runMetadataCqueryForRepo).mockResolvedValueOnce( + mkResult({ + artifacts: [mkArt('com.example:custom:1.0', 'custom')], + repoName: 'my_jars', + }), + ) + const result = await extractBazelToMaven({ + bazelFlags: undefined, + bazelOutputBase: undefined, + bazelRc: undefined, + bin: undefined, + cwd: tmp, + extraMavenRepoNames: ['my_jars'], + out: tmp, + outLayout: 'flat', + verbose: false, + }) + expect(result.status).toBe('complete') + expect(result.artifactCount).toBe(1) + expect(runMetadataCqueryForRepo).toHaveBeenCalledTimes(1) + expect(vi.mocked(runMetadataCqueryForRepo).mock.calls[0]![0]).toMatchObject( + { repoName: 'my_jars' }, + ) + // show_extension must NOT be called in pure WORKSPACE mode. + expect(runBazelModShowMavenExtension).not.toHaveBeenCalled() + }) + it('narrates the per-hub cquery under verbose without changing the outcome', async () => { const logSpy = vi.spyOn(logger, 'log').mockImplementation(() => logger) try { @@ -1192,9 +1235,7 @@ Fetched repositories: expect(result.status).toBe('partial') expect(result.complete).toBe(false) expect(result.manifestPaths).toHaveLength(1) - const loadFailed = result.workspaceOutcomes.filter( - w => w.load === 'failed', - ) + const loadFailed = result.workspaceOutcomes.filter(w => w.load === 'failed') expect(loadFailed).toHaveLength(1) }) diff --git a/src/utils/glob.test.mts b/src/utils/glob.test.mts index 9cbd3c836..8967093e5 100644 --- a/src/utils/glob.test.mts +++ b/src/utils/glob.test.mts @@ -327,7 +327,8 @@ describe('glob utilities', () => { // pyvenv.cfg marker at its root. Manifests inside it must not surface. mockTestFs({ [`${mockFixturePath}/requirements.txt`]: '', - [`${mockFixturePath}/myenv/pyvenv.cfg`]: 'home = /usr/bin\nversion = 3.11.0\n', + [`${mockFixturePath}/myenv/pyvenv.cfg`]: + 'home = /usr/bin\nversion = 3.11.0\n', [`${mockFixturePath}/myenv/requirements.txt`]: '', [`${mockFixturePath}/myenv/lib/python3.11/site-packages/foo/setup.py`]: '',