From fdeec330b58dc29179d45b304fd36cdf2630b862 Mon Sep 17 00:00:00 2001 From: Tobias Schlatter Date: Thu, 18 Jun 2026 10:28:10 +0200 Subject: [PATCH] feat: auto-fetch node versions in module extension More aggressive alternative to #3916: We even do the fetching if facts are not available. --- e2e/nodejs_host/BUILD.bazel | 3 ++ e2e/nodejs_host/MODULE.bazel | 11 ++++ nodejs/BUILD.bazel | 9 +++- nodejs/extensions.bzl | 43 ++++++++++++++-- nodejs/private/BUILD.bazel | 10 ++++ nodejs/private/fetch_node_repositories.bzl | 58 ++++++++++++++++++++++ nodejs/private/version_from_attr.bzl | 28 +++++++++++ nodejs/repositories.bzl | 13 +---- 8 files changed, 160 insertions(+), 15 deletions(-) create mode 100644 nodejs/private/fetch_node_repositories.bzl create mode 100644 nodejs/private/version_from_attr.bzl diff --git a/e2e/nodejs_host/BUILD.bazel b/e2e/nodejs_host/BUILD.bazel index 1871163ff4..9a54ae2893 100644 --- a/e2e/nodejs_host/BUILD.bazel +++ b/e2e/nodejs_host/BUILD.bazel @@ -23,6 +23,7 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test") "node16", "node16_nvmrc", "node17_custom", + "node26_auto_fetch", ] ] @@ -66,5 +67,7 @@ load("@rules_shell//shell:sh_test.bzl", "sh_test") ("node16_nvmrc", "npx"), ("node17_custom", "npm"), ("node17_custom", "npx"), + ("node26_auto_fetch", "npm"), + ("node26_auto_fetch", "npx"), ] ] diff --git a/e2e/nodejs_host/MODULE.bazel b/e2e/nodejs_host/MODULE.bazel index 8c2b4ffc83..ff12b2fa3f 100644 --- a/e2e/nodejs_host/MODULE.bazel +++ b/e2e/nodejs_host/MODULE.bazel @@ -35,6 +35,10 @@ node.toolchain( node_urls = ["https://nodejs.org/dist/v17.0.1/{filename}"], node_version = "17.0.1.custom", ) +node.toolchain( + name = "node26_auto_fetch", + node_version = "26.3.0", +) # FIXME(6.0): a repo rule with name=foo should create a repo named @foo, not @foo_toolchains use_repo( @@ -60,6 +64,13 @@ use_repo( "node17_custom_linux_arm64", "node17_custom_toolchains", "node17_custom_windows_amd64", + "node26_auto_fetch", + "node26_auto_fetch_darwin_amd64", + "node26_auto_fetch_darwin_arm64", + "node26_auto_fetch_linux_amd64", + "node26_auto_fetch_linux_arm64", + "node26_auto_fetch_toolchains", + "node26_auto_fetch_windows_amd64", "nodejs", "nodejs_darwin_amd64", "nodejs_darwin_arm64", diff --git a/nodejs/BUILD.bazel b/nodejs/BUILD.bazel index 622b1f771c..202be5a59d 100644 --- a/nodejs/BUILD.bazel +++ b/nodejs/BUILD.bazel @@ -9,7 +9,13 @@ package(default_visibility = ["//visibility:public"]) bzl_library( name = "extensions", srcs = ["extensions.bzl"], - deps = [":repositories"], + deps = [ + ":repositories", + "//nodejs/private:fetch_node_repositories", + "//nodejs/private:node_versions", + "//nodejs/private:version_from_attr", + "@bazel_features//:features", + ], ) bzl_library( @@ -29,6 +35,7 @@ bzl_library( "//nodejs/private:nodejs_repo_host_os_alias", "//nodejs/private:nodejs_toolchains_repo", "//nodejs/private:os_name", + "//nodejs/private:version_from_attr", ], ) diff --git a/nodejs/extensions.bzl b/nodejs/extensions.bzl index 3b4b53f10c..720b0568bc 100644 --- a/nodejs/extensions.bzl +++ b/nodejs/extensions.bzl @@ -9,6 +9,10 @@ use_repo(node, "nodejs_toolchains") ``` """ +load("@bazel_features//:features.bzl", "bazel_features") +load("//nodejs/private:fetch_node_repositories.bzl", "fetch_node_repositories") +load("//nodejs/private:node_versions.bzl", "NODE_VERSIONS") +load("//nodejs/private:version_from_attr.bzl", "version_from_attr") load( ":repositories.bzl", "DEFAULT_NODE_REPOSITORY", @@ -52,17 +56,50 @@ def _toolchain_extension(module_ctx): else: registrations[toolchain.name] = toolchain + supports_facts = hasattr(module_ctx, "facts") + new_repository_facts = {} + for k, v in registrations.items(): + node_version = version_from_attr(module_ctx, v) + node_repositories = v.node_repositories or NODE_VERSIONS.get(node_version, {}) + + if not node_repositories: + node_repositories = ( + new_repository_facts.get(node_version) or + (module_ctx.facts.get(node_version) if supports_facts else None) or + # TODO: Add support for node_urls? + fetch_node_repositories(module_ctx, node_version) + ) + + new_repository_facts[node_version] = node_repositories + nodejs_register_toolchains( name = k, - node_version = v.node_version, - node_version_from_nvmrc = v.node_version_from_nvmrc, + node_version = node_version, node_urls = v.node_urls, - node_repositories = v.node_repositories, + node_repositories = node_repositories, include_headers = v.include_headers, register = False, ) + if not hasattr(module_ctx, "extension_metadata"): + return # buildifier: disable=return-value (allow no value) + + if not bazel_features.external_deps.extension_metadata_has_reproducible: + return module_ctx.extension_metadata() + + if not supports_facts: + return module_ctx.extension_metadata( + # if we have new_repository_facts is non-empty, we called + # fetch_node_repositories making this invocation non-reproducible. + reproducible = not new_repository_facts, + ) + + return module_ctx.extension_metadata( + reproducible = True, + facts = new_repository_facts, + ) + _ATTRS = { "name": attr.string( doc = "Base name for generated repositories", diff --git a/nodejs/private/BUILD.bazel b/nodejs/private/BUILD.bazel index 7388dc4f8b..9c92ac714f 100644 --- a/nodejs/private/BUILD.bazel +++ b/nodejs/private/BUILD.bazel @@ -41,3 +41,13 @@ bzl_library( srcs = ["user_build_settings.bzl"], deps = ["//nodejs/private/providers:user_build_settings"], ) + +bzl_library( + name = "version_from_attr", + srcs = ["version_from_attr.bzl"], +) + +bzl_library( + name = "fetch_node_repositories", + srcs = ["fetch_node_repositories.bzl"], +) diff --git a/nodejs/private/fetch_node_repositories.bzl b/nodejs/private/fetch_node_repositories.bzl new file mode 100644 index 0000000000..ab0627aaf9 --- /dev/null +++ b/nodejs/private/fetch_node_repositories.bzl @@ -0,0 +1,58 @@ +"""Implementation of node SHASUM fetching for facts.""" + +_REPOSITORY_TYPES = { + "darwin-arm64.tar.gz": "darwin_arm64", + "darwin-x64.tar.gz": "darwin_amd64", + "linux-x64.tar.xz": "linux_amd64", + "linux-arm64.tar.xz": "linux_arm64", + "linux-s390x.tar.xz": "linux_s390x", + "win-x64.zip": "windows_amd64", + "win-arm64.zip": "windows_arm64", + "linux-ppc64le.tar.xz": "linux_ppc64le", +} + +def fetch_node_repositories(module_ctx, version): + """Fetches node repositories for the given node version. + + Port of scripts/update-nodejs-versions.js + + Args: + module_ctx: Module context + version: The node version to fetch repositories for. + + Returns: + A dictionary in the shape of node_repositories. + """ + + shasums_filename = "{version}-SHASUMS256.txt".format(version = version) + url = "https://nodejs.org/dist/v{version}/SHASUMS256.txt".format(version = version) + + result = module_ctx.download(url = url, output = shasums_filename) + if not result.success: + fail("Failed to fetch node shasums:", url, result, sep = "\n") + + shasums = module_ctx.read(shasums_filename) + + result = {} + + for line in shasums.splitlines(): + line = line.strip() + if not line: + continue + + parts = line.split(" ") + if len(parts) != 2: + fail("{url} contains unexpected line:\n{line}".format( + url = url, + line = line, + )) + + sha, filename = parts + type = _REPOSITORY_TYPES.get(filename.removeprefix("node-v%s-" % version)) + if not type: + continue + + strip_prefix = filename.removesuffix(".tar.gz").removesuffix(".tar.xz").removesuffix(".zip") + result[version + "-" + type] = (filename, strip_prefix, sha) + + return result diff --git a/nodejs/private/version_from_attr.bzl b/nodejs/private/version_from_attr.bzl new file mode 100644 index 0000000000..6d7fa20655 --- /dev/null +++ b/nodejs/private/version_from_attr.bzl @@ -0,0 +1,28 @@ +"""Helper to get node version from user config.""" + +def _verify_version_is_valid(version): + major, minor, patch = (version.split(".") + [None, None, None])[:3] + if not major.isdigit() or not minor.isdigit() or not patch.isdigit(): + fail("Invalid node version: %s" % version) + +def version_from_attr(ctx, attr): + """Extract the node version from attr. + + Verifies if the extracted version is valid. + + Args: + ctx: repository or module context + attr: A struct with fields node_version and node_version_from_nvmrc + + Returns: + The node version. + """ + + node_version = attr.node_version + + if attr.node_version_from_nvmrc: + node_version = str(ctx.read(attr.node_version_from_nvmrc)).strip() + + _verify_version_is_valid(node_version) + + return node_version diff --git a/nodejs/repositories.bzl b/nodejs/repositories.bzl index 3a278cf0bd..d5f7ab29ce 100644 --- a/nodejs/repositories.bzl +++ b/nodejs/repositories.bzl @@ -3,6 +3,7 @@ load("//nodejs/private:node_versions.bzl", "NODE_VERSIONS") load("//nodejs/private:nodejs_repo_host_os_alias.bzl", "nodejs_repo_host_os_alias") load("//nodejs/private:nodejs_toolchains_repo.bzl", "PLATFORMS", "nodejs_toolchains_repo") +load("//nodejs/private:version_from_attr.bzl", "version_from_attr") # Default base name for node toolchain repositories # created by the module extension @@ -66,12 +67,7 @@ def _download_node(repository_ctx): # @nodejs_PLATFORM where PLATFORM is one of BUILT_IN_NODE_PLATFORMS host_os = repository_ctx.attr.platform or repository_ctx.name.split("nodejs_", 1)[1] - node_version = repository_ctx.attr.node_version - - if repository_ctx.attr.node_version_from_nvmrc: - node_version = str(repository_ctx.read(repository_ctx.attr.node_version_from_nvmrc)).strip() - - _verify_version_is_valid(node_version) + node_version = version_from_attr(repository_ctx, repository_ctx.attr) node_repositories = repository_ctx.attr.node_repositories @@ -302,11 +298,6 @@ def _strip_bin(path): return path[len("bin/"):] -def _verify_version_is_valid(version): - major, minor, patch = (version.split(".") + [None, None, None])[:3] - if not major.isdigit() or not minor.isdigit() or not patch.isdigit(): - fail("Invalid node version: %s" % version) - def _nodejs_repositories_impl(repository_ctx): reproducible = _download_node(repository_ctx) _prepare_node(repository_ctx)