From 65e967e7d9b5bdb899bfc86676f2e1703ea242e1 Mon Sep 17 00:00:00 2001 From: jennypng <63012604+JennyPng@users.noreply.github.com> Date: Fri, 29 May 2026 14:09:35 -0700 Subject: [PATCH] initial refactor --- .../http-client-python/emitter/src/emitter.ts | 197 +-------------- .../emitter/src/node-runner.browser.ts | 33 +++ .../emitter/src/node-runner.ts | 224 ++++++++++++++++++ .../emitter/src/pyodide-loader.browser.ts | 46 ++++ .../emitter/src/pyodide-loader.ts | 6 + packages/http-client-python/package.json | 4 + 6 files changed, 322 insertions(+), 188 deletions(-) create mode 100644 packages/http-client-python/emitter/src/node-runner.browser.ts create mode 100644 packages/http-client-python/emitter/src/node-runner.ts create mode 100644 packages/http-client-python/emitter/src/pyodide-loader.browser.ts create mode 100644 packages/http-client-python/emitter/src/pyodide-loader.ts diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 3ecd85c5f57..cbcc456fc3e 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -1,24 +1,17 @@ import { createSdkContext } from "@azure-tools/typespec-client-generator-core"; import { EmitContext, emitFile, joinPaths, NoTarget } from "@typespec/compiler"; -import { execSync } from "child_process"; -import fs from "fs"; import jsyaml from "js-yaml"; -import os from "os"; -import path, { dirname } from "path"; -import { loadPyodide, PyodideInterface } from "pyodide"; -import { fileURLToPath } from "url"; +import { loadPyodide, PyodideInterface } from "./pyodide-loader.js"; import pkgJson from "../../package.json" with { type: "json" }; import { emitCodeModel } from "./code-model.js"; import { - blackExcludeDirs, BLOB_STORAGE_BASE_URL, PACKAGE_NAME, PYGEN_WHEEL_FILENAME, PYODIDE_VERSION, } from "./constants.js"; -import { saveCodeModelAsYaml } from "./external-process.js"; import { PythonEmitterOptions, PythonSdkContext, reportDiagnostic } from "./lib.js"; -import { runPython3 } from "./run-python3.js"; +import { runNodeEmit } from "./node-runner.js"; import { getRootNamespace, md2Rst } from "./utils.js"; function getBrowserPygenWheelUrl(): string { @@ -193,7 +186,6 @@ async function onEmitMain(context: EmitContext) { const program = context.program; const sdkContext = await createPythonSdkContext(context); - const outputDir = context.emitterOutputDir; addDefaultOptions(sdkContext); const yamlMap = emitCodeModel(sdkContext); const parsedYamlMap = walkThroughNodes(yamlMap); @@ -255,83 +247,13 @@ async function onEmitMain(context: EmitContext) { await runPyodideGeneration(pyodide, "/output", yamlFilePath, commandArgs); await copyPyodideOutputToHost(context, pyodide, "/output"); } else { - const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); - const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap); - - if (!program.compilerOptions.noEmit && !program.hasError()) { - // If emit-yaml-only mode, just copy YAML to output dir for batch processing - if (resolvedOptions["emit-yaml-only"]) { - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - // Copy YAML to output dir with command args embedded - // Use unique filename to avoid conflicts when multiple specs share output dir - const configId = path.basename(yamlPath, ".yaml"); - const batchConfig = { yamlPath, commandArgs, outputDir }; - fs.writeFileSync( - path.join(outputDir, `.tsp-codegen-${configId}.json`), - JSON.stringify(batchConfig, null, 2), - ); - return; - } - // if not using pyodide and there's no venv, we try to create venv - if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) { - try { - await runPython3(path.join(root, "/eng/scripts/setup/install.py")); - await runPython3(path.join(root, "/eng/scripts/setup/prepare.py")); - } catch { - // if the python env is not ready, we use pyodide instead - resolvedOptions["use-pyodide"] = true; - } - } - - if (resolvedOptions["use-pyodide"]) { - // here we run with pyodide - const pyodide = await setupPyodideCall(root); - // create the output folder if not exists - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - // mount output folder to pyodide - pyodide.FS.mkdirTree("/output"); - pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output"); - // mount yaml file to pyodide - pyodide.FS.mkdirTree("/yaml"); - pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml"); - await runPyodideGeneration( - pyodide, - "/output", - `/yaml/${path.basename(yamlPath)}`, - commandArgs, - ); - } else { - // here we run with native python - let venvPath = path.join(root, "venv"); - if (fs.existsSync(path.join(venvPath, "bin"))) { - venvPath = path.join(venvPath, "bin", "python"); - } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { - venvPath = path.join(venvPath, "Scripts", "python.exe"); - } else { - reportDiagnostic(program, { - code: "pyodide-flag-conflict", - target: NoTarget, - }); - } - commandArgs["output-folder"] = outputDir; - commandArgs["tsp-file"] = yamlPath; - const commandFlags = Object.entries(commandArgs) - .map(([key, value]) => `--${key}=${value}`) - .join(" "); - const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`; - execSync(command); - - const excludePattern = blackExcludeDirs.join("|"); - execSync( - `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`, - ); - await checkForPylintIssues(outputDir, excludePattern); - } - } + await runNodeEmit({ + context, + parsedYamlMap, + commandArgs, + resolvedOptions, + runPyodideGeneration, + }); } } @@ -367,104 +289,3 @@ async function setupPyodideCallBrowser() { return pyodide; } - -async function setupPyodideCall(root: string) { - const pyodide = await loadPyodide({ - indexURL: path.dirname(fileURLToPath(import.meta.resolve("pyodide"))), - }); - const micropipLockPath = path.join(root, "micropip.lock"); - while (true) { - if (fs.existsSync(micropipLockPath)) { - try { - const stats = fs.statSync(micropipLockPath); - const now = new Date().getTime(); - const lockAge = (now - stats.mtime.getTime()) / 1000; - if (lockAge > 300) { - fs.unlinkSync(micropipLockPath); - } - } catch { - // ignore - } - } - try { - const fd = fs.openSync(micropipLockPath, "wx"); - // mount generator to pyodide - pyodide.FS.mkdirTree("/generator"); - pyodide.FS.mount( - pyodide.FS.filesystems.NODEFS, - { root: path.join(root, "generator") }, - "/generator", - ); - await pyodide.loadPackage("micropip"); - const micropip = pyodide.pyimport("micropip"); - await micropip.install(`emfs:/generator/dist/${PYGEN_WHEEL_FILENAME}`); - fs.closeSync(fd); - fs.unlinkSync(micropipLockPath); - break; - } catch { - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - } - return pyodide; -} - -async function checkForPylintIssues(outputDir: string, excludePattern: string) { - const excludeRegex = new RegExp(excludePattern); - - const shouldExcludePath = (filePath: string): boolean => { - const relativePath = path.relative(outputDir, filePath); - const normalizedPath = relativePath.replace(/\\/g, "/"); - return excludeRegex.test(normalizedPath); - }; - - const processFile = async (filePath: string) => { - let fileContent = await fs.promises.readFile(filePath, "utf-8"); - const pylintDisables: string[] = []; - const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n"; - const lines: string[] = fileContent.split(lineEnding); - if (lines.length > 0) { - if (!lines[0].includes("line-too-long") && lines.some((line) => line.length > 120)) { - pylintDisables.push("line-too-long", "useless-suppression"); - } - if (!lines[0].includes("too-many-lines") && lines.length > 1000) { - pylintDisables.push("too-many-lines"); - } - if (pylintDisables.length > 0) { - fileContent = lines[0].includes("pylint: disable=") - ? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding) - : `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent; - await fs.promises.writeFile(filePath, fileContent); - } - } - }; - - const collectPythonFiles = async (dir: string): Promise => { - if (shouldExcludePath(dir)) { - return []; - } - - const entries = await fs.promises.readdir(dir, { withFileTypes: true }); - - const promises = entries.map(async (entry) => { - const filePath = path.join(dir, entry.name); - - if (shouldExcludePath(filePath)) { - return []; - } - - if (entry.isDirectory()) { - return collectPythonFiles(filePath); - } else if (entry.name.endsWith(".py")) { - return [filePath]; - } - return []; - }); - - const results = await Promise.all(promises); - return results.flat(); - }; - - // Collect all Python files first, then process in parallel - const pythonFiles = await collectPythonFiles(outputDir); - await Promise.all(pythonFiles.map(processFile)); -} diff --git a/packages/http-client-python/emitter/src/node-runner.browser.ts b/packages/http-client-python/emitter/src/node-runner.browser.ts new file mode 100644 index 00000000000..06644c791ff --- /dev/null +++ b/packages/http-client-python/emitter/src/node-runner.browser.ts @@ -0,0 +1,33 @@ +// Browser stub for `node-runner.ts`. Swapped in via the `"browser"` field in +// `package.json`. The emitter's browser flow short-circuits before reaching +// `runNodeEmit`, so this stub is defense-in-depth — if it does get called, +// surface a clear diagnostic rather than a cryptic missing-module error. + +import { NoTarget } from "@typespec/compiler"; +import type { PyodideInterface } from "pyodide"; +import type { EmitContext } from "@typespec/compiler"; +import { PythonEmitterOptions, reportDiagnostic } from "./lib.js"; + +export interface RunNodeEmitArgs { + context: EmitContext; + parsedYamlMap: Record; + commandArgs: Record; + resolvedOptions: PythonEmitterOptions; + runPyodideGeneration: ( + pyodide: PyodideInterface, + outputFolder: string, + yamlFile: string, + commandArgs: Record, + ) => Promise; +} + +export async function runNodeEmit({ context }: RunNodeEmitArgs): Promise { + reportDiagnostic(context.program, { + code: "browser-runtime-load-failed", + target: NoTarget, + format: { + details: + " Native Python execution is not supported in the browser; the emitter must run in the in-browser Pyodide branch.", + }, + }); +} diff --git a/packages/http-client-python/emitter/src/node-runner.ts b/packages/http-client-python/emitter/src/node-runner.ts new file mode 100644 index 00000000000..ee8c71626ab --- /dev/null +++ b/packages/http-client-python/emitter/src/node-runner.ts @@ -0,0 +1,224 @@ +// Node-only execution path for the Python emitter. +// +// All direct usage of Node built-ins (`fs`, `os`, `path`, `url`, `child_process`) +// lives here so that the emitter entry can be bundled for the browser. A sibling +// `node-runner.browser.ts` stub is swapped in via the `"browser"` field in +// `package.json` when bundling with `platform: "browser"`. + +import { EmitContext, NoTarget } from "@typespec/compiler"; +import { execSync } from "child_process"; +import fs from "fs"; +import os from "os"; +import path, { dirname } from "path"; +import { loadPyodide, PyodideInterface } from "pyodide"; +import { fileURLToPath } from "url"; +import { blackExcludeDirs, PYGEN_WHEEL_FILENAME } from "./constants.js"; +import { saveCodeModelAsYaml } from "./external-process.js"; +import { PythonEmitterOptions, reportDiagnostic } from "./lib.js"; +import { runPython3 } from "./run-python3.js"; + +export interface RunNodeEmitArgs { + context: EmitContext; + parsedYamlMap: Record; + commandArgs: Record; + resolvedOptions: PythonEmitterOptions; + runPyodideGeneration: ( + pyodide: PyodideInterface, + outputFolder: string, + yamlFile: string, + commandArgs: Record, + ) => Promise; +} + +export async function runNodeEmit({ + context, + parsedYamlMap, + commandArgs, + resolvedOptions, + runPyodideGeneration, +}: RunNodeEmitArgs): Promise { + const program = context.program; + const outputDir = context.emitterOutputDir; + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + const yamlPath = await saveCodeModelAsYaml("python-yaml-path", parsedYamlMap); + + if (program.compilerOptions.noEmit || program.hasError()) { + return; + } + + // If emit-yaml-only mode, just copy YAML to output dir for batch processing + if (resolvedOptions["emit-yaml-only"]) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + // Copy YAML to output dir with command args embedded + // Use unique filename to avoid conflicts when multiple specs share output dir + const configId = path.basename(yamlPath, ".yaml"); + const batchConfig = { yamlPath, commandArgs, outputDir }; + fs.writeFileSync( + path.join(outputDir, `.tsp-codegen-${configId}.json`), + JSON.stringify(batchConfig, null, 2), + ); + return; + } + + // if not using pyodide and there's no venv, we try to create venv + if (!resolvedOptions["use-pyodide"] && !fs.existsSync(path.join(root, "venv"))) { + try { + await runPython3(path.join(root, "/eng/scripts/setup/install.py")); + await runPython3(path.join(root, "/eng/scripts/setup/prepare.py")); + } catch { + // if the python env is not ready, we use pyodide instead + resolvedOptions["use-pyodide"] = true; + } + } + + if (resolvedOptions["use-pyodide"]) { + // here we run with pyodide + const pyodide = await setupPyodideCall(root); + // create the output folder if not exists + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + // mount output folder to pyodide + pyodide.FS.mkdirTree("/output"); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: outputDir }, "/output"); + // mount yaml file to pyodide + pyodide.FS.mkdirTree("/yaml"); + pyodide.FS.mount(pyodide.FS.filesystems.NODEFS, { root: path.dirname(yamlPath) }, "/yaml"); + await runPyodideGeneration( + pyodide, + "/output", + `/yaml/${path.basename(yamlPath)}`, + commandArgs, + ); + return; + } + + // here we run with native python + let venvPath = path.join(root, "venv"); + if (fs.existsSync(path.join(venvPath, "bin"))) { + venvPath = path.join(venvPath, "bin", "python"); + } else if (fs.existsSync(path.join(venvPath, "Scripts"))) { + venvPath = path.join(venvPath, "Scripts", "python.exe"); + } else { + reportDiagnostic(program, { + code: "pyodide-flag-conflict", + target: NoTarget, + }); + } + commandArgs["output-folder"] = outputDir; + commandArgs["tsp-file"] = yamlPath; + const commandFlags = Object.entries(commandArgs) + .map(([key, value]) => `--${key}=${value}`) + .join(" "); + const command = `${venvPath} ${root}/eng/scripts/setup/run_tsp.py ${commandFlags}`; + execSync(command); + + const excludePattern = blackExcludeDirs.join("|"); + execSync( + `${venvPath} -m black --line-length=120 --quiet --fast ${outputDir} --exclude "${excludePattern}"`, + ); + await checkForPylintIssues(outputDir, excludePattern); +} + +async function setupPyodideCall(root: string): Promise { + const pyodide = await loadPyodide({ + indexURL: path.dirname(fileURLToPath(import.meta.resolve("pyodide"))), + }); + const micropipLockPath = path.join(root, "micropip.lock"); + while (true) { + if (fs.existsSync(micropipLockPath)) { + try { + const stats = fs.statSync(micropipLockPath); + const now = new Date().getTime(); + const lockAge = (now - stats.mtime.getTime()) / 1000; + if (lockAge > 300) { + fs.unlinkSync(micropipLockPath); + } + } catch { + // ignore + } + } + try { + const fd = fs.openSync(micropipLockPath, "wx"); + // mount generator to pyodide + pyodide.FS.mkdirTree("/generator"); + pyodide.FS.mount( + pyodide.FS.filesystems.NODEFS, + { root: path.join(root, "generator") }, + "/generator", + ); + await pyodide.loadPackage("micropip"); + const micropip = pyodide.pyimport("micropip"); + await micropip.install(`emfs:/generator/dist/${PYGEN_WHEEL_FILENAME}`); + fs.closeSync(fd); + fs.unlinkSync(micropipLockPath); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + return pyodide; +} + +async function checkForPylintIssues(outputDir: string, excludePattern: string) { + const excludeRegex = new RegExp(excludePattern); + + const shouldExcludePath = (filePath: string): boolean => { + const relativePath = path.relative(outputDir, filePath); + const normalizedPath = relativePath.replace(/\\/g, "/"); + return excludeRegex.test(normalizedPath); + }; + + const processFile = async (filePath: string) => { + let fileContent = await fs.promises.readFile(filePath, "utf-8"); + const pylintDisables: string[] = []; + const lineEnding = fileContent.includes("\r\n") && os.platform() === "win32" ? "\r\n" : "\n"; + const lines: string[] = fileContent.split(lineEnding); + if (lines.length > 0) { + if (!lines[0].includes("line-too-long") && lines.some((line) => line.length > 120)) { + pylintDisables.push("line-too-long", "useless-suppression"); + } + if (!lines[0].includes("too-many-lines") && lines.length > 1000) { + pylintDisables.push("too-many-lines"); + } + if (pylintDisables.length > 0) { + fileContent = lines[0].includes("pylint: disable=") + ? [lines[0] + "," + pylintDisables.join(",")].concat(lines.slice(1)).join(lineEnding) + : `# pylint: disable=${pylintDisables.join(",")}${lineEnding}` + fileContent; + await fs.promises.writeFile(filePath, fileContent); + } + } + }; + + const collectPythonFiles = async (dir: string): Promise => { + if (shouldExcludePath(dir)) { + return []; + } + + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + + const promises = entries.map(async (entry) => { + const filePath = path.join(dir, entry.name); + + if (shouldExcludePath(filePath)) { + return []; + } + + if (entry.isDirectory()) { + return collectPythonFiles(filePath); + } else if (entry.name.endsWith(".py")) { + return [filePath]; + } + return []; + }); + + const results = await Promise.all(promises); + return results.flat(); + }; + + // Collect all Python files first, then process in parallel + const pythonFiles = await collectPythonFiles(outputDir); + await Promise.all(pythonFiles.map(processFile)); +} diff --git a/packages/http-client-python/emitter/src/pyodide-loader.browser.ts b/packages/http-client-python/emitter/src/pyodide-loader.browser.ts new file mode 100644 index 00000000000..f8267c806cd --- /dev/null +++ b/packages/http-client-python/emitter/src/pyodide-loader.browser.ts @@ -0,0 +1,46 @@ +// Browser stub for `pyodide-loader.ts`. Loads pyodide via a `