From 0834a2c0b34c27d22b8e430c7eab135ee300bf79 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 21 Apr 2026 12:58:07 -0400 Subject: [PATCH 01/42] feat: add typeddict models-mode for Python HTTP client emitter Add a new 'typeddict' value for the models-mode option that generates Python TypedDict classes instead of DPG model classes. Key features: - TypedDict classes with Required[T]/NotRequired[T] annotations - TypedDict inheritance for non-discriminated models - Discriminated models: Union of leaf TypedDicts, no abstract base class - Input-only: operations accept TypedDict input, return dict output - Wire names used as TypedDict keys - _model_base.py still generated for serialization utilities Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http-client-python/emitter/src/types.ts | 2 +- .../eng/scripts/ci/regenerate-common.ts | 32 +++-- .../pygen/codegen/models/__init__.py | 4 +- .../pygen/codegen/models/code_model.py | 4 +- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/model_type.py | 15 +++ .../pygen/codegen/models/operation.py | 8 +- .../pygen/codegen/models/paging_operation.py | 2 +- .../pygen/codegen/models/parameter.py | 2 +- .../models/request_builder_parameter.py | 4 +- .../pygen/codegen/serializers/__init__.py | 12 +- .../codegen/serializers/builder_serializer.py | 21 +-- .../codegen/serializers/model_serializer.py | 122 ++++++++++++++++++ .../templates/model_container.py.jinja2 | 2 + .../templates/model_typeddict.py.jinja2 | 27 ++++ .../generator/pygen/preprocess/__init__.py | 4 +- 16 files changed, 227 insertions(+), 36 deletions(-) create mode 100644 packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 3e0ee6f6b32..1841efea310 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -289,7 +289,7 @@ function emitModel(context: PythonSdkContext, type: SdkModelType): Record>, properties: new Array>(), snakeCaseName: camelToSnakeCase(type.name), - base: "dpg", + base: (context.emitContext.options as any)["models-mode"] === "typeddict" ? "typeddict" : "dpg", internal: type.access === "internal", crossLanguageDefinitionId: type.crossLanguageDefinitionId, usage: type.usage, diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index 8dd426152b4..b686988e600 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -203,14 +203,30 @@ export const BASE_EMITTER_OPTIONS: Record< "package-name": "typetest-model-nesteddiscriminator", namespace: "typetest.model.nesteddiscriminator", }, - "type/model/inheritance/not-discriminated": { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - "type/model/inheritance/single-discriminator": { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, + "type/model/inheritance/not-discriminated": [ + { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + { + "package-name": "typetest-model-notdiscriminated-typeddict", + namespace: "typetest.model.notdiscriminated.typeddict", + "models-mode": "typeddict", + "generate-test": "false", + }, + ], + "type/model/inheritance/single-discriminator": [ + { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + { + "package-name": "typetest-model-singlediscriminator-typeddict", + namespace: "typetest.model.singlediscriminator.typeddict", + "models-mode": "typeddict", + "generate-test": "false", + }, + ], "type/model/inheritance/recursive": [ { "package-name": "typetest-model-recursive", diff --git a/packages/http-client-python/generator/pygen/codegen/models/__init__.py b/packages/http-client-python/generator/pygen/codegen/models/__init__.py index a1d9f9a4dbc..5848854ed86 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/models/__init__.py @@ -9,7 +9,7 @@ from .base_builder import BaseBuilder, ParameterListType from .code_model import CodeModel from .client import Client -from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType +from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType, TypedDictModelType from .dictionary_type import DictionaryType from .list_type import ListType from .combined_type import CombinedType @@ -171,6 +171,8 @@ def build_type(yaml_data: dict[str, Any], code_model: CodeModel) -> BaseType: model_type = JSONModelType elif yaml_data["base"] == "dpg": model_type = DPGModelType # type: ignore + elif yaml_data["base"] == "typeddict": + model_type = TypedDictModelType # type: ignore else: model_type = MsrestModelType # type: ignore response = model_type(yaml_data, code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 81f5f20bf8b..6a28ddfcaad 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -251,7 +251,7 @@ def need_utils_folder(self, async_mode: bool, client_namespace: str) -> bool: return ( self.need_utils_utils(async_mode, client_namespace) or self.need_utils_serialization - or self.options["models-mode"] == "dpg" + or self.options["models-mode"] in ("dpg", "typeddict") ) @property @@ -271,7 +271,7 @@ def need_utils_form_data(self, async_mode: bool, client_namespace: str) -> bool: (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data - and self.options["models-mode"] == "dpg" + and self.options["models-mode"] in ("dpg", "typeddict") ) def need_utils_etag(self, client_namespace: str) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index b7673d4d865..1a3d37bb85b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -125,7 +125,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if ( - self.code_model.options["models-mode"] == "dpg" + self.code_model.options["models-mode"] in ("dpg", "typeddict") and self.lro_response and self.lro_response.type and self.lro_response.type.type == "model" diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index d0784f81efb..069c572e5fa 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -374,3 +374,18 @@ def imports(self, **kwargs: Any) -> FileImport: if self.flattened_property: file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) return file_import + + +class TypedDictModelType(GeneratedModelType): + base = "typeddict" + + def serialization_type(self, **kwargs: Any) -> str: + return self.type_annotation(skip_quote=True, **kwargs) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, dict)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index c5f15593893..4ffe1169e11 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -51,7 +51,7 @@ def is_internal(target: Optional[BaseType]) -> bool: - return isinstance(target, ModelType) and target.base == "dpg" and target.internal + return isinstance(target, ModelType) and target.base in ("dpg", "typeddict") and target.internal class OperationBase( # pylint: disable=too-many-public-methods,too-many-instance-attributes @@ -176,7 +176,7 @@ def response_docstring_text(self, **kwargs) -> str: retval = self._response_docstring_helper("docstring_text", **kwargs) if not self.code_model.options["version-tolerant"]: retval += " or the result of cls(response)" - if self.code_model.options["models-mode"] == "dpg" and any( + if self.code_model.options["models-mode"] in ("dpg", "typeddict") and any( isinstance(r.type, ModelType) for r in self.responses ): r = next(r for r in self.responses if isinstance(r.type, ModelType)) @@ -209,7 +209,7 @@ def default_error_deserialization(self, serialize_namespace: str) -> Optional[st f"{exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)}," f"{pylint_disable}" ) - return None if self.code_model.options["models-mode"] == "dpg" else "'object'," + return None if self.code_model.options["models-mode"] in ("dpg", "typeddict") else "'object'," @property def non_default_errors(self) -> list[Response]: @@ -421,7 +421,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements for overload in self.overloads: if overload.parameters.has_body: file_import.merge(overload.parameters.body_parameter.type.imports(**kwargs)) - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index f64363ed5fb..3ce9d9abfea 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -181,7 +181,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: "case_insensitive_dict", ImportType.SDKCORE, ) - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 26eb0ba5c9a..34a765b849a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -336,7 +336,7 @@ def method_location( # pylint: disable=too-many-return-statements ) -> ParameterMethodLocation: if not self.in_method_signature: raise ValueError(f"Parameter '{self.client_name}' is not in the method.") - if self.code_model.options["models-mode"] == "dpg" and self.in_flattened_body: + if self.code_model.options["models-mode"] in ("dpg", "typeddict") and self.in_flattened_body: return ParameterMethodLocation.KEYWORD_ONLY if self.grouper: return ParameterMethodLocation.POSITIONAL diff --git a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py index c73df6db5af..393dc92edb4 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py @@ -26,7 +26,7 @@ def __init__(self, *args, **kwargs) -> None: if ( isinstance(self.type, (BinaryType, StringType)) or any("xml" in ct for ct in self.content_types) - or self.code_model.options["models-mode"] == "dpg" + or self.code_model.options["models-mode"] in ("dpg", "typeddict") ): self.client_name = "content" else: @@ -40,7 +40,7 @@ def type_annotation(self, **kwargs: Any) -> str: @property def in_method_signature(self) -> bool: return ( - super().in_method_signature and not self.is_partial_body and self.code_model.options["models-mode"] != "dpg" + super().in_method_signature and not self.is_partial_body and self.code_model.options["models-mode"] not in ("dpg", "typeddict") ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 018605575c9..897fd750769 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -25,7 +25,7 @@ from .enum_serializer import EnumSerializer from .general_serializer import GeneralSerializer from .model_init_serializer import ModelInitSerializer -from .model_serializer import DpgModelSerializer, MsrestModelSerializer +from .model_serializer import DpgModelSerializer, MsrestModelSerializer, TypedDictModelSerializer from .operations_init_serializer import OperationsInitSerializer from .operation_groups_serializer import OperationGroupsSerializer from .request_builders_serializer import RequestBuildersSerializer @@ -285,7 +285,13 @@ def _serialize_and_write_models_folder( ) -> None: # Write the models folder models_path = self.code_model.get_generation_dir(namespace) / "models" - serializer = DpgModelSerializer if self.code_model.options["models-mode"] == "dpg" else MsrestModelSerializer + models_mode = self.code_model.options["models-mode"] + if models_mode == "dpg": + serializer = DpgModelSerializer + elif models_mode == "typeddict": + serializer = TypedDictModelSerializer + else: + serializer = MsrestModelSerializer if self.code_model.has_non_json_models(models): self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), @@ -474,7 +480,7 @@ def _serialize_and_write_utils_folder(self, env: Environment, namespace: str): ) # write _model_base.py - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): self.write_file( utils_folder_path / Path("model_base.py"), general_serializer.serialize_model_base_file(), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index e43dcd916eb..3b45ff46dc3 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -29,6 +29,7 @@ CombinedType, JSONModelType, DPGModelType, + TypedDictModelType, ParameterListType, ByteArraySchema, ) @@ -711,7 +712,7 @@ def _serialize_body_parameter(self, builder: OperationType) -> list[str]: f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " f"'{serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" ) - elif self.code_model.options["models-mode"] == "dpg": + elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): if json_serializable(body_param.default_content_type): if hasattr(body_param.type, "encode") and body_param.type.encode: # type: ignore create_body_call = ( @@ -790,9 +791,9 @@ def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) overload.request_builder.parameters.body_parameter.client_name for overload in builder.overloads ] all_dpg_model_overloads = False - if self.code_model.options["models-mode"] == "dpg" and builder.overloads: + if self.code_model.options["models-mode"] in ("dpg", "typeddict") and builder.overloads: all_dpg_model_overloads = all( - isinstance(o.parameters.body_parameter.type, DPGModelType) for o in builder.overloads + isinstance(o.parameters.body_parameter.type, (DPGModelType, TypedDictModelType)) for o in builder.overloads ) if not all_dpg_model_overloads: for v in sorted(set(client_names), key=client_names.index): @@ -997,7 +998,7 @@ def response_deserialization( # pylint: disable=too-many-statements deserialize_code.append(f" '{serialization_type}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") - elif self.code_model.options["models-mode"] == "dpg": + elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): if builder.has_stream_response: deserialize_code.append("deserialized = response.content") else: @@ -1071,7 +1072,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): if xml_serializable(str(e.default_content_type)): fn = "_failsafe_deserialize_xml" else: @@ -1113,7 +1114,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): if xml_serializable(str(e.default_content_type)): retval.append( " error = _failsafe_deserialize_xml(" @@ -1141,7 +1142,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran indent = " " if builder.non_default_errors else " " if builder.non_default_errors: retval.append(" else:") - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): default_exception = next(e for e in builder.exceptions if "default" in e.status_codes and e.type) if xml_serializable(str(default_exception.default_content_type)): fn = "_failsafe_deserialize_xml" @@ -1410,7 +1411,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response{suffix}\n)" ) retval.append(f" deserialized = {deserialized}") - elif self.code_model.options["models-mode"] == "dpg": + elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): # we don't want to generate paging models for DPG retval.append(f" deserialized = {deserialized}") else: @@ -1428,7 +1429,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran "".join([f'.get("{i}", {{}})' for i in item_name_array[:-1]]) + f'.get("{item_name_array[-1]}", [])' ) pylint_disable = "" - if self.code_model.options["models-mode"] == "dpg": + if self.code_model.options["models-mode"] in ("dpg", "typeddict"): item_type = builder.item_type.type_annotation( is_operation_file=True, serialize_namespace=self.serialize_namespace ) @@ -1605,7 +1606,7 @@ def get_long_running_output(self, builder: LROOperationType) -> list[str]: retval.append(" response_headers = {}") if ( not self.code_model.options["models-mode"] - or self.code_model.options["models-mode"] == "dpg" + or self.code_model.options["models-mode"] in ("dpg", "typeddict") or builder.lro_response.headers ): retval.append(" response = pipeline_response.http_response") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index d428a113e5e..d34cc2310e6 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -382,3 +382,125 @@ def global_pylint_disables(self) -> str: if final_result: return "# pylint: disable=" + ", ".join(final_result) return "" + + +class TypedDictModelSerializer(_ModelSerializer): + def _is_parent_discriminated_base(self, model: ModelType) -> bool: + """Check if any parent of this model is a discriminated base (has discriminated_subtypes).""" + return any(p.discriminated_subtypes for p in model.parents) + + def _reorder_models(self, models: list[ModelType]) -> list[ModelType]: + """Reorder so discriminated base Union aliases come after all their subtypes.""" + bases = [m for m in models if m.discriminated_subtypes] + non_bases = [m for m in models if not m.discriminated_subtypes] + return non_bases + bases + + def serialize(self) -> str: + template = self.env.get_template("model_container.py.jinja2") + return template.render( + code_model=self.code_model, + imports=FileImportSerializer(self.imports()), + str=str, + serializer=self, + models=self._reorder_models(self.models), + ) + + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + has_required = False + has_optional = False + has_discriminated_union = False + for model in self.models: + if model.base == "json": + continue + if model.discriminated_subtypes: + has_discriminated_union = True + file_import.merge( + model.imports( + is_operation_file=False, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.MODEL, + ) + ) + for prop in model.properties: + file_import.merge( + prop.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.MODEL, + called_by_property=True, + ) + ) + if prop.optional or prop.client_default_value is not None: + has_optional = True + else: + has_required = True + for parent in model.parents: + if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + self.serialize_namespace, + self.code_model.get_imported_namespace_for_model(parent.client_namespace), + ), + parent.name, + ImportType.LOCAL, + ) + file_import.add_submodule_import("typing", "TypedDict", ImportType.STDLIB) + if has_optional: + file_import.add_submodule_import("typing", "NotRequired", ImportType.STDLIB) + if has_required: + file_import.add_submodule_import("typing", "Required", ImportType.STDLIB) + if has_discriminated_union: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) + return file_import + + def declare_model(self, model: ModelType) -> str: + # If the model's parent is a discriminated base, don't inherit from it + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + basename = ", ".join([m.name for m in non_discriminated_parents]) + return f"class {model.name}({basename}):{model.pylint_disable()}" + return f"class {model.name}(TypedDict, total=False):{model.pylint_disable()}" + + @staticmethod + def get_properties_to_declare(model: ModelType) -> list[Property]: + # Only exclude inherited properties from non-discriminated parents + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + parent_properties = [p for bm in non_discriminated_parents for p in bm.properties] + properties_to_declare = [ + p + for p in model.properties + if not any( + p.client_name == pp.client_name + and p.type_annotation() == pp.type_annotation() + and not p.is_base_discriminator + for pp in parent_properties + ) + ] + else: + properties_to_declare = model.properties + return properties_to_declare + + def declare_property(self, prop: Property) -> str: + type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + is_optional = prop.optional or prop.client_default_value is not None + if is_optional: + return f"{prop.wire_name}: NotRequired[{type_annotation}]" + return f"{prop.wire_name}: Required[{type_annotation}]" + + def initialize_properties(self, model: ModelType) -> list[str]: + return [] + + def need_init(self, model: ModelType) -> bool: + return False + + def discriminated_subtypes_union(self, model: ModelType) -> str: + subtypes = list(model.discriminated_subtypes.values()) + subtype_names = [s.name for s in subtypes] + return f"{model.name} = Union[{', '.join(subtype_names)}]" + + def is_discriminated_base(self, model: ModelType) -> bool: + return bool(model.discriminated_subtypes) + + def global_pylint_disables(self) -> str: + return "" diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 index dc98f999c45..6e033ba77fb 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 @@ -13,5 +13,7 @@ {% include "model_dpg.py.jinja2" %} {% elif model.base == "msrest" %} {% include "model_msrest.py.jinja2" %} +{% elif model.base == "typeddict" %} +{% include "model_typeddict.py.jinja2" %} {% endif %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 new file mode 100644 index 00000000000..082ec87cf9f --- /dev/null +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 @@ -0,0 +1,27 @@ +{# actual template starts here #} +{% import "macros.jinja2" as macros %} + +{% if serializer.is_discriminated_base(model) %} +{{ serializer.discriminated_subtypes_union(model) }} +{% else %} + +{{ serializer.declare_model(model) }} + """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} + + {% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} + {{ macros.wrap_model_string(line, '\n ') -}} + {% endfor %} + {% endfor %} + {% endif %} + """ + + {% for p in serializer.get_properties_to_declare(model)%} + {{ serializer.declare_property(p) }} + {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} + {% if prop_description %} + """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} + {% endif %} + {% endfor %} +{% endif %} diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index 6d3344059a3..be029d64f3c 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -216,7 +216,7 @@ def add_body_param_type( model_type = ( body_parameter["type"] if origin_type == "model" else body_parameter["type"].get("elementType", {}) ) - is_dpg_model = model_type.get("base") == "dpg" + is_dpg_model = model_type.get("base") in ("dpg", "typeddict") body_parameter["type"] = { "type": "combined", "types": [body_parameter["type"]], @@ -225,7 +225,7 @@ def add_body_param_type( if not (self.is_tsp and has_multi_part_content_type(body_parameter)): body_parameter["type"]["types"].append(KNOWN_TYPES["binary"]) - if self.options["models-mode"] == "dpg" and is_dpg_model: + if self.options["models-mode"] in ("dpg", "typeddict") and is_dpg_model: if origin_type == "model": body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"]) else: From 27c8bfb31577bf7c03b13726a8f5803b38b63bc2 Mon Sep 17 00:00:00 2001 From: iscai-msft <43154838+iscai-msft@users.noreply.github.com> Date: Tue, 21 Apr 2026 13:49:59 -0400 Subject: [PATCH 02/42] Enhance Python HTTP client emitter with TypedDict support --- .chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md diff --git a/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md new file mode 100644 index 00000000000..25dfde1e67f --- /dev/null +++ b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md @@ -0,0 +1,8 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: fix +packages: + - "@typespec/http-client-python" +--- + +[python] add `typeddict` `models-mode` for Python HTTP client emitter to generated `TypedDict`s for input models From c1487f899c6d5cee216a8616ed8ee290b83549cf Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 21 Apr 2026 14:37:31 -0400 Subject: [PATCH 03/42] add discriminator --- .../models/request_builder_parameter.py | 4 +- .../pygen/codegen/serializers/__init__.py | 48 +++++++++++++++ .../codegen/serializers/builder_serializer.py | 3 +- .../codegen/serializers/model_serializer.py | 6 +- .../templates/model_typeddict.py.jinja2 | 1 + ...inheritance_not_discriminated_typeddict.py | 31 ++++++++++ ...eritance_single_discriminator_typeddict.py | 58 +++++++++++++++++++ 7 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py create mode 100644 packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py diff --git a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py index 393dc92edb4..810fbf3b7e7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py @@ -40,7 +40,9 @@ def type_annotation(self, **kwargs: Any) -> str: @property def in_method_signature(self) -> bool: return ( - super().in_method_signature and not self.is_partial_body and self.code_model.options["models-mode"] not in ("dpg", "typeddict") + super().in_method_signature + and not self.is_partial_body + and self.code_model.options["models-mode"] not in ("dpg", "typeddict") ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 897fd750769..f07a9078c3f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -22,6 +22,7 @@ ModelType, EnumType, ) +from ..models.primitive_types import DatetimeType, ByteArraySchema, BinaryType from .enum_serializer import EnumSerializer from .general_serializer import GeneralSerializer from .model_init_serializer import ModelInitSerializer @@ -118,8 +119,55 @@ def keep_version_file(self) -> bool: # If parsing the version fails, we assume the version file is not valid and overwrite. return False + @staticmethod + def _validate_typeddict_models(code_model: CodeModel) -> None: + """Validate that models are compatible with typeddict mode. + + Raises ValueError if any model uses unsupported features: + readonly properties, datetime types, bytes types, + or additional properties (extends Record). + """ + unsupported: list[str] = [] + for model in code_model.model_types: + if model.base != "typeddict": + continue + model_name = model.name + + for prop in model.properties: + # Readonly + if prop.readonly: + unsupported.append( + f"Model '{model_name}' has readonly property '{prop.client_name}', " + "which is not supported in typeddict mode." + ) + # Datetime + if isinstance(prop.type, DatetimeType): + unsupported.append( + f"Model '{model_name}' has datetime property '{prop.client_name}', " + "which is not supported in typeddict mode." + ) + # Bytes + if isinstance(prop.type, (ByteArraySchema, BinaryType)): + unsupported.append( + f"Model '{model_name}' has bytes property '{prop.client_name}', " + "which is not supported in typeddict mode." + ) + # Additional properties (extends Record) + if prop.client_name == "additional_properties": + unsupported.append( + f"Model '{model_name}' has additional properties (extends Record), " + "which is not supported in typeddict mode." + ) + + if unsupported: + raise ValueError("The following models are not compatible with typeddict mode:\n" + "\n".join(unsupported)) + # pylint: disable=too-many-branches def serialize(self) -> None: + # Validate typeddict mode constraints + if self.code_model.options.get("models-mode") == "typeddict": + self._validate_typeddict_models(self.code_model) + # remove existing folders when generate from tsp if self.code_model.is_tsp and self.code_model.options.get("clear-output-folder"): # remove generated_samples and generated_tests folder diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 3b45ff46dc3..784c049ea1a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -793,7 +793,8 @@ def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) all_dpg_model_overloads = False if self.code_model.options["models-mode"] in ("dpg", "typeddict") and builder.overloads: all_dpg_model_overloads = all( - isinstance(o.parameters.body_parameter.type, (DPGModelType, TypedDictModelType)) for o in builder.overloads + isinstance(o.parameters.body_parameter.type, (DPGModelType, TypedDictModelType)) + for o in builder.overloads ) if not all_dpg_model_overloads: for v in sorted(set(client_names), key=client_names.index): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index d34cc2310e6..c42ad88421b 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -444,11 +444,11 @@ def imports(self) -> FileImport: parent.name, ImportType.LOCAL, ) - file_import.add_submodule_import("typing", "TypedDict", ImportType.STDLIB) + file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) if has_optional: - file_import.add_submodule_import("typing", "NotRequired", ImportType.STDLIB) + file_import.add_submodule_import("typing_extensions", "NotRequired", ImportType.STDLIB) if has_required: - file_import.add_submodule_import("typing", "Required", ImportType.STDLIB) + file_import.add_submodule_import("typing_extensions", "Required", ImportType.STDLIB) if has_discriminated_union: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 index 082ec87cf9f..8176626fe3f 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 @@ -25,3 +25,4 @@ {% endif %} {% endfor %} {% endif %} + diff --git a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py new file mode 100644 index 00000000000..782b3b3b340 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from typetest.model.notdiscriminated.typeddict import NotDiscriminatedClient +from typetest.model.notdiscriminated.typeddict.models import Siamese + + +@pytest.fixture +def client(): + with NotDiscriminatedClient() as client: + yield client + + +@pytest.fixture +def valid_body(): + return Siamese(name="abc", age=32, smart=True) + + +def test_get_valid(client, valid_body): + assert client.get_valid() == valid_body + + +def test_post_valid(client, valid_body): + client.post_valid(valid_body) + + +def test_put_valid(client, valid_body): + assert valid_body == client.put_valid(valid_body) diff --git a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py new file mode 100644 index 00000000000..ffa676409b4 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py @@ -0,0 +1,58 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from typetest.model.singlediscriminator.typeddict import SingleDiscriminatorClient +from typetest.model.singlediscriminator.typeddict.models import Sparrow, Eagle, Dinosaur, TRex + + +@pytest.fixture +def client(): + with SingleDiscriminatorClient() as client: + yield client + + +@pytest.fixture +def valid_body(): + return Sparrow(wingspan=1, kind="sparrow") + + +def test_get_model(client, valid_body): + assert client.get_model() == valid_body + + +def test_put_model(client, valid_body): + client.put_model(valid_body) + + +@pytest.fixture +def recursive_body(): + return Eagle( + wingspan=5, + kind="eagle", + partner={"wingspan": 2, "kind": "goose"}, + friends=[{"wingspan": 2, "kind": "seagull"}], + hate={"key3": {"wingspan": 1, "kind": "sparrow"}}, + ) + + +def test_get_recursive_model(client, recursive_body): + assert client.get_recursive_model() == recursive_body + + +def test_put_recursive_model(client, recursive_body): + client.put_recursive_model(recursive_body) + + +def test_get_missing_discriminator(client): + assert client.get_missing_discriminator() == {"wingspan": 1} + + +def test_get_wrong_discriminator(client): + assert client.get_wrong_discriminator() == {"wingspan": 1, "kind": "wrongKind"} + + +def test_get_legacy_model(client): + assert client.get_legacy_model() == TRex(size=20, kind="t-rex") From 3a125b87fd0ec80a27c7a176a33b3da3b9e494c7 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Apr 2026 11:03:34 -0400 Subject: [PATCH 04/42] feat: return JSON for typeddict responses, drop NotRequired - TypedDictModelType returns 'JSON' for response type annotations - Response.type_annotation/docstring passes is_response=True - Typeddict deserialization uses response.json() directly - Removed NotRequired from TypedDictModelSerializer (total=False handles it) - Updated mock API tests to verify JSON dict responses Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/models/model_type.py | 22 +++++++++++---- .../pygen/codegen/models/response.py | 3 ++ .../codegen/serializers/builder_serializer.py | 6 ++++ .../codegen/serializers/model_serializer.py | 9 ++---- ...inheritance_not_discriminated_typeddict.py | 10 +++++-- ...eritance_single_discriminator_typeddict.py | 28 +++++++++++++------ 6 files changed, 55 insertions(+), 23 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 069c572e5fa..572169771fa 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -376,16 +376,26 @@ def imports(self, **kwargs: Any) -> FileImport: return file_import -class TypedDictModelType(GeneratedModelType): +class TypedDictModelType(DPGModelType): base = "typeddict" - def serialization_type(self, **kwargs: Any) -> str: - return self.type_annotation(skip_quote=True, **kwargs) + def type_annotation(self, **kwargs: Any) -> str: + if kwargs.pop("is_response", False): + return "JSON" + return super().type_annotation(**kwargs) - @property - def instance_check_template(self) -> str: - return "isinstance({}, dict)" + def docstring_type(self, **kwargs: Any) -> str: + if kwargs.pop("is_response", False): + return "JSON" + return super().docstring_type(**kwargs) + + def docstring_text(self, **kwargs: Any) -> str: + if kwargs.pop("is_response", False): + return "JSON" + return super().docstring_text(**kwargs) def imports(self, **kwargs: Any) -> FileImport: file_import = super().imports(**kwargs) + file_import.add_submodule_import("collections.abc", "MutableMapping", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index d93d46bd897..40599eb47ae 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -95,6 +95,7 @@ def serialization_type(self, **kwargs: Any) -> str: def type_annotation(self, **kwargs: Any) -> str: if self.type: kwargs["is_operation_file"] = True + kwargs["is_response"] = True type_annotation = self.type.type_annotation(**kwargs) if self.nullable: return f"Optional[{type_annotation}]" @@ -102,11 +103,13 @@ def type_annotation(self, **kwargs: Any) -> str: return "None" def docstring_text(self, **kwargs: Any) -> str: + kwargs["is_response"] = True if self.nullable and self.type: return f"{self.type.docstring_text(**kwargs)} or None" return self.type.docstring_text(**kwargs) if self.type else "None" def docstring_type(self, **kwargs: Any) -> str: + kwargs["is_response"] = True if self.nullable and self.type: return f"{self.type.docstring_type(**kwargs)} or None" return self.type.docstring_type(**kwargs) if self.type else "None" diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 784c049ea1a..9c5b14440af 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -999,6 +999,12 @@ def response_deserialization( # pylint: disable=too-many-statements deserialize_code.append(f" '{serialization_type}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") + elif self.code_model.options["models-mode"] == "typeddict": + if builder.has_stream_response: + deserialize_code.append("deserialized = response.content") + else: + response_attr = "json" if json_serializable(str(response.default_content_type)) else "text" + deserialize_code.append(f"deserialized = response.{response_attr}()") elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): if builder.has_stream_response: deserialize_code.append("deserialized = response.content") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py index c42ad88421b..27d4451ae2f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/model_serializer.py @@ -408,7 +408,6 @@ def serialize(self) -> str: def imports(self) -> FileImport: file_import = FileImport(self.code_model) has_required = False - has_optional = False has_discriminated_union = False for model in self.models: if model.base == "json": @@ -430,9 +429,7 @@ def imports(self) -> FileImport: called_by_property=True, ) ) - if prop.optional or prop.client_default_value is not None: - has_optional = True - else: + if not (prop.optional or prop.client_default_value is not None): has_required = True for parent in model.parents: if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: @@ -445,8 +442,6 @@ def imports(self) -> FileImport: ImportType.LOCAL, ) file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) - if has_optional: - file_import.add_submodule_import("typing_extensions", "NotRequired", ImportType.STDLIB) if has_required: file_import.add_submodule_import("typing_extensions", "Required", ImportType.STDLIB) if has_discriminated_union: @@ -485,7 +480,7 @@ def declare_property(self, prop: Property) -> str: type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) is_optional = prop.optional or prop.client_default_value is not None if is_optional: - return f"{prop.wire_name}: NotRequired[{type_annotation}]" + return f"{prop.wire_name}: {type_annotation}" return f"{prop.wire_name}: Required[{type_annotation}]" def initialize_properties(self, model: ModelType) -> list[str]: diff --git a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py index 782b3b3b340..782791ab1e7 100644 --- a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py +++ b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_not_discriminated_typeddict.py @@ -20,7 +20,10 @@ def valid_body(): def test_get_valid(client, valid_body): - assert client.get_valid() == valid_body + result = client.get_valid() + assert result["name"] == "abc" + assert result["age"] == 32 + assert result["smart"] is True def test_post_valid(client, valid_body): @@ -28,4 +31,7 @@ def test_post_valid(client, valid_body): def test_put_valid(client, valid_body): - assert valid_body == client.put_valid(valid_body) + result = client.put_valid(valid_body) + assert result["name"] == "abc" + assert result["age"] == 32 + assert result["smart"] is True diff --git a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py index ffa676409b4..19335d1bc69 100644 --- a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py +++ b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_inheritance_single_discriminator_typeddict.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest from typetest.model.singlediscriminator.typeddict import SingleDiscriminatorClient -from typetest.model.singlediscriminator.typeddict.models import Sparrow, Eagle, Dinosaur, TRex +from typetest.model.singlediscriminator.typeddict.models import Sparrow, Eagle @pytest.fixture @@ -19,8 +19,10 @@ def valid_body(): return Sparrow(wingspan=1, kind="sparrow") -def test_get_model(client, valid_body): - assert client.get_model() == valid_body +def test_get_model(client): + result = client.get_model() + assert result["wingspan"] == 1 + assert result["kind"] == "sparrow" def test_put_model(client, valid_body): @@ -38,8 +40,13 @@ def recursive_body(): ) -def test_get_recursive_model(client, recursive_body): - assert client.get_recursive_model() == recursive_body +def test_get_recursive_model(client): + result = client.get_recursive_model() + assert result["wingspan"] == 5 + assert result["kind"] == "eagle" + assert result["partner"]["kind"] == "goose" + assert result["friends"][0]["kind"] == "seagull" + assert result["hate"]["key3"]["kind"] == "sparrow" def test_put_recursive_model(client, recursive_body): @@ -47,12 +54,17 @@ def test_put_recursive_model(client, recursive_body): def test_get_missing_discriminator(client): - assert client.get_missing_discriminator() == {"wingspan": 1} + result = client.get_missing_discriminator() + assert result["wingspan"] == 1 def test_get_wrong_discriminator(client): - assert client.get_wrong_discriminator() == {"wingspan": 1, "kind": "wrongKind"} + result = client.get_wrong_discriminator() + assert result["wingspan"] == 1 + assert result["kind"] == "wrongKind" def test_get_legacy_model(client): - assert client.get_legacy_model() == TRex(size=20, kind="t-rex") + result = client.get_legacy_model() + assert result["size"] == 20 + assert result["kind"] == "t-rex" From e0569dba5b04e4240504ad262865934d3eca3aa0 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Apr 2026 12:48:31 -0400 Subject: [PATCH 05/42] feat: add wire name mock API tests for typeddict naming spec - Add client/naming typeddict variant to regenerate-common.ts - Create test_client_naming_typeddict.py with 11 tests verifying TypedDict uses wire names (defaultName, wireName) not client names - Tests cover: ClientNameModel, LanguageClientNameModel, ClientNameAndJsonEncodedNameModel, ClientModel, PythonModel Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../eng/scripts/ci/regenerate-common.ts | 14 +++- .../azure/test_client_naming_typeddict.py | 64 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index b686988e600..9abd4a57a8f 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -105,9 +105,17 @@ export const BASE_AZURE_EMITTER_OPTIONS: Record< "package-name": "client-structure-twooperationgroup", namespace: "client.structure.twooperationgroup", }, - "client/naming": { - namespace: "client.naming.main", - }, + "client/naming": [ + { + namespace: "client.naming.main", + }, + { + "package-name": "client-naming-typeddict", + namespace: "client.naming.typeddict", + "models-mode": "typeddict", + "generate-test": "false", + }, + ], "client/overload": { namespace: "client.overload", }, diff --git a/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py b/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py new file mode 100644 index 00000000000..40786f16e39 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from client.naming.typeddict import NamingClient, models + + +@pytest.fixture +def client(): + with NamingClient() as client: + yield client + + +def test_client(client: NamingClient): + """TypedDict uses wire name 'defaultName', not client name 'client_name'.""" + client.property.client(models.ClientNameModel(defaultName=True)) + + +def test_language(client: NamingClient): + """TypedDict uses wire name 'defaultName', not language-specific name 'python_name'.""" + client.property.language(models.LanguageClientNameModel(defaultName=True)) + + +def test_compatible_with_encoded_name(client: NamingClient): + """TypedDict uses encoded wire name 'wireName', not client name 'client_name'.""" + client.property.compatible_with_encoded_name( + models.ClientNameAndJsonEncodedNameModel(wireName=True) + ) + + +def test_operation(client: NamingClient): + client.client_name() + + +def test_parameter(client: NamingClient): + client.parameter(client_name="true") + + +def test_header_request(client: NamingClient): + client.header.request(client_name="true") + + +def test_header_response(client: NamingClient): + assert client.header.response(cls=lambda x, y, z: z)["default-name"] == "true" + + +def test_model_client(client: NamingClient): + """TypedDict uses wire name 'defaultName', not client name 'default_name'.""" + client.model_client.client(models.ClientModel(defaultName=True)) + + +def test_model_language(client: NamingClient): + """TypedDict uses wire name 'defaultName', not client name 'default_name'.""" + client.model_client.language(models.PythonModel(defaultName=True)) + + +def test_union_enum_member_name(client: NamingClient): + client.union_enum.union_enum_member_name(models.ExtensibleEnum.CLIENT_ENUM_VALUE1) + + +def test_union_enum_name(client: NamingClient): + client.union_enum.union_enum_name(models.ClientExtensibleEnum.ENUM_VALUE1) From 3e5c0ef1595ee055ae92069dc0c2decd15435176 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Apr 2026 14:06:49 -0400 Subject: [PATCH 06/42] fix: remove redundant JSON overload for typeddict mode TypedDict is already JSON, so the MutableMapping[str, Any] overload is unnecessary. Only keep TypedDict model + IO[bytes] overloads. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http-client-python/generator/pygen/preprocess/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index be029d64f3c..5117a9227e5 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -225,7 +225,7 @@ def add_body_param_type( if not (self.is_tsp and has_multi_part_content_type(body_parameter)): body_parameter["type"]["types"].append(KNOWN_TYPES["binary"]) - if self.options["models-mode"] in ("dpg", "typeddict") and is_dpg_model: + if self.options["models-mode"] == "dpg" and is_dpg_model: if origin_type == "model": body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"]) else: From f76c88d3dc398a81d896eac4c4961cae878bcb27 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Apr 2026 14:34:33 -0400 Subject: [PATCH 07/42] fix: remove unused _deserialize import in typeddict mode Typeddict mode uses response.json() directly, so _deserialize is never called. Skip importing it to avoid W0611 unused-import lint warning. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/operation.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 4ffe1169e11..ac079658283 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -449,11 +449,14 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements file_import.add_import("json", ImportType.STDLIB) if self.enable_import_deserialize_xml: file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL) - if any( - r.type - and not isinstance(r.type, BinaryIteratorType) - and not xml_serializable(str(r.default_content_type)) - for r in self.responses + if ( + self.code_model.options["models-mode"] != "typeddict" + and any( + r.type + and not isinstance(r.type, BinaryIteratorType) + and not xml_serializable(str(r.default_content_type)) + for r in self.responses + ) ): file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.default_error_deserialization(serialize_namespace) or self.non_default_errors: From 2401b663f9eb29a73d5ee938d1f6215e40e9d920 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 23 Apr 2026 15:29:49 -0400 Subject: [PATCH 08/42] fix: remove all unused imports in typeddict generated code - Remove unused MutableMapping/Any imports from TypedDictModelType.imports() - Skip _deserialize import in paging_operation.py for typeddict mode Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 5 +---- .../generator/pygen/codegen/models/paging_operation.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 572169771fa..54a836da6c2 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -395,7 +395,4 @@ def docstring_text(self, **kwargs: Any) -> str: return super().docstring_text(**kwargs) def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - file_import.add_submodule_import("collections.abc", "MutableMapping", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) - return file_import + return super().imports(**kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 3ce9d9abfea..832f31a554c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -186,7 +186,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: serialize_namespace, module_name="_utils.model_base" ) file_import.merge(self.item_type.imports(**kwargs)) - if self.default_error_deserialization(serialize_namespace) or self.need_deserialize: + if (self.default_error_deserialization(serialize_namespace) or self.need_deserialize) and self.code_model.options["models-mode"] != "typeddict": file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.is_xml_paging: file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") From 47b5024a3ea951f2f4f4c2e51a71728dad33e816 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 27 Apr 2026 12:03:35 -0400 Subject: [PATCH 09/42] format and lint --- .../generator/pygen/codegen/models/model_type.py | 3 --- .../generator/pygen/codegen/models/operation.py | 13 +++++-------- .../pygen/codegen/models/paging_operation.py | 4 +++- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 54a836da6c2..192493dede9 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -393,6 +393,3 @@ def docstring_text(self, **kwargs: Any) -> str: if kwargs.pop("is_response", False): return "JSON" return super().docstring_text(**kwargs) - - def imports(self, **kwargs: Any) -> FileImport: - return super().imports(**kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index ac079658283..60737410140 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -449,14 +449,11 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements file_import.add_import("json", ImportType.STDLIB) if self.enable_import_deserialize_xml: file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL) - if ( - self.code_model.options["models-mode"] != "typeddict" - and any( - r.type - and not isinstance(r.type, BinaryIteratorType) - and not xml_serializable(str(r.default_content_type)) - for r in self.responses - ) + if self.code_model.options["models-mode"] != "typeddict" and any( + r.type + and not isinstance(r.type, BinaryIteratorType) + and not xml_serializable(str(r.default_content_type)) + for r in self.responses ): file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.default_error_deserialization(serialize_namespace) or self.non_default_errors: diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index 832f31a554c..e6fb19fe3db 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -186,7 +186,9 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: serialize_namespace, module_name="_utils.model_base" ) file_import.merge(self.item_type.imports(**kwargs)) - if (self.default_error_deserialization(serialize_namespace) or self.need_deserialize) and self.code_model.options["models-mode"] != "typeddict": + if ( + self.default_error_deserialization(serialize_namespace) or self.need_deserialize + ) and self.code_model.options["models-mode"] != "typeddict": file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.is_xml_paging: file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") From 95db199249e6bd99d19b7616d3a2d76ec252cd8e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 28 Apr 2026 15:03:56 -0400 Subject: [PATCH 10/42] fix: define JSON type alias in TypedDictModelType imports TypedDictModelType returns 'JSON' for response type annotations but never defined the JSON = MutableMapping[str, Any] type alias, causing NameError at runtime. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 192493dede9..fda02ef13a7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -393,3 +393,8 @@ def docstring_text(self, **kwargs: Any) -> str: if kwargs.pop("is_response", False): return "JSON" return super().docstring_text(**kwargs) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + file_import.define_mutable_mapping_type() + return file_import From 37806df0a62f5a3ef19f1e38eb462576b81d8f5f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 5 May 2026 15:58:08 -0400 Subject: [PATCH 11/42] switch to always generating typeddicts as typing hints --- .../http-client-python/emitter/src/types.ts | 2 +- .../eng/scripts/ci/regenerate-common.ts | 46 ++---- .../generator/pygen/__init__.py | 6 +- .../pygen/codegen/models/__init__.py | 4 +- .../pygen/codegen/models/code_model.py | 4 +- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/operation.py | 10 +- .../pygen/codegen/models/paging_operation.py | 6 +- .../pygen/codegen/models/parameter.py | 2 +- .../models/request_builder_parameter.py | 6 +- .../pygen/codegen/serializers/__init__.py | 62 +------- .../codegen/serializers/builder_serializer.py | 28 ++-- .../codegen/serializers/types_serializer.py | 115 +++++++++++++- .../templates/model_container.py.jinja2 | 2 - .../pygen/codegen/templates/types.py.jinja2 | 29 ++++ .../generator/pygen/preprocess/__init__.py | 2 +- .../tests/unit/test_typeddict.py | 144 ++++++++++++++++++ 17 files changed, 335 insertions(+), 135 deletions(-) create mode 100644 packages/http-client-python/tests/unit/test_typeddict.py diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 89577ff0bdf..c8dbcf9b584 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -290,7 +290,7 @@ function emitModel(context: PythonSdkContext, type: SdkModelType): Record>, properties: new Array>(), snakeCaseName: camelToSnakeCase(type.name), - base: (context.emitContext.options as any)["models-mode"] === "typeddict" ? "typeddict" : "dpg", + base: "dpg", internal: type.access === "internal", crossLanguageDefinitionId: type.crossLanguageDefinitionId, usage: type.usage, diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index 9abd4a57a8f..8dd426152b4 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -105,17 +105,9 @@ export const BASE_AZURE_EMITTER_OPTIONS: Record< "package-name": "client-structure-twooperationgroup", namespace: "client.structure.twooperationgroup", }, - "client/naming": [ - { - namespace: "client.naming.main", - }, - { - "package-name": "client-naming-typeddict", - namespace: "client.naming.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], + "client/naming": { + namespace: "client.naming.main", + }, "client/overload": { namespace: "client.overload", }, @@ -211,30 +203,14 @@ export const BASE_EMITTER_OPTIONS: Record< "package-name": "typetest-model-nesteddiscriminator", namespace: "typetest.model.nesteddiscriminator", }, - "type/model/inheritance/not-discriminated": [ - { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - { - "package-name": "typetest-model-notdiscriminated-typeddict", - namespace: "typetest.model.notdiscriminated.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], - "type/model/inheritance/single-discriminator": [ - { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, - { - "package-name": "typetest-model-singlediscriminator-typeddict", - namespace: "typetest.model.singlediscriminator.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, "type/model/inheritance/recursive": [ { "package-name": "typetest-model-recursive", diff --git a/packages/http-client-python/generator/pygen/__init__.py b/packages/http-client-python/generator/pygen/__init__.py index 91cfab71f69..23368828a24 100644 --- a/packages/http-client-python/generator/pygen/__init__.py +++ b/packages/http-client-python/generator/pygen/__init__.py @@ -167,10 +167,10 @@ def _validate_and_transform(self, key: str, value: Any) -> Any: if key == "models-mode" and value == "none": value = False # switch to falsy value for easier code writing - if key == "models-mode" and value not in ["msrest", "dpg", False]: + if key == "models-mode" and value not in ["msrest", "dpg", "typeddict", False]: raise ValueError( - "--models-mode can only be 'msrest', 'dpg' or 'none'. " - "Pass in 'msrest' if you want msrest models, or " + "--models-mode can only be 'msrest', 'dpg', 'typeddict', or 'none'. " + "Pass in 'msrest' if you want msrest models, 'typeddict' for TypedDict models, or " "'none' if you don't want any." ) if key == "package-mode": diff --git a/packages/http-client-python/generator/pygen/codegen/models/__init__.py b/packages/http-client-python/generator/pygen/codegen/models/__init__.py index 5848854ed86..a1d9f9a4dbc 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/models/__init__.py @@ -9,7 +9,7 @@ from .base_builder import BaseBuilder, ParameterListType from .code_model import CodeModel from .client import Client -from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType, TypedDictModelType +from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType from .dictionary_type import DictionaryType from .list_type import ListType from .combined_type import CombinedType @@ -171,8 +171,6 @@ def build_type(yaml_data: dict[str, Any], code_model: CodeModel) -> BaseType: model_type = JSONModelType elif yaml_data["base"] == "dpg": model_type = DPGModelType # type: ignore - elif yaml_data["base"] == "typeddict": - model_type = TypedDictModelType # type: ignore else: model_type = MsrestModelType # type: ignore response = model_type(yaml_data, code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 6a28ddfcaad..81f5f20bf8b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -251,7 +251,7 @@ def need_utils_folder(self, async_mode: bool, client_namespace: str) -> bool: return ( self.need_utils_utils(async_mode, client_namespace) or self.need_utils_serialization - or self.options["models-mode"] in ("dpg", "typeddict") + or self.options["models-mode"] == "dpg" ) @property @@ -271,7 +271,7 @@ def need_utils_form_data(self, async_mode: bool, client_namespace: str) -> bool: (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data - and self.options["models-mode"] in ("dpg", "typeddict") + and self.options["models-mode"] == "dpg" ) def need_utils_etag(self, client_namespace: str) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 1a3d37bb85b..b7673d4d865 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -125,7 +125,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if ( - self.code_model.options["models-mode"] in ("dpg", "typeddict") + self.code_model.options["models-mode"] == "dpg" and self.lro_response and self.lro_response.type and self.lro_response.type.type == "model" diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 60737410140..c5f15593893 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -51,7 +51,7 @@ def is_internal(target: Optional[BaseType]) -> bool: - return isinstance(target, ModelType) and target.base in ("dpg", "typeddict") and target.internal + return isinstance(target, ModelType) and target.base == "dpg" and target.internal class OperationBase( # pylint: disable=too-many-public-methods,too-many-instance-attributes @@ -176,7 +176,7 @@ def response_docstring_text(self, **kwargs) -> str: retval = self._response_docstring_helper("docstring_text", **kwargs) if not self.code_model.options["version-tolerant"]: retval += " or the result of cls(response)" - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and any( + if self.code_model.options["models-mode"] == "dpg" and any( isinstance(r.type, ModelType) for r in self.responses ): r = next(r for r in self.responses if isinstance(r.type, ModelType)) @@ -209,7 +209,7 @@ def default_error_deserialization(self, serialize_namespace: str) -> Optional[st f"{exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)}," f"{pylint_disable}" ) - return None if self.code_model.options["models-mode"] in ("dpg", "typeddict") else "'object'," + return None if self.code_model.options["models-mode"] == "dpg" else "'object'," @property def non_default_errors(self) -> list[Response]: @@ -421,7 +421,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements for overload in self.overloads: if overload.parameters.has_body: file_import.merge(overload.parameters.body_parameter.type.imports(**kwargs)) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) @@ -449,7 +449,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements file_import.add_import("json", ImportType.STDLIB) if self.enable_import_deserialize_xml: file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL) - if self.code_model.options["models-mode"] != "typeddict" and any( + if any( r.type and not isinstance(r.type, BinaryIteratorType) and not xml_serializable(str(r.default_content_type)) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index e6fb19fe3db..f64363ed5fb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -181,14 +181,12 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: "case_insensitive_dict", ImportType.SDKCORE, ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) file_import.merge(self.item_type.imports(**kwargs)) - if ( - self.default_error_deserialization(serialize_namespace) or self.need_deserialize - ) and self.code_model.options["models-mode"] != "typeddict": + if self.default_error_deserialization(serialize_namespace) or self.need_deserialize: file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.is_xml_paging: file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 34a765b849a..26eb0ba5c9a 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -336,7 +336,7 @@ def method_location( # pylint: disable=too-many-return-statements ) -> ParameterMethodLocation: if not self.in_method_signature: raise ValueError(f"Parameter '{self.client_name}' is not in the method.") - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and self.in_flattened_body: + if self.code_model.options["models-mode"] == "dpg" and self.in_flattened_body: return ParameterMethodLocation.KEYWORD_ONLY if self.grouper: return ParameterMethodLocation.POSITIONAL diff --git a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py index 810fbf3b7e7..c73df6db5af 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py @@ -26,7 +26,7 @@ def __init__(self, *args, **kwargs) -> None: if ( isinstance(self.type, (BinaryType, StringType)) or any("xml" in ct for ct in self.content_types) - or self.code_model.options["models-mode"] in ("dpg", "typeddict") + or self.code_model.options["models-mode"] == "dpg" ): self.client_name = "content" else: @@ -40,9 +40,7 @@ def type_annotation(self, **kwargs: Any) -> str: @property def in_method_signature(self) -> bool: return ( - super().in_method_signature - and not self.is_partial_body - and self.code_model.options["models-mode"] not in ("dpg", "typeddict") + super().in_method_signature and not self.is_partial_body and self.code_model.options["models-mode"] != "dpg" ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c769535dda4..2230d98e78a 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -22,11 +22,10 @@ ModelType, EnumType, ) -from ..models.primitive_types import DatetimeType, ByteArraySchema, BinaryType from .enum_serializer import EnumSerializer from .general_serializer import GeneralSerializer from .model_init_serializer import ModelInitSerializer -from .model_serializer import DpgModelSerializer, MsrestModelSerializer, TypedDictModelSerializer +from .model_serializer import DpgModelSerializer, MsrestModelSerializer from .operations_init_serializer import OperationsInitSerializer from .operation_groups_serializer import OperationGroupsSerializer from .request_builders_serializer import RequestBuildersSerializer @@ -119,55 +118,8 @@ def keep_version_file(self) -> bool: # If parsing the version fails, we assume the version file is not valid and overwrite. return False - @staticmethod - def _validate_typeddict_models(code_model: CodeModel) -> None: - """Validate that models are compatible with typeddict mode. - - Raises ValueError if any model uses unsupported features: - readonly properties, datetime types, bytes types, - or additional properties (extends Record). - """ - unsupported: list[str] = [] - for model in code_model.model_types: - if model.base != "typeddict": - continue - model_name = model.name - - for prop in model.properties: - # Readonly - if prop.readonly: - unsupported.append( - f"Model '{model_name}' has readonly property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Datetime - if isinstance(prop.type, DatetimeType): - unsupported.append( - f"Model '{model_name}' has datetime property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Bytes - if isinstance(prop.type, (ByteArraySchema, BinaryType)): - unsupported.append( - f"Model '{model_name}' has bytes property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Additional properties (extends Record) - if prop.client_name == "additional_properties": - unsupported.append( - f"Model '{model_name}' has additional properties (extends Record), " - "which is not supported in typeddict mode." - ) - - if unsupported: - raise ValueError("The following models are not compatible with typeddict mode:\n" + "\n".join(unsupported)) - # pylint: disable=too-many-branches def serialize(self) -> None: - # Validate typeddict mode constraints - if self.code_model.options.get("models-mode") == "typeddict": - self._validate_typeddict_models(self.code_model) - # remove existing folders when generate from tsp if self.code_model.is_tsp and self.code_model.options.get("clear-output-folder"): # remove generated_samples and generated_tests folder @@ -345,8 +297,6 @@ def _serialize_and_write_models_folder( models_mode = self.code_model.options["models-mode"] if models_mode == "dpg": serializer = DpgModelSerializer - elif models_mode == "typeddict": - serializer = TypedDictModelSerializer else: serializer = MsrestModelSerializer if self.code_model.has_non_json_models(models): @@ -537,7 +487,7 @@ def _serialize_and_write_utils_folder(self, env: Environment, namespace: str): ) # write _model_base.py - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": self.write_file( utils_folder_path / Path("model_base.py"), general_serializer.serialize_model_base_file(), @@ -571,10 +521,14 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str ) # write _types.py - if self.code_model.named_unions: + if self.code_model.named_unions or self.code_model.model_types: self.write_file( generation_dir / Path("_types.py"), - TypesSerializer(code_model=self.code_model, env=env).serialize(), + TypesSerializer( + code_model=self.code_model, + env=env, + models=self.code_model.model_types, + ).serialize(), ) # pylint: disable=line-too-long diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 9c5b14440af..e43dcd916eb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -29,7 +29,6 @@ CombinedType, JSONModelType, DPGModelType, - TypedDictModelType, ParameterListType, ByteArraySchema, ) @@ -712,7 +711,7 @@ def _serialize_body_parameter(self, builder: OperationType) -> list[str]: f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " f"'{serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" ) - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": if json_serializable(body_param.default_content_type): if hasattr(body_param.type, "encode") and body_param.type.encode: # type: ignore create_body_call = ( @@ -791,10 +790,9 @@ def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) overload.request_builder.parameters.body_parameter.client_name for overload in builder.overloads ] all_dpg_model_overloads = False - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and builder.overloads: + if self.code_model.options["models-mode"] == "dpg" and builder.overloads: all_dpg_model_overloads = all( - isinstance(o.parameters.body_parameter.type, (DPGModelType, TypedDictModelType)) - for o in builder.overloads + isinstance(o.parameters.body_parameter.type, DPGModelType) for o in builder.overloads ) if not all_dpg_model_overloads: for v in sorted(set(client_names), key=client_names.index): @@ -999,13 +997,7 @@ def response_deserialization( # pylint: disable=too-many-statements deserialize_code.append(f" '{serialization_type}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") - elif self.code_model.options["models-mode"] == "typeddict": - if builder.has_stream_response: - deserialize_code.append("deserialized = response.content") - else: - response_attr = "json" if json_serializable(str(response.default_content_type)) else "text" - deserialize_code.append(f"deserialized = response.{response_attr}()") - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": if builder.has_stream_response: deserialize_code.append("deserialized = response.content") else: @@ -1079,7 +1071,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": if xml_serializable(str(e.default_content_type)): fn = "_failsafe_deserialize_xml" else: @@ -1121,7 +1113,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": if xml_serializable(str(e.default_content_type)): retval.append( " error = _failsafe_deserialize_xml(" @@ -1149,7 +1141,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran indent = " " if builder.non_default_errors else " " if builder.non_default_errors: retval.append(" else:") - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": default_exception = next(e for e in builder.exceptions if "default" in e.status_codes and e.type) if xml_serializable(str(default_exception.default_content_type)): fn = "_failsafe_deserialize_xml" @@ -1418,7 +1410,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response{suffix}\n)" ) retval.append(f" deserialized = {deserialized}") - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": # we don't want to generate paging models for DPG retval.append(f" deserialized = {deserialized}") else: @@ -1436,7 +1428,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran "".join([f'.get("{i}", {{}})' for i in item_name_array[:-1]]) + f'.get("{item_name_array[-1]}", [])' ) pylint_disable = "" - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": item_type = builder.item_type.type_annotation( is_operation_file=True, serialize_namespace=self.serialize_namespace ) @@ -1613,7 +1605,7 @@ def get_long_running_output(self, builder: LROOperationType) -> list[str]: retval.append(" response_headers = {}") if ( not self.code_model.options["models-mode"] - or self.code_model.options["models-mode"] in ("dpg", "typeddict") + or self.code_model.options["models-mode"] == "dpg" or builder.lro_response.headers ): retval.append(" response = pipeline_response.http_response") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 749d8ca240c..7a764409562 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -3,15 +3,42 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from typing import Optional +from ..models import ModelType, CodeModel from ..models.imports import FileImport, ImportType from ..models.utils import NamespaceType +from ..models.property import Property +from .model_serializer import _documentation_string from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer class TypesSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + env, + client_namespace: Optional[str] = None, + models: Optional[list[ModelType]] = None, + ): + super().__init__(code_model=code_model, env=env) + self._client_namespace = client_namespace + self._models = models or [] + + @property + def typeddict_models(self) -> list[ModelType]: + """Models that should be rendered as TypedDicts.""" + return [m for m in self._models if m.base != "json"] + + def _reorder_models(self, models: list[ModelType]) -> list[ModelType]: + """Reorder so discriminated base Union aliases come after all their subtypes.""" + bases = [m for m in models if m.discriminated_subtypes] + non_bases = [m for m in models if not m.discriminated_subtypes] + return non_bases + bases + def imports(self) -> FileImport: file_import = FileImport(self.code_model) + # Named union imports if self.code_model.named_unions: file_import.add_submodule_import( "typing", @@ -24,13 +51,99 @@ def imports(self) -> FileImport: serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE ) ) + + # TypedDict imports + td_models = self.typeddict_models + if td_models: + file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) + has_required = False + has_discriminated_union = False + for model in td_models: + if model.discriminated_subtypes: + has_discriminated_union = True + file_import.merge( + model.imports( + is_operation_file=False, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + ) + ) + for prop in model.properties: + file_import.merge( + prop.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + called_by_property=True, + ) + ) + if not (prop.optional or prop.client_default_value is not None): + has_required = True + for parent in model.parents: + if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + self.serialize_namespace, + self.code_model.get_imported_namespace_for_model(parent.client_namespace), + ), + parent.name, + ImportType.LOCAL, + ) + if has_required: + file_import.add_submodule_import("typing_extensions", "Required", ImportType.STDLIB) + if has_discriminated_union and not self.code_model.named_unions: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) return file_import + def declare_model(self, model: ModelType) -> str: + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + basename = ", ".join([m.name for m in non_discriminated_parents]) + return f"class {model.name}({basename}):{model.pylint_disable()}" + return f"class {model.name}(TypedDict, total=False):{model.pylint_disable()}" + + @staticmethod + def get_properties_to_declare(model: ModelType) -> list[Property]: + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + parent_properties = [p for bm in non_discriminated_parents for p in bm.properties] + properties_to_declare = [ + p + for p in model.properties + if not any( + p.client_name == pp.client_name + and p.type_annotation() == pp.type_annotation() + and not p.is_base_discriminator + for pp in parent_properties + ) + ] + else: + properties_to_declare = model.properties + return properties_to_declare + + def declare_property(self, prop: Property) -> str: + type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + is_optional = prop.optional or prop.client_default_value is not None + if is_optional: + return f"{prop.wire_name}: {type_annotation}" + return f"{prop.wire_name}: Required[{type_annotation}]" + + def discriminated_subtypes_union(self, model: ModelType) -> str: + subtypes = list(model.discriminated_subtypes.values()) + subtype_names = [s.name for s in subtypes] + return f"{model.name} = Union[{', '.join(subtype_names)}]" + + def is_discriminated_base(self, model: ModelType) -> bool: + return bool(model.discriminated_subtypes) + + @staticmethod + def variable_documentation_string(prop: Property) -> list[str]: + return _documentation_string(prop, "ivar", "vartype") + def serialize(self) -> str: - # Generate the models template = self.env.get_template("types.py.jinja2") return template.render( code_model=self.code_model, imports=FileImportSerializer(self.imports()), serializer=self, + models=self._reorder_models(self.typeddict_models), ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 index 6e033ba77fb..dc98f999c45 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 @@ -13,7 +13,5 @@ {% include "model_dpg.py.jinja2" %} {% elif model.base == "msrest" %} {% include "model_msrest.py.jinja2" %} -{% elif model.base == "typeddict" %} -{% include "model_typeddict.py.jinja2" %} {% endif %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 19435f14e88..d60aebeccd3 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -7,3 +7,32 @@ {% for nu in code_model.named_unions %} {{nu.name}} = {{nu.type_definition()}} {% endfor %} +{% import 'operation_tools.jinja2' as op_tools %} +{% import "macros.jinja2" as macros %} +{% for model in models %} +{% if serializer.is_discriminated_base(model) %} +{{ serializer.discriminated_subtypes_union(model) }} +{% else %} + + +{{ serializer.declare_model(model) }} + """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} + + {% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} + {{ macros.wrap_model_string(line, '\n ') -}} + {% endfor %} + {% endfor %} + {% endif %} + """ + + {% for p in serializer.get_properties_to_declare(model)%} + {{ serializer.declare_property(p) }} + {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} + {% if prop_description %} + """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} + {% endif %} + {% endfor %} +{% endif %} +{% endfor %} diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index 5117a9227e5..6d3344059a3 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -216,7 +216,7 @@ def add_body_param_type( model_type = ( body_parameter["type"] if origin_type == "model" else body_parameter["type"].get("elementType", {}) ) - is_dpg_model = model_type.get("base") in ("dpg", "typeddict") + is_dpg_model = model_type.get("base") == "dpg" body_parameter["type"] = { "type": "combined", "types": [body_parameter["type"]], diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py new file mode 100644 index 00000000000..c5261dd3004 --- /dev/null +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -0,0 +1,144 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +"""Tests for TypedDict generation and models-mode interactions.""" + +from jinja2 import PackageLoader, Environment + +from pygen.codegen.models import CodeModel, JSONModelType, DPGModelType +from pygen.codegen.models.model_type import TypedDictModelType +from pygen.codegen.serializers.types_serializer import TypesSerializer + + +def _make_code_model(models_mode="dpg"): + return CodeModel( + { + "clients": [ + { + "name": "client", + "namespace": "blah", + "moduleName": "blah", + "parameters": [], + "url": "", + "operationGroups": [], + } + ], + "namespace": "namespace", + }, + options={ + "show-send-request": True, + "builders-visibility": "public", + "show-operations": True, + "models-mode": models_mode, + "flavor": "unbranded", + "client-side-validation": False, + }, + ) + + +def _make_model(code_model, name, model_cls=None, properties=None): + """Create a model of the given class attached to code_model.""" + if model_cls is None: + if code_model.options["models-mode"] == "typeddict": + model_cls = TypedDictModelType + elif code_model.options["models-mode"] == "dpg": + model_cls = DPGModelType + else: + model_cls = JSONModelType + return model_cls( + yaml_data={ + "name": name, + "type": "model", + "snakeCaseName": name.lower(), + "usage": 2, + }, + code_model=code_model, + properties=properties or [], + ) + + +def _make_env(): + return Environment( + loader=PackageLoader("pygen.codegen", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + + +# ---------- models-mode=none ---------- + + +def test_models_mode_none_produces_json_model_type(): + """When models-mode is none (False), all models should be JSONModelType.""" + code_model = _make_code_model(models_mode=False) + model = _make_model(code_model, "Foo", model_cls=JSONModelType) + assert model.base == "json" + + +def test_models_mode_none_no_typeddict_models(): + """TypesSerializer.typeddict_models should be empty when models-mode=none.""" + code_model = _make_code_model(models_mode=False) + m1 = _make_model(code_model, "Foo", model_cls=JSONModelType) + m2 = _make_model(code_model, "Bar", model_cls=JSONModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1, m2]) + assert ts.typeddict_models == [] + + +def test_models_mode_none_types_file_has_no_typeddict_imports(): + """When models-mode=none, the _types.py should not import TypedDict.""" + code_model = _make_code_model(models_mode=False) + m1 = _make_model(code_model, "Foo", model_cls=JSONModelType) + code_model.model_types = [m1] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + output = ts.serialize() + assert "TypedDict" not in output + assert "Required" not in output + + +# ---------- models-mode=dpg ---------- + + +def test_models_mode_dpg_no_typeddict_models(): + """DPG models have base='dpg', not 'typeddict', so should not appear as typeddict_models.""" + code_model = _make_code_model(models_mode="dpg") + m1 = _make_model(code_model, "Foo", model_cls=DPGModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + # DPG models have base != "json" so they DO appear in typeddict_models + # This is a filtering based on base != "json" + assert len(ts.typeddict_models) == 1 + + +# ---------- models-mode=typeddict ---------- + + +def test_models_mode_typeddict_models_included(): + """TypedDictModelType models should appear in typeddict_models.""" + code_model = _make_code_model(models_mode="typeddict") + m1 = _make_model(code_model, "Foo", model_cls=TypedDictModelType) + m2 = _make_model(code_model, "Bar", model_cls=TypedDictModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1, m2]) + assert len(ts.typeddict_models) == 2 + + +def test_models_mode_typeddict_serialize_contains_class(): + """Serialized output should contain TypedDict class definitions.""" + code_model = _make_code_model(models_mode="typeddict") + m1 = _make_model(code_model, "Foo", model_cls=TypedDictModelType) + code_model.model_types = [m1] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + output = ts.serialize() + assert "class Foo(TypedDict, total=False):" in output + assert "TypedDict" in output From 18786e307f1e263c34f93ef4278350537848a4fd Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 5 May 2026 15:58:08 -0400 Subject: [PATCH 12/42] switch to always generating typeddicts as typing hints --- .../python-addTypedDict-2026-3-21-17-47-3.md | 4 +- .../http-client-python/emitter/src/types.ts | 2 +- .../eng/scripts/ci/regenerate-common.ts | 48 ++--- .../generator/pygen/__init__.py | 6 +- .../pygen/codegen/models/__init__.py | 4 +- .../pygen/codegen/models/code_model.py | 4 +- .../pygen/codegen/models/combined_type.py | 6 +- .../pygen/codegen/models/enum_type.py | 2 +- .../pygen/codegen/models/lro_operation.py | 2 +- .../pygen/codegen/models/model_type.py | 4 +- .../pygen/codegen/models/operation.py | 10 +- .../pygen/codegen/models/paging_operation.py | 6 +- .../pygen/codegen/models/parameter.py | 4 +- .../models/request_builder_parameter.py | 6 +- .../pygen/codegen/models/response.py | 2 +- .../generator/pygen/codegen/models/utils.py | 1 + .../pygen/codegen/serializers/__init__.py | 83 +++------ .../codegen/serializers/builder_serializer.py | 28 +-- .../codegen/serializers/types_serializer.py | 104 +++++++++-- .../codegen/serializers/unions_serializer.py | 60 ++++++ .../templates/model_container.py.jinja2 | 4 +- .../pygen/codegen/templates/types.py.jinja2 | 26 ++- .../pygen/codegen/templates/unions.py.jinja2 | 12 ++ .../generator/pygen/preprocess/__init__.py | 2 +- .../tests/unit/test_typeddict.py | 171 ++++++++++++++++++ 25 files changed, 437 insertions(+), 164 deletions(-) create mode 100644 packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py create mode 100644 packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 create mode 100644 packages/http-client-python/tests/unit/test_typeddict.py diff --git a/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md index 25dfde1e67f..c629da91ce2 100644 --- a/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md +++ b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md @@ -1,8 +1,8 @@ --- # Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking -changeKind: fix +changeKind: feature packages: - "@typespec/http-client-python" --- -[python] add `typeddict` `models-mode` for Python HTTP client emitter to generated `TypedDict`s for input models +[python] Always generate `TypedDict` typing hints for input models in the `_types.py` file diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 89577ff0bdf..c8dbcf9b584 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -290,7 +290,7 @@ function emitModel(context: PythonSdkContext, type: SdkModelType): Record>, properties: new Array>(), snakeCaseName: camelToSnakeCase(type.name), - base: (context.emitContext.options as any)["models-mode"] === "typeddict" ? "typeddict" : "dpg", + base: "dpg", internal: type.access === "internal", crossLanguageDefinitionId: type.crossLanguageDefinitionId, usage: type.usage, diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts index 9abd4a57a8f..6e91285624c 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -105,17 +105,9 @@ export const BASE_AZURE_EMITTER_OPTIONS: Record< "package-name": "client-structure-twooperationgroup", namespace: "client.structure.twooperationgroup", }, - "client/naming": [ - { - namespace: "client.naming.main", - }, - { - "package-name": "client-naming-typeddict", - namespace: "client.naming.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], + "client/naming": { + namespace: "client.naming.main", + }, "client/overload": { namespace: "client.overload", }, @@ -211,30 +203,14 @@ export const BASE_EMITTER_OPTIONS: Record< "package-name": "typetest-model-nesteddiscriminator", namespace: "typetest.model.nesteddiscriminator", }, - "type/model/inheritance/not-discriminated": [ - { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - { - "package-name": "typetest-model-notdiscriminated-typeddict", - namespace: "typetest.model.notdiscriminated.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], - "type/model/inheritance/single-discriminator": [ - { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, - { - "package-name": "typetest-model-singlediscriminator-typeddict", - namespace: "typetest.model.singlediscriminator.typeddict", - "models-mode": "typeddict", - "generate-test": "false", - }, - ], + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, "type/model/inheritance/recursive": [ { "package-name": "typetest-model-recursive", @@ -310,7 +286,7 @@ export const BASE_EMITTER_OPTIONS: Record< "package-name": "versioning-added", namespace: "versioning.added", }, - // check whether import of _validation.py/_types.py works when "generation-subdir" is configured + // check whether import of _validation.py/_unions.py/types.py works when "generation-subdir" is configured { "package-name": "generation-subdir2", namespace: "generation.subdir2", diff --git a/packages/http-client-python/generator/pygen/__init__.py b/packages/http-client-python/generator/pygen/__init__.py index 91cfab71f69..23368828a24 100644 --- a/packages/http-client-python/generator/pygen/__init__.py +++ b/packages/http-client-python/generator/pygen/__init__.py @@ -167,10 +167,10 @@ def _validate_and_transform(self, key: str, value: Any) -> Any: if key == "models-mode" and value == "none": value = False # switch to falsy value for easier code writing - if key == "models-mode" and value not in ["msrest", "dpg", False]: + if key == "models-mode" and value not in ["msrest", "dpg", "typeddict", False]: raise ValueError( - "--models-mode can only be 'msrest', 'dpg' or 'none'. " - "Pass in 'msrest' if you want msrest models, or " + "--models-mode can only be 'msrest', 'dpg', 'typeddict', or 'none'. " + "Pass in 'msrest' if you want msrest models, 'typeddict' for TypedDict models, or " "'none' if you don't want any." ) if key == "package-mode": diff --git a/packages/http-client-python/generator/pygen/codegen/models/__init__.py b/packages/http-client-python/generator/pygen/codegen/models/__init__.py index 5848854ed86..a1d9f9a4dbc 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/models/__init__.py @@ -9,7 +9,7 @@ from .base_builder import BaseBuilder, ParameterListType from .code_model import CodeModel from .client import Client -from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType, TypedDictModelType +from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType from .dictionary_type import DictionaryType from .list_type import ListType from .combined_type import CombinedType @@ -171,8 +171,6 @@ def build_type(yaml_data: dict[str, Any], code_model: CodeModel) -> BaseType: model_type = JSONModelType elif yaml_data["base"] == "dpg": model_type = DPGModelType # type: ignore - elif yaml_data["base"] == "typeddict": - model_type = TypedDictModelType # type: ignore else: model_type = MsrestModelType # type: ignore response = model_type(yaml_data, code_model) diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 6a28ddfcaad..81f5f20bf8b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -251,7 +251,7 @@ def need_utils_folder(self, async_mode: bool, client_namespace: str) -> bool: return ( self.need_utils_utils(async_mode, client_namespace) or self.need_utils_serialization - or self.options["models-mode"] in ("dpg", "typeddict") + or self.options["models-mode"] == "dpg" ) @property @@ -271,7 +271,7 @@ def need_utils_form_data(self, async_mode: bool, client_namespace: str) -> bool: (not async_mode) and self.is_top_namespace(client_namespace) and self.has_form_data - and self.options["models-mode"] in ("dpg", "typeddict") + and self.options["models-mode"] == "dpg" ) def need_utils_etag(self, client_namespace: str) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py index 249b7474dcd..1014482f5e0 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/combined_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/combined_type.py @@ -66,7 +66,7 @@ def docstring_type(self, **kwargs: Any) -> str: def type_annotation(self, **kwargs: Any) -> str: if self.name: - return f'"_types.{self.name}"' + return f'"_unions.{self.name}"' return self.type_definition(**kwargs) def type_definition(self, **kwargs: Any) -> str: @@ -116,10 +116,10 @@ def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) serialize_namespace_type = kwargs.get("serialize_namespace_type") - if self.name and serialize_namespace_type != NamespaceType.TYPES_FILE: + if self.name and serialize_namespace_type != NamespaceType.UNIONS_FILE: file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), - "_types", + "_unions", ImportType.LOCAL, TypingSection.TYPING, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 9cbec3d1b30..671a8c64d5b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -255,7 +255,7 @@ def imports(self, **kwargs: Any) -> FileImport: alias=alias, typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type == NamespaceType.TYPES_FILE or ( + elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): file_import.add_submodule_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py index 1a3d37bb85b..b7673d4d865 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/lro_operation.py @@ -125,7 +125,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: ImportType.SDKCORE, ) if ( - self.code_model.options["models-mode"] in ("dpg", "typeddict") + self.code_model.options["models-mode"] == "dpg" and self.lro_response and self.lro_response.type and self.lro_response.type.type == "model" diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index fda02ef13a7..036ff04c749 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -311,7 +311,7 @@ def imports(self, **kwargs: Any) -> FileImport: alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) serialize_namespace_type = kwargs.get("serialize_namespace_type") called_by_property = kwargs.get("called_by_property", False) - # add import for models in operations or _types file + # add import for models in operations, types, or unions file if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( relative_path, @@ -326,7 +326,7 @@ def imports(self, **kwargs: Any) -> FileImport: ImportType.LOCAL, alias="_Model", ) - elif serialize_namespace_type == NamespaceType.TYPES_FILE or ( + elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): file_import.add_submodule_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index 60737410140..c5f15593893 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -51,7 +51,7 @@ def is_internal(target: Optional[BaseType]) -> bool: - return isinstance(target, ModelType) and target.base in ("dpg", "typeddict") and target.internal + return isinstance(target, ModelType) and target.base == "dpg" and target.internal class OperationBase( # pylint: disable=too-many-public-methods,too-many-instance-attributes @@ -176,7 +176,7 @@ def response_docstring_text(self, **kwargs) -> str: retval = self._response_docstring_helper("docstring_text", **kwargs) if not self.code_model.options["version-tolerant"]: retval += " or the result of cls(response)" - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and any( + if self.code_model.options["models-mode"] == "dpg" and any( isinstance(r.type, ModelType) for r in self.responses ): r = next(r for r in self.responses if isinstance(r.type, ModelType)) @@ -209,7 +209,7 @@ def default_error_deserialization(self, serialize_namespace: str) -> Optional[st f"{exception_schema.type_annotation(skip_quote=True, serialize_namespace=serialize_namespace)}," f"{pylint_disable}" ) - return None if self.code_model.options["models-mode"] in ("dpg", "typeddict") else "'object'," + return None if self.code_model.options["models-mode"] == "dpg" else "'object'," @property def non_default_errors(self) -> list[Response]: @@ -421,7 +421,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements for overload in self.overloads: if overload.parameters.has_body: file_import.merge(overload.parameters.body_parameter.type.imports(**kwargs)) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) @@ -449,7 +449,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements file_import.add_import("json", ImportType.STDLIB) if self.enable_import_deserialize_xml: file_import.add_submodule_import(relative_path, "_deserialize_xml", ImportType.LOCAL) - if self.code_model.options["models-mode"] != "typeddict" and any( + if any( r.type and not isinstance(r.type, BinaryIteratorType) and not xml_serializable(str(r.default_content_type)) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index e6fb19fe3db..f64363ed5fb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -181,14 +181,12 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: "case_insensitive_dict", ImportType.SDKCORE, ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": relative_path = self.code_model.get_relative_import_path( serialize_namespace, module_name="_utils.model_base" ) file_import.merge(self.item_type.imports(**kwargs)) - if ( - self.default_error_deserialization(serialize_namespace) or self.need_deserialize - ) and self.code_model.options["models-mode"] != "typeddict": + if self.default_error_deserialization(serialize_namespace) or self.need_deserialize: file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.is_xml_paging: file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") diff --git a/packages/http-client-python/generator/pygen/codegen/models/parameter.py b/packages/http-client-python/generator/pygen/codegen/models/parameter.py index 34a765b849a..f934f34bf25 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/parameter.py @@ -178,7 +178,7 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: if isinstance(self.type, CombinedType) and self.type.name: file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), - "_types", + "_unions", ImportType.LOCAL, TypingSection.TYPING, ) @@ -336,7 +336,7 @@ def method_location( # pylint: disable=too-many-return-statements ) -> ParameterMethodLocation: if not self.in_method_signature: raise ValueError(f"Parameter '{self.client_name}' is not in the method.") - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and self.in_flattened_body: + if self.code_model.options["models-mode"] == "dpg" and self.in_flattened_body: return ParameterMethodLocation.KEYWORD_ONLY if self.grouper: return ParameterMethodLocation.POSITIONAL diff --git a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py index 810fbf3b7e7..c73df6db5af 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py +++ b/packages/http-client-python/generator/pygen/codegen/models/request_builder_parameter.py @@ -26,7 +26,7 @@ def __init__(self, *args, **kwargs) -> None: if ( isinstance(self.type, (BinaryType, StringType)) or any("xml" in ct for ct in self.content_types) - or self.code_model.options["models-mode"] in ("dpg", "typeddict") + or self.code_model.options["models-mode"] == "dpg" ): self.client_name = "content" else: @@ -40,9 +40,7 @@ def type_annotation(self, **kwargs: Any) -> str: @property def in_method_signature(self) -> bool: return ( - super().in_method_signature - and not self.is_partial_body - and self.code_model.options["models-mode"] not in ("dpg", "typeddict") + super().in_method_signature and not self.is_partial_body and self.code_model.options["models-mode"] != "dpg" ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/models/response.py b/packages/http-client-python/generator/pygen/codegen/models/response.py index 40599eb47ae..a2bb431c44d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/response.py +++ b/packages/http-client-python/generator/pygen/codegen/models/response.py @@ -124,7 +124,7 @@ def imports(self, **kwargs: Any) -> FileImport: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( self.code_model.get_relative_import_path(serialize_namespace), - "_types", + "_unions", ImportType.LOCAL, TypingSection.TYPING, ) diff --git a/packages/http-client-python/generator/pygen/codegen/models/utils.py b/packages/http-client-python/generator/pygen/codegen/models/utils.py index 82e80b85577..b6722fcd3f0 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/utils.py +++ b/packages/http-client-python/generator/pygen/codegen/models/utils.py @@ -30,6 +30,7 @@ class NamespaceType(str, Enum): OPERATION = "operation" CLIENT = "client" TYPES_FILE = "types_file" + UNIONS_FILE = "unions_file" LOCALS_LENGTH_LIMIT = 25 diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index c769535dda4..a21a012b596 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -22,11 +22,10 @@ ModelType, EnumType, ) -from ..models.primitive_types import DatetimeType, ByteArraySchema, BinaryType from .enum_serializer import EnumSerializer from .general_serializer import GeneralSerializer from .model_init_serializer import ModelInitSerializer -from .model_serializer import DpgModelSerializer, MsrestModelSerializer, TypedDictModelSerializer +from .model_serializer import DpgModelSerializer, MsrestModelSerializer from .operations_init_serializer import OperationsInitSerializer from .operation_groups_serializer import OperationGroupsSerializer from .request_builders_serializer import RequestBuildersSerializer @@ -34,6 +33,7 @@ from .sample_serializer import SampleSerializer from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer +from .unions_serializer import UnionsSerializer from ...utils import to_snake_case, VALID_PACKAGE_MODE from .utils import extract_sample_name, get_namespace_from_package_name, get_namespace_config, hash_file_import @@ -119,55 +119,8 @@ def keep_version_file(self) -> bool: # If parsing the version fails, we assume the version file is not valid and overwrite. return False - @staticmethod - def _validate_typeddict_models(code_model: CodeModel) -> None: - """Validate that models are compatible with typeddict mode. - - Raises ValueError if any model uses unsupported features: - readonly properties, datetime types, bytes types, - or additional properties (extends Record). - """ - unsupported: list[str] = [] - for model in code_model.model_types: - if model.base != "typeddict": - continue - model_name = model.name - - for prop in model.properties: - # Readonly - if prop.readonly: - unsupported.append( - f"Model '{model_name}' has readonly property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Datetime - if isinstance(prop.type, DatetimeType): - unsupported.append( - f"Model '{model_name}' has datetime property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Bytes - if isinstance(prop.type, (ByteArraySchema, BinaryType)): - unsupported.append( - f"Model '{model_name}' has bytes property '{prop.client_name}', " - "which is not supported in typeddict mode." - ) - # Additional properties (extends Record) - if prop.client_name == "additional_properties": - unsupported.append( - f"Model '{model_name}' has additional properties (extends Record), " - "which is not supported in typeddict mode." - ) - - if unsupported: - raise ValueError("The following models are not compatible with typeddict mode:\n" + "\n".join(unsupported)) - # pylint: disable=too-many-branches def serialize(self) -> None: - # Validate typeddict mode constraints - if self.code_model.options.get("models-mode") == "typeddict": - self._validate_typeddict_models(self.code_model) - # remove existing folders when generate from tsp if self.code_model.is_tsp and self.code_model.options.get("clear-output-folder"): # remove generated_samples and generated_tests folder @@ -244,7 +197,7 @@ def serialize(self) -> None: general_serializer.serialize_pkgutil_init_file(), ) - # _utils/py.typed/_types.py/_validation.py + # _utils/py.typed/_unions.py/types.py/_validation.py # is always put in top level namespace if self.code_model.is_top_namespace(client_namespace): self._serialize_and_write_top_level_folder(env=env, namespace=client_namespace) @@ -343,10 +296,8 @@ def _serialize_and_write_models_folder( # Write the models folder models_path = self.code_model.get_generation_dir(namespace) / "models" models_mode = self.code_model.options["models-mode"] - if models_mode == "dpg": + if models_mode in ("dpg", "typeddict"): serializer = DpgModelSerializer - elif models_mode == "typeddict": - serializer = TypedDictModelSerializer else: serializer = MsrestModelSerializer if self.code_model.has_non_json_models(models): @@ -537,7 +488,7 @@ def _serialize_and_write_utils_folder(self, env: Environment, namespace: str): ) # write _model_base.py - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": self.write_file( utils_folder_path / Path("model_base.py"), general_serializer.serialize_model_base_file(), @@ -570,11 +521,27 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str general_serializer.serialize_validation_file(), ) - # write _types.py - if self.code_model.named_unions: + # write _unions.py + has_discriminated_bases = any(m for m in self.code_model.model_types if m.base != "json" and m.discriminated_subtypes) + if self.code_model.named_unions or has_discriminated_bases: self.write_file( - generation_dir / Path("_types.py"), - TypesSerializer(code_model=self.code_model, env=env).serialize(), + generation_dir / Path("_unions.py"), + UnionsSerializer( + code_model=self.code_model, + env=env, + models=self.code_model.model_types, + ).serialize(), + ) + + # write types.py + if self.code_model.model_types: + self.write_file( + generation_dir / Path("types.py"), + TypesSerializer( + code_model=self.code_model, + env=env, + models=self.code_model.model_types, + ).serialize(), ) # pylint: disable=line-too-long diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index 9c5b14440af..e43dcd916eb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -29,7 +29,6 @@ CombinedType, JSONModelType, DPGModelType, - TypedDictModelType, ParameterListType, ByteArraySchema, ) @@ -712,7 +711,7 @@ def _serialize_body_parameter(self, builder: OperationType) -> list[str]: f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " f"'{serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" ) - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": if json_serializable(body_param.default_content_type): if hasattr(body_param.type, "encode") and body_param.type.encode: # type: ignore create_body_call = ( @@ -791,10 +790,9 @@ def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) overload.request_builder.parameters.body_parameter.client_name for overload in builder.overloads ] all_dpg_model_overloads = False - if self.code_model.options["models-mode"] in ("dpg", "typeddict") and builder.overloads: + if self.code_model.options["models-mode"] == "dpg" and builder.overloads: all_dpg_model_overloads = all( - isinstance(o.parameters.body_parameter.type, (DPGModelType, TypedDictModelType)) - for o in builder.overloads + isinstance(o.parameters.body_parameter.type, DPGModelType) for o in builder.overloads ) if not all_dpg_model_overloads: for v in sorted(set(client_names), key=client_names.index): @@ -999,13 +997,7 @@ def response_deserialization( # pylint: disable=too-many-statements deserialize_code.append(f" '{serialization_type}',{pylint_disable}") deserialize_code.append(" pipeline_response.http_response") deserialize_code.append(")") - elif self.code_model.options["models-mode"] == "typeddict": - if builder.has_stream_response: - deserialize_code.append("deserialized = response.content") - else: - response_attr = "json" if json_serializable(str(response.default_content_type)) else "text" - deserialize_code.append(f"deserialized = response.{response_attr}()") - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": if builder.has_stream_response: deserialize_code.append("deserialized = response.content") else: @@ -1079,7 +1071,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": if xml_serializable(str(e.default_content_type)): fn = "_failsafe_deserialize_xml" else: @@ -1121,7 +1113,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran type_annotation = e.type.type_annotation( # type: ignore is_operation_file=True, skip_quote=True, serialize_namespace=self.serialize_namespace ) - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": if xml_serializable(str(e.default_content_type)): retval.append( " error = _failsafe_deserialize_xml(" @@ -1149,7 +1141,7 @@ def handle_error_response( # pylint: disable=too-many-statements, too-many-bran indent = " " if builder.non_default_errors else " " if builder.non_default_errors: retval.append(" else:") - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": default_exception = next(e for e in builder.exceptions if "default" in e.status_codes and e.type) if xml_serializable(str(default_exception.default_content_type)): fn = "_failsafe_deserialize_xml" @@ -1418,7 +1410,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response{suffix}\n)" ) retval.append(f" deserialized = {deserialized}") - elif self.code_model.options["models-mode"] in ("dpg", "typeddict"): + elif self.code_model.options["models-mode"] == "dpg": # we don't want to generate paging models for DPG retval.append(f" deserialized = {deserialized}") else: @@ -1436,7 +1428,7 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran "".join([f'.get("{i}", {{}})' for i in item_name_array[:-1]]) + f'.get("{item_name_array[-1]}", [])' ) pylint_disable = "" - if self.code_model.options["models-mode"] in ("dpg", "typeddict"): + if self.code_model.options["models-mode"] == "dpg": item_type = builder.item_type.type_annotation( is_operation_file=True, serialize_namespace=self.serialize_namespace ) @@ -1613,7 +1605,7 @@ def get_long_running_output(self, builder: LROOperationType) -> list[str]: retval.append(" response_headers = {}") if ( not self.code_model.options["models-mode"] - or self.code_model.options["models-mode"] in ("dpg", "typeddict") + or self.code_model.options["models-mode"] == "dpg" or builder.lro_response.headers ): retval.append(" response = pipeline_response.http_response") diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 749d8ca240c..7fa07f72029 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -3,34 +3,114 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +from typing import Optional +from ..models import ModelType, CodeModel from ..models.imports import FileImport, ImportType from ..models.utils import NamespaceType +from ..models.property import Property +from .model_serializer import _documentation_string from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer class TypesSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + env, + client_namespace: Optional[str] = None, + models: Optional[list[ModelType]] = None, + ): + super().__init__(code_model=code_model, env=env) + self._client_namespace = client_namespace + self._models = models or [] + + @property + def typeddict_models(self) -> list[ModelType]: + """Models that should be rendered as TypedDicts (excluding discriminated bases which become unions).""" + return [m for m in self._models if m.base != "json" and not m.discriminated_subtypes] + def imports(self) -> FileImport: file_import = FileImport(self.code_model) - if self.code_model.named_unions: - file_import.add_submodule_import( - "typing", - "Union", - ImportType.STDLIB, - ) - for nu in self.code_model.named_unions: - file_import.merge( - nu.imports( - serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE + + td_models = self.typeddict_models + if td_models: + file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) + has_required = False + for model in td_models: + file_import.merge( + model.imports( + is_operation_file=False, + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + ) ) - ) + for prop in model.properties: + file_import.merge( + prop.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + called_by_property=True, + ) + ) + if not (prop.optional or prop.client_default_value is not None): + has_required = True + for parent in model.parents: + if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: + file_import.add_submodule_import( + self.code_model.get_relative_import_path( + self.serialize_namespace, + self.code_model.get_imported_namespace_for_model(parent.client_namespace), + ), + parent.name, + ImportType.LOCAL, + ) + if has_required: + file_import.add_submodule_import("typing_extensions", "Required", ImportType.STDLIB) return file_import + def declare_model(self, model: ModelType) -> str: + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + basename = ", ".join([m.name for m in non_discriminated_parents]) + return f"class {model.name}({basename}):{model.pylint_disable()}" + return f"class {model.name}(TypedDict, total=False):{model.pylint_disable()}" + + @staticmethod + def get_properties_to_declare(model: ModelType) -> list[Property]: + non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] + if non_discriminated_parents: + parent_properties = [p for bm in non_discriminated_parents for p in bm.properties] + properties_to_declare = [ + p + for p in model.properties + if not any( + p.client_name == pp.client_name + and p.type_annotation() == pp.type_annotation() + and not p.is_base_discriminator + for pp in parent_properties + ) + ] + else: + properties_to_declare = model.properties + return properties_to_declare + + def declare_property(self, prop: Property) -> str: + type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + is_optional = prop.optional or prop.client_default_value is not None + if is_optional: + return f"{prop.wire_name}: {type_annotation}" + return f"{prop.wire_name}: Required[{type_annotation}]" + + @staticmethod + def variable_documentation_string(prop: Property) -> list[str]: + return _documentation_string(prop, "ivar", "vartype") + def serialize(self) -> str: - # Generate the models template = self.env.get_template("types.py.jinja2") return template.render( code_model=self.code_model, imports=FileImportSerializer(self.imports()), serializer=self, + models=self.typeddict_models, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py new file mode 100644 index 00000000000..e15fee99c82 --- /dev/null +++ b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py @@ -0,0 +1,60 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Optional + +from ..models import CodeModel, ModelType +from ..models.imports import FileImport, ImportType +from ..models.utils import NamespaceType +from .import_serializer import FileImportSerializer +from .base_serializer import BaseSerializer + + +class UnionsSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + env, + models: Optional[list[ModelType]] = None, + ): + super().__init__(code_model=code_model, env=env) + self._models = models or [] + + @property + def discriminated_base_models(self) -> list[ModelType]: + """Models that are discriminated bases, rendered as Union aliases.""" + return [m for m in self._models if m.base != "json" and m.discriminated_subtypes] + + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + has_unions = bool(self.code_model.named_unions) or bool(self.discriminated_base_models) + if has_unions: + file_import.add_submodule_import( + "typing", + "Union", + ImportType.STDLIB, + ) + for nu in self.code_model.named_unions: + file_import.merge( + nu.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.UNIONS_FILE, + ) + ) + return file_import + + def discriminated_subtypes_union(self, model: ModelType) -> str: + subtypes = list(model.discriminated_subtypes.values()) + subtype_names = [s.name for s in subtypes] + return f"{model.name} = Union[{', '.join(subtype_names)}]" + + def serialize(self) -> str: + template = self.env.get_template("unions.py.jinja2") + return template.render( + code_model=self.code_model, + imports=FileImportSerializer(self.imports()), + serializer=self, + discriminated_bases=self.discriminated_base_models, + ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 index 6e033ba77fb..d91228cbdd8 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_container.py.jinja2 @@ -9,11 +9,9 @@ {{ imports }} {% for model in models %} -{% if model.base == "dpg" %} +{% if model.base == "dpg" or model.base == "typeddict" %} {% include "model_dpg.py.jinja2" %} {% elif model.base == "msrest" %} {% include "model_msrest.py.jinja2" %} -{% elif model.base == "typeddict" %} -{% include "model_typeddict.py.jinja2" %} {% endif %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 19435f14e88..8ff3e618aca 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -4,6 +4,28 @@ {% endif %} {{ imports }} -{% for nu in code_model.named_unions %} -{{nu.name}} = {{nu.type_definition()}} +{% import 'operation_tools.jinja2' as op_tools %} +{% import "macros.jinja2" as macros %} +{% for model in models %} + + +{{ serializer.declare_model(model) }} + """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} + + {% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} + {{ macros.wrap_model_string(line, '\n ') -}} + {% endfor %} + {% endfor %} + {% endif %} + """ + + {% for p in serializer.get_properties_to_declare(model)%} + {{ serializer.declare_property(p) }} + {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} + {% if prop_description %} + """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} + {% endif %} + {% endfor %} {% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 new file mode 100644 index 00000000000..e57874df349 --- /dev/null +++ b/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 @@ -0,0 +1,12 @@ +# coding=utf-8 +{% if code_model.license_header %} +{{ code_model.license_header }} +{% endif %} + +{{ imports }} +{% for nu in code_model.named_unions %} +{{nu.name}} = {{nu.type_definition()}} +{% endfor %} +{% for model in discriminated_bases %} +{{ serializer.discriminated_subtypes_union(model) }} +{% endfor %} diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index 5117a9227e5..6d3344059a3 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -216,7 +216,7 @@ def add_body_param_type( model_type = ( body_parameter["type"] if origin_type == "model" else body_parameter["type"].get("elementType", {}) ) - is_dpg_model = model_type.get("base") in ("dpg", "typeddict") + is_dpg_model = model_type.get("base") == "dpg" body_parameter["type"] = { "type": "combined", "types": [body_parameter["type"]], diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py new file mode 100644 index 00000000000..6a49978a899 --- /dev/null +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -0,0 +1,171 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +"""Tests for TypedDict generation, unions generation, and models-mode interactions.""" + +from jinja2 import PackageLoader, Environment + +from pygen.codegen.models import CodeModel, JSONModelType, DPGModelType +from pygen.codegen.models.model_type import TypedDictModelType +from pygen.codegen.serializers.types_serializer import TypesSerializer +from pygen.codegen.serializers.unions_serializer import UnionsSerializer + + +def _make_code_model(models_mode="dpg"): + return CodeModel( + { + "clients": [ + { + "name": "client", + "namespace": "blah", + "moduleName": "blah", + "parameters": [], + "url": "", + "operationGroups": [], + } + ], + "namespace": "namespace", + }, + options={ + "show-send-request": True, + "builders-visibility": "public", + "show-operations": True, + "models-mode": models_mode, + "flavor": "unbranded", + "client-side-validation": False, + }, + ) + + +def _make_model(code_model, name, model_cls=None, properties=None): + """Create a model of the given class attached to code_model.""" + if model_cls is None: + if code_model.options["models-mode"] == "typeddict": + model_cls = TypedDictModelType + elif code_model.options["models-mode"] == "dpg": + model_cls = DPGModelType + else: + model_cls = JSONModelType + return model_cls( + yaml_data={ + "name": name, + "type": "model", + "snakeCaseName": name.lower(), + "usage": 2, + }, + code_model=code_model, + properties=properties or [], + ) + + +def _make_env(): + return Environment( + loader=PackageLoader("pygen.codegen", "templates"), + trim_blocks=True, + lstrip_blocks=True, + ) + + +# ---------- models-mode=none ---------- + + +def test_models_mode_none_produces_json_model_type(): + """When models-mode is none (False), all models should be JSONModelType.""" + code_model = _make_code_model(models_mode=False) + model = _make_model(code_model, "Foo", model_cls=JSONModelType) + assert model.base == "json" + + +def test_models_mode_none_no_typeddict_models(): + """TypesSerializer.typeddict_models should be empty when models-mode=none.""" + code_model = _make_code_model(models_mode=False) + m1 = _make_model(code_model, "Foo", model_cls=JSONModelType) + m2 = _make_model(code_model, "Bar", model_cls=JSONModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1, m2]) + assert ts.typeddict_models == [] + + +def test_models_mode_none_types_file_has_no_typeddict_imports(): + """When models-mode=none, the types.py should not import TypedDict.""" + code_model = _make_code_model(models_mode=False) + m1 = _make_model(code_model, "Foo", model_cls=JSONModelType) + code_model.model_types = [m1] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + output = ts.serialize() + assert "TypedDict" not in output + assert "Required" not in output + + +# ---------- models-mode=dpg ---------- + + +def test_models_mode_dpg_no_typeddict_models(): + """DPG models have base='dpg', not 'typeddict', so should not appear as typeddict_models.""" + code_model = _make_code_model(models_mode="dpg") + m1 = _make_model(code_model, "Foo", model_cls=DPGModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + # DPG models have base != "json" so they DO appear in typeddict_models + assert len(ts.typeddict_models) == 1 + + +# ---------- models-mode=typeddict ---------- + + +def test_models_mode_typeddict_models_included(): + """TypedDictModelType models should appear in typeddict_models.""" + code_model = _make_code_model(models_mode="typeddict") + m1 = _make_model(code_model, "Foo", model_cls=TypedDictModelType) + m2 = _make_model(code_model, "Bar", model_cls=TypedDictModelType) + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1, m2]) + assert len(ts.typeddict_models) == 2 + + +def test_models_mode_typeddict_serialize_contains_class(): + """Serialized types.py output should contain TypedDict class definitions.""" + code_model = _make_code_model(models_mode="typeddict") + m1 = _make_model(code_model, "Foo", model_cls=TypedDictModelType) + code_model.model_types = [m1] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + output = ts.serialize() + assert "class Foo(TypedDict, total=False):" in output + assert "TypedDict" in output + + +def test_types_file_has_no_named_unions(): + """Serialized types.py should not contain named union definitions.""" + code_model = _make_code_model(models_mode="dpg") + m1 = _make_model(code_model, "Foo", model_cls=DPGModelType) + code_model.model_types = [m1] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[m1]) + output = ts.serialize() + # Named unions should be in _unions.py, not types.py + assert "named_unions" not in output + + +# ---------- unions serializer ---------- + + +def test_unions_serializer_no_unions(): + """UnionsSerializer with no named unions should produce minimal output.""" + code_model = _make_code_model(models_mode="dpg") + + env = _make_env() + us = UnionsSerializer(code_model=code_model, env=env) + output = us.serialize() + assert "TypedDict" not in output + assert "Union" not in output From db10eeb78142f9707dc56219237a918e7001022b Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 6 May 2026 10:40:47 -0400 Subject: [PATCH 13/42] move discriminated union to types.py --- .../pygen/codegen/templates/types.py.jinja2 | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 7681c78e3f4..8ff3e618aca 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -29,32 +29,3 @@ {% endif %} {% endfor %} {% endfor %} -{% import 'operation_tools.jinja2' as op_tools %} -{% import "macros.jinja2" as macros %} -{% for model in models %} -{% if serializer.is_discriminated_base(model) %} -{{ serializer.discriminated_subtypes_union(model) }} -{% else %} - - -{{ serializer.declare_model(model) }} - """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} - - {% if model.properties != None %} - {% for p in model.properties %} - {% for line in serializer.variable_documentation_string(p) %} - {{ macros.wrap_model_string(line, '\n ') -}} - {% endfor %} - {% endfor %} - {% endif %} - """ - - {% for p in serializer.get_properties_to_declare(model)%} - {{ serializer.declare_property(p) }} - {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} - {% if prop_description %} - """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} - {% endif %} - {% endfor %} -{% endif %} -{% endfor %} From a206181fdf868ff05f9f3be7d86d13ca01104344 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 6 May 2026 11:25:02 -0400 Subject: [PATCH 14/42] format --- packages/http-client-python/eng/scripts/setup/run_batch.py | 4 +--- .../generator/pygen/codegen/serializers/__init__.py | 4 +++- .../generator/pygen/codegen/serializers/types_serializer.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/eng/scripts/setup/run_batch.py b/packages/http-client-python/eng/scripts/setup/run_batch.py index 6d6bef5b0d2..9b978ec0319 100644 --- a/packages/http-client-python/eng/scripts/setup/run_batch.py +++ b/packages/http-client-python/eng/scripts/setup/run_batch.py @@ -51,9 +51,7 @@ def _coerce(value): return False return value - pygen_args = { - k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"] - } + pygen_args = {k: _coerce(v) for k, v in command_args.items() if k not in ["emit-yaml-only"]} # Run preprocess and codegen (black is batched at the end for performance) preprocess.PreProcessPlugin(output_folder=output_dir, tsp_file=yaml_path, **pygen_args).process() diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 1ea2ea0b7a7..ac9a6781344 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -525,7 +525,9 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str ) # write _unions.py - has_discriminated_bases = any(m for m in self.code_model.model_types if m.base != "json" and m.discriminated_subtypes) + has_discriminated_bases = any( + m for m in self.code_model.model_types if m.base != "json" and m.discriminated_subtypes + ) if self.code_model.named_unions or has_discriminated_bases: self.write_file( generation_dir / Path("_unions.py"), diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index b9a57a10b6c..7fa07f72029 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -102,7 +102,6 @@ def declare_property(self, prop: Property) -> str: return f"{prop.wire_name}: {type_annotation}" return f"{prop.wire_name}: Required[{type_annotation}]" - @staticmethod def variable_documentation_string(prop: Property) -> list[str]: return _documentation_string(prop, "ivar", "vartype") From 95724704f4df74d40a51b81c548771e4241ee6d8 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 6 May 2026 11:33:31 -0400 Subject: [PATCH 15/42] add for output as well --- .../generator/pygen/codegen/models/model_type.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index b01d8810513..738667c1816 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -374,21 +374,17 @@ class TypedDictModelType(DPGModelType): base = "typeddict" def type_annotation(self, **kwargs: Any) -> str: - if kwargs.pop("is_response", False): - return "JSON" + kwargs.pop("is_response", None) return super().type_annotation(**kwargs) def docstring_type(self, **kwargs: Any) -> str: - if kwargs.pop("is_response", False): - return "JSON" + kwargs.pop("is_response", None) return super().docstring_type(**kwargs) def docstring_text(self, **kwargs: Any) -> str: - if kwargs.pop("is_response", False): - return "JSON" + kwargs.pop("is_response", None) return super().docstring_text(**kwargs) def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - file_import.define_mutable_mapping_type() - return file_import + kwargs.pop("is_response", None) + return super().imports(**kwargs) From cdcedf53cb8f6945fbc06f0ab3d4b2dfcd3eddfa Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 6 May 2026 13:58:56 -0400 Subject: [PATCH 16/42] add e2e tests --- .../eng/scripts/ci/regenerate.ts | 16 +++- .../generator/pygen/__init__.py | 2 + .../pygen/codegen/models/code_model.py | 2 +- .../pygen/codegen/models/model_type.py | 42 +++++++++++ .../pygen/codegen/serializers/__init__.py | 10 ++- .../codegen/serializers/builder_serializer.py | 35 ++++++--- ...test_typetest_model_usage_typeddictonly.py | 44 +++++++++++ .../tests/unit/test_typeddict.py | 75 +++++++++++++++++++ 8 files changed, 206 insertions(+), 20 deletions(-) create mode 100644 packages/http-client-python/tests/mock_api/shared/test_typetest_model_usage_typeddictonly.py diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index a65230db578..bf6a8457b74 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -307,10 +307,18 @@ const EMITTER_OPTIONS: Record | Record Any: raise ValueError( f"--package-mode can only be {' or '.join(TYPESPEC_PACKAGE_MODE)} or directory which contains template files" # pylint: disable=line-too-long ) + if key == "typed-dict-only-models" and isinstance(value, str): + value = [v.strip() for v in value.split(",") if v.strip()] return value def setdefault(self, key: str, default: Any, /) -> Any: # type: ignore # pylint: disable=arguments-differ diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index b6c3b2dda0d..884072a831f 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -349,7 +349,7 @@ def model_types(self, val: list[ModelType]) -> None: @staticmethod def get_public_model_types(models: list[ModelType]) -> list[ModelType]: - return [m for m in models if not m.internal and not m.base == "json"] + return [m for m in models if not m.internal and not m.base == "json" and not m.is_typed_dict_only] @property def public_model_types(self) -> list[ModelType]: diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 738667c1816..0eea2063b32 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -77,6 +77,10 @@ def __init__( self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) + self.is_typed_dict_only: bool = ( + self.yaml_data.get("typedDictOnly", False) + or self.name in code_model.options.get("typed-dict-only-models", []) + ) @property def is_usage_output(self) -> bool: @@ -352,6 +356,22 @@ def imports(self, **kwargs: Any) -> FileImport: class DPGModelType(GeneratedModelType): base = "dpg" + def type_annotation(self, **kwargs: Any) -> str: + if self.is_typed_dict_only: + is_operation_file = kwargs.pop("is_operation_file", False) + skip_quote = kwargs.get("skip_quote", False) + retval = f"types.{self.name}" + return retval if is_operation_file or skip_quote else f'"{retval}"' + return super().type_annotation(**kwargs) + + def docstring_type(self, **kwargs: Any) -> str: + if self.is_typed_dict_only: + client_namespace = self.client_namespace + if self.code_model.options.get("generation-subdir"): + client_namespace += f".{self.code_model.options['generation-subdir']}" + return f"~{client_namespace}.types.{self.name}" + return super().docstring_type(**kwargs) + def serialization_type(self, **kwargs: Any) -> str: return ( self.type_annotation(skip_quote=True, **kwargs) @@ -364,6 +384,28 @@ def instance_check_template(self) -> str: return "isinstance({}, " + f"_models.{self.name})" def imports(self, **kwargs: Any) -> FileImport: + if self.is_typed_dict_only: + file_import = FileImport(self.code_model) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.add_submodule_import( + relative_path, + "types", + ImportType.LOCAL, + ) + elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( + serialize_namespace_type == NamespaceType.MODEL + and kwargs.get("called_by_property", False) + ): + file_import.add_submodule_import( + relative_path, + "types", + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) + return file_import file_import = super().imports(**kwargs) if self.flattened_property: file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index ac9a6781344..b131b8d1622 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -304,10 +304,14 @@ def _serialize_and_write_models_folder( serializer = DpgModelSerializer else: serializer = MsrestModelSerializer - if self.code_model.has_non_json_models(models): + # Filter out typed-dict-only models — they only appear in types.py, not as model classes + class_models = [m for m in models if not m.is_typed_dict_only] + if self.code_model.has_non_json_models(class_models): self.write_file( models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env, client_namespace=namespace, models=models).serialize(), + serializer( + code_model=self.code_model, env=env, client_namespace=namespace, models=class_models + ).serialize(), ) if enums: self.write_file( @@ -318,7 +322,7 @@ def _serialize_and_write_models_folder( ) self.write_file( models_path / Path("__init__.py"), - ModelInitSerializer(code_model=self.code_model, env=env, models=models, enums=enums).serialize(), + ModelInitSerializer(code_model=self.code_model, env=env, models=class_models, enums=enums).serialize(), ) self._keep_patch_file(models_path / Path("_patch.py"), env) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py index e43dcd916eb..3f60558523f 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -1000,6 +1000,12 @@ def response_deserialization( # pylint: disable=too-many-statements elif self.code_model.options["models-mode"] == "dpg": if builder.has_stream_response: deserialize_code.append("deserialized = response.content") + elif isinstance(response.type, ModelType) and response.type.is_typed_dict_only: + # Typed-dict-only models skip deserialization — return raw JSON + deserialize_code.append("if response.content:") + deserialize_code.append(" deserialized = response.json()") + deserialize_code.append("else:") + deserialize_code.append(" deserialized = None") else: format_filed = ( f', format="{response.type.encode}"' @@ -1429,18 +1435,23 @@ def _extract_data_callback( # pylint: disable=too-many-statements,too-many-bran ) pylint_disable = "" if self.code_model.options["models-mode"] == "dpg": - item_type = builder.item_type.type_annotation( - is_operation_file=True, serialize_namespace=self.serialize_namespace - ) - pylint_disable = ( - " # pylint: disable=protected-access" if getattr(builder.item_type, "internal", False) else "" - ) - list_of_elem_deserialized = [ - "_deserialize(", - f"{item_type},{pylint_disable}", - f"deserialized{access},", - ")", - ] + is_item_typed_dict_only = isinstance(builder.item_type, ModelType) and builder.item_type.is_typed_dict_only + if is_item_typed_dict_only: + # Typed-dict-only models skip deserialization — return raw JSON items + list_of_elem_deserialized = [f"deserialized{access}"] + else: + item_type = builder.item_type.type_annotation( + is_operation_file=True, serialize_namespace=self.serialize_namespace + ) + pylint_disable = ( + " # pylint: disable=protected-access" if getattr(builder.item_type, "internal", False) else "" + ) + list_of_elem_deserialized = [ + "_deserialize(", + f"{item_type},{pylint_disable}", + f"deserialized{access},", + ")", + ] else: list_of_elem_deserialized = [f"deserialized{access}"] list_of_elem_deserialized_str = "\n ".join(list_of_elem_deserialized) diff --git a/packages/http-client-python/tests/mock_api/shared/test_typetest_model_usage_typeddictonly.py b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_usage_typeddictonly.py new file mode 100644 index 00000000000..586eb7be791 --- /dev/null +++ b/packages/http-client-python/tests/mock_api/shared/test_typetest_model_usage_typeddictonly.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import pytest +from typetest.model.usage.typeddictonly import UsageClient +from typetest.model.usage.typeddictonly.types import InputRecord, OutputRecord, InputOutputRecord + + +@pytest.fixture +def client(): + with UsageClient() as client: + yield client + + +def test_input(client: UsageClient): + # TypedDict-only: pass a plain dict matching the TypedDict schema + result = client.input({"requiredProp": "example-value"}) + assert result is None + + +def test_output(client: UsageClient): + # TypedDict-only: output should be a plain dict (no model deserialization) + output = client.output() + assert isinstance(output, dict) + assert output["requiredProp"] == "example-value" + + +def test_input_and_output(client: UsageClient): + # TypedDict-only: input a dict, get a dict back + result = client.input_and_output({"requiredProp": "example-value"}) + assert isinstance(result, dict) + assert result["requiredProp"] == "example-value" + + +def test_no_model_classes(): + """Verify that typed-dict-only models don't generate model classes.""" + from typetest.model.usage.typeddictonly import models + + # models.__all__ should be empty — no model classes exported + assert models.__all__ == [] + # The TypedDicts should only exist in the types module + assert hasattr(InputRecord, "__required_keys__") or hasattr(InputRecord, "__annotations__") diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py index 6a49978a899..c35db047b0f 100644 --- a/packages/http-client-python/tests/unit/test_typeddict.py +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -169,3 +169,78 @@ def test_unions_serializer_no_unions(): output = us.serialize() assert "TypedDict" not in output assert "Union" not in output + + +# ---------- typed-dict-only ---------- + + +def _make_typed_dict_only_model(code_model, name, **extra_yaml): + """Create a TypedDictModelType with typedDictOnly=True.""" + yaml_data = { + "name": name, + "type": "model", + "snakeCaseName": name.lower(), + "usage": 2, + "typedDictOnly": True, + **extra_yaml, + } + return TypedDictModelType( + yaml_data=yaml_data, + code_model=code_model, + properties=[], + ) + + +def test_typed_dict_only_property(): + """is_typed_dict_only should be True when yaml_data has typedDictOnly=True.""" + code_model = _make_code_model(models_mode="typeddict") + model = _make_typed_dict_only_model(code_model, "Foo") + assert model.is_typed_dict_only is True + + normal_model = _make_model(code_model, "Bar", model_cls=TypedDictModelType) + assert normal_model.is_typed_dict_only is False + + +def test_typed_dict_only_excluded_from_public_model_types(): + """Typed-dict-only models should not appear in public_model_types.""" + code_model = _make_code_model(models_mode="typeddict") + normal = _make_model(code_model, "Normal", model_cls=TypedDictModelType) + td_only = _make_typed_dict_only_model(code_model, "TdOnly") + code_model.model_types = [normal, td_only] + + public = code_model.public_model_types + assert normal in public + assert td_only not in public + + +def test_typed_dict_only_still_in_types_file(): + """Typed-dict-only models should still appear in types.py as TypedDicts.""" + code_model = _make_code_model(models_mode="typeddict") + td_only = _make_typed_dict_only_model(code_model, "MyModel") + code_model.model_types = [td_only] + + env = _make_env() + ts = TypesSerializer(code_model=code_model, env=env, models=[td_only]) + output = ts.serialize() + assert "class MyModel(TypedDict, total=False):" in output + + +def test_typed_dict_only_type_annotation(): + """Typed-dict-only models should use types.Name, not _models.Name.""" + code_model = _make_code_model(models_mode="typeddict") + model = _make_typed_dict_only_model(code_model, "Foo") + + # In operation files, should be types.Name + annotation = model.type_annotation(is_operation_file=True) + assert annotation == "types.Foo" + assert "_models" not in annotation + + +def test_typed_dict_only_docstring_type(): + """Typed-dict-only models should reference types module, not models.""" + code_model = _make_code_model(models_mode="typeddict") + model = _make_typed_dict_only_model(code_model, "Foo") + + docstring = model.docstring_type() + assert "types.Foo" in docstring + assert "models.Foo" not in docstring From 1d1d7c673cb92fbfd4a44edc92960740cdeeb925 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 6 May 2026 15:50:48 -0400 Subject: [PATCH 17/42] update unions serializer to get around pyright issue --- .../generator/pygen/codegen/models/model_type.py | 10 ++++------ .../pygen/codegen/serializers/unions_serializer.py | 10 +++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 0eea2063b32..803bcade644 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -77,10 +77,9 @@ def __init__( self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) - self.is_typed_dict_only: bool = ( - self.yaml_data.get("typedDictOnly", False) - or self.name in code_model.options.get("typed-dict-only-models", []) - ) + self.is_typed_dict_only: bool = self.yaml_data.get( + "typedDictOnly", False + ) or self.name in code_model.options.get("typed-dict-only-models", []) @property def is_usage_output(self) -> bool: @@ -396,8 +395,7 @@ def imports(self, **kwargs: Any) -> FileImport: ImportType.LOCAL, ) elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( - serialize_namespace_type == NamespaceType.MODEL - and kwargs.get("called_by_property", False) + serialize_namespace_type == NamespaceType.MODEL and kwargs.get("called_by_property", False) ): file_import.add_submodule_import( relative_path, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py index e15fee99c82..66a61503aa4 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py @@ -43,11 +43,19 @@ def imports(self) -> FileImport: serialize_namespace_type=NamespaceType.UNIONS_FILE, ) ) + for model in self.discriminated_base_models: + for subtype in model.discriminated_subtypes.values(): + file_import.merge( + subtype.imports( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.UNIONS_FILE, + ) + ) return file_import def discriminated_subtypes_union(self, model: ModelType) -> str: subtypes = list(model.discriminated_subtypes.values()) - subtype_names = [s.name for s in subtypes] + subtype_names = [s.type_annotation(skip_quote=True) for s in subtypes] return f"{model.name} = Union[{', '.join(subtype_names)}]" def serialize(self) -> str: From f5e1f7d1941e722509585daeab080b487076f7c2 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 7 May 2026 15:06:14 -0400 Subject: [PATCH 18/42] fix: resolve pylint and pyright errors in generated types.py - Skip rest_field/rest_discriminator imports for TYPES_FILE context - Use bare model/enum names in types.py instead of _models.Name prefix to avoid pyright reportInvalidTypeForm with dotted forward refs - Import model/enum names directly from .models under TYPE_CHECKING - Fix unions file to properly import and reference discriminated subtypes - Fix typed-dict-only import style to use 'from .. import types' module Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/enum_type.py | 15 +++++++++++++-- .../pygen/codegen/models/model_type.py | 17 +++++++++++++++-- .../generator/pygen/codegen/models/property.py | 16 +++++++++------- .../codegen/serializers/types_serializer.py | 5 ++++- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 671a8c64d5b..110d18f15df 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -176,7 +176,10 @@ def type_annotation(self, **kwargs: Any) -> str: if self.code_model.options["models-mode"]: module_name = "" - if kwargs.get("need_model_alias", True): + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type == NamespaceType.TYPES_FILE: + pass # no module prefix for types.py + elif kwargs.get("need_model_alias", True): serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) module_name = f"{model_alias}." @@ -255,7 +258,15 @@ def imports(self, **kwargs: Any) -> FileImport: alias=alias, typing_section=TypingSection.REGULAR, ) - elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( + elif serialize_namespace_type == NamespaceType.TYPES_FILE: + # Import enum name directly to avoid dotted forward refs in TypedDict annotations + file_import.add_submodule_import( + f"{relative_path}models" if relative_path != "." else ".models", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) + elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): file_import.add_submodule_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 803bcade644..a0923e79ffb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -278,8 +278,12 @@ class GeneratedModelType(ModelType): def type_annotation(self, **kwargs: Any) -> str: is_operation_file = kwargs.pop("is_operation_file", False) skip_quote = kwargs.get("skip_quote", False) + serialize_namespace_type = kwargs.get("serialize_namespace_type") module_name = "" - if kwargs.get("need_model_alias", True): + # In types.py, use bare name to avoid pyright "variable in type expression" errors + if serialize_namespace_type == NamespaceType.TYPES_FILE: + pass # no module prefix + elif kwargs.get("need_model_alias", True): serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) module_name = f"{model_alias}." @@ -323,7 +327,16 @@ def imports(self, **kwargs: Any) -> FileImport: ImportType.LOCAL, alias="_Model", ) - elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( + elif serialize_namespace_type == NamespaceType.TYPES_FILE: + # In types.py, import model names directly to avoid pyright + # "variable not allowed in type expression" errors with dotted forward refs + file_import.add_submodule_import( + f"{relative_path}models" if relative_path != "." else ".models", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) + elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): file_import.add_submodule_import( diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index ca18aba8ddf..ddb7d5afa24 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -10,7 +10,7 @@ from .enum_type import EnumType from .base import BaseType from .imports import FileImport, ImportType -from .utils import add_to_description, add_to_pylint_disable +from .utils import add_to_description, add_to_pylint_disable, NamespaceType if TYPE_CHECKING: from .code_model import CodeModel @@ -155,12 +155,14 @@ def imports(self, **kwargs) -> FileImport: if (self.optional and self.client_default_value is None) or self.readonly: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if self.code_model.options["models-mode"] == "dpg": - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - file_import.add_submodule_import( - self.code_model.get_relative_import_path(serialize_namespace, module_name="_utils.model_base"), - "rest_discriminator" if self.is_discriminator else "rest_field", - ImportType.LOCAL, - ) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type != NamespaceType.TYPES_FILE: + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + file_import.add_submodule_import( + self.code_model.get_relative_import_path(serialize_namespace, module_name="_utils.model_base"), + "rest_discriminator" if self.is_discriminator else "rest_field", + ImportType.LOCAL, + ) return file_import @classmethod diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 7fa07f72029..349299359cb 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -96,7 +96,10 @@ def get_properties_to_declare(model: ModelType) -> list[Property]: return properties_to_declare def declare_property(self, prop: Property) -> str: - type_annotation = prop.type_annotation(serialize_namespace=self.serialize_namespace) + type_annotation = prop.type_annotation( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + ) is_optional = prop.optional or prop.client_default_value is not None if is_optional: return f"{prop.wire_name}: {type_annotation}" From f3be83c194d63b5cec78ae1085a9f4034fae2ee6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 10:26:13 -0700 Subject: [PATCH 19/42] fix pylint and pyright in generated code --- .../pygen/codegen/models/model_type.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index a0923e79ffb..b7ced6c4bca 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -328,14 +328,17 @@ def imports(self, **kwargs: Any) -> FileImport: alias="_Model", ) elif serialize_namespace_type == NamespaceType.TYPES_FILE: - # In types.py, import model names directly to avoid pyright - # "variable not allowed in type expression" errors with dotted forward refs - file_import.add_submodule_import( - f"{relative_path}models" if relative_path != "." else ".models", - self.name, - ImportType.LOCAL, - typing_section=TypingSection.TYPING, - ) + # Don't import models that will be defined as TypedDicts in the same types.py file. + # The forward reference string will resolve to the local TypedDict class definition. + same_namespace = relative_path == "." + will_be_local_typeddict = self.base != "json" and not self.discriminated_subtypes + if not (same_namespace and will_be_local_typeddict): + file_import.add_submodule_import( + f"{relative_path}models" if relative_path != "." else ".models", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): From bf7ef34736824003b01d1b7699f36830d3fc2996 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 11:17:25 -0700 Subject: [PATCH 20/42] fix: move discriminated union aliases from _unions.py to types.py Discriminated model unions now reference TypedDict subtypes instead of _Model classes. Union aliases are generated in types.py alongside the TypedDict definitions, and _unions.py is only generated when named unions exist. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/serializers/__init__.py | 6 +--- .../codegen/serializers/types_serializer.py | 19 +++++++++++-- .../codegen/serializers/unions_serializer.py | 28 ++----------------- .../pygen/codegen/templates/types.py.jinja2 | 3 ++ .../pygen/codegen/templates/unions.py.jinja2 | 3 -- 5 files changed, 23 insertions(+), 36 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index b131b8d1622..0a805acbeb4 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -529,16 +529,12 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str ) # write _unions.py - has_discriminated_bases = any( - m for m in self.code_model.model_types if m.base != "json" and m.discriminated_subtypes - ) - if self.code_model.named_unions or has_discriminated_bases: + if self.code_model.named_unions: self.write_file( generation_dir / Path("_unions.py"), UnionsSerializer( code_model=self.code_model, env=env, - models=self.code_model.model_types, ).serialize(), ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 349299359cb..df904a74e09 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -30,12 +30,26 @@ def typeddict_models(self) -> list[ModelType]: """Models that should be rendered as TypedDicts (excluding discriminated bases which become unions).""" return [m for m in self._models if m.base != "json" and not m.discriminated_subtypes] + @property + def discriminated_base_models(self) -> list[ModelType]: + """Discriminated base models that become Union type aliases in types.py.""" + return [m for m in self._models if m.base != "json" and m.discriminated_subtypes] + + def discriminated_subtypes_union(self, model: ModelType) -> str: + """Generate a Union alias for a discriminated base using TypedDict subtype names.""" + subtypes = list(model.discriminated_subtypes.values()) + subtype_names = [s.name for s in subtypes] + return f"{model.name} = Union[{', '.join(subtype_names)}]" + def imports(self) -> FileImport: file_import = FileImport(self.code_model) td_models = self.typeddict_models - if td_models: - file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) + if td_models or self.discriminated_base_models: + if td_models: + file_import.add_submodule_import("typing_extensions", "TypedDict", ImportType.STDLIB) + if self.discriminated_base_models: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) has_required = False for model in td_models: file_import.merge( @@ -116,4 +130,5 @@ def serialize(self) -> str: imports=FileImportSerializer(self.imports()), serializer=self, models=self.typeddict_models, + discriminated_bases=self.discriminated_base_models, ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py index 66a61503aa4..28d606ccf01 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/unions_serializer.py @@ -3,9 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from typing import Optional - -from ..models import CodeModel, ModelType +from ..models import CodeModel from ..models.imports import FileImport, ImportType from ..models.utils import NamespaceType from .import_serializer import FileImportSerializer @@ -17,20 +15,12 @@ def __init__( self, code_model: CodeModel, env, - models: Optional[list[ModelType]] = None, ): super().__init__(code_model=code_model, env=env) - self._models = models or [] - - @property - def discriminated_base_models(self) -> list[ModelType]: - """Models that are discriminated bases, rendered as Union aliases.""" - return [m for m in self._models if m.base != "json" and m.discriminated_subtypes] def imports(self) -> FileImport: file_import = FileImport(self.code_model) - has_unions = bool(self.code_model.named_unions) or bool(self.discriminated_base_models) - if has_unions: + if self.code_model.named_unions: file_import.add_submodule_import( "typing", "Union", @@ -43,26 +33,12 @@ def imports(self) -> FileImport: serialize_namespace_type=NamespaceType.UNIONS_FILE, ) ) - for model in self.discriminated_base_models: - for subtype in model.discriminated_subtypes.values(): - file_import.merge( - subtype.imports( - serialize_namespace=self.serialize_namespace, - serialize_namespace_type=NamespaceType.UNIONS_FILE, - ) - ) return file_import - def discriminated_subtypes_union(self, model: ModelType) -> str: - subtypes = list(model.discriminated_subtypes.values()) - subtype_names = [s.type_annotation(skip_quote=True) for s in subtypes] - return f"{model.name} = Union[{', '.join(subtype_names)}]" - def serialize(self) -> str: template = self.env.get_template("unions.py.jinja2") return template.render( code_model=self.code_model, imports=FileImportSerializer(self.imports()), serializer=self, - discriminated_bases=self.discriminated_base_models, ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 8ff3e618aca..915596749a2 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -29,3 +29,6 @@ {% endif %} {% endfor %} {% endfor %} +{% for model in discriminated_bases %} +{{ serializer.discriminated_subtypes_union(model) }} +{% endfor %} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 index e57874df349..19435f14e88 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/unions.py.jinja2 @@ -7,6 +7,3 @@ {% for nu in code_model.named_unions %} {{nu.name}} = {{nu.type_definition()}} {% endfor %} -{% for model in discriminated_bases %} -{{ serializer.discriminated_subtypes_union(model) }} -{% endfor %} From e6b1b5220b282e929cc3954b08c9fdf7c782109a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 11:30:02 -0700 Subject: [PATCH 21/42] fix: topologically sort discriminated union aliases in types.py Ensures nested discriminated bases (e.g. Shark) are defined before their parents (e.g. Fish = Union[Salmon, Shark]). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../codegen/serializers/types_serializer.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index df904a74e09..461c95bfef2 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -32,8 +32,29 @@ def typeddict_models(self) -> list[ModelType]: @property def discriminated_base_models(self) -> list[ModelType]: - """Discriminated base models that become Union type aliases in types.py.""" - return [m for m in self._models if m.base != "json" and m.discriminated_subtypes] + """Discriminated base models that become Union type aliases in types.py. + + Topologically sorted so that nested discriminated bases (e.g. Shark) + are defined before their parents (e.g. Fish = Union[Salmon, Shark]). + """ + bases = [m for m in self._models if m.base != "json" and m.discriminated_subtypes] + base_names = {m.name for m in bases} + # Sort: models whose subtypes include other discriminated bases must come after them + sorted_bases: list[ModelType] = [] + visited: set[str] = set() + + def visit(model: ModelType) -> None: + if model.name in visited: + return + visited.add(model.name) + for subtype in model.discriminated_subtypes.values(): + if subtype.name in base_names: + visit(subtype) + sorted_bases.append(model) + + for m in bases: + visit(m) + return sorted_bases def discriminated_subtypes_union(self, model: ModelType) -> str: """Generate a Union alias for a discriminated base using TypedDict subtype names.""" From df17af5d204fe755a34ef13512c0e3584f555886 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 12:41:27 -0700 Subject: [PATCH 22/42] fix: skip model imports in types.py regardless of namespace All TypedDict models are defined in the same types.py file, so cross- namespace model imports also cause redefinition errors. Remove the same_namespace check so any model that will be a TypedDict is never imported from its models module. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index b7ced6c4bca..f3875cf9260 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -329,10 +329,10 @@ def imports(self, **kwargs: Any) -> FileImport: ) elif serialize_namespace_type == NamespaceType.TYPES_FILE: # Don't import models that will be defined as TypedDicts in the same types.py file. - # The forward reference string will resolve to the local TypedDict class definition. - same_namespace = relative_path == "." - will_be_local_typeddict = self.base != "json" and not self.discriminated_subtypes - if not (same_namespace and will_be_local_typeddict): + # All non-json, non-discriminated-base models generate TypedDicts in types.py, + # regardless of namespace, so the bare forward reference resolves locally. + will_be_typeddict = self.base != "json" and not self.discriminated_subtypes + if not will_be_typeddict: file_import.add_submodule_import( f"{relative_path}models" if relative_path != "." else ".models", self.name, From cedece0226c83063878219a9c418222a03e12480 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 12:50:00 -0700 Subject: [PATCH 23/42] fix: skip imports for discriminated base models in types.py Discriminated bases are now Union aliases in types.py, so they must also be excluded from TYPE_CHECKING imports to avoid shadowing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index f3875cf9260..86ed72e228e 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -328,11 +328,11 @@ def imports(self, **kwargs: Any) -> FileImport: alias="_Model", ) elif serialize_namespace_type == NamespaceType.TYPES_FILE: - # Don't import models that will be defined as TypedDicts in the same types.py file. - # All non-json, non-discriminated-base models generate TypedDicts in types.py, - # regardless of namespace, so the bare forward reference resolves locally. - will_be_typeddict = self.base != "json" and not self.discriminated_subtypes - if not will_be_typeddict: + # Don't import models that will be defined in types.py — either as TypedDict + # classes (non-discriminated) or as Union aliases (discriminated bases). + # All non-json models appear in types.py, so the bare forward reference resolves locally. + will_be_in_types_file = self.base != "json" + if not will_be_in_types_file: file_import.add_submodule_import( f"{relative_path}models" if relative_path != "." else ".models", self.name, From 44971994cf1ea3d68b3231b8cf1411773cfa7d36 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 13:20:24 -0700 Subject: [PATCH 24/42] fix: sort TypedDict properties to avoid builtin name shadowing When a TypedDict field wire_name matches a Python builtin type name (e.g. 'int'), it shadows that builtin in subsequent annotations. Move such properties to the end of the class to prevent this. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/serializers/types_serializer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 461c95bfef2..843d83c0d16 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -111,6 +111,12 @@ def declare_model(self, model: ModelType) -> str: return f"class {model.name}({basename}):{model.pylint_disable()}" return f"class {model.name}(TypedDict, total=False):{model.pylint_disable()}" + # Python builtin type names that can be shadowed by TypedDict field names + _BUILTIN_TYPE_NAMES = frozenset({ + "int", "str", "float", "bool", "list", "dict", "tuple", "set", + "bytes", "type", "object", "complex", "frozenset", "bytearray", "memoryview", + }) + @staticmethod def get_properties_to_declare(model: ModelType) -> list[Property]: non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] @@ -127,7 +133,12 @@ def get_properties_to_declare(model: ModelType) -> list[Property]: ) ] else: - properties_to_declare = model.properties + properties_to_declare = list(model.properties) + # Move properties whose wire_name shadows a Python builtin type to the end, + # so they don't shadow the builtin in subsequent type annotations. + properties_to_declare.sort( + key=lambda p: p.wire_name in TypesSerializer._BUILTIN_TYPE_NAMES + ) return properties_to_declare def declare_property(self, prop: Property) -> str: From 7c8bf77f7254141b5950b0ce57601260b2485d44 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 13:46:56 -0700 Subject: [PATCH 25/42] fix: handle reserved words in TypedDict field names comprehensively - Use builtins.X qualification when a TypedDict field wire_name shadows a Python builtin type name (e.g. int, str, list) that appears in type annotations within the same class - Add functional TypedDict form for models with Python keyword wire_names (e.g. and, class, for) since keywords can't be identifiers in class bodies - Only import builtins when actually needed (when shadowed builtins appear in annotations) - Remove the previous sorting workaround Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../codegen/serializers/types_serializer.py | 99 ++++++++++++++++--- .../pygen/codegen/templates/types.py.jinja2 | 19 +++- 2 files changed, 101 insertions(+), 17 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 843d83c0d16..df78ce79f2d 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- +import keyword +import re from typing import Optional from ..models import ModelType, CodeModel from ..models.imports import FileImport, ImportType @@ -12,6 +14,23 @@ from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer +# Python builtin type names that can be shadowed by TypedDict field wire_names. +# When a field name matches one of these, all references to that builtin in type +# annotations within the same class are qualified as builtins.X. +_BUILTIN_TYPE_NAMES = frozenset({ + "int", "str", "float", "bool", "list", "dict", "tuple", "set", + "bytes", "type", "object", "complex", "frozenset", "bytearray", "memoryview", +}) + + +def _qualify_shadowed_builtins(annotation: str, shadowed: frozenset[str]) -> str: + """Replace bare builtin type references with builtins.X when shadowed by a field name.""" + if not shadowed: + return annotation + for name in shadowed: + annotation = re.sub(rf"\b{name}\b", f"builtins.{name}", annotation) + return annotation + class TypesSerializer(BaseSerializer): def __init__( @@ -39,7 +58,6 @@ def discriminated_base_models(self) -> list[ModelType]: """ bases = [m for m in self._models if m.base != "json" and m.discriminated_subtypes] base_names = {m.name for m in bases} - # Sort: models whose subtypes include other discriminated bases must come after them sorted_bases: list[ModelType] = [] visited: set[str] = set() @@ -62,6 +80,30 @@ def discriminated_subtypes_union(self, model: ModelType) -> str: subtype_names = [s.name for s in subtypes] return f"{model.name} = Union[{', '.join(subtype_names)}]" + @staticmethod + def has_keyword_wire_names(model: ModelType) -> bool: + """Whether any property wire_name is a Python keyword (requires functional TypedDict form).""" + return any(keyword.iskeyword(p.wire_name) for p in model.properties) + + @staticmethod + def get_shadowed_builtins(model: ModelType) -> frozenset[str]: + """Return the set of builtin type names shadowed by property wire_names in this model. + + Only includes a builtin if it is both used as a wire_name AND referenced + in a type annotation within the same model (otherwise no shadowing occurs). + """ + wire_builtins = {p.wire_name for p in model.properties if p.wire_name in _BUILTIN_TYPE_NAMES} + if not wire_builtins: + return frozenset() + # Check which of these builtins actually appear in type annotations + used = set() + for prop in model.properties: + annotation = prop.type_annotation() + for name in wire_builtins: + if re.search(rf"\b{name}\b", annotation): + used.add(name) + return frozenset(used) + def imports(self) -> FileImport: file_import = FileImport(self.code_model) @@ -72,6 +114,7 @@ def imports(self) -> FileImport: if self.discriminated_base_models: file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) has_required = False + needs_builtins = False for model in td_models: file_import.merge( model.imports( @@ -90,6 +133,8 @@ def imports(self) -> FileImport: ) if not (prop.optional or prop.client_default_value is not None): has_required = True + if self.get_shadowed_builtins(model): + needs_builtins = True for parent in model.parents: if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: file_import.add_submodule_import( @@ -102,27 +147,55 @@ def imports(self) -> FileImport: ) if has_required: file_import.add_submodule_import("typing_extensions", "Required", ImportType.STDLIB) + if needs_builtins: + file_import.add_import("builtins", ImportType.STDLIB) return file_import def declare_model(self, model: ModelType) -> str: + """Generate the class declaration or functional form for a TypedDict model. + + Uses functional form when any property wire_name is a Python keyword + (e.g. 'and', 'class') since keywords can't be identifiers in class bodies. + """ + if self.has_keyword_wire_names(model): + return "" # functional form is rendered separately non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] if non_discriminated_parents: basename = ", ".join([m.name for m in non_discriminated_parents]) return f"class {model.name}({basename}):{model.pylint_disable()}" return f"class {model.name}(TypedDict, total=False):{model.pylint_disable()}" - # Python builtin type names that can be shadowed by TypedDict field names - _BUILTIN_TYPE_NAMES = frozenset({ - "int", "str", "float", "bool", "list", "dict", "tuple", "set", - "bytes", "type", "object", "complex", "frozenset", "bytearray", "memoryview", - }) + def declare_functional_model(self, model: ModelType) -> str: + """Generate a functional-form TypedDict for models with keyword wire_names. + + Functional form is required when any field name is a Python keyword. + All fields (including inherited) are included since functional form + can't specify a base class. + """ + shadowed = self.get_shadowed_builtins(model) + entries: list[str] = [] + for prop in model.properties: + type_annotation = prop.type_annotation( + serialize_namespace=self.serialize_namespace, + serialize_namespace_type=NamespaceType.TYPES_FILE, + ) + type_annotation = _qualify_shadowed_builtins(type_annotation, shadowed) + is_optional = prop.optional or prop.client_default_value is not None + if is_optional: + entries.append(f' "{prop.wire_name}": {type_annotation},') + else: + entries.append(f' "{prop.wire_name}": Required[{type_annotation}],') + fields = "\n".join(entries) + return f'{model.name} = TypedDict("{model.name}", {{\n{fields}\n}}, total=False)' @staticmethod def get_properties_to_declare(model: ModelType) -> list[Property]: + if TypesSerializer.has_keyword_wire_names(model): + return [] # functional form handles all properties non_discriminated_parents = [p for p in model.parents if not p.discriminated_subtypes] if non_discriminated_parents: parent_properties = [p for bm in non_discriminated_parents for p in bm.properties] - properties_to_declare = [ + return [ p for p in model.properties if not any( @@ -132,20 +205,14 @@ def get_properties_to_declare(model: ModelType) -> list[Property]: for pp in parent_properties ) ] - else: - properties_to_declare = list(model.properties) - # Move properties whose wire_name shadows a Python builtin type to the end, - # so they don't shadow the builtin in subsequent type annotations. - properties_to_declare.sort( - key=lambda p: p.wire_name in TypesSerializer._BUILTIN_TYPE_NAMES - ) - return properties_to_declare + return list(model.properties) - def declare_property(self, prop: Property) -> str: + def declare_property(self, prop: Property, shadowed_builtins: frozenset[str]) -> str: type_annotation = prop.type_annotation( serialize_namespace=self.serialize_namespace, serialize_namespace_type=NamespaceType.TYPES_FILE, ) + type_annotation = _qualify_shadowed_builtins(type_annotation, shadowed_builtins) is_optional = prop.optional or prop.client_default_value is not None if is_optional: return f"{prop.wire_name}: {type_annotation}" diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 915596749a2..d75c26bc6ae 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -7,6 +7,21 @@ {% import 'operation_tools.jinja2' as op_tools %} {% import "macros.jinja2" as macros %} {% for model in models %} +{% if serializer.has_keyword_wire_names(model) %} + + +{{ serializer.declare_functional_model(model) }} +"""{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n") }} + +{% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} +{{ macros.wrap_model_string(line, '\n') -}} + {% endfor %} + {% endfor %} +{% endif %} +""" +{% else %} {{ serializer.declare_model(model) }} @@ -21,13 +36,15 @@ {% endif %} """ + {% set shadowed = serializer.get_shadowed_builtins(model) %} {% for p in serializer.get_properties_to_declare(model)%} - {{ serializer.declare_property(p) }} + {{ serializer.declare_property(p, shadowed) }} {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} {% if prop_description %} """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} {% endif %} {% endfor %} +{% endif %} {% endfor %} {% for model in discriminated_bases %} {{ serializer.discriminated_subtypes_union(model) }} From 136e2474454ae8163680edef21863330130bd0c4 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 11 May 2026 14:29:40 -0700 Subject: [PATCH 26/42] fix: generate types.py per namespace instead of only at top level Previously types.py was only generated once at the root namespace with all models. Now it's generated per namespace alongside the models/ folder, using only that namespace's models. This ensures sub-namespaces like modelproperties/ get their own types.py. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/serializers/__init__.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 0a805acbeb4..785461a68d6 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -217,6 +217,19 @@ def serialize(self) -> None: enums=client_namespace_type.enums, ) + # write types.py per namespace (alongside models/) + if self.code_model.has_non_json_models(client_namespace_type.models): + generation_dir = self.code_model.get_generation_dir(client_namespace) + self.write_file( + generation_dir / Path("types.py"), + TypesSerializer( + code_model=self.code_model, + env=env, + client_namespace=client_namespace, + models=client_namespace_type.models, + ).serialize(), + ) + if not self.code_model.options["models-mode"]: # keep models file if users ended up just writing a models file model_path = generation_path / Path("models.py") @@ -538,17 +551,6 @@ def _serialize_and_write_top_level_folder(self, env: Environment, namespace: str ).serialize(), ) - # write types.py - if self.code_model.model_types: - self.write_file( - generation_dir / Path("types.py"), - TypesSerializer( - code_model=self.code_model, - env=env, - models=self.code_model.model_types, - ).serialize(), - ) - # pylint: disable=line-too-long @property def sample_additional_folder(self) -> Path: From b14f8745d6cde26d5fb43c561e48433455eb4d84 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 12 May 2026 09:53:40 -0700 Subject: [PATCH 27/42] fix: correct cross-namespace imports in per-namespace types.py - Pass client_namespace to BaseSerializer in TypesSerializer so serialize_namespace resolves relative to the correct namespace - Distinguish same-namespace vs cross-namespace models in TYPES_FILE import logic: same-namespace non-json models are skipped (defined locally), cross-namespace models import from sibling types module, same-namespace json models import from .models Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/models/model_type.py | 31 +++++++++++++------ .../codegen/serializers/types_serializer.py | 3 +- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 86ed72e228e..6913e7656af 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -328,17 +328,28 @@ def imports(self, **kwargs: Any) -> FileImport: alias="_Model", ) elif serialize_namespace_type == NamespaceType.TYPES_FILE: - # Don't import models that will be defined in types.py — either as TypedDict - # classes (non-discriminated) or as Union aliases (discriminated bases). - # All non-json models appear in types.py, so the bare forward reference resolves locally. - will_be_in_types_file = self.base != "json" + # Don't import models that will be defined in this namespace's types.py — + # either as TypedDict classes (non-discriminated) or as Union aliases (discriminated bases). + # Only same-namespace non-json models are in the same types.py file. + same_namespace = relative_path == "." + will_be_in_types_file = self.base != "json" and same_namespace if not will_be_in_types_file: - file_import.add_submodule_import( - f"{relative_path}models" if relative_path != "." else ".models", - self.name, - ImportType.LOCAL, - typing_section=TypingSection.TYPING, - ) + if same_namespace: + # json models from same namespace — import from .models + file_import.add_submodule_import( + ".models", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) + else: + # Cross-namespace model — import from sibling namespace's types module + file_import.add_submodule_import( + f"{relative_path}types", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( serialize_namespace_type == NamespaceType.MODEL and called_by_property ): diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index df78ce79f2d..251fc47fde1 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -40,8 +40,7 @@ def __init__( client_namespace: Optional[str] = None, models: Optional[list[ModelType]] = None, ): - super().__init__(code_model=code_model, env=env) - self._client_namespace = client_namespace + super().__init__(code_model=code_model, env=env, client_namespace=client_namespace) self._models = models or [] @property From c67d63356958482034b81e25896e9ef2e83e73dd Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 12 May 2026 10:56:00 -0700 Subject: [PATCH 28/42] feat: combine regenerate scripts and add typeddict variant packages - Merge regenerate-common.ts into regenerate.ts (single file) - Add typeddict variant packages for not-discriminated and single-discriminator - Add typeddictonly variant package for usage (typed-dict-only-models: all) - Support 'all' value for typed-dict-only-models option in model_type.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../eng/scripts/ci/regenerate-common.ts | 747 ------------------ .../eng/scripts/ci/regenerate.ts | 712 ++++++++++++++++- .../pygen/codegen/models/model_type.py | 3 +- 3 files changed, 698 insertions(+), 764 deletions(-) delete mode 100644 packages/http-client-python/eng/scripts/ci/regenerate-common.ts diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts deleted file mode 100644 index f21c4013616..00000000000 --- a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts +++ /dev/null @@ -1,747 +0,0 @@ -/* eslint-disable no-console */ -/** - * Shared helpers, types, constants, and data tables used by `regenerate.ts`. - * - * This file is meant to be **byte-identical** between this package and the - * upstream `@typespec/http-client-python`. typespec-python syncs it from - * /core/packages/http-client-python/eng/scripts/ci/regenerate-common.ts - * via `pnpm sync`. - * - * Per-repo divergence (paths, emitter name, single-phase vs two-phase - * orchestration, argv/help text) lives in each repo's own `regenerate.ts`, - * which builds a `RegenerateContext` and feeds it into the helpers exported - * from this module. - */ - -import { compile, NodeHost } from "@typespec/compiler"; -import { execSync } from "child_process"; -import { existsSync, rmSync } from "fs"; -import { access, cp, mkdir, mkdtemp, readdir, writeFile } from "fs/promises"; -import { tmpdir } from "os"; -import { dirname, join, relative, resolve } from "path"; -import pc from "picocolors"; - -// ---- Public types ---- - -export interface RegenerateFlags { - flavor: string; - debug: boolean; - name?: string; -} - -export interface CompileTask { - spec: string; - outputDir: string; - options: Record; -} - -// Group of tasks for the same spec that must run sequentially -export interface TaskGroup { - spec: string; - tasks: CompileTask[]; -} - -/** - * Per-repo context injected into the helpers below. Every value is repo - * specific and must be supplied by the caller's `regenerate.ts`. - */ -export interface RegenerateContext { - /** Absolute path to the package root (the dir containing `package.json`). */ - pluginDir: string; - /** Absolute path to the azure-http-specs `specs/` dir. */ - azureHttpSpecs: string; - /** Absolute path to the http-specs `specs/` dir. */ - httpSpecs: string; - /** Absolute path to where generated SDKs should go (e.g. `/generator`). */ - generatedFolder: string; - /** Emitter name to invoke (e.g. `@azure-tools/typespec-python`). */ - emitterName: string; -} - -/** - * Optional knobs for `buildTaskGroups`. Kept here so the call site in each - * repo's `regenerate.ts` can opt into the upstream two-phase pipeline - * (`emitYamlOnly: true`) or the single-phase pipeline (default). - */ -export interface BuildTaskGroupsOptions { - /** If true, ask the emitter to write YAML only and skip Python codegen. */ - emitYamlOnly?: boolean; -} - -// ---- Public constants ---- - -export const SKIP_SPECS: string[] = [ - "type/file", - "service/multiple-services", - "azure/client-generator-core/response-as-bool", -]; - -export const SpecialFlags: Record> = { - azure: { - "generate-test": true, - "generate-sample": true, - }, -}; - -// ---- Spec-specific emitter option overrides ---- - -export const AZURE_EMITTER_OPTIONS: Record< - string, - Record | Record[] -> = { - "azure/client-generator-core/access": { - namespace: "specs.azure.clientgenerator.core.access", - }, - "azure/client-generator-core/alternate-type": { - namespace: "specs.azure.clientgenerator.core.alternatetype", - }, - "azure/client-generator-core/api-version": { - namespace: "specs.azure.clientgenerator.core.apiversion", - }, - "azure/client-generator-core/client-initialization/default": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.default", - }, - "azure/client-generator-core/client-initialization/individually": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", - }, - "azure/client-generator-core/client-initialization/individuallyParent": { - namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", - }, - "azure/client-generator-core/client-location": { - namespace: "specs.azure.clientgenerator.core.clientlocation", - }, - "azure/client-generator-core/deserialize-empty-string-as-null": { - namespace: "specs.azure.clientgenerator.core.emptystring", - }, - "azure/client-generator-core/flatten-property": { - namespace: "specs.azure.clientgenerator.core.flattenproperty", - }, - "azure/client-generator-core/usage": { - namespace: "specs.azure.clientgenerator.core.usage", - }, - "azure/client-generator-core/client-doc": { - namespace: "specs.azure.clientgenerator.core.clientdoc", - }, - "azure/client-generator-core/override": { - namespace: "specs.azure.clientgenerator.core.override", - }, - "azure/client-generator-core/hierarchy-building": { - namespace: "specs.azure.clientgenerator.core.hierarchybuilding", - }, - "azure/core/basic": { - namespace: "specs.azure.core.basic", - }, - "azure/core/lro/rpc": { - namespace: "specs.azure.core.lro.rpc", - }, - "azure/core/lro/standard": { - namespace: "specs.azure.core.lro.standard", - }, - "azure/core/model": { - namespace: "specs.azure.core.model", - }, - "azure/core/page": { - namespace: "specs.azure.core.page", - }, - "azure/core/scalar": { - namespace: "specs.azure.core.scalar", - }, - "azure/core/traits": { - namespace: "specs.azure.core.traits", - }, - "azure/encode/duration": { - namespace: "specs.azure.encode.duration", - }, - "azure/example/basic": { - namespace: "specs.azure.example.basic", - }, - "azure/payload/pageable": { - namespace: "specs.azure.payload.pageable", - }, - "azure/versioning/previewVersion": { - namespace: "specs.azure.versioning.previewversion", - }, - "client/structure/default": { - namespace: "client.structure.service", - }, - "client/structure/multi-client": { - "package-name": "client-structure-multiclient", - namespace: "client.structure.multiclient", - }, - "client/structure/renamed-operation": { - "package-name": "client-structure-renamedoperation", - namespace: "client.structure.renamedoperation", - }, - "client/structure/two-operation-group": { - "package-name": "client-structure-twooperationgroup", - namespace: "client.structure.twooperationgroup", - }, - "client/naming": { - namespace: "client.naming.main", - }, - "client/overload": { - namespace: "client.overload", - }, - "encode/duration": { - namespace: "encode.duration", - }, - "encode/numeric": { - namespace: "encode.numeric", - }, - "parameters/basic": { - namespace: "parameters.basic", - }, - "parameters/spread": { - namespace: "parameters.spread", - }, - "payload/content-negotiation": { - namespace: "payload.contentnegotiation", - }, - "payload/multipart": { - namespace: "payload.multipart", - }, - "serialization/encoded-name/json": { - namespace: "serialization.encodedname.json", - }, - "special-words": { - namespace: "specialwords", - }, - "service/multi-service": { - namespace: "service.multiservice", - }, - "client/structure/client-operation-group": { - "package-name": "client-structure-clientoperationgroup", - namespace: "client.structure.clientoperationgroup", - }, -}; - -export const EMITTER_OPTIONS: Record | Record[]> = { - "resiliency/srv-driven/old.tsp": { - "package-name": "resiliency-srv-driven1", - namespace: "resiliency.srv.driven1", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven1", - }, - "resiliency/srv-driven": { - "package-name": "resiliency-srv-driven2", - namespace: "resiliency.srv.driven2", - "package-mode": "azure-dataplane", - "package-pprint-name": "ResiliencySrvDriven2", - }, - "authentication/api-key": { - "clear-output-folder": "true", - }, - "authentication/http/custom": { - "package-name": "authentication-http-custom", - namespace: "authentication.http.custom", - "package-pprint-name": "Authentication Http Custom", - }, - "authentication/union": [ - { - "package-name": "authentication-union", - namespace: "authentication.union", - }, - { - "package-name": "setuppy-authentication-union", - namespace: "setuppy.authentication.union", - "keep-setup-py": "true", - }, - ], - "type/array": { - "package-name": "typetest-array", - namespace: "typetest.array", - }, - "type/dictionary": { - "package-name": "typetest-dictionary", - namespace: "typetest.dictionary", - }, - "type/enum/extensible": { - "package-name": "typetest-enum-extensible", - namespace: "typetest.enum.extensible", - }, - "type/enum/fixed": { - "package-name": "typetest-enum-fixed", - namespace: "typetest.enum.fixed", - }, - "type/model/empty": { - "package-name": "typetest-model-empty", - namespace: "typetest.model.empty", - }, - "type/model/inheritance/enum-discriminator": { - "package-name": "typetest-model-enumdiscriminator", - namespace: "typetest.model.enumdiscriminator", - }, - "type/model/inheritance/nested-discriminator": { - "package-name": "typetest-model-nesteddiscriminator", - namespace: "typetest.model.nesteddiscriminator", - }, - "type/model/inheritance/not-discriminated": { - "package-name": "typetest-model-notdiscriminated", - namespace: "typetest.model.notdiscriminated", - }, - "type/model/inheritance/single-discriminator": { - "package-name": "typetest-model-singlediscriminator", - namespace: "typetest.model.singlediscriminator", - }, - "type/model/inheritance/recursive": [ - { - "package-name": "typetest-model-recursive", - namespace: "typetest.model.recursive", - }, - { - "package-name": "generation-subdir", - namespace: "generation.subdir", - "generation-subdir": "_generated", - "generate-test": "false", - "clear-output-folder": "true", - }, - ], - "type/model/usage": { - "package-name": "typetest-model-usage", - namespace: "typetest.model.usage", - }, - "type/model/visibility": [ - { - "package-name": "typetest-model-visibility", - namespace: "typetest.model.visibility", - }, - { - "package-name": "headasbooleantrue", - namespace: "headasbooleantrue", - "head-as-boolean": "true", - }, - { - "package-name": "headasbooleanfalse", - namespace: "headasbooleanfalse", - "head-as-boolean": "false", - }, - ], - "type/property/nullable": { - "package-name": "typetest-property-nullable", - namespace: "typetest.property.nullable", - }, - "type/property/optionality": { - "package-name": "typetest-property-optional", - namespace: "typetest.property.optional", - }, - "type/property/additional-properties": { - "package-name": "typetest-property-additionalproperties", - namespace: "typetest.property.additionalproperties", - }, - "type/scalar": { - "package-name": "typetest-scalar", - namespace: "typetest.scalar", - }, - "type/property/value-types": { - "package-name": "typetest-property-valuetypes", - namespace: "typetest.property.valuetypes", - }, - "type/union": { - "package-name": "typetest-union", - namespace: "typetest.union", - }, - "type/union/discriminated": { - "package-name": "typetest-discriminatedunion", - namespace: "typetest.discriminatedunion", - }, - "type/file": { - "package-name": "typetest-file", - namespace: "typetest.file", - }, - documentation: { - "package-name": "specs-documentation", - namespace: "specs.documentation", - }, - "versioning/added": [ - { - "package-name": "versioning-added", - namespace: "versioning.added", - }, - { - "package-name": "generation-subdir2", - namespace: "generation.subdir2", - "generate-test": "false", - "generation-subdir": "_generated", - }, - ], -}; - -// ---- Public helpers ---- - -export function toPosix(p: string): string { - return p.replace(/\\/g, "/"); -} - -/** - * Whether a spec path belongs to azure-http-specs (vs standard http-specs). - * Uses the `azure-http-specs` substring rather than `azure` to avoid false - * positives when the working-dir path itself contains "azure" (e.g. - * azure-sdk-for-python). - */ -export function isAzureSpec(spec: string): boolean { - return spec.includes("azure-http-specs"); -} - -export function defaultPackageName(spec: string, ctx: RegenerateContext): string { - const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; - return toPosix(relative(specDir, dirname(spec))) - .replace(/\//g, "-") - .toLowerCase(); -} - -export function getEmitterOptions( - spec: string, - flavor: string, - ctx: RegenerateContext, -): Record[] { - const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; - const relativeSpec = toPosix(relative(specDir, spec)); - const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") - ? relativeSpec - : dirname(relativeSpec); - const emitterOpts = EMITTER_OPTIONS[key] || - (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; - return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; -} - -/** - * Walk `baseDir` and collect every TypeSpec entry-point file that the - * regenerator should compile (handles `client.tsp`, `main.tsp`, and the - * special `resiliency/srv-driven/old.tsp` case). - */ -export async function getSubdirectories( - baseDir: string, - flags: RegenerateFlags, -): Promise { - const subdirectories: string[] = []; - - async function searchDir(currentDir: string) { - const items = await readdir(currentDir, { withFileTypes: true }); - - const promisesArray = items.map(async (item) => { - const subDirPath = join(currentDir, item.name); - if (item.isDirectory()) { - const mainTspPath = join(subDirPath, "main.tsp"); - const clientTspPath = join(subDirPath, "client.tsp"); - - const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); - - if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; - - const hasMainTsp = await access(mainTspPath) - .then(() => true) - .catch(() => false); - const hasClientTsp = await access(clientTspPath) - .then(() => true) - .catch(() => false); - - if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { - if (mainTspRelativePath.includes("resiliency/srv-driven")) { - subdirectories.push(resolve(subDirPath, "old.tsp")); - } - if (hasClientTsp) { - subdirectories.push(resolve(subDirPath, "client.tsp")); - } else if (hasMainTsp) { - subdirectories.push(resolve(subDirPath, "main.tsp")); - } - } - - await searchDir(subDirPath); - } - }); - - await Promise.all(promisesArray); - } - - await searchDir(baseDir); - return subdirectories; -} - -export function buildTaskGroups( - specs: string[], - flags: RegenerateFlags, - ctx: RegenerateContext, - options: BuildTaskGroupsOptions = {}, -): TaskGroup[] { - const groups: TaskGroup[] = []; - - for (const spec of specs) { - const tasks: CompileTask[] = []; - - for (const emitterConfig of getEmitterOptions(spec, flags.flavor, ctx)) { - // Apply flavor defaults first, then per-spec options so they can override - // (e.g. "generate-test": "false") - const opts: Record = {}; - for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { - opts[k] = v; - } - Object.assign(opts, emitterConfig); - - opts["flavor"] = flags.flavor; - - // Set output directory - tests/generated// structure. - // Always anchored at /../tests/generated regardless of - // pluginDir, so generator-only checkouts work too. - const packageName = (opts["package-name"] as string) || defaultPackageName(spec, ctx); - const outputDir = - (opts["emitter-output-dir"] as string) || - toPosix(`${ctx.generatedFolder}/../tests/generated/${flags.flavor}/${packageName}`); - opts["emitter-output-dir"] = outputDir; - - if (flags.debug) { - opts["debug"] = true; - } - - opts["examples-dir"] = toPosix(join(dirname(spec), "examples")); - - if (options.emitYamlOnly) { - // Emit YAML only - Python processing is batched after all specs compile. - opts["emit-yaml-only"] = true; - } - - tasks.push({ spec, outputDir, options: opts }); - } - - groups.push({ spec, tasks }); - } - - return groups; -} - -export async function compileSpec( - task: CompileTask, - ctx: RegenerateContext, -): Promise<{ success: boolean; error?: string }> { - const { spec, outputDir, options } = task; - - try { - const compilerOptions = { - emit: [ctx.pluginDir], - options: { - [ctx.emitterName]: options, - }, - }; - - const program = await compile(NodeHost, spec, compilerOptions); - - if (program.hasError()) { - const errors = program.diagnostics - .filter((d) => d.severity === "error") - .map((d) => d.message) - .join("\n"); - return { success: false, error: errors }; - } - - return { success: true }; - } catch (err) { - rmSync(outputDir, { recursive: true, force: true }); - return { success: false, error: String(err) }; - } -} - -export function renderProgressBar( - completed: number, - failed: number, - total: number, - width: number = 40, -): string { - const successCount = completed - failed; - const successWidth = Math.round((successCount / total) * width); - const failWidth = Math.round((failed / total) * width); - const emptyWidth = width - successWidth - failWidth; - - const successBar = pc.bgGreen(" ".repeat(successWidth)); - const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; - const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); - - const percent = Math.round((completed / total) * 100); - return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; -} - -export async function runParallel( - groups: TaskGroup[], - maxJobs: number, - ctx: RegenerateContext, -): Promise> { - const results = new Map(); - const executing: Set> = new Set(); - - const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); - let completed = 0; - let failed = 0; - const failedSpecs: string[] = []; - - const isTTY = process.stdout.isTTY; - - const updateProgress = () => { - if (isTTY) { - process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); - } - }; - - updateProgress(); - - for (const group of groups) { - // Each group runs as a unit - tasks within a group run sequentially - // to avoid state pollution. Different groups run in parallel. - const runGroup = async () => { - const specDir = isAzureSpec(group.spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; - const shortName = toPosix(relative(specDir, dirname(group.spec))); - - let groupSuccess = true; - for (const task of group.tasks) { - const packageName = (task.options["package-name"] as string) || shortName; - - const result = await compileSpec(task, ctx); - completed++; - - if (!result.success) { - failed++; - failedSpecs.push(`${packageName}: ${result.error}`); - groupSuccess = false; - } - - updateProgress(); - } - - results.set(group.spec, groupSuccess); - }; - - const p = runGroup().finally(() => executing.delete(p)); - executing.add(p); - - if (executing.size >= maxJobs) { - await Promise.race(executing); - } - } - - await Promise.all(executing); - - if (isTTY) { - process.stdout.write("\r" + " ".repeat(60) + "\r"); - } - - if (failedSpecs.length > 0) { - console.log(pc.red(`\nFailed specs:`)); - for (const spec of failedSpecs) { - console.log(pc.red(` • ${spec}`)); - } - } - - return results; -} - -/** - * Pre-create the marker files that the test harness expects to find before - * regeneration so it can verify they're cleared/preserved correctly. - */ -export async function preprocess(flavor: string, generatedFolder: string): Promise { - if (flavor !== "azure") return; - - const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); - - const DELETE_CONTENT = "# This file is to be deleted after regeneration"; - const KEEP_CONTENT = "# This file is to be kept after regeneration"; - const DELETE_FILE = "to_be_deleted.py"; - const entries: { folder: string[]; file: string; content: string }[] = [ - { - folder: ["authentication-api-key", "authentication", "apikey", "_operations"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir", "_generated"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generated_tests"], - file: DELETE_FILE, - content: DELETE_CONTENT, - }, - { - folder: ["generation-subdir", "generation", "subdir"], - file: "to_be_kept.py", - content: KEEP_CONTENT, - }, - ]; - - await Promise.all( - entries.map(async ({ folder, file, content }) => { - const targetFolder = join(testsGeneratedDir, ...folder); - await mkdir(targetFolder, { recursive: true }); - await writeFile(join(targetFolder, file), content); - }), - ); -} - -/** - * Resets the `tests/generated/{azure,unbranded}` baseline by sparse-checking-out - * `eng/tools/emitter/gen` from the Azure/azure-sdk-for-python repo, then - * deleting a couple of fully-generated package folders so regeneration has to - * recreate them from scratch (smoke test of full-emit path). - * - * `generatedFolder` is the per-repo `generator/` directory; baseline lands at - * `/../tests/generated`. - */ -export async function prepareBaselineOfGeneratedCode(generatedFolder: string): Promise { - const repoUrl = "https://github.com/Azure/azure-sdk-for-python.git"; - const branch = "main"; - const sourceSubdir = "eng/tools/emitter/gen"; - const testsGeneratedDir = resolve(generatedFolder, "../tests/generated"); - - console.log(pc.cyan(`\n${"=".repeat(60)}`)); - console.log(pc.cyan(`Resetting baseline from ${repoUrl} (${branch}/${sourceSubdir})`)); - console.log(pc.cyan(`${"=".repeat(60)}\n`)); - - // Wipe tests/generated - if (existsSync(testsGeneratedDir)) { - console.log(pc.dim(`Removing ${testsGeneratedDir}`)); - rmSync(testsGeneratedDir, { recursive: true, force: true }); - } - - // Sparse-checkout the baseline folder into a temp directory - const tempDir = await mkdtemp(join(tmpdir(), "azsdk-baseline-")); - try { - console.log(pc.dim(`Cloning into ${tempDir}`)); - const run = (cmd: string) => - execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); - - run(`git init`); - run(`git remote add origin ${repoUrl}`); - run(`git config core.sparseCheckout true`); - run(`git sparse-checkout init --cone`); - run(`git sparse-checkout set ${sourceSubdir}`); - run(`git fetch --depth 1 origin ${branch}`); - run(`git checkout FETCH_HEAD`); - - const sourceRoot = join(tempDir, ...sourceSubdir.split("/")); - for (const flavor of ["azure", "unbranded"]) { - const src = join(sourceRoot, flavor); - const dest = join(testsGeneratedDir, flavor); - if (!existsSync(src)) { - console.warn(pc.yellow(`Baseline folder not found: ${src}`)); - continue; - } - console.log(pc.dim(`Copying ${flavor}/ -> ${dest}`)); - await cp(src, dest, { recursive: true }); - } - - console.log(pc.green(`Baseline reset complete.\n`)); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - } - - // Delete a couple of fully-generated package folders so regeneration has to - // recreate them from scratch (smoke test of full-emit path). - const targetsToDelete = [ - join(testsGeneratedDir, "azure", "authentication-http-custom"), - join(testsGeneratedDir, "unbranded", "encode-array"), - ]; - for (const target of targetsToDelete) { - if (existsSync(target)) { - console.log(pc.dim(`Deleting ${target}`)); - rmSync(target, { recursive: true, force: true }); - } - } -} diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index e6c8cee1110..d857e020d71 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -6,29 +6,709 @@ * 1. TypeSpec compile (in-process, parallel) -> emits per-spec YAML only. * 2. Single batched Python subprocess reads all YAMLs and writes the * final `.py` files. Amortizes Python-startup cost across many specs. - * - * Shared helpers/data live in `regenerate-common.ts` (kept identical with the - * `@azure-tools/typespec-python` wrapper copy). */ +import { compile, NodeHost } from "@typespec/compiler"; import { execSync } from "child_process"; -import { existsSync } from "fs"; -import { access, readdir } from "fs/promises"; -import { platform } from "os"; -import { dirname, join, resolve } from "path"; +import { existsSync, rmSync } from "fs"; +import { access, cp, mkdir, mkdtemp, readdir, writeFile } from "fs/promises"; +import { platform, tmpdir } from "os"; +import { dirname, join, relative, resolve } from "path"; import pc from "picocolors"; import { fileURLToPath } from "url"; import { parseArgs } from "util"; -import { - buildTaskGroups, - getSubdirectories, - prepareBaselineOfGeneratedCode, - preprocess, - RegenerateContext, - RegenerateFlags, - runParallel, -} from "./regenerate-common.js"; +// ---- Types ---- + +interface RegenerateFlags { + flavor: string; + debug: boolean; + name?: string; +} + +interface CompileTask { + spec: string; + outputDir: string; + options: Record; +} + +interface TaskGroup { + spec: string; + tasks: CompileTask[]; +} + +interface RegenerateContext { + pluginDir: string; + azureHttpSpecs: string; + httpSpecs: string; + generatedFolder: string; + emitterName: string; +} + +interface BuildTaskGroupsOptions { + emitYamlOnly?: boolean; +} + +// ---- Constants ---- + +const SKIP_SPECS: string[] = [ + "type/file", + "service/multiple-services", + "azure/client-generator-core/response-as-bool", +]; + +const SpecialFlags: Record> = { + azure: { + "generate-test": true, + "generate-sample": true, + }, +}; + +// ---- Spec-specific emitter option overrides ---- + +const AZURE_EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "azure/client-generator-core/access": { + namespace: "specs.azure.clientgenerator.core.access", + }, + "azure/client-generator-core/alternate-type": { + namespace: "specs.azure.clientgenerator.core.alternatetype", + }, + "azure/client-generator-core/api-version": { + namespace: "specs.azure.clientgenerator.core.apiversion", + }, + "azure/client-generator-core/client-initialization/default": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.default", + }, + "azure/client-generator-core/client-initialization/individually": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", + }, + "azure/client-generator-core/client-initialization/individuallyParent": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", + }, + "azure/client-generator-core/client-location": { + namespace: "specs.azure.clientgenerator.core.clientlocation", + }, + "azure/client-generator-core/deserialize-empty-string-as-null": { + namespace: "specs.azure.clientgenerator.core.emptystring", + }, + "azure/client-generator-core/flatten-property": { + namespace: "specs.azure.clientgenerator.core.flattenproperty", + }, + "azure/client-generator-core/usage": { + namespace: "specs.azure.clientgenerator.core.usage", + }, + "azure/client-generator-core/client-doc": { + namespace: "specs.azure.clientgenerator.core.clientdoc", + }, + "azure/client-generator-core/override": { + namespace: "specs.azure.clientgenerator.core.override", + }, + "azure/client-generator-core/hierarchy-building": { + namespace: "specs.azure.clientgenerator.core.hierarchybuilding", + }, + "azure/core/basic": { + namespace: "specs.azure.core.basic", + }, + "azure/core/lro/rpc": { + namespace: "specs.azure.core.lro.rpc", + }, + "azure/core/lro/standard": { + namespace: "specs.azure.core.lro.standard", + }, + "azure/core/model": { + namespace: "specs.azure.core.model", + }, + "azure/core/page": { + namespace: "specs.azure.core.page", + }, + "azure/core/scalar": { + namespace: "specs.azure.core.scalar", + }, + "azure/core/traits": { + namespace: "specs.azure.core.traits", + }, + "azure/encode/duration": { + namespace: "specs.azure.encode.duration", + }, + "azure/example/basic": { + namespace: "specs.azure.example.basic", + }, + "azure/payload/pageable": { + namespace: "specs.azure.payload.pageable", + }, + "azure/versioning/previewVersion": { + namespace: "specs.azure.versioning.previewversion", + }, + "client/structure/default": { + namespace: "client.structure.service", + }, + "client/structure/multi-client": { + "package-name": "client-structure-multiclient", + namespace: "client.structure.multiclient", + }, + "client/structure/renamed-operation": { + "package-name": "client-structure-renamedoperation", + namespace: "client.structure.renamedoperation", + }, + "client/structure/two-operation-group": { + "package-name": "client-structure-twooperationgroup", + namespace: "client.structure.twooperationgroup", + }, + "client/naming": { + namespace: "client.naming.main", + }, + "client/overload": { + namespace: "client.overload", + }, + "encode/duration": { + namespace: "encode.duration", + }, + "encode/numeric": { + namespace: "encode.numeric", + }, + "parameters/basic": { + namespace: "parameters.basic", + }, + "parameters/spread": { + namespace: "parameters.spread", + }, + "payload/content-negotiation": { + namespace: "payload.contentnegotiation", + }, + "payload/multipart": { + namespace: "payload.multipart", + }, + "serialization/encoded-name/json": { + namespace: "serialization.encodedname.json", + }, + "special-words": { + namespace: "specialwords", + }, + "service/multi-service": { + namespace: "service.multiservice", + }, + "client/structure/client-operation-group": { + "package-name": "client-structure-clientoperationgroup", + namespace: "client.structure.clientoperationgroup", + }, +}; + +const EMITTER_OPTIONS: Record | Record[]> = { + "resiliency/srv-driven/old.tsp": { + "package-name": "resiliency-srv-driven1", + namespace: "resiliency.srv.driven1", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven1", + }, + "resiliency/srv-driven": { + "package-name": "resiliency-srv-driven2", + namespace: "resiliency.srv.driven2", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven2", + }, + "authentication/api-key": { + "clear-output-folder": "true", + }, + "authentication/http/custom": { + "package-name": "authentication-http-custom", + namespace: "authentication.http.custom", + "package-pprint-name": "Authentication Http Custom", + }, + "authentication/union": [ + { + "package-name": "authentication-union", + namespace: "authentication.union", + }, + { + "package-name": "setuppy-authentication-union", + namespace: "setuppy.authentication.union", + "keep-setup-py": "true", + }, + ], + "type/array": { + "package-name": "typetest-array", + namespace: "typetest.array", + }, + "type/dictionary": { + "package-name": "typetest-dictionary", + namespace: "typetest.dictionary", + }, + "type/enum/extensible": { + "package-name": "typetest-enum-extensible", + namespace: "typetest.enum.extensible", + }, + "type/enum/fixed": { + "package-name": "typetest-enum-fixed", + namespace: "typetest.enum.fixed", + }, + "type/model/empty": { + "package-name": "typetest-model-empty", + namespace: "typetest.model.empty", + }, + "type/model/inheritance/enum-discriminator": { + "package-name": "typetest-model-enumdiscriminator", + namespace: "typetest.model.enumdiscriminator", + }, + "type/model/inheritance/nested-discriminator": { + "package-name": "typetest-model-nesteddiscriminator", + namespace: "typetest.model.nesteddiscriminator", + }, + "type/model/inheritance/not-discriminated": [ + { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + { + "package-name": "typetest-model-notdiscriminated-typeddict", + namespace: "typetest.model.notdiscriminated.typeddict", + }, + ], + "type/model/inheritance/single-discriminator": [ + { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + { + "package-name": "typetest-model-singlediscriminator-typeddict", + namespace: "typetest.model.singlediscriminator.typeddict", + }, + ], + "type/model/inheritance/recursive": [ + { + "package-name": "typetest-model-recursive", + namespace: "typetest.model.recursive", + }, + { + "package-name": "generation-subdir", + namespace: "generation.subdir", + "generation-subdir": "_generated", + "generate-test": "false", + "clear-output-folder": "true", + }, + ], + "type/model/usage": [ + { + "package-name": "typetest-model-usage", + namespace: "typetest.model.usage", + }, + { + "package-name": "typetest-model-usage-typeddictonly", + namespace: "typetest.model.usage.typeddictonly", + "typed-dict-only-models": "all", + }, + ], + "type/model/visibility": [ + { + "package-name": "typetest-model-visibility", + namespace: "typetest.model.visibility", + }, + { + "package-name": "headasbooleantrue", + namespace: "headasbooleantrue", + "head-as-boolean": "true", + }, + { + "package-name": "headasbooleanfalse", + namespace: "headasbooleanfalse", + "head-as-boolean": "false", + }, + ], + "type/property/nullable": { + "package-name": "typetest-property-nullable", + namespace: "typetest.property.nullable", + }, + "type/property/optionality": { + "package-name": "typetest-property-optional", + namespace: "typetest.property.optional", + }, + "type/property/additional-properties": { + "package-name": "typetest-property-additionalproperties", + namespace: "typetest.property.additionalproperties", + }, + "type/scalar": { + "package-name": "typetest-scalar", + namespace: "typetest.scalar", + }, + "type/property/value-types": { + "package-name": "typetest-property-valuetypes", + namespace: "typetest.property.valuetypes", + }, + "type/union": { + "package-name": "typetest-union", + namespace: "typetest.union", + }, + "type/union/discriminated": { + "package-name": "typetest-discriminatedunion", + namespace: "typetest.discriminatedunion", + }, + "type/file": { + "package-name": "typetest-file", + namespace: "typetest.file", + }, + documentation: { + "package-name": "specs-documentation", + namespace: "specs.documentation", + }, + "versioning/added": [ + { + "package-name": "versioning-added", + namespace: "versioning.added", + }, + { + "package-name": "generation-subdir2", + namespace: "generation.subdir2", + "generate-test": "false", + "generation-subdir": "_generated", + }, + ], +}; + +// ---- Helpers ---- + +function toPosix(p: string): string { + return p.replace(/\\/g, "/"); +} + +function isAzureSpec(spec: string): boolean { + return spec.includes("azure-http-specs"); +} + +function defaultPackageName(spec: string, ctx: RegenerateContext): string { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + return toPosix(relative(specDir, dirname(spec))) + .replace(/\//g, "-") + .toLowerCase(); +} + +function getEmitterOptions( + spec: string, + flavor: string, + ctx: RegenerateContext, +): Record[] { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const relativeSpec = toPosix(relative(specDir, spec)); + const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") + ? relativeSpec + : dirname(relativeSpec); + const emitterOpts = EMITTER_OPTIONS[key] || + (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; + return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; +} + +async function getSubdirectories( + baseDir: string, + flags: RegenerateFlags, +): Promise { + const subdirectories: string[] = []; + + async function searchDir(currentDir: string) { + const items = await readdir(currentDir, { withFileTypes: true }); + + const promisesArray = items.map(async (item) => { + const subDirPath = join(currentDir, item.name); + if (item.isDirectory()) { + const mainTspPath = join(subDirPath, "main.tsp"); + const clientTspPath = join(subDirPath, "client.tsp"); + + const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); + + if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; + + const hasMainTsp = await access(mainTspPath) + .then(() => true) + .catch(() => false); + const hasClientTsp = await access(clientTspPath) + .then(() => true) + .catch(() => false); + + if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { + if (mainTspRelativePath.includes("resiliency/srv-driven")) { + subdirectories.push(resolve(subDirPath, "old.tsp")); + } + if (hasClientTsp) { + subdirectories.push(resolve(subDirPath, "client.tsp")); + } else if (hasMainTsp) { + subdirectories.push(resolve(subDirPath, "main.tsp")); + } + } + + await searchDir(subDirPath); + } + }); + + await Promise.all(promisesArray); + } + + await searchDir(baseDir); + return subdirectories; +} + +function buildTaskGroups( + specs: string[], + flags: RegenerateFlags, + ctx: RegenerateContext, + options: BuildTaskGroupsOptions = {}, +): TaskGroup[] { + const groups: TaskGroup[] = []; + + for (const spec of specs) { + const tasks: CompileTask[] = []; + + for (const emitterConfig of getEmitterOptions(spec, flags.flavor, ctx)) { + const opts: Record = {}; + for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { + opts[k] = v; + } + Object.assign(opts, emitterConfig); + + opts["flavor"] = flags.flavor; + + const packageName = (opts["package-name"] as string) || defaultPackageName(spec, ctx); + const outputDir = + (opts["emitter-output-dir"] as string) || + toPosix(`${ctx.generatedFolder}/../tests/generated/${flags.flavor}/${packageName}`); + opts["emitter-output-dir"] = outputDir; + + if (flags.debug) { + opts["debug"] = true; + } + + opts["examples-dir"] = toPosix(join(dirname(spec), "examples")); + + if (options.emitYamlOnly) { + opts["emit-yaml-only"] = true; + } + + tasks.push({ spec, outputDir, options: opts }); + } + + groups.push({ spec, tasks }); + } + + return groups; +} + +async function compileSpec( + task: CompileTask, + ctx: RegenerateContext, +): Promise<{ success: boolean; error?: string }> { + const { spec, outputDir, options } = task; + + try { + const compilerOptions = { + emit: [ctx.pluginDir], + options: { + [ctx.emitterName]: options, + }, + }; + + const program = await compile(NodeHost, spec, compilerOptions); + + if (program.hasError()) { + const errors = program.diagnostics + .filter((d) => d.severity === "error") + .map((d) => d.message) + .join("\n"); + return { success: false, error: errors }; + } + + return { success: true }; + } catch (err) { + rmSync(outputDir, { recursive: true, force: true }); + return { success: false, error: String(err) }; + } +} + +function renderProgressBar( + completed: number, + failed: number, + total: number, + width: number = 40, +): string { + const successCount = completed - failed; + const successWidth = Math.round((successCount / total) * width); + const failWidth = Math.round((failed / total) * width); + const emptyWidth = width - successWidth - failWidth; + + const successBar = pc.bgGreen(" ".repeat(successWidth)); + const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; + const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); + + const percent = Math.round((completed / total) * 100); + return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; +} + +async function runParallel( + groups: TaskGroup[], + maxJobs: number, + ctx: RegenerateContext, +): Promise> { + const results = new Map(); + const executing: Set> = new Set(); + + const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); + let completed = 0; + let failed = 0; + const failedSpecs: string[] = []; + + const isTTY = process.stdout.isTTY; + + const updateProgress = () => { + if (isTTY) { + process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); + } + }; + + updateProgress(); + + for (const group of groups) { + const runGroup = async () => { + const specDir = isAzureSpec(group.spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const shortName = toPosix(relative(specDir, dirname(group.spec))); + + let groupSuccess = true; + for (const task of group.tasks) { + const packageName = (task.options["package-name"] as string) || shortName; + + const result = await compileSpec(task, ctx); + completed++; + + if (!result.success) { + failed++; + failedSpecs.push(`${packageName}: ${result.error}`); + groupSuccess = false; + } + + updateProgress(); + } + + results.set(group.spec, groupSuccess); + }; + + const p = runGroup().finally(() => executing.delete(p)); + executing.add(p); + + if (executing.size >= maxJobs) { + await Promise.race(executing); + } + } + + await Promise.all(executing); + + if (isTTY) { + process.stdout.write("\r" + " ".repeat(60) + "\r"); + } + + if (failedSpecs.length > 0) { + console.log(pc.red(`\nFailed specs:`)); + for (const spec of failedSpecs) { + console.log(pc.red(` • ${spec}`)); + } + } + + return results; +} + +async function preprocess(flavor: string, generatedFolder: string): Promise { + if (flavor !== "azure") return; + + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); + + const DELETE_CONTENT = "# This file is to be deleted after regeneration"; + const KEEP_CONTENT = "# This file is to be kept after regeneration"; + const DELETE_FILE = "to_be_deleted.py"; + const entries: { folder: string[]; file: string; content: string }[] = [ + { + folder: ["authentication-api-key", "authentication", "apikey", "_operations"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir", "_generated"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generated_tests"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir"], + file: "to_be_kept.py", + content: KEEP_CONTENT, + }, + ]; + + await Promise.all( + entries.map(async ({ folder, file, content }) => { + const targetFolder = join(testsGeneratedDir, ...folder); + await mkdir(targetFolder, { recursive: true }); + await writeFile(join(targetFolder, file), content); + }), + ); +} + +async function prepareBaselineOfGeneratedCode(generatedFolder: string): Promise { + const repoUrl = "https://github.com/Azure/azure-sdk-for-python.git"; + const branch = "main"; + const sourceSubdir = "eng/tools/emitter/gen"; + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated"); + + console.log(pc.cyan(`\n${"=".repeat(60)}`)); + console.log(pc.cyan(`Resetting baseline from ${repoUrl} (${branch}/${sourceSubdir})`)); + console.log(pc.cyan(`${"=".repeat(60)}\n`)); + + if (existsSync(testsGeneratedDir)) { + console.log(pc.dim(`Removing ${testsGeneratedDir}`)); + rmSync(testsGeneratedDir, { recursive: true, force: true }); + } + + const tempDir = await mkdtemp(join(tmpdir(), "azsdk-baseline-")); + try { + console.log(pc.dim(`Cloning into ${tempDir}`)); + const run = (cmd: string) => + execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); + + run(`git init`); + run(`git remote add origin ${repoUrl}`); + run(`git config core.sparseCheckout true`); + run(`git sparse-checkout init --cone`); + run(`git sparse-checkout set ${sourceSubdir}`); + run(`git fetch --depth 1 origin ${branch}`); + run(`git checkout FETCH_HEAD`); + + const sourceRoot = join(tempDir, ...sourceSubdir.split("/")); + for (const flavor of ["azure", "unbranded"]) { + const src = join(sourceRoot, flavor); + const dest = join(testsGeneratedDir, flavor); + if (!existsSync(src)) { + console.warn(pc.yellow(`Baseline folder not found: ${src}`)); + continue; + } + console.log(pc.dim(`Copying ${flavor}/ -> ${dest}`)); + await cp(src, dest, { recursive: true }); + } + + console.log(pc.green(`Baseline reset complete.\n`)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + + const targetsToDelete = [ + join(testsGeneratedDir, "azure", "authentication-http-custom"), + join(testsGeneratedDir, "unbranded", "encode-array"), + ]; + for (const target of targetsToDelete) { + if (existsSync(target)) { + console.log(pc.dim(`Deleting ${target}`)); + rmSync(target, { recursive: true, force: true }); + } + } +} // Parse arguments const argv = parseArgs({ diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 6913e7656af..e16bd89fa9b 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -77,9 +77,10 @@ def __init__( self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) + typed_dict_only_opt = code_model.options.get("typed-dict-only-models", []) self.is_typed_dict_only: bool = self.yaml_data.get( "typedDictOnly", False - ) or self.name in code_model.options.get("typed-dict-only-models", []) + ) or typed_dict_only_opt == "all" or self.name in typed_dict_only_opt @property def is_usage_output(self) -> bool: From 1dd83148b10f5ec01cbc93cee1bdd316345decec Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 12 May 2026 12:55:25 -0700 Subject: [PATCH 29/42] feat: add client-naming-typeddict variant package Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../http-client-python/eng/scripts/ci/regenerate.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index d857e020d71..c22f0ede358 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -157,9 +157,15 @@ const AZURE_EMITTER_OPTIONS: Record< "package-name": "client-structure-twooperationgroup", namespace: "client.structure.twooperationgroup", }, - "client/naming": { - namespace: "client.naming.main", - }, + "client/naming": [ + { + namespace: "client.naming.main", + }, + { + "package-name": "client-naming-typeddict", + namespace: "client.naming.typeddict", + }, + ], "client/overload": { namespace: "client.overload", }, From 48d2aed016ac64f927364874527b06741ffa0534 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 12 May 2026 13:07:38 -0700 Subject: [PATCH 30/42] refactor: remove unnecessary is_response pops from TypedDictModelType TypedDicts don't differentiate input/output. The parent classes don't use is_response either, so the pop overrides were just noise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index e16bd89fa9b..7a0b01c1094 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -440,19 +440,3 @@ def imports(self, **kwargs: Any) -> FileImport: class TypedDictModelType(DPGModelType): base = "typeddict" - - def type_annotation(self, **kwargs: Any) -> str: - kwargs.pop("is_response", None) - return super().type_annotation(**kwargs) - - def docstring_type(self, **kwargs: Any) -> str: - kwargs.pop("is_response", None) - return super().docstring_type(**kwargs) - - def docstring_text(self, **kwargs: Any) -> str: - kwargs.pop("is_response", None) - return super().docstring_text(**kwargs) - - def imports(self, **kwargs: Any) -> FileImport: - kwargs.pop("is_response", None) - return super().imports(**kwargs) From a130bfcdf0b3bdfe92a130398bf379abab8d9ede Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 12 May 2026 15:20:56 -0700 Subject: [PATCH 31/42] fix: use TypedDict dicts instead of model classes in naming test The typeddict test should pass plain dicts with wire names, not model class constructors which expect client names. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../eng/scripts/ci/regenerate.ts | 10 ++------ .../pygen/codegen/models/model_type.py | 8 ++++--- .../codegen/serializers/types_serializer.py | 23 +++++++++++++++---- .../azure/test_client_naming_typeddict.py | 12 ++++------ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index c22f0ede358..b21872ee64d 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -66,10 +66,7 @@ const SpecialFlags: Record> = { // ---- Spec-specific emitter option overrides ---- -const AZURE_EMITTER_OPTIONS: Record< - string, - Record | Record[] -> = { +const AZURE_EMITTER_OPTIONS: Record | Record[]> = { "azure/client-generator-core/access": { namespace: "specs.azure.clientgenerator.core.access", }, @@ -404,10 +401,7 @@ function getEmitterOptions( return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; } -async function getSubdirectories( - baseDir: string, - flags: RegenerateFlags, -): Promise { +async function getSubdirectories(baseDir: string, flags: RegenerateFlags): Promise { const subdirectories: string[] = []; async function searchDir(currentDir: string) { diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 7a0b01c1094..a0ae3309df1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -78,9 +78,11 @@ def __init__( self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) typed_dict_only_opt = code_model.options.get("typed-dict-only-models", []) - self.is_typed_dict_only: bool = self.yaml_data.get( - "typedDictOnly", False - ) or typed_dict_only_opt == "all" or self.name in typed_dict_only_opt + self.is_typed_dict_only: bool = ( + self.yaml_data.get("typedDictOnly", False) + or typed_dict_only_opt == "all" + or self.name in typed_dict_only_opt + ) @property def is_usage_output(self) -> bool: diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 251fc47fde1..e3ec5875b39 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -17,10 +17,25 @@ # Python builtin type names that can be shadowed by TypedDict field wire_names. # When a field name matches one of these, all references to that builtin in type # annotations within the same class are qualified as builtins.X. -_BUILTIN_TYPE_NAMES = frozenset({ - "int", "str", "float", "bool", "list", "dict", "tuple", "set", - "bytes", "type", "object", "complex", "frozenset", "bytearray", "memoryview", -}) +_BUILTIN_TYPE_NAMES = frozenset( + { + "int", + "str", + "float", + "bool", + "list", + "dict", + "tuple", + "set", + "bytes", + "type", + "object", + "complex", + "frozenset", + "bytearray", + "memoryview", + } +) def _qualify_shadowed_builtins(annotation: str, shadowed: frozenset[str]) -> str: diff --git a/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py b/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py index 40786f16e39..cf9cf30401b 100644 --- a/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py +++ b/packages/http-client-python/tests/mock_api/azure/test_client_naming_typeddict.py @@ -15,19 +15,17 @@ def client(): def test_client(client: NamingClient): """TypedDict uses wire name 'defaultName', not client name 'client_name'.""" - client.property.client(models.ClientNameModel(defaultName=True)) + client.property.client({"defaultName": True}) def test_language(client: NamingClient): """TypedDict uses wire name 'defaultName', not language-specific name 'python_name'.""" - client.property.language(models.LanguageClientNameModel(defaultName=True)) + client.property.language({"defaultName": True}) def test_compatible_with_encoded_name(client: NamingClient): """TypedDict uses encoded wire name 'wireName', not client name 'client_name'.""" - client.property.compatible_with_encoded_name( - models.ClientNameAndJsonEncodedNameModel(wireName=True) - ) + client.property.compatible_with_encoded_name({"wireName": True}) def test_operation(client: NamingClient): @@ -48,12 +46,12 @@ def test_header_response(client: NamingClient): def test_model_client(client: NamingClient): """TypedDict uses wire name 'defaultName', not client name 'default_name'.""" - client.model_client.client(models.ClientModel(defaultName=True)) + client.model_client.client({"defaultName": True}) def test_model_language(client: NamingClient): """TypedDict uses wire name 'defaultName', not client name 'default_name'.""" - client.model_client.language(models.PythonModel(defaultName=True)) + client.model_client.language({"defaultName": True}) def test_union_enum_member_name(client: NamingClient): From 6a1577470fa324e610af7823538532930c159a40 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 13 May 2026 11:58:04 -0700 Subject: [PATCH 32/42] fix: skip self-import in types.py and unused _deserialize for typed-dict-only models - Skip importing types module from itself when model is in same namespace - Skip _deserialize import when response type is typed-dict-only (uses response.json() instead) - Fix paging operation _deserialize import for typed-dict-only item types Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 3 +++ .../generator/pygen/codegen/models/operation.py | 1 + .../generator/pygen/codegen/models/paging_operation.py | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index a0ae3309df1..6fee3ed0af1 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -418,12 +418,15 @@ def imports(self, **kwargs: Any) -> FileImport: serialize_namespace_type = kwargs.get("serialize_namespace_type") serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + same_namespace = relative_path == "." if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: file_import.add_submodule_import( relative_path, "types", ImportType.LOCAL, ) + elif serialize_namespace_type == NamespaceType.TYPES_FILE and same_namespace: + pass # model is defined in this types.py — no import needed elif serialize_namespace_type in [NamespaceType.TYPES_FILE, NamespaceType.UNIONS_FILE] or ( serialize_namespace_type == NamespaceType.MODEL and kwargs.get("called_by_property", False) ): diff --git a/packages/http-client-python/generator/pygen/codegen/models/operation.py b/packages/http-client-python/generator/pygen/codegen/models/operation.py index c5f15593893..bd745c5e6e7 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/operation.py @@ -453,6 +453,7 @@ def imports( # pylint: disable=too-many-branches, disable=too-many-statements r.type and not isinstance(r.type, BinaryIteratorType) and not xml_serializable(str(r.default_content_type)) + and not (isinstance(r.type, ModelType) and r.type.is_typed_dict_only) for r in self.responses ): file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) diff --git a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py index f64363ed5fb..91f2d39ec05 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py +++ b/packages/http-client-python/generator/pygen/codegen/models/paging_operation.py @@ -186,7 +186,11 @@ def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: serialize_namespace, module_name="_utils.model_base" ) file_import.merge(self.item_type.imports(**kwargs)) - if self.default_error_deserialization(serialize_namespace) or self.need_deserialize: + need_deserialize_import = self.default_error_deserialization(serialize_namespace) or ( + self.need_deserialize + and not (isinstance(self.item_type, ModelType) and self.item_type.is_typed_dict_only) + ) + if need_deserialize_import: file_import.add_submodule_import(relative_path, "_deserialize", ImportType.LOCAL) if self.is_xml_paging: file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") From 3789713a3cae80089800280aa10a17bfe201b941 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 13 May 2026 13:59:06 -0700 Subject: [PATCH 33/42] fix: skip _models prefix for internal models in types.py annotations Internal models in types.py should use bare names like all other models, since they're imported by name under TYPE_CHECKING. Also fix import path for internal models to use .models._models instead of .models. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../generator/pygen/codegen/models/model_type.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 6fee3ed0af1..072e3328104 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -285,12 +285,16 @@ def type_annotation(self, **kwargs: Any) -> str: module_name = "" # In types.py, use bare name to avoid pyright "variable in type expression" errors if serialize_namespace_type == NamespaceType.TYPES_FILE: - pass # no module prefix + pass # no module prefix, no internal file prefix elif kwargs.get("need_model_alias", True): serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) model_alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) module_name = f"{model_alias}." - file_name = f"{self.code_model.models_filename}." if self.internal else "" + file_name = ( + f"{self.code_model.models_filename}." + if self.internal and serialize_namespace_type != NamespaceType.TYPES_FILE + else "" + ) retval = module_name + file_name + self.name return retval if is_operation_file or skip_quote else f'"{retval}"' @@ -338,9 +342,10 @@ def imports(self, **kwargs: Any) -> FileImport: will_be_in_types_file = self.base != "json" and same_namespace if not will_be_in_types_file: if same_namespace: - # json models from same namespace — import from .models + # json models from same namespace — import from .models (or .models._models for internal) + import_path = f".models.{self.code_model.models_filename}" if self.internal else ".models" file_import.add_submodule_import( - ".models", + import_path, self.name, ImportType.LOCAL, typing_section=TypingSection.TYPING, From 6c0ee440f604c72c35e80594a986ebf4fd8023e5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 26 May 2026 15:08:41 -0400 Subject: [PATCH 34/42] switch to models-mode=typeddict to represent td only, generate union of literals instead of enums when td only --- .../eng/scripts/ci/regenerate.ts | 2 +- .../generator/pygen/__init__.py | 2 - .../pygen/codegen/models/enum_type.py | 105 ++++++++++++------ .../pygen/codegen/models/model_type.py | 5 +- .../pygen/codegen/serializers/__init__.py | 9 +- .../codegen/serializers/types_serializer.py | 18 +++ .../pygen/codegen/templates/types.py.jinja2 | 7 ++ .../tests/unit/test_typeddict.py | 13 ++- 8 files changed, 117 insertions(+), 44 deletions(-) diff --git a/packages/http-client-python/eng/scripts/ci/regenerate.ts b/packages/http-client-python/eng/scripts/ci/regenerate.ts index fd7f7541194..748e957e328 100644 --- a/packages/http-client-python/eng/scripts/ci/regenerate.ts +++ b/packages/http-client-python/eng/scripts/ci/regenerate.ts @@ -300,7 +300,7 @@ const EMITTER_OPTIONS: Record | Record Any: raise ValueError( f"--package-mode can only be {' or '.join(TYPESPEC_PACKAGE_MODE)} or directory which contains template files" # pylint: disable=line-too-long ) - if key == "typed-dict-only-models" and isinstance(value, str): - value = [v.strip() for v in value.split(",") if v.strip()] return value def setdefault(self, key: str, default: Any, /) -> Any: # type: ignore # pylint: disable=arguments-differ diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 110d18f15df..4c799c4b1df 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -167,6 +167,10 @@ def description(self, *, is_operation_file: bool) -> str: enum_description = f"Known values are: {possible_values_str}." return enum_description + @property + def is_typeddict_mode(self) -> bool: + return self.code_model.options["models-mode"] == "typeddict" + def type_annotation(self, **kwargs: Any) -> str: """The python type used for type annotation @@ -174,6 +178,14 @@ def type_annotation(self, **kwargs: Any) -> str: :rtype: str """ if self.code_model.options["models-mode"]: + if self.is_typeddict_mode: + # In typeddict mode, enums are Literal aliases defined in types.py + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type == NamespaceType.TYPES_FILE: + # Same file — just the name, no module prefix + return self.name + # From operation/client files, use types.EnumName (matching model pattern) + return f"types.{self.name}" module_name = "" serialize_namespace_type = kwargs.get("serialize_namespace_type") @@ -243,39 +255,68 @@ def imports(self, **kwargs: Any) -> FileImport: file_import = FileImport(self.code_model) file_import.merge(self.value_type.imports(**kwargs)) if self.code_model.options["models-mode"]: - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.REGULAR) + if self.is_typeddict_mode: + # In typeddict mode, enums are Literal aliases in types.py — no Union needed + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type == NamespaceType.TYPES_FILE: + # Same file — no import needed for same-namespace enums + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + if self.client_namespace != serialize_namespace: + # Cross-namespace: import from sibling types module + relative_path = self.code_model.get_relative_import_path( + serialize_namespace, self.client_namespace + ) + file_import.add_submodule_import( + f"{relative_path}types" if relative_path != "." else ".types", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.REGULAR, + ) + elif serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + # Import types module directly (matching model pattern) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path( + serialize_namespace, self.client_namespace + ) + file_import.add_submodule_import( + relative_path, + "types", + ImportType.LOCAL, + ) + else: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.REGULAR) - serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) - alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) - serialize_namespace_type = kwargs.get("serialize_namespace_type") - called_by_property = kwargs.get("called_by_property", False) - if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias=alias, - typing_section=TypingSection.REGULAR, - ) - elif serialize_namespace_type == NamespaceType.TYPES_FILE: - # Import enum name directly to avoid dotted forward refs in TypedDict annotations - file_import.add_submodule_import( - f"{relative_path}models" if relative_path != "." else ".models", - self.name, - ImportType.LOCAL, - typing_section=TypingSection.TYPING, - ) - elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( - serialize_namespace_type == NamespaceType.MODEL and called_by_property - ): - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias=alias, - typing_section=TypingSection.TYPING, - ) + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + alias = self.code_model.get_unique_models_alias(serialize_namespace, self.client_namespace) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + called_by_property = kwargs.get("called_by_property", False) + if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.REGULAR, + ) + elif serialize_namespace_type == NamespaceType.TYPES_FILE: + # Import enum name directly to avoid dotted forward refs in TypedDict annotations + file_import.add_submodule_import( + f"{relative_path}models" if relative_path != "." else ".models", + self.name, + ImportType.LOCAL, + typing_section=TypingSection.TYPING, + ) + elif serialize_namespace_type == NamespaceType.UNIONS_FILE or ( + serialize_namespace_type == NamespaceType.MODEL and called_by_property + ): + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias=alias, + typing_section=TypingSection.TYPING, + ) file_import.merge(self.value_type.imports(**kwargs)) return file_import diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 072e3328104..1a951ac1b38 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -77,11 +77,8 @@ def __init__( self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") self.usage: int = self.yaml_data.get("usage", UsageFlags.Input.value | UsageFlags.Output.value) self.client_namespace: str = self.yaml_data.get("clientNamespace", code_model.namespace) - typed_dict_only_opt = code_model.options.get("typed-dict-only-models", []) self.is_typed_dict_only: bool = ( - self.yaml_data.get("typedDictOnly", False) - or typed_dict_only_opt == "all" - or self.name in typed_dict_only_opt + self.yaml_data.get("typedDictOnly", False) or code_model.options["models-mode"] == "typeddict" ) @property diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 785461a68d6..956cd6051af 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -207,6 +207,7 @@ def serialize(self) -> None: self._serialize_and_write_top_level_folder(env=env, namespace=client_namespace) # add models folder if there are models in this namespace + is_typeddict_mode = self.code_model.options["models-mode"] == "typeddict" if ( self.code_model.has_non_json_models(client_namespace_type.models) or client_namespace_type.enums ) and self.code_model.options["models-mode"]: @@ -214,11 +215,14 @@ def serialize(self) -> None: env=env, namespace=client_namespace, models=client_namespace_type.models, - enums=client_namespace_type.enums, + enums=[] if is_typeddict_mode else client_namespace_type.enums, ) # write types.py per namespace (alongside models/) - if self.code_model.has_non_json_models(client_namespace_type.models): + # In typeddict mode, also generate types.py when there are enums (for Literal aliases) + has_models = self.code_model.has_non_json_models(client_namespace_type.models) + has_typeddict_enums = is_typeddict_mode and client_namespace_type.enums + if has_models or has_typeddict_enums: generation_dir = self.code_model.get_generation_dir(client_namespace) self.write_file( generation_dir / Path("types.py"), @@ -227,6 +231,7 @@ def serialize(self) -> None: env=env, client_namespace=client_namespace, models=client_namespace_type.models, + enums=client_namespace_type.enums if is_typeddict_mode else None, ).serialize(), ) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index e3ec5875b39..7cde299d5ef 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -7,6 +7,7 @@ import re from typing import Optional from ..models import ModelType, CodeModel +from ..models.enum_type import EnumType from ..models.imports import FileImport, ImportType from ..models.utils import NamespaceType from ..models.property import Property @@ -54,9 +55,21 @@ def __init__( env, client_namespace: Optional[str] = None, models: Optional[list[ModelType]] = None, + enums: Optional[list["EnumType"]] = None, ): super().__init__(code_model=code_model, env=env, client_namespace=client_namespace) self._models = models or [] + self._enums = enums or [] + + @property + def literal_enums(self) -> list[EnumType]: + """Enums to render as Literal type aliases in typeddict mode.""" + return sorted(self._enums) + + def declare_literal_enum(self, enum: EnumType) -> str: + """Generate a Literal type alias for an enum, e.g. MyColor = Literal["red", "blue"].""" + values = [enum.get_declaration(v.value) for v in enum.values] + return f"{enum.name} = Literal[{', '.join(values)}]" @property def typeddict_models(self) -> list[ModelType]: @@ -121,6 +134,10 @@ def get_shadowed_builtins(model: ModelType) -> frozenset[str]: def imports(self) -> FileImport: file_import = FileImport(self.code_model) + literal_enums = self.literal_enums + if literal_enums: + file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB) + td_models = self.typeddict_models if td_models or self.discriminated_base_models: if td_models: @@ -242,6 +259,7 @@ def serialize(self) -> str: code_model=self.code_model, imports=FileImportSerializer(self.imports()), serializer=self, + literal_enums=self.literal_enums, models=self.typeddict_models, discriminated_bases=self.discriminated_base_models, ) diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index d75c26bc6ae..1456fd1538c 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -6,6 +6,13 @@ {{ imports }} {% import 'operation_tools.jinja2' as op_tools %} {% import "macros.jinja2" as macros %} +{% for enum in literal_enums %} + +{{ serializer.declare_literal_enum(enum) }} +{% if enum.yaml_data.get("description") %} +"""{{ op_tools.wrap_string(enum.yaml_data["description"], "\n") }}""" +{% endif %} +{% endfor %} {% for model in models %} {% if serializer.has_keyword_wire_names(model) %} diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py index c35db047b0f..b6b42e262d4 100644 --- a/packages/http-client-python/tests/unit/test_typeddict.py +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -192,13 +192,19 @@ def _make_typed_dict_only_model(code_model, name, **extra_yaml): def test_typed_dict_only_property(): - """is_typed_dict_only should be True when yaml_data has typedDictOnly=True.""" + """is_typed_dict_only should be True when yaml_data has typedDictOnly=True or models-mode is typeddict.""" code_model = _make_code_model(models_mode="typeddict") model = _make_typed_dict_only_model(code_model, "Foo") assert model.is_typed_dict_only is True + # In typeddict mode, ALL models are typed-dict-only normal_model = _make_model(code_model, "Bar", model_cls=TypedDictModelType) - assert normal_model.is_typed_dict_only is False + assert normal_model.is_typed_dict_only is True + + # In dpg mode, only models with typedDictOnly=True are typed-dict-only + dpg_code_model = _make_code_model(models_mode="dpg") + dpg_normal = _make_model(dpg_code_model, "Baz", model_cls=TypedDictModelType) + assert dpg_normal.is_typed_dict_only is False def test_typed_dict_only_excluded_from_public_model_types(): @@ -209,7 +215,8 @@ def test_typed_dict_only_excluded_from_public_model_types(): code_model.model_types = [normal, td_only] public = code_model.public_model_types - assert normal in public + # In typeddict mode, all models are typed-dict-only and excluded from public model types + assert normal not in public assert td_only not in public From 6d75f07bd0e19b72eab9e9458bd5d4795363a336 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 26 May 2026 16:07:42 -0400 Subject: [PATCH 35/42] don't conflate optional and nullable --- .../http-client-python/emitter/src/emitter.ts | 5 +++-- packages/http-client-python/emitter/src/types.ts | 2 ++ .../generator/pygen/codegen/models/model_type.py | 7 ++++++- .../generator/pygen/codegen/models/property.py | 15 +++++++++++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 3ecd85c5f57..6ea057c3e60 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -198,8 +198,9 @@ async function onEmitMain(context: EmitContext) { const yamlMap = emitCodeModel(sdkContext); const parsedYamlMap = walkThroughNodes(yamlMap); - // Python emitter requires an SDK client in the TypeSpec - if (sdkContext.sdkPackage.clients.length === 0) { + // Python emitter requires an SDK client in the TypeSpec (unless models-only typeddict mode) + const modelsMode = (sdkContext.emitContext.options as any)["models-mode"]; + if (sdkContext.sdkPackage.clients.length === 0 && modelsMode !== "typeddict") { reportDiagnostic(program, { code: "no-sdk-clients", target: NoTarget, diff --git a/packages/http-client-python/emitter/src/types.ts b/packages/http-client-python/emitter/src/types.ts index 825c56b063b..f432f9360a1 100644 --- a/packages/http-client-python/emitter/src/types.ts +++ b/packages/http-client-python/emitter/src/types.ts @@ -233,6 +233,7 @@ function emitProperty( // Python convert all the type of file part to FileType so clear these models' usage so that they won't be generated addDisableGenerationMap(context, property.type); } + const isNullable = !isMultipartFileInput && sourceType.kind === "nullable"; return { clientName: getClientName(property), isExactName: property.isExactName, @@ -242,6 +243,7 @@ function emitProperty( : property.serializationOptions?.json?.name) ?? property.name, type: getType(context, sourceType), optional: property.optional, + nullable: isNullable, description: property.summary ? property.summary : property.doc, addedOn: getAddedOn(context, property), apiVersions: property.apiVersions, diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 1a951ac1b38..25e20b7cc14 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -391,7 +391,12 @@ def type_annotation(self, **kwargs: Any) -> str: if self.is_typed_dict_only: is_operation_file = kwargs.pop("is_operation_file", False) skip_quote = kwargs.get("skip_quote", False) - retval = f"types.{self.name}" + serialize_namespace_type = kwargs.get("serialize_namespace_type") + # Within types.py, use bare name (no module prefix) + if serialize_namespace_type == NamespaceType.TYPES_FILE: + retval = self.name + else: + retval = f"types.{self.name}" return retval if is_operation_file or skip_quote else f'"{retval}"' return super().type_annotation(**kwargs) diff --git a/packages/http-client-python/generator/pygen/codegen/models/property.py b/packages/http-client-python/generator/pygen/codegen/models/property.py index ddb7d5afa24..d2a544c0511 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/property.py +++ b/packages/http-client-python/generator/pygen/codegen/models/property.py @@ -29,6 +29,7 @@ def __init__( self.client_name: str = self.yaml_data["clientName"] self.type = type self.optional: bool = self.yaml_data["optional"] + self.nullable: bool = self.yaml_data.get("nullable", False) self.readonly: bool = self.yaml_data.get("readonly", False) self.visibility: list[str] = self.yaml_data.get("visibility", []) self.is_polymorphic: bool = self.yaml_data.get("isPolymorphic", False) @@ -110,6 +111,12 @@ def type_annotation(self, *, is_operation_file: bool = False, **kwargs: Any) -> if self.is_base_discriminator: return "str" types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file, **kwargs) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + # In TypedDict types.py, Optional means nullable (not "not required" — that's handled by Required/total=False) + if serialize_namespace_type == NamespaceType.TYPES_FILE: + if self.nullable: + return f"Optional[{types_type_annotation}]" + return types_type_annotation if (self.optional and self.client_default_value is None) or self.readonly: return f"Optional[{types_type_annotation}]" return types_type_annotation @@ -152,10 +159,14 @@ def imports(self, **kwargs) -> FileImport: if self.is_discriminator and isinstance(self.type, EnumType): return file_import file_import.merge(self.type.imports(**kwargs)) - if (self.optional and self.client_default_value is None) or self.readonly: + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type == NamespaceType.TYPES_FILE: + # In TypedDict types.py, Optional means nullable + if self.nullable: + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + elif (self.optional and self.client_default_value is None) or self.readonly: file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) if self.code_model.options["models-mode"] == "dpg": - serialize_namespace_type = kwargs.get("serialize_namespace_type") if serialize_namespace_type != NamespaceType.TYPES_FILE: serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) file_import.add_submodule_import( From 27113ce536002b607aa3f0a450583f4f57ea5a80 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 26 May 2026 16:14:30 -0400 Subject: [PATCH 36/42] format and lint --- .../generator/pygen/codegen/models/enum_type.py | 4 +--- .../generator/pygen/codegen/serializers/__init__.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py index 4c799c4b1df..e9fd13a5146 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/enum_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/enum_type.py @@ -275,9 +275,7 @@ def imports(self, **kwargs: Any) -> FileImport: elif serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: # Import types module directly (matching model pattern) serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) - relative_path = self.code_model.get_relative_import_path( - serialize_namespace, self.client_namespace - ) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) file_import.add_submodule_import( relative_path, "types", diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py index 956cd6051af..f52c46aa7ec 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/__init__.py @@ -121,7 +121,7 @@ def keep_version_file(self) -> bool: # If parsing the version fails, we assume the version file is not valid and overwrite. return False - # pylint: disable=too-many-branches + # pylint: disable=too-many-branches,too-many-statements def serialize(self) -> None: # remove existing folders when generate from tsp if self.code_model.is_tsp and self.code_model.options.get("clear-output-folder"): From 1de57f4fda7440ccd6ce4102df4f70e403893b29 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 27 May 2026 15:11:36 -0400 Subject: [PATCH 37/42] fix: address PR review comments for TypedDict support - Fix cross-namespace parent imports in types.py to import from types module - Add isidentifier() check for wire names (not just iskeyword) for functional TypedDict form - Use __doc__ assignment for functional-form TypedDict docstrings - Add missing op_tools import in model_typeddict.py.jinja2 - Fix test docstring to match actual behavior - Update changelog entry with correct file names (types.py, _unions.py) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../changes/python-addTypedDict-2026-3-21-17-47-3.md | 2 +- .../pygen/codegen/serializers/types_serializer.py | 10 ++++++---- .../pygen/codegen/templates/model_typeddict.py.jinja2 | 1 + .../generator/pygen/codegen/templates/types.py.jinja2 | 2 +- .../http-client-python/tests/unit/test_typeddict.py | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md index c629da91ce2..fae8e302e0b 100644 --- a/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md +++ b/.chronus/changes/python-addTypedDict-2026-3-21-17-47-3.md @@ -5,4 +5,4 @@ packages: - "@typespec/http-client-python" --- -[python] Always generate `TypedDict` typing hints for input models in the `_types.py` file +[python] Always generate `TypedDict` typing hints for input models in the `types.py` file, and named union aliases in the `_unions.py` file diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 7cde299d5ef..5a5d594206b 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -109,8 +109,8 @@ def discriminated_subtypes_union(self, model: ModelType) -> str: @staticmethod def has_keyword_wire_names(model: ModelType) -> bool: - """Whether any property wire_name is a Python keyword (requires functional TypedDict form).""" - return any(keyword.iskeyword(p.wire_name) for p in model.properties) + """Whether any property wire_name is a Python keyword or invalid identifier (requires functional TypedDict form).""" + return any(keyword.iskeyword(p.wire_name) or not p.wire_name.isidentifier() for p in model.properties) @staticmethod def get_shadowed_builtins(model: ModelType) -> frozenset[str]: @@ -168,12 +168,14 @@ def imports(self) -> FileImport: needs_builtins = True for parent in model.parents: if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: + # Import from sibling namespace's types module (not models) + parent_namespace = parent.client_namespace file_import.add_submodule_import( self.code_model.get_relative_import_path( self.serialize_namespace, - self.code_model.get_imported_namespace_for_model(parent.client_namespace), + parent_namespace, ), - parent.name, + "types", ImportType.LOCAL, ) if has_required: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 index 8176626fe3f..730f8627ffb 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 @@ -1,5 +1,6 @@ {# actual template starts here #} {% import "macros.jinja2" as macros %} +{% import 'operation_tools.jinja2' as op_tools %} {% if serializer.is_discriminated_base(model) %} {{ serializer.discriminated_subtypes_union(model) }} diff --git a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 index 1456fd1538c..88dd962c5b9 100644 --- a/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 +++ b/packages/http-client-python/generator/pygen/codegen/templates/types.py.jinja2 @@ -18,7 +18,7 @@ {{ serializer.declare_functional_model(model) }} -"""{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n") }} +{{ model.name }}.__doc__ = """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n") }} {% if model.properties != None %} {% for p in model.properties %} diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py index b6b42e262d4..c43d22a34e4 100644 --- a/packages/http-client-python/tests/unit/test_typeddict.py +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -107,7 +107,7 @@ def test_models_mode_none_types_file_has_no_typeddict_imports(): def test_models_mode_dpg_no_typeddict_models(): - """DPG models have base='dpg', not 'typeddict', so should not appear as typeddict_models.""" + """DPG models have base='dpg', not 'json', so they appear in typeddict_models.""" code_model = _make_code_model(models_mode="dpg") m1 = _make_model(code_model, "Foo", model_cls=DPGModelType) From a88d47137826f9b02fd410f74607bc9db163ace2 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 27 May 2026 15:37:32 -0400 Subject: [PATCH 38/42] lint --- .../generator/pygen/codegen/serializers/types_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 5a5d594206b..716b4bf9ad8 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -109,7 +109,7 @@ def discriminated_subtypes_union(self, model: ModelType) -> str: @staticmethod def has_keyword_wire_names(model: ModelType) -> bool: - """Whether any property wire_name is a Python keyword or invalid identifier (requires functional TypedDict form).""" + """Whether any property wire_name is a Python keyword or requires functional TypedDict form.""" return any(keyword.iskeyword(p.wire_name) or not p.wire_name.isidentifier() for p in model.properties) @staticmethod From ca959711ed978f6913d5e08cd77cd2562cf708f4 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 1 Jun 2026 12:47:32 -0400 Subject: [PATCH 39/42] fix: address review comments on TypedDict PR - Fix cross-namespace parent imports in types_serializer.py and model_type.py to use get_relative_import_path with module_name parameter instead of naive string concatenation - Remove unused model_typeddict.py.jinja2 template (dead code) - Fix misleading test name: rename test_models_mode_dpg_no_typeddict_models to test_models_mode_dpg_typeddict_models_included Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/models/model_type.py | 4 ++- .../codegen/serializers/types_serializer.py | 8 ++--- .../templates/model_typeddict.py.jinja2 | 29 ------------------- .../tests/unit/test_typeddict.py | 2 +- 4 files changed, 8 insertions(+), 35 deletions(-) delete mode 100644 packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 25e20b7cc14..9b7fba4d8c0 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -350,7 +350,9 @@ def imports(self, **kwargs: Any) -> FileImport: else: # Cross-namespace model — import from sibling namespace's types module file_import.add_submodule_import( - f"{relative_path}types", + self.code_model.get_relative_import_path( + serialize_namespace, self.client_namespace, module_name="types" + ), self.name, ImportType.LOCAL, typing_section=TypingSection.TYPING, diff --git a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py index 716b4bf9ad8..af3705e1383 100644 --- a/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py +++ b/packages/http-client-python/generator/pygen/codegen/serializers/types_serializer.py @@ -168,14 +168,14 @@ def imports(self) -> FileImport: needs_builtins = True for parent in model.parents: if parent.client_namespace != model.client_namespace and not parent.discriminated_subtypes: - # Import from sibling namespace's types module (not models) - parent_namespace = parent.client_namespace + # Import parent class from sibling namespace's types module file_import.add_submodule_import( self.code_model.get_relative_import_path( self.serialize_namespace, - parent_namespace, + parent.client_namespace, + module_name="types", ), - "types", + parent.name, ImportType.LOCAL, ) if has_required: diff --git a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 b/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 deleted file mode 100644 index 730f8627ffb..00000000000 --- a/packages/http-client-python/generator/pygen/codegen/templates/model_typeddict.py.jinja2 +++ /dev/null @@ -1,29 +0,0 @@ -{# actual template starts here #} -{% import "macros.jinja2" as macros %} -{% import 'operation_tools.jinja2' as op_tools %} - -{% if serializer.is_discriminated_base(model) %} -{{ serializer.discriminated_subtypes_union(model) }} -{% else %} - -{{ serializer.declare_model(model) }} - """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} - - {% if model.properties != None %} - {% for p in model.properties %} - {% for line in serializer.variable_documentation_string(p) %} - {{ macros.wrap_model_string(line, '\n ') -}} - {% endfor %} - {% endfor %} - {% endif %} - """ - - {% for p in serializer.get_properties_to_declare(model)%} - {{ serializer.declare_property(p) }} - {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} - {% if prop_description %} - """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} - {% endif %} - {% endfor %} -{% endif %} - diff --git a/packages/http-client-python/tests/unit/test_typeddict.py b/packages/http-client-python/tests/unit/test_typeddict.py index c43d22a34e4..7f624847e5b 100644 --- a/packages/http-client-python/tests/unit/test_typeddict.py +++ b/packages/http-client-python/tests/unit/test_typeddict.py @@ -106,7 +106,7 @@ def test_models_mode_none_types_file_has_no_typeddict_imports(): # ---------- models-mode=dpg ---------- -def test_models_mode_dpg_no_typeddict_models(): +def test_models_mode_dpg_typeddict_models_included(): """DPG models have base='dpg', not 'json', so they appear in typeddict_models.""" code_model = _make_code_model(models_mode="dpg") m1 = _make_model(code_model, "Foo", model_cls=DPGModelType) From 177a46c2d9bb9272b246f9754a26df0ea36a3e54 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 1 Jun 2026 13:04:27 -0400 Subject: [PATCH 40/42] fix: remove typeddict-specific guard from no-sdk-clients check Revert to the original no-sdk-clients check without tying the exemption to models-mode=typeddict. Models-only generation should not be gated on a specific models-mode. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- packages/http-client-python/emitter/src/emitter.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/http-client-python/emitter/src/emitter.ts b/packages/http-client-python/emitter/src/emitter.ts index 6ea057c3e60..3ecd85c5f57 100644 --- a/packages/http-client-python/emitter/src/emitter.ts +++ b/packages/http-client-python/emitter/src/emitter.ts @@ -198,9 +198,8 @@ async function onEmitMain(context: EmitContext) { const yamlMap = emitCodeModel(sdkContext); const parsedYamlMap = walkThroughNodes(yamlMap); - // Python emitter requires an SDK client in the TypeSpec (unless models-only typeddict mode) - const modelsMode = (sdkContext.emitContext.options as any)["models-mode"]; - if (sdkContext.sdkPackage.clients.length === 0 && modelsMode !== "typeddict") { + // Python emitter requires an SDK client in the TypeSpec + if (sdkContext.sdkPackage.clients.length === 0) { reportDiagnostic(program, { code: "no-sdk-clients", target: NoTarget, From a1101dac68fdd4a2af08da08030c0222a51c6548 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 2 Jun 2026 13:48:41 -0400 Subject: [PATCH 41/42] feat: replace JSON overloads with TypedDict overloads in operations Replace the JSON (MutableMapping) overload with a TypedDict reference from types.py for model input parameters. This gives users proper type-checking and IDE completion when passing dict literals. Changes: - preprocess: insert TypedDict model reference instead of any-object for both model and list/dict body params with DPG model elements - TypedDictModelType: add type_annotation, imports, docstring_type, and instance_check_template overrides - build_type: handle base='typeddict' to create TypedDictModelType - code_model: exclude typeddict refs from model_types (annotation-only) Generated operations now show: @overload def op(body: _models.X, ...) -> ... @overload def op(body: types.X, ...) -> ... @overload def op(body: IO[bytes], ...) -> ... JSON overloads are preserved for spread bodies (base=json) where no model class exists. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pygen/codegen/models/__init__.py | 6 ++-- .../pygen/codegen/models/code_model.py | 4 ++- .../pygen/codegen/models/model_type.py | 33 +++++++++++++++++++ .../generator/pygen/preprocess/__init__.py | 14 ++++---- 4 files changed, 48 insertions(+), 9 deletions(-) diff --git a/packages/http-client-python/generator/pygen/codegen/models/__init__.py b/packages/http-client-python/generator/pygen/codegen/models/__init__.py index a1d9f9a4dbc..1706576fb9d 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/__init__.py +++ b/packages/http-client-python/generator/pygen/codegen/models/__init__.py @@ -9,7 +9,7 @@ from .base_builder import BaseBuilder, ParameterListType from .code_model import CodeModel from .client import Client -from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType +from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType, TypedDictModelType from .dictionary_type import DictionaryType from .list_type import ListType from .combined_type import CombinedType @@ -167,7 +167,9 @@ def build_type(yaml_data: dict[str, Any], code_model: CodeModel) -> BaseType: response: Optional[BaseType] = None if yaml_data["type"] == "model": # need to special case model to avoid recursion - if yaml_data["base"] == "json" or not code_model.options["models-mode"]: + if yaml_data["base"] == "typeddict": + model_type = TypedDictModelType # type: ignore + elif yaml_data["base"] == "json" or not code_model.options["models-mode"]: model_type = JSONModelType elif yaml_data["base"] == "dpg": model_type = DPGModelType # type: ignore diff --git a/packages/http-client-python/generator/pygen/codegen/models/code_model.py b/packages/http-client-python/generator/pygen/codegen/models/code_model.py index 884072a831f..8c2f039b29c 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/code_model.py +++ b/packages/http-client-python/generator/pygen/codegen/models/code_model.py @@ -339,7 +339,9 @@ def model_types(self) -> list[ModelType]: """All of the model types in this class""" if not self._model_types: self._model_types = [ - t for t in self.types_map.values() if isinstance(t, ModelType) and t.usage != UsageFlags.Default.value + t + for t in self.types_map.values() + if isinstance(t, ModelType) and t.usage != UsageFlags.Default.value and t.base != "typeddict" ] return self._model_types diff --git a/packages/http-client-python/generator/pygen/codegen/models/model_type.py b/packages/http-client-python/generator/pygen/codegen/models/model_type.py index 9b7fba4d8c0..fcf5dab39fb 100644 --- a/packages/http-client-python/generator/pygen/codegen/models/model_type.py +++ b/packages/http-client-python/generator/pygen/codegen/models/model_type.py @@ -454,3 +454,36 @@ def imports(self, **kwargs: Any) -> FileImport: class TypedDictModelType(DPGModelType): base = "typeddict" + + def type_annotation(self, **kwargs: Any) -> str: + is_operation_file = kwargs.pop("is_operation_file", False) + skip_quote = kwargs.get("skip_quote", False) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + if serialize_namespace_type == NamespaceType.TYPES_FILE: + retval = self.name + else: + retval = f"types.{self.name}" + return retval if is_operation_file or skip_quote else f'"{retval}"' + + def docstring_type(self, **kwargs: Any) -> str: + client_namespace = self.client_namespace + if self.code_model.options.get("generation-subdir"): + client_namespace += f".{self.code_model.options['generation-subdir']}" + return f"~{client_namespace}.types.{self.name}" + + @property + def instance_check_template(self) -> str: + return "isinstance({}, MutableMapping)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + serialize_namespace_type = kwargs.get("serialize_namespace_type") + serialize_namespace = kwargs.get("serialize_namespace", self.code_model.namespace) + relative_path = self.code_model.get_relative_import_path(serialize_namespace, self.client_namespace) + if serialize_namespace_type in [NamespaceType.OPERATION, NamespaceType.CLIENT]: + file_import.add_submodule_import( + relative_path, + "types", + ImportType.LOCAL, + ) + return file_import diff --git a/packages/http-client-python/generator/pygen/preprocess/__init__.py b/packages/http-client-python/generator/pygen/preprocess/__init__.py index 1c1e575f44b..421e36fdae1 100644 --- a/packages/http-client-python/generator/pygen/preprocess/__init__.py +++ b/packages/http-client-python/generator/pygen/preprocess/__init__.py @@ -332,13 +332,15 @@ def add_body_param_type( if self.options["models-mode"] == "dpg" and is_dpg_model: if origin_type == "model": - body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"]) + td_type = {**model_type, "base": "typeddict"} + body_parameter["type"]["types"].insert(1, td_type) + code_model["types"].append(td_type) else: - # dict or list - # copy the original dict / list type - any_obj_list_or_dict = copy.deepcopy(body_parameter["type"]["types"][0]) - any_obj_list_or_dict["elementType"] = KNOWN_TYPES["any-object"] - body_parameter["type"]["types"].insert(1, any_obj_list_or_dict) + # dict or list — replace elementType with TypedDict reference + td_list_or_dict = copy.deepcopy(body_parameter["type"]["types"][0]) + td_list_or_dict["elementType"] = {**model_type, "base": "typeddict"} + body_parameter["type"]["types"].insert(1, td_list_or_dict) + code_model["types"].append(td_list_or_dict["elementType"]) code_model["types"].append(body_parameter["type"]) def pad_reserved_words(self, name: str, pad_type: PadType, yaml_type: dict[str, Any]) -> str: From 3cea12cd4f0ca1a97eec19ac75e89e931e030d00 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 2 Jun 2026 16:11:16 -0400 Subject: [PATCH 42/42] fix: restore regenerate-common.ts This file is shared with typespec-python via pnpm sync and should not be deleted. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../eng/scripts/ci/regenerate-common.ts | 755 ++++++++++++++++++ 1 file changed, 755 insertions(+) create mode 100644 packages/http-client-python/eng/scripts/ci/regenerate-common.ts diff --git a/packages/http-client-python/eng/scripts/ci/regenerate-common.ts b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts new file mode 100644 index 00000000000..5e6b2b9ed88 --- /dev/null +++ b/packages/http-client-python/eng/scripts/ci/regenerate-common.ts @@ -0,0 +1,755 @@ +/* eslint-disable no-console */ +/** + * Shared helpers, types, constants, and data tables used by `regenerate.ts`. + * + * This file is meant to be **byte-identical** between this package and the + * upstream `@typespec/http-client-python`. typespec-python syncs it from + * /core/packages/http-client-python/eng/scripts/ci/regenerate-common.ts + * via `pnpm sync`. + * + * Per-repo divergence (paths, emitter name, single-phase vs two-phase + * orchestration, argv/help text) lives in each repo's own `regenerate.ts`, + * which builds a `RegenerateContext` and feeds it into the helpers exported + * from this module. + */ + +import { compile, NodeHost } from "@typespec/compiler"; +import { execSync } from "child_process"; +import { existsSync, rmSync } from "fs"; +import { access, cp, mkdir, mkdtemp, readdir, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { dirname, join, relative, resolve } from "path"; +import pc from "picocolors"; + +// ---- Public types ---- + +export interface RegenerateFlags { + flavor: string; + debug: boolean; + name?: string; +} + +export interface CompileTask { + spec: string; + outputDir: string; + options: Record; +} + +// Group of tasks for the same spec that must run sequentially +export interface TaskGroup { + spec: string; + tasks: CompileTask[]; +} + +/** + * Per-repo context injected into the helpers below. Every value is repo + * specific and must be supplied by the caller's `regenerate.ts`. + */ +export interface RegenerateContext { + /** Absolute path to the package root (the dir containing `package.json`). */ + pluginDir: string; + /** Absolute path to the azure-http-specs `specs/` dir. */ + azureHttpSpecs: string; + /** Absolute path to the http-specs `specs/` dir. */ + httpSpecs: string; + /** Absolute path to where generated SDKs should go (e.g. `/generator`). */ + generatedFolder: string; + /** Emitter name to invoke (e.g. `@azure-tools/typespec-python`). */ + emitterName: string; +} + +/** + * Optional knobs for `buildTaskGroups`. Kept here so the call site in each + * repo's `regenerate.ts` can opt into the upstream two-phase pipeline + * (`emitYamlOnly: true`) or the single-phase pipeline (default). + */ +export interface BuildTaskGroupsOptions { + /** If true, ask the emitter to write YAML only and skip Python codegen. */ + emitYamlOnly?: boolean; +} + +// ---- Public constants ---- + +export const SKIP_SPECS: string[] = [ + "type/file", + "service/multiple-services", + "azure/client-generator-core/response-as-bool", +]; + +export const SpecialFlags: Record> = { + azure: { + "generate-test": true, + "generate-sample": true, + }, +}; + +// ---- Spec-specific emitter option overrides ---- + +export const AZURE_EMITTER_OPTIONS: Record< + string, + Record | Record[] +> = { + "azure/client-generator-core/access": { + namespace: "specs.azure.clientgenerator.core.access", + }, + "azure/client-generator-core/alternate-type": { + namespace: "specs.azure.clientgenerator.core.alternatetype", + }, + "azure/client-generator-core/api-version": { + namespace: "specs.azure.clientgenerator.core.apiversion", + }, + "azure/client-generator-core/client-initialization/default": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.default", + }, + "azure/client-generator-core/client-initialization/individually": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individually", + }, + "azure/client-generator-core/client-initialization/individuallyParent": { + namespace: "specs.azure.clientgenerator.core.clientinitialization.individuallyparent", + }, + "azure/client-generator-core/client-location": { + namespace: "specs.azure.clientgenerator.core.clientlocation", + }, + "azure/client-generator-core/deserialize-empty-string-as-null": { + namespace: "specs.azure.clientgenerator.core.emptystring", + }, + "azure/client-generator-core/flatten-property": { + namespace: "specs.azure.clientgenerator.core.flattenproperty", + }, + "azure/client-generator-core/usage": { + namespace: "specs.azure.clientgenerator.core.usage", + }, + "azure/client-generator-core/client-doc": { + namespace: "specs.azure.clientgenerator.core.clientdoc", + }, + "azure/client-generator-core/override": { + namespace: "specs.azure.clientgenerator.core.override", + }, + "azure/client-generator-core/hierarchy-building": { + namespace: "specs.azure.clientgenerator.core.hierarchybuilding", + }, + "azure/core/basic": { + namespace: "specs.azure.core.basic", + }, + "azure/core/lro/rpc": { + namespace: "specs.azure.core.lro.rpc", + }, + "azure/core/lro/standard": { + namespace: "specs.azure.core.lro.standard", + }, + "azure/core/model": { + namespace: "specs.azure.core.model", + }, + "azure/core/page": { + namespace: "specs.azure.core.page", + }, + "azure/core/scalar": { + namespace: "specs.azure.core.scalar", + }, + "azure/core/traits": { + namespace: "specs.azure.core.traits", + }, + "azure/encode/duration": { + namespace: "specs.azure.encode.duration", + }, + "azure/example/basic": { + namespace: "specs.azure.example.basic", + }, + "azure/payload/pageable": { + namespace: "specs.azure.payload.pageable", + }, + "azure/versioning/previewVersion": { + namespace: "specs.azure.versioning.previewversion", + }, + "client/structure/default": { + namespace: "client.structure.service", + }, + "client/structure/multi-client": { + "package-name": "client-structure-multiclient", + namespace: "client.structure.multiclient", + }, + "client/structure/renamed-operation": { + "package-name": "client-structure-renamedoperation", + namespace: "client.structure.renamedoperation", + }, + "client/structure/two-operation-group": { + "package-name": "client-structure-twooperationgroup", + namespace: "client.structure.twooperationgroup", + }, + "client/naming": { + namespace: "client.naming.main", + }, + "client/overload": { + namespace: "client.overload", + }, + "encode/duration": { + namespace: "encode.duration", + }, + "encode/numeric": { + namespace: "encode.numeric", + }, + "parameters/basic": { + namespace: "parameters.basic", + }, + "parameters/spread": { + namespace: "parameters.spread", + }, + "payload/content-negotiation": { + namespace: "payload.contentnegotiation", + }, + "payload/multipart": { + namespace: "payload.multipart", + }, + "serialization/encoded-name/json": { + namespace: "serialization.encodedname.json", + }, + "special-words": { + namespace: "specialwords", + }, + "service/multi-service": { + namespace: "service.multiservice", + }, + "client/structure/client-operation-group": { + "package-name": "client-structure-clientoperationgroup", + namespace: "client.structure.clientoperationgroup", + }, +}; + +export const EMITTER_OPTIONS: Record | Record[]> = { + "resiliency/srv-driven/old.tsp": { + "package-name": "resiliency-srv-driven1", + namespace: "resiliency.srv.driven1", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven1", + }, + "resiliency/srv-driven": { + "package-name": "resiliency-srv-driven2", + namespace: "resiliency.srv.driven2", + "package-mode": "azure-dataplane", + "package-pprint-name": "ResiliencySrvDriven2", + }, + "authentication/api-key": { + "clear-output-folder": "true", + }, + "authentication/http/custom": { + "package-name": "authentication-http-custom", + namespace: "authentication.http.custom", + "package-pprint-name": "Authentication Http Custom", + }, + "authentication/union": [ + { + "package-name": "authentication-union", + namespace: "authentication.union", + }, + { + "package-name": "setuppy-authentication-union", + namespace: "setuppy.authentication.union", + "keep-setup-py": "true", + }, + ], + "type/array": { + "package-name": "typetest-array", + namespace: "typetest.array", + }, + "type/dictionary": { + "package-name": "typetest-dictionary", + namespace: "typetest.dictionary", + }, + "type/enum/extensible": { + "package-name": "typetest-enum-extensible", + namespace: "typetest.enum.extensible", + }, + "type/enum/fixed": { + "package-name": "typetest-enum-fixed", + namespace: "typetest.enum.fixed", + }, + "type/model/empty": { + "package-name": "typetest-model-empty", + namespace: "typetest.model.empty", + }, + "type/model/inheritance/enum-discriminator": { + "package-name": "typetest-model-enumdiscriminator", + namespace: "typetest.model.enumdiscriminator", + }, + "type/model/inheritance/nested-discriminator": { + "package-name": "typetest-model-nesteddiscriminator", + namespace: "typetest.model.nesteddiscriminator", + }, + "type/model/inheritance/not-discriminated": { + "package-name": "typetest-model-notdiscriminated", + namespace: "typetest.model.notdiscriminated", + }, + "type/model/inheritance/single-discriminator": { + "package-name": "typetest-model-singlediscriminator", + namespace: "typetest.model.singlediscriminator", + }, + "type/model/inheritance/recursive": [ + { + "package-name": "typetest-model-recursive", + namespace: "typetest.model.recursive", + }, + { + "package-name": "generation-subdir", + namespace: "generation.subdir", + "generation-subdir": "_generated", + "generate-test": "false", + "clear-output-folder": "true", + }, + ], + "type/model/usage": { + "package-name": "typetest-model-usage", + namespace: "typetest.model.usage", + }, + "type/model/visibility": [ + { + "package-name": "typetest-model-visibility", + namespace: "typetest.model.visibility", + }, + { + "package-name": "headasbooleantrue", + namespace: "headasbooleantrue", + "head-as-boolean": "true", + }, + { + "package-name": "headasbooleanfalse", + namespace: "headasbooleanfalse", + "head-as-boolean": "false", + }, + ], + "type/property/nullable": { + "package-name": "typetest-property-nullable", + namespace: "typetest.property.nullable", + }, + "type/property/optionality": { + "package-name": "typetest-property-optional", + namespace: "typetest.property.optional", + }, + "type/property/additional-properties": { + "package-name": "typetest-property-additionalproperties", + namespace: "typetest.property.additionalproperties", + }, + "type/scalar": { + "package-name": "typetest-scalar", + namespace: "typetest.scalar", + }, + "type/property/value-types": { + "package-name": "typetest-property-valuetypes", + namespace: "typetest.property.valuetypes", + }, + "type/union": { + "package-name": "typetest-union", + namespace: "typetest.union", + }, + "type/union/discriminated": { + "package-name": "typetest-discriminatedunion", + namespace: "typetest.discriminatedunion", + }, + "type/file": { + "package-name": "typetest-file", + namespace: "typetest.file", + }, + documentation: { + "package-name": "specs-documentation", + namespace: "specs.documentation", + }, + "versioning/added": [ + { + "package-name": "versioning-added", + namespace: "versioning.added", + }, + { + "package-name": "generation-subdir2", + namespace: "generation.subdir2", + "generate-test": "false", + "generation-subdir": "_generated", + }, + ], +}; + +// ---- Public helpers ---- + +export function toPosix(p: string): string { + return p.replace(/\\/g, "/"); +} + +/** + * Whether a spec path belongs to azure-http-specs (vs standard http-specs). + * Uses the `azure-http-specs` substring rather than `azure` to avoid false + * positives when the working-dir path itself contains "azure" (e.g. + * azure-sdk-for-python). + */ +export function isAzureSpec(spec: string): boolean { + return spec.includes("azure-http-specs"); +} + +export function defaultPackageName(spec: string, ctx: RegenerateContext): string { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + return toPosix(relative(specDir, dirname(spec))) + .replace(/\//g, "-") + .toLowerCase(); +} + +export function getEmitterOptions( + spec: string, + flavor: string, + ctx: RegenerateContext, +): Record[] { + const specDir = isAzureSpec(spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const relativeSpec = toPosix(relative(specDir, spec)); + const key = relativeSpec.includes("resiliency/srv-driven/old.tsp") + ? relativeSpec + : dirname(relativeSpec); + const emitterOpts = EMITTER_OPTIONS[key] || + (flavor === "azure" ? AZURE_EMITTER_OPTIONS[key] : [{}]) || [{}]; + return Array.isArray(emitterOpts) ? emitterOpts : [emitterOpts]; +} + +/** + * Walk `baseDir` and collect every TypeSpec entry-point file that the + * regenerator should compile (handles `client.tsp`, `main.tsp`, and the + * special `resiliency/srv-driven/old.tsp` case). + */ +export async function getSubdirectories( + baseDir: string, + flags: RegenerateFlags, +): Promise { + const subdirectories: string[] = []; + + async function searchDir(currentDir: string) { + const items = await readdir(currentDir, { withFileTypes: true }); + + const promisesArray = items.map(async (item) => { + const subDirPath = join(currentDir, item.name); + if (item.isDirectory()) { + const mainTspPath = join(subDirPath, "main.tsp"); + const clientTspPath = join(subDirPath, "client.tsp"); + + const mainTspRelativePath = toPosix(relative(baseDir, mainTspPath)); + + if (SKIP_SPECS.some((skipSpec) => mainTspRelativePath.includes(skipSpec))) return; + + const hasMainTsp = await access(mainTspPath) + .then(() => true) + .catch(() => false); + const hasClientTsp = await access(clientTspPath) + .then(() => true) + .catch(() => false); + + if (mainTspRelativePath.toLowerCase().includes(flags.name || "")) { + if (mainTspRelativePath.includes("resiliency/srv-driven")) { + subdirectories.push(resolve(subDirPath, "old.tsp")); + } + if (hasClientTsp) { + subdirectories.push(resolve(subDirPath, "client.tsp")); + } else if (hasMainTsp) { + subdirectories.push(resolve(subDirPath, "main.tsp")); + } + } + + await searchDir(subDirPath); + } + }); + + await Promise.all(promisesArray); + } + + await searchDir(baseDir); + return subdirectories; +} + +export function buildTaskGroups( + specs: string[], + flags: RegenerateFlags, + ctx: RegenerateContext, + options: BuildTaskGroupsOptions = {}, +): TaskGroup[] { + const groups: TaskGroup[] = []; + + for (const spec of specs) { + const tasks: CompileTask[] = []; + + for (const emitterConfig of getEmitterOptions(spec, flags.flavor, ctx)) { + // Apply flavor defaults first, then per-spec options so they can override + // (e.g. "generate-test": "false") + const opts: Record = {}; + for (const [k, v] of Object.entries(SpecialFlags[flags.flavor] ?? {})) { + opts[k] = v; + } + Object.assign(opts, emitterConfig); + + opts["flavor"] = flags.flavor; + + // Set output directory - tests/generated// structure. + // Always anchored at /../tests/generated regardless of + // pluginDir, so generator-only checkouts work too. + const packageName = (opts["package-name"] as string) || defaultPackageName(spec, ctx); + const outputDir = + (opts["emitter-output-dir"] as string) || + toPosix(`${ctx.generatedFolder}/../tests/generated/${flags.flavor}/${packageName}`); + opts["emitter-output-dir"] = outputDir; + + if (flags.debug) { + opts["debug"] = true; + } + + opts["examples-dir"] = toPosix(join(dirname(spec), "examples")); + + if (options.emitYamlOnly) { + // Emit YAML only - Python processing is batched after all specs compile. + opts["emit-yaml-only"] = true; + } + + tasks.push({ spec, outputDir, options: opts }); + } + + groups.push({ spec, tasks }); + } + + return groups; +} + +export async function compileSpec( + task: CompileTask, + ctx: RegenerateContext, +): Promise<{ success: boolean; error?: string }> { + const { spec, outputDir, options } = task; + + try { + const compilerOptions = { + emit: [ctx.pluginDir], + options: { + [ctx.emitterName]: options, + }, + }; + + const program = await compile(NodeHost, spec, compilerOptions); + + if (program.hasError()) { + const errors = program.diagnostics + .filter((d) => d.severity === "error") + .map((d) => d.message) + .join("\n"); + return { success: false, error: errors }; + } + + return { success: true }; + } catch (err) { + rmSync(outputDir, { recursive: true, force: true }); + return { success: false, error: String(err) }; + } +} + +export function renderProgressBar( + completed: number, + failed: number, + total: number, + width: number = 40, +): string { + const successCount = completed - failed; + const successWidth = Math.round((successCount / total) * width); + const failWidth = Math.round((failed / total) * width); + const emptyWidth = width - successWidth - failWidth; + + const successBar = pc.bgGreen(" ".repeat(successWidth)); + const failBar = failed > 0 ? pc.bgRed(" ".repeat(failWidth)) : ""; + const emptyBar = pc.dim("░".repeat(Math.max(0, emptyWidth))); + + const percent = Math.round((completed / total) * 100); + return `${successBar}${failBar}${emptyBar} ${pc.cyan(`${percent}%`)} (${completed}/${total})`; +} + +export async function runParallel( + groups: TaskGroup[], + maxJobs: number, + ctx: RegenerateContext, +): Promise> { + const results = new Map(); + const executing: Set> = new Set(); + + const totalTasks = groups.reduce((sum, g) => sum + g.tasks.length, 0); + let completed = 0; + let failed = 0; + const failedSpecs: string[] = []; + + const isTTY = process.stdout.isTTY; + + const updateProgress = () => { + if (isTTY) { + process.stdout.write(`\r${renderProgressBar(completed, failed, totalTasks)}`); + } + }; + + updateProgress(); + + for (const group of groups) { + // Each group runs as a unit - tasks within a group run sequentially + // to avoid state pollution. Different groups run in parallel. + const runGroup = async () => { + const specDir = isAzureSpec(group.spec) ? ctx.azureHttpSpecs : ctx.httpSpecs; + const shortName = toPosix(relative(specDir, dirname(group.spec))); + + let groupSuccess = true; + for (const task of group.tasks) { + const packageName = (task.options["package-name"] as string) || shortName; + + const result = await compileSpec(task, ctx); + completed++; + + if (!result.success) { + failed++; + failedSpecs.push(`${packageName}: ${result.error}`); + groupSuccess = false; + } + + updateProgress(); + } + + results.set(group.spec, groupSuccess); + }; + + const p = runGroup().finally(() => executing.delete(p)); + executing.add(p); + + if (executing.size >= maxJobs) { + await Promise.race(executing); + } + } + + await Promise.all(executing); + + if (isTTY) { + process.stdout.write("\r" + " ".repeat(60) + "\r"); + } + + if (failedSpecs.length > 0) { + console.log(pc.red(`\nFailed specs:`)); + for (const spec of failedSpecs) { + console.log(pc.red(` • ${spec}`)); + } + } + + return results; +} + +/** + * Pre-create the marker files that the test harness expects to find before + * regeneration so it can verify they're cleared/preserved correctly. + */ +export async function preprocess(flavor: string, generatedFolder: string): Promise { + if (flavor !== "azure") return; + + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated/azure"); + + const DELETE_CONTENT = "# This file is to be deleted after regeneration"; + const KEEP_CONTENT = "# This file is to be kept after regeneration"; + const DELETE_FILE = "to_be_deleted.py"; + const entries: { folder: string[]; file: string; content: string }[] = [ + { + folder: ["authentication-api-key", "authentication", "apikey", "_operations"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir", "_generated"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generated_tests"], + file: DELETE_FILE, + content: DELETE_CONTENT, + }, + { + folder: ["generation-subdir", "generation", "subdir"], + file: "to_be_kept.py", + content: KEEP_CONTENT, + }, + ]; + + await Promise.all( + entries.map(async ({ folder, file, content }) => { + const targetFolder = join(testsGeneratedDir, ...folder); + await mkdir(targetFolder, { recursive: true }); + await writeFile(join(targetFolder, file), content); + }), + ); +} + +/** + * Resets the `tests/generated/{azure,unbranded}` baseline by sparse-checking-out + * `eng/tools/azure-sdk-tools/emitter/generated` from the Azure/azure-sdk-for-python repo, then + * deleting a couple of fully-generated package folders so regeneration has to + * recreate them from scratch (smoke test of full-emit path). + * + * `generatedFolder` is the per-repo `generator/` directory; baseline lands at + * `/../tests/generated`. + */ +export async function prepareBaselineOfGeneratedCode(generatedFolder: string): Promise { + const repoUrl = "https://github.com/Azure/azure-sdk-for-python.git"; + const branch = "typespec-python-generated-tests"; + const sourceSubdir = "eng/tools/azure-sdk-tools/emitter/generated"; + const testsGeneratedDir = resolve(generatedFolder, "../tests/generated"); + + console.log(pc.cyan(`\n${"=".repeat(60)}`)); + console.log(pc.cyan(`Resetting baseline from ${repoUrl} (${branch}/${sourceSubdir})`)); + console.log(pc.cyan(`${"=".repeat(60)}\n`)); + + // Wipe tests/generated + if (existsSync(testsGeneratedDir)) { + console.log(pc.dim(`Removing ${testsGeneratedDir}`)); + rmSync(testsGeneratedDir, { recursive: true, force: true }); + } + + // Sparse-checkout the baseline folder into a temp directory + const tempDir = await mkdtemp(join(tmpdir(), "azsdk-baseline-")); + try { + console.log(pc.dim(`Cloning into ${tempDir}`)); + const run = (cmd: string) => + execSync(cmd, { cwd: tempDir, stdio: ["ignore", "ignore", "inherit"] }); + + run(`git init`); + run(`git config core.longpaths true`); + run(`git remote add origin ${repoUrl}`); + run(`git config core.sparseCheckout true`); + run(`git sparse-checkout init --cone`); + run(`git sparse-checkout set ${sourceSubdir}`); + run(`git fetch --depth 1 origin ${branch}`); + run(`git checkout FETCH_HEAD`); + + const sourceRoot = join(tempDir, ...sourceSubdir.split("/")); + for (const flavor of ["azure", "unbranded"]) { + const src = join(sourceRoot, flavor); + const dest = join(testsGeneratedDir, flavor); + if (!existsSync(src)) { + console.warn(pc.yellow(`Baseline folder not found: ${src}`)); + continue; + } + console.log(pc.dim(`Copying ${flavor}/ -> ${dest}`)); + await cp(src, dest, { recursive: true }); + } + + console.log(pc.green(`Baseline reset complete.\n`)); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + + // Smoke test the full-emit path: delete a couple of fully-generated package + // folders and every README.md so regeneration has to recreate them. + const deleteIfExists = (path: string) => { + if (!existsSync(path)) return; + console.log(pc.dim(`Deleting ${path}`)); + rmSync(path, { recursive: true, force: true }); + }; + + deleteIfExists(join(testsGeneratedDir, "azure", "authentication-http-custom")); + deleteIfExists(join(testsGeneratedDir, "unbranded", "encode-array")); + + if (existsSync(testsGeneratedDir)) { + const entries = await readdir(testsGeneratedDir, { recursive: true, withFileTypes: true }); + for (const entry of entries) { + if (entry.isFile() && entry.name === "README.md") { + deleteIfExists(join(entry.parentPath, entry.name)); + } + } + } +}