From a04454caa6a1a3739313ef72408b135f6130aa34 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Wed, 8 Jul 2026 19:42:21 -0700 Subject: [PATCH 01/37] Add config parsing --- Herebyfile.mjs | 1 + .../src/enums/scriptKind.enum.ts | 4 +- .../native-preview/src/enums/scriptKind.ts | 3 +- internal/api/session.go | 1 - internal/compiler/fileloader.go | 2 +- internal/compiler/program.go | 9 +- internal/core/contentmapper.go | 9 ++ internal/core/parsedoptions.go | 1 + internal/core/scriptkind.go | 20 ++- .../core/scriptkind_stringer_generated.go | 20 +-- internal/diagnostics/diagnostics_generated.go | 12 ++ .../diagnostics/extraDiagnosticMessages.json | 12 ++ internal/ls/string_completions.go | 6 +- internal/modulespecifiers/util.go | 13 +- internal/testrunner/test_case_parser.go | 1 - internal/tsoptions/commandlineoption.go | 4 + internal/tsoptions/parsedcommandline.go | 19 ++- internal/tsoptions/parsinghelpers.go | 45 +++++++ internal/tsoptions/tsconfigparsing.go | 88 ++++++++----- internal/tsoptions/tsconfigparsing_test.go | 123 ++++++++++++++++-- .../tsoptionstest/parsedcommandline.go | 2 +- 21 files changed, 305 insertions(+), 90 deletions(-) create mode 100644 internal/core/contentmapper.go diff --git a/Herebyfile.mjs b/Herebyfile.mjs index 47ef8dd6a9b..d40560b8fb9 100644 --- a/Herebyfile.mjs +++ b/Herebyfile.mjs @@ -345,6 +345,7 @@ const enumDefs = [ { name: "ModuleDetectionKind", goPrefix: "ModuleDetectionKind", goFile: "internal/core/compileroptions.go", outDir: "_packages/native-preview/src/enums" }, { name: "NewLineKind", goPrefix: "NewLineKind", goFile: "internal/core/compileroptions.go", outDir: "_packages/native-preview/src/enums" }, { name: "JsxEmit", goPrefix: "JsxEmit", goFile: "internal/core/compileroptions.go", outDir: "_packages/native-preview/src/enums" }, + { name: "ScriptKind", goPrefix: "ScriptKind", goFile: "internal/core/scriptkind.go", outDir: "_packages/native-preview/src/enums" }, { name: "TokenFlags", goPrefix: "TokenFlags", goFile: "internal/ast/tokenflags.go", outDir: "_packages/native-preview/src/enums" }, { name: "NodeBuilderFlags", goPrefix: "Flags", goFile: "internal/nodebuilder/types.go", outDir: "_packages/native-preview/src/enums" }, { name: "CompletionItemKind", goPrefix: "CompletionItemKind", goFile: "internal/lsp/lsproto/lsp_generated.go", outDir: "_packages/native-preview/src/enums" }, diff --git a/_packages/native-preview/src/enums/scriptKind.enum.ts b/_packages/native-preview/src/enums/scriptKind.enum.ts index b089f2f1af1..391aa7ba24f 100644 --- a/_packages/native-preview/src/enums/scriptKind.enum.ts +++ b/_packages/native-preview/src/enums/scriptKind.enum.ts @@ -1,10 +1,10 @@ +// Code generated by Herebyfile.mjs generate:enums from internal/core/scriptkind.go. DO NOT EDIT. + export enum ScriptKind { Unknown = 0, JS = 1, JSX = 2, TS = 3, TSX = 4, - External = 5, JSON = 6, - Deferred = 7, } diff --git a/_packages/native-preview/src/enums/scriptKind.ts b/_packages/native-preview/src/enums/scriptKind.ts index afa05b63f17..3c682541369 100644 --- a/_packages/native-preview/src/enums/scriptKind.ts +++ b/_packages/native-preview/src/enums/scriptKind.ts @@ -1,3 +1,4 @@ +// Code generated by Herebyfile.mjs generate:enums from internal/core/scriptkind.go. DO NOT EDIT. export var ScriptKind: any; (function (ScriptKind) { ScriptKind[ScriptKind["Unknown"] = 0] = "Unknown"; @@ -5,7 +6,5 @@ export var ScriptKind: any; ScriptKind[ScriptKind["JSX"] = 2] = "JSX"; ScriptKind[ScriptKind["TS"] = 3] = "TS"; ScriptKind[ScriptKind["TSX"] = 4] = "TSX"; - ScriptKind[ScriptKind["External"] = 5] = "External"; ScriptKind[ScriptKind["JSON"] = 6] = "JSON"; - ScriptKind[ScriptKind["Deferred"] = 7] = "Deferred"; })(ScriptKind || (ScriptKind = {})); diff --git a/internal/api/session.go b/internal/api/session.go index 7e29c44b6ff..e725cf6db50 100644 --- a/internal/api/session.go +++ b/internal/api/session.go @@ -1031,7 +1031,6 @@ func (s *Session) handleParseConfigFile(ctx context.Context, params *ParseConfig nil, /*existingOptionsRaw*/ configFileName, nil, /*resolutionStack*/ - nil, /*extraFileExtensions*/ nil, /*extendedConfigCache*/ ) diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index f8d006e9258..94cba4244f6 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -125,7 +125,7 @@ func processAllProgramFiles( ) processedFiles { compilerOptions := opts.Config.CompilerOptions() rootFiles := opts.Config.FileNames() - supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, nil /*extraFileExtensions*/) + supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, opts.Config.ContentMapperExtensions()) supportedExtensionsWithJsonIfResolveJsonModule := tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions, supportedExtensions) var maxNodeModuleJsDepth int if p := opts.Config.CompilerOptions().MaxNodeModuleJsDepth; p != nil { diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 74b2504d9ec..1dd71306054 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -234,7 +234,7 @@ func (p *Program) GetSourceFileFromReference(origin *ast.SourceFile, ref *ast.Fi // Still, without the failed lookup reporting that only the loader does, this isn't terribly complicated fileName := tspath.ResolvePath(tspath.GetDirectoryPath(origin.FileName()), ref.FileName) - supportedExtensionsBase := tsoptions.GetSupportedExtensions(p.Options(), nil /*extraFileExtensions*/) + supportedExtensionsBase := tsoptions.GetSupportedExtensions(p.Options(), p.CommandLine().ContentMapperExtensions()) supportedExtensions := tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(p.Options(), supportedExtensionsBase) allowNonTsExtensions := p.Options().AllowNonTsExtensions.IsTrue() if tspath.HasExtension(fileName) { @@ -701,7 +701,7 @@ func (p *Program) canIncludeBindAndCheckDiagnostics(sourceFile *ast.SourceFile) return false } - if sourceFile.ScriptKind == core.ScriptKindTS || sourceFile.ScriptKind == core.ScriptKindTSX || sourceFile.ScriptKind == core.ScriptKindExternal { + if sourceFile.ScriptKind == core.ScriptKindTS || sourceFile.ScriptKind == core.ScriptKindTSX { return true } @@ -709,11 +709,10 @@ func (p *Program) canIncludeBindAndCheckDiagnostics(sourceFile *ast.SourceFile) isCheckJS := isJS && ast.IsCheckJSEnabledForFile(sourceFile, p.Options()) isPlainJS := ast.IsPlainJSFile(sourceFile, p.Options().CheckJs) - // By default, only type-check .ts, .tsx, Deferred, plain JS, checked JS and External + // By default, only type-check .ts, .tsx, plain JS, and checked JS // - plain JS: .js files with no // ts-check and checkJs: undefined // - check JS: .js files with either // ts-check or checkJs: true - // - external: files that are added by plugins - return isPlainJS || isCheckJS || sourceFile.ScriptKind == core.ScriptKindDeferred + return isPlainJS || isCheckJS } func (p *Program) getSourceFilesToEmit(targetSourceFile *ast.SourceFile, forceDtsEmit bool) []*ast.SourceFile { diff --git a/internal/core/contentmapper.go b/internal/core/contentmapper.go new file mode 100644 index 00000000000..7e155a3b191 --- /dev/null +++ b/internal/core/contentmapper.go @@ -0,0 +1,9 @@ +package core + +// ContentMapper describes an external content mapper declared at the top level of a tsconfig. +// A content mapper registers foreign file extensions whose contents are transformed into +// TypeScript during program construction. +type ContentMapper struct { + Command []string `json:"command"` + Extensions []string `json:"extensions"` +} diff --git a/internal/core/parsedoptions.go b/internal/core/parsedoptions.go index 3aba067df0d..7b9ba89a170 100644 --- a/internal/core/parsedoptions.go +++ b/internal/core/parsedoptions.go @@ -7,4 +7,5 @@ type ParsedOptions struct { FileNames []string `json:"fileNames"` ProjectReferences []*ProjectReference `json:"projectReferences"` + ContentMappers []*ContentMapper `json:"contentMappers"` } diff --git a/internal/core/scriptkind.go b/internal/core/scriptkind.go index e7066ae2240..2d98bd4c503 100644 --- a/internal/core/scriptkind.go +++ b/internal/core/scriptkind.go @@ -6,16 +6,12 @@ package core type ScriptKind int32 const ( - ScriptKindUnknown ScriptKind = iota - ScriptKindJS - ScriptKindJSX - ScriptKindTS - ScriptKindTSX - ScriptKindExternal - ScriptKindJSON - /** - * Used on extensions that doesn't define the ScriptKind but the content defines it. - * Deferred extensions are going to be included in all project contexts. - */ - ScriptKindDeferred + ScriptKindUnknown ScriptKind = 0 + ScriptKindJS ScriptKind = 1 + ScriptKindJSX ScriptKind = 2 + ScriptKindTS ScriptKind = 3 + ScriptKindTSX ScriptKind = 4 + // Value 5 is reserved (formerly ScriptKindExternal). + ScriptKindJSON ScriptKind = 6 + // Value 7 is reserved (formerly ScriptKindDeferred). ) diff --git a/internal/core/scriptkind_stringer_generated.go b/internal/core/scriptkind_stringer_generated.go index 97de80e4ca4..8da807e5045 100644 --- a/internal/core/scriptkind_stringer_generated.go +++ b/internal/core/scriptkind_stringer_generated.go @@ -1,4 +1,4 @@ -// Code generated by "stringer -type=ScriptKind -output=scriptkind_stringer_generated.go"; DO NOT EDIT. +// Code generated by "stringer -type=ScriptKind -output=internal/core/scriptkind_stringer_generated.go ./internal/core"; DO NOT EDIT. package core @@ -13,19 +13,23 @@ func _() { _ = x[ScriptKindJSX-2] _ = x[ScriptKindTS-3] _ = x[ScriptKindTSX-4] - _ = x[ScriptKindExternal-5] _ = x[ScriptKindJSON-6] - _ = x[ScriptKindDeferred-7] } -const _ScriptKind_name = "ScriptKindUnknownScriptKindJSScriptKindJSXScriptKindTSScriptKindTSXScriptKindExternalScriptKindJSONScriptKindDeferred" +const ( + _ScriptKind_name_0 = "ScriptKindUnknownScriptKindJSScriptKindJSXScriptKindTSScriptKindTSX" + _ScriptKind_name_1 = "ScriptKindJSON" +) -var _ScriptKind_index = [...]uint8{0, 17, 29, 42, 54, 67, 85, 99, 117} +var _ScriptKind_index_0 = [...]uint8{0, 17, 29, 42, 54, 67} func (i ScriptKind) String() string { - idx := int(i) - 0 - if i < 0 || idx >= len(_ScriptKind_index)-1 { + switch { + case 0 <= i && i <= 4: + return _ScriptKind_name_0[_ScriptKind_index_0[i]:_ScriptKind_index_0[i+1]] + case i == 6: + return _ScriptKind_name_1 + default: return "ScriptKind(" + strconv.FormatInt(int64(i), 10) + ")" } - return _ScriptKind_name[_ScriptKind_index[idx]:_ScriptKind_index[idx+1]] } diff --git a/internal/diagnostics/diagnostics_generated.go b/internal/diagnostics/diagnostics_generated.go index 86ffdcea417..cfd424061df 100644 --- a/internal/diagnostics/diagnostics_generated.go +++ b/internal/diagnostics/diagnostics_generated.go @@ -4310,6 +4310,12 @@ var Sort_Imports = &Message{code: 100018, category: CategoryMessage, key: "Sort_ var JSDoc_comment = &Message{code: 100019, category: CategoryMessage, key: "JSDoc_comment_100019", text: "JSDoc comment"} +var Content_mapper_file_extension_0_must_begin_with_a = &Message{code: 100020, category: CategoryError, key: "Content_mapper_file_extension_0_must_begin_with_a_100020", text: "Content mapper file extension '{0}' must begin with a '.'."} + +var Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper = &Message{code: 100021, category: CategoryError, key: "Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper_100021", text: "Content mapper file extension '{0}' is a built-in extension and cannot be registered by a content mapper."} + +var Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper = &Message{code: 100022, category: CategoryError, key: "Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper_100022", text: "Content mapper file extension '{0}' is registered by more than one content mapper."} + func keyToMessage(key Key) *Message { switch key { case "Unterminated_string_literal_1002": @@ -8620,6 +8626,12 @@ func keyToMessage(key Key) *Message { return Sort_Imports case "JSDoc_comment_100019": return JSDoc_comment + case "Content_mapper_file_extension_0_must_begin_with_a_100020": + return Content_mapper_file_extension_0_must_begin_with_a + case "Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper_100021": + return Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper + case "Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper_100022": + return Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper default: return nil } diff --git a/internal/diagnostics/extraDiagnosticMessages.json b/internal/diagnostics/extraDiagnosticMessages.json index 10ea4408af6..af395934784 100644 --- a/internal/diagnostics/extraDiagnosticMessages.json +++ b/internal/diagnostics/extraDiagnosticMessages.json @@ -130,5 +130,17 @@ "JSDoc comment": { "category": "Message", "code": 100019 + }, + "Content mapper file extension '{0}' must begin with a '.'.": { + "category": "Error", + "code": 100020 + }, + "Content mapper file extension '{0}' is a built-in extension and cannot be registered by a content mapper.": { + "category": "Error", + "code": 100021 + }, + "Content mapper file extension '{0}' is registered by more than one content mapper.": { + "category": "Error", + "code": 100022 } } diff --git a/internal/ls/string_completions.go b/internal/ls/string_completions.go index f5240bcf329..18afc5aceff 100644 --- a/internal/ls/string_completions.go +++ b/internal/ls/string_completions.go @@ -1007,7 +1007,7 @@ func (l *LanguageService) getExtensionOptions( mode core.ResolutionMode, checker *checker.Checker, ) *extensionOptions { - extensionsToSearch := getSupportedExtensionsForModuleResolution(options, checker) + extensionsToSearch := getSupportedExtensionsForModuleResolution(options, l.GetProgram().CommandLine().ContentMapperExtensions(), checker) return &extensionOptions{ extensionsToSearch: extensionsToSearch, @@ -1018,7 +1018,7 @@ func (l *LanguageService) getExtensionOptions( } } -func getSupportedExtensionsForModuleResolution(options *core.CompilerOptions, checker *checker.Checker) []string { +func getSupportedExtensionsForModuleResolution(options *core.CompilerOptions, extraExtensions []string, checker *checker.Checker) []string { /** file extensions from ambient modules declarations e.g. *.css */ var extensions []string if checker != nil { @@ -1031,7 +1031,7 @@ func getSupportedExtensionsForModuleResolution(options *core.CompilerOptions, ch extensions = append(extensions, name[1:]) } } - supportedExtensions := tsoptions.GetSupportedExtensions(options, nil /*extraFileExtensions*/) + supportedExtensions := tsoptions.GetSupportedExtensions(options, extraExtensions) for _, ext := range supportedExtensions { extensions = append(extensions, ext...) } diff --git a/internal/modulespecifiers/util.go b/internal/modulespecifiers/util.go index a765e88429e..27230b42756 100644 --- a/internal/modulespecifiers/util.go +++ b/internal/modulespecifiers/util.go @@ -198,18 +198,7 @@ func tryGetAnyFileFromPath(host ModuleSpecifierGenerationHost, path string) bool &core.CompilerOptions{ AllowJs: core.TSTrue, }, - []tsoptions.FileExtensionInfo{ - { - Extension: "node", - IsMixedContent: false, - ScriptKind: core.ScriptKindExternal, - }, - { - Extension: "json", - IsMixedContent: false, - ScriptKind: core.ScriptKindJSON, - }, - }, + []string{".node", ".json"}, ) for _, exts := range extGroups { for _, e := range exts { diff --git a/internal/testrunner/test_case_parser.go b/internal/testrunner/test_case_parser.go index 1baec4087e0..50718afba34 100644 --- a/internal/testrunner/test_case_parser.go +++ b/internal/testrunner/test_case_parser.go @@ -91,7 +91,6 @@ func makeUnitsFromTest(code string, fileName string) testCaseContent { nil, /*existingOptionsRaw*/ configFileName, nil, /*resolutionStack*/ - nil, /*extraFileExtensions*/ nil, /*extendedConfigCache*/ ) tsConfigFileUnitData = data diff --git a/internal/tsoptions/commandlineoption.go b/internal/tsoptions/commandlineoption.go index de7c8c723ec..baa2b35c6a0 100644 --- a/internal/tsoptions/commandlineoption.go +++ b/internal/tsoptions/commandlineoption.go @@ -139,6 +139,10 @@ var commandLineOptionElements = map[string]*CommandLineOption{ Name: "references", Kind: CommandLineOptionTypeObject, }, + "contentMappers": { + Name: "contentMappers", + Kind: CommandLineOptionTypeObject, + }, "files": { Name: "files", Kind: CommandLineOptionTypeString, diff --git a/internal/tsoptions/parsedcommandline.go b/internal/tsoptions/parsedcommandline.go index 9f0229165bc..ef05e625a22 100644 --- a/internal/tsoptions/parsedcommandline.go +++ b/internal/tsoptions/parsedcommandline.go @@ -37,7 +37,6 @@ type ParsedCommandLine struct { wildcardDirectories map[string]bool includeGlobsOnce sync.Once includeGlobs []*glob.Glob - extraFileExtensions []FileExtensionInfo sourceAndOutputMapsOnce sync.Once sourceToProjectReference map[tspath.Path]*SourceOutputAndProjectReference @@ -310,6 +309,21 @@ func (p *ParsedCommandLine) ProjectReferences() []*core.ProjectReference { return p.ParsedConfig.ProjectReferences } +func (p *ParsedCommandLine) ContentMappers() []*core.ContentMapper { + if p == nil || p.ParsedConfig == nil { + return nil + } + return p.ParsedConfig.ContentMappers +} + +// ContentMapperExtensions returns the flattened list of file extensions registered by the +// config's content mappers. +func (p *ParsedCommandLine) ContentMapperExtensions() []string { + return core.FlatMap(p.ContentMappers(), func(m *core.ContentMapper) []string { + return m.Extensions + }) +} + func (p *ParsedCommandLine) ResolvedProjectReferencePaths() []string { p.resolvedProjectReferencePathsOnce.Do(func() { p.resolvedProjectReferencePaths = core.Map(p.ParsedConfig.ProjectReferences, core.ResolveProjectReferencePath) @@ -397,7 +411,7 @@ func (p *ParsedCommandLine) ReloadFileNamesOfParsedCommandLine(fs vfs.FS) *Parse p.GetCurrentDirectory(), p.CompilerOptions(), fs, - p.extraFileExtensions, + p.ContentMapperExtensions(), ) parsedConfig.FileNames = fileNames parsedCommandLine := ParsedCommandLine{ @@ -409,7 +423,6 @@ func (p *ParsedCommandLine) ReloadFileNamesOfParsedCommandLine(fs vfs.FS) *Parse comparePathsOptions: p.comparePathsOptions, wildcardDirectories: p.wildcardDirectories, includeGlobs: p.includeGlobs, - extraFileExtensions: p.extraFileExtensions, literalFileNamesLen: literalFileNamesLen, } return &parsedCommandLine diff --git a/internal/tsoptions/parsinghelpers.go b/internal/tsoptions/parsinghelpers.go index 47a4d476bf4..04751c8df67 100644 --- a/internal/tsoptions/parsinghelpers.go +++ b/internal/tsoptions/parsinghelpers.go @@ -85,6 +85,48 @@ func parseProjectReference(json any) []*core.ProjectReference { return result } +func parseContentMapper(json any) (*core.ContentMapper, []*ast.Diagnostic) { + v, ok := json.(*collections.OrderedMap[string, any]) + if !ok { + return nil, nil + } + var errors []*ast.Diagnostic + mapper := &core.ContentMapper{} + if command, ok := v.Get("command"); ok { + if strs, valid := parseStringArrayStrict(command); valid { + mapper.Command = strs + } else { + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Compiler_option_0_requires_a_value_of_type_1, "contentMapper.command", "string[]")) + } + } + if extensions, ok := v.Get("extensions"); ok { + if strs, valid := parseStringArrayStrict(extensions); valid { + mapper.Extensions = strs + } else { + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Compiler_option_0_requires_a_value_of_type_1, "contentMapper.extensions", "string[]")) + } + } + return mapper, errors +} + +// parseStringArrayStrict returns the string slice and true only if value is an array whose +// elements are all strings. A missing element or wrong element type yields false. +func parseStringArrayStrict(value any) ([]string, bool) { + arr, ok := value.([]any) + if !ok { + return nil, false + } + result := make([]string, 0, len(arr)) + for _, v := range arr { + str, ok := v.(string) + if !ok { + return nil, false + } + result = append(result, str) + } + return result, true +} + func parseJsonToStringKey(json any) *collections.OrderedMap[string, any] { result := collections.NewOrderedMapWithSizeHint[string, any](6) if m, ok := json.(*collections.OrderedMap[string, any]); ok { @@ -100,6 +142,9 @@ func parseJsonToStringKey(json any) *collections.OrderedMap[string, any] { if v, ok := m.Get("references"); ok { result.Set("references", v) } + if v, ok := m.Get("contentMappers"); ok { + result.Set("contentMappers", v) + } if v, ok := m.Get("extends"); ok { if str, ok := v.(string); ok { result.Set("extends", []any{str}) diff --git a/internal/tsoptions/tsconfigparsing.go b/internal/tsoptions/tsconfigparsing.go index 4f85782753d..f5ca2194207 100644 --- a/internal/tsoptions/tsconfigparsing.go +++ b/internal/tsoptions/tsconfigparsing.go @@ -65,6 +65,10 @@ var tsconfigRootOptionsMap = &CommandLineOption{ Kind: CommandLineOptionTypeList, // should be a list of projectReference // Category: diagnostics.Projects, }, + { + Name: "contentMappers", + Kind: CommandLineOptionTypeList, // list of content mapper objects + }, { Name: "files", Kind: CommandLineOptionTypeList, @@ -145,12 +149,6 @@ func (c *configFileSpecs) getMatchedFileSpec(fileName string, comparePathsOption return "" } -type FileExtensionInfo struct { - Extension string - IsMixedContent bool - ScriptKind core.ScriptKind -} - type ExtendedConfigCache interface { GetExtendedConfig(fileName string, path tspath.Path, resolutionStack []tspath.Path, host ParseConfigHost) *ExtendedConfigCacheEntry } @@ -731,11 +729,10 @@ func ParseJsonSourceFileConfigFileContent( existingOptionsRaw *collections.OrderedMap[string, any], configFileName string, resolutionStack []tspath.Path, - extraFileExtensions []FileExtensionInfo, extendedConfigCache ExtendedConfigCache, ) *ParsedCommandLine { // tracing?.push(tracing.Phase.Parse, "parseJsonSourceFileConfigFileContent", { path: sourceFile.fileName }); - result := parseJsonConfigFileContentWorker(nil /*json*/, sourceFile, host, basePath, existingOptions, existingOptionsRaw, configFileName, resolutionStack, extraFileExtensions, extendedConfigCache) + result := parseJsonConfigFileContentWorker(nil /*json*/, sourceFile, host, basePath, existingOptions, existingOptionsRaw, configFileName, resolutionStack, extendedConfigCache) // tracing?.pop(); return result } @@ -870,8 +867,8 @@ func convertPropertyValueToJson(sourceFile *ast.SourceFile, valueExpression *ast // jsonNode: The contents of the config file to parse // host: Instance of ParseConfigHost used to enumerate files in folder. // basePath: A root directory to resolve relative path entries in the config file to. e.g. outDir -func ParseJsonConfigFileContent(json any, host ParseConfigHost, basePath string, existingOptions *core.CompilerOptions, configFileName string, resolutionStack []tspath.Path, extraFileExtensions []FileExtensionInfo, extendedConfigCache ExtendedConfigCache) *ParsedCommandLine { - result := parseJsonConfigFileContentWorker(parseJsonToStringKey(json), nil /*sourceFile*/, host, basePath, existingOptions, nil /*existingOptionsRaw*/, configFileName, resolutionStack, extraFileExtensions, extendedConfigCache) +func ParseJsonConfigFileContent(json any, host ParseConfigHost, basePath string, existingOptions *core.CompilerOptions, configFileName string, resolutionStack []tspath.Path, extendedConfigCache ExtendedConfigCache) *ParsedCommandLine { + result := parseJsonConfigFileContentWorker(parseJsonToStringKey(json), nil /*sourceFile*/, host, basePath, existingOptions, nil /*existingOptionsRaw*/, configFileName, resolutionStack, extendedConfigCache) return result } @@ -1192,7 +1189,6 @@ func parseJsonConfigFileContentWorker( existingOptionsRaw *collections.OrderedMap[string, any], configFileName string, resolutionStack []tspath.Path, - extraFileExtensions []FileExtensionInfo, extendedConfigCache ExtendedConfigCache, ) *ParsedCommandLine { debug.Assert((json == nil && sourceFile != nil) || (json != nil && sourceFile == nil)) @@ -1326,9 +1322,43 @@ func parseJsonConfigFileContentWorker( sourceFile.configFileSpecs = &configFileSpecs } + var contentMappers []*core.ContentMapper + contentMappersOfRaw := getPropFromRaw("contentMappers", func(element any) bool { return reflect.TypeOf(element) == orderedMapType }, "object") + for _, element := range contentMappersOfRaw.sliceValue { + mapper, mapperErrors := parseContentMapper(element) + errors = append(errors, mapperErrors...) + if mapper != nil { + contentMappers = append(contentMappers, mapper) + } + } + totalContentMapperExtensions := 0 + for _, mapper := range contentMappers { + totalContentMapperExtensions += len(mapper.Extensions) + } + seenContentMapperExtensions := make(map[string]struct{}, totalContentMapperExtensions) + contentMapperExtensions := make([]string, 0, totalContentMapperExtensions) + nativeExtensions := core.Flatten(tspath.AllSupportedExtensionsWithJson) + for _, mapper := range contentMappers { + for _, ext := range mapper.Extensions { + switch { + case !strings.HasPrefix(ext, "."): + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_must_begin_with_a, ext)) + case slices.Contains(nativeExtensions, ext): + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper, ext)) + default: + if _, seen := seenContentMapperExtensions[ext]; seen { + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper, ext)) + } else { + seenContentMapperExtensions[ext] = struct{}{} + contentMapperExtensions = append(contentMapperExtensions, ext) + } + } + } + } + getFileNames := func(basePath string) ([]string, int) { parsedConfigOptions := parsedConfig.options - fileNames, literalFileNamesLen := getFileNamesFromConfigSpecs(configFileSpecs, basePath, parsedConfigOptions, host.FS(), extraFileExtensions) + fileNames, literalFileNamesLen := getFileNamesFromConfigSpecs(configFileSpecs, basePath, parsedConfigOptions, host.FS(), contentMapperExtensions) if shouldReportNoInputFiles(fileNames, canJsonReportNoInputFiles(rawConfig), resolutionStack) { includeSpecs := configFileSpecs.includeSpecs excludeSpecs := configFileSpecs.excludeSpecs @@ -1375,12 +1405,12 @@ func parseJsonConfigFileContentWorker( // WatchOptions: nil, FileNames: fileNames, ProjectReferences: getProjectReferences(basePathForFileNames), + ContentMappers: contentMappers, }, ConfigFile: sourceFile, Raw: parsedConfig.raw, Errors: errors, - extraFileExtensions: extraFileExtensions, comparePathsOptions: tspath.ComparePathsOptions{ UseCaseSensitiveFileNames: host.FS().UseCaseSensitiveFileNames(), CurrentDirectory: basePathForFileNames, @@ -1665,15 +1695,14 @@ func removeWildcardFilesWithLowerPriorityExtension(file string, wildcardFiles *c // basePath is the base path for any relative file specifications. // options is the Compiler options. // host is the host used to resolve files and directories. -// extraFileExtensions optionally file extra file extension information from host +// extraExtensions are additional file extensions (e.g. from content mappers) to treat as supported. func getFileNamesFromConfigSpecs( configFileSpecs configFileSpecs, basePath string, // considering this is the current directory options *core.CompilerOptions, host vfs.FS, - extraFileExtensions []FileExtensionInfo, + extraExtensions []string, ) ([]string, int) { - extraFileExtensions = []FileExtensionInfo{} basePath = tspath.NormalizePath(basePath) keyMappper := func(value string) string { return tspath.GetCanonicalFileName(value, host.UseCaseSensitiveFileNames()) } // Literal file names (provided via the "files" array in tsconfig.json) are stored in a @@ -1693,7 +1722,7 @@ func getFileNamesFromConfigSpecs( validatedExcludeSpecs := configFileSpecs.validatedExcludeSpecs // Rather than re-query this for each file and filespec, we query the supported extensions // once and store it on the expansion context. - supportedExtensions := GetSupportedExtensions(options, extraFileExtensions) + supportedExtensions := GetSupportedExtensions(options, extraExtensions) supportedExtensionsWithJsonIfResolveJsonModule := GetSupportedExtensionsWithJsonIfResolveJsonModule(options, supportedExtensions) // Literal files are always included verbatim. An "include" or "exclude" specification cannot // remove a literal file. @@ -1759,30 +1788,28 @@ func getFileNamesFromConfigSpecs( return files, literalFileMap.Size() } -func GetSupportedExtensions(compilerOptions *core.CompilerOptions, extraFileExtensions []FileExtensionInfo) [][]string { +func GetSupportedExtensions(compilerOptions *core.CompilerOptions, extraExtensions []string) [][]string { needJSExtensions := compilerOptions.GetAllowJS() - if len(extraFileExtensions) == 0 { - if needJSExtensions { - return tspath.AllSupportedExtensions - } else { - return tspath.SupportedTSExtensions - } - } var builtins [][]string if needJSExtensions { builtins = tspath.AllSupportedExtensions } else { builtins = tspath.SupportedTSExtensions } + if len(extraExtensions) == 0 { + return builtins + } flatBuiltins := core.Flatten(builtins) var result [][]string - for _, x := range extraFileExtensions { - if x.ScriptKind == core.ScriptKindDeferred || (needJSExtensions && (x.ScriptKind == core.ScriptKindJS || x.ScriptKind == core.ScriptKindJSX)) && !slices.Contains(flatBuiltins, x.Extension) { - result = append(result, []string{x.Extension}) + for _, ext := range extraExtensions { + if !slices.Contains(flatBuiltins, ext) { + result = append(result, []string{ext}) } } - extensions := slices.Concat(builtins, result) - return extensions + if len(result) == 0 { + return builtins + } + return slices.Concat(builtins, result) } func GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions *core.CompilerOptions, supportedExtensions [][]string) [][]string { @@ -1836,7 +1863,6 @@ func GetParsedCommandLineOfConfigFilePath( optionsRaw, configFileName, nil, - nil, extendedConfigCache, ), nil } diff --git a/internal/tsoptions/tsconfigparsing_test.go b/internal/tsoptions/tsconfigparsing_test.go index a53b3a66d27..840ba644ef1 100644 --- a/internal/tsoptions/tsconfigparsing_test.go +++ b/internal/tsoptions/tsconfigparsing_test.go @@ -6,6 +6,7 @@ import ( "io/fs" "maps" "path/filepath" + "slices" "strings" "testing" @@ -826,7 +827,6 @@ func getParsedWithJsonApi(config testConfig, host tsoptions.ParseConfigHost, bas nil, configFileName, /*resolutionStack*/ nil, - /*extraFileExtensions*/ nil, /*extendedConfigCache*/ nil, ) } @@ -869,7 +869,6 @@ func TestParseJsonSourceFileConfigFileContentReportsInvalidExtendedConfig(t *tes configFileName, nil, nil, - nil, ) parseErrors := core.Filter(parsed.Errors, func(diagnostic *ast.Diagnostic) bool { @@ -914,7 +913,6 @@ func TestParseJsonSourceFileConfigFileContentWithEmptyExtendedConfig(t *testing. configFileName, nil, nil, - nil, ) assert.Assert(t, parsed != nil) @@ -1006,6 +1004,120 @@ func TestParseNullEnumCompilerOptions(t *testing.T) { } } +func TestContentMappers(t *testing.T) { + t.Parallel() + + config := testConfig{ + jsonText: `{ + "contentMappers": [ + { "command": ["vue-mapper"], "extensions": [".vue"] } + ], + "include": ["src"] + }`, + configFileName: "tsconfig.json", + basePath: "/", + allFileList: map[string]string{ + "/src/app.ts": "export {}", + "/src/Component.vue": "", + }, + } + for name, getParsed := range map[string]func(testConfig, tsoptions.ParseConfigHost, string) *tsoptions.ParsedCommandLine{ + "json api": getParsedWithJsonApi, + "jsonSourceFile api": getParsedWithJsonSourceFileApi, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + allFileLists := make(map[string]string, len(config.allFileList)+1) + maps.Copy(allFileLists, config.allFileList) + allFileLists["/tsconfig.json"] = config.jsonText + host := tsoptionstest.NewVFSParseConfigHost(allFileLists, config.basePath, true /*useCaseSensitiveFileNames*/) + parsed := getParsed(config, host, config.basePath) + + assert.Equal(t, len(parsed.Errors), 0) + + mappers := parsed.ContentMappers() + assert.Equal(t, len(mappers), 1) + assert.DeepEqual(t, mappers[0].Command, []string{"vue-mapper"}) + assert.DeepEqual(t, mappers[0].Extensions, []string{".vue"}) + assert.DeepEqual(t, parsed.ContentMapperExtensions(), []string{".vue"}) + + // The .vue file is picked up by the include glob because its extension is registered. + assert.Assert(t, slices.Contains(parsed.FileNames(), "/src/Component.vue"), "expected /src/Component.vue in %v", parsed.FileNames()) + assert.Assert(t, slices.Contains(parsed.FileNames(), "/src/app.ts"), "expected /src/app.ts in %v", parsed.FileNames()) + }) + } +} + +func TestContentMappersValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + contentMappers string + expectedCode int32 + }{ + { + name: "extension without leading dot", + contentMappers: `[{ "command": ["vue-mapper"], "extensions": ["vue"] }]`, + expectedCode: diagnostics.Content_mapper_file_extension_0_must_begin_with_a.Code(), + }, + { + name: "built-in extension", + contentMappers: `[{ "command": ["x"], "extensions": [".ts"] }]`, + expectedCode: diagnostics.Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper.Code(), + }, + { + name: "duplicate extension across mappers", + contentMappers: `[{ "command": ["a"], "extensions": [".vue"] }, { "command": ["b"], "extensions": [".vue"] }]`, + expectedCode: diagnostics.Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper.Code(), + }, + { + name: "extensions is not an array", + contentMappers: `[{ "command": ["x"], "extensions": ".vue" }]`, + expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), + }, + { + name: "extensions contains a non-string", + contentMappers: `[{ "command": ["x"], "extensions": [".vue", 1] }]`, + expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), + }, + { + name: "command is not an array", + contentMappers: `[{ "command": "x", "extensions": [".vue"] }]`, + expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + config := testConfig{ + jsonText: `{ "contentMappers": ` + test.contentMappers + ` }`, + configFileName: "tsconfig.json", + basePath: "/", + allFileList: map[string]string{"/app.ts": "export {}"}, + } + for apiName, getParsed := range map[string]func(testConfig, tsoptions.ParseConfigHost, string) *tsoptions.ParsedCommandLine{ + "json api": getParsedWithJsonApi, + "jsonSourceFile api": getParsedWithJsonSourceFileApi, + } { + t.Run(apiName, func(t *testing.T) { + t.Parallel() + allFileLists := map[string]string{"/tsconfig.json": config.jsonText} + maps.Copy(allFileLists, config.allFileList) + host := tsoptionstest.NewVFSParseConfigHost(allFileLists, config.basePath, true /*useCaseSensitiveFileNames*/) + parsed := getParsed(config, host, config.basePath) + found := slices.ContainsFunc(parsed.Errors, func(d *ast.Diagnostic) bool { + return d.Code() == test.expectedCode + }) + assert.Assert(t, found, "expected diagnostic %d, got errors: %v", test.expectedCode, parsed.Errors) + }) + } + }) + } +} + func getParsedWithJsonSourceFileApi(config testConfig, host tsoptions.ParseConfigHost, basePath string) *tsoptions.ParsedCommandLine { configFileName := tspath.GetNormalizedAbsolutePath(config.configFileName, basePath) path := tspath.ToPath(config.configFileName, basePath, host.FS().UseCaseSensitiveFileNames()) @@ -1024,7 +1136,6 @@ func getParsedWithJsonSourceFileApi(config testConfig, host tsoptions.ParseConfi nil, configFileName, /*resolutionStack*/ nil, - /*extraFileExtensions*/ nil, /*extendedConfigCache*/ nil, ) } @@ -1248,7 +1359,6 @@ func TestParseSrcCompiler(t *testing.T) { nil, tsconfigFileName, /*resolutionStack*/ nil, - /*extraFileExtensions*/ nil, /*extendedConfigCache*/ nil, ) @@ -1413,7 +1523,6 @@ func BenchmarkParseSrcCompiler(b *testing.B) { nil, tsconfigFileName, /*resolutionStack*/ nil, - /*extraFileExtensions*/ nil, /*extendedConfigCache*/ nil, ) } @@ -1474,7 +1583,6 @@ func TestExtendedConfigErrorsAppearOnCacheHit(t *testing.T) { nil, configFileName, nil, - nil, cache, ) } @@ -1519,7 +1627,6 @@ func TestExtendedConfigErrorsAppearOnCacheHit(t *testing.T) { nil, configFileName, nil, - nil, cache, ) } diff --git a/internal/tsoptions/tsoptionstest/parsedcommandline.go b/internal/tsoptions/tsoptionstest/parsedcommandline.go index c5dc7b1c203..02f5c99b845 100644 --- a/internal/tsoptions/tsoptionstest/parsedcommandline.go +++ b/internal/tsoptions/tsoptionstest/parsedcommandline.go @@ -10,5 +10,5 @@ func GetParsedCommandLine(t assert.TestingT, jsonText string, files map[string]s host := NewVFSParseConfigHost(files, currentDirectory, useCaseSensitiveFileNames) configFileName := tspath.CombinePaths(currentDirectory, "tsconfig.json") tsconfigSourceFile := tsoptions.NewTsconfigSourceFileFromFilePath(configFileName, tspath.ToPath(configFileName, currentDirectory, useCaseSensitiveFileNames), jsonText) - return tsoptions.ParseJsonSourceFileConfigFileContent(tsconfigSourceFile, host, currentDirectory, nil, nil, configFileName, nil, nil, nil) + return tsoptions.ParseJsonSourceFileConfigFileContent(tsconfigSourceFile, host, currentDirectory, nil, nil, configFileName, nil, nil) } From 7b5485c6e71d2d5382ddc05706736e1badd56916 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 9 Jul 2026 15:39:08 -0700 Subject: [PATCH 02/37] Add span mapping, wire into file loading and module resolution with mock test mappers --- internal/ast/ast.go | 51 ++- internal/ast/diagnostic.go | 31 +- internal/compiler/contentmapper_test.go | 351 ++++++++++++++++++ internal/compiler/contentmapperrunner.go | 31 ++ internal/compiler/fileloader.go | 184 ++++++++- internal/compiler/filesparser.go | 2 + internal/compiler/program.go | 16 +- internal/core/compileroptions.go | 41 +- internal/core/contentmapper.go | 18 + internal/diagnostics/diagnostics_generated.go | 36 ++ .../diagnostics/extraDiagnosticMessages.json | 36 ++ internal/diagnosticwriter/diagnosticwriter.go | 104 +++++- internal/fourslash/fourslash.go | 4 + .../ls/autoimport/aliasresolver_crash_test.go | 2 +- internal/ls/sourcedefinition.go | 2 +- internal/module/resolver.go | 24 +- internal/module/resolver_test.go | 8 +- internal/module/types.go | 17 +- internal/module/util.go | 4 + internal/project/ata/ata.go | 4 +- internal/spanmap/spanmap.go | 238 ++++++++++++ internal/spanmap/spanmap_test.go | 168 +++++++++ internal/tsoptions/declscompiler.go | 8 + internal/tsoptions/parsinghelpers.go | 2 + internal/tsoptions/tsconfigparsing.go | 3 + internal/tsoptions/tsconfigparsing_test.go | 52 ++- .../contentMapperDiagnostics.txt | 22 ++ ...entMapperDisabledAfterRepeatedFailures.txt | 72 ++++ .../contentMapperFileFailure.txt | 20 + .../contentMapperGeneratedCode.txt | 25 ++ .../contentMapperSpanMapping.txt | 24 ++ .../reference/tsc/commandLine/help-all.js | 3 + 32 files changed, 1531 insertions(+), 72 deletions(-) create mode 100644 internal/compiler/contentmapper_test.go create mode 100644 internal/compiler/contentmapperrunner.go create mode 100644 internal/spanmap/spanmap.go create mode 100644 internal/spanmap/spanmap_test.go create mode 100644 testdata/baselines/reference/contentMappers/contentMapperDiagnostics.txt create mode 100644 testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt create mode 100644 testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt create mode 100644 testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt create mode 100644 testdata/baselines/reference/contentMappers/contentMapperSpanMapping.txt diff --git a/internal/ast/ast.go b/internal/ast/ast.go index 6dcd606b57e..ef3ebd3fe47 100644 --- a/internal/ast/ast.go +++ b/internal/ast/ast.go @@ -9,6 +9,7 @@ import ( "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/spanmap" "github.com/microsoft/typescript-go/internal/stringutil" "github.com/microsoft/typescript-go/internal/tspath" "github.com/zeebo/xxh3" @@ -2471,8 +2472,11 @@ type SourceFile struct { fileName string // For debugging convenience parseOptions SourceFileParseOptions text string - Statements *NodeList // NodeList[*Statement] - EndOfFileToken *TokenNode // TokenNode[*EndOfFileToken] + originalText string // For content-mapped files, the untransformed source text; empty otherwise. + spanMap *spanmap.SpanMap // For content-mapped files, maps transformed positions back to the original text. + contentMapper string // For content-mapped files, the identity of the mapper that produced this file. + Statements *NodeList // NodeList[*Statement] + EndOfFileToken *TokenNode // TokenNode[*EndOfFileToken] // Fields for lazily-computed data owned by packages outside ast. dataMu sync.Mutex @@ -2565,6 +2569,49 @@ func (node *SourceFile) Text() string { return node.text } +// OriginalText returns the untransformed source text for content-mapped files, or Text() otherwise. +func (node *SourceFile) OriginalText() string { + if node.originalText != "" { + return node.originalText + } + return node.text +} + +// SetOriginalText records the untransformed source text of a content-mapped file, whose Text() holds +// the transformed TypeScript. +func (node *SourceFile) SetOriginalText(text string) { + node.originalText = text +} + +// CanMapToOriginal reports whether this file carries a span map, i.e. whether positions in its +// transformed Text() can be mapped back to the original content. +func (node *SourceFile) CanMapToOriginal() bool { + return node.spanMap != nil +} + +// ContentMapper returns the identity of the content mapper that produced this file, or "" if the file +// was not produced by a content mapper (or the mapper did not identify itself). +func (node *SourceFile) ContentMapper() string { + return node.contentMapper +} + +// SetContentMapper records the identity of the content mapper that produced this file. +func (node *SourceFile) SetContentMapper(identity string) { + node.contentMapper = identity +} + +// SetSpanMap records the span map that maps positions in this file's transformed Text() back to its +// original, untransformed content. +func (node *SourceFile) SetSpanMap(spanMap *spanmap.SpanMap) { + node.spanMap = spanMap +} + +// MapRangeToOriginal maps a range in this file's transformed Text() to the corresponding range in its +// original content, along with the fidelity of the result. Files without a span map map identically. +func (node *SourceFile) MapRangeToOriginal(r core.TextRange) (core.TextRange, spanmap.Fidelity) { + return node.spanMap.MapSpan(r) +} + func (node *SourceFile) FileName() string { return node.parseOptions.FileName } diff --git a/internal/ast/diagnostic.go b/internal/ast/diagnostic.go index 8eadefd4b20..fe49ac28215 100644 --- a/internal/ast/diagnostic.go +++ b/internal/ast/diagnostic.go @@ -35,8 +35,15 @@ type Diagnostic struct { loc core.TextRange code int32 category diagnostics.Category + // source, when non-empty, is a custom prefix (e.g. a content mapper's name) shown instead of "TS" + // before the code. It marks the diagnostic as coming from an external source whose ranges point + // into the file's original, untransformed text. + source string // Original message; may be nil. - message *diagnostics.Message + message *diagnostics.Message + // messageText is an already-localized message used when message is nil, e.g. a diagnostic + // deserialized from an external process that owns its own localization. + messageText string messageKey diagnostics.Key messageArgs []string messageChain []*Diagnostic @@ -54,6 +61,7 @@ func (d *Diagnostic) Len() int { return d.loc.L func (d *Diagnostic) Loc() core.TextRange { return d.loc } func (d *Diagnostic) Code() int32 { return d.code } func (d *Diagnostic) Category() diagnostics.Category { return d.category } +func (d *Diagnostic) Source() string { return d.source } func (d *Diagnostic) MessageKey() diagnostics.Key { return d.messageKey } func (d *Diagnostic) MessageArgs() []string { return d.messageArgs } func (d *Diagnostic) MessageChain() []*Diagnostic { return d.messageChain } @@ -99,11 +107,17 @@ func (d *Diagnostic) Clone() *Diagnostic { } func (d *Diagnostic) Localize(locale locale.Locale) string { + if d.message == nil && d.messageText != "" { + return d.messageText + } return diagnostics.Localize(locale, d.message, d.messageKey, d.messageArgs...) } // For debugging only. func (d *Diagnostic) String() string { + if d.message == nil && d.messageText != "" { + return d.messageText + } return diagnostics.Localize(locale.Default, d.message, d.messageKey, d.messageArgs...) } @@ -160,6 +174,21 @@ func NewCompilerDiagnostic(message *diagnostics.Message, args ...any) *Diagnosti return NewDiagnostic(nil, core.UndefinedTextRange(), message, args...) } +// NewExternalDiagnostic creates a diagnostic reported by an external source such as a content mapper. +// The message text is already localized (the external source owns localization) and the code is shown +// with the given source prefix (e.g. "vue") instead of "TS". The location refers to the file's original, +// untransformed content. +func NewExternalDiagnostic(file *SourceFile, loc core.TextRange, source string, category diagnostics.Category, code int32, messageText string) *Diagnostic { + return &Diagnostic{ + file: file, + loc: loc, + code: code, + category: category, + source: source, + messageText: messageText, + } +} + type DiagnosticsCollection struct { mu sync.Mutex count int diff --git a/internal/compiler/contentmapper_test.go b/internal/compiler/contentmapper_test.go new file mode 100644 index 00000000000..126b87615e8 --- /dev/null +++ b/internal/compiler/contentmapper_test.go @@ -0,0 +1,351 @@ +package compiler_test + +import ( + "context" + "errors" + "fmt" + "regexp" + "slices" + "strings" + "testing" + + "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/bundled" + "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/diagnostics" + "github.com/microsoft/typescript-go/internal/diagnosticwriter" + "github.com/microsoft/typescript-go/internal/spanmap" + "github.com/microsoft/typescript-go/internal/testutil/baseline" + "github.com/microsoft/typescript-go/internal/tsoptions" + "github.com/microsoft/typescript-go/internal/tspath" + "github.com/microsoft/typescript-go/internal/vfs/vfstest" + "gotest.tools/v3/assert" +) + +type fakeContentMapperRunner struct { + transform func(fileName string, content string) (compiler.ContentMapperResult, error) +} + +func (r fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { + return r.transform(fileName, content) +} + +const vueRawContent = "" + +// synthesizedSpanMap maps an entirely generated transformed text to no original location (a fully +// synthesized file). It satisfies the span map contract: it tiles the transformed text and its (empty) +// original span stays within the original content. +func synthesizedSpanMap(transformed string) *spanmap.SpanMap { + return spanmap.New([]spanmap.Segment{{ + GenStart: 0, + GenEnd: core.TextPos(len(transformed)), + OrigStart: 0, + OrigEnd: 0, + Kind: spanmap.KindSynthesized, + }}) +} + +func newContentMapperProgram(t *testing.T, runner compiler.ContentMapperRunner, files map[string]string, rootFiles []string) *compiler.Program { + t.Helper() + if !bundled.Embedded { + t.Skip("bundled files are not embedded") + } + fs := vfstest.FromMap[any](nil, false /*useCaseSensitiveFileNames*/) + fs = bundled.WrapFS(fs) + for name, content := range files { + _ = fs.WriteFile(name, content) + } + + config := &tsoptions.ParsedCommandLine{ + ParsedConfig: &core.ParsedOptions{ + FileNames: rootFiles, + CompilerOptions: &core.CompilerOptions{ + SkipLibCheck: core.TSTrue, + Module: core.ModuleKindESNext, + ModuleResolution: core.ModuleResolutionKindBundler, + }, + ContentMappers: []*core.ContentMapper{{Command: []string{"vue"}, Extensions: []string{".vue"}, Name: "vue-mapper", Version: "1.0.0"}}, + }, + } + return compiler.NewProgram(compiler.ProgramOptions{ + Config: config, + Host: compiler.NewCompilerHost("/src", fs, bundled.LibPath(), nil, nil), + ContentMapperRunner: runner, + // Load files on the calling goroutine for deterministic diagnostics ordering. + SingleThreaded: core.TSTrue, + }) +} + +func collectContentMapperDiagnostics(program *compiler.Program) []*ast.Diagnostic { + ctx := context.Background() + return slices.Concat( + program.GetSyntacticDiagnostics(ctx, nil), + program.GetSemanticDiagnostics(ctx, nil), + program.GetProgramDiagnostics(), + ) +} + +func TestContentMapperDiagnostics(t *testing.T) { + t.Parallel() + + // A mapper reports problems it finds in the file's content (e.g. a syntax error in the embedded + // script) as result.Diagnostics. A real runner deserializes these from another process: it does + // not have our diagnostics.Message values, so it builds each one from an already-localized message, + // a code namespaced by the mapper's own source prefix, and a range into the original content. + const componentVue = ` + + +` + files := map[string]string{ + "/src/app.ts": `import { greeting } from "./Component.vue"; +console.log(greeting);`, + "/src/Component.vue": componentVue, + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + // The mapper turns the \n" + scriptStart := strings.Index(componentVue, scriptBody) + + files := map[string]string{ + "/src/app.ts": `import { greeting } from "./Component.vue"; +console.log(greeting);`, + "/src/Component.vue": componentVue, + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{ + Text: scriptBody, + ScriptKind: core.ScriptKindTS, + Mappings: spanmap.New([]spanmap.Segment{{ + GenStart: 0, + GenEnd: core.TextPos(len(scriptBody)), + OrigStart: core.TextPos(scriptStart), + OrigEnd: core.TextPos(scriptStart + len(scriptBody)), + Kind: spanmap.KindVerbatim, + }}), + }, nil + }, + } + program := newContentMapperProgram(t, runner, files, []string{"/src/app.ts"}) + baseline.Run(t, "contentMapperSpanMapping.txt", contentMapperBaseline(program, collectContentMapperDiagnostics(program)), baseline.Options{Subfolder: "contentMappers"}) +} + +func TestContentMapperGeneratedCode(t *testing.T) { + t.Parallel() + + // Some transformed code is synthesized by the mapper and has no counterpart in the original file + // (e.g. a generated runtime call). A compiler error there cannot be pointed at the original, so it is + // shown against the transformed text with a note that the location is generated. + const transformed = "export const el = jsxRuntime(Widget);\n" + files := map[string]string{ + "/src/app.ts": `import "./Component.vue";`, + "/src/Component.vue": "\n", + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{ + Text: transformed, + ScriptKind: core.ScriptKindTS, + Mappings: synthesizedSpanMap(transformed), + }, nil + }, + } + program := newContentMapperProgram(t, runner, files, []string{"/src/app.ts"}) + baseline.Run(t, "contentMapperGeneratedCode.txt", contentMapperBaseline(program, collectContentMapperDiagnostics(program)), baseline.Options{Subfolder: "contentMappers"}) +} + +func TestContentMapperInvalidMappings(t *testing.T) { + t.Parallel() + + // Mappings are a required, enforced part of the content mapper contract. Each malformed map is + // attributed to the mapper with a specific diagnostic instead of surfacing untrustworthy positions. + const transformed = "export const x = 1;\n" + const original = "\n" + + verbatimAll := func(kind spanmap.Kind, origEnd int) *spanmap.SpanMap { + return spanmap.New([]spanmap.Segment{{ + GenStart: 0, GenEnd: core.TextPos(len(transformed)), + OrigStart: 0, OrigEnd: core.TextPos(origEnd), Kind: kind, + }}) + } + + testCases := []struct { + name string + mappings *spanmap.SpanMap + wantCode int32 + }{ + {"missing", nil, 100027}, + { + "coverage", + spanmap.New([]spanmap.Segment{{GenStart: 0, GenEnd: 3, OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}}), + 100028, + }, + { + "outOfBounds", + verbatimAll(spanmap.KindSynthesized, len(original)+50), + 100029, + }, + { + // A verbatim segment whose original text differs from the transformed text. + "verbatimMismatch", + verbatimAll(spanmap.KindVerbatim, len(transformed)), + 100030, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + files := map[string]string{ + "/src/app.ts": `import "./Component.vue";`, + "/src/Component.vue": original, + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{ + Text: transformed, + ScriptKind: core.ScriptKindTS, + Mappings: tc.mappings, + }, nil + }, + } + program := newContentMapperProgram(t, runner, files, []string{"/src/app.ts"}) + diags := collectContentMapperDiagnostics(program) + found := slices.ContainsFunc(diags, func(d *ast.Diagnostic) bool { return d.Code() == tc.wantCode }) + assert.Assert(t, found, "expected diagnostic TS%d attributing the invalid mapping, got: %v", tc.wantCode, diags) + }) + } +} + +func TestContentMapperFileFailure(t *testing.T) { + t.Parallel() + + files := map[string]string{ + "/src/app.ts": `import { greeting } from "./Component.vue"; +export const bad: number = greeting;`, + "/src/Component.vue": vueRawContent, + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{}, errors.New("broken pipe") + }, + } + program := newContentMapperProgram(t, runner, files, []string{"/src/app.ts"}) + baseline.Run(t, "contentMapperFileFailure.txt", contentMapperBaseline(program, collectContentMapperDiagnostics(program)), baseline.Options{Subfolder: "contentMappers"}) +} + +func TestContentMapperDisabledAfterRepeatedFailures(t *testing.T) { + t.Parallel() + + // A mapper that keeps failing is disabled after a bounded number of failures: the individual + // failures are reported, then a single program diagnostic notes the mapper was disabled, and the + // remaining files it would have handled are silently substituted with empty files. + files := map[string]string{ + "/src/app.ts": `import "./A.vue"; +import "./B.vue"; +import "./C.vue"; +import "./D.vue"; +import "./E.vue"; +import "./F.vue"; +import "./G.vue";`, + "/src/A.vue": vueRawContent, + "/src/B.vue": vueRawContent, + "/src/C.vue": vueRawContent, + "/src/D.vue": vueRawContent, + "/src/E.vue": vueRawContent, + "/src/F.vue": vueRawContent, + "/src/G.vue": vueRawContent, + } + runner := fakeContentMapperRunner{ + transform: func(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{}, errors.New("mapper protocol version mismatch") + }, + } + program := newContentMapperProgram(t, runner, files, []string{"/src/app.ts"}) + baseline.Run(t, "contentMapperDisabledAfterRepeatedFailures.txt", contentMapperBaseline(program, collectContentMapperDiagnostics(program)), baseline.Options{Subfolder: "contentMappers"}) +} + +// contentMapperBaseline renders the content-mapped source files (original and transformed text, script +// kind, and associated content mapper) followed by the program's diagnostics as the CLI would show them: +// with a source code frame for each diagnostic. Terminal color escapes are stripped so the baseline +// matches tsc output redirected to a file. +func contentMapperBaseline(program *compiler.Program, diagnostics []*ast.Diagnostic) string { + var b strings.Builder + formatOpts := &diagnosticwriter.FormattingOptions{ + NewLine: "\n", + ComparePathsOptions: tspath.ComparePathsOptions{ + CurrentDirectory: "/src", + UseCaseSensitiveFileNames: false, + }, + } + relative := func(fileName string) string { + return tspath.ConvertToRelativePath(fileName, formatOpts.ComparePathsOptions) + } + + b.WriteString("=== Content-mapped files ===\n") + for _, file := range program.GetSourceFiles() { + mapper := program.GetContentMapper(file) + if mapper == nil { + continue + } + fmt.Fprintf(&b, "\n//// [%s] (ScriptKind: %s, ContentMapper: %v)\n", relative(file.FileName()), file.ScriptKind, mapper.Extensions) + b.WriteString("--- Original ---\n") + b.WriteString(file.OriginalText()) + b.WriteString("\n--- Transformed ---\n") + b.WriteString(file.Text()) + if !strings.HasSuffix(file.Text(), "\n") { + b.WriteString("\n") + } + } + + b.WriteString("\n=== Diagnostics ===\n") + if len(diagnostics) == 0 { + b.WriteString("\n" + baseline.NoContent + "\n") + } else { + b.WriteString("\n") + var rendered strings.Builder + diagnosticwriter.FormatDiagnosticsWithColorAndContext(&rendered, diagnosticwriter.ToDiagnostics(diagnosticwriter.WrapASTDiagnostics(diagnostics)), formatOpts) + b.WriteString(ansiEscape.ReplaceAllString(rendered.String(), "")) + b.WriteString("\n") + } + return b.String() +} + +var ansiEscape = regexp.MustCompile("\x1b\\[[0-9;]*m") diff --git a/internal/compiler/contentmapperrunner.go b/internal/compiler/contentmapperrunner.go new file mode 100644 index 00000000000..4328326db89 --- /dev/null +++ b/internal/compiler/contentmapperrunner.go @@ -0,0 +1,31 @@ +package compiler + +import ( + "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/spanmap" +) + +// ContentMapperResult is the outcome of transforming a foreign file's content into TypeScript. +type ContentMapperResult struct { + // Text is the transformed TypeScript source text that is parsed into the program. + Text string + // ScriptKind is how Text should be parsed. + ScriptKind core.ScriptKind + // Diagnostics are syntax errors in the original content. + Diagnostics []*ast.Diagnostic + // Mappings maps positions in Text back to the original content, so that diagnostics the compiler + // produces against the transformed text can be reported at their original locations. A nil map + // means positions are used as-is. + Mappings *spanmap.SpanMap +} + +// ContentMapperRunner transforms foreign file content into TypeScript during program construction. +type ContentMapperRunner interface { + // Transform maps a foreign file's content to TypeScript. + // + // A non-nil error indicates the mapper itself failed to produce a result — for example the + // runner/coordinator hit a broken pipe, a process crash, or could not deserialize the mapper's + // response. + Transform(fileName string, content string) (result ContentMapperResult, err error) +} diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 94cba4244f6..088f0bbfa93 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -2,6 +2,7 @@ package compiler import ( "cmp" + "fmt" "slices" "strings" "sync" @@ -12,6 +13,8 @@ import ( "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/module" + "github.com/microsoft/typescript-go/internal/parser" + "github.com/microsoft/typescript-go/internal/spanmap" "github.com/microsoft/typescript-go/internal/tracing" "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" @@ -24,6 +27,10 @@ type libResolution struct { trace []module.DiagAndArgs } +// maxContentMapperFailures is the number of transform failures a single content mapper may accumulate +// before it is disabled for the rest of the program. +const maxContentMapperFailures = 5 + type LibFile struct { Name string path string @@ -42,6 +49,7 @@ type fileLoader struct { comparePathsOptions tspath.ComparePathsOptions supportedExtensions [][]string supportedExtensionsWithJsonIfResolveJsonModule [][]string + contentMapperExtensions []string filesParser *filesParser rootTasks []*parseTask @@ -57,6 +65,13 @@ type fileLoader struct { pathForLibFileCache collections.SyncMap[string, *LibFile] pathForLibFileResolutions collections.SyncMap[tspath.Path, *libResolution] + + // contentMapperMu guards the content-mapper bookkeeping below, which is written concurrently as + // content-mapped files are parsed across worker goroutines. + contentMapperMu sync.Mutex + contentMapperForFile map[tspath.Path]*core.ContentMapper + contentMapperFailures map[*core.ContentMapper]int + contentMapperDiagnostics []*ast.Diagnostic } type redirectsFile struct { @@ -111,7 +126,11 @@ type processedFiles struct { redirectTargetsMap map[tspath.Path][]string // filesByPath for redirect files redirectFilesByPath map[tspath.Path]*redirectsFile - finishedProcessing bool + // Association from a content-mapped source file path to the content mapper that produced it. + contentMapperForFile map[tspath.Path]*core.ContentMapper + // Program-level diagnostics reported when a content mapper fails fatally (reported once per mapper). + contentMapperDiagnostics []*ast.Diagnostic + finishedProcessing bool } type jsxRuntimeImportSpecifier struct { @@ -145,9 +164,10 @@ func processAllProgramFiles( rootTasks: make([]*parseTask, 0, len(rootFiles)+len(compilerOptions.Lib)), supportedExtensions: supportedExtensions, supportedExtensionsWithJsonIfResolveJsonModule: supportedExtensionsWithJsonIfResolveJsonModule, + contentMapperExtensions: opts.Config.ContentMapperExtensions(), } loader.addProjectReferenceTasks(singleThreaded) - loader.resolver = module.NewResolver(loader.projectReferenceFileMapper.host, compilerOptions, opts.TypingsLocation, opts.ProjectName) + loader.resolver = module.NewResolver(loader.projectReferenceFileMapper.host, compilerOptions, opts.TypingsLocation, opts.ProjectName, opts.Config.ContentMapperExtensions()) if opts.Tracing != nil { defer opts.Tracing.Push(tracing.PhaseProgram, "processRootFiles", map[string]any{"count": len(rootFiles)}, false)() } @@ -367,14 +387,168 @@ func (p *fileLoader) parseSourceFile(t *parseTask) *ast.SourceFile { } path := p.toPath(t.normalizedFilePath) options := p.projectReferenceFileMapper.getCompilerOptionsForFile(t) - sourceFile := p.opts.Host.GetSourceFile(ast.SourceFileParseOptions{ + parseOptions := ast.SourceFileParseOptions{ FileName: t.normalizedFilePath, Path: path, ExternalModuleIndicatorOptions: ast.GetExternalModuleIndicatorOptions(t.normalizedFilePath, options, t.metadata), - }) + } + if tspath.FileExtensionIsOneOf(t.normalizedFilePath, p.contentMapperExtensions) { + return p.parseContentMappedFile(parseOptions) + } + return p.opts.Host.GetSourceFile(parseOptions) +} + +// parseContentMappedFile reads a foreign file, transforms its content into TypeScript via the content +// mapper runner, and parses the result, preserving the original file name and retaining the +// untransformed text on the source file. Content mapper extensions only reach the parser when content +// mappers are configured, and configured content mappers require a runner; a content-mapped file that +// arrives without a runner is an invalid internal state and panics. +// +// When the runner returns an error the file is added with empty content and a per-file diagnostic +// describing the failure. To avoid drowning the output when a mapper is systematically broken (e.g. a +// version mismatch that makes it fail on every file), a mapper is disabled after maxContentMapperFailures +// failures: at that point a single program diagnostic reports that the mapper was disabled, and every +// subsequent file it would have handled is silently substituted with an empty file. It returns nil only +// if the file cannot be read. +func (p *fileLoader) parseContentMappedFile(opts ast.SourceFileParseOptions) *ast.SourceFile { + content, ok := p.opts.Host.FS().ReadFile(opts.FileName) + if !ok { + return nil + } + if p.opts.ContentMapperRunner == nil { + panic(fmt.Sprintf("content mapper runner is required to load content-mapped file %q", opts.FileName)) + } + mapper := p.matchContentMapper(opts.FileName) + p.recordContentMapper(opts.Path, mapper) + // The mapper's identity is part of its contract (reported at initialize); it is what we attribute + // failures to, so we use it directly rather than substituting anything when it is absent. + label := mapper.Identity() + if p.contentMapperDisabled(mapper) { + // The mapper already exceeded its failure budget; add the file empty without re-reporting. + return p.emptyContentMappedFile(opts, content) + } + result, err := p.opts.ContentMapperRunner.Transform(opts.FileName, content) + if err != nil { + sourceFile := p.emptyContentMappedFile(opts, content) + if p.recordContentMapperFailure(mapper, label) { + sourceFile.SetDiagnostics([]*ast.Diagnostic{ + ast.NewDiagnostic( + sourceFile, + core.NewTextRange(0, 0), + diagnostics.The_content_mapper_0_failed_to_transform_this_file_Colon_1, + label, + err.Error(), + ), + }) + } + return sourceFile + } + if problem := result.Mappings.Validate(result.Text, content); problem != nil { + sourceFile := p.emptyContentMappedFile(opts, content) + if p.recordContentMapperFailure(mapper, label) { + sourceFile.SetDiagnostics([]*ast.Diagnostic{contentMapperMappingDiagnostic(sourceFile, label, problem)}) + } + return sourceFile + } + sourceFile := parser.ParseSourceFile(opts, result.Text, result.ScriptKind) + if result.Text != content { + sourceFile.SetOriginalText(content) + } + sourceFile.SetSpanMap(result.Mappings) + sourceFile.SetContentMapper(label) + if len(result.Diagnostics) > 0 { + // The runner produces diagnostics without a source file (it doesn't have one yet); associate + // them with the file now so they are reported against it. + for _, diagnostic := range result.Diagnostics { + diagnostic.SetFile(sourceFile) + } + sourceFile.SetDiagnostics(append(sourceFile.Diagnostics(), result.Diagnostics...)) + } return sourceFile } +// contentMapperMappingDiagnostic builds the diagnostic reported against a mapper that produced an +// invalid span map, including the offsets involved so the mapper's author can locate the problem. +func contentMapperMappingDiagnostic(file *ast.SourceFile, label string, problem *spanmap.Problem) *ast.Diagnostic { + loc := core.NewTextRange(0, 0) + switch problem.Kind { + case spanmap.ProblemCoverage: + return ast.NewDiagnostic(file, loc, diagnostics.The_content_mapper_0_produced_position_mappings_that_do_not_cover_the_entire_transformed_output_near_output_offset_1, label, int(problem.GenPos)) + case spanmap.ProblemOutOfBounds: + return ast.NewDiagnostic(file, loc, diagnostics.The_content_mapper_0_produced_a_position_mapping_that_points_outside_the_original_content_original_offset_1, label, int(problem.OrigPos)) + case spanmap.ProblemVerbatimMismatch: + return ast.NewDiagnostic(file, loc, diagnostics.The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_offset_1_original_offset_2, label, int(problem.GenPos), int(problem.OrigPos)) + default: + return ast.NewDiagnostic(file, loc, diagnostics.The_content_mapper_0_did_not_provide_the_required_position_mappings, label) + } +} + +// emptyContentMappedFile produces an empty TypeScript source file for a content-mapped file whose +// transform could not be used, retaining the original content for diagnostics. Importers see it as an +// empty module rather than triggering a "cannot find module" error. +func (p *fileLoader) emptyContentMappedFile(opts ast.SourceFileParseOptions, content string) *ast.SourceFile { + sourceFile := parser.ParseSourceFile(opts, "", core.ScriptKindTS) + sourceFile.SetOriginalText(content) + return sourceFile +} + +// matchContentMapper returns the configured content mapper whose extensions include fileName. +func (p *fileLoader) matchContentMapper(fileName string) *core.ContentMapper { + for _, mapper := range p.opts.Config.ContentMappers() { + if tspath.FileExtensionIsOneOf(fileName, mapper.Extensions) { + return mapper + } + } + return nil +} + +// recordContentMapper associates a content-mapped file with the mapper that produced it, so the +// program can report the mapping without re-matching extensions. +func (p *fileLoader) recordContentMapper(path tspath.Path, mapper *core.ContentMapper) { + if mapper == nil { + return + } + p.contentMapperMu.Lock() + defer p.contentMapperMu.Unlock() + if p.contentMapperForFile == nil { + p.contentMapperForFile = make(map[tspath.Path]*core.ContentMapper) + } + p.contentMapperForFile[path] = mapper +} + +// contentMapperDisabled reports whether mapper has exceeded its failure budget and been disabled. +func (p *fileLoader) contentMapperDisabled(mapper *core.ContentMapper) bool { + if mapper == nil { + return false + } + p.contentMapperMu.Lock() + defer p.contentMapperMu.Unlock() + return p.contentMapperFailures[mapper] >= maxContentMapperFailures +} + +// recordContentMapperFailure counts a transform failure for mapper. It returns whether the failure +// should be reported for this file (false once the mapper is already disabled). On the failure that +// reaches maxContentMapperFailures it appends a single program diagnostic disabling the mapper. +func (p *fileLoader) recordContentMapperFailure(mapper *core.ContentMapper, label string) bool { + p.contentMapperMu.Lock() + defer p.contentMapperMu.Unlock() + if p.contentMapperFailures == nil { + p.contentMapperFailures = make(map[*core.ContentMapper]int) + } + if p.contentMapperFailures[mapper] >= maxContentMapperFailures { + return false + } + p.contentMapperFailures[mapper]++ + if p.contentMapperFailures[mapper] >= maxContentMapperFailures { + p.contentMapperDiagnostics = append(p.contentMapperDiagnostics, ast.NewCompilerDiagnostic( + diagnostics.The_content_mapper_0_failed_1_times_and_will_not_be_used, + label, + maxContentMapperFailures, + )) + } + return true +} + func (p *fileLoader) isSupportedExtension(canonicalFileName string) bool { for _, group := range p.supportedExtensionsWithJsonIfResolveJsonModule { if tspath.FileExtensionIsOneOf(canonicalFileName, group) { @@ -591,7 +765,7 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(t *parseTask) { resolvedFileName := resolvedModule.ResolvedFileName isFromNodeModulesSearch := resolvedModule.IsExternalLibraryImport // Don't treat redirected files as JS files. - isJsFile := !tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat) && p.projectReferenceFileMapper.getRedirectParsedCommandLineForResolution(ast.NewHasFileName(resolvedFileName, p.toPath(resolvedFileName))) == nil + isJsFile := !resolvedModule.ResolvedUsingExtraExtensions && !tspath.FileExtensionIsOneOf(resolvedFileName, tspath.SupportedTSExtensionsWithJsonFlat) && p.projectReferenceFileMapper.getRedirectParsedCommandLineForResolution(ast.NewHasFileName(resolvedFileName, p.toPath(resolvedFileName))) == nil isJsFileFromNodeModules := isFromNodeModulesSearch && isJsFile && strings.Contains(resolvedFileName, "/node_modules/") // add file to program only if: diff --git a/internal/compiler/filesparser.go b/internal/compiler/filesparser.go index 5817fc6bb86..c46dd158fde 100644 --- a/internal/compiler/filesparser.go +++ b/internal/compiler/filesparser.go @@ -551,6 +551,8 @@ func (w *filesParser) getProcessedFiles(loader *fileLoader) processedFiles { outputFileToProjectReferenceSource: outputFileToProjectReferenceSource, redirectTargetsMap: redirectTargetsMap, redirectFilesByPath: redirectFilesByPath, + contentMapperForFile: loader.contentMapperForFile, + contentMapperDiagnostics: loader.contentMapperDiagnostics, } } diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 1dd71306054..0487535fe0d 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -41,6 +41,7 @@ type ProgramOptions struct { TypingsLocation string ProjectName string Tracing *tracing.Tracing + ContentMapperRunner ContentMapperRunner } func (p *ProgramOptions) canUseProjectReferenceSource() bool { @@ -402,9 +403,15 @@ func equalCheckJSDirectives(d1 *ast.CheckJsDirective, d2 *ast.CheckJsDirective) func (p *Program) SourceFiles() []*ast.SourceFile { return p.files } func (p *Program) DuplicateSourceFiles() []*DuplicateSourceFile { return p.duplicateSourceFiles } func (p *Program) Options() *core.CompilerOptions { return p.opts.Config.CompilerOptions() } -func (p *Program) CommandLine() *tsoptions.ParsedCommandLine { return p.opts.Config } -func (p *Program) Host() CompilerHost { return p.opts.Host } -func (p *Program) Tracing() *tracing.Tracing { return p.opts.Tracing } + +// GetContentMapper returns the content mapper that produced the given source file, or nil if the +// file was not produced by a content mapper. +func (p *Program) GetContentMapper(file *ast.SourceFile) *core.ContentMapper { + return p.contentMapperForFile[file.Path()] +} +func (p *Program) CommandLine() *tsoptions.ParsedCommandLine { return p.opts.Config } +func (p *Program) Host() CompilerHost { return p.opts.Host } +func (p *Program) Tracing() *tracing.Tracing { return p.opts.Tracing } func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic { return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics()) } @@ -674,8 +681,9 @@ func (p *Program) GetSuggestionDiagnostics(ctx context.Context, sourceFile *ast. } func (p *Program) GetProgramDiagnostics() []*ast.Diagnostic { - return SortAndDeduplicateDiagnostics(core.Concatenate( + return SortAndDeduplicateDiagnostics(slices.Concat( p.programDiagnostics, + p.contentMapperDiagnostics, p.includeProcessor.getDiagnostics(p).GetGlobalDiagnostics(), )) } diff --git a/internal/core/compileroptions.go b/internal/core/compileroptions.go index b079097c540..c4f87e3e509 100644 --- a/internal/core/compileroptions.go +++ b/internal/core/compileroptions.go @@ -133,26 +133,27 @@ type CompilerOptions struct { OutFile string `json:"outFile,omitzero"` // Internal fields - ConfigFilePath string `json:"configFilePath,omitzero"` - NoDtsResolution Tristate `json:"noDtsResolution,omitzero"` - PathsBasePath string `json:"pathsBasePath,omitzero"` - Diagnostics Tristate `json:"diagnostics,omitzero"` - ExtendedDiagnostics Tristate `json:"extendedDiagnostics,omitzero"` - GenerateCpuProfile string `json:"generateCpuProfile,omitzero"` - GenerateTrace string `json:"generateTrace,omitzero"` - ListEmittedFiles Tristate `json:"listEmittedFiles,omitzero"` - ListFiles Tristate `json:"listFiles,omitzero"` - ExplainFiles Tristate `json:"explainFiles,omitzero"` - ListFilesOnly Tristate `json:"listFilesOnly,omitzero"` - NoEmitForJsFiles Tristate `json:"noEmitForJsFiles,omitzero"` - PreserveWatchOutput Tristate `json:"preserveWatchOutput,omitzero"` - Pretty Tristate `json:"pretty,omitzero"` - Version Tristate `json:"version,omitzero"` - Watch Tristate `json:"watch,omitzero"` - ShowConfig Tristate `json:"showConfig,omitzero"` - Build Tristate `json:"build,omitzero"` - Help Tristate `json:"help,omitzero"` - All Tristate `json:"all,omitzero"` + ConfigFilePath string `json:"configFilePath,omitzero"` + NoDtsResolution Tristate `json:"noDtsResolution,omitzero"` + PathsBasePath string `json:"pathsBasePath,omitzero"` + Diagnostics Tristate `json:"diagnostics,omitzero"` + ExtendedDiagnostics Tristate `json:"extendedDiagnostics,omitzero"` + GenerateCpuProfile string `json:"generateCpuProfile,omitzero"` + GenerateTrace string `json:"generateTrace,omitzero"` + ListEmittedFiles Tristate `json:"listEmittedFiles,omitzero"` + ListFiles Tristate `json:"listFiles,omitzero"` + ExplainFiles Tristate `json:"explainFiles,omitzero"` + ListFilesOnly Tristate `json:"listFilesOnly,omitzero"` + NoEmitForJsFiles Tristate `json:"noEmitForJsFiles,omitzero"` + PreserveWatchOutput Tristate `json:"preserveWatchOutput,omitzero"` + Pretty Tristate `json:"pretty,omitzero"` + Version Tristate `json:"version,omitzero"` + Watch Tristate `json:"watch,omitzero"` + ShowConfig Tristate `json:"showConfig,omitzero"` + Build Tristate `json:"build,omitzero"` + Help Tristate `json:"help,omitzero"` + All Tristate `json:"all,omitzero"` + DangerouslyLoadExternalPlugins Tristate `json:"dangerouslyLoadExternalPlugins,omitzero"` PprofDir string `json:"pprofDir,omitzero"` SingleThreaded Tristate `json:"singleThreaded,omitzero"` diff --git a/internal/core/contentmapper.go b/internal/core/contentmapper.go index 7e155a3b191..a472ad4aad5 100644 --- a/internal/core/contentmapper.go +++ b/internal/core/contentmapper.go @@ -6,4 +6,22 @@ package core type ContentMapper struct { Command []string `json:"command"` Extensions []string `json:"extensions"` + // Name and Version identify the mapper implementation. They are reported by the mapper itself + // (e.g. during an initialize handshake) rather than declared in the tsconfig, and are used to + // attribute diagnostics to a specific mapper. + Name string `json:"-"` + Version string `json:"-"` +} + +// Identity returns a human-readable "name@version" identifier for the mapper, or just the name when no +// version is reported, or an empty string when the mapper has not identified itself. +func (m *ContentMapper) Identity() string { + switch { + case m.Name == "": + return "" + case m.Version == "": + return m.Name + default: + return m.Name + "@" + m.Version + } } diff --git a/internal/diagnostics/diagnostics_generated.go b/internal/diagnostics/diagnostics_generated.go index cfd424061df..d790ed0144a 100644 --- a/internal/diagnostics/diagnostics_generated.go +++ b/internal/diagnostics/diagnostics_generated.go @@ -4316,6 +4316,24 @@ var Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_regist var Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper = &Message{code: 100022, category: CategoryError, key: "Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper_100022", text: "Content mapper file extension '{0}' is registered by more than one content mapper."} +var Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation = &Message{code: 100023, category: CategoryMessage, key: "Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation_100023", text: "Allow loading external content mapper plugins that execute code during compilation."} + +var Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled = &Message{code: 100024, category: CategoryError, key: "Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled_100024", text: "Content mappers require the '--dangerouslyLoadExternalPlugins' command line flag to be enabled."} + +var The_content_mapper_0_failed_to_transform_this_file_Colon_1 = &Message{code: 100025, category: CategoryError, key: "The_content_mapper_0_failed_to_transform_this_file_Colon_1_100025", text: "The content mapper '{0}' failed to transform this file: {1}"} + +var The_content_mapper_0_failed_1_times_and_will_not_be_used = &Message{code: 100026, category: CategoryError, key: "The_content_mapper_0_failed_1_times_and_will_not_be_used_100026", text: "The content mapper '{0}' failed {1} times and will not be used."} + +var The_content_mapper_0_did_not_provide_the_required_position_mappings = &Message{code: 100027, category: CategoryError, key: "The_content_mapper_0_did_not_provide_the_required_position_mappings_100027", text: "The content mapper '{0}' did not provide the required position mappings."} + +var The_content_mapper_0_produced_position_mappings_that_do_not_cover_the_entire_transformed_output_near_output_offset_1 = &Message{code: 100028, category: CategoryError, key: "The_content_mapper_0_produced_position_mappings_that_do_not_cover_the_entire_transformed_output_near_100028", text: "The content mapper '{0}' produced position mappings that do not cover the entire transformed output (near output offset {1})."} + +var The_content_mapper_0_produced_a_position_mapping_that_points_outside_the_original_content_original_offset_1 = &Message{code: 100029, category: CategoryError, key: "The_content_mapper_0_produced_a_position_mapping_that_points_outside_the_original_content_original_o_100029", text: "The content mapper '{0}' produced a position mapping that points outside the original content (original offset {1})."} + +var The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_offset_1_original_offset_2 = &Message{code: 100030, category: CategoryError, key: "The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_off_100030", text: "The content mapper '{0}' produced a verbatim mapping that does not match the original content (output offset {1}, original offset {2})."} + +var This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the_original_file = &Message{code: 100031, category: CategoryMessage, key: "This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the__100031", text: "This location is in code generated by the content mapper '{0}' and has no corresponding location in the original file."} + func keyToMessage(key Key) *Message { switch key { case "Unterminated_string_literal_1002": @@ -8632,6 +8650,24 @@ func keyToMessage(key Key) *Message { return Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper case "Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper_100022": return Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper + case "Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation_100023": + return Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation + case "Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled_100024": + return Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled + case "The_content_mapper_0_failed_to_transform_this_file_Colon_1_100025": + return The_content_mapper_0_failed_to_transform_this_file_Colon_1 + case "The_content_mapper_0_failed_1_times_and_will_not_be_used_100026": + return The_content_mapper_0_failed_1_times_and_will_not_be_used + case "The_content_mapper_0_did_not_provide_the_required_position_mappings_100027": + return The_content_mapper_0_did_not_provide_the_required_position_mappings + case "The_content_mapper_0_produced_position_mappings_that_do_not_cover_the_entire_transformed_output_near_100028": + return The_content_mapper_0_produced_position_mappings_that_do_not_cover_the_entire_transformed_output_near_output_offset_1 + case "The_content_mapper_0_produced_a_position_mapping_that_points_outside_the_original_content_original_o_100029": + return The_content_mapper_0_produced_a_position_mapping_that_points_outside_the_original_content_original_offset_1 + case "The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_off_100030": + return The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_offset_1_original_offset_2 + case "This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the__100031": + return This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the_original_file default: return nil } diff --git a/internal/diagnostics/extraDiagnosticMessages.json b/internal/diagnostics/extraDiagnosticMessages.json index af395934784..90a2deed438 100644 --- a/internal/diagnostics/extraDiagnosticMessages.json +++ b/internal/diagnostics/extraDiagnosticMessages.json @@ -142,5 +142,41 @@ "Content mapper file extension '{0}' is registered by more than one content mapper.": { "category": "Error", "code": 100022 + }, + "Allow loading external content mapper plugins that execute code during compilation.": { + "category": "Message", + "code": 100023 + }, + "Content mappers require the '--dangerouslyLoadExternalPlugins' command line flag to be enabled.": { + "category": "Error", + "code": 100024 + }, + "The content mapper '{0}' failed to transform this file: {1}": { + "category": "Error", + "code": 100025 + }, + "The content mapper '{0}' failed {1} times and will not be used.": { + "category": "Error", + "code": 100026 + }, + "The content mapper '{0}' did not provide the required position mappings.": { + "category": "Error", + "code": 100027 + }, + "The content mapper '{0}' produced position mappings that do not cover the entire transformed output (near output offset {1}).": { + "category": "Error", + "code": 100028 + }, + "The content mapper '{0}' produced a position mapping that points outside the original content (original offset {1}).": { + "category": "Error", + "code": 100029 + }, + "The content mapper '{0}' produced a verbatim mapping that does not match the original content (output offset {1}, original offset {2}).": { + "category": "Error", + "code": 100030 + }, + "This location is in code generated by the content mapper '{0}' and has no corresponding location in the original file.": { + "category": "Message", + "code": 100031 } } diff --git a/internal/diagnosticwriter/diagnosticwriter.go b/internal/diagnosticwriter/diagnosticwriter.go index f038badc6dc..b056574cc9d 100644 --- a/internal/diagnosticwriter/diagnosticwriter.go +++ b/internal/diagnosticwriter/diagnosticwriter.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/locale" "github.com/microsoft/typescript-go/internal/scanner" + "github.com/microsoft/typescript-go/internal/spanmap" "github.com/microsoft/typescript-go/internal/tspath" ) @@ -31,6 +32,9 @@ type Diagnostic interface { Len() int Code() int32 Category() diagnostics.Category + // Source is a custom prefix shown before the code instead of "TS" (empty means "TS"). A non-empty + // value marks the diagnostic as coming from an external source (e.g. a content mapper). + Source() string Localize(locale locale.Locale) string MessageChain() []Diagnostic RelatedInformation() []Diagnostic @@ -51,17 +55,92 @@ func (d *ASTDiagnostic) RelatedInformation() []Diagnostic { } func (d *ASTDiagnostic) File() FileLike { - if file := d.Diagnostic.File(); file != nil { - return file + file := d.Diagnostic.File() + if file == nil { + return nil + } + if d.resolve().useOriginal { + // The mapper's own diagnostics (Source != "") already carry original ranges; compiler + // diagnostics have their transformed ranges mapped back. Both render against the original, + // untransformed text. Diagnostics in generated code (see resolve) keep the transformed text. + return newOriginalTextFile(file) + } + return file +} + +func (d *ASTDiagnostic) Source() string { + return d.Diagnostic.Source() +} + +func (d *ASTDiagnostic) Pos() int { return d.resolve().loc.Pos() } +func (d *ASTDiagnostic) End() int { return d.resolve().loc.End() } +func (d *ASTDiagnostic) Len() int { return d.resolve().loc.Len() } + +// resolvedLocation describes how a diagnostic on a content-mapped file should be reported. +type resolvedLocation struct { + loc core.TextRange + useOriginal bool // render against the file's original, untransformed text + generated bool // the range is in generated code with no corresponding original location +} + +// resolve determines where and against which text a diagnostic should be reported. A content mapper's +// own diagnostics already carry original ranges. A compiler diagnostic on a content-mapped file has its +// transformed range mapped back to the original; if it falls entirely within generated code, there is no +// original location, so it is shown against the transformed text and flagged as generated. +func (d *ASTDiagnostic) resolve() resolvedLocation { + loc := d.Diagnostic.Loc() + file := d.Diagnostic.File() + switch { + case file == nil: + return resolvedLocation{loc: loc} + case d.Diagnostic.Source() != "": + return resolvedLocation{loc: loc, useOriginal: true} + case file.CanMapToOriginal(): + mapped, fidelity := file.MapRangeToOriginal(loc) + if fidelity == spanmap.FidelityNone { + return resolvedLocation{loc: loc, generated: true} + } + return resolvedLocation{loc: mapped, useOriginal: true} + default: + return resolvedLocation{loc: loc} + } +} + +// originalTextFile presents a source file's original (untransformed) text as a FileLike, so that +// diagnostics whose ranges point into that text render at the correct locations. +type originalTextFile struct { + fileName string + text string + lineMap []core.TextPos +} + +func newOriginalTextFile(file *ast.SourceFile) *originalTextFile { + text := file.OriginalText() + return &originalTextFile{ + fileName: file.FileName(), + text: text, + lineMap: []core.TextPos(core.ComputeECMALineStarts(text)), } - return nil } +func (f *originalTextFile) FileName() string { return f.fileName } +func (f *originalTextFile) Text() string { return f.text } +func (f *originalTextFile) ECMALineMap() []core.TextPos { return f.lineMap } + func (d *ASTDiagnostic) MessageChain() []Diagnostic { chain := d.Diagnostic.MessageChain() - result := make([]Diagnostic, len(chain)) - for i, c := range chain { - result[i] = &ASTDiagnostic{c} + result := make([]Diagnostic, 0, len(chain)+1) + for _, c := range chain { + result = append(result, &ASTDiagnostic{c}) + } + if d.resolve().generated { + // The diagnostic points into generated code; make clear the shown location is not in the + // original file, and which content mapper generated it. + note := ast.NewCompilerDiagnostic( + diagnostics.This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the_original_file, + d.Diagnostic.File().ContentMapper(), + ) + result = append(result, WrapASTDiagnostic(note)) } return result } @@ -140,7 +219,7 @@ func FormatDiagnosticWithColorAndContext(output io.Writer, diagnostic Diagnostic } writeWithStyleAndReset(output, diagnostic.Category().Name(), getCategoryFormat(diagnostic.Category())) - fmt.Fprintf(output, "%s TS%d: %s", foregroundColorEscapeGrey, diagnostic.Code(), resetEscapeSequence) + fmt.Fprintf(output, "%s %s%d: %s", foregroundColorEscapeGrey, diagnosticPrefix(diagnostic), diagnostic.Code(), resetEscapeSequence) WriteFlattenedDiagnosticMessage(output, diagnostic, formatOpts.NewLine, formatOpts.Locale) if diagnostic.File() != nil && diagnostic.Code() != diagnostics.File_appears_to_be_binary.Code() { @@ -280,6 +359,15 @@ func flattenDiagnosticMessageChain(writer io.Writer, chain Diagnostic, newLine s } } +// diagnosticPrefix returns the prefix shown before a diagnostic's code, e.g. "TS" for compiler +// diagnostics or a content mapper's custom source for its diagnostics. +func diagnosticPrefix(diagnostic Diagnostic) string { + if source := diagnostic.Source(); source != "" { + return source + } + return "TS" +} + func getCategoryFormat(category diagnostics.Category) string { switch category { case diagnostics.CategoryError: @@ -472,7 +560,7 @@ func WriteFormatDiagnostic(output io.Writer, diagnostic Diagnostic, formatOpts * fmt.Fprintf(output, "%s(%d,%d): ", relativeFileName, line+1, int(character)+1) } - fmt.Fprintf(output, "%s TS%d: ", diagnostic.Category().Name(), diagnostic.Code()) + fmt.Fprintf(output, "%s %s%d: ", diagnostic.Category().Name(), diagnosticPrefix(diagnostic), diagnostic.Code()) WriteFlattenedDiagnosticMessage(output, diagnostic, formatOpts.NewLine, formatOpts.Locale) fmt.Fprint(output, formatOpts.NewLine) } diff --git a/internal/fourslash/fourslash.go b/internal/fourslash/fourslash.go index dcd76dbedfa..512fbfed4a7 100644 --- a/internal/fourslash/fourslash.go +++ b/internal/fourslash/fourslash.go @@ -5352,6 +5352,10 @@ func (d *fourslashDiagnostic) Category() diagnostics.Category { return d.category } +func (d *fourslashDiagnostic) Source() string { + return "" +} + func (d *fourslashDiagnostic) Localize(locale locale.Locale) string { return d.message } diff --git a/internal/ls/autoimport/aliasresolver_crash_test.go b/internal/ls/autoimport/aliasresolver_crash_test.go index 897261c15d2..22a48bd1e8a 100644 --- a/internal/ls/autoimport/aliasresolver_crash_test.go +++ b/internal/ls/autoimport/aliasresolver_crash_test.go @@ -56,7 +56,7 @@ func TestAliasResolverGetDiagnosticsDoesNotPanic(t *testing.T) { }, text, core.ScriptKindTS) binder.BindSourceFile(sourceFile) - resolver := module.NewResolver(host, core.EmptyCompilerOptions, "", "") + resolver := module.NewResolver(host, core.EmptyCompilerOptions, "", "", nil) r := newAliasResolver( []*ast.SourceFile{sourceFile}, nil, diff --git a/internal/ls/sourcedefinition.go b/internal/ls/sourcedefinition.go index 023e6da1a59..49d5dc8e222 100644 --- a/internal/ls/sourcedefinition.go +++ b/internal/ls/sourcedefinition.go @@ -131,7 +131,7 @@ func (l *LanguageService) newSourceDefResolver( options: options, getSourceFile: program.GetSourceFile, resolveFrom: resolveFrom, - resolver: module.NewResolver(program.Host(), noDtsOptions, program.GetGlobalTypingsCacheLocation(), ""), + resolver: module.NewResolver(program.Host(), noDtsOptions, program.GetGlobalTypingsCacheLocation(), "", program.CommandLine().ContentMapperExtensions()), } } diff --git a/internal/module/resolver.go b/internal/module/resolver.go index 52400dc2b98..26ac09c686a 100644 --- a/internal/module/resolver.go +++ b/internal/module/resolver.go @@ -18,11 +18,12 @@ import ( ) type resolved struct { - path string - extension string - packageId PackageId - originalPath string - resolvedUsingTsExtension bool + path string + extension string + packageId PackageId + originalPath string + resolvedUsingTsExtension bool + resolvedUsingExtraExtensions bool } func (r *resolved) shouldContinueSearching() bool { @@ -158,6 +159,7 @@ type Resolver struct { compilerOptions *core.CompilerOptions typingsLocation string projectName string + extraExtensions []string // reportDiagnostic: DiagnosticReporter } @@ -170,6 +172,7 @@ func NewResolver( options *core.CompilerOptions, typingsLocation string, projectName string, + extraExtensions []string, ) *Resolver { return &Resolver{ host: host, @@ -177,6 +180,7 @@ func NewResolver( compilerOptions: options, typingsLocation: typingsLocation, projectName: projectName, + extraExtensions: extraExtensions, } } @@ -1178,6 +1182,7 @@ func (r *resolutionState) createResolvedModule(resolved *resolved, isExternalLib resolvedModule.OriginalPath = resolved.originalPath resolvedModule.IsExternalLibraryImport = isExternalLibraryImport resolvedModule.ResolvedUsingTsExtension = resolved.resolvedUsingTsExtension + resolvedModule.ResolvedUsingExtraExtensions = resolved.resolvedUsingExtraExtensions resolvedModule.Extension = resolved.extension resolvedModule.PackageId = resolved.packageId } @@ -1558,6 +1563,13 @@ func (r *resolutionState) tryAddingExtensions(extensionless string, extensions e } return continueSearching() default: + if slices.Contains(r.resolver.extraExtensions, originalExtension) { + // A fully specified import of an extraExtension resolves directly to the file. + if resolved := r.tryExtension(originalExtension, extensionless, false); !resolved.shouldContinueSearching() { + resolved.resolvedUsingExtraExtensions = true + return resolved + } + } if extensions&extensionsDeclaration != 0 && !tspath.IsDeclarationFileName(extensionless+originalExtension) { if resolved := r.tryExtension(".d"+originalExtension+".ts", extensionless, false); !resolved.shouldContinueSearching() { return resolved @@ -2075,7 +2087,7 @@ func extensionIsOk(extensions extensions, extension string) bool { } func ResolveConfig(moduleName string, containingFile string, host ResolutionHost) *ResolvedModule { - resolver := NewResolver(host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") + resolver := NewResolver(host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "", nil) return resolver.resolveConfig(moduleName, containingFile) } diff --git a/internal/module/resolver_test.go b/internal/module/resolver_test.go index c15209769ad..9902713f15d 100644 --- a/internal/module/resolver_test.go +++ b/internal/module/resolver_test.go @@ -39,7 +39,7 @@ func TestResolveModuleNameTrailingSlash(t *testing.T) { Module: core.ModuleKindESNext, Target: core.ScriptTargetESNext, } - resolver := module.NewResolver(host, opts, "", "") + resolver := module.NewResolver(host, opts, "", "", nil) for _, name := range []string{"pkg", "pkg/"} { r, _ := resolver.ResolveModuleName(name, "/repo/src/file.ts", core.ModuleKindESNext, nil) @@ -168,7 +168,7 @@ func TestResolveModuleNameTrailingSlashRace(t *testing.T) { Module: core.ModuleKindESNext, Target: core.ScriptTargetESNext, } - resolver := module.NewResolver(host, opts, "", "") + resolver := module.NewResolver(host, opts, "", "", nil) type resolutionResult struct { name string @@ -240,7 +240,7 @@ func TestResolveSubpathNilContentsRace(t *testing.T) { Module: core.ModuleKindESNext, Target: core.ScriptTargetESNext, } - resolver := module.NewResolver(host, opts, "", "") + resolver := module.NewResolver(host, opts, "", "", nil) var panicked atomic.Bool type resolutionResult struct { @@ -363,7 +363,7 @@ func TestResolvePeerDependencyNilContentsRace(t *testing.T) { Module: core.ModuleKindESNext, Target: core.ScriptTargetESNext, } - resolver := module.NewResolver(host, opts, "", "") + resolver := module.NewResolver(host, opts, "", "", nil) var panicked atomic.Bool type resolutionResult struct { diff --git a/internal/module/types.go b/internal/module/types.go index 32d18530295..0662959bc1c 100644 --- a/internal/module/types.go +++ b/internal/module/types.go @@ -63,14 +63,15 @@ func (p *PackageId) PackageName() string { } type ResolvedModule struct { - ResolutionDiagnostics []*ast.Diagnostic - ResolvedFileName string - OriginalPath string - Extension string - ResolvedUsingTsExtension bool - PackageId PackageId - IsExternalLibraryImport bool - AlternateResult string + ResolutionDiagnostics []*ast.Diagnostic + ResolvedFileName string + OriginalPath string + Extension string + ResolvedUsingTsExtension bool + ResolvedUsingExtraExtensions bool + PackageId PackageId + IsExternalLibraryImport bool + AlternateResult string } func (r *ResolvedModule) IsResolved() bool { diff --git a/internal/module/util.go b/internal/module/util.go index bb48dedcb11..79f1f6489fa 100644 --- a/internal/module/util.go +++ b/internal/module/util.go @@ -151,6 +151,10 @@ func GetResolutionDiagnostic(options *core.CompilerOptions, resolvedModule *Reso return diagnostics.Module_0_was_resolved_to_1_but_allowArbitraryExtensions_is_not_set } + if resolvedModule.ResolvedUsingExtraExtensions { + return nil + } + switch resolvedModule.Extension { case tspath.ExtensionTs, tspath.ExtensionDts, tspath.ExtensionMts, tspath.ExtensionDmts, diff --git a/internal/project/ata/ata.go b/internal/project/ata/ata.go index f877f718d47..7018d96c63f 100644 --- a/internal/project/ata/ata.go +++ b/internal/project/ata/ata.go @@ -186,7 +186,7 @@ func (ti *TypingsInstaller) installTypings( if packageNames, ok := ti.installWorker(projectID, requestID, scopedTypings, logger); ok { logger.Log(fmt.Sprintf("ATA:: Installed typings %v", packageNames)) var installedTypingFiles []string - resolver := module.NewResolver(ti.host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") + resolver := module.NewResolver(ti.host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "", nil) for _, packageName := range filteredTypings { typingFile := ti.typingToFileName(resolver, packageName) if typingFile == "" { @@ -416,7 +416,7 @@ func (ti *TypingsInstaller) processCacheLocation(projectID string, fs vfs.FS, lo logger.Log("ATA:: Loaded content of " + packageLockJson + ": " + npmLockContents) // !!! sheetal strada uses Node10 - resolver := module.NewResolver(ti.host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "") + resolver := module.NewResolver(ti.host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindNodeNext}, "", "", nil) if npmConfig.DevDependencies != nil && (npmLock.Packages != nil || npmLock.Dependencies != nil) { for key := range npmConfig.DevDependencies { npmLockValue, npmLockValueExists := npmLock.Packages["node_modules/"+key] diff --git a/internal/spanmap/spanmap.go b/internal/spanmap/spanmap.go new file mode 100644 index 00000000000..18b10c4eff7 --- /dev/null +++ b/internal/spanmap/spanmap.go @@ -0,0 +1,238 @@ +// Package spanmap provides a span-aware mapping from positions in a content mapper's transformed +// output back to positions in the original, untransformed source. Unlike a source map, which records +// point correspondences and leaves spans and "no origin" implicit, a SpanMap tiles the generated text +// into explicit segments so that a generated span can be mapped to an original span with a known +// fidelity. All positions are absolute offsets (core.TextPos), matching the compiler's TextRange model. +package spanmap + +import ( + "slices" + + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/json" +) + +// Kind describes how positions inside a segment relate the generated span to the original span. +type Kind int32 + +const ( + // KindVerbatim segments are length-preserving: the generated and original spans have the same + // length and interior positions map 1:1 (origPos = pos - GenStart + OrigStart). A generated span + // fully within a verbatim segment maps to an exact original span. + KindVerbatim Kind = iota + // KindAtom segments map a generated span to an original span as a whole; interior positions are not + // interpolatable (the lengths may differ), so positions within clamp to the segment's endpoints. + // Used for renamed identifiers or short expressions. + KindAtom + // KindSynthesized segments are generated content with no original counterpart (e.g. a synthesized + // import). Their OrigStart == OrigEnd marks the point in the original text where the content was + // inserted, used as a fallback location. + KindSynthesized +) + +// Fidelity describes how faithfully a mapped span reflects the original. +type Fidelity int32 + +const ( + // FidelityExact means the span fell entirely within a single verbatim segment and maps precisely. + FidelityExact Fidelity = iota + // FidelityAtom means the span fell within a single atom segment and maps to that atom's span. + FidelityAtom + // FidelityApproximate means the span crossed segment boundaries; its endpoints were mapped and clamped. + FidelityApproximate + // FidelityNone means the span had no original counterpart (it was entirely synthesized). + FidelityNone +) + +// Segment maps a contiguous generated span to an original span. +type Segment struct { + GenStart core.TextPos + GenEnd core.TextPos + OrigStart core.TextPos + OrigEnd core.TextPos + Kind Kind +} + +// SpanMap is a sorted, gap-free tiling of a content mapper's generated text. +type SpanMap struct { + segments []Segment +} + +// Validation failures. A content mapper is required to provide a valid span map; these describe the +// ways a map can be malformed, so the compiler can attribute the failure to the mapper precisely and +// point the mapper's author at the offending location. +type ProblemKind int32 + +const ( + // ProblemMissing means no mappings were provided at all. + ProblemMissing ProblemKind = iota + // ProblemCoverage means the segments do not tile the entire transformed text (a gap, overlap, or a + // segment extending past the end of the transformed text). + ProblemCoverage + // ProblemOutOfBounds means a segment's original span lies outside the original text. + ProblemOutOfBounds + // ProblemVerbatimMismatch means a verbatim segment's generated and original text differ. + ProblemVerbatimMismatch +) + +// Problem describes a single span map validation failure, including the offsets involved so the mapper's +// author can locate it. GenPos is an offset into the transformed output; OrigPos is an offset into the +// original content. Either may be unused (zero) depending on Kind. +type Problem struct { + Kind ProblemKind + GenPos core.TextPos + OrigPos core.TextPos +} + +// Validate enforces the content-mapper span map contract against the transformed and original text: the +// segments must tile the whole transformed text with no gaps or overlaps, every original span must lie +// within the original text, and every verbatim segment's text must match the original exactly. It +// returns the first violation found, or nil if the map is valid. +func (m *SpanMap) Validate(transformed, original string) *Problem { + if m == nil || len(m.segments) == 0 { + return &Problem{Kind: ProblemMissing} + } + genLen := core.TextPos(len(transformed)) + origLen := core.TextPos(len(original)) + var expectedGenStart core.TextPos + for i := range m.segments { + s := &m.segments[i] + if s.GenStart != expectedGenStart || s.GenEnd < s.GenStart || s.GenEnd > genLen { + return &Problem{Kind: ProblemCoverage, GenPos: expectedGenStart} + } + expectedGenStart = s.GenEnd + if s.OrigStart < 0 || s.OrigEnd < s.OrigStart || s.OrigEnd > origLen { + return &Problem{Kind: ProblemOutOfBounds, GenPos: s.GenStart, OrigPos: s.OrigEnd} + } + if s.Kind == KindVerbatim { + if s.GenEnd-s.GenStart != s.OrigEnd-s.OrigStart || + transformed[s.GenStart:s.GenEnd] != original[s.OrigStart:s.OrigEnd] { + return &Problem{Kind: ProblemVerbatimMismatch, GenPos: s.GenStart, OrigPos: s.OrigStart} + } + } + } + if expectedGenStart != genLen { + return &Problem{Kind: ProblemCoverage, GenPos: expectedGenStart} + } + return nil +} + +// New builds a SpanMap from segments, which are sorted by generated start. The segments are expected to +// tile the generated text; gaps and overlaps are tolerated but map on a best-effort basis. +func New(segments []Segment) *SpanMap { + sorted := slices.Clone(segments) + slices.SortFunc(sorted, func(a, b Segment) int { + return int(a.GenStart - b.GenStart) + }) + return &SpanMap{segments: sorted} +} + +// MapSpan maps a generated range to an original range, along with the fidelity of the result. A nil or +// empty SpanMap maps identically. +func (m *SpanMap) MapSpan(r core.TextRange) (core.TextRange, Fidelity) { + if m == nil || len(m.segments) == 0 { + return r, FidelityExact + } + genStart := core.TextPos(r.Pos()) + genEnd := max(core.TextPos(r.End()), genStart) + + startIdx := m.findSegmentIndex(genStart) + endProbe := genEnd + if genEnd > genStart { + endProbe = genEnd - 1 + } + endIdx := m.findSegmentIndex(endProbe) + + startSeg := &m.segments[startIdx] + endSeg := &m.segments[endIdx] + + origStart := mapLow(genStart, startSeg) + origEnd := max(mapHigh(genEnd, endSeg), origStart) + + var fidelity Fidelity + if startIdx == endIdx { + switch startSeg.Kind { + case KindVerbatim: + fidelity = FidelityExact + case KindAtom: + fidelity = FidelityAtom + default: + fidelity = FidelityNone + } + } else { + fidelity = FidelityApproximate + } + + return core.NewTextRange(int(origStart), int(origEnd)), fidelity +} + +// findSegmentIndex returns the index of the segment containing pos, clamping to the first or last +// segment when pos lies outside the tiling. +func (m *SpanMap) findSegmentIndex(pos core.TextPos) int { + idx, found := slices.BinarySearchFunc(m.segments, pos, func(s Segment, p core.TextPos) int { + return int(s.GenStart - p) + }) + if found { + return idx + } + if idx == 0 { + return 0 + } + return idx - 1 +} + +func mapLow(pos core.TextPos, seg *Segment) core.TextPos { + if seg.Kind == KindVerbatim { + return clamp(seg.OrigStart+(pos-seg.GenStart), seg.OrigStart, seg.OrigEnd) + } + return seg.OrigStart +} + +func mapHigh(pos core.TextPos, seg *Segment) core.TextPos { + if seg.Kind == KindVerbatim { + return clamp(seg.OrigStart+(pos-seg.GenStart), seg.OrigStart, seg.OrigEnd) + } + return seg.OrigEnd +} + +func clamp(v, lo, hi core.TextPos) core.TextPos { + return max(lo, min(v, hi)) +} + +// wireSegment is the JSON tuple form exchanged with an out-of-process content mapper: +// [genStart, genLength, origStart, origLength, kind]. +type wireSegment [5]int32 + +// Unmarshal decodes a SpanMap from the JSON tuple form produced by an out-of-process content mapper. +func Unmarshal(data []byte) (*SpanMap, error) { + var tuples []wireSegment + if err := json.Unmarshal(data, &tuples); err != nil { + return nil, err + } + segments := make([]Segment, len(tuples)) + for i, t := range tuples { + segments[i] = Segment{ + GenStart: core.TextPos(t[0]), + GenEnd: core.TextPos(t[0] + t[1]), + OrigStart: core.TextPos(t[2]), + OrigEnd: core.TextPos(t[2] + t[3]), + Kind: Kind(t[4]), + } + } + return New(segments), nil +} + +// Marshal encodes a SpanMap into the JSON tuple form. +func (m *SpanMap) Marshal() ([]byte, error) { + tuples := make([]wireSegment, len(m.segments)) + for i, s := range m.segments { + tuples[i] = wireSegment{ + int32(s.GenStart), + int32(s.GenEnd - s.GenStart), + int32(s.OrigStart), + int32(s.OrigEnd - s.OrigStart), + int32(s.Kind), + } + } + return json.Marshal(tuples) +} diff --git a/internal/spanmap/spanmap_test.go b/internal/spanmap/spanmap_test.go new file mode 100644 index 00000000000..95858ca8f36 --- /dev/null +++ b/internal/spanmap/spanmap_test.go @@ -0,0 +1,168 @@ +package spanmap_test + +import ( + "testing" + + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/spanmap" + "gotest.tools/v3/assert" +) + +func TestMapSpanVerbatim(t *testing.T) { + t.Parallel() + + // Generated [0,10) is a verbatim copy of original [100,110). + m := spanmap.New([]spanmap.Segment{ + {GenStart: 0, GenEnd: 10, OrigStart: 100, OrigEnd: 110, Kind: spanmap.KindVerbatim}, + }) + + got, fidelity := m.MapSpan(core.NewTextRange(3, 7)) + assert.Equal(t, got.Pos(), 103) + assert.Equal(t, got.End(), 107) + assert.Equal(t, fidelity, spanmap.FidelityExact) +} + +func TestMapSpanAtom(t *testing.T) { + t.Parallel() + + // Generated [0,3) ("jsx") maps to nothing; [3,14) ("MyComponent") is an atom of the original. + m := spanmap.New([]spanmap.Segment{ + {GenStart: 0, GenEnd: 3, OrigStart: 50, OrigEnd: 50, Kind: spanmap.KindSynthesized}, + {GenStart: 3, GenEnd: 14, OrigStart: 60, OrigEnd: 71, Kind: spanmap.KindAtom}, + }) + + // A span inside the atom maps to the whole atom span. + got, fidelity := m.MapSpan(core.NewTextRange(5, 9)) + assert.Equal(t, got.Pos(), 60) + assert.Equal(t, got.End(), 71) + assert.Equal(t, fidelity, spanmap.FidelityAtom) +} + +func TestMapSpanSynthesized(t *testing.T) { + t.Parallel() + + m := spanmap.New([]spanmap.Segment{ + {GenStart: 0, GenEnd: 30, OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}, + }) + + got, fidelity := m.MapSpan(core.NewTextRange(5, 10)) + assert.Equal(t, got.Pos(), 0) + assert.Equal(t, got.End(), 0) + assert.Equal(t, fidelity, spanmap.FidelityNone) +} + +func TestMapSpanCrossingSegments(t *testing.T) { + t.Parallel() + + m := spanmap.New([]spanmap.Segment{ + {GenStart: 0, GenEnd: 10, OrigStart: 100, OrigEnd: 110, Kind: spanmap.KindVerbatim}, + {GenStart: 10, GenEnd: 20, OrigStart: 200, OrigEnd: 210, Kind: spanmap.KindVerbatim}, + }) + + got, fidelity := m.MapSpan(core.NewTextRange(5, 15)) + assert.Equal(t, got.Pos(), 105) + assert.Equal(t, got.End(), 205) + assert.Equal(t, fidelity, spanmap.FidelityApproximate) +} + +func TestMapSpanNilAndEmptyIdentity(t *testing.T) { + t.Parallel() + + var m *spanmap.SpanMap + got, fidelity := m.MapSpan(core.NewTextRange(3, 7)) + assert.Equal(t, got.Pos(), 3) + assert.Equal(t, got.End(), 7) + assert.Equal(t, fidelity, spanmap.FidelityExact) +} + +func TestMarshalRoundTrip(t *testing.T) { + t.Parallel() + + original := spanmap.New([]spanmap.Segment{ + {GenStart: 0, GenEnd: 3, OrigStart: 50, OrigEnd: 50, Kind: spanmap.KindSynthesized}, + {GenStart: 3, GenEnd: 14, OrigStart: 60, OrigEnd: 71, Kind: spanmap.KindAtom}, + {GenStart: 14, GenEnd: 24, OrigStart: 71, OrigEnd: 81, Kind: spanmap.KindVerbatim}, + }) + + data, err := original.Marshal() + assert.NilError(t, err) + decoded, err := spanmap.Unmarshal(data) + assert.NilError(t, err) + + for _, r := range []core.TextRange{core.NewTextRange(1, 2), core.NewTextRange(4, 10), core.NewTextRange(16, 20)} { + wantRange, wantFidelity := original.MapSpan(r) + gotRange, gotFidelity := decoded.MapSpan(r) + assert.Equal(t, gotRange, wantRange) + assert.Equal(t, gotFidelity, wantFidelity) + } +} + +func TestValidate(t *testing.T) { + t.Parallel() + + const transformed = "const greeting = 1;\n" + const original = "const greeting = 1;\n" + scriptStart := 3 // index of "const" in original + + testCases := []struct { + name string + segs []spanmap.Segment + wantKind spanmap.ProblemKind + wantOK bool + }{ + { + name: "valid verbatim", + segs: []spanmap.Segment{{GenStart: 0, GenEnd: core.TextPos(len(transformed)), OrigStart: core.TextPos(scriptStart), OrigEnd: core.TextPos(scriptStart + len(transformed)), Kind: spanmap.KindVerbatim}}, + wantOK: true, + }, + { + name: "valid synthesized", + segs: []spanmap.Segment{{GenStart: 0, GenEnd: core.TextPos(len(transformed)), OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}}, + wantOK: true, + }, + { + name: "gap leaves transformed uncovered", + segs: []spanmap.Segment{{GenStart: 0, GenEnd: 3, OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}}, + wantKind: spanmap.ProblemCoverage, + }, + { + name: "overlap", + segs: []spanmap.Segment{ + {GenStart: 0, GenEnd: 10, OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}, + {GenStart: 5, GenEnd: core.TextPos(len(transformed)), OrigStart: 0, OrigEnd: 0, Kind: spanmap.KindSynthesized}, + }, + wantKind: spanmap.ProblemCoverage, + }, + { + name: "original out of bounds", + segs: []spanmap.Segment{{GenStart: 0, GenEnd: core.TextPos(len(transformed)), OrigStart: 0, OrigEnd: core.TextPos(len(original) + 10), Kind: spanmap.KindSynthesized}}, + wantKind: spanmap.ProblemOutOfBounds, + }, + { + name: "verbatim text mismatch", + segs: []spanmap.Segment{{GenStart: 0, GenEnd: core.TextPos(len(transformed)), OrigStart: 0, OrigEnd: core.TextPos(len(transformed)), Kind: spanmap.KindVerbatim}}, + wantKind: spanmap.ProblemVerbatimMismatch, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + problem := spanmap.New(tc.segs).Validate(transformed, original) + if tc.wantOK { + assert.Assert(t, problem == nil, "expected valid, got %+v", problem) + return + } + assert.Assert(t, problem != nil, "expected a problem") + assert.Equal(t, problem.Kind, tc.wantKind) + }) + } +} + +func TestValidateNilIsRequired(t *testing.T) { + t.Parallel() + var m *spanmap.SpanMap + problem := m.Validate("abc", "abc") + assert.Assert(t, problem != nil) + assert.Equal(t, problem.Kind, spanmap.ProblemMissing) +} diff --git a/internal/tsoptions/declscompiler.go b/internal/tsoptions/declscompiler.go index cde4e2556bc..c16e085be5c 100644 --- a/internal/tsoptions/declscompiler.go +++ b/internal/tsoptions/declscompiler.go @@ -307,6 +307,14 @@ var optionsForCompiler = []*CommandLineOption{ Description: diagnostics.Print_names_of_files_that_are_part_of_the_compilation_and_then_stop_processing, DefaultValueDescription: false, }, + { + Name: "dangerouslyLoadExternalPlugins", + Kind: CommandLineOptionTypeBoolean, + Category: diagnostics.Command_line_Options, + IsCommandLineOnly: true, + Description: diagnostics.Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation, + DefaultValueDescription: false, + }, { Name: "ignoreConfig", Kind: CommandLineOptionTypeBoolean, diff --git a/internal/tsoptions/parsinghelpers.go b/internal/tsoptions/parsinghelpers.go index 04751c8df67..7b5dad767c2 100644 --- a/internal/tsoptions/parsinghelpers.go +++ b/internal/tsoptions/parsinghelpers.go @@ -517,6 +517,8 @@ func parseCompilerOptions(key string, value any, allOptions *core.CompilerOption allOptions.Quiet = ParseTristate(value) case "checkers": allOptions.Checkers = parseNumber(value) + case "dangerouslyLoadExternalPlugins": + allOptions.DangerouslyLoadExternalPlugins = ParseTristate(value) default: // different than any key above return false diff --git a/internal/tsoptions/tsconfigparsing.go b/internal/tsoptions/tsconfigparsing.go index f5ca2194207..eb786e0b28b 100644 --- a/internal/tsoptions/tsconfigparsing.go +++ b/internal/tsoptions/tsconfigparsing.go @@ -1355,6 +1355,9 @@ func parseJsonConfigFileContentWorker( } } } + if len(contentMappers) > 0 && !(parsedConfig.options != nil && parsedConfig.options.DangerouslyLoadExternalPlugins.IsTrue()) { + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled)) + } getFileNames := func(basePath string) ([]string, int) { parsedConfigOptions := parsedConfig.options diff --git a/internal/tsoptions/tsconfigparsing_test.go b/internal/tsoptions/tsconfigparsing_test.go index 840ba644ef1..038ac8a6fde 100644 --- a/internal/tsoptions/tsconfigparsing_test.go +++ b/internal/tsoptions/tsconfigparsing_test.go @@ -30,10 +30,11 @@ import ( ) type testConfig struct { - jsonText string - configFileName string - basePath string - allFileList map[string]string + jsonText string + configFileName string + basePath string + allFileList map[string]string + existingOptions *core.CompilerOptions } var parseConfigFileTextToJsonTests = []struct { @@ -824,7 +825,7 @@ func getParsedWithJsonApi(config testConfig, host tsoptions.ParseConfigHost, bas parsed, host, basePath, - nil, + config.existingOptions, configFileName, /*resolutionStack*/ nil, /*extendedConfigCache*/ nil, @@ -1020,6 +1021,7 @@ func TestContentMappers(t *testing.T) { "/src/app.ts": "export {}", "/src/Component.vue": "", }, + existingOptions: &core.CompilerOptions{DangerouslyLoadExternalPlugins: core.TSTrue}, } for name, getParsed := range map[string]func(testConfig, tsoptions.ParseConfigHost, string) *tsoptions.ParsedCommandLine{ "json api": getParsedWithJsonApi, @@ -1049,6 +1051,35 @@ func TestContentMappers(t *testing.T) { } } +func TestContentMappersRequireFlag(t *testing.T) { + t.Parallel() + + config := testConfig{ + jsonText: `{ "contentMappers": [{ "command": ["vue-mapper"], "extensions": [".vue"] }] }`, + configFileName: "tsconfig.json", + basePath: "/", + allFileList: map[string]string{"/app.ts": "export {}"}, + // existingOptions omitted: --dangerouslyLoadExternalPlugins is not set. + } + expectedCode := diagnostics.Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled.Code() + for name, getParsed := range map[string]func(testConfig, tsoptions.ParseConfigHost, string) *tsoptions.ParsedCommandLine{ + "json api": getParsedWithJsonApi, + "jsonSourceFile api": getParsedWithJsonSourceFileApi, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + allFileLists := map[string]string{"/tsconfig.json": config.jsonText} + maps.Copy(allFileLists, config.allFileList) + host := tsoptionstest.NewVFSParseConfigHost(allFileLists, config.basePath, true /*useCaseSensitiveFileNames*/) + parsed := getParsed(config, host, config.basePath) + found := slices.ContainsFunc(parsed.Errors, func(d *ast.Diagnostic) bool { + return d.Code() == expectedCode + }) + assert.Assert(t, found, "expected diagnostic %d, got errors: %v", expectedCode, parsed.Errors) + }) + } +} + func TestContentMappersValidation(t *testing.T) { t.Parallel() @@ -1093,10 +1124,11 @@ func TestContentMappersValidation(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() config := testConfig{ - jsonText: `{ "contentMappers": ` + test.contentMappers + ` }`, - configFileName: "tsconfig.json", - basePath: "/", - allFileList: map[string]string{"/app.ts": "export {}"}, + jsonText: `{ "contentMappers": ` + test.contentMappers + ` }`, + configFileName: "tsconfig.json", + basePath: "/", + allFileList: map[string]string{"/app.ts": "export {}"}, + existingOptions: &core.CompilerOptions{DangerouslyLoadExternalPlugins: core.TSTrue}, } for apiName, getParsed := range map[string]func(testConfig, tsoptions.ParseConfigHost, string) *tsoptions.ParsedCommandLine{ "json api": getParsedWithJsonApi, @@ -1132,7 +1164,7 @@ func getParsedWithJsonSourceFileApi(config testConfig, host tsoptions.ParseConfi tsConfigSourceFile, host, host.GetCurrentDirectory(), - nil, + config.existingOptions, nil, configFileName, /*resolutionStack*/ nil, diff --git a/testdata/baselines/reference/contentMappers/contentMapperDiagnostics.txt b/testdata/baselines/reference/contentMappers/contentMapperDiagnostics.txt new file mode 100644 index 00000000000..6bcd10e3b52 --- /dev/null +++ b/testdata/baselines/reference/contentMappers/contentMapperDiagnostics.txt @@ -0,0 +1,22 @@ +=== Content-mapped files === + +//// [Component.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + + + + +--- Transformed --- +export const greeting = "hello"; + +=== Diagnostics === + +Component.vue:6:33 - error vue1002: Unexpected token. + +6 export const greeting = "hello" oops + ~~~~ + diff --git a/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt b/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt new file mode 100644 index 00000000000..434888d8ce6 --- /dev/null +++ b/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt @@ -0,0 +1,72 @@ +=== Content-mapped files === + +//// [A.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [B.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [C.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [D.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [E.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [F.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +//// [G.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +=== Diagnostics === + +C.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch + +1 + ~ + +D.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch + +1 + ~ + +E.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch + +1 + ~ + +F.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch + +1 + ~ + +G.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch + +1 + ~ + +error TS100026: The content mapper 'vue-mapper@1.0.0' failed 5 times and will not be used. diff --git a/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt b/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt new file mode 100644 index 00000000000..440f0c75d92 --- /dev/null +++ b/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt @@ -0,0 +1,20 @@ +=== Content-mapped files === + +//// [Component.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + +--- Transformed --- + + +=== Diagnostics === + +Component.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: broken pipe + +1 + ~ + +app.ts:1:26 - error TS2306: File '/src/Component.vue' is not a module. + +1 import { greeting } from "./Component.vue"; + ~~~~~~~~~~~~~~~~~ + diff --git a/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt b/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt new file mode 100644 index 00000000000..e6149858fe2 --- /dev/null +++ b/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt @@ -0,0 +1,25 @@ +=== Content-mapped files === + +//// [Component.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + + +--- Transformed --- +export const el = jsxRuntime(Widget); + +=== Diagnostics === + +Component.vue:1:19 - error TS2304: Cannot find name 'jsxRuntime'. + This location is in code generated by the content mapper 'vue-mapper@1.0.0' and has no corresponding location in the original file. + +1 export const el = jsxRuntime(Widget); + ~~~~~~~~~~ + +Component.vue:1:30 - error TS2304: Cannot find name 'Widget'. + This location is in code generated by the content mapper 'vue-mapper@1.0.0' and has no corresponding location in the original file. + +1 export const el = jsxRuntime(Widget); + ~~~~~~ + diff --git a/testdata/baselines/reference/contentMappers/contentMapperSpanMapping.txt b/testdata/baselines/reference/contentMappers/contentMapperSpanMapping.txt new file mode 100644 index 00000000000..8e5cee7aac7 --- /dev/null +++ b/testdata/baselines/reference/contentMappers/contentMapperSpanMapping.txt @@ -0,0 +1,24 @@ +=== Content-mapped files === + +//// [Component.vue] (ScriptKind: ScriptKindTS, ContentMapper: [.vue]) +--- Original --- + + + + +--- Transformed --- +const greeting: number = "not a number"; +export { greeting }; + +=== Diagnostics === + +Component.vue:6:7 - error TS2322: Type 'string' is not assignable to type 'number'. + +6 const greeting: number = "not a number"; + ~~~~~~~~ + diff --git a/testdata/baselines/reference/tsc/commandLine/help-all.js b/testdata/baselines/reference/tsc/commandLine/help-all.js index 4e0012b2d8c..b537d4d2f0e 100644 --- a/testdata/baselines/reference/tsc/commandLine/help-all.js +++ b/testdata/baselines/reference/tsc/commandLine/help-all.js @@ -20,6 +20,9 @@ Build one or more projects and their dependencies, if out of date --checkers Set the number of checkers per project. +--dangerouslyLoadExternalPlugins +Allow loading external content mapper plugins that execute code during compilation. + --help, -h Print this message. From 40dc747ee016f844a6a103a99a9d72b1ece1ded4 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Thu, 9 Jul 2026 18:59:51 -0700 Subject: [PATCH 03/37] Wire into build mode / tsbuildinfo --- internal/compiler/contentmapper_test.go | 51 +++++- internal/compiler/contentmapperrunner.go | 32 ++++ internal/compiler/fileloader.go | 4 +- internal/compiler/program.go | 6 + internal/core/contentmapper.go | 18 -- internal/execute/build/buildtask.go | 10 +- internal/execute/incremental/buildInfo.go | 18 +- .../buildinfo_contentmapper_test.go | 72 ++++++++ internal/execute/incremental/incremental.go | 7 +- .../incremental/snapshottobuildinfo.go | 1 + internal/execute/tsc.go | 17 +- internal/execute/tsc/compile.go | 10 ++ internal/execute/tsctests/runner.go | 17 +- internal/execute/tsctests/sys.go | 40 +++++ internal/execute/tsctests/tscbuild_test.go | 37 ++++ internal/execute/watcher.go | 7 +- internal/testutil/harnessutil/harnessutil.go | 2 +- internal/tsoptions/declscompiler.go | 16 +- .../reference/tsbuild/commandLine/help.js | 3 + .../reference/tsbuild/commandLine/locale.js | 3 + ...t-mapper-identity-change-forces-rebuild.js | 168 ++++++++++++++++++ 21 files changed, 483 insertions(+), 56 deletions(-) create mode 100644 internal/execute/incremental/buildinfo_contentmapper_test.go create mode 100644 testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js diff --git a/internal/compiler/contentmapper_test.go b/internal/compiler/contentmapper_test.go index 126b87615e8..f0df553eed3 100644 --- a/internal/compiler/contentmapper_test.go +++ b/internal/compiler/contentmapper_test.go @@ -25,12 +25,61 @@ import ( type fakeContentMapperRunner struct { transform func(fileName string, content string) (compiler.ContentMapperResult, error) + identity string } func (r fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { return r.transform(fileName, content) } +func (r fakeContentMapperRunner) Identity(mapper *core.ContentMapper) string { + if r.identity == "" { + return "vue-mapper@1.0.0" + } + return r.identity +} + +// identityRunner is a runner whose only interesting behavior is the identity it reports per mapper. +type identityRunner func(*core.ContentMapper) string + +func (f identityRunner) Identity(mapper *core.ContentMapper) string { return f(mapper) } + +func (identityRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{}, nil +} + +func TestContentMapperIdentities(t *testing.T) { + t.Parallel() + + config := &tsoptions.ParsedCommandLine{ + ParsedConfig: &core.ParsedOptions{ + CompilerOptions: &core.CompilerOptions{}, + ContentMappers: []*core.ContentMapper{ + {Command: []string{"vue"}}, + {Command: []string{"svelte"}}, + {Command: []string{"anon"}}, + }, + }, + } + + // No runner means no identities to record. + assert.Assert(t, compiler.ContentMapperIdentities(nil, config) == nil) + + // Identities come from the runner; "anon" reports none and is omitted, and the result is sorted so + // reordering content mappers in tsconfig does not change it. + runner := identityRunner(func(mapper *core.ContentMapper) string { + switch mapper.Command[0] { + case "vue": + return "vue@2.0.0" + case "svelte": + return "svelte@3.0.0" + default: + return "" + } + }) + assert.DeepEqual(t, compiler.ContentMapperIdentities(runner, config), []string{"svelte@3.0.0", "vue@2.0.0"}) +} + const vueRawContent = "" // synthesizedSpanMap maps an entirely generated transformed text to no original location (a fully @@ -65,7 +114,7 @@ func newContentMapperProgram(t *testing.T, runner compiler.ContentMapperRunner, Module: core.ModuleKindESNext, ModuleResolution: core.ModuleResolutionKindBundler, }, - ContentMappers: []*core.ContentMapper{{Command: []string{"vue"}, Extensions: []string{".vue"}, Name: "vue-mapper", Version: "1.0.0"}}, + ContentMappers: []*core.ContentMapper{{Command: []string{"vue"}, Extensions: []string{".vue"}}}, }, } return compiler.NewProgram(compiler.ProgramOptions{ diff --git a/internal/compiler/contentmapperrunner.go b/internal/compiler/contentmapperrunner.go index 4328326db89..e4b19f21d46 100644 --- a/internal/compiler/contentmapperrunner.go +++ b/internal/compiler/contentmapperrunner.go @@ -1,9 +1,12 @@ package compiler import ( + "slices" + "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/spanmap" + "github.com/microsoft/typescript-go/internal/tsoptions" ) // ContentMapperResult is the outcome of transforming a foreign file's content into TypeScript. @@ -28,4 +31,33 @@ type ContentMapperRunner interface { // runner/coordinator hit a broken pipe, a process crash, or could not deserialize the mapper's // response. Transform(fileName string, content string) (result ContentMapperResult, err error) + + // Identity returns a stable identifier for the given mapper (e.g. "name@version"), used both to + // attribute diagnostics and to detect when a mapper implementation has changed between builds. How + // the identity is resolved — declared in the config, read from a manifest, reported during an + // initialize handshake, or some combination — is the runner's concern. An empty string means the + // mapper has no identity to report. + Identity(mapper *core.ContentMapper) string +} + +// ContentMapperIdentities returns the sorted identities that runner assigns to the content mappers in +// config, used to detect when a mapper implementation has changed between builds. Mappers with no +// identity are omitted, and the result is sorted so that merely reordering content mappers in tsconfig +// does not force a rebuild. Returns nil when there is no runner or no mapper reports an identity. +func ContentMapperIdentities(runner ContentMapperRunner, config *tsoptions.ParsedCommandLine) []string { + if runner == nil { + return nil + } + mappers := config.ContentMappers() + identities := make([]string, 0, len(mappers)) + for _, mapper := range mappers { + if identity := runner.Identity(mapper); identity != "" { + identities = append(identities, identity) + } + } + if len(identities) == 0 { + return nil + } + slices.Sort(identities) + return identities } diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 088f0bbfa93..57f4f43d422 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -420,9 +420,7 @@ func (p *fileLoader) parseContentMappedFile(opts ast.SourceFileParseOptions) *as } mapper := p.matchContentMapper(opts.FileName) p.recordContentMapper(opts.Path, mapper) - // The mapper's identity is part of its contract (reported at initialize); it is what we attribute - // failures to, so we use it directly rather than substituting anything when it is absent. - label := mapper.Identity() + label := p.opts.ContentMapperRunner.Identity(mapper) if p.contentMapperDisabled(mapper) { // The mapper already exceeded its failure budget; add the file empty without re-reporting. return p.emptyContentMappedFile(opts, content) diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 0487535fe0d..4af61024772 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -409,6 +409,12 @@ func (p *Program) Options() *core.CompilerOptions { return p.opts. func (p *Program) GetContentMapper(file *ast.SourceFile) *core.ContentMapper { return p.contentMapperForFile[file.Path()] } + +// ContentMapperIdentities returns the sorted identities of the configured content mappers, as reported +// by the content mapper runner, for recording in build info. +func (p *Program) ContentMapperIdentities() []string { + return ContentMapperIdentities(p.opts.ContentMapperRunner, p.opts.Config) +} func (p *Program) CommandLine() *tsoptions.ParsedCommandLine { return p.opts.Config } func (p *Program) Host() CompilerHost { return p.opts.Host } func (p *Program) Tracing() *tracing.Tracing { return p.opts.Tracing } diff --git a/internal/core/contentmapper.go b/internal/core/contentmapper.go index a472ad4aad5..7e155a3b191 100644 --- a/internal/core/contentmapper.go +++ b/internal/core/contentmapper.go @@ -6,22 +6,4 @@ package core type ContentMapper struct { Command []string `json:"command"` Extensions []string `json:"extensions"` - // Name and Version identify the mapper implementation. They are reported by the mapper itself - // (e.g. during an initialize handshake) rather than declared in the tsconfig, and are used to - // attribute diagnostics to a specific mapper. - Name string `json:"-"` - Version string `json:"-"` -} - -// Identity returns a human-readable "name@version" identifier for the mapper, or just the name when no -// version is reported, or an empty string when the mapper has not identified itself. -func (m *ContentMapper) Identity() string { - switch { - case m.Name == "": - return "" - case m.Version == "": - return m.Name - default: - return m.Name + "@" + m.Version - } } diff --git a/internal/execute/build/buildtask.go b/internal/execute/build/buildtask.go index 97381244caa..b71cb4bb46f 100644 --- a/internal/execute/build/buildtask.go +++ b/internal/execute/build/buildtask.go @@ -198,9 +198,10 @@ func (t *BuildTask) compileAndEmit(orchestrator *Orchestrator, path tspath.Path) configTime, _ := orchestrator.host.configTimes.Load(path) compileTimes.ConfigTime = configTime buildInfoReadStart := orchestrator.opts.Sys.Now() + mapperRunner := tsc.ResolveContentMapperRunner(orchestrator.opts.Testing, t.resolved) var oldProgram *incremental.Program if !orchestrator.opts.Command.BuildOptions.Force.IsTrue() { - oldProgram = incremental.ReadBuildInfoProgram(t.resolved, orchestrator.host, orchestrator.host) + oldProgram = incremental.ReadBuildInfoProgram(t.resolved, mapperRunner, orchestrator.host, orchestrator.host) } compileTimes.BuildInfoReadTime = orchestrator.opts.Sys.Now().Sub(buildInfoReadStart) parseStart := orchestrator.opts.Sys.Now() @@ -210,6 +211,7 @@ func (t *BuildTask) compileAndEmit(orchestrator *Orchestrator, path tspath.Path) host: orchestrator.host, trace: tsc.GetTraceWithWriterFromSys(&t.result.builder, orchestrator.opts.Command.Locale(), orchestrator.opts.Testing), }, + ContentMapperRunner: mapperRunner, }) compileTimes.ParseTime = orchestrator.opts.Sys.Now().Sub(parseStart) changesComputeStart := orchestrator.opts.Sys.Now() @@ -343,6 +345,12 @@ func (t *BuildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tsp return &upToDateStatus{kind: upToDateStatusTypeTsVersionOutputOfDate, data: buildInfo.Version} } + // If a configured content mapper's identity has changed, files it produced may be stale. + mapperRunner := tsc.ResolveContentMapperRunner(orchestrator.opts.Testing, t.resolved) + if !buildInfo.ContentMapperIdentitiesMatch(compiler.ContentMapperIdentities(mapperRunner, t.resolved)) { + return &upToDateStatus{kind: upToDateStatusTypeOutOfDateOptions, data: buildInfoPath} + } + // Report errors if build info indicates errors if buildInfo.Errors || // Errors that need to be reported irrespective of "--noCheck" (!t.resolved.CompilerOptions().NoCheck.IsTrue() && (buildInfo.SemanticErrors || buildInfo.CheckPending)) { // Errors without --noCheck diff --git a/internal/execute/incremental/buildInfo.go b/internal/execute/incremental/buildInfo.go index 19a308064b7..388c0ce400e 100644 --- a/internal/execute/incremental/buildInfo.go +++ b/internal/execute/incremental/buildInfo.go @@ -3,6 +3,7 @@ package incremental import ( "fmt" "iter" + "slices" "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" @@ -463,11 +464,12 @@ type BuildInfo struct { Version string `json:"version,omitzero"` // Common between incremental and tsc -b buildinfo for non incremental programs - Errors bool `json:"errors,omitzero"` - CheckPending bool `json:"checkPending,omitzero"` - Root []*BuildInfoRoot `json:"root,omitzero"` - PackageJsons []string `json:"packageJsons,omitzero"` - MissingPackageJsons []string `json:"missingPackageJsons,omitzero"` + Errors bool `json:"errors,omitzero"` + CheckPending bool `json:"checkPending,omitzero"` + Root []*BuildInfoRoot `json:"root,omitzero"` + PackageJsons []string `json:"packageJsons,omitzero"` + MissingPackageJsons []string `json:"missingPackageJsons,omitzero"` + ContentMapperIdentities []string `json:"contentMapperIdentities,omitzero"` // IncrementalProgram info FileNames []string `json:"fileNames,omitzero"` @@ -491,6 +493,12 @@ func (b *BuildInfo) IsValidVersion() bool { return b.Version == core.Version() } +// ContentMapperIdentitiesMatch reports whether the content mapper identities recorded in this build info +// match the given current identities (as produced by compiler.ContentMapperIdentities). +func (b *BuildInfo) ContentMapperIdentitiesMatch(current []string) bool { + return slices.Equal(b.ContentMapperIdentities, current) +} + func (b *BuildInfo) IsIncremental() bool { return b != nil && len(b.FileNames) != 0 } diff --git a/internal/execute/incremental/buildinfo_contentmapper_test.go b/internal/execute/incremental/buildinfo_contentmapper_test.go new file mode 100644 index 00000000000..8cf33639991 --- /dev/null +++ b/internal/execute/incremental/buildinfo_contentmapper_test.go @@ -0,0 +1,72 @@ +package incremental_test + +import ( + "testing" + + "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/execute/incremental" + "github.com/microsoft/typescript-go/internal/tsoptions" + "gotest.tools/v3/assert" +) + +func configWithMappers(mappers ...*core.ContentMapper) *tsoptions.ParsedCommandLine { + return &tsoptions.ParsedCommandLine{ + ParsedConfig: &core.ParsedOptions{ + CompilerOptions: &core.CompilerOptions{}, + ContentMappers: mappers, + }, + } +} + +// fakeContentMapperRunner reports a fixed identity for every mapper; Transform is never exercised here. +type fakeContentMapperRunner struct { + identity string +} + +func (r fakeContentMapperRunner) Identity(*core.ContentMapper) string { + return r.identity +} + +func (fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{}, nil +} + +func TestContentMapperIdentitiesMatch(t *testing.T) { + t.Parallel() + + buildInfo := &incremental.BuildInfo{ContentMapperIdentities: []string{"vue@1.0.0"}} + assert.Assert(t, buildInfo.ContentMapperIdentitiesMatch([]string{"vue@1.0.0"})) + assert.Assert(t, !buildInfo.ContentMapperIdentitiesMatch([]string{"vue@2.0.0"})) + assert.Assert(t, !buildInfo.ContentMapperIdentitiesMatch(nil)) + + empty := &incremental.BuildInfo{} + assert.Assert(t, empty.ContentMapperIdentitiesMatch(nil)) + assert.Assert(t, !empty.ContentMapperIdentitiesMatch([]string{"vue@1.0.0"})) +} + +type fakeBuildInfoReader struct { + buildInfo *incremental.BuildInfo +} + +func (r fakeBuildInfoReader) ReadBuildInfo(*tsoptions.ParsedCommandLine) *incremental.BuildInfo { + return r.buildInfo +} + +func TestReadBuildInfoProgramContentMapperIdentityMismatch(t *testing.T) { + t.Parallel() + + // An otherwise-valid, incremental build info whose recorded mapper identity differs from the one the + // runner now reports cannot be reused: the old program is discarded (nil) so the project is rebuilt. + // The host is never touched because we bail before reconstructing the snapshot. + buildInfo := &incremental.BuildInfo{ + Version: core.Version(), + FileNames: []string{"/src/a.ts"}, + ContentMapperIdentities: []string{"vue@1.0.0"}, + } + config := configWithMappers(&core.ContentMapper{Extensions: []string{".vue"}}) + runner := fakeContentMapperRunner{identity: "vue@2.0.0"} + + program := incremental.ReadBuildInfoProgram(config, runner, fakeBuildInfoReader{buildInfo}, nil) + assert.Assert(t, program == nil, "expected the old program to be discarded when the mapper identity changed") +} diff --git a/internal/execute/incremental/incremental.go b/internal/execute/incremental/incremental.go index fc29c318487..e2e770b00a3 100644 --- a/internal/execute/incremental/incremental.go +++ b/internal/execute/incremental/incremental.go @@ -41,12 +41,17 @@ func NewBuildInfoReader( return &buildInfoReader{host: host} } -func ReadBuildInfoProgram(config *tsoptions.ParsedCommandLine, reader BuildInfoReader, host compiler.CompilerHost) *Program { +func ReadBuildInfoProgram(config *tsoptions.ParsedCommandLine, runner compiler.ContentMapperRunner, reader BuildInfoReader, host compiler.CompilerHost) *Program { // Read buildInfo file buildInfo := reader.ReadBuildInfo(config) if buildInfo == nil || !buildInfo.IsValidVersion() || !buildInfo.IsIncremental() { return nil } + // If any configured content mapper's identity has changed, files it produced may be stale, so the + // old program cannot be reused. + if !buildInfo.ContentMapperIdentitiesMatch(compiler.ContentMapperIdentities(runner, config)) { + return nil + } // Convert to information that can be used to create incremental program incrementalProgram := &Program{ diff --git a/internal/execute/incremental/snapshottobuildinfo.go b/internal/execute/incremental/snapshottobuildinfo.go index 3b2c3c7736d..8e9f4e3a5bc 100644 --- a/internal/execute/incremental/snapshottobuildinfo.go +++ b/internal/execute/incremental/snapshottobuildinfo.go @@ -52,6 +52,7 @@ func snapshotToBuildInfo(snapshot *snapshot, program *compiler.Program, buildInf buildInfo.Errors = snapshot.hasErrors.IsTrue() buildInfo.SemanticErrors = snapshot.hasSemanticErrors buildInfo.CheckPending = snapshot.checkPending + buildInfo.ContentMapperIdentities = program.ContentMapperIdentities() to.setPackageJsons() return buildInfo } diff --git a/internal/execute/tsc.go b/internal/execute/tsc.go index 17baf4e51c5..576c78fa383 100644 --- a/internal/execute/tsc.go +++ b/internal/execute/tsc.go @@ -291,17 +291,19 @@ func performIncrementalCompilation( testing tsc.CommandLineTesting, ) tsc.CommandLineResult { host := compiler.NewCachedFSCompilerHost(sys.GetCurrentDirectory(), sys.FS(), sys.DefaultLibraryPath(), extendedConfigCache, getTraceFromSys(sys, config.Locale(), testing)) + mapperRunner := tsc.ResolveContentMapperRunner(testing, config) buildInfoReadStart := sys.Now() - oldProgram := incremental.ReadBuildInfoProgram(config, incremental.NewBuildInfoReader(host), host) + oldProgram := incremental.ReadBuildInfoProgram(config, mapperRunner, incremental.NewBuildInfoReader(host), host) compileTimes.BuildInfoReadTime = sys.Now().Sub(buildInfoReadStart) tr := startTracingIfNeeded(sys, config, testing) parseStart := sys.Now() program := compiler.NewProgram(compiler.ProgramOptions{ - Config: config, - Host: host, - Tracing: tr, + Config: config, + Host: host, + Tracing: tr, + ContentMapperRunner: mapperRunner, }) compileTimes.ParseTime = sys.Now().Sub(parseStart) changesComputeStart := sys.Now() @@ -345,9 +347,10 @@ func performCompilation( parseStart := sys.Now() program := compiler.NewProgram(compiler.ProgramOptions{ - Config: config, - Host: host, - Tracing: tr, + Config: config, + Host: host, + Tracing: tr, + ContentMapperRunner: tsc.ResolveContentMapperRunner(testing, config), }) compileTimes.ParseTime = sys.Now().Sub(parseStart) result, _ := tsc.EmitAndReportStatistics(tsc.EmitInput{ diff --git a/internal/execute/tsc/compile.go b/internal/execute/tsc/compile.go index 4f69345efaa..be77d53faac 100644 --- a/internal/execute/tsc/compile.go +++ b/internal/execute/tsc/compile.go @@ -7,9 +7,11 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/execute/incremental" "github.com/microsoft/typescript-go/internal/locale" + "github.com/microsoft/typescript-go/internal/tsoptions" "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs" ) @@ -60,6 +62,14 @@ type CommandLineTesting interface { OnWatchStatusReportEnd() GetTrace(w io.Writer, locale locale.Locale) func(msg *diagnostics.Message, args ...any) OnProgram(program *incremental.Program) + GetContentMapperRunner(mappers []*core.ContentMapper) compiler.ContentMapperRunner +} + +func ResolveContentMapperRunner(testing CommandLineTesting, config *tsoptions.ParsedCommandLine) compiler.ContentMapperRunner { + if testing == nil { + return nil + } + return testing.GetContentMapperRunner(config.ContentMappers()) } type CompileTimes struct { diff --git a/internal/execute/tsctests/runner.go b/internal/execute/tsctests/runner.go index 38c7e9c2089..cebee80f315 100644 --- a/internal/execute/tsctests/runner.go +++ b/internal/execute/tsctests/runner.go @@ -31,14 +31,15 @@ var noChangeOnlyEdit = []*tscEdit{ } type tscInput struct { - subScenario string - commandLineArgs []string - files FileMap - cwd string - edits []*tscEdit - env map[string]string - ignoreCase bool - windowsStyleRoot string + subScenario string + commandLineArgs []string + files FileMap + cwd string + edits []*tscEdit + env map[string]string + ignoreCase bool + windowsStyleRoot string + contentMapperVersion string } func (test *tscInput) executeCommand(sys *TestSys, baselineBuilder *strings.Builder, commandLineArgs []string) tsc.CommandLineResult { diff --git a/internal/execute/tsctests/sys.go b/internal/execute/tsctests/sys.go index 2d75a36d4ca..eff58b138ae 100644 --- a/internal/execute/tsctests/sys.go +++ b/internal/execute/tsctests/sys.go @@ -19,6 +19,7 @@ import ( "github.com/microsoft/typescript-go/internal/execute/tsc" "github.com/microsoft/typescript-go/internal/execute/watchmanager" "github.com/microsoft/typescript-go/internal/locale" + "github.com/microsoft/typescript-go/internal/spanmap" "github.com/microsoft/typescript-go/internal/testutil/fsbaselineutil" "github.com/microsoft/typescript-go/internal/testutil/harnessutil" "github.com/microsoft/typescript-go/internal/testutil/stringtestutil" @@ -133,6 +134,7 @@ func newTestSys(tscInput *tscInput, forIncrementalCorrectness bool) *TestSys { }, currentWrite) sys.env = tscInput.env sys.forIncrementalCorrectness = forIncrementalCorrectness + sys.contentMapperVersion = tscInput.contentMapperVersion sys.mockWatchBackend = NewMockWatchBackend() sys.mockWatchBackend.DirectoryExists = sys.fs.FS.DirectoryExists sys.fsDiffer = &fsbaselineutil.FSDiffer{ @@ -166,6 +168,8 @@ type TestSys struct { cwd string env map[string]string clock *TestClock + + contentMapperVersion string } var ( @@ -322,6 +326,42 @@ func (s *TestSys) WatchBackend() watchmanager.WatchBackend { return s.mockWatchBackend } +// GetContentMapperRunner stands in for a real mapper host, returning a runner that transforms foreign +// files verbatim and derives each mapper's identity from its command and the sys-controlled version (so +// a test can bump the version between builds to simulate a mapper upgrade). +func (s *TestSys) GetContentMapperRunner(mappers []*core.ContentMapper) compiler.ContentMapperRunner { + if len(mappers) == 0 { + return nil + } + return testContentMapperRunner{version: s.contentMapperVersion} +} + +// testContentMapperRunner transforms a foreign file's content verbatim into TypeScript. The content is +// therefore expected to already be valid TypeScript, which keeps the identity-preserving span map valid. +type testContentMapperRunner struct { + version string +} + +func (r testContentMapperRunner) Identity(mapper *core.ContentMapper) string { + name := mapper.Command[0] + if r.version == "" { + return name + } + return name + "@" + r.version +} + +func (testContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { + return compiler.ContentMapperResult{ + Text: content, + ScriptKind: core.ScriptKindTS, + Mappings: spanmap.New([]spanmap.Segment{{ + GenEnd: core.TextPos(len(content)), + OrigEnd: core.TextPos(len(content)), + Kind: spanmap.KindVerbatim, + }}), + }, nil +} + func (s *TestSys) OnProgram(program *incremental.Program) { s.writeHeaderToBaseline(&s.programBaselines, program) diff --git a/internal/execute/tsctests/tscbuild_test.go b/internal/execute/tsctests/tscbuild_test.go index 8c2339ffb60..a9f68ffef5e 100644 --- a/internal/execute/tsctests/tscbuild_test.go +++ b/internal/execute/tsctests/tscbuild_test.go @@ -297,6 +297,43 @@ console.log(tsconfig);`, } } +func TestBuildContentMapperIdentity(t *testing.T) { + t.Parallel() + testCases := []*tscInput{ + { + subScenario: "content mapper identity change forces rebuild", + files: FileMap{ + "/home/src/workspaces/project/tsconfig.json": stringtestutil.Dedent(` + { + "compilerOptions": { + "incremental": true + }, + "contentMappers": [ + { "command": ["vue"], "extensions": [".vue"] } + ] + }`), + "/home/src/workspaces/project/index.ts": `export const local = 1;`, + "/home/src/workspaces/project/app.vue": `export const app = 1;`, + }, + commandLineArgs: []string{"--build", "--verbose", "--dangerouslyLoadExternalPlugins"}, + contentMapperVersion: "1.0.0", + edits: []*tscEdit{ + noChange, + { + caption: "bump content mapper version", + edit: func(sys *TestSys) { + sys.contentMapperVersion = "2.0.0" + }, + }, + noChange, + }, + }, + } + for _, test := range testCases { + test.run(t, "contentMapperIdentity") + } +} + func TestBuildClean(t *testing.T) { t.Parallel() testCases := []*tscInput{ diff --git a/internal/execute/watcher.go b/internal/execute/watcher.go index 9b11c2a4dd4..4dbbf516148 100644 --- a/internal/execute/watcher.go +++ b/internal/execute/watcher.go @@ -115,7 +115,7 @@ func (w *Watcher) start(ctx context.Context) { w.wm.Lock() w.extendedConfigCache = &tsc.ExtendedConfigCache{} host := compiler.NewCompilerHost(w.sys.GetCurrentDirectory(), w.sys.FS(), w.sys.DefaultLibraryPath(), w.extendedConfigCache, getTraceFromSys(w.sys, w.config.Locale(), w.testing)) - w.program = incremental.ReadBuildInfoProgram(w.config, incremental.NewBuildInfoReader(host), host) + w.program = incremental.ReadBuildInfoProgram(w.config, tsc.ResolveContentMapperRunner(w.testing, w.config), incremental.NewBuildInfoReader(host), host) if w.configFileName != "" { w.configFilePaths = append([]string{w.configFileName}, w.config.ExtendedSourceFiles()...) @@ -304,8 +304,9 @@ func (w *Watcher) doBuild() error { } w.program = incremental.NewProgram(compiler.NewProgram(compiler.ProgramOptions{ - Config: w.config, - Host: host, + Config: w.config, + Host: host, + ContentMapperRunner: tsc.ResolveContentMapperRunner(w.testing, w.config), }), w.program, nil, w.testing != nil) result := w.compileAndEmit() diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index a571b4ce7ee..6067abd7af6 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -949,7 +949,7 @@ func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLi } program := compiler.NewProgram(programOptions) if config.CompilerOptions().Incremental.IsTrue() { - oldProgram := incremental.ReadBuildInfoProgram(config, getTestBuildInfoReader(host), host) + oldProgram := incremental.ReadBuildInfoProgram(config, programOptions.ContentMapperRunner, getTestBuildInfoReader(host), host) incrementalProgram := incremental.NewProgram(program, oldProgram, incremental.CreateHost(host), false) return incrementalProgram } diff --git a/internal/tsoptions/declscompiler.go b/internal/tsoptions/declscompiler.go index c16e085be5c..68683b831ac 100644 --- a/internal/tsoptions/declscompiler.go +++ b/internal/tsoptions/declscompiler.go @@ -250,6 +250,14 @@ var commonOptionsWithBuild = []*CommandLineOption{ DefaultValueDescription: diagnostics.X_4_unless_singleThreaded_is_passed, minValue: 1, }, + { + Name: "dangerouslyLoadExternalPlugins", + Kind: CommandLineOptionTypeBoolean, + Category: diagnostics.Command_line_Options, + IsCommandLineOnly: true, + Description: diagnostics.Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation, + DefaultValueDescription: false, + }, } var optionsForCompiler = []*CommandLineOption{ @@ -307,14 +315,6 @@ var optionsForCompiler = []*CommandLineOption{ Description: diagnostics.Print_names_of_files_that_are_part_of_the_compilation_and_then_stop_processing, DefaultValueDescription: false, }, - { - Name: "dangerouslyLoadExternalPlugins", - Kind: CommandLineOptionTypeBoolean, - Category: diagnostics.Command_line_Options, - IsCommandLineOnly: true, - Description: diagnostics.Allow_loading_external_content_mapper_plugins_that_execute_code_during_compilation, - DefaultValueDescription: false, - }, { Name: "ignoreConfig", Kind: CommandLineOptionTypeBoolean, diff --git a/testdata/baselines/reference/tsbuild/commandLine/help.js b/testdata/baselines/reference/tsbuild/commandLine/help.js index 37d3b89b958..b1bb466a2ac 100644 --- a/testdata/baselines/reference/tsbuild/commandLine/help.js +++ b/testdata/baselines/reference/tsbuild/commandLine/help.js @@ -134,6 +134,9 @@ Generate pprof CPU/memory profiles to the given directory. --checkers Set the number of checkers per project. +--dangerouslyLoadExternalPlugins +Allow loading external content mapper plugins that execute code during compilation. + --verbose, -v Enable verbose logging. diff --git a/testdata/baselines/reference/tsbuild/commandLine/locale.js b/testdata/baselines/reference/tsbuild/commandLine/locale.js index bbdd99f5836..da0b50e7def 100644 --- a/testdata/baselines/reference/tsbuild/commandLine/locale.js +++ b/testdata/baselines/reference/tsbuild/commandLine/locale.js @@ -134,6 +134,9 @@ Generate pprof CPU/memory profiles to the given directory. --checkers Set the number of checkers per project. +--dangerouslyLoadExternalPlugins +Allow loading external content mapper plugins that execute code during compilation. + --verbose, -v Enable verbose logging. diff --git a/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js b/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js new file mode 100644 index 00000000000..b169e09ace1 --- /dev/null +++ b/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js @@ -0,0 +1,168 @@ +currentDirectory::/home/src/workspaces/project +useCaseSensitiveFileNames::true +Input:: +//// [/home/src/workspaces/project/app.vue] *new* +export const app = 1; +//// [/home/src/workspaces/project/index.ts] *new* +export const local = 1; +//// [/home/src/workspaces/project/tsconfig.json] *new* +{ + "compilerOptions": { + "incremental": true + }, + "contentMappers": [ + { "command": ["vue"], "extensions": [".vue"] } + ] +} + +tsgo --build --verbose --dangerouslyLoadExternalPlugins +ExitStatus:: Success +Output:: +[HH:MM:SS AM] Projects in this build: + * tsconfig.json + +[HH:MM:SS AM] Project 'tsconfig.json' is out of date because output file 'tsconfig.tsbuildinfo' does not exist + +[HH:MM:SS AM] Building project 'tsconfig.json'... + +//// [/home/src/tslibs/TS/Lib/lib.es2025.full.d.ts] *Lib* +/// +interface Boolean {} +interface Function {} +interface CallableFunction {} +interface NewableFunction {} +interface IArguments {} +interface Number { toExponential: any; } +interface Object {} +interface RegExp {} +interface String { charAt: any; } +interface Array { length: number; [n: number]: T; } +interface ReadonlyArray {} +interface SymbolConstructor { + (desc?: string | number): symbol; + for(name: string): symbol; + readonly toStringTag: symbol; +} +declare var Symbol: SymbolConstructor; +interface Symbol { + readonly [Symbol.toStringTag]: string; +} +declare const console: { log(msg: any): void; }; +//// [/home/src/workspaces/project/app.vue.js] *new* +export const app = 1; + +//// [/home/src/workspaces/project/index.js] *new* +export const local = 1; + +//// [/home/src/workspaces/project/tsconfig.tsbuildinfo] *new* +{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue@1.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} +//// [/home/src/workspaces/project/tsconfig.tsbuildinfo.readable.baseline.txt] *new* +{ + "version": "FakeTSVersion", + "root": [ + { + "files": [ + "./app.vue", + "./index.ts" + ], + "original": [ + 2, + 3 + ] + } + ], + "fileNames": [ + "lib.es2025.full.d.ts", + "./app.vue", + "./index.ts" + ], + "fileInfos": [ + { + "fileName": "lib.es2025.full.d.ts", + "version": "8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };", + "signature": "8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };", + "affectsGlobalScope": true, + "impliedNodeFormat": "CommonJS", + "original": { + "version": "8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };", + "affectsGlobalScope": true, + "impliedNodeFormat": 1 + } + }, + { + "fileName": "./app.vue", + "version": "8e06c0ee6938980d692eefcee0c71e42-export const app = 1;", + "signature": "8e06c0ee6938980d692eefcee0c71e42-export const app = 1;", + "impliedNodeFormat": "None", + "original": { + "version": "8e06c0ee6938980d692eefcee0c71e42-export const app = 1;" + } + }, + { + "fileName": "./index.ts", + "version": "652b2b5c9eb278600aaa728787469dc8-export const local = 1;", + "signature": "652b2b5c9eb278600aaa728787469dc8-export const local = 1;", + "impliedNodeFormat": "CommonJS" + } + ], + "size": 1056 +} + +tsconfig.json:: +SemanticDiagnostics:: +*refresh* /home/src/tslibs/TS/Lib/lib.es2025.full.d.ts +*refresh* /home/src/workspaces/project/app.vue +*refresh* /home/src/workspaces/project/index.ts +Signatures:: + + +Edit [0]:: no change + +tsgo --build --verbose --dangerouslyLoadExternalPlugins +ExitStatus:: Success +Output:: +[HH:MM:SS AM] Projects in this build: + * tsconfig.json + +[HH:MM:SS AM] Project 'tsconfig.json' is up to date because newest input 'index.ts' is older than output 'tsconfig.tsbuildinfo' + + + + +Edit [1]:: bump content mapper version + +tsgo --build --verbose --dangerouslyLoadExternalPlugins +ExitStatus:: Success +Output:: +[HH:MM:SS AM] Projects in this build: + * tsconfig.json + +[HH:MM:SS AM] Project 'tsconfig.json' is out of date because buildinfo file 'tsconfig.tsbuildinfo' indicates there is change in compilerOptions + +[HH:MM:SS AM] Building project 'tsconfig.json'... + +//// [/home/src/workspaces/project/app.vue.js] *rewrite with same content* +//// [/home/src/workspaces/project/index.js] *rewrite with same content* +//// [/home/src/workspaces/project/tsconfig.tsbuildinfo] *modified* +{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue@2.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} +//// [/home/src/workspaces/project/tsconfig.tsbuildinfo.readable.baseline.txt] *rewrite with same content* + +tsconfig.json:: +SemanticDiagnostics:: +*refresh* /home/src/tslibs/TS/Lib/lib.es2025.full.d.ts +*refresh* /home/src/workspaces/project/app.vue +*refresh* /home/src/workspaces/project/index.ts +Signatures:: + + +Edit [2]:: no change + +tsgo --build --verbose --dangerouslyLoadExternalPlugins +ExitStatus:: Success +Output:: +[HH:MM:SS AM] Projects in this build: + * tsconfig.json + +[HH:MM:SS AM] Project 'tsconfig.json' is up to date because newest input 'index.ts' is older than output 'tsconfig.tsbuildinfo' + + From b825fb8cfc45d0886ac0c8e86979f674f57380f6 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Fri, 10 Jul 2026 13:27:41 -0700 Subject: [PATCH 04/37] Use node_modules package.json as manifest/identity --- internal/compiler/contentmapper_test.go | 52 +-------- internal/compiler/contentmapperrunner.go | 32 ------ internal/compiler/fileloader.go | 21 ++-- internal/compiler/program.go | 9 +- internal/contentmapper/contentmapper.go | 44 +++++++ internal/core/contentmapper.go | 9 -- internal/core/parsedoptions.go | 8 +- internal/diagnostics/diagnostics_generated.go | 20 ++++ .../diagnostics/extraDiagnosticMessages.json | 20 ++++ internal/execute/build/buildtask.go | 5 +- internal/execute/incremental/buildInfo.go | 21 +++- .../buildinfo_contentmapper_test.go | 33 +++--- internal/execute/incremental/incremental.go | 4 +- .../incremental/snapshottobuildinfo.go | 2 +- internal/execute/tsc.go | 2 +- internal/execute/tsc/compile.go | 5 +- internal/execute/tsctests/runner.go | 17 ++- internal/execute/tsctests/sys.go | 28 ++--- internal/execute/tsctests/tscbuild_test.go | 19 +++- internal/execute/watcher.go | 2 +- internal/packagejson/packagejson.go | 5 + internal/packagejson/packagejson_test.go | 2 + internal/testutil/harnessutil/harnessutil.go | 2 +- internal/tsoptions/contentmappers.go | 51 +++++++++ internal/tsoptions/contentmappers_test.go | 83 ++++++++++++++ internal/tsoptions/parsedcommandline.go | 5 +- internal/tsoptions/parsinghelpers.go | 13 ++- internal/tsoptions/tsconfigparsing.go | 107 ++++++++++++++++-- internal/tsoptions/tsconfigparsing_test.go | 41 ++++--- ...entMapperDisabledAfterRepeatedFailures.txt | 12 +- .../contentMapperFileFailure.txt | 2 +- .../contentMapperGeneratedCode.txt | 4 +- ...t-mapper-identity-change-forces-rebuild.js | 22 +++- 33 files changed, 483 insertions(+), 219 deletions(-) create mode 100644 internal/contentmapper/contentmapper.go delete mode 100644 internal/core/contentmapper.go create mode 100644 internal/tsoptions/contentmappers.go create mode 100644 internal/tsoptions/contentmappers_test.go diff --git a/internal/compiler/contentmapper_test.go b/internal/compiler/contentmapper_test.go index f0df553eed3..90bf2393e57 100644 --- a/internal/compiler/contentmapper_test.go +++ b/internal/compiler/contentmapper_test.go @@ -12,6 +12,7 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/bundled" "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/diagnosticwriter" @@ -25,61 +26,12 @@ import ( type fakeContentMapperRunner struct { transform func(fileName string, content string) (compiler.ContentMapperResult, error) - identity string } func (r fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { return r.transform(fileName, content) } -func (r fakeContentMapperRunner) Identity(mapper *core.ContentMapper) string { - if r.identity == "" { - return "vue-mapper@1.0.0" - } - return r.identity -} - -// identityRunner is a runner whose only interesting behavior is the identity it reports per mapper. -type identityRunner func(*core.ContentMapper) string - -func (f identityRunner) Identity(mapper *core.ContentMapper) string { return f(mapper) } - -func (identityRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { - return compiler.ContentMapperResult{}, nil -} - -func TestContentMapperIdentities(t *testing.T) { - t.Parallel() - - config := &tsoptions.ParsedCommandLine{ - ParsedConfig: &core.ParsedOptions{ - CompilerOptions: &core.CompilerOptions{}, - ContentMappers: []*core.ContentMapper{ - {Command: []string{"vue"}}, - {Command: []string{"svelte"}}, - {Command: []string{"anon"}}, - }, - }, - } - - // No runner means no identities to record. - assert.Assert(t, compiler.ContentMapperIdentities(nil, config) == nil) - - // Identities come from the runner; "anon" reports none and is omitted, and the result is sorted so - // reordering content mappers in tsconfig does not change it. - runner := identityRunner(func(mapper *core.ContentMapper) string { - switch mapper.Command[0] { - case "vue": - return "vue@2.0.0" - case "svelte": - return "svelte@3.0.0" - default: - return "" - } - }) - assert.DeepEqual(t, compiler.ContentMapperIdentities(runner, config), []string{"svelte@3.0.0", "vue@2.0.0"}) -} - const vueRawContent = "" // synthesizedSpanMap maps an entirely generated transformed text to no original location (a fully @@ -114,7 +66,7 @@ func newContentMapperProgram(t *testing.T, runner compiler.ContentMapperRunner, Module: core.ModuleKindESNext, ModuleResolution: core.ModuleResolutionKindBundler, }, - ContentMappers: []*core.ContentMapper{{Command: []string{"vue"}, Extensions: []string{".vue"}}}, + ContentMappers: []*contentmapper.Mapper{{Definition: contentmapper.Definition{Package: "vue", Extensions: []string{".vue"}}, Manifest: contentmapper.Manifest{Name: "vue-mapper", Version: "1.0.0"}}}, }, } return compiler.NewProgram(compiler.ProgramOptions{ diff --git a/internal/compiler/contentmapperrunner.go b/internal/compiler/contentmapperrunner.go index e4b19f21d46..4328326db89 100644 --- a/internal/compiler/contentmapperrunner.go +++ b/internal/compiler/contentmapperrunner.go @@ -1,12 +1,9 @@ package compiler import ( - "slices" - "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/spanmap" - "github.com/microsoft/typescript-go/internal/tsoptions" ) // ContentMapperResult is the outcome of transforming a foreign file's content into TypeScript. @@ -31,33 +28,4 @@ type ContentMapperRunner interface { // runner/coordinator hit a broken pipe, a process crash, or could not deserialize the mapper's // response. Transform(fileName string, content string) (result ContentMapperResult, err error) - - // Identity returns a stable identifier for the given mapper (e.g. "name@version"), used both to - // attribute diagnostics and to detect when a mapper implementation has changed between builds. How - // the identity is resolved — declared in the config, read from a manifest, reported during an - // initialize handshake, or some combination — is the runner's concern. An empty string means the - // mapper has no identity to report. - Identity(mapper *core.ContentMapper) string -} - -// ContentMapperIdentities returns the sorted identities that runner assigns to the content mappers in -// config, used to detect when a mapper implementation has changed between builds. Mappers with no -// identity are omitted, and the result is sorted so that merely reordering content mappers in tsconfig -// does not force a rebuild. Returns nil when there is no runner or no mapper reports an identity. -func ContentMapperIdentities(runner ContentMapperRunner, config *tsoptions.ParsedCommandLine) []string { - if runner == nil { - return nil - } - mappers := config.ContentMappers() - identities := make([]string, 0, len(mappers)) - for _, mapper := range mappers { - if identity := runner.Identity(mapper); identity != "" { - identities = append(identities, identity) - } - } - if len(identities) == 0 { - return nil - } - slices.Sort(identities) - return identities } diff --git a/internal/compiler/fileloader.go b/internal/compiler/fileloader.go index 57f4f43d422..45be93576fa 100644 --- a/internal/compiler/fileloader.go +++ b/internal/compiler/fileloader.go @@ -10,6 +10,7 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/module" @@ -69,8 +70,8 @@ type fileLoader struct { // contentMapperMu guards the content-mapper bookkeeping below, which is written concurrently as // content-mapped files are parsed across worker goroutines. contentMapperMu sync.Mutex - contentMapperForFile map[tspath.Path]*core.ContentMapper - contentMapperFailures map[*core.ContentMapper]int + contentMapperForFile map[tspath.Path]*contentmapper.Mapper + contentMapperFailures map[*contentmapper.Mapper]int contentMapperDiagnostics []*ast.Diagnostic } @@ -127,7 +128,7 @@ type processedFiles struct { // filesByPath for redirect files redirectFilesByPath map[tspath.Path]*redirectsFile // Association from a content-mapped source file path to the content mapper that produced it. - contentMapperForFile map[tspath.Path]*core.ContentMapper + contentMapperForFile map[tspath.Path]*contentmapper.Mapper // Program-level diagnostics reported when a content mapper fails fatally (reported once per mapper). contentMapperDiagnostics []*ast.Diagnostic finishedProcessing bool @@ -420,7 +421,7 @@ func (p *fileLoader) parseContentMappedFile(opts ast.SourceFileParseOptions) *as } mapper := p.matchContentMapper(opts.FileName) p.recordContentMapper(opts.Path, mapper) - label := p.opts.ContentMapperRunner.Identity(mapper) + label := mapper.Name if p.contentMapperDisabled(mapper) { // The mapper already exceeded its failure budget; add the file empty without re-reporting. return p.emptyContentMappedFile(opts, content) @@ -491,7 +492,7 @@ func (p *fileLoader) emptyContentMappedFile(opts ast.SourceFileParseOptions, con } // matchContentMapper returns the configured content mapper whose extensions include fileName. -func (p *fileLoader) matchContentMapper(fileName string) *core.ContentMapper { +func (p *fileLoader) matchContentMapper(fileName string) *contentmapper.Mapper { for _, mapper := range p.opts.Config.ContentMappers() { if tspath.FileExtensionIsOneOf(fileName, mapper.Extensions) { return mapper @@ -502,20 +503,20 @@ func (p *fileLoader) matchContentMapper(fileName string) *core.ContentMapper { // recordContentMapper associates a content-mapped file with the mapper that produced it, so the // program can report the mapping without re-matching extensions. -func (p *fileLoader) recordContentMapper(path tspath.Path, mapper *core.ContentMapper) { +func (p *fileLoader) recordContentMapper(path tspath.Path, mapper *contentmapper.Mapper) { if mapper == nil { return } p.contentMapperMu.Lock() defer p.contentMapperMu.Unlock() if p.contentMapperForFile == nil { - p.contentMapperForFile = make(map[tspath.Path]*core.ContentMapper) + p.contentMapperForFile = make(map[tspath.Path]*contentmapper.Mapper) } p.contentMapperForFile[path] = mapper } // contentMapperDisabled reports whether mapper has exceeded its failure budget and been disabled. -func (p *fileLoader) contentMapperDisabled(mapper *core.ContentMapper) bool { +func (p *fileLoader) contentMapperDisabled(mapper *contentmapper.Mapper) bool { if mapper == nil { return false } @@ -527,11 +528,11 @@ func (p *fileLoader) contentMapperDisabled(mapper *core.ContentMapper) bool { // recordContentMapperFailure counts a transform failure for mapper. It returns whether the failure // should be reported for this file (false once the mapper is already disabled). On the failure that // reaches maxContentMapperFailures it appends a single program diagnostic disabling the mapper. -func (p *fileLoader) recordContentMapperFailure(mapper *core.ContentMapper, label string) bool { +func (p *fileLoader) recordContentMapperFailure(mapper *contentmapper.Mapper, label string) bool { p.contentMapperMu.Lock() defer p.contentMapperMu.Unlock() if p.contentMapperFailures == nil { - p.contentMapperFailures = make(map[*core.ContentMapper]int) + p.contentMapperFailures = make(map[*contentmapper.Mapper]int) } if p.contentMapperFailures[mapper] >= maxContentMapperFailures { return false diff --git a/internal/compiler/program.go b/internal/compiler/program.go index 4af61024772..3e32e106e07 100644 --- a/internal/compiler/program.go +++ b/internal/compiler/program.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/typescript-go/internal/binder" "github.com/microsoft/typescript-go/internal/checker" "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/json" @@ -406,15 +407,9 @@ func (p *Program) Options() *core.CompilerOptions { return p.opts. // GetContentMapper returns the content mapper that produced the given source file, or nil if the // file was not produced by a content mapper. -func (p *Program) GetContentMapper(file *ast.SourceFile) *core.ContentMapper { +func (p *Program) GetContentMapper(file *ast.SourceFile) *contentmapper.Mapper { return p.contentMapperForFile[file.Path()] } - -// ContentMapperIdentities returns the sorted identities of the configured content mappers, as reported -// by the content mapper runner, for recording in build info. -func (p *Program) ContentMapperIdentities() []string { - return ContentMapperIdentities(p.opts.ContentMapperRunner, p.opts.Config) -} func (p *Program) CommandLine() *tsoptions.ParsedCommandLine { return p.opts.Config } func (p *Program) Host() CompilerHost { return p.opts.Host } func (p *Program) Tracing() *tracing.Tracing { return p.opts.Tracing } diff --git a/internal/contentmapper/contentmapper.go b/internal/contentmapper/contentmapper.go new file mode 100644 index 00000000000..9c260a79ebc --- /dev/null +++ b/internal/contentmapper/contentmapper.go @@ -0,0 +1,44 @@ +// Package contentmapper defines the types describing an external content mapper: a plugin that +// transforms foreign file content (e.g. .vue) into TypeScript during program construction. +// +// A mapper is declared in tsconfig (Definition), its implementation is described by fields in its npm +// package's package.json (Manifest), and the two are combined once the package is resolved (Mapper). +// Resolution itself lives in the tsoptions package (it needs node module resolution), keeping this +// package free of dependencies so lower layers can reference these types. +package contentmapper + +// Definition is a content mapper as declared in a tsconfig's "contentMappers": the npm package that +// implements the mapper and the foreign file extensions it registers. +type Definition struct { + Package string `json:"package"` + Extensions []string `json:"extensions"` +} + +// Manifest is the content-mapper information read from a package's package.json: its name and version +// (which form the mapper's identity) and the argv used to run it. +type Manifest struct { + Name string + Version string + Exec []string +} + +// Mapper is a resolved content mapper: its tsconfig Definition combined with the Manifest resolved from +// the package's package.json, plus the package directory used as the mapper's working directory. +type Mapper struct { + Definition + Manifest `json:"-"` + PackageDirectory string `json:"-"` +} + +// Identity returns the mapper's "name@version" identity, or just the name when it declares no version, +// or an empty string when the mapper has not been resolved to a name. +func (m *Mapper) Identity() string { + switch { + case m.Name == "": + return "" + case m.Version == "": + return m.Name + default: + return m.Name + "@" + m.Version + } +} diff --git a/internal/core/contentmapper.go b/internal/core/contentmapper.go deleted file mode 100644 index 7e155a3b191..00000000000 --- a/internal/core/contentmapper.go +++ /dev/null @@ -1,9 +0,0 @@ -package core - -// ContentMapper describes an external content mapper declared at the top level of a tsconfig. -// A content mapper registers foreign file extensions whose contents are transformed into -// TypeScript during program construction. -type ContentMapper struct { - Command []string `json:"command"` - Extensions []string `json:"extensions"` -} diff --git a/internal/core/parsedoptions.go b/internal/core/parsedoptions.go index 7b9ba89a170..7fbcdec6be9 100644 --- a/internal/core/parsedoptions.go +++ b/internal/core/parsedoptions.go @@ -1,11 +1,13 @@ package core +import "github.com/microsoft/typescript-go/internal/contentmapper" + type ParsedOptions struct { CompilerOptions *CompilerOptions `json:"compilerOptions"` WatchOptions *WatchOptions `json:"watchOptions"` TypeAcquisition *TypeAcquisition `json:"typeAcquisition"` - FileNames []string `json:"fileNames"` - ProjectReferences []*ProjectReference `json:"projectReferences"` - ContentMappers []*ContentMapper `json:"contentMappers"` + FileNames []string `json:"fileNames"` + ProjectReferences []*ProjectReference `json:"projectReferences"` + ContentMappers []*contentmapper.Mapper `json:"contentMappers"` } diff --git a/internal/diagnostics/diagnostics_generated.go b/internal/diagnostics/diagnostics_generated.go index d790ed0144a..5006c77b202 100644 --- a/internal/diagnostics/diagnostics_generated.go +++ b/internal/diagnostics/diagnostics_generated.go @@ -4334,6 +4334,16 @@ var The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_ori var This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the_original_file = &Message{code: 100031, category: CategoryMessage, key: "This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the__100031", text: "This location is in code generated by the content mapper '{0}' and has no corresponding location in the original file."} +var The_content_mapper_package_0_could_not_be_resolved = &Message{code: 100032, category: CategoryError, key: "The_content_mapper_package_0_could_not_be_resolved_100032", text: "The content mapper package '{0}' could not be resolved."} + +var The_package_json_of_the_content_mapper_package_0_could_not_be_parsed = &Message{code: 100033, category: CategoryError, key: "The_package_json_of_the_content_mapper_package_0_could_not_be_parsed_100033", text: "The 'package.json' of the content mapper package '{0}' could not be parsed."} + +var The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name = &Message{code: 100034, category: CategoryError, key: "The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name_100034", text: "The 'package.json' of the content mapper package '{0}' does not specify a 'name'."} + +var The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object = &Message{code: 100035, category: CategoryError, key: "The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object_100035", text: "The 'package.json' of the content mapper package '{0}' does not declare a 'tsContentMapper' object."} + +var The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings = &Message{code: 100036, category: CategoryError, key: "The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings_100036", text: "The 'tsContentMapper.exec' of the content mapper package '{0}' must be a non-empty array of strings."} + func keyToMessage(key Key) *Message { switch key { case "Unterminated_string_literal_1002": @@ -8668,6 +8678,16 @@ func keyToMessage(key Key) *Message { return The_content_mapper_0_produced_a_verbatim_mapping_that_does_not_match_the_original_content_output_offset_1_original_offset_2 case "This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the__100031": return This_location_is_in_code_generated_by_the_content_mapper_0_and_has_no_corresponding_location_in_the_original_file + case "The_content_mapper_package_0_could_not_be_resolved_100032": + return The_content_mapper_package_0_could_not_be_resolved + case "The_package_json_of_the_content_mapper_package_0_could_not_be_parsed_100033": + return The_package_json_of_the_content_mapper_package_0_could_not_be_parsed + case "The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name_100034": + return The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name + case "The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object_100035": + return The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object + case "The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings_100036": + return The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings default: return nil } diff --git a/internal/diagnostics/extraDiagnosticMessages.json b/internal/diagnostics/extraDiagnosticMessages.json index 90a2deed438..391aa9f6c34 100644 --- a/internal/diagnostics/extraDiagnosticMessages.json +++ b/internal/diagnostics/extraDiagnosticMessages.json @@ -178,5 +178,25 @@ "This location is in code generated by the content mapper '{0}' and has no corresponding location in the original file.": { "category": "Message", "code": 100031 + }, + "The content mapper package '{0}' could not be resolved.": { + "category": "Error", + "code": 100032 + }, + "The 'package.json' of the content mapper package '{0}' could not be parsed.": { + "category": "Error", + "code": 100033 + }, + "The 'package.json' of the content mapper package '{0}' does not specify a 'name'.": { + "category": "Error", + "code": 100034 + }, + "The 'package.json' of the content mapper package '{0}' does not declare a 'tsContentMapper' object.": { + "category": "Error", + "code": 100035 + }, + "The 'tsContentMapper.exec' of the content mapper package '{0}' must be a non-empty array of strings.": { + "category": "Error", + "code": 100036 } } diff --git a/internal/execute/build/buildtask.go b/internal/execute/build/buildtask.go index b71cb4bb46f..a0b74f337b2 100644 --- a/internal/execute/build/buildtask.go +++ b/internal/execute/build/buildtask.go @@ -201,7 +201,7 @@ func (t *BuildTask) compileAndEmit(orchestrator *Orchestrator, path tspath.Path) mapperRunner := tsc.ResolveContentMapperRunner(orchestrator.opts.Testing, t.resolved) var oldProgram *incremental.Program if !orchestrator.opts.Command.BuildOptions.Force.IsTrue() { - oldProgram = incremental.ReadBuildInfoProgram(t.resolved, mapperRunner, orchestrator.host, orchestrator.host) + oldProgram = incremental.ReadBuildInfoProgram(t.resolved, orchestrator.host, orchestrator.host) } compileTimes.BuildInfoReadTime = orchestrator.opts.Sys.Now().Sub(buildInfoReadStart) parseStart := orchestrator.opts.Sys.Now() @@ -346,8 +346,7 @@ func (t *BuildTask) getUpToDateStatus(orchestrator *Orchestrator, configPath tsp } // If a configured content mapper's identity has changed, files it produced may be stale. - mapperRunner := tsc.ResolveContentMapperRunner(orchestrator.opts.Testing, t.resolved) - if !buildInfo.ContentMapperIdentitiesMatch(compiler.ContentMapperIdentities(mapperRunner, t.resolved)) { + if !buildInfo.ContentMapperIdentitiesMatch(incremental.ContentMapperIdentities(t.resolved)) { return &upToDateStatus{kind: upToDateStatusTypeOutOfDateOptions, data: buildInfoPath} } diff --git a/internal/execute/incremental/buildInfo.go b/internal/execute/incremental/buildInfo.go index 388c0ce400e..cbe86320641 100644 --- a/internal/execute/incremental/buildInfo.go +++ b/internal/execute/incremental/buildInfo.go @@ -493,8 +493,27 @@ func (b *BuildInfo) IsValidVersion() bool { return b.Version == core.Version() } +// ContentMapperIdentities returns the sorted identities of the content mappers configured in config, as +// resolved during tsconfig parsing, used to detect when a mapper implementation has changed between +// builds. Mappers with no resolved name are omitted, and the result is sorted so that merely reordering +// content mappers in tsconfig does not force a rebuild. Returns nil when no mapper has an identity. +func ContentMapperIdentities(config *tsoptions.ParsedCommandLine) []string { + mappers := config.ContentMappers() + identities := make([]string, 0, len(mappers)) + for _, mapper := range mappers { + if identity := mapper.Identity(); identity != "" { + identities = append(identities, identity) + } + } + if len(identities) == 0 { + return nil + } + slices.Sort(identities) + return identities +} + // ContentMapperIdentitiesMatch reports whether the content mapper identities recorded in this build info -// match the given current identities (as produced by compiler.ContentMapperIdentities). +// match the given current identities (as produced by ContentMapperIdentities). func (b *BuildInfo) ContentMapperIdentitiesMatch(current []string) bool { return slices.Equal(b.ContentMapperIdentities, current) } diff --git a/internal/execute/incremental/buildinfo_contentmapper_test.go b/internal/execute/incremental/buildinfo_contentmapper_test.go index 8cf33639991..629c1ece99c 100644 --- a/internal/execute/incremental/buildinfo_contentmapper_test.go +++ b/internal/execute/incremental/buildinfo_contentmapper_test.go @@ -3,14 +3,14 @@ package incremental_test import ( "testing" - "github.com/microsoft/typescript-go/internal/compiler" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/execute/incremental" "github.com/microsoft/typescript-go/internal/tsoptions" "gotest.tools/v3/assert" ) -func configWithMappers(mappers ...*core.ContentMapper) *tsoptions.ParsedCommandLine { +func configWithMappers(mappers ...*contentmapper.Mapper) *tsoptions.ParsedCommandLine { return &tsoptions.ParsedCommandLine{ ParsedConfig: &core.ParsedOptions{ CompilerOptions: &core.CompilerOptions{}, @@ -19,17 +19,19 @@ func configWithMappers(mappers ...*core.ContentMapper) *tsoptions.ParsedCommandL } } -// fakeContentMapperRunner reports a fixed identity for every mapper; Transform is never exercised here. -type fakeContentMapperRunner struct { - identity string -} +func TestContentMapperIdentities(t *testing.T) { + t.Parallel() -func (r fakeContentMapperRunner) Identity(*core.ContentMapper) string { - return r.identity -} + assert.Assert(t, incremental.ContentMapperIdentities(configWithMappers()) == nil) -func (fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { - return compiler.ContentMapperResult{}, nil + // Identities come from the mappers' resolved name/version; a mapper with no name is omitted, and the + // result is sorted so reordering content mappers in tsconfig does not change it. + config := configWithMappers( + &contentmapper.Mapper{Definition: contentmapper.Definition{Package: "vue"}, Manifest: contentmapper.Manifest{Name: "vue", Version: "2.0.0"}}, + &contentmapper.Mapper{Definition: contentmapper.Definition{Package: "svelte"}, Manifest: contentmapper.Manifest{Name: "svelte", Version: "3.0.0"}}, + &contentmapper.Mapper{Definition: contentmapper.Definition{Package: "anon"}}, + ) + assert.DeepEqual(t, incremental.ContentMapperIdentities(config), []string{"svelte@3.0.0", "vue@2.0.0"}) } func TestContentMapperIdentitiesMatch(t *testing.T) { @@ -57,16 +59,15 @@ func TestReadBuildInfoProgramContentMapperIdentityMismatch(t *testing.T) { t.Parallel() // An otherwise-valid, incremental build info whose recorded mapper identity differs from the one the - // runner now reports cannot be reused: the old program is discarded (nil) so the project is rebuilt. - // The host is never touched because we bail before reconstructing the snapshot. + // config now resolves to cannot be reused: the old program is discarded (nil) so the project is + // rebuilt. The host is never touched because we bail before reconstructing the snapshot. buildInfo := &incremental.BuildInfo{ Version: core.Version(), FileNames: []string{"/src/a.ts"}, ContentMapperIdentities: []string{"vue@1.0.0"}, } - config := configWithMappers(&core.ContentMapper{Extensions: []string{".vue"}}) - runner := fakeContentMapperRunner{identity: "vue@2.0.0"} + config := configWithMappers(&contentmapper.Mapper{Definition: contentmapper.Definition{Package: "vue", Extensions: []string{".vue"}}, Manifest: contentmapper.Manifest{Name: "vue", Version: "2.0.0"}}) - program := incremental.ReadBuildInfoProgram(config, runner, fakeBuildInfoReader{buildInfo}, nil) + program := incremental.ReadBuildInfoProgram(config, fakeBuildInfoReader{buildInfo}, nil) assert.Assert(t, program == nil, "expected the old program to be discarded when the mapper identity changed") } diff --git a/internal/execute/incremental/incremental.go b/internal/execute/incremental/incremental.go index e2e770b00a3..804ab1ec792 100644 --- a/internal/execute/incremental/incremental.go +++ b/internal/execute/incremental/incremental.go @@ -41,7 +41,7 @@ func NewBuildInfoReader( return &buildInfoReader{host: host} } -func ReadBuildInfoProgram(config *tsoptions.ParsedCommandLine, runner compiler.ContentMapperRunner, reader BuildInfoReader, host compiler.CompilerHost) *Program { +func ReadBuildInfoProgram(config *tsoptions.ParsedCommandLine, reader BuildInfoReader, host compiler.CompilerHost) *Program { // Read buildInfo file buildInfo := reader.ReadBuildInfo(config) if buildInfo == nil || !buildInfo.IsValidVersion() || !buildInfo.IsIncremental() { @@ -49,7 +49,7 @@ func ReadBuildInfoProgram(config *tsoptions.ParsedCommandLine, runner compiler.C } // If any configured content mapper's identity has changed, files it produced may be stale, so the // old program cannot be reused. - if !buildInfo.ContentMapperIdentitiesMatch(compiler.ContentMapperIdentities(runner, config)) { + if !buildInfo.ContentMapperIdentitiesMatch(ContentMapperIdentities(config)) { return nil } diff --git a/internal/execute/incremental/snapshottobuildinfo.go b/internal/execute/incremental/snapshottobuildinfo.go index 8e9f4e3a5bc..f4f0eec9fa5 100644 --- a/internal/execute/incremental/snapshottobuildinfo.go +++ b/internal/execute/incremental/snapshottobuildinfo.go @@ -52,7 +52,7 @@ func snapshotToBuildInfo(snapshot *snapshot, program *compiler.Program, buildInf buildInfo.Errors = snapshot.hasErrors.IsTrue() buildInfo.SemanticErrors = snapshot.hasSemanticErrors buildInfo.CheckPending = snapshot.checkPending - buildInfo.ContentMapperIdentities = program.ContentMapperIdentities() + buildInfo.ContentMapperIdentities = ContentMapperIdentities(program.CommandLine()) to.setPackageJsons() return buildInfo } diff --git a/internal/execute/tsc.go b/internal/execute/tsc.go index 576c78fa383..47871b9ae69 100644 --- a/internal/execute/tsc.go +++ b/internal/execute/tsc.go @@ -293,7 +293,7 @@ func performIncrementalCompilation( host := compiler.NewCachedFSCompilerHost(sys.GetCurrentDirectory(), sys.FS(), sys.DefaultLibraryPath(), extendedConfigCache, getTraceFromSys(sys, config.Locale(), testing)) mapperRunner := tsc.ResolveContentMapperRunner(testing, config) buildInfoReadStart := sys.Now() - oldProgram := incremental.ReadBuildInfoProgram(config, mapperRunner, incremental.NewBuildInfoReader(host), host) + oldProgram := incremental.ReadBuildInfoProgram(config, incremental.NewBuildInfoReader(host), host) compileTimes.BuildInfoReadTime = sys.Now().Sub(buildInfoReadStart) tr := startTracingIfNeeded(sys, config, testing) diff --git a/internal/execute/tsc/compile.go b/internal/execute/tsc/compile.go index be77d53faac..2e61332ff18 100644 --- a/internal/execute/tsc/compile.go +++ b/internal/execute/tsc/compile.go @@ -7,7 +7,6 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler" - "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/execute/incremental" "github.com/microsoft/typescript-go/internal/locale" @@ -62,14 +61,14 @@ type CommandLineTesting interface { OnWatchStatusReportEnd() GetTrace(w io.Writer, locale locale.Locale) func(msg *diagnostics.Message, args ...any) OnProgram(program *incremental.Program) - GetContentMapperRunner(mappers []*core.ContentMapper) compiler.ContentMapperRunner + GetContentMapperRunner(config *tsoptions.ParsedCommandLine) compiler.ContentMapperRunner } func ResolveContentMapperRunner(testing CommandLineTesting, config *tsoptions.ParsedCommandLine) compiler.ContentMapperRunner { if testing == nil { return nil } - return testing.GetContentMapperRunner(config.ContentMappers()) + return testing.GetContentMapperRunner(config) } type CompileTimes struct { diff --git a/internal/execute/tsctests/runner.go b/internal/execute/tsctests/runner.go index cebee80f315..38c7e9c2089 100644 --- a/internal/execute/tsctests/runner.go +++ b/internal/execute/tsctests/runner.go @@ -31,15 +31,14 @@ var noChangeOnlyEdit = []*tscEdit{ } type tscInput struct { - subScenario string - commandLineArgs []string - files FileMap - cwd string - edits []*tscEdit - env map[string]string - ignoreCase bool - windowsStyleRoot string - contentMapperVersion string + subScenario string + commandLineArgs []string + files FileMap + cwd string + edits []*tscEdit + env map[string]string + ignoreCase bool + windowsStyleRoot string } func (test *tscInput) executeCommand(sys *TestSys, baselineBuilder *strings.Builder, commandLineArgs []string) tsc.CommandLineResult { diff --git a/internal/execute/tsctests/sys.go b/internal/execute/tsctests/sys.go index eff58b138ae..60c01da9574 100644 --- a/internal/execute/tsctests/sys.go +++ b/internal/execute/tsctests/sys.go @@ -134,7 +134,6 @@ func newTestSys(tscInput *tscInput, forIncrementalCorrectness bool) *TestSys { }, currentWrite) sys.env = tscInput.env sys.forIncrementalCorrectness = forIncrementalCorrectness - sys.contentMapperVersion = tscInput.contentMapperVersion sys.mockWatchBackend = NewMockWatchBackend() sys.mockWatchBackend.DirectoryExists = sys.fs.FS.DirectoryExists sys.fsDiffer = &fsbaselineutil.FSDiffer{ @@ -168,8 +167,6 @@ type TestSys struct { cwd string env map[string]string clock *TestClock - - contentMapperVersion string } var ( @@ -327,28 +324,17 @@ func (s *TestSys) WatchBackend() watchmanager.WatchBackend { } // GetContentMapperRunner stands in for a real mapper host, returning a runner that transforms foreign -// files verbatim and derives each mapper's identity from its command and the sys-controlled version (so -// a test can bump the version between builds to simulate a mapper upgrade). -func (s *TestSys) GetContentMapperRunner(mappers []*core.ContentMapper) compiler.ContentMapperRunner { - if len(mappers) == 0 { +// files verbatim. Mapper identity is resolved during tsconfig parsing, so the runner only performs +// transformation. The content is expected to already be valid TypeScript, which keeps the +// identity-preserving span map valid. +func (s *TestSys) GetContentMapperRunner(config *tsoptions.ParsedCommandLine) compiler.ContentMapperRunner { + if len(config.ContentMappers()) == 0 { return nil } - return testContentMapperRunner{version: s.contentMapperVersion} -} - -// testContentMapperRunner transforms a foreign file's content verbatim into TypeScript. The content is -// therefore expected to already be valid TypeScript, which keeps the identity-preserving span map valid. -type testContentMapperRunner struct { - version string + return testContentMapperRunner{} } -func (r testContentMapperRunner) Identity(mapper *core.ContentMapper) string { - name := mapper.Command[0] - if r.version == "" { - return name - } - return name + "@" + r.version -} +type testContentMapperRunner struct{} func (testContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { return compiler.ContentMapperResult{ diff --git a/internal/execute/tsctests/tscbuild_test.go b/internal/execute/tsctests/tscbuild_test.go index a9f68ffef5e..d6993e4a54b 100644 --- a/internal/execute/tsctests/tscbuild_test.go +++ b/internal/execute/tsctests/tscbuild_test.go @@ -309,20 +309,29 @@ func TestBuildContentMapperIdentity(t *testing.T) { "incremental": true }, "contentMappers": [ - { "command": ["vue"], "extensions": [".vue"] } + { "package": "vue-ts-mapper", "extensions": [".vue"] } ] }`), "/home/src/workspaces/project/index.ts": `export const local = 1;`, "/home/src/workspaces/project/app.vue": `export const app = 1;`, + "/home/src/workspaces/project/node_modules/vue-ts-mapper/package.json": stringtestutil.Dedent(` + { + "name": "vue-ts-mapper", + "version": "1.0.0", + "tsContentMapper": { "exec": ["node", "./mapper.js"] } + }`), }, - commandLineArgs: []string{"--build", "--verbose", "--dangerouslyLoadExternalPlugins"}, - contentMapperVersion: "1.0.0", + commandLineArgs: []string{"--build", "--verbose", "--dangerouslyLoadExternalPlugins"}, edits: []*tscEdit{ noChange, { - caption: "bump content mapper version", + caption: "upgrade the content mapper package to a new version", edit: func(sys *TestSys) { - sys.contentMapperVersion = "2.0.0" + sys.replaceFileText( + "/home/src/workspaces/project/node_modules/vue-ts-mapper/package.json", + `"version": "1.0.0"`, + `"version": "2.0.0"`, + ) }, }, noChange, diff --git a/internal/execute/watcher.go b/internal/execute/watcher.go index 4dbbf516148..1750a1ca99f 100644 --- a/internal/execute/watcher.go +++ b/internal/execute/watcher.go @@ -115,7 +115,7 @@ func (w *Watcher) start(ctx context.Context) { w.wm.Lock() w.extendedConfigCache = &tsc.ExtendedConfigCache{} host := compiler.NewCompilerHost(w.sys.GetCurrentDirectory(), w.sys.FS(), w.sys.DefaultLibraryPath(), w.extendedConfigCache, getTraceFromSys(w.sys, w.config.Locale(), w.testing)) - w.program = incremental.ReadBuildInfoProgram(w.config, tsc.ResolveContentMapperRunner(w.testing, w.config), incremental.NewBuildInfoReader(host), host) + w.program = incremental.ReadBuildInfoProgram(w.config, incremental.NewBuildInfoReader(host), host) if w.configFileName != "" { w.configFilePaths = append([]string{w.configFileName}, w.config.ExtendedSourceFiles()...) diff --git a/internal/packagejson/packagejson.go b/internal/packagejson/packagejson.go index d9ff92879a9..2dd792ffda8 100644 --- a/internal/packagejson/packagejson.go +++ b/internal/packagejson/packagejson.go @@ -111,6 +111,11 @@ type Fields struct { HeaderFields PathFields DependencyFields + ContentMapper Expected[ContentMapperFields] `json:"tsContentMapper"` +} + +type ContentMapperFields struct { + Exec Expected[[]string] `json:"exec"` } func Parse(data []byte) (Fields, error) { diff --git a/internal/packagejson/packagejson_test.go b/internal/packagejson/packagejson_test.go index 832ec5221a7..4af6fa5af90 100644 --- a/internal/packagejson/packagejson_test.go +++ b/internal/packagejson/packagejson_test.go @@ -96,6 +96,8 @@ func TestParse(t *testing.T) { packagejson.HeaderFields{}, packagejson.Expected[string]{}, packagejson.Expected[map[string]string]{}, + packagejson.Expected[[]string]{}, + packagejson.Expected[packagejson.ContentMapperFields]{}, packagejson.ExportsOrImports{}, )) }) diff --git a/internal/testutil/harnessutil/harnessutil.go b/internal/testutil/harnessutil/harnessutil.go index 6067abd7af6..a571b4ce7ee 100644 --- a/internal/testutil/harnessutil/harnessutil.go +++ b/internal/testutil/harnessutil/harnessutil.go @@ -949,7 +949,7 @@ func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLi } program := compiler.NewProgram(programOptions) if config.CompilerOptions().Incremental.IsTrue() { - oldProgram := incremental.ReadBuildInfoProgram(config, programOptions.ContentMapperRunner, getTestBuildInfoReader(host), host) + oldProgram := incremental.ReadBuildInfoProgram(config, getTestBuildInfoReader(host), host) incrementalProgram := incremental.NewProgram(program, oldProgram, incremental.CreateHost(host), false) return incrementalProgram } diff --git a/internal/tsoptions/contentmappers.go b/internal/tsoptions/contentmappers.go new file mode 100644 index 00000000000..b408078e162 --- /dev/null +++ b/internal/tsoptions/contentmappers.go @@ -0,0 +1,51 @@ +package tsoptions + +import ( + "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/contentmapper" + "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/diagnostics" + "github.com/microsoft/typescript-go/internal/module" + "github.com/microsoft/typescript-go/internal/packagejson" + "github.com/microsoft/typescript-go/internal/tspath" +) + +// resolveContentMapperManifest locates packageName in node_modules (walking up from the directory of +// containingFile via node module resolution) and reads its package.json to produce the mapper's manifest +// and package directory. It never executes the package. On failure it returns a diagnostic describing why +// the mapper could not be resolved; on success the diagnostic is nil. +func resolveContentMapperManifest(host ParseConfigHost, containingFile string, packageName string) (contentmapper.Manifest, string, *ast.Diagnostic) { + resolver := module.NewResolver(host, &core.CompilerOptions{ModuleResolution: core.ModuleResolutionKindBundler}, "", "", nil) + resolved := resolver.ResolvePackageDirectory(packageName, containingFile, core.ResolutionModeNone, nil) + if resolved == nil || resolved.ResolvedFileName == "" { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_content_mapper_package_0_could_not_be_resolved, packageName) + } + packageDirectory := resolved.ResolvedFileName + + packageJsonPath := tspath.CombinePaths(packageDirectory, "package.json") + contents, ok := host.FS().ReadFile(packageJsonPath) + if !ok { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_content_mapper_package_0_could_not_be_resolved, packageName) + } + fields, err := packagejson.Parse([]byte(contents)) + if err != nil { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_package_json_of_the_content_mapper_package_0_could_not_be_parsed, packageName) + } + name, _ := fields.Name.GetValue() + if name == "" { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name, packageName) + } + version, _ := fields.Version.GetValue() + + // A content mapper package must declare how to run it: a "tsContentMapper" object with a non-empty + // "exec" array of strings. + cm, ok := fields.ContentMapper.GetValue() + if !ok { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object, packageName) + } + exec, ok := cm.Exec.GetValue() + if !ok || len(exec) == 0 { + return contentmapper.Manifest{}, "", ast.NewCompilerDiagnostic(diagnostics.The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings, packageName) + } + return contentmapper.Manifest{Name: name, Version: version, Exec: exec}, packageDirectory, nil +} diff --git a/internal/tsoptions/contentmappers_test.go b/internal/tsoptions/contentmappers_test.go new file mode 100644 index 00000000000..e80e93f4da9 --- /dev/null +++ b/internal/tsoptions/contentmappers_test.go @@ -0,0 +1,83 @@ +package tsoptions + +import ( + "testing" + + "github.com/microsoft/typescript-go/internal/diagnostics" + "github.com/microsoft/typescript-go/internal/vfs" + "github.com/microsoft/typescript-go/internal/vfs/vfstest" + "gotest.tools/v3/assert" +) + +type resolveContentMapperHost struct { + fs vfs.FS +} + +func (h resolveContentMapperHost) FS() vfs.FS { return h.fs } +func (h resolveContentMapperHost) GetCurrentDirectory() string { return "/home/project" } + +func TestResolveContentMapperManifest(t *testing.T) { + t.Parallel() + + host := resolveContentMapperHost{fs: vfstest.FromMap(map[string]string{ + "/home/project/node_modules/vue-ts-mapper/package.json": `{ + "name": "vue-ts-mapper", + "version": "1.2.3", + "tsContentMapper": { "exec": ["node", "./dist/mapper.js"] } + }`, + "/home/node_modules/@scope/noversion/package.json": `{ + "name": "@scope/noversion", + "tsContentMapper": { "exec": ["run"] } + }`, + "/home/project/node_modules/no-name/package.json": `{ + "version": "1.0.0" + }`, + "/home/project/node_modules/no-manifest/package.json": `{ + "name": "no-manifest" + }`, + "/home/project/node_modules/no-exec/package.json": `{ + "name": "no-exec", + "tsContentMapper": {} + }`, + "/home/project/node_modules/bad-exec/package.json": `{ + "name": "bad-exec", + "tsContentMapper": { "exec": "node ./mapper.js" } + }`, + }, true /*useCaseSensitiveFileNames*/)} + + // Name, version, and the verbatim exec argv are preserved. + manifest, packageDirectory, diagnostic := resolveContentMapperManifest(host, "/home/project/tsconfig.json", "vue-ts-mapper") + assert.Assert(t, diagnostic == nil) + assert.Equal(t, manifest.Name, "vue-ts-mapper") + assert.Equal(t, manifest.Version, "1.2.3") + assert.Equal(t, packageDirectory, "/home/project/node_modules/vue-ts-mapper") + assert.DeepEqual(t, manifest.Exec, []string{"node", "./dist/mapper.js"}) + + // Resolution walks up node_modules; a package with no version resolves to a name and empty version. + manifest, _, diagnostic = resolveContentMapperManifest(host, "/home/project/src/tsconfig.json", "@scope/noversion") + assert.Assert(t, diagnostic == nil) + assert.Equal(t, manifest.Name, "@scope/noversion") + assert.Equal(t, manifest.Version, "") + + // A package that is not installed reports a resolution diagnostic. + _, _, diagnostic = resolveContentMapperManifest(host, "/home/project/tsconfig.json", "missing-mapper") + assert.Assert(t, diagnostic != nil) + assert.Equal(t, diagnostic.Code(), diagnostics.The_content_mapper_package_0_could_not_be_resolved.Code()) + + // A package whose package.json has no name reports a diagnostic. + _, _, diagnostic = resolveContentMapperManifest(host, "/home/project/tsconfig.json", "no-name") + assert.Assert(t, diagnostic != nil) + assert.Equal(t, diagnostic.Code(), diagnostics.The_package_json_of_the_content_mapper_package_0_does_not_specify_a_name.Code()) + + // A package that does not declare a "tsContentMapper" object reports a diagnostic. + _, _, diagnostic = resolveContentMapperManifest(host, "/home/project/tsconfig.json", "no-manifest") + assert.Assert(t, diagnostic != nil) + assert.Equal(t, diagnostic.Code(), diagnostics.The_package_json_of_the_content_mapper_package_0_does_not_declare_a_tsContentMapper_object.Code()) + + // A "tsContentMapper" with no "exec", or an "exec" of the wrong type, reports a diagnostic. + for _, pkg := range []string{"no-exec", "bad-exec"} { + _, _, diagnostic = resolveContentMapperManifest(host, "/home/project/tsconfig.json", pkg) + assert.Assert(t, diagnostic != nil, "expected a diagnostic for %s", pkg) + assert.Equal(t, diagnostic.Code(), diagnostics.The_tsContentMapper_exec_of_the_content_mapper_package_0_must_be_a_non_empty_array_of_strings.Code()) + } +} diff --git a/internal/tsoptions/parsedcommandline.go b/internal/tsoptions/parsedcommandline.go index ef05e625a22..152e059b40a 100644 --- a/internal/tsoptions/parsedcommandline.go +++ b/internal/tsoptions/parsedcommandline.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/microsoft/typescript-go/internal/ast" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/glob" @@ -309,7 +310,7 @@ func (p *ParsedCommandLine) ProjectReferences() []*core.ProjectReference { return p.ParsedConfig.ProjectReferences } -func (p *ParsedCommandLine) ContentMappers() []*core.ContentMapper { +func (p *ParsedCommandLine) ContentMappers() []*contentmapper.Mapper { if p == nil || p.ParsedConfig == nil { return nil } @@ -319,7 +320,7 @@ func (p *ParsedCommandLine) ContentMappers() []*core.ContentMapper { // ContentMapperExtensions returns the flattened list of file extensions registered by the // config's content mappers. func (p *ParsedCommandLine) ContentMapperExtensions() []string { - return core.FlatMap(p.ContentMappers(), func(m *core.ContentMapper) []string { + return core.FlatMap(p.ContentMappers(), func(m *contentmapper.Mapper) []string { return m.Extensions }) } diff --git a/internal/tsoptions/parsinghelpers.go b/internal/tsoptions/parsinghelpers.go index 7b5dad767c2..6b45e3f36f0 100644 --- a/internal/tsoptions/parsinghelpers.go +++ b/internal/tsoptions/parsinghelpers.go @@ -6,6 +6,7 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/tspath" @@ -85,18 +86,18 @@ func parseProjectReference(json any) []*core.ProjectReference { return result } -func parseContentMapper(json any) (*core.ContentMapper, []*ast.Diagnostic) { +func parseContentMapper(json any) (*contentmapper.Mapper, []*ast.Diagnostic) { v, ok := json.(*collections.OrderedMap[string, any]) if !ok { return nil, nil } var errors []*ast.Diagnostic - mapper := &core.ContentMapper{} - if command, ok := v.Get("command"); ok { - if strs, valid := parseStringArrayStrict(command); valid { - mapper.Command = strs + mapper := &contentmapper.Mapper{} + if pkg, ok := v.Get("package"); ok { + if str, valid := pkg.(string); valid && str != "" { + mapper.Package = str } else { - errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Compiler_option_0_requires_a_value_of_type_1, "contentMapper.command", "string[]")) + errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Compiler_option_0_requires_a_value_of_type_1, "contentMapper.package", "string")) } } if extensions, ok := v.Get("extensions"); ok { diff --git a/internal/tsoptions/tsconfigparsing.go b/internal/tsoptions/tsconfigparsing.go index eb786e0b28b..216611abe93 100644 --- a/internal/tsoptions/tsconfigparsing.go +++ b/internal/tsoptions/tsconfigparsing.go @@ -8,6 +8,7 @@ import ( "github.com/microsoft/typescript-go/internal/ast" "github.com/microsoft/typescript-go/internal/collections" + "github.com/microsoft/typescript-go/internal/contentmapper" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/debug" "github.com/microsoft/typescript-go/internal/diagnostics" @@ -15,6 +16,7 @@ import ( "github.com/microsoft/typescript-go/internal/locale" "github.com/microsoft/typescript-go/internal/module" "github.com/microsoft/typescript-go/internal/parser" + "github.com/microsoft/typescript-go/internal/scanner" "github.com/microsoft/typescript-go/internal/tspath" "github.com/microsoft/typescript-go/internal/vfs" "github.com/microsoft/typescript-go/internal/vfs/vfsmatch" @@ -1322,13 +1324,21 @@ func parseJsonConfigFileContentWorker( sourceFile.configFileSpecs = &configFileSpecs } - var contentMappers []*core.ContentMapper + var contentMapperSourceFile *ast.SourceFile + if sourceFile != nil { + contentMapperSourceFile = sourceFile.SourceFile + } + var contentMappers []*contentmapper.Mapper + var contentMapperIndices []int contentMappersOfRaw := getPropFromRaw("contentMappers", func(element any) bool { return reflect.TypeOf(element) == orderedMapType }, "object") - for _, element := range contentMappersOfRaw.sliceValue { + for i, element := range contentMappersOfRaw.sliceValue { mapper, mapperErrors := parseContentMapper(element) - errors = append(errors, mapperErrors...) + for _, mapperError := range mapperErrors { + errors = append(errors, setContentMapperDiagnosticLocation(mapperError, contentMapperSourceFile, getContentMapperSyntax(contentMapperSourceFile, i, ""))) + } if mapper != nil { contentMappers = append(contentMappers, mapper) + contentMapperIndices = append(contentMapperIndices, i) } } totalContentMapperExtensions := 0 @@ -1338,16 +1348,17 @@ func parseJsonConfigFileContentWorker( seenContentMapperExtensions := make(map[string]struct{}, totalContentMapperExtensions) contentMapperExtensions := make([]string, 0, totalContentMapperExtensions) nativeExtensions := core.Flatten(tspath.AllSupportedExtensionsWithJson) - for _, mapper := range contentMappers { + for j, mapper := range contentMappers { for _, ext := range mapper.Extensions { + extNode := getContentMapperExtensionSyntax(contentMapperSourceFile, contentMapperIndices[j], ext) switch { case !strings.HasPrefix(ext, "."): - errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_must_begin_with_a, ext)) + errors = append(errors, setContentMapperDiagnosticLocation(ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_must_begin_with_a, ext), contentMapperSourceFile, extNode)) case slices.Contains(nativeExtensions, ext): - errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper, ext)) + errors = append(errors, setContentMapperDiagnosticLocation(ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper, ext), contentMapperSourceFile, extNode)) default: if _, seen := seenContentMapperExtensions[ext]; seen { - errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper, ext)) + errors = append(errors, setContentMapperDiagnosticLocation(ast.NewCompilerDiagnostic(diagnostics.Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper, ext), contentMapperSourceFile, extNode)) } else { seenContentMapperExtensions[ext] = struct{}{} contentMapperExtensions = append(contentMapperExtensions, ext) @@ -1356,7 +1367,23 @@ func parseJsonConfigFileContentWorker( } } if len(contentMappers) > 0 && !(parsedConfig.options != nil && parsedConfig.options.DangerouslyLoadExternalPlugins.IsTrue()) { - errors = append(errors, ast.NewCompilerDiagnostic(diagnostics.Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled)) + errors = append(errors, setContentMapperDiagnosticLocation(ast.NewCompilerDiagnostic(diagnostics.Content_mappers_require_the_dangerouslyLoadExternalPlugins_command_line_flag_to_be_enabled), contentMapperSourceFile, getContentMappersKeySyntax(contentMapperSourceFile))) + } else if len(contentMappers) > 0 { + // Resolve each mapper's package.json now so its name, version, and run command are available to + // everything downstream (diagnostics, build-info staleness) without executing anything. + containingFile := configFileName + if containingFile == "" { + containingFile = tspath.CombinePaths(basePathForFileNames, "tsconfig.json") + } + for j, mapper := range contentMappers { + manifest, packageDirectory, diagnostic := resolveContentMapperManifest(host, containingFile, mapper.Package) + if diagnostic != nil { + errors = append(errors, setContentMapperDiagnosticLocation(diagnostic, contentMapperSourceFile, getContentMapperSyntax(contentMapperSourceFile, contentMapperIndices[j], "package"))) + continue + } + mapper.Manifest = manifest + mapper.PackageDirectory = packageDirectory + } } getFileNames := func(basePath string) ([]string, int) { @@ -1542,6 +1569,70 @@ func GetOptionsSyntaxByArrayElementValue(objectLiteral *ast.ObjectLiteralExpress return ForEachPropertyAssignment(objectLiteral, propKey, GetCallbackForFindingPropertyAssignmentByValue(elementValue)) } +// getContentMapperSyntax returns the tsconfig JSON node to attribute a diagnostic about the content +// mapper at index to: the value of subKey within that mapper's object (when subKey is non-empty), +// falling back to the mapper element, then to the "contentMappers" array. An index outside the array +// (e.g. -1) yields the array itself. Returns nil when there is no source file (JSON API). +func getContentMapperSyntax(sourceFile *ast.SourceFile, index int, subKey string) *ast.Node { + if sourceFile == nil { + return nil + } + return ForEachTsConfigPropArray(sourceFile, "contentMappers", func(property *ast.PropertyAssignment) *ast.Node { + if !ast.IsArrayLiteralExpression(property.Initializer) { + return property.Initializer + } + elements := property.Initializer.Elements() + if index < 0 || index >= len(elements) { + return property.Initializer + } + element := elements[index] + if subKey != "" && ast.IsObjectLiteralExpression(element) { + if node := ForEachPropertyAssignment(element.AsObjectLiteralExpression(), subKey, func(property *ast.PropertyAssignment) *ast.Node { + return property.Initializer + }); node != nil { + return node + } + } + return element + }) +} + +// getContentMappersKeySyntax returns the "contentMappers" property key node, used to attribute a +// diagnostic about the setting as a whole rather than a specific mapper. +func getContentMappersKeySyntax(sourceFile *ast.SourceFile) *ast.Node { + if sourceFile == nil { + return nil + } + return ForEachTsConfigPropArray(sourceFile, "contentMappers", func(property *ast.PropertyAssignment) *ast.Node { + return property.Name() + }) +} + +// getContentMapperExtensionSyntax returns the node for a specific extension string within the content +// mapper at index, falling back to the "extensions" array or the mapper element. +func getContentMapperExtensionSyntax(sourceFile *ast.SourceFile, index int, ext string) *ast.Node { + node := getContentMapperSyntax(sourceFile, index, "extensions") + if node != nil && ast.IsArrayLiteralExpression(node) { + if element := core.Find(node.Elements(), func(element *ast.Node) bool { + return ast.IsStringLiteral(element) && element.Text() == ext + }); element != nil { + return element + } + } + return node +} + +// setContentMapperDiagnosticLocation attaches a source location to a content mapper diagnostic when a +// tsconfig source file and node are available (the jsonSourceFile API), leaving it as a location-less +// compiler diagnostic otherwise (the JSON API). +func setContentMapperDiagnosticLocation(diagnostic *ast.Diagnostic, sourceFile *ast.SourceFile, node *ast.Node) *ast.Diagnostic { + if sourceFile != nil && node != nil { + diagnostic.SetFile(sourceFile) + diagnostic.SetLocation(core.NewTextRange(scanner.SkipTrivia(sourceFile.Text(), node.Pos()), node.End())) + } + return diagnostic +} + func ForEachPropertyAssignment[T any](objectLiteral *ast.ObjectLiteralExpression, key string, callback func(property *ast.PropertyAssignment) *T, key2 ...string) *T { if objectLiteral != nil { for _, property := range objectLiteral.Properties.Nodes { diff --git a/internal/tsoptions/tsconfigparsing_test.go b/internal/tsoptions/tsconfigparsing_test.go index 038ac8a6fde..00a61d3747d 100644 --- a/internal/tsoptions/tsconfigparsing_test.go +++ b/internal/tsoptions/tsconfigparsing_test.go @@ -1011,15 +1011,16 @@ func TestContentMappers(t *testing.T) { config := testConfig{ jsonText: `{ "contentMappers": [ - { "command": ["vue-mapper"], "extensions": [".vue"] } + { "package": "vue-mapper", "extensions": [".vue"] } ], "include": ["src"] }`, configFileName: "tsconfig.json", basePath: "/", allFileList: map[string]string{ - "/src/app.ts": "export {}", - "/src/Component.vue": "", + "/src/app.ts": "export {}", + "/src/Component.vue": "", + "/node_modules/vue-mapper/package.json": `{ "name": "vue-mapper", "version": "1.2.3", "tsContentMapper": { "exec": ["node", "./mapper.js"] } }`, }, existingOptions: &core.CompilerOptions{DangerouslyLoadExternalPlugins: core.TSTrue}, } @@ -1040,10 +1041,16 @@ func TestContentMappers(t *testing.T) { mappers := parsed.ContentMappers() assert.Equal(t, len(mappers), 1) - assert.DeepEqual(t, mappers[0].Command, []string{"vue-mapper"}) + assert.Equal(t, mappers[0].Package, "vue-mapper") assert.DeepEqual(t, mappers[0].Extensions, []string{".vue"}) assert.DeepEqual(t, parsed.ContentMapperExtensions(), []string{".vue"}) + // The package.json is resolved during parsing, populating name, version, and exec. + assert.Equal(t, mappers[0].Name, "vue-mapper") + assert.Equal(t, mappers[0].Version, "1.2.3") + assert.DeepEqual(t, mappers[0].Exec, []string{"node", "./mapper.js"}) + assert.Equal(t, mappers[0].PackageDirectory, "/node_modules/vue-mapper") + // The .vue file is picked up by the include glob because its extension is registered. assert.Assert(t, slices.Contains(parsed.FileNames(), "/src/Component.vue"), "expected /src/Component.vue in %v", parsed.FileNames()) assert.Assert(t, slices.Contains(parsed.FileNames(), "/src/app.ts"), "expected /src/app.ts in %v", parsed.FileNames()) @@ -1055,7 +1062,7 @@ func TestContentMappersRequireFlag(t *testing.T) { t.Parallel() config := testConfig{ - jsonText: `{ "contentMappers": [{ "command": ["vue-mapper"], "extensions": [".vue"] }] }`, + jsonText: `{ "contentMappers": [{ "package": "vue-mapper", "extensions": [".vue"] }] }`, configFileName: "tsconfig.json", basePath: "/", allFileList: map[string]string{"/app.ts": "export {}"}, @@ -1090,32 +1097,32 @@ func TestContentMappersValidation(t *testing.T) { }{ { name: "extension without leading dot", - contentMappers: `[{ "command": ["vue-mapper"], "extensions": ["vue"] }]`, + contentMappers: `[{ "package": "vue-mapper", "extensions": ["vue"] }]`, expectedCode: diagnostics.Content_mapper_file_extension_0_must_begin_with_a.Code(), }, { name: "built-in extension", - contentMappers: `[{ "command": ["x"], "extensions": [".ts"] }]`, + contentMappers: `[{ "package": "x", "extensions": [".ts"] }]`, expectedCode: diagnostics.Content_mapper_file_extension_0_is_a_built_in_extension_and_cannot_be_registered_by_a_content_mapper.Code(), }, { name: "duplicate extension across mappers", - contentMappers: `[{ "command": ["a"], "extensions": [".vue"] }, { "command": ["b"], "extensions": [".vue"] }]`, + contentMappers: `[{ "package": "a", "extensions": [".vue"] }, { "package": "b", "extensions": [".vue"] }]`, expectedCode: diagnostics.Content_mapper_file_extension_0_is_registered_by_more_than_one_content_mapper.Code(), }, { name: "extensions is not an array", - contentMappers: `[{ "command": ["x"], "extensions": ".vue" }]`, + contentMappers: `[{ "package": "x", "extensions": ".vue" }]`, expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), }, { name: "extensions contains a non-string", - contentMappers: `[{ "command": ["x"], "extensions": [".vue", 1] }]`, + contentMappers: `[{ "package": "x", "extensions": [".vue", 1] }]`, expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), }, { - name: "command is not an array", - contentMappers: `[{ "command": "x", "extensions": [".vue"] }]`, + name: "package is not a string", + contentMappers: `[{ "package": ["x"], "extensions": [".vue"] }]`, expectedCode: diagnostics.Compiler_option_0_requires_a_value_of_type_1.Code(), }, } @@ -1140,10 +1147,16 @@ func TestContentMappersValidation(t *testing.T) { maps.Copy(allFileLists, config.allFileList) host := tsoptionstest.NewVFSParseConfigHost(allFileLists, config.basePath, true /*useCaseSensitiveFileNames*/) parsed := getParsed(config, host, config.basePath) - found := slices.ContainsFunc(parsed.Errors, func(d *ast.Diagnostic) bool { + diagnostic := core.Find(parsed.Errors, func(d *ast.Diagnostic) bool { return d.Code() == test.expectedCode }) - assert.Assert(t, found, "expected diagnostic %d, got errors: %v", test.expectedCode, parsed.Errors) + assert.Assert(t, diagnostic != nil, "expected diagnostic %d, got errors: %v", test.expectedCode, parsed.Errors) + + // With the jsonSourceFile API the diagnostic is located at the offending tsconfig syntax. + if apiName == "jsonSourceFile api" { + assert.Assert(t, diagnostic.File() != nil, "expected diagnostic %d to have a source file", test.expectedCode) + assert.Assert(t, diagnostic.Len() > 0, "expected diagnostic %d to have a non-empty location", test.expectedCode) + } }) } }) diff --git a/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt b/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt index 434888d8ce6..688dfc375c0 100644 --- a/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt +++ b/testdata/baselines/reference/contentMappers/contentMapperDisabledAfterRepeatedFailures.txt @@ -44,29 +44,29 @@ === Diagnostics === -C.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch +C.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: mapper protocol version mismatch 1 ~ -D.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch +D.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: mapper protocol version mismatch 1 ~ -E.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch +E.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: mapper protocol version mismatch 1 ~ -F.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch +F.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: mapper protocol version mismatch 1 ~ -G.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: mapper protocol version mismatch +G.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: mapper protocol version mismatch 1 ~ -error TS100026: The content mapper 'vue-mapper@1.0.0' failed 5 times and will not be used. +error TS100026: The content mapper 'vue-mapper' failed 5 times and will not be used. diff --git a/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt b/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt index 440f0c75d92..ea5253eaf06 100644 --- a/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt +++ b/testdata/baselines/reference/contentMappers/contentMapperFileFailure.txt @@ -8,7 +8,7 @@ === Diagnostics === -Component.vue:1:1 - error TS100025: The content mapper 'vue-mapper@1.0.0' failed to transform this file: broken pipe +Component.vue:1:1 - error TS100025: The content mapper 'vue-mapper' failed to transform this file: broken pipe 1 ~ diff --git a/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt b/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt index e6149858fe2..cd615c5b068 100644 --- a/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt +++ b/testdata/baselines/reference/contentMappers/contentMapperGeneratedCode.txt @@ -12,13 +12,13 @@ export const el = jsxRuntime(Widget); === Diagnostics === Component.vue:1:19 - error TS2304: Cannot find name 'jsxRuntime'. - This location is in code generated by the content mapper 'vue-mapper@1.0.0' and has no corresponding location in the original file. + This location is in code generated by the content mapper 'vue-mapper' and has no corresponding location in the original file. 1 export const el = jsxRuntime(Widget); ~~~~~~~~~~ Component.vue:1:30 - error TS2304: Cannot find name 'Widget'. - This location is in code generated by the content mapper 'vue-mapper@1.0.0' and has no corresponding location in the original file. + This location is in code generated by the content mapper 'vue-mapper' and has no corresponding location in the original file. 1 export const el = jsxRuntime(Widget); ~~~~~~ diff --git a/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js b/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js index b169e09ace1..b5b0c99580f 100644 --- a/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js +++ b/testdata/baselines/reference/tsbuild/contentMapperIdentity/content-mapper-identity-change-forces-rebuild.js @@ -5,13 +5,19 @@ Input:: export const app = 1; //// [/home/src/workspaces/project/index.ts] *new* export const local = 1; +//// [/home/src/workspaces/project/node_modules/vue-ts-mapper/package.json] *new* +{ + "name": "vue-ts-mapper", + "version": "1.0.0", + "tsContentMapper": { "exec": ["node", "./mapper.js"] } +} //// [/home/src/workspaces/project/tsconfig.json] *new* { "compilerOptions": { "incremental": true }, "contentMappers": [ - { "command": ["vue"], "extensions": [".vue"] } + { "package": "vue-ts-mapper", "extensions": [".vue"] } ] } @@ -55,7 +61,7 @@ export const app = 1; export const local = 1; //// [/home/src/workspaces/project/tsconfig.tsbuildinfo] *new* -{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue@1.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} +{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue-ts-mapper@1.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} //// [/home/src/workspaces/project/tsconfig.tsbuildinfo.readable.baseline.txt] *new* { "version": "FakeTSVersion", @@ -105,7 +111,7 @@ export const local = 1; "impliedNodeFormat": "CommonJS" } ], - "size": 1056 + "size": 1066 } tsconfig.json:: @@ -129,7 +135,13 @@ Output:: -Edit [1]:: bump content mapper version +Edit [1]:: upgrade the content mapper package to a new version +//// [/home/src/workspaces/project/node_modules/vue-ts-mapper/package.json] *modified* +{ + "name": "vue-ts-mapper", + "version": "2.0.0", + "tsContentMapper": { "exec": ["node", "./mapper.js"] } +} tsgo --build --verbose --dangerouslyLoadExternalPlugins ExitStatus:: Success @@ -144,7 +156,7 @@ Output:: //// [/home/src/workspaces/project/app.vue.js] *rewrite with same content* //// [/home/src/workspaces/project/index.js] *rewrite with same content* //// [/home/src/workspaces/project/tsconfig.tsbuildinfo] *modified* -{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue@2.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} +{"version":"FakeTSVersion","root":[[2,3]],"contentMapperIdentities":["vue-ts-mapper@2.0.0"],"fileNames":["lib.es2025.full.d.ts","./app.vue","./index.ts"],"fileInfos":[{"version":"8859c12c614ce56ba9a18e58384a198f-/// \ninterface Boolean {}\ninterface Function {}\ninterface CallableFunction {}\ninterface NewableFunction {}\ninterface IArguments {}\ninterface Number { toExponential: any; }\ninterface Object {}\ninterface RegExp {}\ninterface String { charAt: any; }\ninterface Array { length: number; [n: number]: T; }\ninterface ReadonlyArray {}\ninterface SymbolConstructor {\n (desc?: string | number): symbol;\n for(name: string): symbol;\n readonly toStringTag: symbol;\n}\ndeclare var Symbol: SymbolConstructor;\ninterface Symbol {\n readonly [Symbol.toStringTag]: string;\n}\ndeclare const console: { log(msg: any): void; };","affectsGlobalScope":true,"impliedNodeFormat":1},{"version":"8e06c0ee6938980d692eefcee0c71e42-export const app = 1;"},"652b2b5c9eb278600aaa728787469dc8-export const local = 1;"]} //// [/home/src/workspaces/project/tsconfig.tsbuildinfo.readable.baseline.txt] *rewrite with same content* tsconfig.json:: From 9ccd3aa0621db5a2e8e18e1c7d6d900e8bdb6ba3 Mon Sep 17 00:00:00 2001 From: Andrew Branch Date: Mon, 13 Jul 2026 15:45:37 -0700 Subject: [PATCH 05/37] IPC infrastructure --- cmd/tsgo/sys.go | 44 +++ internal/api/callbackfs.go | 5 +- internal/api/protocol_msgpack.go | 7 +- internal/api/server.go | 15 +- internal/api/session.go | 3 +- internal/compiler/contentmapper_test.go | 69 ++-- internal/compiler/contentmapperrunner.go | 31 -- internal/compiler/fileloader.go | 20 +- internal/compiler/program.go | 3 +- .../contentmapperhost/contentmapperhost.go | 47 +++ internal/contentmapperhost/host.go | 296 ++++++++++++++++++ internal/contentmapperhost/host_test.go | 169 ++++++++++ internal/execute/build/buildtask.go | 4 +- internal/execute/build/orchestrator.go | 5 + internal/execute/tsc.go | 22 +- internal/execute/tsc/compile.go | 9 +- internal/execute/tsctests/sys.go | 70 +++-- internal/execute/watcher.go | 11 +- internal/{api => ipc}/conn.go | 6 +- internal/{api => ipc}/conn_async.go | 12 +- internal/{api => ipc}/conn_sync.go | 16 +- internal/{api => ipc}/protocol.go | 2 +- internal/{api => ipc}/protocol_jsonrpc.go | 2 +- internal/{api => ipc}/timing.go | 9 +- internal/{api => ipc}/timing_test.go | 2 +- internal/{api => ipc}/transport.go | 2 +- internal/{api => ipc}/transport_unix.go | 2 +- internal/{api => ipc}/transport_windows.go | 2 +- internal/lsp/server.go | 7 +- 29 files changed, 743 insertions(+), 149 deletions(-) delete mode 100644 internal/compiler/contentmapperrunner.go create mode 100644 internal/contentmapperhost/contentmapperhost.go create mode 100644 internal/contentmapperhost/host.go create mode 100644 internal/contentmapperhost/host_test.go rename internal/{api => ipc}/conn.go (91%) rename internal/{api => ipc}/conn_async.go (94%) rename internal/{api => ipc}/conn_sync.go (92%) rename internal/{api => ipc}/protocol.go (98%) rename internal/{api => ipc}/protocol_jsonrpc.go (99%) rename internal/{api => ipc}/timing.go (93%) rename internal/{api => ipc}/timing_test.go (99%) rename internal/{api => ipc}/transport.go (99%) rename internal/{api => ipc}/transport_unix.go (97%) rename internal/{api => ipc}/transport_windows.go (96%) diff --git a/cmd/tsgo/sys.go b/cmd/tsgo/sys.go index e5b2b8d569a..2dea06e45a5 100644 --- a/cmd/tsgo/sys.go +++ b/cmd/tsgo/sys.go @@ -1,9 +1,11 @@ package main import ( + "errors" "fmt" "io" "os" + "os/exec" "time" "github.com/microsoft/typescript-go/internal/bundled" @@ -59,6 +61,48 @@ func (s *osSys) GetEnvironmentVariable(name string) string { return os.Getenv(name) } +func (s *osSys) Spawn(command []string, dir string) (io.ReadWriteCloser, error) { + cmd := exec.Command(command[0], command[1:]...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + return &contentMapperProcess{cmd: cmd, stdin: stdin, stdout: stdout}, nil +} + +// contentMapperProcess adapts a child process's stdout (read) and stdin (write) into one +// io.ReadWriteCloser. Close kills and reaps the process. +type contentMapperProcess struct { + cmd *exec.Cmd + stdin io.WriteCloser + stdout io.ReadCloser +} + +func (p *contentMapperProcess) Read(b []byte) (int, error) { return p.stdout.Read(b) } +func (p *contentMapperProcess) Write(b []byte) (int, error) { return p.stdin.Write(b) } + +func (p *contentMapperProcess) Close() error { + // Kill guarantees the process is gone even if it is ignoring stdin's EOF; Wait then reaps it and + // closes the stdio pipes it created. Kill is best-effort because Wait reports the real outcome, and a + // "signal: killed" ExitError is the expected result of that kill, so only an unexpected Wait error is + // surfaced. + _ = p.cmd.Process.Kill() + err := p.cmd.Wait() + if _, ok := errors.AsType[*exec.ExitError](err); ok { + return nil + } + return err +} + func newSystem() *osSys { cwd, err := os.Getwd() if err != nil { diff --git a/internal/api/callbackfs.go b/internal/api/callbackfs.go index daf170e7181..df69ae73e9f 100644 --- a/internal/api/callbackfs.go +++ b/internal/api/callbackfs.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/microsoft/typescript-go/internal/ipc" "github.com/microsoft/typescript-go/internal/json" "github.com/microsoft/typescript-go/internal/vfs" ) @@ -21,7 +22,7 @@ type callbackFS struct { enabledCallbacks map[string]bool // conn and ctx are set after connection is established - conn Conn + conn ipc.Conn ctx context.Context } @@ -67,7 +68,7 @@ func newCallbackFS(base vfs.FS, callbacks []string) *callbackFS { // SetConnection sets the RPC connection for callbacks. // This must be called after the transport connection is established // but before any filesystem operations that need callbacks. -func (fs *callbackFS) SetConnection(ctx context.Context, conn Conn) { +func (fs *callbackFS) SetConnection(ctx context.Context, conn ipc.Conn) { fs.ctx = ctx fs.conn = conn } diff --git a/internal/api/protocol_msgpack.go b/internal/api/protocol_msgpack.go index b8b9d87d66b..a4d5a73f5e5 100644 --- a/internal/api/protocol_msgpack.go +++ b/internal/api/protocol_msgpack.go @@ -6,6 +6,7 @@ import ( "fmt" "io" + "github.com/microsoft/typescript-go/internal/ipc" "github.com/microsoft/typescript-go/internal/json" "github.com/microsoft/typescript-go/internal/jsonrpc" ) @@ -43,7 +44,7 @@ type MessagePackProtocol struct { w *bufio.Writer } -var _ Protocol = (*MessagePackProtocol)(nil) +var _ ipc.Protocol = (*MessagePackProtocol)(nil) // NewMessagePackProtocol creates a new msgpack protocol handler. func NewMessagePackProtocol(rw io.ReadWriter) *MessagePackProtocol { @@ -54,14 +55,14 @@ func NewMessagePackProtocol(rw io.ReadWriter) *MessagePackProtocol { } // ReadMessage implements Protocol. -func (p *MessagePackProtocol) ReadMessage() (*Message, error) { +func (p *MessagePackProtocol) ReadMessage() (*ipc.Message, error) { msgType, method, payload, err := p.readTuple() if err != nil { return nil, err } // Convert msgpack message type to JSON-RPC message - msg := &Message{} + msg := &ipc.Message{} switch msgType { case MessageTypeRequest: diff --git a/internal/api/server.go b/internal/api/server.go index ef5ca2a99ee..5d32754b678 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -6,6 +6,7 @@ import ( "io" "github.com/microsoft/typescript-go/internal/bundled" + "github.com/microsoft/typescript-go/internal/ipc" "github.com/microsoft/typescript-go/internal/lsp/lsproto" "github.com/microsoft/typescript-go/internal/project" "github.com/microsoft/typescript-go/internal/vfs/osvfs" @@ -55,16 +56,16 @@ func NewStdioServer(options *StdioServerOptions) *StdioServer { // Run starts the server and blocks until the connection closes. func (s *StdioServer) Run(ctx context.Context) error { - var transport Transport + var transport ipc.Transport if s.options.PipePath != "" { - t, err := NewPipeTransport(s.options.PipePath) + t, err := ipc.NewPipeTransport(s.options.PipePath) if err != nil { return fmt.Errorf("failed to create pipe transport: %w", err) } defer t.Close() transport = t } else { - t := NewStdioTransport(s.options.In, s.options.Out) + t := ipc.NewStdioTransport(s.options.In, s.options.Out) defer t.Close() transport = t } @@ -102,15 +103,15 @@ func (s *StdioServer) Run(ctx context.Context) error { } // Create protocol and connection based on async mode - var conn Conn + var conn ipc.Conn if s.options.Async { - protocol := NewJSONRPCProtocol(rwc) - asyncConn := NewAsyncConnWithProtocol(rwc, protocol, session) + protocol := ipc.NewJSONRPCProtocol(rwc) + asyncConn := ipc.NewAsyncConnWithProtocol(rwc, protocol, session) asyncConn.SetCollectTiming(s.options.CollectTiming) conn = asyncConn } else { protocol := NewMessagePackProtocol(rwc) - syncConn := NewSyncConn(rwc, protocol, session) + syncConn := ipc.NewSyncConn(rwc, protocol, session) syncConn.SetCollectTiming(s.options.CollectTiming) conn = syncConn } diff --git a/internal/api/session.go b/internal/api/session.go index e725cf6db50..09b9f98204b 100644 --- a/internal/api/session.go +++ b/internal/api/session.go @@ -17,6 +17,7 @@ import ( "github.com/microsoft/typescript-go/internal/collections" "github.com/microsoft/typescript-go/internal/compiler" "github.com/microsoft/typescript-go/internal/core" + "github.com/microsoft/typescript-go/internal/ipc" "github.com/microsoft/typescript-go/internal/json" "github.com/microsoft/typescript-go/internal/ls" "github.com/microsoft/typescript-go/internal/lsp/lsproto" @@ -404,7 +405,7 @@ type Session struct { } // Ensure Session implements Handler -var _ Handler = (*Session)(nil) +var _ ipc.Handler = (*Session)(nil) // SessionOptions configures an API session. type SessionOptions struct { diff --git a/internal/compiler/contentmapper_test.go b/internal/compiler/contentmapper_test.go index 90bf2393e57..e73efa4fcec 100644 --- a/internal/compiler/contentmapper_test.go +++ b/internal/compiler/contentmapper_test.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/typescript-go/internal/bundled" "github.com/microsoft/typescript-go/internal/compiler" "github.com/microsoft/typescript-go/internal/contentmapper" + "github.com/microsoft/typescript-go/internal/contentmapperhost" "github.com/microsoft/typescript-go/internal/core" "github.com/microsoft/typescript-go/internal/diagnostics" "github.com/microsoft/typescript-go/internal/diagnosticwriter" @@ -24,14 +25,16 @@ import ( "gotest.tools/v3/assert" ) -type fakeContentMapperRunner struct { - transform func(fileName string, content string) (compiler.ContentMapperResult, error) +type fakeContentMapperHost struct { + transform func(fileName string, content string) (contentmapperhost.Result, error) } -func (r fakeContentMapperRunner) Transform(fileName string, content string) (compiler.ContentMapperResult, error) { - return r.transform(fileName, content) +func (r fakeContentMapperHost) Transform(mapper *contentmapper.Mapper, request contentmapperhost.Request) (contentmapperhost.Result, error) { + return r.transform(request.FileName, request.Content) } +func (fakeContentMapperHost) Close() error { return nil } + const vueRawContent = "" // synthesizedSpanMap maps an entirely generated transformed text to no original location (a fully @@ -47,7 +50,7 @@ func synthesizedSpanMap(transformed string) *spanmap.SpanMap { }}) } -func newContentMapperProgram(t *testing.T, runner compiler.ContentMapperRunner, files map[string]string, rootFiles []string) *compiler.Program { +func newContentMapperProgram(t *testing.T, contentMapperHost contentmapperhost.Host, files map[string]string, rootFiles []string) *compiler.Program { t.Helper() if !bundled.Embedded { t.Skip("bundled files are not embedded") @@ -70,9 +73,9 @@ func newContentMapperProgram(t *testing.T, runner compiler.ContentMapperRunner, }, } return compiler.NewProgram(compiler.ProgramOptions{ - Config: config, - Host: compiler.NewCompilerHost("/src", fs, bundled.LibPath(), nil, nil), - ContentMapperRunner: runner, + Config: config, + Host: compiler.NewCompilerHost("/src", fs, bundled.LibPath(), nil, nil), + ContentMapperHost: contentMapperHost, // Load files on the calling goroutine for deterministic diagnostics ordering. SingleThreaded: core.TSTrue, }) @@ -91,7 +94,7 @@ func TestContentMapperDiagnostics(t *testing.T) { t.Parallel() // A mapper reports problems it finds in the file's content (e.g. a syntax error in the embedded - // script) as result.Diagnostics. A real runner deserializes these from another process: it does + // script) as result.Diagnostics. A real host deserializes these from another process: it does // not have our diagnostics.Message values, so it builds each one from an already-localized message, // a code namespaced by the mapper's own source prefix, and a range into the original content. const componentVue = ` +// +// + +// === /format.ts === +// export function [|format|](value: string): string { return value; } +// + + + +// === findAllReferences === +// === /ProfileCard.vue === +// +//