diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index 87fcd750640..ed48bd28953 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -145,6 +145,7 @@ * Implied argument names for function-to-delegate coercions now fall back to the delegate's `Invoke` parameter names when the function has no recoverable names (e.g. a partial application like `System.Func((+) 1)`), instead of synthetic `delegateArg0`, `delegateArg1`, … names. ([PR #20001](https://github.com/dotnet/fsharp/pull/20001)) * Add internal `ResetCompilerGeneratedNameState` to `CompilerGlobalState` name generators so warm-checker re-compilation can produce fresh-process-identical generated names. ([PR #20017](https://github.com/dotnet/fsharp/pull/20017)) * Add Roslyn-format EnC CustomDebugInformation codec and portable PDB method CDI emission support to AbstractIL. ([PR #20018](https://github.com/dotnet/fsharp/pull/20018)) +* Add internal hot reload baseline reading for recorded EnC state and synthesized-name snapshot PDB data. ([PR #20026](https://github.com/dotnet/fsharp/pull/20026)) ### Improved diff --git a/src/Compiler/AbstractIL/EncMethodDebugInformation.fs b/src/Compiler/AbstractIL/EncMethodDebugInformation.fs index e605b2208a4..36160e5c981 100644 --- a/src/Compiler/AbstractIL/EncMethodDebugInformation.fs +++ b/src/Compiler/AbstractIL/EncMethodDebugInformation.fs @@ -31,8 +31,11 @@ open System.IO open System.Reflection.Metadata open System.Reflection.Metadata.Ecma335 open System.Runtime.InteropServices +open System.Text open Microsoft.FSharp.NativeInterop +open FSharp.Compiler.AbstractIL.ILPdbWriter + /// Portable-PDB CustomDebugInformation kind GUIDs for the EnC blobs, copied verbatim /// from roslyn/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs. [] @@ -47,6 +50,10 @@ module PortableCustomDebugInfoKinds = /// EnC State Machine State Map CDI kind. let encStateMachineStateMap = Guid("8B78CD68-2EDE-420B-980B-E15884B8AAA3") + /// F#-owned hot reload synthesized-name snapshot CDI kind. The blob records + /// FSharpSynthesizedTypeMaps.Snapshot bucket arrays in allocation-slot order. + let fsharpSynthesizedNameSnapshot = Guid("49DDB47E-9C74-46EC-8626-0350676571EB") + /// Closure ordinal of a lambda that is lowered to a static (non-capturing) method. /// Mirrors Roslyn's LambdaDebugInfo.StaticClosureOrdinal. [] @@ -167,7 +174,7 @@ let private MaxOccurrenceKey = 0x1FFFFFFD /// 16-bit segments, least-significant segment = the innermost ordinal; an enclosing /// ordinal p is stored as (p + 1) shifted left 16 so that depth-1 keys (< 0x10000) and /// depth-2 keys (>= 0x10000) never collide. Fails closed (None) past the limits: chains -/// deeper than 2, ordinals > 0xFFFF, or keys exceeding the compressed-integer budget — +/// deeper than 2, ordinals > 0xFFFF, or keys exceeding the compressed-integer budget, /// callers must then treat the chain as unmappable, never truncate. let tryEncodeOccurrenceKey (ordinalChain: int list) : int option = match ordinalChain with @@ -209,6 +216,134 @@ let private invalidData (blobName: string) (offset: int) = // nullness model, so guard with box (FS3261-safe) rather than dropping the check. let private isEmpty (blob: byte[]) = isNull (box blob) || blob.Length = 0 +// --------------------------------------------------------------------------- +// F# hot reload module CDI: synthesized-name allocation snapshot +// Format: +// compressed(version = 1), compressed(bucket count), +// then buckets sorted by key for deterministic PDB bytes: +// string key, compressed(name count), string name in allocation-slot order. +// Strings are compressed(byte length) followed by UTF-8 bytes. +// --------------------------------------------------------------------------- + +[] +let private SynthesizedNameSnapshotBlobVersion = 1 + +let private writeUtf8String (builder: BlobBuilder) (value: string) = + if isNull (box value) then + invalidArg (nameof value) "snapshot strings must be non-null" + + let bytes = Encoding.UTF8.GetBytes value + builder.WriteCompressedInteger bytes.Length + builder.WriteBytes bytes + +let private readUtf8String (blobName: string) (reader: byref) = + let length = reader.ReadCompressedInteger() + + if length < 0 || length > reader.RemainingBytes then + invalidData blobName reader.Offset + + let bytes = reader.ReadBytes length + Encoding.UTF8.GetString(bytes, 0, bytes.Length) + +let private materializeSynthesizedNameSnapshot (snapshot: seq) = + snapshot + |> Seq.map (fun struct (key, names) -> + if isNull (box key) then + invalidArg (nameof snapshot) "snapshot keys must be non-null" + + if isNull (box names) then + invalidArg (nameof snapshot) $"snapshot bucket '{key}' must be non-null" + + key, Array.copy names) + |> Seq.sortBy fst + |> Seq.toArray + +/// Serializes an allocation-ordered synthesized-name snapshot into the F#-owned module +/// CDI blob. An empty snapshot returns an empty blob so no CDI row needs to be emitted. +let serializeSynthesizedNameSnapshot (snapshot: seq) : byte[] = + let buckets = materializeSynthesizedNameSnapshot snapshot + + if buckets.Length = 0 then + Array.empty + else + let builder = BlobBuilder() + builder.WriteCompressedInteger SynthesizedNameSnapshotBlobVersion + builder.WriteCompressedInteger buckets.Length + + for key, names in buckets do + writeUtf8String builder key + builder.WriteCompressedInteger names.Length + + for name in names do + writeUtf8String builder name + + builder.ToArray() + +/// Deserializes the F#-owned synthesized-name snapshot CDI blob. Bucket order in the +/// blob is deterministic only; each bucket array is returned exactly in recorded slot order. +let deserializeSynthesizedNameSnapshot (blob: byte[]) : Map = + if isEmpty blob then + Map.empty + else + let handle = GCHandle.Alloc(blob, GCHandleType.Pinned) + + try + let mutable reader = + BlobReader(NativePtr.ofNativeInt (handle.AddrOfPinnedObject()), blob.Length) + + try + let version = reader.ReadCompressedInteger() + + if version <> SynthesizedNameSnapshotBlobVersion then + invalidData "synthesized name snapshot" reader.Offset + + let bucketCount = reader.ReadCompressedInteger() + + if bucketCount <= 0 || bucketCount > reader.RemainingBytes / 2 then + invalidData "synthesized name snapshot" reader.Offset + + let buckets = ResizeArray() + + for _ in 1..bucketCount do + let key = readUtf8String "synthesized name snapshot" &reader + let nameCount = reader.ReadCompressedInteger() + + // Every serialized name consumes at least one byte for its UTF-8 + // length, so this check bounds allocation before Array.zeroCreate. + if nameCount < 0 || nameCount > reader.RemainingBytes then + invalidData "synthesized name snapshot" reader.Offset + + let names = Array.zeroCreate nameCount + + for i in 0 .. nameCount - 1 do + names[i] <- readUtf8String "synthesized name snapshot" &reader + + buckets.Add(key, names) + + if reader.RemainingBytes <> 0 then + invalidData "synthesized name snapshot" reader.Offset + + buckets |> Seq.map id |> Map.ofSeq + with :? BadImageFormatException -> + invalidData "synthesized name snapshot" reader.Offset + finally + handle.Free() + +/// Creates the module-level CustomDebugInformation row for the allocation-ordered +/// synthesized-name snapshot. Empty snapshots emit no row. +let computeSynthesizedNameSnapshotCustomDebugInfoRows (snapshot: seq) : PdbModuleCustomDebugInfo list = + let blob = serializeSynthesizedNameSnapshot snapshot + + if blob.Length = 0 then + [] + else + [ + { + KindGuid = PortableCustomDebugInfoKinds.fsharpSynthesizedNameSnapshot + Blob = blob + } + ] + // --------------------------------------------------------------------------- // EnC Local Slot Map // Format (EditAndContinueMethodDebugInformation.cs, SerializeLocalSlots lines 145-191, @@ -555,3 +690,35 @@ let readEncMethodDebugInfoFromPortablePdb (pdbBytes: byte[]) : Map option = + if isEmpty pdbBytes then + None + else + try + use provider = + MetadataReaderProvider.FromPortablePdbImage(ImmutableArray.CreateRange pdbBytes) + + let reader = provider.GetMetadataReader() + + let blobs = + [ + for cdiHandle in reader.CustomDebugInformation do + let cdi = reader.GetCustomDebugInformation cdiHandle + + if cdi.Parent.Kind = HandleKind.ModuleDefinition then + let kind = reader.GetGuid cdi.Kind + + if kind = PortableCustomDebugInfoKinds.fsharpSynthesizedNameSnapshot then + reader.GetBlobBytes cdi.Value + ] + + match blobs with + | [ blob ] -> Some(deserializeSynthesizedNameSnapshot blob) + | _ -> None + with + | :? BadImageFormatException + | :? InvalidDataException -> None diff --git a/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi b/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi index 1e2ba76e7c8..6d83c3a42b6 100644 --- a/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi +++ b/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi @@ -36,6 +36,9 @@ module PortableCustomDebugInfoKinds = /// EnC State Machine State Map CDI kind. val encStateMachineStateMap: System.Guid + /// F#-owned hot reload synthesized-name snapshot CDI kind. + val fsharpSynthesizedNameSnapshot: System.Guid + /// Closure ordinal of a lambda that is lowered to a static (non-capturing) method. /// Mirrors Roslyn's LambdaDebugInfo.StaticClosureOrdinal. [] @@ -135,6 +138,19 @@ val tryEncodeOccurrenceKey: ordinalChain: int list -> int option /// root-first ordinal chain. val decodeOccurrenceKey: key: int -> int list +/// Serializes an allocation-ordered synthesized-name snapshot into the F#-owned module +/// CDI blob. An empty snapshot returns an empty blob so no CDI row needs to be emitted. +val serializeSynthesizedNameSnapshot: snapshot: seq -> byte[] + +/// Deserializes the F#-owned synthesized-name snapshot CDI blob. Bucket order in the +/// blob is deterministic only; each bucket array is returned exactly in recorded slot order. +val deserializeSynthesizedNameSnapshot: blob: byte[] -> Map + +/// Creates the module-level CustomDebugInformation row for the allocation-ordered +/// synthesized-name snapshot. Empty snapshots emit no row. +val computeSynthesizedNameSnapshotCustomDebugInfoRows: + snapshot: seq -> FSharp.Compiler.AbstractIL.ILPdbWriter.PdbModuleCustomDebugInfo list + /// Serializes the EnC Local Slot Map blob for 'info', byte-for-byte as Roslyn's /// SerializeLocalSlots. Returns the empty array when there are no slots (no CDI row /// should be emitted then). @@ -176,3 +192,8 @@ val deserialize: /// Fail safe: a null/empty or non-PDB image yields the empty map, and a method whose /// blobs do not decode is omitted rather than guessed. val readEncMethodDebugInfoFromPortablePdb: pdbBytes: byte[] -> Map + +/// Reads the F#-owned allocation-ordered synthesized-name snapshot from a portable PDB. +/// None means either the record is absent or invalid; callers must fall back to IL +/// reconstruction rather than trusting a partial layout. +val readSynthesizedNameSnapshotFromPortablePdb: pdbBytes: byte[] -> Map option diff --git a/src/Compiler/AbstractIL/ilwrite.fs b/src/Compiler/AbstractIL/ilwrite.fs index bf6277bf485..5c387dc7e41 100644 --- a/src/Compiler/AbstractIL/ilwrite.fs +++ b/src/Compiler/AbstractIL/ilwrite.fs @@ -3863,8 +3863,11 @@ type options = referenceAssemblyAttribOpt: ILAttribute option referenceAssemblySignatureHash : int option pathMap: PathMap + /// Hot reload baseline side channel: module-level CustomDebugInformation rows for + /// F#-owned records in the portable PDB. Empty for ordinary compiles. + moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list /// Per-method EnC CustomDebugInformation rows for the portable PDB writer, keyed by - /// IL method name. Empty for ordinary compiles, so flag-off output stays byte-identical. + /// IL method name. Empty for ordinary compiles. methodCustomDebugInfoRows: Map } let writeBinaryAux (stream: Stream, options: options, modul, normalizeAssemblyRefs) = @@ -4028,7 +4031,15 @@ let writeBinaryAux (stream: Stream, options: options, modul, normalizeAssemblyRe match options.pdbfile, options.portablePDB with | Some _, true -> let pdbInfo = - generatePortablePdb options.embedAllSource options.embedSourceList options.sourceLink options.checksumAlgorithm pdbData options.pathMap options.methodCustomDebugInfoRows + generatePortablePdb + options.embedAllSource + options.embedSourceList + options.sourceLink + options.checksumAlgorithm + pdbData + options.pathMap + options.moduleCustomDebugInfoRows + options.methodCustomDebugInfoRows if options.embeddedPDB then let uncompressedLength, contentId, stream, algorithmName, checkSum = pdbInfo diff --git a/src/Compiler/AbstractIL/ilwrite.fsi b/src/Compiler/AbstractIL/ilwrite.fsi index 08321664c2f..3045fe8980b 100644 --- a/src/Compiler/AbstractIL/ilwrite.fsi +++ b/src/Compiler/AbstractIL/ilwrite.fsi @@ -28,6 +28,9 @@ type options = referenceAssemblyAttribOpt: ILAttribute option referenceAssemblySignatureHash: int option pathMap: PathMap + /// Hot reload baseline side channel: module-level CustomDebugInformation rows for + /// F#-owned records in the portable PDB. Empty for ordinary compiles. + moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list /// Per-method EnC CustomDebugInformation rows for the portable PDB writer, keyed by /// IL method name. Empty for ordinary compiles, so flag-off output stays byte-identical. methodCustomDebugInfoRows: Map diff --git a/src/Compiler/AbstractIL/ilwritepdb.fs b/src/Compiler/AbstractIL/ilwritepdb.fs index 70f88b471d7..bfa9cafef99 100644 --- a/src/Compiler/AbstractIL/ilwritepdb.fs +++ b/src/Compiler/AbstractIL/ilwritepdb.fs @@ -122,6 +122,10 @@ type PdbMethodData = /// definition row in the portable PDB. type PdbMethodCustomDebugInfo = { KindGuid: Guid; Blob: byte[] } +/// A pre-serialized CustomDebugInformation row (kind GUID + blob) to attach to the +/// module definition row in the portable PDB. +type PdbModuleCustomDebugInfo = { KindGuid: Guid; Blob: byte[] } + module SequencePoint = let orderBySource sp1 sp2 = let c1 = compare sp1.Document sp2.Document @@ -348,6 +352,7 @@ type PortablePdbGenerator checksumAlgorithm, info: PdbData, pathMap: PathMap, + moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list, methodCustomDebugInfoRows: Map ) = @@ -484,6 +489,14 @@ type PortablePdbGenerator ) |> ignore + for cdiRow in moduleCustomDebugInfoRows |> List.sortBy (fun row -> row.KindGuid) do + metadata.AddCustomDebugInformation( + ModuleDefinitionHandle.op_Implicit EntityHandle.ModuleDefinition, + metadata.GetOrAddGuid cdiRow.KindGuid, + metadata.GetOrAddBlob cdiRow.Blob + ) + |> ignore + index let mutable lastLocalVariableHandle = Unchecked.defaultof @@ -881,10 +894,20 @@ let generatePortablePdb checksumAlgorithm (info: PdbData) (pathMap: PathMap) + (moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list) (methodCustomDebugInfoRows: Map) = let generator = - PortablePdbGenerator(embedAllSource, embedSourceList, sourceLink, checksumAlgorithm, info, pathMap, methodCustomDebugInfoRows) + PortablePdbGenerator( + embedAllSource, + embedSourceList, + sourceLink, + checksumAlgorithm, + info, + pathMap, + moduleCustomDebugInfoRows, + methodCustomDebugInfoRows + ) generator.Emit() diff --git a/src/Compiler/AbstractIL/ilwritepdb.fsi b/src/Compiler/AbstractIL/ilwritepdb.fsi index 09d380e44cc..3aa0679178a 100644 --- a/src/Compiler/AbstractIL/ilwritepdb.fsi +++ b/src/Compiler/AbstractIL/ilwritepdb.fsi @@ -73,6 +73,11 @@ type PdbMethodData = /// one method row (fail closed on ambiguity). type PdbMethodCustomDebugInfo = { KindGuid: System.Guid; Blob: byte[] } +/// A pre-serialized CustomDebugInformation row to attach to the module definition row +/// in the portable PDB (kind GUID + blob). Supplied by hot reload for F#-owned +/// deterministic baseline records. +type PdbModuleCustomDebugInfo = { KindGuid: System.Guid; Blob: byte[] } + [] type PdbData = { @@ -115,6 +120,7 @@ val generatePortablePdb: checksumAlgorithm: HashAlgorithm -> info: PdbData -> pathMap: PathMap -> + moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list -> methodCustomDebugInfoRows: Map -> int64 * BlobContentId * MemoryStream * string * byte[] diff --git a/src/Compiler/CodeGen/HotReloadBaseline.fs b/src/Compiler/CodeGen/HotReloadBaseline.fs new file mode 100644 index 00000000000..f68e3b10419 --- /dev/null +++ b/src/Compiler/CodeGen/HotReloadBaseline.fs @@ -0,0 +1,456 @@ +module internal FSharp.Compiler.HotReloadBaseline + +open System +open System.Collections.Generic +open System.Collections.Immutable + +open FSharp.Compiler.AbstractIL.EncMethodDebugInformation +open FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.CodeGen +open FSharp.Compiler.CompilerGeneratedNameMapState +open FSharp.Compiler.GeneratedNames +open FSharp.Compiler.Syntax.PrettyNaming + +[] +type SynthesizedNameSnapshotSource = + | Recorded + | Reconstructed + +type PortablePdbSnapshot = + { + Bytes: byte[] + TableRowCounts: ImmutableArray + EntryPointToken: int option + } + +type TypeDefinitionKey = + { + RowId: int + Namespace: string + Name: string + } + +type MethodDefinitionKey = + { + DeclaringType: TypeDefinitionKey + Name: string + Signature: byte list + } + +type FieldDefinitionKey = + { + DeclaringType: TypeDefinitionKey + Name: string + Signature: byte list + } + +type PropertyDefinitionKey = + { + DeclaringType: TypeDefinitionKey + Name: string + Signature: byte list + } + +type EventDefinitionKey = + { + DeclaringType: TypeDefinitionKey + Name: string + EventType: int + } + +type BaselineTokenMaps = + { + TypeTokens: Map + MethodTokens: Map + FieldTokens: Map + PropertyTokens: Map + EventTokens: Map + } + +type FSharpEmitBaseline = + { + ModuleId: Guid + Metadata: ILBaselineReader.MetadataSnapshot + PortablePdb: PortablePdbSnapshot option + TokenMaps: BaselineTokenMaps + SynthesizedNameSnapshot: Map + SynthesizedNameSnapshotSource: SynthesizedNameSnapshotSource + EncMethodDebugInfos: Map + EncClosureNames: Map> + } + +let private typeDefToken rowId = (0x02 <<< 24) ||| rowId +let private fieldToken rowId = (0x04 <<< 24) ||| rowId +let private methodDefToken rowId = (0x06 <<< 24) ||| rowId +let private eventToken rowId = (0x14 <<< 24) ||| rowId +let private propertyToken rowId = (0x17 <<< 24) ||| rowId + +let private typeFullName (key: TypeDefinitionKey) = + if String.IsNullOrEmpty key.Namespace then + key.Name + else + key.Namespace + "." + key.Name + +let private signatureList (bytes: byte[]) = bytes |> Array.toList + +let private buildTypeKeys (reader: ILBaselineReader.BaselineMetadataReader) = + [ + for rowId in 1 .. reader.TypeDefCount do + match reader.GetTypeDef rowId with + | Some row -> + yield + rowId, + { + RowId = rowId + Namespace = reader.GetString row.NamespaceOffset + Name = reader.GetString row.NameOffset + } + | None -> () + ] + |> Map.ofList + +let private emptyTokenMaps = + { + TypeTokens = Map.empty + MethodTokens = Map.empty + FieldTokens = Map.empty + PropertyTokens = Map.empty + EventTokens = Map.empty + } + +let private buildTokenMaps (reader: ILBaselineReader.BaselineMetadataReader) = + let typeKeys = buildTypeKeys reader + + let typeTokens: Map = + typeKeys + |> Map.toSeq + |> Seq.map (fun (rowId, key) -> key, typeDefToken rowId) + |> Map.ofSeq + + let methodTokens: Map = + seq { + for KeyValue(typeRowId, typeKey) in typeKeys do + match reader.GetTypeMethodRange typeRowId with + | None -> () + | Some(firstMethod, lastMethod) -> + for methodRowId in firstMethod..lastMethod do + match reader.GetMethodDef methodRowId with + | None -> () + | Some methodDef -> + let key: MethodDefinitionKey = + { + DeclaringType = typeKey + Name = reader.GetString methodDef.NameOffset + Signature = reader.GetBlob methodDef.SignatureOffset |> signatureList + } + + yield key, methodDefToken methodRowId + } + |> Map.ofSeq + + let fieldTokens: Map = + seq { + for KeyValue(typeRowId, typeKey) in typeKeys do + match reader.GetTypeFieldRange typeRowId with + | None -> () + | Some(firstField, lastField) -> + for fieldRowId in firstField..lastField do + match reader.GetField fieldRowId with + | None -> () + | Some fieldDef -> + let key: FieldDefinitionKey = + { + DeclaringType = typeKey + Name = reader.GetString fieldDef.NameOffset + Signature = reader.GetBlob fieldDef.SignatureOffset |> signatureList + } + + yield key, fieldToken fieldRowId + } + |> Map.ofSeq + + let propertyTokens: Map = + seq { + for propertyMapRowId in 1 .. reader.PropertyMapCount do + match reader.GetPropertyMapRange propertyMapRowId with + | Some(parentTypeRowId, firstProperty, lastProperty) -> + match Map.tryFind parentTypeRowId typeKeys with + | None -> () + | Some typeKey -> + for propertyRowId in firstProperty..lastProperty do + match reader.GetProperty propertyRowId with + | None -> () + | Some propertyDef -> + let key: PropertyDefinitionKey = + { + DeclaringType = typeKey + Name = reader.GetString propertyDef.NameOffset + Signature = reader.GetBlob propertyDef.SignatureOffset |> signatureList + } + + yield key, propertyToken propertyRowId + | None -> () + } + |> Map.ofSeq + + let eventTokens: Map = + seq { + for eventMapRowId in 1 .. reader.EventMapCount do + match reader.GetEventMapRange eventMapRowId with + | Some(parentTypeRowId, firstEvent, lastEvent) -> + match Map.tryFind parentTypeRowId typeKeys with + | None -> () + | Some typeKey -> + for eventRowId in firstEvent..lastEvent do + match reader.GetEvent eventRowId with + | None -> () + | Some eventDef -> + let key: EventDefinitionKey = + { + DeclaringType = typeKey + Name = reader.GetString eventDef.NameOffset + EventType = eventDef.EventType + } + + yield key, eventToken eventRowId + | None -> () + } + |> Map.ofSeq + + { + TypeTokens = typeTokens + MethodTokens = methodTokens + FieldTokens = fieldTokens + PropertyTokens = propertyTokens + EventTokens = eventTokens + } + +let private addSynthesizedName (buckets: Dictionary>) (name: string) = + if not (String.IsNullOrWhiteSpace name) && IsCompilerGeneratedName name then + let basicName = GetBasicNameOfPossibleCompilerGeneratedName name + let mapKey = SynthesizedNameMapKey basicName + + if not (String.IsNullOrWhiteSpace mapKey) then + let bucket = + match buckets.TryGetValue mapKey with + | true, existing -> existing + | _ -> + let created = ResizeArray() + buckets[mapKey] <- created + created + + if not (bucket.Contains name) then + bucket.Add name + +let private snapshotFromBuckets (buckets: Dictionary>) = + buckets + |> Seq.map (fun (KeyValue(key, bucket)) -> key, bucket.ToArray()) + |> Map.ofSeq + +let internal collectSynthesizedNameSnapshot (ilModule: ILModuleDef) = + let buckets = Dictionary>(StringComparer.Ordinal) + + let rec collectTypeDef (typeDef: ILTypeDef) = + addSynthesizedName buckets typeDef.Name + + typeDef.Fields.AsList() + |> List.iter (fun fieldDef -> addSynthesizedName buckets fieldDef.Name) + + typeDef.Methods.AsList() + |> List.iter (fun methodDef -> addSynthesizedName buckets methodDef.Name) + + typeDef.Properties.AsList() + |> List.iter (fun propertyDef -> addSynthesizedName buckets propertyDef.Name) + + typeDef.Events.AsList() + |> List.iter (fun eventDef -> addSynthesizedName buckets eventDef.Name) + + typeDef.NestedTypes.AsList() |> List.iter collectTypeDef + + ilModule.TypeDefs.AsList() |> List.iter collectTypeDef + snapshotFromBuckets buckets + +let internal collectRecordedSynthesizedNameSnapshot (_compilerGlobalState: obj) (map: ICompilerGeneratedNameMap) = map.Snapshot + +let private collectSynthesizedNameSnapshotFromTokens (tokenMaps: BaselineTokenMaps) = + let buckets = Dictionary>(StringComparer.Ordinal) + + for KeyValue(typeKey, _) in tokenMaps.TypeTokens do + addSynthesizedName buckets typeKey.Name + + for KeyValue(methodKey, _) in tokenMaps.MethodTokens do + addSynthesizedName buckets methodKey.Name + + for KeyValue(fieldKey, _) in tokenMaps.FieldTokens do + addSynthesizedName buckets fieldKey.Name + + for KeyValue(propertyKey, _) in tokenMaps.PropertyTokens do + addSynthesizedName buckets propertyKey.Name + + for KeyValue(eventKey, _) in tokenMaps.EventTokens do + addSynthesizedName buckets eventKey.Name + + snapshotFromBuckets buckets + +let private formatOccurrenceChainKey (ordinalChain: int list) = + ordinalChain |> List.map string |> String.concat "_" + +let private formatGenerationSuffixedClosureName baseName generation ordinalChain = + CompilerGeneratedNameSuffix baseName $"hotreload#g{generation}_o{formatOccurrenceChainKey ordinalChain}" + +let private cleanUpGeneratedTypeName (name: string) = + if name.IndexOfAny IllegalCharactersInTypeAndNamespaceNames = -1 then + name + else + (name, IllegalCharactersInTypeAndNamespaceNames) + ||> Array.fold (fun acc c -> acc.Replace(string c, "-")) + +let private typeDefSimpleNames (tokenMaps: BaselineTokenMaps) = + tokenMaps.TypeTokens + |> Map.toSeq + |> Seq.map (fun (key, _) -> key.Name) + |> Set.ofSeq + +let private methodNamesByToken (methodTokens: Map) = + methodTokens + |> Map.toSeq + |> Seq.map (fun (key, token) -> token, key.Name) + |> Map.ofSeq + +let deriveEncClosureNamesFromEncDebugInfos + (encMethodDebugInfos: Map) + (methodNamesByToken: Map) + (typeDefSimpleNames: Set) + : Map> = + + if Map.isEmpty encMethodDebugInfos then + Map.empty + else + let hasMidSessionClosureNames = + typeDefSimpleNames + |> Set.exists (fun name -> + match TryGetHotReloadNameGeneration name with + | Some generation -> generation >= 1 + | None -> false) + + if hasMidSessionClosureNames then + Map.empty + else + let hasReplayNamedTypeDef nameBase = + let prefix = nameBase + "@hotreload" + + typeDefSimpleNames + |> Set.exists (fun name -> + name.StartsWith(prefix, StringComparison.Ordinal) + && not (IsHotReloadGenerationSuffixedName name)) + + let derivedRows = + encMethodDebugInfos + |> Map.toList + |> List.choose (fun (methodToken, info) -> + match info.Closures, Map.tryFind methodToken methodNamesByToken with + | [], _ + | _, None -> None + | closures, Some methodName -> + let nameBase = cleanUpGeneratedTypeName methodName + + let rows = + closures + |> List.choose (fun closure -> + let chain = decodeOccurrenceKey closure.SyntaxOffset + let name = formatGenerationSuffixedClosureName nameBase 0 chain + + if Set.contains name typeDefSimpleNames then + Some(chain, name) + else + None) + + Some(methodToken, nameBase, rows)) + + let hasReplayOnlyCdiMethod = + derivedRows + |> List.exists (fun (_, nameBase, rows) -> List.isEmpty rows && hasReplayNamedTypeDef nameBase) + + if hasReplayOnlyCdiMethod then + Map.empty + else + derivedRows + |> List.choose (fun (methodToken, _, rows) -> + match rows with + | [] -> None + | _ -> Some(methodToken, Map.ofList rows)) + |> Map.ofList + +let private toPortablePdbSnapshot (expectedContentId: byte[]) (pdbBytes: byte[]) = + ILBaselineReader.readPortablePdbMetadata pdbBytes + |> Option.filter (fun metadata -> metadata.ContentId.AsSpan().SequenceEqual(expectedContentId)) + |> Option.map (fun metadata -> + { + Bytes = Array.copy pdbBytes + TableRowCounts = ImmutableArray.CreateRange metadata.TableRowCounts + EntryPointToken = metadata.EntryPointToken + }) + +let private createCore moduleId metadata portablePdb tokenMaps = + let reconstructedSynthesizedNames = + collectSynthesizedNameSnapshotFromTokens tokenMaps + + let synthesizedNames, synthesizedNameSnapshotSource = + match + portablePdb + |> Option.bind (fun snapshot -> readSynthesizedNameSnapshotFromPortablePdb snapshot.Bytes) + with + | Some recordedSnapshot -> recordedSnapshot, SynthesizedNameSnapshotSource.Recorded + | None -> reconstructedSynthesizedNames, SynthesizedNameSnapshotSource.Reconstructed + + let encMethodDebugInfos = + portablePdb + |> Option.map (fun snapshot -> readEncMethodDebugInfoFromPortablePdb snapshot.Bytes) + |> Option.defaultValue Map.empty + + { + ModuleId = moduleId + Metadata = metadata + PortablePdb = portablePdb + TokenMaps = tokenMaps + SynthesizedNameSnapshot = synthesizedNames + SynthesizedNameSnapshotSource = synthesizedNameSnapshotSource + EncMethodDebugInfos = encMethodDebugInfos + EncClosureNames = + deriveEncClosureNamesFromEncDebugInfos + encMethodDebugInfos + (methodNamesByToken tokenMaps.MethodTokens) + (typeDefSimpleNames tokenMaps) + } + +let tryReadFromAssemblyAndPdbBytes (assemblyBytes: byte[]) (portablePdbBytes: byte[] option) = + try + match + ILBaselineReader.metadataSnapshotFromBytes assemblyBytes, + ILBaselineReader.BaselineMetadataReader.Create assemblyBytes, + ILBaselineReader.readModuleMvidFromBytes assemblyBytes + with + | Some metadata, Some reader, Some moduleId when moduleId <> Guid.Empty -> + let portablePdb = + match ILBaselineReader.readCodeViewContentIdFromBytes assemblyBytes with + | Some expectedContentId -> portablePdbBytes |> Option.bind (toPortablePdbSnapshot expectedContentId) + | None -> None + + Some(createCore moduleId metadata portablePdb (buildTokenMaps reader)) + | _ -> None + with + | :? BadImageFormatException + | :? IO.IOException + | :? ArgumentException + | :? IndexOutOfRangeException + | :? InvalidOperationException + | :? OverflowException -> None + +let readFromAssemblyAndPdbBytes (assemblyBytes: byte[]) (portablePdbBytes: byte[] option) = + match tryReadFromAssemblyAndPdbBytes assemblyBytes portablePdbBytes with + | Some baseline -> baseline + | None -> invalidArg (nameof assemblyBytes) "assembly bytes do not contain readable CLI metadata" + +let metadataSnapshotFromBytes = ILBaselineReader.metadataSnapshotFromBytes + +let readModuleMvid = ILBaselineReader.readModuleMvidFromBytes diff --git a/src/Compiler/CodeGen/ILBaselineReader.fs b/src/Compiler/CodeGen/ILBaselineReader.fs new file mode 100644 index 00000000000..e41faadd39e --- /dev/null +++ b/src/Compiler/CodeGen/ILBaselineReader.fs @@ -0,0 +1,1015 @@ +/// Minimal binary reader for baseline PE and portable PDB metadata. +module internal FSharp.Compiler.CodeGen.ILBaselineReader + +open System +open System.Collections.Immutable +open System.IO +open System.Reflection.PortableExecutable +open System.Text + +type MetadataHeapSizes = + { + StringHeapSize: int + UserStringHeapSize: int + BlobHeapSize: int + GuidHeapSize: int + } + +type MetadataSnapshot = + { + HeapSizes: MetadataHeapSizes + TableRowCounts: int[] + GuidHeapStart: int + } + +type PortablePdbMetadata = + { + ContentId: byte[] + TableRowCounts: int[] + EntryPointToken: int option + } + +let private readUInt16 (bytes: byte[]) (offset: int) = + uint16 bytes[offset] ||| (uint16 bytes[offset + 1] <<< 8) + +let private readInt32 (bytes: byte[]) (offset: int) = + int bytes[offset] + ||| (int bytes[offset + 1] <<< 8) + ||| (int bytes[offset + 2] <<< 16) + ||| (int bytes[offset + 3] <<< 24) + +/// Reads an unsigned 64-bit little-endian value without sign-extending either half. +let internal readUInt64 (bytes: byte[]) (offset: int) = + uint64 (uint32 (readInt32 bytes offset)) + ||| (uint64 (uint32 (readInt32 bytes (offset + 4))) <<< 32) + +[] +let private tableCount = 64 + +module private TableIndices = + let Module = 0 + let TypeRef = 1 + let TypeDef = 2 + let FieldPtr = 3 + let Field = 4 + let MethodPtr = 5 + let MethodDef = 6 + let ParamPtr = 7 + let Param = 8 + let InterfaceImpl = 9 + let MemberRef = 10 + let Constant = 11 + let FieldMarshal = 13 + let DeclSecurity = 14 + let ClassLayout = 15 + let FieldLayout = 16 + let StandAloneSig = 17 + let EventMap = 18 + let EventPtr = 19 + let Event = 20 + let PropertyMap = 21 + let PropertyPtr = 22 + let Property = 23 + let MethodSemantics = 24 + let MethodImpl = 25 + let ModuleRef = 26 + let TypeSpec = 27 + let ImplMap = 28 + let FieldRVA = 29 + let Assembly = 32 + let AssemblyRef = 35 + let File = 38 + let ExportedType = 39 + let ManifestResource = 40 + let NestedClass = 41 + let GenericParam = 42 + let MethodSpec = 43 + let GenericParamConstraint = 44 + +type private StreamHeader = + { Offset: int; Size: int; Name: string } + +let private tryRvaToOffset (bytes: byte[]) (coffHeader: int) (optionalHeader: int) (sizeOfOptionalHeader: int) (rva: int) = + let numberOfSections = int (readUInt16 bytes (coffHeader + 2)) + let sectionHeadersStart = optionalHeader + sizeOfOptionalHeader + + let rec loop sectionIndex = + if sectionIndex >= numberOfSections then + None + else + let sectionOffset = sectionHeadersStart + sectionIndex * 40 + + if sectionOffset + 40 > bytes.Length then + None + else + let virtualSize = readInt32 bytes (sectionOffset + 8) + let virtualAddress = readInt32 bytes (sectionOffset + 12) + let rawSize = readInt32 bytes (sectionOffset + 16) + let pointerToRawData = readInt32 bytes (sectionOffset + 20) + let span = max virtualSize rawSize + + if rva >= virtualAddress && rva < virtualAddress + span then + Some(rva - virtualAddress + pointerToRawData) + else + loop (sectionIndex + 1) + + loop 0 + +let private findMetadataRoot (bytes: byte[]) : int option = + try + if bytes.Length < 64 || bytes[0] <> 0x4Duy || bytes[1] <> 0x5Auy then + None + else + let peOffset = readInt32 bytes 0x3C + + if peOffset < 0 || peOffset + 24 > bytes.Length then + None + elif + bytes[peOffset] <> 0x50uy + || bytes[peOffset + 1] <> 0x45uy + || bytes[peOffset + 2] <> 0uy + || bytes[peOffset + 3] <> 0uy + then + None + else + let coffHeader = peOffset + 4 + let sizeOfOptionalHeader = int (readUInt16 bytes (coffHeader + 16)) + let optionalHeader = coffHeader + 20 + let magic = readUInt16 bytes optionalHeader + + let dataDirectoryStart = + if magic = 0x20Bus then + optionalHeader + 112 + else + optionalHeader + 96 + + let cliDirectory = dataDirectoryStart + 14 * 8 + + if cliDirectory + 8 > bytes.Length then + None + else + let cliHeaderRva = readInt32 bytes cliDirectory + + if cliHeaderRva = 0 then + None + else + match tryRvaToOffset bytes coffHeader optionalHeader sizeOfOptionalHeader cliHeaderRva with + | None -> None + | Some cliHeaderOffset when cliHeaderOffset + 12 > bytes.Length -> None + | Some cliHeaderOffset -> + let metadataRva = readInt32 bytes (cliHeaderOffset + 8) + tryRvaToOffset bytes coffHeader optionalHeader sizeOfOptionalHeader metadataRva + with + | :? IndexOutOfRangeException + | :? ArgumentOutOfRangeException -> None + +let private parseStreamHeaders (bytes: byte[]) (metadataRoot: int) : StreamHeader list = + let signature = readInt32 bytes metadataRoot + + if signature <> 0x424A5342 then + [] + else + let versionLength = readInt32 bytes (metadataRoot + 12) + let paddedVersionLength = (versionLength + 3) &&& ~~~3 + let streamsOffset = metadataRoot + 16 + paddedVersionLength + let numberOfStreams = int (readUInt16 bytes (streamsOffset + 2)) + let mutable currentOffset = streamsOffset + 4 + let headers = ResizeArray() + + for _ in 1..numberOfStreams do + let offset = readInt32 bytes currentOffset + let size = readInt32 bytes (currentOffset + 4) + let mutable nameEnd = currentOffset + 8 + + while nameEnd < bytes.Length && bytes[nameEnd] <> 0uy do + nameEnd <- nameEnd + 1 + + if nameEnd >= bytes.Length then + invalidArg (nameof bytes) "invalid metadata stream header" + + let name = + Encoding.ASCII.GetString(bytes, currentOffset + 8, nameEnd - currentOffset - 8) + + let paddedNameLength = ((nameEnd - currentOffset - 8 + 1) + 3) &&& ~~~3 + + headers.Add( + { + Offset = metadataRoot + offset + Size = size + Name = name + } + ) + + currentOffset <- currentOffset + 8 + paddedNameLength + + headers |> Seq.toList + +let private findStream (headers: StreamHeader list) (name: string) = + headers |> List.tryFind (fun header -> header.Name = name) + +let private parseTablesStream (bytes: byte[]) (tablesStream: StreamHeader) = + let offset = tablesStream.Offset + let heapSizes = bytes[offset + 6] + let valid = readUInt64 bytes (offset + 8) + let rowCounts = Array.zeroCreate tableCount + let mutable rowCountOffset = offset + 24 + + for i in 0..63 do + if (valid &&& (1UL <<< i)) <> 0UL then + let rowCount = readInt32 bytes rowCountOffset + + if rowCount < 0 then + invalidArg (nameof bytes) "metadata table row counts must be non-negative" + + rowCounts[i] <- rowCount + rowCountOffset <- rowCountOffset + 4 + + heapSizes, rowCounts, offset, valid + +/// Computes the first table-row offset from the table header's valid-table mask. +let internal tableDataStart tablesOffset (valid: uint64) = + let mutable remaining = valid + let mutable presentTableCount = 0 + + while remaining <> 0UL do + presentTableCount <- presentTableCount + 1 + remaining <- remaining &&& (remaining - 1UL) + + tablesOffset + 24 + (presentTableCount * 4) + +let metadataSnapshotFromBytes (bytes: byte[]) : MetadataSnapshot option = + try + match findMetadataRoot bytes with + | None -> None + | Some metadataRoot -> + let streamHeaders = parseStreamHeaders bytes metadataRoot + let stringsStream = findStream streamHeaders "#Strings" + let userStringsStream = findStream streamHeaders "#US" + let blobStream = findStream streamHeaders "#Blob" + let guidStream = findStream streamHeaders "#GUID" + + let tablesStream = + findStream streamHeaders "#~" |> Option.orElse (findStream streamHeaders "#-") + + match tablesStream with + | None -> None + | Some tables -> + let _, rowCounts, _, _ = parseTablesStream bytes tables + + let trimmedStringHeapSize = + match stringsStream with + | None -> 0 + | Some stream -> + if stream.Size = 0 then + 0 + else + let last = stream.Offset + stream.Size - 1 + let mutable i = last + + while i >= stream.Offset && bytes[i] = 0uy do + i <- i - 1 + + if i = last then stream.Size else i - stream.Offset + 2 + + let heapSizes = + { + StringHeapSize = trimmedStringHeapSize + UserStringHeapSize = + userStringsStream + |> Option.map (fun stream -> stream.Size) + |> Option.defaultValue 0 + BlobHeapSize = blobStream |> Option.map (fun stream -> stream.Size) |> Option.defaultValue 0 + GuidHeapSize = guidStream |> Option.map (fun stream -> stream.Size) |> Option.defaultValue 0 + } + + Some + { + HeapSizes = heapSizes + TableRowCounts = rowCounts + GuidHeapStart = heapSizes.GuidHeapSize + } + with + | :? IndexOutOfRangeException + | :? ArgumentOutOfRangeException -> None + +let private readGuidFromBytes (bytes: byte[]) (guidIndex: int) = + if guidIndex <= 0 then + None + else + match findMetadataRoot bytes with + | None -> None + | Some metadataRoot -> + let streamHeaders = parseStreamHeaders bytes metadataRoot + + match findStream streamHeaders "#GUID" with + | None -> None + | Some guidStream -> + let offset = guidStream.Offset + (guidIndex - 1) * 16 + let streamEnd = int64 guidStream.Offset + int64 guidStream.Size + let guidEnd = int64 offset + 16L + + if + guidStream.Offset < 0 + || guidStream.Size < 0 + || streamEnd > int64 bytes.Length + || offset < guidStream.Offset + || guidEnd > streamEnd + then + None + else + Some(Guid(bytes[offset .. offset + 15])) + +/// Reads the portable CodeView content ID embedded in a PE debug directory. +let readCodeViewContentIdFromBytes (bytes: byte[]) : byte[] option = + try + use peReader = new PEReader(ImmutableArray.CreateRange bytes) + + peReader.ReadDebugDirectory() + |> Seq.tryFind (fun entry -> entry.IsPortableCodeView) + |> Option.map (fun entry -> + let data = peReader.ReadCodeViewDebugDirectoryData entry + let contentId = Array.zeroCreate 20 + data.Guid.ToByteArray().CopyTo(contentId, 0) + BitConverter.GetBytes(entry.Stamp).CopyTo(contentId, 16) + contentId) + with + | :? BadImageFormatException + | :? IOException + | :? InvalidOperationException -> None + +/// Parsed metadata context for reading table rows. +/// Internal (not private): tiny reader members can get cross-module inlined in Release +/// builds, and inlined code referencing a module-private type fails CLR visibility +/// checks at runtime. +type internal MetadataContext = + { + Bytes: byte[] + HeapSizes: byte + RowCounts: int[] + TablesStart: int + StringIndexSize: int + GuidIndexSize: int + BlobIndexSize: int + StringsStreamOffset: int + StringsStreamSize: int + BlobStreamOffset: int + } + +let private tableIndexSize (rowCounts: int[]) tableIndex = + if rowCounts[tableIndex] <= 65535 then 2 else 4 + +let private codedIndexSize (rowCounts: int[]) (tableIndices: int[]) tagBits = + let maxRows = + tableIndices + |> Array.map (fun tableIndex -> if tableIndex < tableCount then rowCounts[tableIndex] else 0) + |> Array.max + + let maxValue = (maxRows <<< tagBits) ||| ((1 <<< tagBits) - 1) + if maxValue <= 65535 then 2 else 4 + +let private resolutionScopeSize rowCounts = + codedIndexSize + rowCounts + [| + TableIndices.Module + TableIndices.ModuleRef + TableIndices.AssemblyRef + TableIndices.TypeRef + |] + 2 + +let private typeDefOrRefSize rowCounts = + codedIndexSize rowCounts [| TableIndices.TypeDef; TableIndices.TypeRef; TableIndices.TypeSpec |] 2 + +let private hasConstantSize rowCounts = + codedIndexSize rowCounts [| TableIndices.Field; TableIndices.Param; TableIndices.Property |] 2 + +let private hasCustomAttributeSize rowCounts = + codedIndexSize + rowCounts + [| + TableIndices.MethodDef + TableIndices.Field + TableIndices.TypeRef + TableIndices.TypeDef + TableIndices.Param + TableIndices.InterfaceImpl + TableIndices.MemberRef + TableIndices.Module + TableIndices.DeclSecurity + TableIndices.Property + TableIndices.Event + TableIndices.StandAloneSig + TableIndices.ModuleRef + TableIndices.TypeSpec + TableIndices.Assembly + TableIndices.AssemblyRef + TableIndices.File + TableIndices.ExportedType + TableIndices.ManifestResource + TableIndices.GenericParam + TableIndices.GenericParamConstraint + TableIndices.MethodSpec + |] + 5 + +let private hasFieldMarshalSize rowCounts = + codedIndexSize rowCounts [| TableIndices.Field; TableIndices.Param |] 1 + +let private hasDeclSecuritySize rowCounts = + codedIndexSize rowCounts [| TableIndices.TypeDef; TableIndices.MethodDef; TableIndices.Assembly |] 2 + +let private memberRefParentSize rowCounts = + codedIndexSize + rowCounts + [| + TableIndices.TypeDef + TableIndices.TypeRef + TableIndices.ModuleRef + TableIndices.MethodDef + TableIndices.TypeSpec + |] + 3 + +let private hasSemanticsSize rowCounts = + codedIndexSize rowCounts [| TableIndices.Event; TableIndices.Property |] 1 + +let private methodDefOrRefSize rowCounts = + codedIndexSize rowCounts [| TableIndices.MethodDef; TableIndices.MemberRef |] 1 + +let private memberForwardedSize rowCounts = + codedIndexSize rowCounts [| TableIndices.Field; TableIndices.MethodDef |] 1 + +let private implementationSize rowCounts = + codedIndexSize rowCounts [| TableIndices.File; TableIndices.AssemblyRef; TableIndices.ExportedType |] 2 + +let private customAttributeTypeSize rowCounts = + codedIndexSize rowCounts [| 0; 0; TableIndices.MethodDef; TableIndices.MemberRef; 0 |] 3 + +let private typeOrMethodDefSize rowCounts = + codedIndexSize rowCounts [| TableIndices.TypeDef; TableIndices.MethodDef |] 1 + +let private calculateTableRowSizes (ctx: MetadataContext) = + let rowCounts = ctx.RowCounts + let strIdx = ctx.StringIndexSize + let guidIdx = ctx.GuidIndexSize + let blobIdx = ctx.BlobIndexSize + let sizes = Array.zeroCreate tableCount + + sizes[0] <- 2 + strIdx + guidIdx + guidIdx + guidIdx + sizes[1] <- resolutionScopeSize rowCounts + strIdx + strIdx + + sizes[2] <- + 4 + + strIdx + + strIdx + + typeDefOrRefSize rowCounts + + tableIndexSize rowCounts TableIndices.Field + + tableIndexSize rowCounts TableIndices.MethodDef + + sizes[4] <- 2 + strIdx + blobIdx + sizes[6] <- 4 + 2 + 2 + strIdx + blobIdx + tableIndexSize rowCounts TableIndices.Param + sizes[8] <- 2 + 2 + strIdx + sizes[9] <- tableIndexSize rowCounts TableIndices.TypeDef + typeDefOrRefSize rowCounts + sizes[10] <- memberRefParentSize rowCounts + strIdx + blobIdx + sizes[11] <- 2 + hasConstantSize rowCounts + blobIdx + sizes[12] <- hasCustomAttributeSize rowCounts + customAttributeTypeSize rowCounts + blobIdx + sizes[13] <- hasFieldMarshalSize rowCounts + blobIdx + sizes[14] <- 2 + hasDeclSecuritySize rowCounts + blobIdx + sizes[15] <- 2 + 4 + tableIndexSize rowCounts TableIndices.TypeDef + sizes[16] <- 4 + tableIndexSize rowCounts TableIndices.Field + sizes[17] <- blobIdx + + sizes[18] <- + tableIndexSize rowCounts TableIndices.TypeDef + + tableIndexSize rowCounts TableIndices.Event + + sizes[20] <- 2 + strIdx + typeDefOrRefSize rowCounts + + sizes[21] <- + tableIndexSize rowCounts TableIndices.TypeDef + + tableIndexSize rowCounts TableIndices.Property + + sizes[23] <- 2 + strIdx + blobIdx + sizes[24] <- 2 + tableIndexSize rowCounts TableIndices.MethodDef + hasSemanticsSize rowCounts + + sizes[25] <- + tableIndexSize rowCounts TableIndices.TypeDef + + methodDefOrRefSize rowCounts + + methodDefOrRefSize rowCounts + + sizes[26] <- strIdx + sizes[27] <- blobIdx + + sizes[28] <- + 2 + + memberForwardedSize rowCounts + + strIdx + + tableIndexSize rowCounts TableIndices.ModuleRef + + sizes[29] <- 4 + tableIndexSize rowCounts TableIndices.Field + sizes[32] <- 4 + 2 + 2 + 2 + 2 + 4 + blobIdx + strIdx + strIdx + sizes[35] <- 2 + 2 + 2 + 2 + 4 + blobIdx + strIdx + strIdx + blobIdx + sizes[38] <- 4 + strIdx + blobIdx + sizes[39] <- 4 + 4 + strIdx + strIdx + implementationSize rowCounts + sizes[40] <- 4 + 4 + strIdx + implementationSize rowCounts + + sizes[41] <- + tableIndexSize rowCounts TableIndices.TypeDef + + tableIndexSize rowCounts TableIndices.TypeDef + + sizes[42] <- 2 + 2 + typeOrMethodDefSize rowCounts + strIdx + sizes[43] <- methodDefOrRefSize rowCounts + blobIdx + sizes[44] <- tableIndexSize rowCounts TableIndices.GenericParam + typeDefOrRefSize rowCounts + sizes + +let private calculateTableOffsets (ctx: MetadataContext) (rowSizes: int[]) = + let offsets = Array.zeroCreate tableCount + let mutable currentOffset = ctx.TablesStart + + for i in 0 .. tableCount - 1 do + offsets[i] <- currentOffset + currentOffset <- currentOffset + rowSizes[i] * ctx.RowCounts[i] + + offsets + +let private readHeapIndex (bytes: byte[]) offset indexSize = + if indexSize = 2 then + int (readUInt16 bytes offset) + else + readInt32 bytes offset + +let private createMetadataContext (bytes: byte[]) = + match findMetadataRoot bytes with + | None -> None + | Some metadataRoot -> + let streamHeaders = parseStreamHeaders bytes metadataRoot + + let tablesStream = + findStream streamHeaders "#~" |> Option.orElse (findStream streamHeaders "#-") + + match tablesStream with + | None -> None + | Some stream -> + let heapSizes, rowCounts, tablesOffset, valid = parseTablesStream bytes stream + + let pointerTables = + [| + TableIndices.FieldPtr + TableIndices.MethodPtr + TableIndices.ParamPtr + TableIndices.EventPtr + TableIndices.PropertyPtr + |] + + // The #- stream permits pointer-table indirection. This reader consumes the + // definition tables directly, so accepting a non-empty pointer table would + // associate members with the wrong declaring type. + if + stream.Name = "#-" + && pointerTables |> Array.exists (fun table -> rowCounts[table] <> 0) + then + None + else + let stringsBig = (heapSizes &&& 0x01uy) <> 0uy + let guidsBig = (heapSizes &&& 0x02uy) <> 0uy + let blobsBig = (heapSizes &&& 0x04uy) <> 0uy + + let stringsStream = + streamHeaders |> List.tryFind (fun header -> header.Name = "#Strings") + + Some + { + Bytes = bytes + HeapSizes = heapSizes + RowCounts = rowCounts + TablesStart = tableDataStart tablesOffset valid + StringIndexSize = if stringsBig then 4 else 2 + GuidIndexSize = if guidsBig then 4 else 2 + BlobIndexSize = if blobsBig then 4 else 2 + StringsStreamOffset = + stringsStream + |> Option.map (fun header -> header.Offset) + |> Option.defaultValue 0 + StringsStreamSize = stringsStream |> Option.map (fun header -> header.Size) |> Option.defaultValue 0 + BlobStreamOffset = + streamHeaders + |> List.tryFind (fun h -> h.Name = "#Blob") + |> Option.map (fun h -> h.Offset) + |> Option.defaultValue 0 + } + +let private readStringFromHeap (ctx: MetadataContext) offset = + if offset = 0 then + "" + else + let streamStart = int64 ctx.StringsStreamOffset + let streamSize = int64 ctx.StringsStreamSize + let streamEnd = streamStart + streamSize + let stringStart = streamStart + int64 offset + + // Metadata indices are scoped to #Strings, not to the containing PE image. + // Failing before decoding prevents malformed offsets from reading an adjacent heap. + if + offset < 0 + || streamStart < 0L + || streamSize < 0L + || streamEnd > int64 ctx.Bytes.Length + || stringStart < streamStart + || stringStart >= streamEnd + then + raise (BadImageFormatException("String heap index is outside the #Strings stream.")) + + let start = int stringStart + let streamEnd = int streamEnd + let mutable endPos = start + + while endPos < streamEnd && ctx.Bytes[endPos] <> 0uy do + endPos <- endPos + 1 + + if endPos = streamEnd then + raise (BadImageFormatException("String heap value is not terminated inside the #Strings stream.")) + + Encoding.UTF8.GetString(ctx.Bytes, start, endPos - start) + +let private readBlobFromHeap (ctx: MetadataContext) offset = + if offset <= 0 then + Array.empty + else + let start = ctx.BlobStreamOffset + offset + let b0 = int ctx.Bytes[start] + + let length, headerSize = + if b0 &&& 0x80 = 0 then + b0, 1 + elif b0 &&& 0xC0 = 0x80 then + ((b0 &&& 0x3F) <<< 8) ||| int ctx.Bytes[start + 1], 2 + else + (((b0 &&& 0x1F) <<< 24) + ||| (int ctx.Bytes[start + 1] <<< 16) + ||| (int ctx.Bytes[start + 2] <<< 8) + ||| int ctx.Bytes[start + 3]), + 4 + + if length = 0 then + Array.empty + else + ctx.Bytes[start + headerSize .. start + headerSize + length - 1] + +type TypeDefRowData = + { + Flags: int + NameOffset: int + NamespaceOffset: int + Extends: int + FieldList: int + MethodList: int + } + +type FieldRowData = + { + Flags: int + NameOffset: int + SignatureOffset: int + } + +type MethodDefRowData = + { + RVA: int + ImplFlags: int + Flags: int + NameOffset: int + SignatureOffset: int + ParamList: int + } + +type PropertyMapRowData = { Parent: int; PropertyList: int } + +type PropertyRowData = + { + Flags: int + NameOffset: int + SignatureOffset: int + } + +type EventMapRowData = { Parent: int; EventList: int } + +type EventRowData = + { + Flags: int + NameOffset: int + EventType: int + } + +type ModuleRowData = + { + Generation: int + NameOffset: int + MvidIndex: int + EncIdIndex: int + EncBaseIdIndex: int + } + +let private rowOffset (ctx: MetadataContext) (rowSizes: int[]) (tableOffsets: int[]) tableIndex rowId = + if rowId < 1 || rowId > ctx.RowCounts[tableIndex] then + None + else + Some(tableOffsets[tableIndex] + (rowId - 1) * rowSizes[tableIndex]) + +let private readTypeDefRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.TypeDef rowId + |> Option.map (fun offset -> + let extendsOffset = offset + 4 + ctx.StringIndexSize + ctx.StringIndexSize + + { + Flags = readInt32 ctx.Bytes offset + NameOffset = readHeapIndex ctx.Bytes (offset + 4) ctx.StringIndexSize + NamespaceOffset = readHeapIndex ctx.Bytes (offset + 4 + ctx.StringIndexSize) ctx.StringIndexSize + Extends = readHeapIndex ctx.Bytes extendsOffset (typeDefOrRefSize ctx.RowCounts) + FieldList = + readHeapIndex ctx.Bytes (extendsOffset + typeDefOrRefSize ctx.RowCounts) (tableIndexSize ctx.RowCounts TableIndices.Field) + MethodList = + readHeapIndex + ctx.Bytes + (extendsOffset + + typeDefOrRefSize ctx.RowCounts + + tableIndexSize ctx.RowCounts TableIndices.Field) + (tableIndexSize ctx.RowCounts TableIndices.MethodDef) + }) + +let private readFieldRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.Field rowId + |> Option.map (fun offset -> + { + Flags = int (readUInt16 ctx.Bytes offset) + NameOffset = readHeapIndex ctx.Bytes (offset + 2) ctx.StringIndexSize + SignatureOffset = readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize) ctx.BlobIndexSize + }) + +let private readMethodDefRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.MethodDef rowId + |> Option.map (fun offset -> + { + RVA = readInt32 ctx.Bytes offset + ImplFlags = int (readUInt16 ctx.Bytes (offset + 4)) + Flags = int (readUInt16 ctx.Bytes (offset + 6)) + NameOffset = readHeapIndex ctx.Bytes (offset + 8) ctx.StringIndexSize + SignatureOffset = readHeapIndex ctx.Bytes (offset + 8 + ctx.StringIndexSize) ctx.BlobIndexSize + ParamList = + readHeapIndex + ctx.Bytes + (offset + 8 + ctx.StringIndexSize + ctx.BlobIndexSize) + (tableIndexSize ctx.RowCounts TableIndices.Param) + }) + +let private readPropertyMapRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.PropertyMap rowId + |> Option.map (fun offset -> + { + Parent = readHeapIndex ctx.Bytes offset (tableIndexSize ctx.RowCounts TableIndices.TypeDef) + PropertyList = + readHeapIndex + ctx.Bytes + (offset + tableIndexSize ctx.RowCounts TableIndices.TypeDef) + (tableIndexSize ctx.RowCounts TableIndices.Property) + }) + +let private readPropertyRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.Property rowId + |> Option.map (fun offset -> + { + Flags = int (readUInt16 ctx.Bytes offset) + NameOffset = readHeapIndex ctx.Bytes (offset + 2) ctx.StringIndexSize + SignatureOffset = readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize) ctx.BlobIndexSize + }) + +let private readEventMapRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.EventMap rowId + |> Option.map (fun offset -> + { + Parent = readHeapIndex ctx.Bytes offset (tableIndexSize ctx.RowCounts TableIndices.TypeDef) + EventList = + readHeapIndex + ctx.Bytes + (offset + tableIndexSize ctx.RowCounts TableIndices.TypeDef) + (tableIndexSize ctx.RowCounts TableIndices.Event) + }) + +let private readEventRow ctx rowSizes tableOffsets rowId = + rowOffset ctx rowSizes tableOffsets TableIndices.Event rowId + |> Option.map (fun offset -> + { + Flags = int (readUInt16 ctx.Bytes offset) + NameOffset = readHeapIndex ctx.Bytes (offset + 2) ctx.StringIndexSize + EventType = readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize) (typeDefOrRefSize ctx.RowCounts) + }) + +let private readModuleRow (ctx: MetadataContext) (tableOffsets: int[]) = + if ctx.RowCounts[TableIndices.Module] < 1 then + None + else + let offset = tableOffsets[TableIndices.Module] + + Some + { + Generation = int (readUInt16 ctx.Bytes offset) + NameOffset = readHeapIndex ctx.Bytes (offset + 2) ctx.StringIndexSize + MvidIndex = readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize) ctx.GuidIndexSize + EncIdIndex = readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize + ctx.GuidIndexSize) ctx.GuidIndexSize + EncBaseIdIndex = + readHeapIndex ctx.Bytes (offset + 2 + ctx.StringIndexSize + ctx.GuidIndexSize + ctx.GuidIndexSize) ctx.GuidIndexSize + } + +type BaselineMetadataReader private (ctx: MetadataContext, rowSizes: int[], tableOffsets: int[]) = + + static member Create(bytes: byte[]) = + try + match createMetadataContext bytes with + | None -> None + | Some ctx -> + let rowSizes = calculateTableRowSizes ctx + let tableOffsets = calculateTableOffsets ctx rowSizes + Some(BaselineMetadataReader(ctx, rowSizes, tableOffsets)) + with + | :? IndexOutOfRangeException + | :? ArgumentOutOfRangeException -> None + + member _.RowCounts = ctx.RowCounts + + member _.TypeDefCount = ctx.RowCounts[TableIndices.TypeDef] + + member _.FieldCount = ctx.RowCounts[TableIndices.Field] + + member _.MethodDefCount = ctx.RowCounts[TableIndices.MethodDef] + + member _.PropertyMapCount = ctx.RowCounts[TableIndices.PropertyMap] + + member _.PropertyCount = ctx.RowCounts[TableIndices.Property] + + member _.EventMapCount = ctx.RowCounts[TableIndices.EventMap] + + member _.EventCount = ctx.RowCounts[TableIndices.Event] + + member _.GetModule() = readModuleRow ctx tableOffsets + + member _.GetTypeDef(rowId: int) = + readTypeDefRow ctx rowSizes tableOffsets rowId + + member _.GetField(rowId: int) = + readFieldRow ctx rowSizes tableOffsets rowId + + member _.GetMethodDef(rowId: int) = + readMethodDefRow ctx rowSizes tableOffsets rowId + + member _.GetPropertyMap(rowId: int) = + readPropertyMapRow ctx rowSizes tableOffsets rowId + + member _.GetProperty(rowId: int) = + readPropertyRow ctx rowSizes tableOffsets rowId + + member _.GetEventMap(rowId: int) = + readEventMapRow ctx rowSizes tableOffsets rowId + + member _.GetEvent(rowId: int) = + readEventRow ctx rowSizes tableOffsets rowId + + member _.GetString(offset: int) = readStringFromHeap ctx offset + + member _.GetBlob(offset: int) = readBlobFromHeap ctx offset + + member this.GetTypeFieldRange(typeRowId: int) = + match this.GetTypeDef typeRowId with + | None -> None + | Some typeDef -> + let firstField = typeDef.FieldList + + let lastField = + if typeRowId < ctx.RowCounts[TableIndices.TypeDef] then + match this.GetTypeDef(typeRowId + 1) with + | Some next -> next.FieldList - 1 + | None -> ctx.RowCounts[TableIndices.Field] + else + ctx.RowCounts[TableIndices.Field] + + if firstField <= 0 || firstField > lastField then + None + else + Some(firstField, lastField) + + member this.GetTypeMethodRange(typeRowId: int) = + match this.GetTypeDef typeRowId with + | None -> None + | Some typeDef -> + let firstMethod = typeDef.MethodList + + let lastMethod = + if typeRowId < ctx.RowCounts[TableIndices.TypeDef] then + match this.GetTypeDef(typeRowId + 1) with + | Some next -> next.MethodList - 1 + | None -> ctx.RowCounts[TableIndices.MethodDef] + else + ctx.RowCounts[TableIndices.MethodDef] + + if firstMethod <= 0 || firstMethod > lastMethod then + None + else + Some(firstMethod, lastMethod) + + member this.GetPropertyMapRange(propertyMapRowId: int) = + match this.GetPropertyMap propertyMapRowId with + | None -> None + | Some map -> + let firstProperty = map.PropertyList + + let lastProperty = + if propertyMapRowId < ctx.RowCounts[TableIndices.PropertyMap] then + match this.GetPropertyMap(propertyMapRowId + 1) with + | Some next -> next.PropertyList - 1 + | None -> ctx.RowCounts[TableIndices.Property] + else + ctx.RowCounts[TableIndices.Property] + + if firstProperty <= 0 || firstProperty > lastProperty then + None + else + Some(map.Parent, firstProperty, lastProperty) + + member this.GetEventMapRange(eventMapRowId: int) = + match this.GetEventMap eventMapRowId with + | None -> None + | Some map -> + let firstEvent = map.EventList + + let lastEvent = + if eventMapRowId < ctx.RowCounts[TableIndices.EventMap] then + match this.GetEventMap(eventMapRowId + 1) with + | Some next -> next.EventList - 1 + | None -> ctx.RowCounts[TableIndices.Event] + else + ctx.RowCounts[TableIndices.Event] + + if firstEvent <= 0 || firstEvent > lastEvent then + None + else + Some(map.Parent, firstEvent, lastEvent) + +let readModuleMvidFromBytes (bytes: byte[]) : Guid option = + try + match BaselineMetadataReader.Create bytes with + | None -> None + | Some reader -> reader.GetModule() |> Option.bind (fun m -> readGuidFromBytes bytes m.MvidIndex) + with + | :? IndexOutOfRangeException + | :? ArgumentOutOfRangeException -> None + +let private parsePdbStream (bytes: byte[]) (pdbStream: StreamHeader) = + if pdbStream.Size < 24 then + None + else + let entryPointToken = readInt32 bytes (pdbStream.Offset + 20) + if entryPointToken = 0 then None else Some entryPointToken + +let private parsePdbTablesStream (bytes: byte[]) (tablesStream: StreamHeader) = + let offset = tablesStream.Offset + let valid = readUInt64 bytes (offset + 8) + let pdbRowCounts = Array.zeroCreate 8 + let mutable rowCountOffset = offset + 24 + + for i in 0..63 do + if (valid &&& (1UL <<< i)) <> 0UL then + let count = readInt32 bytes rowCountOffset + + if i >= 0x30 && i <= 0x37 then + pdbRowCounts[i - 0x30] <- count + + rowCountOffset <- rowCountOffset + 4 + + pdbRowCounts + +let readPortablePdbMetadata (pdbBytes: byte[]) = + if pdbBytes.Length < 4 then + None + else + try + if readInt32 pdbBytes 0 <> 0x424A5342 then + None + else + let streamHeaders = parseStreamHeaders pdbBytes 0 + + let tablesStream = + findStream streamHeaders "#~" |> Option.orElse (findStream streamHeaders "#-") + + let pdbStream = findStream streamHeaders "#Pdb" + + Option.map2 + (fun stream pdb -> + { + ContentId = pdbBytes[pdb.Offset .. pdb.Offset + 19] + TableRowCounts = parsePdbTablesStream pdbBytes stream + EntryPointToken = parsePdbStream pdbBytes pdb + }) + tablesStream + (pdbStream |> Option.filter (fun stream -> stream.Size >= 24)) + with + | :? IndexOutOfRangeException + | :? ArgumentOutOfRangeException -> None diff --git a/src/Compiler/Driver/fsc.fs b/src/Compiler/Driver/fsc.fs index f5aa287b6a7..7988c47ce76 100644 --- a/src/Compiler/Driver/fsc.fs +++ b/src/Compiler/Driver/fsc.fs @@ -1149,6 +1149,7 @@ let main6 referenceAssemblyAttribOpt = referenceAssemblyAttribOpt referenceAssemblySignatureHash = refAssemblySignatureHash pathMap = tcConfig.pathMap + moduleCustomDebugInfoRows = [] methodCustomDebugInfoRows = Map.empty }, ilxMainModule, @@ -1181,6 +1182,7 @@ let main6 referenceAssemblyAttribOpt = None referenceAssemblySignatureHash = None pathMap = tcConfig.pathMap + moduleCustomDebugInfoRows = [] methodCustomDebugInfoRows = Map.empty }, ilxMainModule, diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index 1f5278f6ecc..69875f6e913 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -236,10 +236,10 @@ - - + + @@ -449,6 +449,8 @@ + + diff --git a/src/Compiler/Interactive/fsi.fs b/src/Compiler/Interactive/fsi.fs index a41b658cab1..3f81b67f9da 100644 --- a/src/Compiler/Interactive/fsi.fs +++ b/src/Compiler/Interactive/fsi.fs @@ -1941,6 +1941,7 @@ type internal FsiDynamicCompiler referenceAssemblyAttribOpt = None referenceAssemblySignatureHash = None pathMap = tcConfig.pathMap + moduleCustomDebugInfoRows = [] methodCustomDebugInfoRows = Map.empty } diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs b/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs index 831d9c6f020..55f8dff3282 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs @@ -16,6 +16,7 @@ open FSharp.Compiler.AbstractIL.IL open FSharp.Compiler.AbstractIL.ILBinaryWriter open FSharp.Compiler.AbstractIL.ILPdbWriter open FSharp.Compiler.AbstractIL.EncMethodDebugInformation +open FSharp.Compiler.HotReloadBaseline // ----------------------------------------------------------------------- // Round-trip properties (pure codec) @@ -183,6 +184,44 @@ let ``Full record round-trips through the three blobs`` () = Assert.Equal(info, decoded) +[] +let ``Synthesized name snapshot preserves mixed bucket allocation order`` () = + let expectedNames = + [| + "endpoints@hotreload" + "endpoints@hotreload#g0_o0" + "endpoints@hotreload-2" + "endpoints@hotreload#g0_o1" + "endpoints@hotreload-4" + "endpoints@hotreload#g0_o2" + "endpoints@hotreload#g0_o3" + "endpoints@hotreload#g0_o4" + |] + + let snapshot = [ struct ("endpoints", expectedNames) ] + + let blob = serializeSynthesizedNameSnapshot snapshot + let decoded = deserializeSynthesizedNameSnapshot blob + + match Map.tryFind "endpoints" decoded with + | Some actualNames -> Assert.Equal(expectedNames, actualNames) + | None -> failwith "expected endpoints bucket to round-trip" + +[] +let ``Synthesized name snapshot rejects a present but empty payload`` () = + // Canonical empty snapshots are represented by an absent CDI row. Accepting this + // non-canonical payload would suppress reconstruction from the assembly token maps. + let blob = [| 1uy; 0uy |] + Assert.Throws(fun () -> deserializeSynthesizedNameSnapshot blob |> ignore) + |> ignore + +[] +let ``Synthesized name snapshot bounds name allocation by remaining payload`` () = + // version=1, buckets=1, key="k", names=0x1fffffff, with no name payload. + let blob = [| 1uy; 1uy; 1uy; byte 'k'; 0xDFuy; 0xFFuy; 0xFFuy; 0xFFuy |] + Assert.Throws(fun () -> deserializeSynthesizedNameSnapshot blob |> ignore) + |> ignore + // ----------------------------------------------------------------------- // Occurrence-key packing // ----------------------------------------------------------------------- @@ -450,11 +489,14 @@ module private Plumbing = let private ilg = mkILGlobals (ILScopeRef.Assembly primaryAssemblyRef, [], ILScopeRef.Assembly primaryAssemblyRef) - let private mkMethod (name: string) (body: MethodBody) : ILMethodDef = - mkILNonGenericStaticMethod (name, ILMemberAccess.Public, [], mkILReturn ILType.Void, body) + let private mkAbstractMethod (name: string) : ILMethodDef = + // MethodBody.Abstract is the smallest body shape the IL writer accepts: it still + // gets a full PdbMethodData row (token, name), but skips code/IL-body generation + // entirely, which is all this test needs. + mkILNonGenericStaticMethod (name, ILMemberAccess.Public, [], mkILReturn ILType.Void, MethodBody.Abstract) - let private mkType (typeName: string) (methods: (string * MethodBody) list) : ILTypeDef = - let methods = methods |> List.map (fun (name, body) -> mkMethod name body) |> mkILMethods + let private mkType (typeName: string) (methodNames: string list) : ILTypeDef = + let methods = methodNames |> List.map mkAbstractMethod |> mkILMethods ILTypeDef( typeName, @@ -478,8 +520,8 @@ module private Plumbing = /// method table forbids two same-named methods of the same arity *within one type* /// (unrelated to CDI), but the CDI name-keying this test exercises is per-assembly, /// so cross-type name clashes are exactly the ambiguous case to cover. - let buildModuleOfMethodBodies (types: (string * (string * MethodBody) list) list) : ILModuleDef = - let typeDefs = types |> List.map (fun (typeName, methods) -> mkType typeName methods) + let buildModuleOfTypes (types: (string * string list) list) : ILModuleDef = + let typeDefs = types |> List.map (fun (typeName, methodNames) -> mkType typeName methodNames) let assemblyName = "EncCdiPlumbing_" + Guid.NewGuid().ToString("N") @@ -496,19 +538,17 @@ module private Plumbing = (mkILExportedTypes []) "v4.0.30319" // Non-empty: pins the metadata version explicitly rather than relying on primaryAssemblyRef's. - let buildModuleOfTypes (types: (string * string list) list) : ILModuleDef = - types - |> List.map (fun (typeName, methodNames) -> - typeName, methodNames |> List.map (fun name -> name, MethodBody.Abstract)) - |> buildModuleOfMethodBodies - /// Builds a minimal in-memory module with one type "T" declaring 'methodNames'. let buildModule (methodNames: string list) : ILModuleDef = buildModuleOfTypes [ "T", methodNames ] /// Writes 'modul' through the same in-memory ILBinaryWriter entry point fsi.fs uses for /// dynamic assembly emission, attaching 'methodCustomDebugInfoRows' as the CDI side /// channel. No hot reload flag or session state is involved. - let writeInMemory (modul: ILModuleDef) (methodCustomDebugInfoRows: Map) = + let writeInMemoryWithModuleRows + (modul: ILModuleDef) + (moduleCustomDebugInfoRows: PdbModuleCustomDebugInfo list) + (methodCustomDebugInfoRows: Map) + = let options: options = { ilg = ilg @@ -529,6 +569,7 @@ module private Plumbing = referenceAssemblyAttribOpt = None referenceAssemblySignatureHash = None pathMap = PathMap.empty + moduleCustomDebugInfoRows = moduleCustomDebugInfoRows methodCustomDebugInfoRows = methodCustomDebugInfoRows } @@ -536,6 +577,9 @@ module private Plumbing = | assemblyBytes, Some pdbBytes -> assemblyBytes, pdbBytes | _, None -> failwith "expected a portable PDB to be produced" + let writeInMemory (modul: ILModuleDef) (methodCustomDebugInfoRows: Map) = + writeInMemoryWithModuleRows modul [] methodCustomDebugInfoRows + type CdiRow = { MethodName: string option @@ -569,6 +613,123 @@ module private Plumbing = Blob = pdbMdReader.GetBlobBytes cdi.Value } ] + type private StreamHeader = + { + HeaderOffset: int + DataOffset: int + Size: int + Name: string + } + + let private readInt32 (bytes: byte[]) offset = + int bytes[offset] + ||| (int bytes[offset + 1] <<< 8) + ||| (int bytes[offset + 2] <<< 16) + ||| (int bytes[offset + 3] <<< 24) + + let private writeInt32 (bytes: byte[]) offset value = + bytes[offset] <- byte value + bytes[offset + 1] <- byte (value >>> 8) + bytes[offset + 2] <- byte (value >>> 16) + bytes[offset + 3] <- byte (value >>> 24) + + let private metadataStreamHeaders (assemblyBytes: byte[]) = + use peReader = new PEReader(ImmutableArray.CreateRange assemblyBytes) + let metadataRoot = peReader.PEHeaders.MetadataStartOffset + let versionLength = readInt32 assemblyBytes (metadataRoot + 12) + let streamsOffset = metadataRoot + 16 + ((versionLength + 3) &&& ~~~3) + let streamCount = int assemblyBytes[streamsOffset + 2] ||| (int assemblyBytes[streamsOffset + 3] <<< 8) + let headers = ResizeArray() + let mutable headerOffset = streamsOffset + 4 + + for _ in 1..streamCount do + let relativeOffset = readInt32 assemblyBytes headerOffset + let size = readInt32 assemblyBytes (headerOffset + 4) + let mutable nameEnd = headerOffset + 8 + + while assemblyBytes[nameEnd] <> 0uy do + nameEnd <- nameEnd + 1 + + let name = Text.Encoding.ASCII.GetString(assemblyBytes, headerOffset + 8, nameEnd - headerOffset - 8) + let paddedNameLength = ((nameEnd - headerOffset - 8 + 1) + 3) &&& ~~~3 + + headers.Add + { + HeaderOffset = headerOffset + DataOffset = metadataRoot + relativeOffset + Size = size + Name = name + } + + headerOffset <- headerOffset + 8 + paddedNameLength + + headers |> Seq.toList + + let private mutateStream name mutation (assemblyBytes: byte[]) = + let copy = Array.copy assemblyBytes + + let stream = + metadataStreamHeaders copy + |> List.find (fun stream -> stream.Name = name) + + mutation copy stream + copy + + let zeroMvidGuid assemblyBytes = + mutateStream "#GUID" (fun bytes stream -> Array.Clear(bytes, stream.DataOffset, 16)) assemblyBytes + + let truncateGuidStream assemblyBytes = + mutateStream "#GUID" (fun bytes stream -> writeInt32 bytes (stream.HeaderOffset + 4) 0) assemblyBytes + + let truncateStringsStreamBeforeReferencedStrings assemblyBytes = + // Keep only the reserved empty string so every non-zero table index points + // outside the declared heap, while the following metadata bytes remain present. + mutateStream "#Strings" (fun bytes stream -> writeInt32 bytes (stream.HeaderOffset + 4) 1) assemblyBytes + + let truncateStringsStreamBeforeMethodNameTerminator methodName assemblyBytes = + let copy = Array.copy assemblyBytes + + let methodNameOffset = + use peReader = new PEReader(ImmutableArray.CreateRange copy) + let metadataReader = peReader.GetMetadataReader() + + metadataReader.MethodDefinitions + |> Seq.map metadataReader.GetMethodDefinition + |> Seq.find (fun methodDef -> metadataReader.GetString(methodDef.Name) = methodName) + |> fun methodDef -> MetadataTokens.GetHeapOffset(methodDef.Name) + + let stringsStream = + metadataStreamHeaders copy + |> List.find (fun stream -> stream.Name = "#Strings") + + let terminatorOffset = methodNameOffset + Text.Encoding.UTF8.GetByteCount(methodName) + + // The name bytes remain inside #Strings, but its terminating zero is now the + // first byte outside the declared stream. + writeInt32 copy (stringsStream.HeaderOffset + 4) terminatorOffset + copy + + let addUnsupportedFieldPointerTable assemblyBytes = + mutateStream "#~" (fun bytes stream -> + bytes[stream.HeaderOffset + 8] <- byte '#' + bytes[stream.HeaderOffset + 9] <- byte '-' + + let validLow = readInt32 bytes (stream.DataOffset + 8) + writeInt32 bytes (stream.DataOffset + 8) (validLow ||| (1 <<< 3))) assemblyBytes + + let redirectBlobHeapPastEnd assemblyBytes = + let copy = Array.copy assemblyBytes + + use peReader = new PEReader(ImmutableArray.CreateRange copy) + let metadataRoot = peReader.PEHeaders.MetadataStartOffset + + let blobStream = + metadataStreamHeaders copy + |> List.find (fun stream -> stream.Name = "#Blob") + + writeInt32 copy blobStream.HeaderOffset (copy.Length - metadataRoot + 16) + copy + [] let ``Synthetic CustomDebugInformation row attaches to the right MethodDef`` () = let modul = Plumbing.buildModule [ "Foo"; "Bar" ] @@ -580,7 +741,7 @@ let ``Synthetic CustomDebugInformation row attaches to the right MethodDef`` () Closures = [ { SyntaxOffset = 0 } ] Lambdas = [ { SyntaxOffset = 5; ClosureOrdinal = 0 } ] } - let rows = + let rows: Map = Map.ofList [ "Foo", [ { KindGuid = PortableCustomDebugInfoKinds.encLambdaAndClosureMap; Blob = blob } ] ] let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows @@ -600,7 +761,7 @@ let ``Synthetic CustomDebugInformation row attaches to the right MethodDef`` () [] let ``Empty map produces zero CustomDebugInformation rows`` () = let modul = Plumbing.buildModule [ "Foo" ] - let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul Map.empty + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul (Map.empty) Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) [] @@ -614,7 +775,7 @@ let ``A method name absent from the module attaches nothing`` () = { EncMethodDebugInformation.Empty with StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 1 } ] } - let rows = + let rows: Map = Map.ofList [ "DoesNotExist", [ { KindGuid = PortableCustomDebugInfoKinds.encStateMachineStateMap; Blob = blob } ] ] let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows @@ -634,26 +795,199 @@ let ``An ambiguous method name attaches to neither method`` () = { EncMethodDebugInformation.Empty with StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 1 } ] } - let rows = + let rows: Map = Map.ofList [ "Dup", [ { KindGuid = PortableCustomDebugInfoKinds.encStateMachineStateMap; Blob = blob } ] ] let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) [] -let ``A method name shared with unavailable metadata attaches to neither method`` () = +let ``Synthetic module CustomDebugInformation row round-trips synthesized snapshot`` () = + let modul = Plumbing.buildModule [ "Foo" ] + + let expected = + [ struct ("endpoints", [| "endpoints@hotreload#g0_o0"; "endpoints@hotreload"; "endpoints@hotreload#g0_o1" |]) ] + + let moduleRows = computeSynthesizedNameSnapshotCustomDebugInfoRows expected + let _, pdbBytes = + Plumbing.writeInMemoryWithModuleRows modul moduleRows (Map.empty) + + match readSynthesizedNameSnapshotFromPortablePdb pdbBytes with + | Some snapshot -> + match Map.tryFind "endpoints" snapshot with + | Some names -> + let struct (_, expectedNames) = List.head expected + Assert.Equal(expectedNames, names) + | None -> failwith "expected endpoints bucket" + | None -> failwith "expected recorded synthesized-name snapshot" + +[] +let ``Baseline reader populates token maps and uses reconstructed synthesized snapshot when no record exists`` () = let modul = - Plumbing.buildModuleOfMethodBodies - [ "T1", [ "Dup", MethodBody.Abstract ] - "T2", [ "Dup", MethodBody.NotAvailable ] ] + Plumbing.buildModuleOfTypes + [ + "T", [ "Compute" ] + "endpoints@hotreload#g0_o0", [] + ] + + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul (Map.empty) + let baseline = readFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes) + + Assert.Equal(SynthesizedNameSnapshotSource.Reconstructed, baseline.SynthesizedNameSnapshotSource) + + Assert.True( + baseline.TokenMaps.TypeTokens + |> Seq.exists (fun kvp -> kvp.Key.Name = "T" && (kvp.Value &&& 0xFF000000) = 0x02000000), + "expected T TypeDef token") + + Assert.True( + baseline.TokenMaps.MethodTokens + |> Seq.exists (fun kvp -> + kvp.Key.Name = "Compute" + && kvp.Key.DeclaringType.Name = "T" + && (kvp.Value &&& 0xFF000000) = 0x06000000), + "expected Compute MethodDef token") + + match Map.tryFind "endpoints" baseline.SynthesizedNameSnapshot with + | Some names -> Assert.Contains("endpoints@hotreload#g0_o0", names) + | None -> failwith "expected reconstructed endpoints bucket" - let blob = - serializeStateMachineStates - { EncMethodDebugInformation.Empty with - StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 1 } ] } +[] +let ``Baseline reader does not trust a portable PDB from another assembly`` () = + let assemblyBytes, _ = + Plumbing.writeInMemory (Plumbing.buildModule [ "First" ]) (Map.empty) - let rows = - Map.ofList [ "Dup", [ { KindGuid = PortableCustomDebugInfoKinds.encStateMachineStateMap; Blob = blob } ] ] + let _, unrelatedPdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Second"; "Third" ]) (Map.empty) - let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows - Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) + let baseline = readFromAssemblyAndPdbBytes assemblyBytes (Some unrelatedPdbBytes) + + Assert.True(baseline.PortablePdb.IsNone, "a mismatched PDB must not contribute baseline state") + Assert.Equal(SynthesizedNameSnapshotSource.Reconstructed, baseline.SynthesizedNameSnapshotSource) + +[] +let ``Baseline reader rejects an empty MVID`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.zeroMvidGuid assemblyBytes + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline reader bounds the MVID read to the GUID stream`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.truncateGuidStream assemblyBytes + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline reader rejects pointer-table indirection in an uncompressed stream`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.addUnsupportedFieldPointerTable assemblyBytes + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline table data starts after valid tables even when their row count is zero`` () = + let valid = 1UL <<< 3 + + Assert.Equal(128, FSharp.Compiler.CodeGen.ILBaselineReader.tableDataStart 100 valid) + +[] +let ``Baseline valid-mask reader preserves the unsigned high bit`` () = + let bytes = [| 0uy; 0uy; 0uy; 0uy; 0uy; 0uy; 0uy; 0x80uy |] + + Assert.Equal(0x8000000000000000UL, FSharp.Compiler.CodeGen.ILBaselineReader.readUInt64 bytes 0) + +[] +let ``Baseline reader rejects a malformed signature blob without throwing`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.redirectBlobHeapPastEnd assemblyBytes + + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline reader rejects a string offset outside the strings stream`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.truncateStringsStreamBeforeReferencedStrings assemblyBytes + + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline reader rejects a string without a terminator inside the strings stream`` () = + let assemblyBytes, pdbBytes = + Plumbing.writeInMemory (Plumbing.buildModule [ "Compute" ]) (Map.empty) + + let assemblyBytes = Plumbing.truncateStringsStreamBeforeMethodNameTerminator "Compute" assemblyBytes + + Assert.True((tryReadFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes)).IsNone) + +[] +let ``Baseline reader gives recorded synthesized snapshot precedence over reconstruction`` () = + let modul = + Plumbing.buildModuleOfTypes + [ + "T", [ "Compute" ] + "endpoints@hotreload#g0_o0", [] + ] + + let recordedNames = + [| "recorded@hotreload"; "recorded@hotreload#g0_o0"; "recorded@hotreload-2" |] + + let moduleRows = computeSynthesizedNameSnapshotCustomDebugInfoRows [ struct ("endpoints", recordedNames) ] + let assemblyBytes, pdbBytes = + Plumbing.writeInMemoryWithModuleRows modul moduleRows (Map.empty) + let baseline = readFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes) + + Assert.Equal(SynthesizedNameSnapshotSource.Recorded, baseline.SynthesizedNameSnapshotSource) + + match Map.tryFind "endpoints" baseline.SynthesizedNameSnapshot with + | Some names -> Assert.Equal(recordedNames, names) + | None -> failwith "expected recorded endpoints bucket" + +[] +let ``Baseline reader reconstructs closure names from method CDI rows`` () = + let modul = + Plumbing.buildModuleOfTypes + [ + "T", [ "Compute" ] + "Compute@hotreload#g0_o0", [] + ] + + let lambdaMap = + serializeLambdaMap + { EncMethodDebugInformation.Empty with + MethodOrdinal = 0 + Closures = [ { SyntaxOffset = 0 } ] } + + let methodRows: Map = + Map.ofList + [ + "Compute", + [ + { + KindGuid = PortableCustomDebugInfoKinds.encLambdaAndClosureMap + Blob = lambdaMap + } + ] + ] + + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul methodRows + let baseline = readFromAssemblyAndPdbBytes assemblyBytes (Some pdbBytes) + + Assert.False(Map.isEmpty baseline.EncMethodDebugInfos, "expected method EnC debug information") + Assert.False(Map.isEmpty baseline.EncClosureNames, "expected reconstructed closure-name table") + + let closureNames = + baseline.EncClosureNames + |> Map.toSeq + |> Seq.collect (fun (_, rows) -> rows |> Map.toSeq |> Seq.map snd) + |> Set.ofSeq + + Assert.Contains("Compute@hotreload#g0_o0", closureNames)