[cDAC] Add EnC metadata and caching

docs/design/datacontracts/EcmaMetadata.md

@@ -23,9 +23,36 @@ Data descriptors used:
 | --- | --- | --- |
 | `Module` | `Base` | Pointer to start of PE file in memory |
 | `Module` | `DynamicMetadata` | Pointer to saved metadata for reflection emit modules |
+| `Module` | `MetadataGeneration` | Counter incremented each time a dynamic module re-serializes its saved metadata copy |
 | `Module` | `FieldDefToDescMap` | Mapping table |
 | `DynamicMetadata` | `Size` | Size of the dynamic metadata blob (as a 32bit uint) |
 | `DynamicMetadata` | `Data` | Start of dynamic metadata data array |
+| `PEAssembly` | `MDImport` | An `MDInternalRW` when `MetadataGeneration` is non-zero and module is not dynamic |
+| `MDInternalRW` | `Stgdb` | Pointer to the read-write storage database |
+| `CLiteWeightStgdbRW` | `MiniMd` | Address of the embedded `CMiniMdRW` model |
+| `CLiteWeightStgdbRW` | `MetadataAddress` | Pointer to the metadata image |
+| `CMiniMdRW` | `Schema` | Address of the embedded `CMiniMdSchema` |
+| `CMiniMdRW` | `TableCount` | Number of valid tables |
+| `CMiniMdRW` | `All4ByteColumns` | Whether all variable-width columns are 4 bytes wide |
+| `CMiniMdRW` | `Tables` | Address of the first table's record storage pool |
+| `CMiniMdRW` | `StringHeap` | Address of the string heap's storage pool |
+| `CMiniMdRW` | `BlobHeap` | Address of the blob heap's storage pool |
+| `CMiniMdRW` | `UserStringHeap` | Address of the user-string heap's storage pool |
+| `CMiniMdRW` | `GuidHeap` | Address of the GUID heap's storage pool |
+| `CMiniMdSchema` | `Heaps` | Heap-size flags byte |
+| `CMiniMdSchema` | `Sorted` | Sorted-table bit mask |
+| `CMiniMdSchema` | `RecordCounts` | Address of the inline per-table row count array |
+| `StgPool` | `SegData` | Pointer to the head segment's data |
+| `StgPool` | `NextSegment` | Pointer to the next pool segment |
+| `StgPool` | `DataSize` | Live byte count of the head segment |
+| `StgPoolSeg` | `SegData` | Pointer to this extension segment's data |
+| `StgPoolSeg` | `NextSegment` | Pointer to the next pool segment, or null |
+| `StgPoolSeg` | `DataSize` | Live byte count of this extension segment |
+
+Contracts used:
+| Contract Name |
+| --- |
+| `Loader` |
 
 
 ```csharp
@@ -75,152 +102,12 @@ MetadataReader? GetMetadata(ModuleHandle handle)
         }
         case AvailableMetadataType.ReadWrite:
         {
-            var targetEcmaMetadata = GetReadWriteMetadata(handle);
-
-            // From the multiple different target spans, we need to build a single
-            // contiguous ECMA-335 metadata blob.
-            BlobBuilder builder = new BlobBuilder();
-            builder.WriteUInt32(0x424A5342);
-
-            // major version
-            builder.WriteUInt16(1);
-
-            // minor version
-            builder.WriteUInt16(1);
-
-            // reserved
-            builder.WriteUInt32(0);
-
-            string version = targetEcmaMetadata.Schema.MetadataVersion;
-            builder.WriteInt32(AlignUp(version.Length, 4));
-            Write4ByteAlignedString(builder, version);
-
-            // reserved
-            builder.WriteUInt16(0);
-
-            // number of streams
-            ushort numStreams = 5; // #Strings, #US, #Blob, #GUID, #~ (metadata)
-            if (targetEcmaMetadata.Schema.VariableSizedColumnsAreAll4BytesLong)
-            {
-                // We direct MetadataReader to use 4-byte encoding for all variable-sized columns
-                // by providing the marker stream for a "minimal delta" image.
-                numStreams++;
-            }
-            builder.WriteUInt16(numStreams);
-
-            // Write Stream headers
-            if (targetEcmaMetadata.Schema.VariableSizedColumnsAreAll4BytesLong)
-            {
-                // Write the #JTD stream to indicate that all variable-sized columns are 4 bytes long.
-                WriteStreamHeader(builder, "#JTD", 0).WriteInt32(builder.Count);
-            }
-
-            BlobWriter stringsOffset = WriteStreamHeader(builder, "#Strings", (int)AlignUp(targetEcmaMetadata.StringHeap.Size, 4ul));
-            BlobWriter blobOffset = WriteStreamHeader(builder, "#Blob", (int)targetEcmaMetadata.BlobHeap.Size);
-            BlobWriter guidOffset = WriteStreamHeader(builder, "#GUID", (int)targetEcmaMetadata.GuidHeap.Size);
-            BlobWriter userStringOffset = WriteStreamHeader(builder, "#US", (int)targetEcmaMetadata.UserStringHeap.Size);
-
-            // We'll use the "uncompressed" tables stream name as the runtime may have created the *Ptr tables
-            // that are only present in the uncompressed tables stream.
-            BlobWriter tablesOffset = WriteStreamHeader(builder, "#-", 0);
-
-            // Write the heap-style Streams
-
-            stringsOffset.WriteInt32(builder.Count);
-            WriteTargetSpan(builder, targetEcmaMetadata.StringHeap);
-            for (ulong i = targetEcmaMetadata.StringHeap.Size; i < AlignUp(targetEcmaMetadata.StringHeap.Size, 4ul); i++)
-            {
-                builder.WriteByte(0);
-            }
-
-            blobOffset.WriteInt32(builder.Count);
-            WriteTargetSpan(builder, targetEcmaMetadata.BlobHeap);
-
-            guidOffset.WriteInt32(builder.Count);
-            WriteTargetSpan(builder, targetEcmaMetadata.GuidHeap);
-
-            userStringOffset.WriteInt32(builder.Count);
-            WriteTargetSpan(builder, targetEcmaMetadata.UserStringHeap);
-
-            // Write tables stream
-            tablesOffset.WriteInt32(builder.Count);
-
-            // Write tables stream header
-            builder.WriteInt32(0); // reserved
-            builder.WriteByte(2); // major version
-            builder.WriteByte(0); // minor version
-            uint heapSizes =
-                (targetEcmaMetadata.Schema.LargeStringHeap ? 1u << 0 : 0) |
-                (targetEcmaMetadata.Schema.LargeBlobHeap ? 1u << 1 : 0) |
-                (targetEcmaMetadata.Schema.LargeGuidHeap ? 1u << 2 : 0);
-
-            builder.WriteByte((byte)heapSizes);
-            builder.WriteByte(1); // reserved
-
-            ulong validTables = 0;
-            for (int i = 0; i < targetEcmaMetadata.Schema.RowCount.Length; i++)
-            {
-                if (targetEcmaMetadata.Schema.RowCount[i] != 0)
-                {
-                    validTables |= 1ul << i;
-                }
-            }
-
-            ulong sortedTables = 0;
-            for (int i = 0; i < targetEcmaMetadata.Schema.IsSorted.Length; i++)
-            {
-                if (targetEcmaMetadata.Schema.IsSorted[i])
-                {
-                    sortedTables |= 1ul << i;
-                }
-            }
-
-            builder.WriteUInt64(validTables);
-            builder.WriteUInt64(sortedTables);
-
-            foreach (int rowCount in targetEcmaMetadata.Schema.RowCount)
-            {
-                if (rowCount > 0)
-                {
-                    builder.WriteInt32(rowCount);
-                }
-            }
-
-            // Write the tables
-            foreach (TargetSpan span in targetEcmaMetadata.Tables)
-            {
-                WriteTargetSpan(builder, span);
-            }
-
-            MemoryStream metadataStream = new MemoryStream();
-            builder.WriteContentTo(metadataStream);
-            return MetadataReaderProvider.FromMetadataStream(metadataStream).GetMetadataReader();
-
-            void WriteTargetSpan(BlobBuilder builder, TargetSpan span)
-            {
-                Blob blob = builder.ReserveBytes(checked((int)span.Size));
-                _target.ReadBuffer(span.Address, blob.GetBytes().AsSpan());
-            }
-
-            static BlobWriter WriteStreamHeader(BlobBuilder builder, string name, int size)
-            {
-                BlobWriter offset = new(builder.ReserveBytes(4));
-                builder.WriteInt32(size);
-                Write4ByteAlignedString(builder, name);
-                return offset;
-            }
-
-            static void Write4ByteAlignedString(BlobBuilder builder, string value)
-            {
-                int bufferStart = builder.Count;
-                builder.WriteUTF8(value);
-                builder.WriteByte(0);
-                int stringEnd = builder.Count;
-                for (int i = stringEnd; i < bufferStart + AlignUp(value.Length, 4); i++)
-                {
-                    builder.WriteByte(0);
-                }
-            }
+            // From the ModuleHandle, walk Module -> PEAssembly -> MDInternalRW -> CLiteWeightStgdbRW -> CMiniMdRW.
+            // Read the schema and whether every variable-width column is 4 bytes, which selects the #JTD "minimal delta" marker when reserializing.
+            // Each heap and each entry of the Tables array is a storage pool; read its head segment with the
+            // StgPool descriptor and walk the rest of the chain as bare StgPoolSeg via NextSegment, concatenating every segment's
+            // data into a contiguous buffer. The resulting heaps and per-table record blobs,
+            // together with the schema, are returned as TargetEcmaMetadata and reserialized into an ECMA-335 image.
         }
     }
 }
@@ -229,68 +116,6 @@ MetadataReader? GetMetadata(ModuleHandle handle)
 ### Helper Methods
 
 ``` csharp
-using System;
-using System.Numerics;
-
-struct EcmaMetadataSchema
-{
-    public EcmaMetadataSchema(string metadataVersion, bool largeStringHeap, bool largeBlobHeap, bool largeGuidHeap, int[] rowCount, bool[] isSorted, bool variableSizedColumnsAre4BytesLong)
-    {
-        MetadataVersion = metadataVersion;
-        LargeStringHeap = largeStringHeap;
-        LargeBlobHeap = largeBlobHeap;
-        LargeGuidHeap = largeGuidHeap;
-
-        _rowCount = rowCount;
-        _isSorted = isSorted;
-
-        VariableSizedColumnsAreAll4BytesLong = variableSizedColumnsAre4BytesLong;
-    }
-
-    public readonly string MetadataVersion;
-
-    public readonly bool LargeStringHeap;
-    public readonly bool LargeBlobHeap;
-    public readonly bool LargeGuidHeap;
-
-    // Table data, these structures hold MetadataTable.Count entries
-    private readonly int[] _rowCount;
-    public readonly ReadOnlySpan<int> RowCount => _rowCount;
-
-    private readonly bool[] _isSorted;
-    public readonly ReadOnlySpan<bool> IsSorted => _isSorted;
-
-    // In certain scenarios the size of the tables is forced to be the maximum size
-    // Otherwise the size of columns should be computed based on RowSize/the various heap flags
-    public readonly bool VariableSizedColumnsAreAll4BytesLong;
-}
-
-class TargetEcmaMetadata
-{
-    public TargetEcmaMetadata(EcmaMetadataSchema schema,
-                        TargetSpan[] tables,
-                        TargetSpan stringHeap,
-                        TargetSpan userStringHeap,
-                        TargetSpan blobHeap,
-                        TargetSpan guidHeap)
-    {
-        Schema = schema;
-        _tables = tables;
-        StringHeap = stringHeap;
-        UserStringHeap = userStringHeap;
-        BlobHeap = blobHeap;
-        GuidHeap = guidHeap;
-    }
-
-    public EcmaMetadataSchema Schema { get; init; }
-
-    private TargetSpan[] _tables;
-    public ReadOnlySpan<TargetSpan> Tables => _tables;
-    public TargetSpan StringHeap { get; init; }
-    public TargetSpan UserStringHeap { get; init; }
-    public TargetSpan BlobHeap { get; init; }
-    public TargetSpan GuidHeap { get; init; }
-}
 
 [Flags]
 enum AvailableMetadataType
@@ -303,38 +128,32 @@ enum AvailableMetadataType
 
 AvailableMetadataType GetAvailableMetadataType(ModuleHandle handle)
 {
-    Data.Module module = new Data.Module(Target, handle.Address);
-
     AvailableMetadataType flags = AvailableMetadataType.None;
 
     TargetPointer dynamicMetadata = Target.ReadPointer(handle.Address + /* Module::DynamicMetadata offset */);
+    uint metadataGeneration = Target.Read<uint>(handle.Address + /* Module::MetadataGeneration offset */);
 
     if (dynamicMetadata != TargetPointer.Null)
+    {
         flags |= AvailableMetadataType.ReadWriteSavedCopy;
+    }
+    else if (metadataGeneration != 0)
+    {
+        flags |= AvailableMetadataType.ReadWrite;
+    }
     else
+    {
         flags |= AvailableMetadataType.ReadOnly;
+    }
 
     return flags;
 }
 
 TargetSpan GetReadWriteSavedMetadataAddress(ModuleHandle handle)
 {
-    Data.Module module = new Data.Module(Target, handle.Address);
     TargetPointer dynamicMetadata = Target.ReadPointer(handle.Address + /* Module::DynamicMetadata offset */);
-
     ulong size = Target.Read<uint>(handle.Address + /* DynamicMetadata::Size offset */);
     TargetPointer result = handle.Address + /* DynamicMetadata::Data offset */;
     return new(result, size);
 }
-
-TargetEcmaMetadata GetReadWriteMetadata(ModuleHandle handle)
-{
-    // [cdac] TODO.
-}
-
-T AlignUp<T>(T input, T alignment)
-    where T : IBinaryInteger<T>
-{
-    return input + (alignment - T.One) & ~(alignment - T.One);
-}
 ```

src/coreclr/md/enc/liteweightstgdbrw.cpp

@@ -186,6 +186,7 @@ CLiteWeightStgdbRW::InitFileForRead(
     if (SUCCEEDED(pStorage->OpenStream(MINIMAL_MD_STREAM, &cbData, &pvData)))
     {
         m_MiniMd.m_fMinimalDelta = TRUE;
+        m_MiniMd.m_fAll4ByteColumns = TRUE;
     }
 
     // Load the string pool.

src/coreclr/md/enc/metamodelrw.cpp

@@ -711,6 +711,7 @@ CMiniMdRW::CMiniMdRW()
     m_pHostFilter(0),
     m_pTokenRemapManager(0),
     m_fMinimalDelta(FALSE),
+    m_fAll4ByteColumns(FALSE),
     m_rENCRecs(0)
 {
 #ifdef _DEBUG
@@ -1276,6 +1277,7 @@ CMiniMdRW::ComputeGrowLimits(
         m_limIx = USHRT_MAX << 1;
         m_limRid = USHRT_MAX << 1;
         m_eGrow = eg_grown;
+        m_fAll4ByteColumns = TRUE;
     }
 } // CMiniMdRW::ComputeGrowLimits
 
@@ -3556,6 +3558,7 @@ CMiniMdRW::ExpandTables()
 
     // Remember that we've grown.
     m_eGrow = eg_grown;
+    m_fAll4ByteColumns = TRUE;
     m_maxRid = m_maxIx = UINT32_MAX;
 
 ErrExit:

src/coreclr/md/inc/VerifyLayouts.inc

@@ -200,6 +200,7 @@ ALIGN_FIELD(CMiniMdRW, m_bSortable, sizeof(BYTE)*TBL_COUNT, sizeof(BYTE))
 FIELD(CMiniMdRW, dbg_m_pLock, sizeof(void*))
 #endif
 FIELD(CMiniMdRW, m_fMinimalDelta, 4)
+FIELD(CMiniMdRW, m_fAll4ByteColumns, 4)
 FIELD(CMiniMdRW, m_rENCRecs, sizeof(void*))
 END_TYPE(CMiniMdRW, 8)
 

src/coreclr/md/inc/liteweightstgdb.h

@@ -15,6 +15,7 @@
 #include "metadata.h"
 #include "metamodelro.h"
 #include "metamodelrw.h"
+#include "cdacdata.h"
 
 #include "stgtiggerstorage.h"
 
@@ -79,6 +80,7 @@ void CLiteWeightStgdb<MiniMd>::Uninit()
 
 class CLiteWeightStgdbRW : public CLiteWeightStgdb<CMiniMdRW>
 {
+    friend struct ::cdac_data<CLiteWeightStgdbRW>;
     friend class RegMeta;
     friend class VerifyLayoutsMD;
     friend HRESULT TranslateSigHelper(
@@ -235,4 +237,11 @@ class CLiteWeightStgdbRW : public CLiteWeightStgdb<CMiniMdRW>
 #endif
 };  // class CLiteWeightStgdbRW
 
+template<>
+struct cdac_data<CLiteWeightStgdbRW>
+{
+    static constexpr size_t MiniMd = offsetof(CLiteWeightStgdbRW, m_MiniMd);
+    static constexpr size_t MetadataAddress = offsetof(CLiteWeightStgdbRW, m_pvMd);
+};
+
 #endif // __LiteWeightStgdb_h__