[lldb] Add a GetSubsetExtractorSP method to DataExtractor (llvm#177309)

We have many places where an ObjectFile subclass will take the
DataExtractor representing the entire binary, create a subsection of
that in a new DataExtractor for processing. For instance, an object file
might have symbol table entries with offsets into the string table. A
common code pattern is to create a DataExtractor representing the string
table, and then pulling out the c-strings based on those offsets from
the string table DataExtractor.

When code does this, it creates a new DataExtractor, copies the
Endianness and Wordsize from the original, copies the DataBufferSP from
the original, and specifies a new start and offset into the DataBuffer.

However, if the binary is actaully stored in a VirtualDataExtractor,
this code pattern loses the correct virtual-to-physical table
translation and will not work correctly. This new method simplifies this
common pattern, and correctly takes a subset of a VirtualDataExtractor.

The current implementation only allows a subset of a
VirtualDataExtractor that is contained within a single virtual entry
(LookupTable entry) and returns a DataExtractor with the corret offsets
calculated from the LookupTable. If we need to a VirtualDataExtractor to
create a Subset DataExtractor representing multiple separate virtual
ranges of data, we'll need to copy over the LookupTable entries that
cover all the bytes, and update them to be relative to the new
VirtualDataExtractor. It's a bit of work, and it's not needed right now,
so I'm not tackling that.

I am working on a larger PR which needs this new method. This PR
contains a unit test that uses it.

rdar://148939795

Author: jasonmolenda

lldb/include/lldb/Utility/DataExtractor.h

@@ -16,6 +16,7 @@
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h"
 #include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/SwapByteOrder.h"
 
@@ -818,6 +819,33 @@ class DataExtractor {
   ///     The extracted unsigned integer value.
   uint64_t GetULEB128(lldb::offset_t *offset_ptr) const;
 
+  /// Return a new DataExtractor which represents a subset of an existing
+  /// data extractor's bytes, copying all other fields from the existing
+  /// data extractor.
+  ///
+  /// \param[in] offset
+  ///     The starting byte offset into the shared data buffer.
+  /// \param[in] length
+  ///     The length of bytes that the new extractor can operate on.
+  ///
+  /// \return
+  ///     A shared pointer to a new DataExtractor.
+  virtual lldb::DataExtractorSP GetSubsetExtractorSP(lldb::offset_t offset,
+                                                     lldb::offset_t length);
+
+  /// Return a new DataExtractor which represents a subset of an existing
+  /// data extractor's bytes, copying all other fields from the existing
+  /// data extractor.  The length will be the largest contiguous region that
+  /// can be provided starting at \a offset; it is safe to read any bytes
+  /// within the returned subset Extractor.
+  ///
+  /// \param[in] offset
+  ///     The starting byte offset into the shared data buffer.
+  ///
+  /// \return
+  ///     A shared pointer to a new DataExtractor.
+  virtual lldb::DataExtractorSP GetSubsetExtractorSP(lldb::offset_t offset);
+
   lldb::DataBufferSP &GetSharedDataBuffer() { return m_data_sp; }
 
   bool HasData() { return m_start && m_end && m_end - m_start > 0; }
@@ -997,10 +1025,16 @@ class DataExtractor {
 
   void Checksum(llvm::SmallVectorImpl<uint8_t> &dest, uint64_t max_data = 0);
 
-  llvm::ArrayRef<uint8_t> GetData() const {
+  virtual llvm::ArrayRef<uint8_t> GetData() const {
     return {GetDataStart(), size_t(GetByteSize())};
   }
 
+  llvm::DWARFDataExtractor GetAsLLVMDWARF() const {
+    return llvm::DWARFDataExtractor(GetData(),
+                                    GetByteOrder() == lldb::eByteOrderLittle,
+                                    GetAddressByteSize());
+  }
+
   llvm::DataExtractor GetAsLLVM() const {
     return {GetData(), GetByteOrder() == lldb::eByteOrderLittle,
             uint8_t(GetAddressByteSize())};

lldb/include/lldb/Utility/VirtualDataExtractor.h

@@ -43,12 +43,22 @@ class VirtualDataExtractor : public DataExtractor {
                        lldb::ByteOrder byte_order, uint32_t addr_size,
                        LookupTable lookup_table);
 
+  VirtualDataExtractor(const lldb::DataBufferSP &data_sp,
+                       LookupTable lookup_table);
+
   const void *GetData(lldb::offset_t *offset_ptr,
                       lldb::offset_t length) const override;
 
   const uint8_t *PeekData(lldb::offset_t offset,
                           lldb::offset_t length) const override;
 
+  lldb::DataExtractorSP GetSubsetExtractorSP(lldb::offset_t offset,
+                                             lldb::offset_t length) override;
+
+  lldb::DataExtractorSP GetSubsetExtractorSP(lldb::offset_t offset) override;
+
+  llvm::ArrayRef<uint8_t> GetData() const override;
+
   /// Unchecked overrides
   /// @{
   uint8_t GetU8_unchecked(lldb::offset_t *offset_ptr) const override;

lldb/source/Plugins/ObjectContainer/BSD-Archive/ObjectContainerBSDArchive.cpp

@@ -417,14 +417,14 @@ ObjectFileSP ObjectContainerBSDArchive::GetObjectFile(const FileSpec *file) {
           lldb::offset_t data_offset = 0;
           DataExtractorSP extractor_sp =
               std::make_shared<DataExtractor>(child_data_sp);
-          return ObjectFile::FindPlugin(
+          return lldb_private::ObjectFile::FindPlugin(
               module_sp, &child, m_offset + object->file_offset,
               object->file_size, extractor_sp, data_offset);
         }
         lldb::offset_t data_offset = object->file_offset;
         DataExtractorSP extractor_sp =
             std::make_shared<DataExtractor>(m_archive_sp->GetData());
-        return ObjectFile::FindPlugin(
+        return lldb_private::ObjectFile::FindPlugin(
             module_sp, file, m_offset + object->file_offset, object->file_size,
             extractor_sp, data_offset);
       }
@@ -476,8 +476,8 @@ size_t ObjectContainerBSDArchive::GetModuleSpecifications(
             continue;
           FileSpec child = GetChildFileSpecificationsFromThin(
               object->ar_name.GetStringRef(), file);
-          if (ObjectFile::GetModuleSpecifications(child, 0, object->file_size,
-                                                  specs)) {
+          if (lldb_private::ObjectFile::GetModuleSpecifications(
+                  child, 0, object->file_size, specs)) {
             ModuleSpec &spec =
                 specs.GetModuleSpecRefAtIndex(specs.GetSize() - 1);
             llvm::sys::TimePoint<> object_mod_time(
@@ -492,7 +492,7 @@ size_t ObjectContainerBSDArchive::GetModuleSpecifications(
         const lldb::offset_t object_file_offset =
             file_offset + object->file_offset;
         if (object->file_offset < file_size && file_size > object_file_offset) {
-          if (ObjectFile::GetModuleSpecifications(
+          if (lldb_private::ObjectFile::GetModuleSpecifications(
                   file, object_file_offset, file_size - object_file_offset,
                   specs)) {
             ModuleSpec &spec =

lldb/source/Utility/DataExtractor.cpp

@@ -1050,3 +1050,16 @@ void DataExtractor::Checksum(llvm::SmallVectorImpl<uint8_t> &dest,
   dest.clear();
   dest.append(result.begin(), result.end());
 }
+
+DataExtractorSP DataExtractor::GetSubsetExtractorSP(offset_t offset,
+                                                    offset_t length) {
+  DataExtractorSP new_sp = std::make_shared<DataExtractor>(
+      GetSharedDataBuffer(), GetByteOrder(), GetAddressByteSize());
+  new_sp->SetData(GetSharedDataBuffer(), GetSharedDataOffset() + offset,
+                  length);
+  return new_sp;
+}
+
+DataExtractorSP DataExtractor::GetSubsetExtractorSP(offset_t offset) {
+  return GetSubsetExtractorSP(offset, GetByteSize() - offset);
+}

lldb/source/Utility/VirtualDataExtractor.cpp

@@ -31,6 +31,12 @@ VirtualDataExtractor::VirtualDataExtractor(const DataBufferSP &data_sp,
   m_lookup_table.Sort();
 }
 
+VirtualDataExtractor::VirtualDataExtractor(const DataBufferSP &data_sp,
+                                           LookupTable lookup_table)
+    : DataExtractor(data_sp), m_lookup_table(std::move(lookup_table)) {
+  m_lookup_table.Sort();
+}
+
 const VirtualDataExtractor::LookupTable::Entry *
 VirtualDataExtractor::FindEntry(offset_t virtual_addr) const {
   // Use RangeDataVector's binary search instead of linear search.
@@ -137,3 +143,70 @@ uint64_t VirtualDataExtractor::GetU64_unchecked(offset_t *offset_ptr) const {
   *offset_ptr += 8;
   return result;
 }
+
+DataExtractorSP
+VirtualDataExtractor::GetSubsetExtractorSP(offset_t virtual_offset,
+                                           offset_t virtual_length) {
+  const LookupTable::Entry *entry = FindEntry(virtual_offset);
+  assert(
+      entry &&
+      "VirtualDataExtractor subset extractor requires valid virtual address");
+  if (!entry)
+    return {};
+
+  // Entry::data is the offset into the DataBuffer's actual start/end range
+  // Entry::base is the virtual address at the start of this region of data
+  offset_t offset_into_entry_range = virtual_offset - entry->base;
+  assert(
+      offset_into_entry_range + virtual_length <= entry->size &&
+      "VirtualDataExtractor subset may not span multiple LookupTable entries");
+  if (offset_into_entry_range + virtual_length > entry->size)
+    return {};
+
+  // We could support a Subset VirtualDataExtractor which covered
+  // multiple LookupTable virtual entries, but we'd need to mutate
+  // all of the LookupTable entries that were properly included in
+  // the Subset, a bit tricky.  So we won't implement that until it's
+  // needed.
+
+  offset_t physical_start = entry->data + offset_into_entry_range;
+  std::shared_ptr<DataExtractor> new_sp = std::make_shared<DataExtractor>(
+      GetSharedDataBuffer(), GetByteOrder(), GetAddressByteSize());
+  new_sp->SetData(GetSharedDataBuffer(), physical_start, virtual_length);
+  return new_sp;
+}
+
+// Return a DataExtractorSP that contains a single LookupTable's entry; all
+// bytes are guaranteed to be readable.
+DataExtractorSP
+VirtualDataExtractor::GetSubsetExtractorSP(offset_t virtual_offset) {
+  const LookupTable::Entry *entry = FindEntry(virtual_offset);
+  assert(
+      entry &&
+      "VirtualDataExtractor subset extractor requires valid virtual address");
+  if (!entry)
+    return {};
+
+  // Entry::data is the offset into the DataBuffer's actual start/end range
+  // Entry::base is the virtual address at the start of this region of data
+  offset_t offset_into_entry_range = virtual_offset - entry->base;
+
+  offset_t physical_start = entry->data + offset_into_entry_range;
+  std::shared_ptr<DataExtractor> new_sp = std::make_shared<DataExtractor>(
+      GetSharedDataBuffer(), GetByteOrder(), GetAddressByteSize());
+  new_sp->SetData(GetSharedDataBuffer(), physical_start,
+                  entry->size - offset_into_entry_range);
+  return new_sp;
+}
+
+// Return an ArrayRef to the first contiguous region of the LookupTable
+// only.  The LookupTable entries may have gaps of unmapped data, and we
+// can't include those in the ArrayRef or something may touch those pages.
+llvm::ArrayRef<uint8_t> VirtualDataExtractor::GetData() const {
+  const LookupTable::Entry *entry = FindEntry(0);
+  assert(entry &&
+         "VirtualDataExtractor GetData requires valid virtual address");
+  if (!entry)
+    return {};
+  return {m_start + entry->data, entry->size};
+}

lldb/unittests/Utility/VirtualDataExtractorTest.cpp

@@ -581,3 +581,75 @@ TEST(VirtualDataExtractorTest, ReadExactlyAtEntryEnd) {
   EXPECT_EQ(extractor->GetU8(&virtual_offset), 0x04U);
   EXPECT_EQ(virtual_offset, 0x1004U);
 }
+
+TEST(VirtualDataExtractorTest, SubsetExtractorGetU32) {
+  uint32_t buffer[16];
+  // 0x11111111 0x22222222 ... 0xffffffff
+  for (int i = 0; i < 16; i++)
+    buffer[i] =
+        i << 28 | i << 24 | i << 20 | i << 16 | i << 12 | i << 8 | i << 4 | i;
+  DataBufferSP buffer_sp =
+      std::make_shared<DataBufferUnowned>((uint8_t *)&buffer, sizeof(buffer));
+  lldb::DataExtractorSP extractor = std::make_shared<VirtualDataExtractor>(
+      buffer_sp, eByteOrderLittle, 8,
+      Table{Entry(0x0, 4 * sizeof(uint32_t), 12 * sizeof(uint32_t)),
+            Entry(0x10, 4 * sizeof(uint32_t), 0 * sizeof(uint32_t)),
+            Entry(0x20, 4 * sizeof(uint32_t), 8 * sizeof(uint32_t)),
+            Entry(0x30, 4 * sizeof(uint32_t), 4 * sizeof(uint32_t))});
+
+  offset_t virtual_offset = 0;
+  // Entry(0x0, 4*sizeof(uint32_t), 12*sizeof(uint32_t))
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xccccccccU);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xddddddddU);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xeeeeeeeeU);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xffffffffU);
+  // Entry(0x10, 4*sizeof(uint32_t), 0*sizeof(uint32_t))
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x00000000U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x11111111U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x22222222U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x33333333U);
+  // Entry(0x20, 4*sizeof(uint32_t), 8*sizeof(uint32_t))
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x88888888U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x99999999U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xAAAAAAAAU);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0xBBBBBBBBU);
+  // Entry(0x30, 4*sizeof(uint32_t), 4*sizeof(uint32_t))
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x44444444U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x55555555U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x66666666U);
+  EXPECT_EQ(extractor->GetU32(&virtual_offset), 0x77777777U);
+
+  // sub_extractor starts at buffer[4] for 4 uint32_t's, aligned
+  // to the start of a LookupTable entry.
+  lldb::DataExtractorSP aligned_sub_extractor = extractor->GetSubsetExtractorSP(
+      4 * sizeof(uint32_t), 4 * sizeof(uint32_t));
+
+  virtual_offset = 0;
+  // Entry(0x10, 4*sizeof(uint32_t), 0*sizeof(uint32_t))
+  // {subset virtual offset: 0x0}
+  EXPECT_EQ(aligned_sub_extractor->GetU32(&virtual_offset), 0x00000000U);
+  EXPECT_EQ(aligned_sub_extractor->GetU32(&virtual_offset), 0x11111111U);
+  EXPECT_EQ(aligned_sub_extractor->GetU32(&virtual_offset), 0x22222222U);
+  EXPECT_EQ(aligned_sub_extractor->GetU32(&virtual_offset), 0x33333333U);
+
+  // sub_extractor starts at buffer[10] for 2 uint32_t's,
+  // only PART of a LookupTable entry.
+  lldb::DataExtractorSP misaligned_sub_extractor =
+      extractor->GetSubsetExtractorSP(10 * sizeof(uint32_t),
+                                      2 * sizeof(uint32_t));
+  virtual_offset = 0;
+  EXPECT_EQ(misaligned_sub_extractor->GetU32(&virtual_offset), 0xAAAAAAAAU);
+  EXPECT_EQ(misaligned_sub_extractor->GetU32(&virtual_offset), 0xBBBBBBBBU);
+
+  lldb::DataExtractorSP contiguous_subset = extractor->GetSubsetExtractorSP(0);
+  EXPECT_EQ(contiguous_subset->GetByteSize(), 4 * sizeof(uint32_t));
+
+  lldb::DataExtractorSP misaligned_contiguous_subset =
+      extractor->GetSubsetExtractorSP(2 * sizeof(uint32_t));
+  EXPECT_EQ(misaligned_contiguous_subset->GetByteSize(), 2 * sizeof(uint32_t));
+
+  // Ask for a subset in the second LookupTable entry.
+  lldb::DataExtractorSP middle_contiguous_subset =
+      extractor->GetSubsetExtractorSP(4 * sizeof(uint32_t));
+  EXPECT_EQ(middle_contiguous_subset->GetByteSize(), 4 * sizeof(uint32_t));
+}