DOC: Add thorough Doxygen and inline documentation for OOC architecture

Add comprehensive documentation to all new methods, type aliases,
classes, and algorithms introduced in the OOC architecture rewrite.
Every new public API now has Doxygen explaining what it does, how it
works, and why it is needed. Algorithm implementations have step-by-
step inline comments explaining the logic.

Signed-off-by: Joey Kleingers <joey.kleingers@bluequartz.net>

Author: joeykleingers

src/Plugins/SimplnxCore/src/SimplnxCore/utils/VtkUtilities.hpp

@@ -134,6 +134,9 @@ std::string TypeForPrimitive(const IFilter::MessageHandler& messageHandler)
   return "";
 }
 
+// -----------------------------------------------------------------------------
+// Functor for writing a DataArray to a VTK legacy file via FILE* I/O.
+// Supports both binary (big-endian) and ASCII output modes.
 // -----------------------------------------------------------------------------
 struct WriteVtkDataArrayFunctor
 {
@@ -161,15 +164,31 @@ struct WriteVtkDataArrayFunctor
     fprintf(outputFile, "LOOKUP_TABLE default\n");
     if(binary)
     {
-      // Read data into buffer, byte-swap in memory, and write.
-      // This avoids modifying the DataStore (critical for OOC stores
-      // where per-element byte-swap via setValue would be very slow).
+      // ---------------------------------------------------------------
+      // Chunked binary write pattern for OOC compatibility
+      // ---------------------------------------------------------------
+      // The original code used dataArray->data() for a single fwrite,
+      // which requires the entire array to be resident in memory. This
+      // fails for OOC stores where data lives on disk and data() is
+      // not available.
+      //
+      // Instead, we read 4096 elements at a time into a local buffer
+      // via copyIntoBuffer (the OOC-compatible bulk read API), perform
+      // an in-place byte swap to big-endian (VTK legacy binary format
+      // requires big-endian), and fwrite the buffer. This keeps memory
+      // usage constant regardless of array size.
+      //
+      // For bool arrays, copyIntoBuffer is not available (bool is not
+      // a supported span type), so we use per-element getValue() and
+      // convert to uint8 (0 or 1).
+      // ---------------------------------------------------------------
       constexpr usize k_ChunkSize = 4096;
       for(usize offset = 0; offset < totalElements; offset += k_ChunkSize)
       {
         usize count = std::min(k_ChunkSize, totalElements - offset);
         if constexpr(std::is_same_v<T, bool>)
         {
+          // Bool special case: convert to uint8 via per-element access.
           std::vector<uint8> buf(count);
           for(usize i = 0; i < count; i++)
           {
@@ -179,10 +198,14 @@ struct WriteVtkDataArrayFunctor
         }
         else
         {
+          // General case: bulk read into buffer, byte-swap, then write.
           std::vector<T> buf(count);
           dataStore.copyIntoBuffer(offset, nonstd::span<T>(buf.data(), count));
           if constexpr(endian::little == endian::native)
           {
+            // VTK legacy binary requires big-endian. Swap in the local
+            // buffer rather than mutating the DataStore, which would be
+            // slow for OOC stores and would modify shared data.
             for(usize i = 0; i < count; i++)
             {
               buf[i] = nx::core::byteswap(buf[i]);

src/simplnx/Core/Preferences.cpp

@@ -120,14 +120,17 @@ void Preferences::setLargeDataFormat(std::string dataFormat)
   if(dataFormat.empty())
   {
     // Remove the key so the default (set by plugins) can take effect.
-    // An empty string means "not configured", not "in-core" — use
-    // k_InMemoryFormat for an explicit in-core choice.
+    // An empty string means "not configured", not "in-core". To explicitly
+    // request in-core storage, pass k_InMemoryFormat instead. This distinction
+    // matters because the SimplnxOoc plugin sets a default OOC format on startup,
+    // and erasing the user value lets that default take effect.
     m_Values.erase(k_PreferredLargeDataFormat_Key);
   }
   else
   {
     m_Values[k_PreferredLargeDataFormat_Key] = dataFormat;
   }
+  // Recompute the cached m_UseOoc flag after any format change
   checkUseOoc();
 }
 
@@ -271,9 +274,14 @@ Result<> Preferences::loadFromFile(const std::filesystem::path& filepath)
 
   m_Values = parsedResult;
 
-  // Migrate legacy format strings from saved preferences:
-  // - Empty string: legacy "not configured" state → remove so plugin defaults take effect
-  // - "In-Memory": legacy explicit in-core choice → remove so plugin defaults take effect
+  // Migrate legacy format strings from saved preferences files that were
+  // written before the OOC architecture was finalized. Two legacy values
+  // need cleanup:
+  //   - Empty string (""):   Old "not configured" state. Removing the key
+  //     lets the plugin-supplied default (e.g., "HDF5-OOC") take effect.
+  //   - "In-Memory":         Old explicit in-core sentinel. Replaced by
+  //     k_InMemoryFormat ("Simplnx-Default-In-Memory"). Removing the key
+  //     avoids confusion with the new sentinel value.
   if(m_Values.contains(k_PreferredLargeDataFormat_Key) && m_Values[k_PreferredLargeDataFormat_Key].is_string())
   {
     const std::string savedFormat = m_Values[k_PreferredLargeDataFormat_Key].get<std::string>();
@@ -283,19 +291,27 @@ Result<> Preferences::loadFromFile(const std::filesystem::path& filepath)
     }
   }
 
+  // Recompute derived state from the loaded (and possibly migrated) values
   checkUseOoc();
   updateMemoryDefaults();
   return {};
 }
 
 void Preferences::checkUseOoc()
 {
+  // Resolve the format from user values first, then default values (via value())
   auto formatJson = value(k_PreferredLargeDataFormat_Key);
+
+  // If no format is configured (null/non-string), OOC is not active
   if(formatJson.is_null() || !formatJson.is_string())
   {
     m_UseOoc = false;
     return;
   }
+
+  // OOC is active when the format is a non-empty string that is NOT the
+  // explicit in-memory sentinel. This means a plugin (e.g., SimplnxOoc)
+  // has registered a real OOC format like "HDF5-OOC".
   const std::string format = formatJson.get<std::string>();
   m_UseOoc = !format.empty() && format != k_InMemoryFormat;
 }
@@ -325,9 +341,15 @@ void Preferences::setForceOocData(bool forceOoc)
 
 void Preferences::updateMemoryDefaults()
 {
+  // Reserve headroom equal to 2x the single-array large-data threshold.
+  // This leaves room for the OS, the application, and at least one large
+  // array being constructed while the DataStructure holds existing data.
   const uint64 minimumRemaining = 2 * defaultValueAs<uint64>(k_LargeDataSize_Key);
   const uint64 totalMemory = Memory::GetTotalMemory();
   uint64 targetValue = totalMemory - minimumRemaining;
+
+  // On low-memory systems where the reservation exceeds total RAM,
+  // fall back to using half of total RAM as the threshold
   if(minimumRemaining >= totalMemory)
   {
     targetValue = totalMemory / 2;
@@ -368,8 +390,12 @@ void Preferences::setOocRangeScanTimeoutSeconds(uint32 seconds)
 
 uint64 Preferences::oocMemoryBudgetBytes() const
 {
-  // Default: 8 GB (the application will set the real default from
-  // OocMemoryBudgetManager::defaultBudgetBytes() on startup)
+  // Hard-coded fallback of 8 GB. This conservative default is used when:
+  //   1. The user has never saved an explicit budget preference, AND
+  //   2. The SimplnxOoc plugin has not yet called setOocMemoryBudgetBytes()
+  //      with its computed default (50% of system RAM).
+  // Using m_Values.value() (not the value() member) reads directly from
+  // user-set values with the fallback, bypassing the default-value layer.
   static constexpr uint64 k_DefaultBudget = 8ULL * 1024 * 1024 * 1024;
   return m_Values.value(k_OocMemoryBudgetBytes_Key, k_DefaultBudget);
 }

src/simplnx/Core/Preferences.hpp

@@ -24,14 +24,49 @@ class SIMPLNX_EXPORT Preferences
   friend class AbstractPlugin;
 
 public:
-  static inline constexpr StringLiteral k_LargeDataSize_Key = "large_data_size";                             // bytes
-  static inline constexpr StringLiteral k_PreferredLargeDataFormat_Key = "large_data_format";                // string
-  static inline constexpr StringLiteral k_InMemoryFormat = "Simplnx-Default-In-Memory";                      // explicit in-core; empty string means "not configured"
-  static inline constexpr StringLiteral k_LargeDataStructureSize_Key = "large_datastructure_size";           // bytes
-  static inline constexpr StringLiteral k_ForceOocData_Key = "force_ooc_data";                               // boolean
-  static inline constexpr nx::core::StringLiteral k_OoCTempDirectory_ID = "ooc_temp_directory";              // Out-of-Core temp directory
-  static inline constexpr StringLiteral k_OocRangeScanTimeoutSeconds_Key = "ooc_range_scan_timeout_seconds"; // uint32, seconds
-  static inline constexpr StringLiteral k_OocMemoryBudgetBytes_Key = "ooc_memory_budget_bytes";              // uint64, bytes
+  /// @name Preference Keys
+  /// JSON keys used to store and retrieve preference values. These keys appear
+  /// in the serialized preferences.json file and are used internally by the
+  /// getter/setter methods below.
+  /// @{
+
+  /// Byte-size threshold above which a single DataArray is considered "large"
+  /// and may be written to an OOC-capable format instead of in-memory storage.
+  static inline constexpr StringLiteral k_LargeDataSize_Key = "large_data_size";
+
+  /// Name of the preferred storage format for large DataArrays (e.g., "HDF5-OOC").
+  /// An empty string means "not yet configured by the user or plugin".
+  static inline constexpr StringLiteral k_PreferredLargeDataFormat_Key = "large_data_format";
+
+  /// Sentinel value for k_PreferredLargeDataFormat_Key that explicitly requests
+  /// in-memory storage. This is distinct from an empty string, which means
+  /// "not configured" and falls back to plugin-supplied defaults.
+  static inline constexpr StringLiteral k_InMemoryFormat = "Simplnx-Default-In-Memory";
+
+  /// Byte-size threshold for the entire DataStructure. When total memory usage
+  /// approaches this value, the application may switch to OOC storage for new arrays.
+  /// The default is computed dynamically by updateMemoryDefaults() based on system RAM.
+  static inline constexpr StringLiteral k_LargeDataStructureSize_Key = "large_datastructure_size";
+
+  /// Boolean flag that, when true, forces all new DataArrays to use OOC storage
+  /// regardless of their size. Only takes effect when an OOC format is active.
+  static inline constexpr StringLiteral k_ForceOocData_Key = "force_ooc_data";
+
+  /// Filesystem path to the directory where OOC temporary files (chunk stores,
+  /// backing HDF5 files) are created during filter execution.
+  static inline constexpr nx::core::StringLiteral k_OoCTempDirectory_ID = "ooc_temp_directory";
+
+  /// Timeout in seconds for the background thread that scans OOC DataArrays to
+  /// compute their value ranges (min/max). If the scan does not complete within
+  /// this window, the range is reported as unknown. Default: 30 seconds.
+  static inline constexpr StringLiteral k_OocRangeScanTimeoutSeconds_Key = "ooc_range_scan_timeout_seconds";
+
+  /// Total memory budget in bytes shared across all OOC caching subsystems
+  /// (chunk cache, stride cache, partition cache). The OOC memory budget manager
+  /// distributes this budget via global LRU eviction. Default: 8 GB.
+  static inline constexpr StringLiteral k_OocMemoryBudgetBytes_Key = "ooc_memory_budget_bytes";
+
+  /// @}
 
   /**
    * @brief Returns the default file path for storing preferences based on the application name.
@@ -230,7 +265,15 @@ class SIMPLNX_EXPORT Preferences
   void setForceOocData(bool forceOoc);
 
   /**
-   * @brief Updates memory-related default values based on system capabilities.
+   * @brief Recomputes the default value for k_LargeDataStructureSize_Key based
+   *        on the current system's total physical RAM.
+   *
+   * The target value is (totalRAM - 2 * k_LargeDataSize), which reserves
+   * headroom for the OS and the application itself. If the reservation would
+   * exceed total RAM (e.g., on a low-memory system), the fallback is totalRAM / 2.
+   *
+   * Called automatically during construction, after loadFromFile(), and after
+   * clear(). Can also be called explicitly after changing k_LargeDataSize_Key.
    */
   void updateMemoryDefaults();
 
@@ -253,32 +296,54 @@ class SIMPLNX_EXPORT Preferences
   void setOocTempDirectory(const std::string& path);
 
   /**
-   * @brief Gets the timeout (in seconds) for the background OOC range scan.
+   * @brief Gets the timeout for the background OOC range scan.
+   *
+   * The range scan runs on a background thread after an OOC DataArray is loaded,
+   * computing min/max values by reading through all chunks sequentially. If the
+   * scan does not complete within this timeout, the range is reported as unknown
+   * and the UI shows "N/A" for the array's value range.
+   *
    * @return Timeout in seconds (default 30)
    */
   uint32 oocRangeScanTimeoutSeconds() const;
 
   /**
-   * @brief Sets the timeout (in seconds) for the background OOC range scan.
-   * @param seconds Timeout value in seconds
+   * @brief Sets the timeout for the background OOC range scan.
+   * @param seconds Timeout value in seconds. A value of 0 effectively disables
+   *        the range scan by expiring it immediately.
    */
   void setOocRangeScanTimeoutSeconds(uint32 seconds);
 
   /**
-   * @brief Gets the total memory budget for all OOC caching (chunk cache, stride cache, partition cache).
+   * @brief Gets the total memory budget for all OOC caching subsystems.
+   *
+   * The OOC memory budget manager distributes this budget across the chunk
+   * cache, stride cache, and partition cache using global LRU eviction. When
+   * the combined memory usage of all caches exceeds this budget, the least
+   * recently used entries are evicted to make room for new data.
    *
-   * The budget manager distributes this across subsystems via global LRU eviction.
-   * The default (8 GB) is a safe fallback; the SimplnxOoc plugin overrides this on
-   * startup with OocMemoryBudgetManager::defaultBudgetBytes() (50% of system RAM)
-   * if the user has not saved an explicit preference.
+   * The default value (8 GB) is a conservative fallback used when no plugin
+   * has configured a more appropriate value. On startup, the SimplnxOoc plugin
+   * calls OocMemoryBudgetManager::defaultBudgetBytes() (50% of system RAM) and
+   * sets that as the budget, unless the user has already saved an explicit
+   * preference via the UI.
    *
-   * @return Budget in bytes (default 8 GB)
+   * @note This reads from m_Values (user-set) directly, NOT from m_DefaultValues,
+   *       because the default is hard-coded as a compile-time constant.
+   *
+   * @return Budget in bytes (default 8 GB if not explicitly set)
    */
   uint64 oocMemoryBudgetBytes() const;
 
   /**
-   * @brief Sets the total memory budget for all OOC caching.
-   * @param bytes Budget in bytes
+   * @brief Sets the total memory budget for all OOC caching subsystems.
+   *
+   * The new budget takes effect immediately for subsequent cache eviction
+   * decisions. Existing cached data that exceeds the new budget will be
+   * evicted lazily as new cache entries are requested.
+   *
+   * @param bytes Budget in bytes. Must be > 0; passing 0 would effectively
+   *        disable caching.
    */
   void setOocMemoryBudgetBytes(uint64 bytes);
 
@@ -297,7 +362,13 @@ class SIMPLNX_EXPORT Preferences
   void addDefaultValues(std::string pluginName, std::string valueName, const nlohmann::json& value);
 
   /**
-   * @brief Checks and updates whether out-of-core mode should be used based on current settings.
+   * @brief Recomputes the cached m_UseOoc flag based on the current value of
+   *        k_PreferredLargeDataFormat_Key.
+   *
+   * OOC mode is considered active when the resolved format string is non-empty
+   * and is not the sentinel value k_InMemoryFormat. This method is called after
+   * any operation that could change the format: construction, loadFromFile(),
+   * setLargeDataFormat(), and setDefaultLargeDataFormat().
    */
   void checkUseOoc();
 

src/simplnx/DataStructure/AbstractDataStore.hpp

@@ -412,22 +412,59 @@ class AbstractDataStore : public IDataStore
 
   /**
    * @brief Copies a contiguous range of values from this data store into the
-   * provided buffer. The buffer must be large enough to hold the requested
-   * range. Each store subclass implements its own optimal version: in-memory
-   * stores use std::copy; out-of-core stores use bulk chunk I/O.
+   * provided caller-owned buffer.
+   *
+   * This is the primary bulk-read API for algorithms that need to process data
+   * in contiguous blocks. It replaces the earlier chunk-based API and provides
+   * a single uniform interface that works identically for both in-memory and
+   * out-of-core (OOC) data stores:
+   *
+   * - **In-memory (DataStore):** Performs a direct std::copy from the backing
+   *   array into the buffer. This is essentially zero-overhead.
+   * - **Out-of-core (OOC stores):** The OOC subclass translates the flat
+   *   element range into the appropriate chunk reads from the backing HDF5
+   *   file, coalescing I/O where possible. The caller does not need to know
+   *   the chunk layout.
+   * - **Empty (EmptyDataStore):** Throws std::runtime_error because no data
+   *   exists.
+   *
+   * The number of elements to copy is determined by `buffer.size()`. The caller
+   * is responsible for ensuring the buffer is large enough and that the range
+   * `[startIndex, startIndex + buffer.size())` does not exceed `getSize()`.
    *
    * @param startIndex The starting flat element index to read from
-   * @param buffer A span to receive the copied values (buffer.size() determines count)
+   * @param buffer A span to receive the copied values; its size determines how
+   *               many elements are read
+   * @throw std::out_of_range If the requested range exceeds the store's size
+   * @throw std::runtime_error If called on an EmptyDataStore
    */
   virtual void copyIntoBuffer(usize startIndex, nonstd::span<T> buffer) const = 0;
 
   /**
-   * @brief Copies values from the provided buffer into a contiguous range of
-   * this data store. Each store subclass implements its own optimal version:
-   * in-memory stores use std::copy; out-of-core stores use bulk chunk I/O.
+   * @brief Copies values from the provided caller-owned buffer into a
+   * contiguous range of this data store.
+   *
+   * This is the primary bulk-write API, the write-side counterpart of
+   * copyIntoBuffer(). It provides a single uniform interface for both
+   * in-memory and out-of-core (OOC) data stores:
+   *
+   * - **In-memory (DataStore):** Performs a direct std::copy from the buffer
+   *   into the backing array.
+   * - **Out-of-core (OOC stores):** The OOC subclass translates the flat
+   *   element range into the appropriate chunk writes to the backing HDF5
+   *   file.
+   * - **Empty (EmptyDataStore):** Throws std::runtime_error because no data
+   *   exists.
+   *
+   * The number of elements to copy is determined by `buffer.size()`. The caller
+   * is responsible for ensuring the range `[startIndex, startIndex + buffer.size())`
+   * does not exceed `getSize()`.
    *
    * @param startIndex The starting flat element index to write to
-   * @param buffer A span containing the values to copy into the store
+   * @param buffer A span containing the values to copy into the store; its
+   *               size determines how many elements are written
+   * @throw std::out_of_range If the requested range exceeds the store's size
+   * @throw std::runtime_error If called on an EmptyDataStore
    */
   virtual void copyFromBuffer(usize startIndex, nonstd::span<const T> buffer) = 0;