Merge pull request #29 from apache/expand_docs

Expand docs

Author: jmalkin

datasketches/KernelFunction.py

@@ -24,8 +24,12 @@
 # Each implementation must extend the KernelFunction class
 # and define the __call__ method
 
-# Implements a basic Gaussian Kernel
 class GaussianKernel(KernelFunction):
+  '''Implements a basic Gaussian kernel
+
+     :param bandwidth: The kernel bandwidth, default 1.0
+     :type bandwidth: float
+    '''
   def __init__(self, bandwidth: float=1.0):
     KernelFunction.__init__(self)
     self._bw = bandwidth

datasketches/PySerDe.py

@@ -32,12 +32,12 @@
 #     returns a tuple with the newly reconstructed object and the
 #     total number of bytes beyond the offset read from the input data.
 
-# Implements a simple string-encoding scheme where a string is
-# written as <num_bytes> <string_contents>, with no null termination.
-# This format allows pre-allocating each string, at the cost of
-# additional storage. Using this format, the serialized string consumes
-# 4 + len(item) bytes.
 class PyStringsSerDe(PyObjectSerDe):
+  '''Implements a simple string-encoding scheme where a string is
+     written as `<num_bytes> <string_contents>`, with no null termination.
+     This format allows pre-allocating each string, at the cost of
+     additional storage. Using this format, the serialized string consumes
+     ``4 + len(item)`` bytes.'''
   def get_size(self, item):
     return int(4 + len(item))
 
@@ -54,9 +54,9 @@ def from_bytes(self, data: bytes, offset: int):
     str = data[offset+4:offset+4+num_chars].decode()
     return (str, 4+num_chars)
 
-# Implements an integer encoding scheme where each integer is written
-# as a 32-bit (4 byte) little-endian value.
 class PyIntsSerDe(PyObjectSerDe):
+  '''Implements an integer encoding scheme where each integer is written
+     as a 32-bit (4 byte) little-endian value.'''
   def get_size(self, item):
     return int(4)
 
@@ -68,9 +68,9 @@ def from_bytes(self, data: bytes, offset: int):
     return (val, 4)
 
 
-# Implements an integer encoding scheme where each integer is written
-# as a 64-bit (8 byte) little-endian value.
 class PyLongsSerDe(PyObjectSerDe):
+  '''Implements an integer encoding scheme where each integer is written
+     as a 64-bit (8 byte) little-endian value.'''
   def get_size(self, item):
     return int(8)
 
@@ -82,9 +82,9 @@ def from_bytes(self, data: bytes, offset: int):
     return (val, 8)
 
 
-# Implements a floating point encoding scheme where each value is written
-# as a 32-bit floating point value.
 class PyFloatsSerDe(PyObjectSerDe):
+  '''Implements a floating point encoding scheme where each value is written
+     as a 32-bit floating point value.'''
   def get_size(self, item):
     return int(4)
 
@@ -96,9 +96,9 @@ def from_bytes(self, data: bytes, offset: int):
     return (val, 4)
 
 
-# Implements a floating point encoding scheme where each value is written
-# as a 64-bit floating point value.
 class PyDoublesSerDe(PyObjectSerDe):
+  '''Implements a floating point encoding scheme where each value is written
+     as a 64-bit floating point value.'''
   def get_size(self, item):
     return int(8)
 

datasketches/TuplePolicy.py

@@ -22,16 +22,17 @@
 # This file provides an example Python Tuple Policy implementation.
 #
 # Each implementation must extend the PyTuplePolicy class and define
-# two methods:
+# the following methods:
 #   * create_summary() returns a new Summary object
 #   * update_summary(summary, update) applies the relevant policy to update the
 #     provided summary with the data in update.
 #   * __call__ may be similar to update_summary but allows a different
 #     implementation for set operations (union and intersection)
 
-# Implements an accumulator summary policy, where new values are
-# added to the existing value.
 class AccumulatorPolicy(TuplePolicy):
+  '''Implements an accumulatory summary policy, where new values
+  are added to the existing value.'''
+
   def __init__(self):
     TuplePolicy.__init__(self)
 
@@ -47,8 +48,9 @@ def __call__(self, summary: int, update: int) -> int:
     return summary
 
 
-# Implements a MAX rule, where the largest integer value is always kept
 class MaxIntPolicy(TuplePolicy):
+  '''Implements a MAX rule, where the largest integer value is always kept.'''
+
   def __init__(self):
     TuplePolicy.__init__(self)
 
@@ -62,8 +64,9 @@ def __call__(self, summary: int, update: int) -> int:
     return max(summary, update)
 
 
-# Implements a MIN rule, where the smallest integer value is always kept
 class MinIntPolicy(TuplePolicy):
+  '''Implements a MIN rule, where the smallest integer value is always kept.'''
+
   def __init__(self):
     TuplePolicy.__init__(self)
 

docs/source/count_min_sketch.rst

@@ -17,7 +17,6 @@ heavy hitters.
     :members:
     :undoc-members:
     :exclude-members: deserialize, suggest_num_buckets, suggest_num_hashes
-    :member-order: groupwise
 
     .. rubric:: Static Methods:
 

docs/source/cpc.rst

@@ -13,12 +13,20 @@ For additional security this sketch can be configured with a user-specified hash
 .. autoclass:: _datasketches.cpc_sketch
     :members:
     :undoc-members:
-    :exclude-members: deserialize,
-    :member-order: groupwise
+    :exclude-members: deserialize
 
     .. rubric:: Static Methods:
 
     .. automethod:: deserialize
 
     .. rubric:: Non-static Methods:
 
+    .. automethod:: __init__
+
+
+.. autoclass:: _datasketches.cpc_union
+    :members:
+    :undoc-members:
+    :exclude-members: deserialize
+
+    .. automethod:: __init__

docs/source/density_sketch.rst

@@ -1,5 +1,8 @@
 Density Sketch
 --------------
+
+.. currentmodule:: datasketches
+
 Builds a coreset from the given set of input points. 
 Provides density estimate at a given point.
 
@@ -9,9 +12,17 @@ https://proceedings.mlr.press/v99/karnin19a/karnin19a.pdf
 
 Inspired by the following implementation: https://github.com/edoliberty/streaming-quantiles/blob/f688c8161a25582457b0a09deb4630a81406293b/gde.py
 
-.. autoclass:: datasketches.density_sketch
+Requires the use of a :class:`KernelFunction` to compute the distance between two vectors.
+
+.. autoclass:: density_sketch
     :members:
     :undoc-members:
-    
-.. autoclass:: datasketches.GaussianKernel
-    :members:
+    :exclude-members: deserialize
+
+    .. rubric:: Static Methods:
+
+    .. automethod:: deserialize
+
+    .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__

docs/source/ebpps.rst

@@ -0,0 +1,33 @@
+Exact and Bounded, Probabilitiy Proportional to Size (EBPPS) Sampling
+---------------------------------------------------------------------
+
+.. currentmodule:: datasketches
+
+An EBPPS sketch produces a randome sample of data from a stream of items, ensuring that the probability
+of including an item is always exactly equal to the item's size. The size of an item is defined as its
+weight relative to the total weight of all items seen so far by the sketch. In contrast to VarOpt sampling,
+this sketch may return fewer than `k` items in order to keep the probability of including an item strictly
+proportional to its size.
+
+This sketch is based on: B. Hentschel, P. J. Haas, Y. Tian
+"Exact PPS Sampling with Bounded Sample Size",
+Information Processing Letters, 2023.
+
+EBPPS sampling is related to reservoir sampling, but handles unequal item weights.
+Feeding the sketch items with a uniform weight value will produce a sample equivalent to reservoir sampling.
+
+.. note::
+    Serializing and deserializing this sketch requires the use of a :class:`PyObjectSerDe`.
+
+.. autoclass:: ebpps_sketch
+    :members:
+    :undoc-members:
+    :exclude-members: deserialize
+
+    .. rubric:: Static Methods:
+
+    .. automethod:: deserialize
+
+    .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__

docs/source/frequent_items.rst

@@ -1,8 +1,10 @@
 Frequent Items
 --------------
 
-This sketch is useful for tracking approximate frequencies of items of type `<T>` with optional associated counts `(<T> item, int count)` 
-that are members of a multiset of such items. 
+.. currentmodule:: datasketches
+
+This sketch is useful for tracking approximate frequencies of items (``object`` or ``string``) with optional associated
+integer counts that are members of a multiset of such items. 
 The true frequency of an item is defined to be the sum of associated counts.
 
 This implementation provides the following capabilities:
@@ -16,38 +18,38 @@ This implementation provides the following capabilities:
 
 **Space Usage**
 
-The sketch is initialized with a maximum map size, `maxMapSize`, that specifies the maximum physical length of the internal hash map of the form `(<T> item, int count)`. 
-The maximum map size is always a power of 2, defined through the variables `lg_max_map_size`.
+The sketch is initialized with a maximum map size, ``maxMapSize``, that specifies the maximum physical length of the internal hash map of the form ``(object item, int count)``. 
+The maximum map size is always a power of 2, defined through the variables ``lg_max_map_size``.
 
 The hash map starts at a very small size (8 entries) and grows as needed up to the specified maximum map size.
 
-Excluding external space required for the item objects, the internal memory space usage of this sketch is `18 * mapSize bytes` (assuming 8 bytes for each reference), 
+Excluding external space required for the item objects, the internal memory space usage of this sketch is ``18 * mapSize`` bytes (assuming 8 bytes for each reference), 
 plus a small constant number of additional bytes. 
-The internal memory space usage of this sketch will never exceed `18 * maxMapSize` bytes, plus a small constant number of additional bytes.
+The internal memory space usage of this sketch will never exceed ``18 * maxMapSize`` bytes, plus a small constant number of additional bytes.
 
 **Maximum Capacity of the Sketch**
 
-The `LOAD_FACTOR` for the hash map is internally set at :math:`75\%`, which means at any time the map capacity of `(item, count)` pairs is `mapCap = 0.75 * mapSize`. 
-The maximum capacity of `(item, count)`` pairs of the sketch is `maxMapCap = 0.75 * maxMapSize`.
+The ``LOAD_FACTOR`` for the hash map is internally set at :math:`75\%`, which means at any time the map capacity of ``(item, count)`` pairs is ``mapCap = 0.75 * mapSize``. 
+The maximum capacity of ``(item, count)`` pairs of the sketch is ``maxMapCap = 0.75 * maxMapSize``.
 
-**Updating the sketch with `(item, count)` pairs**
+**Updating the sketch with ``(item, count)`` pairs**
 
-If the item is found in the hash map, the mapped count field (the "counter") is incremented by the incoming count; otherwise, a new counter `"(item, count) pair"` is created. 
+If the item is found in the hash map, the mapped count field (the "counter") is incremented by the incoming count; otherwise, a new counter ``(item, count)`` pair is created. 
 If the number of tracked counters reaches the maximum capacity of the hash map, the sketch decrements all of the counters (by an approximately computed median) 
 and removes any non-positive counters.
 
 **Accuracy**
 
-If fewer than `0.75 * maxMapSize` different items are inserted into the sketch, the estimated frequencies returned by the sketch will be exact.
+If fewer than ``0.75 * maxMapSize`` different items are inserted into the sketch, the estimated frequencies returned by the sketch will be exact.
 
 The logic of the frequent items sketch is such that the stored counts and true counts are never too different. 
 More specifically, for any item, the sketch can return an estimate of the true frequency of item, along with upper and lower bounds on the frequency (that hold deterministically).
 
 For this implementation and for a specific active item, it is guaranteed that the true frequency will be between the Upper Bound (UB) and the Lower Bound (LB) computed for that item. 
-Specifically, `(UB- LB) ≤ W * epsilon`, where :math:`W` denotes the sum of all item counts, and :math:`epsilon = 3.5/M`, where :math:`epsilon = M` is the maxMapSize.
+Specifically, ``(UB- LB) ≤ W * epsilon``, where :math:`W` denotes the sum of all item counts, and :math:`epsilon = 3.5/M`, where :math:`epsilon = M` is the maxMapSize.
 
 This is a worst-case guarantee that applies to arbitrary inputs.
-For inputs typically seen in practice, `(UB-LB)` is usually much smaller.
+For inputs typically seen in practice, ``(UB-LB)`` is usually much smaller.
 
 **Background**
 
@@ -63,12 +65,45 @@ Variants of it were discovered and rediscovered and redesigned several times ove
 For speed, we do employ some randomization that introduces a small probability that our proof of the worst-case bound might not apply to a given run. 
 However, we have ensured that this probability is extremely small. 
 For example, if the stream causes one table purge (rebuild), our proof of the worst-case bound applies with a probability of at least `1 - 1E-14`. 
-If the stream causes `1E9` purges, our proof applies with a probability of at least `1 - 1E-5`.
+If the stream causes ``1E9`` purges, our proof applies with a probability of at least ``1 - 1E-5``.
+
+There are two flavors of Frequent Items Sketches, one with generic items (objects) and another specific to strings.
+The string version is a legacy name from before the library supported generic objects and is retained
+only for backwards compatibility.
+
+.. note::
+    The :class:`frequent_items_sketch` uses an input object's ``__hash__`` and ``__eq__`` methods.
+
+.. note::
+    Serializing and deserializing the :class:`frequent_items_sketch` requires the use of a :class:`PyObjectSerDe`.
+
+.. autoclass:: frequent_items_error_type
+
+    .. autoattribute:: NO_FALSE_POSITIVES
+        :annotation: : Returns only true positives but may miss some heavy hitters.
+
+    .. autoattribute:: NO_FALSE_NEGATIVES
+        :annotation: : Does not miss any heavy hitters but may return false positives.
+
+
+.. autoclass:: frequent_items_sketch
+    :members:
+    :undoc-members:
+    :exclude-members: deserialize, get_epsilon_for_lg_size, get_apriori_error
+    :member-order: groupwise
 
-Parameter: <T> The type of item to be tracked by this sketch
+    .. rubric:: Static Methods:
 
+    .. automethod:: deserialize
+    .. automethod:: get_epsilon_for_lg_size
+    .. automethod:: get_apriori_error
 
-.. autoclass:: _datasketches.frequent_items_sketch
+    .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__
+
+
+.. autoclass:: frequent_strings_sketch
     :members:
     :undoc-members:
     :exclude-members: deserialize, get_epsilon_for_lg_size, get_apriori_error
@@ -81,3 +116,5 @@ Parameter: <T> The type of item to be tracked by this sketch
     .. automethod:: get_apriori_error
 
     .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__

docs/source/hyper_log_log.rst

@@ -7,26 +7,49 @@ If the ONLY use case for sketching is counting uniques and merging, the HLL sket
 This implementation offers three different types of HLL sketch, each with different trade-offs with accuracy, space and performance. 
 These types are specified with the target_hll_type parameter.
 
-In terms of accuracy, all three types, for the same lg_config_k, have the same error distribution as a function of n, the number of unique values fed to the sketch.
-The configuration parameter `lg_config_k` is the log-base-2 of `K`, where `K` is the number of buckets or slots for the sketch.
+In terms of accuracy, all three types, for the same lg_config_k, have the same error distribution as a function of ``n``, the number of unique values fed to the sketch.
+The configuration parameter ``lg_config_k`` is the log-base-2 of ``k``, where ``k`` is the number of buckets or slots for the sketch.
 
-During warmup, when the sketch has only received a small number of unique items (up to about 10% of `K`), this implementation leverages a new class of estimator algorithms with significantly better accuracy.
+During warmup, when the sketch has only received a small number of unique items (up to about 10% of ``k``), this implementation leverages a new class of estimator algorithms with significantly better accuracy.
+
+
+.. autoclass:: _datasketches.tgt_hll_type
+
+    .. autoattribute:: HLL_4
+        :annotation: : 4 bits per entry
+
+    .. autoattribute:: HLL_6
+        :annotation: : 6 bits per entry
+
+    .. autoattribute:: HLL_8
+        :annotation: : 8 bits per entry
 
-This sketch also offers the capability of operating off-heap. 
-Given a WritableMemory object created by the user, the sketch will perform all of its updates and internal phase transitions in that object, which can actually reside either on-heap or off-heap based on how it is configured. 
-In large systems that must update and merge many millions of sketches, having the sketch operate off-heap avoids the serialization and deserialization costs of moving sketches to and from off-heap memory-mapped files, for example, and eliminates big garbage collection delays.
 
 .. autoclass:: _datasketches.hll_sketch
     :members:
     :undoc-members:
     :exclude-members: deserialize, get_max_updatable_serialization_bytes, get_rel_err 
 
-    :member-order: groupwise
-
     .. rubric:: Static Methods:
 
     .. automethod:: deserialize
     .. automethod:: get_max_updatable_serialization_bytes
     .. automethod:: get_rel_err
 
     .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__
+
+.. autoclass:: _datasketches.hll_union
+    :members:
+    :undoc-members:
+    :exclude-members: get_rel_err 
+
+    .. rubric:: Static Methods:
+
+    .. automethod:: get_rel_err
+
+    .. rubric:: Non-static Methods:
+
+    .. automethod:: __init__
+