Merge pull request #404 from apache/cleanup-before-5.0.0

Cleanup before 5.0.0

Author: AlexanderSaydakov

common/include/common_defs.hpp

@@ -35,20 +35,19 @@ static const uint64_t DEFAULT_SEED = 9001;
 
 enum resize_factor { X1 = 0, X2, X4, X8 };
 
-template<typename A> using AllocChar = typename std::allocator_traits<A>::template rebind_alloc<char>;
-template<typename A> using string = std::basic_string<char, std::char_traits<char>, AllocChar<A>>;
-
-// thread-safe random bit
-static thread_local std::independent_bits_engine<std::mt19937, 1, uint32_t>
-  random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count() 
-    + std::hash<std::thread::id>{}(std::this_thread::get_id())));
+template<typename A> using string = std::basic_string<char, std::char_traits<char>, typename std::allocator_traits<A>::template rebind_alloc<char>>;
 
 // common random declarations
 namespace random_utils {
   static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2
   static thread_local std::mt19937_64 rand(rd());
   static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0);
 
+  // thread-safe random bit
+  static thread_local std::independent_bits_engine<std::mt19937, 1, uint32_t>
+    random_bit(static_cast<uint32_t>(std::chrono::system_clock::now().time_since_epoch().count()
+      + std::hash<std::thread::id>{}(std::this_thread::get_id())));
+
   inline void override_seed(uint64_t s) {
     rand.seed(s);
   }

common/include/count_zeros.hpp

@@ -22,8 +22,6 @@
 
 #include <cstdint>
 
-#include <stdio.h>
-
 namespace datasketches {
 
 static const uint8_t byte_leading_zeros_table[256] = {

common/include/quantiles_sorted_view.hpp

@@ -214,8 +214,6 @@ class quantiles_sorted_view<T, C, A>::const_iterator: public quantiles_sorted_vi
   using Base = typename quantiles_sorted_view<T, C, A>::Container::const_iterator;
   using value_type = typename std::conditional<std::is_arithmetic<T>::value, typename Base::value_type, std::pair<const T&, const uint64_t>>::type;
 
-  const_iterator(const Base& it, const Base& begin): Base(it), begin(begin) {}
-
   template<typename TT = T, typename std::enable_if<std::is_arithmetic<TT>::value, int>::type = 0>
   const value_type operator*() const { return Base::operator*(); }
 
@@ -239,6 +237,9 @@ class quantiles_sorted_view<T, C, A>::const_iterator: public quantiles_sorted_vi
 
 private:
   Base begin;
+
+  friend class quantiles_sorted_view<T, C, A>;
+  const_iterator(const Base& it, const Base& begin): Base(it), begin(begin) {}
 };
 
 } /* namespace datasketches */

cpc/include/cpc_common.hpp

@@ -26,45 +26,39 @@
 
 namespace datasketches {
 
+/// CPC constants
 namespace cpc_constants {
-    const uint8_t MIN_LG_K = 4;
-    const uint8_t MAX_LG_K = 26;
-    const uint8_t DEFAULT_LG_K = 11;
+  /// min log2 of K
+  const uint8_t MIN_LG_K = 4;
+  /// max log2 of K
+  const uint8_t MAX_LG_K = 26;
+  /// default log2 of K
+  const uint8_t DEFAULT_LG_K = 11;
 }
 
-// TODO: Redundant and deprecated. Will be removed in next major version release.
-static const uint8_t CPC_MIN_LG_K = cpc_constants::MIN_LG_K;
-static const uint8_t CPC_MAX_LG_K = cpc_constants::MAX_LG_K;
-static const uint8_t CPC_DEFAULT_LG_K = cpc_constants::DEFAULT_LG_K;
-
-template<typename A> using AllocU8 = typename std::allocator_traits<A>::template rebind_alloc<uint8_t>;
-template<typename A> using AllocU16 = typename std::allocator_traits<A>::template rebind_alloc<uint16_t>;
-template<typename A> using AllocU32 = typename std::allocator_traits<A>::template rebind_alloc<uint32_t>;
-template<typename A> using AllocU64 = typename std::allocator_traits<A>::template rebind_alloc<uint64_t>;
-
-template<typename A> using vector_u8 = std::vector<uint8_t, AllocU8<A>>;
-template<typename A> using vector_u32 = std::vector<uint32_t, AllocU32<A>>;
-template<typename A> using vector_u64 = std::vector<uint64_t, AllocU64<A>>;
-
 // forward declaration
 template<typename A> class u32_table;
 
 template<typename A>
 struct compressed_state {
+  using vector_u32 = std::vector<uint32_t, typename std::allocator_traits<A>::template rebind_alloc<uint32_t>>;
+
   explicit compressed_state(const A& allocator): table_data(allocator), table_data_words(0), table_num_entries(0),
       window_data(allocator), window_data_words(0) {}
-  vector_u32<A> table_data;
+  vector_u32 table_data;
   uint32_t table_data_words;
   uint32_t table_num_entries; // can be different from the number of entries in the sketch in hybrid mode
-  vector_u32<A> window_data;
+  vector_u32 window_data;
   uint32_t window_data_words;
 };
 
 template<typename A>
 struct uncompressed_state {
+  using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+
   explicit uncompressed_state(const A& allocator): table(allocator), window(allocator) {}
   u32_table<A> table;
-  vector_u8<A> window;
+  vector_bytes window;
 };
 
 } /* namespace datasketches */

cpc/include/cpc_compressor.hpp

@@ -47,6 +47,9 @@ inline cpc_compressor<A>& get_compressor();
 template<typename A>
 class cpc_compressor {
 public:
+  using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+  using vector_u32 = std::vector<uint32_t, typename std::allocator_traits<A>::template rebind_alloc<uint32_t>>;
+
   void compress(const cpc_sketch_alloc<A>& source, compressed_state<A>& target) const;
   void uncompress(const compressed_state<A>& source, uncompressed_state<A>& target, uint8_t lg_k, uint32_t num_coupons) const;
 
@@ -126,17 +129,17 @@ class cpc_compressor {
   uint16_t* make_decoding_table(const uint16_t* encoding_table, unsigned num_byte_values);
   void validate_decoding_table(const uint16_t* decoding_table, const uint16_t* encoding_table) const;
 
-  void compress_surprising_values(const vector_u32<A>& pairs, uint8_t lg_k, compressed_state<A>& result) const;
+  void compress_surprising_values(const vector_u32& pairs, uint8_t lg_k, compressed_state<A>& result) const;
   void compress_sliding_window(const uint8_t* window, uint8_t lg_k, uint32_t num_coupons, compressed_state<A>& target) const;
 
-  vector_u32<A> uncompress_surprising_values(const uint32_t* data, uint32_t data_words, uint32_t num_pairs, uint8_t lg_k, const A& allocator) const;
-  void uncompress_sliding_window(const uint32_t* data, uint32_t data_words, vector_u8<A>& window, uint8_t lg_k, uint32_t num_coupons) const;
+  vector_u32 uncompress_surprising_values(const uint32_t* data, uint32_t data_words, uint32_t num_pairs, uint8_t lg_k, const A& allocator) const;
+  void uncompress_sliding_window(const uint32_t* data, uint32_t data_words, vector_bytes& window, uint8_t lg_k, uint32_t num_coupons) const;
 
   static size_t safe_length_for_compressed_pair_buf(uint32_t k, uint32_t num_pairs, uint8_t num_base_bits);
   static size_t safe_length_for_compressed_window_buf(uint32_t k);
   static uint8_t determine_pseudo_phase(uint8_t lg_k, uint32_t c);
 
-  static inline vector_u32<A> tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get, uint32_t empty_space, const A& allocator);
+  static inline vector_u32 tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get, uint32_t empty_space, const A& allocator);
   static inline uint8_t golomb_choose_number_of_base_bits(uint32_t k, uint64_t count);
 };
 

cpc/include/cpc_compressor_impl.hpp

@@ -183,7 +183,7 @@ void cpc_compressor<A>::uncompress(const compressed_state<A>& source, uncompress
 template<typename A>
 void cpc_compressor<A>::compress_sparse_flavor(const cpc_sketch_alloc<A>& source, compressed_state<A>& result) const {
   if (source.sliding_window.size() > 0) throw std::logic_error("unexpected sliding window");
-  vector_u32<A> pairs = source.surprising_value_table.unwrapping_get_items();
+  vector_u32 pairs = source.surprising_value_table.unwrapping_get_items();
   u32_table<A>::introspective_insertion_sort(pairs.data(), 0, pairs.size());
   compress_surprising_values(pairs, source.get_lg_k(), result);
 }
@@ -192,7 +192,7 @@ template<typename A>
 void cpc_compressor<A>::uncompress_sparse_flavor(const compressed_state<A>& source, uncompressed_state<A>& target, uint8_t lg_k) const {
   if (source.window_data.size() > 0) throw std::logic_error("unexpected sliding window");
   if (source.table_data.size() == 0) throw std::logic_error("table is expected");
-  vector_u32<A> pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries,
+  vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries,
       lg_k, source.table_data.get_allocator());
   target.table = u32_table<A>::make_from_pairs(pairs.data(), source.table_num_entries, lg_k, pairs.get_allocator());
 }
@@ -204,12 +204,12 @@ void cpc_compressor<A>::compress_hybrid_flavor(const cpc_sketch_alloc<A>& source
   if (source.sliding_window.size() == 0) throw std::logic_error("no sliding window");
   if (source.window_offset != 0) throw std::logic_error("window_offset != 0");
   const uint32_t k = 1 << source.get_lg_k();
-  vector_u32<A> pairs_from_table = source.surprising_value_table.unwrapping_get_items();
+  vector_u32 pairs_from_table = source.surprising_value_table.unwrapping_get_items();
   const uint32_t num_pairs_from_table = static_cast<uint32_t>(pairs_from_table.size());
   if (num_pairs_from_table > 0) u32_table<A>::introspective_insertion_sort(pairs_from_table.data(), 0, num_pairs_from_table);
   const uint32_t num_pairs_from_window = source.get_num_coupons() - num_pairs_from_table; // because the window offset is zero
 
-  vector_u32<A> all_pairs = tricky_get_pairs_from_window(source.sliding_window.data(), k, num_pairs_from_window, num_pairs_from_table, source.get_allocator());
+  vector_u32 all_pairs = tricky_get_pairs_from_window(source.sliding_window.data(), k, num_pairs_from_window, num_pairs_from_table, source.get_allocator());
 
   u32_table<A>::merge(
       pairs_from_table.data(), 0, pairs_from_table.size(),
@@ -224,7 +224,7 @@ template<typename A>
 void cpc_compressor<A>::uncompress_hybrid_flavor(const compressed_state<A>& source, uncompressed_state<A>& target, uint8_t lg_k) const {
   if (source.window_data.size() > 0) throw std::logic_error("window is not expected");
   if (source.table_data.size() == 0) throw std::logic_error("table is expected");
-  vector_u32<A> pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries,
+  vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, source.table_num_entries,
       lg_k, source.table_data.get_allocator());
 
   // In the hybrid flavor, some of these pairs actually
@@ -250,7 +250,7 @@ void cpc_compressor<A>::uncompress_hybrid_flavor(const compressed_state<A>& sour
 template<typename A>
 void cpc_compressor<A>::compress_pinned_flavor(const cpc_sketch_alloc<A>& source, compressed_state<A>& result) const {
   compress_sliding_window(source.sliding_window.data(), source.get_lg_k(), source.get_num_coupons(), result);
-  vector_u32<A> pairs = source.surprising_value_table.unwrapping_get_items();
+  vector_u32 pairs = source.surprising_value_table.unwrapping_get_items();
   if (pairs.size() > 0) {
     // Here we subtract 8 from the column indices. Because they are stored in the low 6 bits
     // of each row_col pair, and because no column index is less than 8 for a "Pinned" sketch,
@@ -277,7 +277,7 @@ void cpc_compressor<A>::uncompress_pinned_flavor(const compressed_state<A>& sour
     target.table = u32_table<A>(2, 6 + lg_k, source.table_data.get_allocator());
   } else {
     if (source.table_data.size() == 0) throw std::logic_error("table is expected");
-    vector_u32<A> pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs,
+    vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs,
         lg_k, source.table_data.get_allocator());
     // undo the compressor's 8-column shift
     for (uint32_t i = 0; i < num_pairs; i++) {
@@ -291,7 +291,7 @@ void cpc_compressor<A>::uncompress_pinned_flavor(const compressed_state<A>& sour
 template<typename A>
 void cpc_compressor<A>::compress_sliding_flavor(const cpc_sketch_alloc<A>& source, compressed_state<A>& result) const {
   compress_sliding_window(source.sliding_window.data(), source.get_lg_k(), source.get_num_coupons(), result);
-  vector_u32<A> pairs = source.surprising_value_table.unwrapping_get_items();
+  vector_u32 pairs = source.surprising_value_table.unwrapping_get_items();
   if (pairs.size() > 0) {
     // Here we apply a complicated transformation to the column indices, which
     // changes the implied ordering of the pairs, so we must do it before sorting.
@@ -330,7 +330,7 @@ void cpc_compressor<A>::uncompress_sliding_flavor(const compressed_state<A>& sou
     target.table = u32_table<A>(2, 6 + lg_k, source.table_data.get_allocator());
   } else {
     if (source.table_data.size() == 0) throw std::logic_error("table is expected");
-    vector_u32<A> pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs,
+    vector_u32 pairs = uncompress_surprising_values(source.table_data.data(), source.table_data_words, num_pairs,
         lg_k, source.table_data.get_allocator());
 
     const uint8_t pseudo_phase = determine_pseudo_phase(lg_k, num_coupons);
@@ -356,7 +356,7 @@ void cpc_compressor<A>::uncompress_sliding_flavor(const compressed_state<A>& sou
 }
 
 template<typename A>
-void cpc_compressor<A>::compress_surprising_values(const vector_u32<A>& pairs, uint8_t lg_k, compressed_state<A>& result) const {
+void cpc_compressor<A>::compress_surprising_values(const vector_u32& pairs, uint8_t lg_k, compressed_state<A>& result) const {
   const uint32_t k = 1 << lg_k;
   const uint32_t num_pairs = static_cast<uint32_t>(pairs.size());
   const uint8_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs);
@@ -374,10 +374,10 @@ void cpc_compressor<A>::compress_surprising_values(const vector_u32<A>& pairs, u
 }
 
 template<typename A>
-vector_u32<A> cpc_compressor<A>::uncompress_surprising_values(const uint32_t* data, uint32_t data_words, uint32_t num_pairs,
-    uint8_t lg_k, const A& allocator) const {
+auto cpc_compressor<A>::uncompress_surprising_values(const uint32_t* data, uint32_t data_words, uint32_t num_pairs,
+    uint8_t lg_k, const A& allocator) const -> vector_u32 {
   const uint32_t k = 1 << lg_k;
-  vector_u32<A> pairs(num_pairs, 0, allocator);
+  vector_u32 pairs(num_pairs, 0, allocator);
   const uint8_t num_base_bits = golomb_choose_number_of_base_bits(k + num_pairs, num_pairs);
   low_level_uncompress_pairs(pairs.data(), num_pairs, num_base_bits, data, data_words);
   return pairs;
@@ -399,7 +399,7 @@ void cpc_compressor<A>::compress_sliding_window(const uint8_t* window, uint8_t l
 }
 
 template<typename A>
-void cpc_compressor<A>::uncompress_sliding_window(const uint32_t* data, uint32_t data_words, vector_u8<A>& window,
+void cpc_compressor<A>::uncompress_sliding_window(const uint32_t* data, uint32_t data_words, vector_bytes& window,
     uint8_t lg_k, uint32_t num_coupons) const {
   const uint32_t k = 1 << lg_k;
   window.resize(k); // zeroing not needed here (unlike the Hybrid Flavor)
@@ -722,10 +722,10 @@ void write_unary(
 // The empty space that this leaves at the beginning of the output array
 // will be filled in later by the caller.
 template<typename A>
-vector_u32<A> cpc_compressor<A>::tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get,
-    uint32_t empty_space, const A& allocator) {
+auto cpc_compressor<A>::tricky_get_pairs_from_window(const uint8_t* window, uint32_t k, uint32_t num_pairs_to_get,
+    uint32_t empty_space, const A& allocator) -> vector_u32 {
   const size_t output_length = empty_space + num_pairs_to_get;
-  vector_u32<A> pairs(output_length, 0, allocator);
+  vector_u32 pairs(output_length, 0, allocator);
   size_t pair_index = empty_space;
   for (unsigned row_index = 0; row_index < k; row_index++) {
     uint8_t byte = window[row_index];

cpc/include/cpc_sketch.hpp

@@ -33,7 +33,7 @@
 
 namespace datasketches {
 
-// forward-declarations
+// forward declarations
 template<typename A> class cpc_sketch_alloc;
 template<typename A> class cpc_union_alloc;
 
@@ -64,6 +64,8 @@ template<typename A>
 class cpc_sketch_alloc {
 public:
   using allocator_type = A;
+  using vector_bytes = std::vector<uint8_t, typename std::allocator_traits<A>::template rebind_alloc<uint8_t>>;
+  using vector_u64 = std::vector<uint64_t, typename std::allocator_traits<A>::template rebind_alloc<uint64_t>>;
 
   /**
    * Creates an instance of the sketch given the lg_k parameter and hash seed.
@@ -204,10 +206,6 @@ class cpc_sketch_alloc {
    */
   void serialize(std::ostream& os) const;
 
-  // This is a convenience alias for users
-  // The type returned by the following serialize method
-  using vector_bytes = vector_u8<A>;
-
   /**
    * This method serializes the sketch as a vector of bytes.
    * An optional header can be reserved in front of the sketch.
@@ -278,7 +276,7 @@ class cpc_sketch_alloc {
   uint32_t num_coupons; // the number of coupons collected so far
 
   u32_table<A> surprising_value_table;
-  vector_u8<A> sliding_window;
+  vector_bytes sliding_window;
   uint8_t window_offset; // derivable from num_coupons, but made explicit for speed
   uint8_t first_interesting_column; // This is part of a speed optimization
 
@@ -287,7 +285,7 @@ class cpc_sketch_alloc {
 
   // for deserialization and cpc_union::get_result()
   cpc_sketch_alloc(uint8_t lg_k, uint32_t num_coupons, uint8_t first_interesting_column, u32_table<A>&& table,
-      vector_u8<A>&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed);
+      vector_bytes&& window, bool has_hip, double kxp, double hip_est_accum, uint64_t seed);
 
   inline void row_col_update(uint32_t row_col);
   inline void update_sparse(uint32_t row_col);
@@ -310,7 +308,7 @@ class cpc_sketch_alloc {
   static inline uint8_t determine_correct_offset(uint8_t lg_k, uint64_t c);
 
   // this produces a full-size k-by-64 bit matrix
-  vector_u64<A> build_bit_matrix() const;
+  vector_u64 build_bit_matrix() const;
 
   static uint8_t get_preamble_ints(uint32_t num_coupons, bool has_hip, bool has_table, bool has_window);
   inline void write_hip(std::ostream& os) const;