core+proto: Generalize histogram utilities

Histogram functionality currently used in the Measure module will soon
be used in other modules. This commit moves some helper functions and
protobufs so they can be shared.

Author: dtnaylor

core/modules/measure.cc

@@ -142,22 +142,6 @@ void Measure::ProcessBatch(Context *ctx, bess::PacketBatch *batch) {
   RunNextModule(ctx, batch);
 }
 
-template <typename T>
-static void SetHistogram(
-    bess::pb::MeasureCommandGetSummaryResponse::Histogram *r, const T &hist,
-    uint64_t bucket_width) {
-  r->set_count(hist.count);
-  r->set_above_range(hist.above_range);
-  r->set_resolution_ns(bucket_width);
-  r->set_min_ns(hist.min);
-  r->set_max_ns(hist.max);
-  r->set_avg_ns(hist.avg);
-  r->set_total_ns(hist.total);
-  for (const auto &val : hist.percentile_values) {
-    r->add_percentile_values_ns(val);
-  }
-}
-
 void Measure::Clear() {
   // vector initialization is expensive thus should be out of critical section
   decltype(rtt_hist_) new_rtt_hist(rtt_hist_.num_buckets(),
@@ -175,16 +159,6 @@ void Measure::Clear() {
   mcs_unlock(&lock_, &mynode);
 }
 
-static bool IsValidPercentiles(const std::vector<double> &percentiles) {
-  if (percentiles.empty()) {
-    return true;
-  }
-
-  return std::is_sorted(percentiles.cbegin(), percentiles.cend()) &&
-         *std::min_element(percentiles.cbegin(), percentiles.cend()) >= 0.0 &&
-         *std::max_element(percentiles.cbegin(), percentiles.cend()) <= 100.0;
-}
-
 CommandResponse Measure::CommandGetSummary(
     const bess::pb::MeasureCommandGetSummaryArg &arg) {
   bess::pb::MeasureCommandGetSummaryResponse r;
@@ -211,8 +185,8 @@ CommandResponse Measure::CommandGetSummary(
   const auto &rtt = rtt_hist_.Summarize(latency_percentiles);
   const auto &jitter = jitter_hist_.Summarize(jitter_percentiles);
 
-  SetHistogram(r.mutable_latency(), rtt, rtt_hist_.bucket_width());
-  SetHistogram(r.mutable_jitter(), jitter, jitter_hist_.bucket_width());
+  SetHistogram(r.mutable_latency_ns(), rtt);
+  SetHistogram(r.mutable_jitter_ns(), jitter);
 
   if (arg.clear()) {
     // Note that some samples might be lost due to the small gap between

core/utils/histogram.cc

@@ -0,0 +1,40 @@
+// Copyright (c) 2016-2019, Nefeli Networks, Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice, this
+// list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+//
+// * Neither the names of the copyright holders nor the names of their
+// contributors may be used to endorse or promote products derived from this
+// software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+
+#include "histogram.h"
+
+bool IsValidPercentiles(const std::vector<double> &percentiles) {
+  if (percentiles.empty()) {
+    return true;
+  }
+
+  return std::is_sorted(percentiles.cbegin(), percentiles.cend()) &&
+         *std::min_element(percentiles.cbegin(), percentiles.cend()) >= 0.0 &&
+         *std::max_element(percentiles.cbegin(), percentiles.cend()) <= 100.0;
+}

core/utils/histogram.h

@@ -1,5 +1,5 @@
 // Copyright (c) 2014-2016, The Regents of the University of California.
-// Copyright (c) 2016-2017, Nefeli Networks, Inc.
+// Copyright (c) 2016-2019, Nefeli Networks, Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -42,6 +42,8 @@
 
 #include <glog/logging.h>
 
+#include "../pb/util_msg.pb.h"
+
 // Class for general purpose histogram. T generally should be an
 // integral type, though floating point types will also work.
 // A bin b_i corresponds for the range [i * width, (i + 1) * width)
@@ -52,7 +54,9 @@ class Histogram {
  public:
   static_assert(std::is_arithmetic<T>::value, "Arithmetic type required.");
   struct Summary {
-    size_t count;        // # of all samples. If 0, min, max and avg are also 0
+    size_t num_buckets;  // Number of buckets in the histogram
+    size_t bucket_width; // Resolution of the measured data
+    size_t count;        // # of samples (including above_range). If 0, min, max and avg are also 0
     size_t above_range;  // # of samples beyond the histogram range
     T min;               // Min value
     T max;               // Max value. May be underestimated if above_range > 0
@@ -124,6 +128,8 @@ class Histogram {
   // percentile_values
   const Summary Summarize(const std::vector<double> &percentiles = {}) const {
     Summary ret = {};
+    ret.num_buckets = num_buckets();
+    ret.bucket_width = bucket_width_;
     uint64_t count = std::accumulate(buckets_.begin(), buckets_.end(), 0);
     ret.count = count;
     ret.above_range = buckets_.back();
@@ -201,4 +207,21 @@ class Histogram {
   std::vector<std::atomic<uint64_t>> buckets_;
 };
 
+bool IsValidPercentiles(const std::vector<double> &percentiles);
+
+template <typename T>
+void SetHistogram(bess::pb::HistogramSummary *r, const T &summary) {
+  r->set_num_buckets(summary.num_buckets);
+  r->set_bucket_width(summary.bucket_width);
+  r->set_count(summary.count);
+  r->set_above_range(summary.above_range);
+  r->set_min(summary.min);
+  r->set_max(summary.max);
+  r->set_avg(summary.avg);
+  r->set_total(summary.total);
+  for (const auto &val : summary.percentile_values) {
+    r->add_percentile_values(val);
+  }
+}
+
 #endif  // BESS_UTILS_HISTOGRAM_H_

protobuf/module_msg.proto

@@ -243,22 +243,11 @@ message MeasureCommandGetSummaryArg {
  * may not be a multiple of the resolution.
  */
 message MeasureCommandGetSummaryResponse {
-  message Histogram {
-    uint64 count = 1; /// Total # of measured data points, including above_range
-    uint64 above_range = 2; /// # of data points for the "too large value" bucket
-    uint64 resolution_ns = 8; /// resolution of measured data
-    uint64 min_ns = 3;
-    uint64 avg_ns = 4;
-    uint64 max_ns = 5;
-    uint64 total_ns = 6;
-    repeated uint64 percentile_values_ns = 7;
-  }
-
   double timestamp = 1; /// Seconds since boot.
   uint64 packets = 2; /// Total # of packets seen by this module.
   uint64 bits = 3; /// Total # of bits seen by this module.
-  Histogram latency = 4;
-  Histogram jitter = 5;
+  HistogramSummary latency_ns = 4;
+  HistogramSummary jitter_ns = 5;
 }
 
 

protobuf/util_msg.proto

@@ -49,3 +49,15 @@ message FieldData {
   }
 }
 
+/// The HistogramSummary message carries summary statistics about a histogram.
+message HistogramSummary {
+  uint64 num_buckets = 1;  /// Number of buckets in the histogram
+  uint64 bucket_width = 2; /// Resolution of the measured data
+  uint64 count = 3;        /// # of samples (including above_range). If 0, min, max and avg are also 0
+  uint64 above_range = 4;  /// # of samples beyond the histogram range
+  uint64 min = 5;          /// Min value
+  uint64 max = 6;          /// Max value. May be underestimated if above_range > 0
+  uint64 avg = 7;          /// Average of all samples (== total / count)
+  uint64 total = 8;        /// Total sum of all samples
+  repeated uint64 percentile_values = 9;
+}