Drop samples with duplicate timestamps, presumably due to truncation

Signed-off-by: Brian Brazil <brian.brazil@robustperception.io>

Author: brian-brazil

prometheus_client/openmetrics/parser.py

@@ -317,7 +317,6 @@ def build_metric(name, documentation, typ, unit, samples):
             _check_histogram(samples, name)
         metric = core.Metric(name, documentation, typ, unit)
         # TODO: check labelvalues are valid utf8
-        # TODO: Check for duplicate samples
         metric.samples = samples
         return metric
 
@@ -347,6 +346,7 @@ def build_metric(name, documentation, typ, unit, samples):
                 group = None
                 seen_groups = set()
                 group_timestamp = None
+                group_timestamp_samples = set()
                 samples = []
                 allowed_names = [parts[2]]
 
@@ -387,13 +387,12 @@ def build_metric(name, documentation, typ, unit, samples):
                 documentation = None
                 unit = None
                 typ = 'unknown'
-                samples = [sample]
+                samples = []
                 group = None
                 group_timestamp = None
+                group_timestamp_samples = set()
                 seen_groups = set()
                 allowed_names = [sample.name]
-            else:
-                samples.append(sample)
 
             if typ == 'stateset' and name not in sample.labels:
                 raise ValueError("Stateset missing label: " + line)
@@ -412,6 +411,15 @@ def build_metric(name, documentation, typ, unit, samples):
                     raise ValueError("Mix of timestamp presence within a group: " + line)
                 if group_timestamp is not None and group_timestamp > sample.timestamp and typ != 'info':
                     raise ValueError("Timestamps went backwards within a group: " + line)
+            else:
+                group_timestamp_samples = set()
+
+            series_id = (sample.name, tuple(sorted(sample.labels.items())))
+            if sample.timestamp != group_timestamp or series_id not in group_timestamp_samples:
+                # Not a duplicate due to timestamp truncation.
+                samples.append(sample)
+            group_timestamp_samples.add(series_id)
+
             group = g
             group_timestamp = sample.timestamp
             seen_groups.add(g)

tests/openmetrics/test_parser.py

@@ -185,6 +185,22 @@ def test_simple_stateset(self):
 """)
         self.assertEqual([StateSetMetricFamily("a", "help", {'foo': True, 'bar': False})], list(families))
 
+    def test_duplicate_timestamps(self):
+        families = text_string_to_metric_families("""# TYPE a gauge
+# HELP a help
+a{a="1",foo="bar"} 1 0.0000000000
+a{a="1",foo="bar"} 2 0.0000000001
+a{a="1",foo="bar"} 3 0.0000000010
+a{a="2",foo="bar"} 4 0.0000000000
+a{a="2",foo="bar"} 5 0.0000000001
+# EOF
+""")
+        imf = GaugeMetricFamily("a", "help")
+        imf.add_sample("a", {"a": "1", "foo": "bar"}, 1, Timestamp(0, 0))
+        imf.add_sample("a", {"a": "1", "foo": "bar"}, 3, Timestamp(0, 1))
+        imf.add_sample("a", {"a": "2", "foo": "bar"}, 4, Timestamp(0, 0))
+        self.assertEqual([imf], list(families))
+
     def test_no_metadata(self):
         families = text_string_to_metric_families("""a 1
 # EOF
@@ -533,7 +549,6 @@ def test_invalid_input(self):
                 ('# TYPE a gaugehistogram\na_gsum 1\n# EOF\n'),
                 ('# TYPE a histogram\na_count 1\na_bucket{le="+Inf"} 0\n# EOF\n'),
                 ('# TYPE a histogram\na_bucket{le="+Inf"} 0\na_count 1\n# EOF\n'),
-                ('# TYPE a histogram\na_bucket{le="+Inf"} 0\na_bucket{le="+Inf"} 0\n# EOF\n'),
                 ('# TYPE a histogram\na_bucket{le="2"} 0\na_bucket{le="1"} 0\na_bucket{le="+Inf"} 0\n# EOF\n'),
                 ('# TYPE a histogram\na_bucket{le="1"} 1\na_bucket{le="2"} 1\na_bucket{le="+Inf"} 0\n# EOF\n'),
                 # Bad grouping or ordering.