From 0294058df41187dae873c7529d88e4afac3d9bd4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 15:14:07 -0400 Subject: [PATCH 01/70] Add Hashtable and LongHashingUtils to datadog.trace.util Two general-purpose utilities used by the client-side stats aggregator work (PR #11382 and follow-ups), extracted into their own change so the metrics-specific PRs can build on a smaller, reviewable foundation. - Hashtable: a generic open-addressed-ish bucket table abstraction keyed by a 64-bit hash, with a public abstract Entry type so client code can subclass it for higher-arity keys. The metrics aggregator uses it to back its AggregateTable. - LongHashingUtils: chained 64-bit hash combiners with primitive overloads (boolean, short, int, long, Object). Used in place of varargs combiners to avoid Object[] allocation and boxing on the hot path. No callers within internal-api itself yet -- the metrics aggregator PR will introduce the first usages. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 553 ++++++++++++++++++ .../datadog/trace/util/LongHashingUtils.java | 158 +++++ 2 files changed, 711 insertions(+) create mode 100644 internal-api/src/main/java/datadog/trace/util/Hashtable.java create mode 100644 internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java new file mode 100644 index 00000000000..d7f49dcae00 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -0,0 +1,553 @@ +package datadog.trace.util; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Light weight simple Hashtable system that can be useful when HashMap would + * be unnecessarily heavy. + * + * + * + * Convenience classes are provided for lower key dimensions. + * + * For higher key dimensions, client code must implement its own class, + * but can still use the support class to ease the implementation complexity. + */ +public abstract class Hashtable { + /** + * Internal base class for entries. Stores the precomputed 64-bit keyHash and + * the chain-next pointer used to link colliding entries within a single bucket. + * + *

Subclasses add the actual key field(s) and a {@code matches(...)} method + * tailored to their key arity. See {@link D1.Entry} and {@link D2.Entry}; for + * higher arities, client code can subclass this directly and use {@link Support} + * to drive the table mechanics. + */ + public static abstract class Entry { + public final long keyHash; + Entry next = null; + + protected Entry(long keyHash) { + this.keyHash = keyHash; + } + + public final void setNext(TEntry next) { + this.next = next; + } + + @SuppressWarnings("unchecked") + public final TEntry next() { + return (TEntry)this.next; + } + } + + /** + * Single-key open hash table with chaining. + * + *

The user supplies an {@link D1.Entry} subclass that carries the key and + * whatever value fields they want to mutate in place, then instantiates this + * class over that entry type. The main advantage over {@code HashMap} + * is that mutating an existing entry's value fields requires no allocation: + * call {@link #get} once and write directly to the returned entry's fields. + * For counter-style workloads this can be several times faster than + * {@code HashMap} and produces effectively zero GC pressure. + * + *

Capacity is fixed at construction. The table does not resize, so the + * caller is responsible for choosing a capacity appropriate to the working + * set. Actual bucket-array length is rounded up to the next power of two. + * + *

Null keys are permitted; they collapse to a single bucket via the + * sentinel hash {@link Long#MIN_VALUE} defined in {@link D1.Entry#hash}. + * + *

Not thread-safe. Concurrent access (including mixing reads with + * writes) requires external synchronization. + * + * @param the key type + * @param the user's {@link D1.Entry D1.Entry<K>} subclass + */ + public static final class D1> { + /** + * Abstract base for {@link D1} entries. Subclass to add value fields you + * wish to mutate in place after retrieving the entry via {@link D1#get}. + * + *

The key is captured at construction and stored alongside its + * precomputed 64-bit hash. {@link #matches(Object)} uses + * {@link Objects#equals} by default; override if a different equality + * semantics is needed (e.g. reference equality for interned keys). + * + * @param the key type + */ + public static abstract class Entry extends Hashtable.Entry { + final K key; + + protected Entry(K key) { + super(hash(key)); + this.key = key; + } + + public boolean matches(Object key) { + return Objects.equals(this.key, key); + } + + public static long hash(Object key) { + return (key == null ) ? Long.MIN_VALUE : key.hashCode(); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D1(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K key) { + long keyHash = D1.Entry.hash(key); + Hashtable.Entry[] thisBuckets = this.buckets; + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key)) return te; + } + } + return null; + } + + public TEntry remove(K key) { + long keyHash = D1.Entry.hash(key); + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + + this.size += 1; + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + } + + /** + * Two-key (composite-key) hash table with chaining. + * + *

The user supplies a {@link D2.Entry} subclass carrying both key parts + * and any value fields. Compared to {@code HashMap} this avoids the + * per-lookup {@code Pair} (or record) allocation: both key parts are passed + * directly through {@link #get}, {@link #remove}, {@link #insert}, and + * {@link #insertOrReplace}. Combined with in-place value mutation, this + * makes {@code D2} substantially less GC-intensive than the equivalent + * {@code HashMap} for counter-style workloads. + * + *

Capacity is fixed at construction; the table does not resize. Actual + * bucket-array length is rounded up to the next power of two. + * + *

Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; + * see {@link D2.Entry#hash(Object, Object)}. + * + *

Not thread-safe. + * + * @param first key type + * @param second key type + * @param the user's {@link D2.Entry D2.Entry<K1, K2>} subclass + */ + public static final class D2> { + /** + * Abstract base for {@link D2} entries. Subclass to add value fields you + * wish to mutate in place. + * + *

Both key parts are captured at construction and stored alongside their + * combined 64-bit hash. {@link #matches(Object, Object)} uses + * {@link Objects#equals} pairwise on the two parts. + * + * @param first key type + * @param second key type + */ + public static abstract class Entry extends Hashtable.Entry { + final K1 key1; + final K2 key2; + + protected Entry(K1 key1, K2 key2) { + super(hash(key1, key2)); + this.key1 = key1; + this.key2 = key2; + } + + public boolean matches(K1 key1, K2 key2) { + return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); + } + + public static long hash(Object key1, Object key2) { + return LongHashingUtils.hash(key1, key2); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D2(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + Hashtable.Entry[] thisBuckets = this.buckets; + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key1, key2)) return te; + } + } + return null; + } + + public TEntry remove(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key1, key2)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + + this.size += 1; + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key1, newEntry.key2)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + } + + /** + * Internal building blocks for hash-table operations. + * + *

Used by {@link D1} and {@link D2}, and available to package code that + * wants to assemble its own higher-arity table (3+ key parts) without + * re-implementing the bucket-array mechanics. The typical recipe: + * + *

+ * + *

All bucket arrays produced by {@link #create(int)} have a power-of-two + * length, so {@link #bucketIndex(Object[], long)} can use a bit mask. + * + *

Methods on this class are package-private; the class itself is public + * only so that its nested {@link BucketIterator} can be referenced by + * callers in other packages. + */ + public static final class Support { + public static final Hashtable.Entry[] create(int capacity) { + return new Entry[sizeFor(capacity)]; + } + + static final int sizeFor(int requestedCapacity) { + int pow; + for ( pow = 1; pow < requestedCapacity; pow *= 2 ); + return pow; + } + + public static final void clear(Hashtable.Entry[] buckets) { + Arrays.fill(buckets, null); + } + + public static final BucketIterator bucketIterator(Hashtable.Entry[] buckets, long keyHash) { + return new BucketIterator(buckets, keyHash); + } + + public static final MutatingBucketIterator mutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + return new MutatingBucketIterator(buckets, keyHash); + } + + public static final int bucketIndex(Object[] buckets, long keyHash) { + return (int)(keyHash & buckets.length - 1); + } + } + + /** + * Read-only iterator over entries in a single bucket whose {@code keyHash} + * matches a specific search hash. Cheaper than {@link MutatingBucketIterator} + * because it does not track the previous-node pointers required for + * splicing — use it when you only need to walk the chain. + * + *

For {@code remove} or {@code replace} operations, use + * {@link MutatingBucketIterator} instead. + */ + public static final class BucketIterator implements Iterator { + private final long keyHash; + private Hashtable.Entry nextEntry; + + BucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.keyHash = keyHash; + Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; + while (cur != null && cur.keyHash != keyHash) cur = cur.next; + this.nextEntry = cur; + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry cur = this.nextEntry; + if (cur == null) throw new NoSuchElementException("no next!"); + + Hashtable.Entry advance = cur.next; + while (advance != null && advance.keyHash != keyHash) advance = advance.next; + this.nextEntry = advance; + + return (TEntry) cur; + } + } + + /** + * Mutating iterator over entries in a single bucket whose {@code keyHash} + * matches a specific search hash. Supports {@link #remove()} and + * {@link #replace(Entry)} to splice the chain in place. + * + *

Carries previous-node pointers for the current entry and the next-match + * entry so that {@code remove} and {@code replace} can fix up the chain in + * O(1) without re-walking from the bucket head. After {@code remove} or + * {@code replace}, iteration may continue with another {@link #next()}. + */ + public static final class MutatingBucketIterator implements Iterator { + private final long keyHash; + + private final Hashtable.Entry[] buckets; + + /** + * The entry prior to the last entry returned by next + * Used for mutating operations + */ + private Hashtable.Entry curPrevEntry; + + /** + * The entry that was last returned by next + */ + private Hashtable.Entry curEntry; + + /** + * The entry prior to the next entry + */ + private Hashtable.Entry nextPrevEntry; + + /** + * The next entry to be returned by next + */ + private Hashtable.Entry nextEntry; + + MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.buckets = buckets; + this.keyHash = keyHash; + + int bucketIndex = Support.bucketIndex(buckets, keyHash); + Hashtable.Entry headEntry = this.buckets[bucketIndex]; + if ( headEntry == null ) { + this.nextEntry = null; + this.nextPrevEntry = null; + + this.curEntry = null; + this.curPrevEntry = null; + } else { + Hashtable.Entry prev, cur; + for ( prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next() ) { + if ( cur.keyHash == keyHash ) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + this.curEntry = null; + this.curPrevEntry = null; + } + } + + @Override + public boolean hasNext() { + return (this.nextEntry != null); + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry curEntry = this.nextEntry; + if ( curEntry == null ) throw new NoSuchElementException("no next!"); + + this.curEntry = curEntry; + this.curPrevEntry = this.nextPrevEntry; + + Hashtable.Entry prev, cur; + for ( prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next() ) { + if ( cur.keyHash == keyHash ) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + return (TEntry) curEntry; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if ( oldCurEntry == null ) throw new IllegalStateException(); + + this.setPrevNext(oldCurEntry.next()); + + // If the next match was directly after oldCurEntry, its predecessor is now + // curPrevEntry (oldCurEntry was just unlinked from the chain). + if ( this.nextPrevEntry == oldCurEntry ) { + this.nextPrevEntry = this.curPrevEntry; + } + this.curEntry = null; + } + + public void replace(TEntry replacementEntry) { + Hashtable.Entry oldCurEntry = this.curEntry; + if ( oldCurEntry == null ) throw new IllegalStateException(); + + replacementEntry.setNext(oldCurEntry.next()); + this.setPrevNext(replacementEntry); + + // If the next match was directly after oldCurEntry, its predecessor is now + // the replacement entry (which took oldCurEntry's chain slot). + if ( this.nextPrevEntry == oldCurEntry ) { + this.nextPrevEntry = replacementEntry; + } + this.curEntry = replacementEntry; + } + + void setPrevNext(Hashtable.Entry nextEntry) { + if ( this.curPrevEntry == null ) { + Hashtable.Entry[] buckets = this.buckets; + buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; + } else { + this.curPrevEntry.setNext(nextEntry); + } + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java new file mode 100644 index 00000000000..bc53bc4ecb6 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -0,0 +1,158 @@ +package datadog.trace.util; + +/** + * This class is intended to be a drop-in replacement for the hashing portions of java.util.Objects. + * This class provides more convenience methods for hashing primitives and includes overrides for + * hash that take many argument lengths to avoid var-args allocation. + */ +public final class LongHashingUtils { + private LongHashingUtils() {} + + public static final long hashCodeX(Object obj) { + return obj == null ? Long.MIN_VALUE : obj.hashCode(); + } + + public static final long hash(boolean value) { + return Boolean.hashCode(value); + } + + public static final long hash(char value) { + return Character.hashCode(value); + } + + public static final long hash(byte value) { + return Byte.hashCode(value); + } + + public static final long hash(short value) { + return Short.hashCode(value); + } + + public static final long hash(int value) { + return Integer.hashCode(value); + } + + public static final long hash(long value) { + return value; + } + + public static final long hash(float value) { + return Float.hashCode(value); + } + + public static final long hash(double value) { + return Double.doubleToRawLongBits(value); + } + + public static final long hash(Object obj0, Object obj1) { + return hash(intHash(obj0), intHash(obj1)); + } + + public static final long hash(int hash0, int hash1) { + return 31L * hash0 + hash1; + } + + private static final int intHash(Object obj) { + return obj == null ? 0 : obj.hashCode(); + } + + public static final long hash(Object obj0, Object obj1, Object obj2) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2)); + } + + public static final long hash(long hash0, long hash1, long hash2) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * hash0 + 31L * hash1 + hash2; + } + + public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3)); + } + + public static final long hash(int hash0, int hash1, int hash2, int hash3) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * 31L * hash0 + 31L * 31L * hash1 + 31L * hash2 + hash3; + } + + public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3, Object obj4) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3), intHash(obj4)); + } + + public static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * 31L * 31L * hash0 + 31L * 31L * 31L * hash1 + 31L * 31L * hash2 + 31L * hash3 + hash4; + } + + @Deprecated + public static final long hash(int[] hashes) { + long result = 0; + for (int hash : hashes) { + result = addToHash(result, hash); + } + return result; + } + + public static final long addToHash(long hash, int value) { + return 31L * hash + value; + } + + public static final long addToHash(long hash, Object obj) { + return addToHash(hash, intHash(obj)); + } + + public static final long addToHash(long hash, boolean value) { + return addToHash(hash, Boolean.hashCode(value)); + } + + public static final long addToHash(long hash, char value) { + return addToHash(hash, Character.hashCode(value)); + } + + public static final long addToHash(long hash, byte value) { + return addToHash(hash, Byte.hashCode(value)); + } + + public static final long addToHash(long hash, short value) { + return addToHash(hash, Short.hashCode(value)); + } + + public static final long addToHash(long hash, long value) { + return addToHash(hash, Long.hashCode(value)); + } + + public static final long addToHash(long hash, float value) { + return addToHash(hash, Float.hashCode(value)); + } + + public static final long addToHash(long hash, double value) { + return addToHash(hash, Double.hashCode(value)); + } + + public static final long hash(Iterable objs) { + long result = 0; + for (Object obj : objs) { + result = addToHash(result, obj); + } + return result; + } + + /** + * Calling this var-arg version can result in large amounts of allocation (see HashingBenchmark) + * Rather than calliing this method, add another override of hash that handles a larger number of + * arguments or use calls to addToHash. + */ + @Deprecated + public static final long hash(Object[] objs) { + long result = 0; + for (Object obj : objs) { + result = addToHash(result, obj); + } + return result; + } +} From f751ab4e32718dc76e49d2aa82669d386015f78d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 14:18:17 -0400 Subject: [PATCH 02/70] Add AggregateTable + AggregateEntry backed by Hashtable Standalone classes for swapping the consumer-side LRUCache with a multi-key Hashtable in the next commit. No call sites use them yet. - AggregateEntry extends Hashtable.Entry, holds the canonical MetricKey, the mutable AggregateMetric, and copies of the 13 raw SpanSnapshot fields for matches(). The 64-bit lookup hash is computed via chained LongHashingUtils.addToHash calls (no varargs, no boxing of short/boolean). - AggregateTable wraps a Hashtable.Entry[] from Hashtable.Support.create. findOrInsert(SpanSnapshot) walks the bucket comparing raw fields, falling back to MetricKeys.fromSnapshot on a true miss. On cap overrun, it scans for an entry with hitCount==0 and unlinks it; if none, it returns null and the caller drops the data point. - MetricKeys.fromSnapshot extracts the canonicalization logic (DDCache lookups + UTF8 encoding) from Aggregator.buildMetricKey, so the helper can be called from AggregateTable on miss. This also commits Hashtable and LongHashingUtils (added earlier, previously uncommitted) and lifts Hashtable.Entry / Hashtable.Support visibility so client code outside datadog.trace.util can build higher-arity tables -- the case the javadoc describes but the original visibility didn't actually support. Specifically: Entry is now public abstract with a protected ctor; keyHash, next(), and setNext() are public; Support's create / clear / bucketIndex / bucketIterator / mutatingBucketIterator methods are public. Tests: AggregateTableTest covers hit, miss, distinct-by-spanKind, peer-tag identity (including null vs non-null), cap overrun with stale victim, cap overrun with no victim (returns null), expungeStaleAggregates, forEach, clear, and that the canonical MetricKey is built at insert. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 98 ++++++++ .../trace/common/metrics/AggregateTable.java | 134 ++++++++++ .../trace/common/metrics/MetricKeys.java | 65 +++++ .../common/metrics/AggregateTableTest.java | 234 ++++++++++++++++++ 4 files changed, 531 insertions(+) create mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java create mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java create mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java new file mode 100644 index 00000000000..10e256620f5 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -0,0 +1,98 @@ +package datadog.trace.common.metrics; + +import datadog.trace.util.Hashtable; +import datadog.trace.util.LongHashingUtils; +import java.util.Arrays; +import java.util.Objects; + +/** + * Hashtable entry pairing the raw {@link SpanSnapshot} key fields with their canonical {@link + * MetricKey} (built once on miss) and the mutable {@link AggregateMetric}. + * + *

Lookups compare the snapshot's raw fields against the entry's stored copies, so the consumer + * never has to build a {@link MetricKey} just to do a HashMap lookup. The {@code MetricKey} field + * is retained because the serializer ({@link MetricWriter#add}) needs it at report time. + */ +final class AggregateEntry extends Hashtable.Entry { + final MetricKey key; + final AggregateMetric aggregate; + + // Raw snapshot fields, used by matches(SpanSnapshot). Stored as captured at insert time; + // the canonical MetricKey above holds the UTF8BytesString-encoded forms. + private final CharSequence resourceName; + private final String serviceName; + private final CharSequence operationName; + private final CharSequence serviceNameSource; + private final CharSequence spanType; + private final short httpStatusCode; + private final boolean synthetic; + private final boolean traceRoot; + private final String spanKind; + private final String[] peerTagPairs; + private final String httpMethod; + private final String httpEndpoint; + private final String grpcStatusCode; + + AggregateEntry(MetricKey key, SpanSnapshot s, AggregateMetric aggregate) { + super(hashOf(s)); + this.key = key; + this.aggregate = aggregate; + this.resourceName = s.resourceName; + this.serviceName = s.serviceName; + this.operationName = s.operationName; + this.serviceNameSource = s.serviceNameSource; + this.spanType = s.spanType; + this.httpStatusCode = s.httpStatusCode; + this.synthetic = s.synthetic; + this.traceRoot = s.traceRoot; + this.spanKind = s.spanKind; + this.peerTagPairs = s.peerTagPairs; + this.httpMethod = s.httpMethod; + this.httpEndpoint = s.httpEndpoint; + this.grpcStatusCode = s.grpcStatusCode; + } + + boolean matches(SpanSnapshot s) { + return httpStatusCode == s.httpStatusCode + && synthetic == s.synthetic + && traceRoot == s.traceRoot + && Objects.equals(resourceName, s.resourceName) + && Objects.equals(serviceName, s.serviceName) + && Objects.equals(operationName, s.operationName) + && Objects.equals(serviceNameSource, s.serviceNameSource) + && Objects.equals(spanType, s.spanType) + && Objects.equals(spanKind, s.spanKind) + && Arrays.equals(peerTagPairs, s.peerTagPairs) + && Objects.equals(httpMethod, s.httpMethod) + && Objects.equals(httpEndpoint, s.httpEndpoint) + && Objects.equals(grpcStatusCode, s.grpcStatusCode); + } + + /** + * Computes the 64-bit lookup hash for a {@link SpanSnapshot}. Chained per-field calls -- no + * varargs / Object[] allocation, no autoboxing on primitive overloads. The constructor's + * super({@code hashOf(s)}) call uses the same function so an entry built from a snapshot hashes + * to the same bucket the snapshot itself looks up. + */ + static long hashOf(SpanSnapshot s) { + long h = 0; + h = LongHashingUtils.addToHash(h, s.resourceName); + h = LongHashingUtils.addToHash(h, s.serviceName); + h = LongHashingUtils.addToHash(h, s.operationName); + h = LongHashingUtils.addToHash(h, s.serviceNameSource); + h = LongHashingUtils.addToHash(h, s.spanType); + h = LongHashingUtils.addToHash(h, s.httpStatusCode); + h = LongHashingUtils.addToHash(h, s.synthetic); + h = LongHashingUtils.addToHash(h, s.traceRoot); + h = LongHashingUtils.addToHash(h, s.spanKind); + if (s.peerTagPairs != null) { + for (String p : s.peerTagPairs) { + h = LongHashingUtils.addToHash(h, p); + } + } + h = LongHashingUtils.addToHash(h, s.httpMethod); + h = LongHashingUtils.addToHash(h, s.httpEndpoint); + h = LongHashingUtils.addToHash(h, s.grpcStatusCode); + return h; + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java new file mode 100644 index 00000000000..98260a2e2b3 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -0,0 +1,134 @@ +package datadog.trace.common.metrics; + +import datadog.trace.util.Hashtable; +import java.util.function.BiConsumer; + +/** + * Consumer-side {@link AggregateMetric} store, keyed on the raw fields of a {@link SpanSnapshot}. + * + *

Replaces the prior {@code LRUCache}. The win is on the + * steady-state hit path: a snapshot lookup is a 64-bit hash compute + bucket walk + field-wise + * {@code matches}, with no {@link MetricKey} allocation and no UTF8 cache lookups. The canonical + * {@link MetricKey} (with UTF8-encoded forms) is only built once per unique key, at insert time, + * and lives on the {@link AggregateEntry}. + * + *

Not thread-safe. The aggregator thread is the sole writer; {@link #clear()} must be + * routed through the inbox rather than called from arbitrary threads. + */ +final class AggregateTable { + + private final Hashtable.Entry[] buckets; + private final int maxAggregates; + private int size; + + AggregateTable(int maxAggregates) { + this.buckets = Hashtable.Support.create(maxAggregates * 4 / 3); + this.maxAggregates = maxAggregates; + } + + int size() { + return size; + } + + boolean isEmpty() { + return size == 0; + } + + /** + * Returns the {@link AggregateMetric} to update for {@code snapshot}, lazily creating an entry on + * miss. Returns {@code null} when the table is at capacity and no stale entry can be evicted -- + * the caller should drop the data point in that case. + */ + AggregateMetric findOrInsert(SpanSnapshot snapshot) { + long keyHash = AggregateEntry.hashOf(snapshot); + int bucketIndex = Hashtable.Support.bucketIndex(buckets, keyHash); + for (Hashtable.Entry e = buckets[bucketIndex]; e != null; e = e.next()) { + if (e.keyHash == keyHash) { + AggregateEntry candidate = (AggregateEntry) e; + if (candidate.matches(snapshot)) { + return candidate.aggregate; + } + } + } + if (size >= maxAggregates && !evictOneStale()) { + return null; + } + AggregateEntry entry = + new AggregateEntry(MetricKeys.fromSnapshot(snapshot), snapshot, new AggregateMetric()); + entry.setNext(buckets[bucketIndex]); + buckets[bucketIndex] = entry; + size++; + return entry.aggregate; + } + + /** Unlink the first entry whose {@code AggregateMetric.getHitCount() == 0}. */ + private boolean evictOneStale() { + for (int i = 0; i < buckets.length; i++) { + Hashtable.Entry head = buckets[i]; + if (head == null) { + continue; + } + if (((AggregateEntry) head).aggregate.getHitCount() == 0) { + buckets[i] = head.next(); + size--; + return true; + } + Hashtable.Entry prev = head; + Hashtable.Entry cur = head.next(); + while (cur != null) { + if (((AggregateEntry) cur).aggregate.getHitCount() == 0) { + prev.setNext(cur.next()); + size--; + return true; + } + prev = cur; + cur = cur.next(); + } + } + return false; + } + + void forEach(BiConsumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + AggregateEntry entry = (AggregateEntry) e; + consumer.accept(entry.key, entry.aggregate); + } + } + } + + /** Removes entries whose {@code AggregateMetric.getHitCount() == 0}. */ + void expungeStaleAggregates() { + for (int i = 0; i < buckets.length; i++) { + // unlink leading stale entries + Hashtable.Entry head = buckets[i]; + while (head != null && ((AggregateEntry) head).aggregate.getHitCount() == 0) { + head = head.next(); + size--; + } + buckets[i] = head; + if (head == null) { + continue; + } + // unlink stale entries in the chain + Hashtable.Entry prev = head; + Hashtable.Entry cur = head.next(); + while (cur != null) { + if (((AggregateEntry) cur).aggregate.getHitCount() == 0) { + Hashtable.Entry skipped = cur.next(); + prev.setNext(skipped); + size--; + cur = skipped; + } else { + prev = cur; + cur = cur.next(); + } + } + } + } + + void clear() { + Hashtable.Support.clear(buckets); + size = 0; + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java new file mode 100644 index 00000000000..2e03c3730d3 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java @@ -0,0 +1,65 @@ +package datadog.trace.common.metrics; + +import static datadog.trace.api.Functions.UTF8_ENCODE; +import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE; +import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE_ADDER; +import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SERVICE_NAMES; +import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SPAN_KINDS; + +import datadog.trace.api.Pair; +import datadog.trace.api.cache.DDCache; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +/** + * Canonicalization helpers for {@link MetricKey}: applies the static {@link + * ConflatingMetricsAggregator#SERVICE_NAMES} / {@link ConflatingMetricsAggregator#SPAN_KINDS} / + * {@link ConflatingMetricsAggregator#PEER_TAGS_CACHE} caches to a {@link SpanSnapshot}. + * + *

Called only on a true miss in {@link AggregateTable}, so the CHM lookups inside the DDCaches + * happen once per unique key rather than once per snapshot. + */ +final class MetricKeys { + private MetricKeys() {} + + static MetricKey fromSnapshot(SpanSnapshot s) { + return new MetricKey( + s.resourceName, + SERVICE_NAMES.computeIfAbsent(s.serviceName, UTF8_ENCODE), + s.operationName, + s.serviceNameSource, + s.spanType, + s.httpStatusCode, + s.synthetic, + s.traceRoot, + SPAN_KINDS.computeIfAbsent(s.spanKind, UTF8BytesString::create), + materializePeerTags(s.peerTagPairs), + s.httpMethod, + s.httpEndpoint, + s.grpcStatusCode); + } + + private static List materializePeerTags(String[] pairs) { + if (pairs == null || pairs.length == 0) { + return Collections.emptyList(); + } + if (pairs.length == 2) { + // single-entry fast path (matches the original singletonList shape for INTERNAL spans) + return Collections.singletonList(encodePeerTag(pairs[0], pairs[1])); + } + List tags = new ArrayList<>(pairs.length / 2); + for (int i = 0; i < pairs.length; i += 2) { + tags.add(encodePeerTag(pairs[i], pairs[i + 1])); + } + return tags; + } + + private static UTF8BytesString encodePeerTag(String name, String value) { + final Pair, Function> + cacheAndCreator = PEER_TAGS_CACHE.computeIfAbsent(name, PEER_TAGS_CACHE_ADDER); + return cacheAndCreator.getLeft().computeIfAbsent(value, cacheAndCreator.getRight()); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java new file mode 100644 index 00000000000..6c4839e4e4f --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -0,0 +1,234 @@ +package datadog.trace.common.metrics; + +import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG; +import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.metrics.agent.AgentMeter; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.DDSketchHistograms; +import datadog.metrics.impl.MonitoringImpl; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class AggregateTableTest { + + @BeforeAll + static void initAgentMeter() { + // AggregateMetric.recordOneDuration -> Histogram.accept needs AgentMeter to be initialized. + // Mirror what AggregateMetricTest does. + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + AgentMeter.registerIfAbsent(StatsDClient.NO_OP, monitoring, DDSketchHistograms.FACTORY); + monitoring.newTimer("test.init"); + } + + @Test + void insertOnMissReturnsNewAggregate() { + AggregateTable table = new AggregateTable(8); + SpanSnapshot s = snapshot("svc", "op", "client"); + + AggregateMetric agg = table.findOrInsert(s); + + assertNotNull(agg); + assertEquals(1, table.size()); + assertEquals(0, agg.getHitCount()); + } + + @Test + void hitReturnsSameAggregateInstance() { + AggregateTable table = new AggregateTable(8); + SpanSnapshot s1 = snapshot("svc", "op", "client"); + SpanSnapshot s2 = snapshot("svc", "op", "client"); + + AggregateMetric first = table.findOrInsert(s1); + AggregateMetric second = table.findOrInsert(s2); + + assertSame(first, second); + assertEquals(1, table.size()); + } + + @Test + void differentKindFieldsAreDistinct() { + AggregateTable table = new AggregateTable(8); + + AggregateMetric clientAgg = table.findOrInsert(snapshot("svc", "op", "client")); + AggregateMetric serverAgg = table.findOrInsert(snapshot("svc", "op", "server")); + + assertNotSame(clientAgg, serverAgg); + assertEquals(2, table.size()); + } + + @Test + void peerTagPairsParticipateInIdentity() { + AggregateTable table = new AggregateTable(8); + SpanSnapshot withTags = + builder("svc", "op", "client").peerTags("peer.hostname", "host-a").build(); + SpanSnapshot otherTags = + builder("svc", "op", "client").peerTags("peer.hostname", "host-b").build(); + SpanSnapshot noTags = builder("svc", "op", "client").build(); + + AggregateMetric a = table.findOrInsert(withTags); + AggregateMetric b = table.findOrInsert(otherTags); + AggregateMetric c = table.findOrInsert(noTags); + + assertNotSame(a, b); + assertNotSame(a, c); + assertNotSame(b, c); + assertEquals(3, table.size()); + } + + @Test + void capOverrunEvictsStaleEntry() { + AggregateTable table = new AggregateTable(2); + + AggregateMetric stale = table.findOrInsert(snapshot("svc-a", "op", "client")); + // do not record on stale -> hitCount stays at 0 + + AggregateMetric live = table.findOrInsert(snapshot("svc-b", "op", "client")); + live.recordOneDuration(10L | TOP_LEVEL_TAG); // hitCount=1, not evictable + + // table is full (size=2). Inserting a third should evict the stale one and succeed. + AggregateMetric newcomer = table.findOrInsert(snapshot("svc-c", "op", "client")); + assertNotNull(newcomer); + assertEquals(2, table.size()); + + // re-inserting the stale snapshot should miss now (it was evicted) and produce a fresh entry + AggregateMetric staleAgain = table.findOrInsert(snapshot("svc-a", "op", "client")); + assertNotSame(stale, staleAgain); + } + + @Test + void capOverrunWithNoStaleReturnsNull() { + AggregateTable table = new AggregateTable(2); + + AggregateMetric a = table.findOrInsert(snapshot("svc-a", "op", "client")); + AggregateMetric b = table.findOrInsert(snapshot("svc-b", "op", "client")); + a.recordOneDuration(10L); + b.recordOneDuration(20L); + + AggregateMetric c = table.findOrInsert(snapshot("svc-c", "op", "client")); + assertNull(c); + assertEquals(2, table.size()); + } + + @Test + void expungeStaleAggregatesRemovesZeroHitsOnly() { + AggregateTable table = new AggregateTable(16); + + AggregateMetric live = table.findOrInsert(snapshot("svc-live", "op", "client")); + live.recordOneDuration(10L); + AggregateMetric stale1 = table.findOrInsert(snapshot("svc-stale1", "op", "client")); + AggregateMetric stale2 = table.findOrInsert(snapshot("svc-stale2", "op", "client")); + assertEquals(3, table.size()); + assertEquals(0, stale1.getHitCount()); + assertEquals(0, stale2.getHitCount()); + + table.expungeStaleAggregates(); + + assertEquals(1, table.size()); + // the live entry must still be reachable + assertSame(live, table.findOrInsert(snapshot("svc-live", "op", "client"))); + } + + @Test + void forEachVisitsEveryEntry() { + AggregateTable table = new AggregateTable(8); + table.findOrInsert(snapshot("a", "op", "client")).recordOneDuration(1L); + table.findOrInsert(snapshot("b", "op", "client")).recordOneDuration(2L); + table.findOrInsert(snapshot("c", "op", "client")).recordOneDuration(3L | ERROR_TAG); + + Map visited = new HashMap<>(); + table.forEach((key, agg) -> visited.put(key.getService().toString(), agg.getDuration())); + + assertEquals(3, visited.size()); + assertEquals(1L, visited.get("a")); + assertEquals(2L, visited.get("b")); + assertEquals(3L, visited.get("c")); + } + + @Test + void clearEmptiesTheTable() { + AggregateTable table = new AggregateTable(8); + table.findOrInsert(snapshot("a", "op", "client")); + table.findOrInsert(snapshot("b", "op", "client")); + assertEquals(2, table.size()); + + table.clear(); + + assertTrue(table.isEmpty()); + assertEquals(0, table.size()); + // and re-insertion works after clear + assertNotNull(table.findOrInsert(snapshot("a", "op", "client"))); + } + + @Test + void canonicalMetricKeyIsBuiltOnInsert() { + AggregateTable table = new AggregateTable(4); + List seen = new ArrayList<>(); + table.findOrInsert(snapshot("svc", "op", "client")); + table.forEach((key, agg) -> seen.add(key)); + + assertEquals(1, seen.size()); + MetricKey k = seen.get(0); + assertEquals("svc", k.getService().toString()); + assertEquals("op", k.getOperationName().toString()); + assertEquals("client", k.getSpanKind().toString()); + } + + // ---------- helpers ---------- + + private static SpanSnapshot snapshot(String service, String operation, String spanKind) { + return builder(service, operation, spanKind).build(); + } + + private static SnapshotBuilder builder(String service, String operation, String spanKind) { + return new SnapshotBuilder(service, operation, spanKind); + } + + private static final class SnapshotBuilder { + private final String service; + private final String operation; + private final String spanKind; + private String[] peerTagPairs; + private long tagAndDuration = 0L; + + SnapshotBuilder(String service, String operation, String spanKind) { + this.service = service; + this.operation = operation; + this.spanKind = spanKind; + } + + SnapshotBuilder peerTags(String... namesAndValues) { + this.peerTagPairs = namesAndValues; + return this; + } + + SpanSnapshot build() { + return new SpanSnapshot( + "resource", + service, + operation, + null, + "web", + (short) 200, + false, + true, + spanKind, + peerTagPairs, + null, + null, + null, + tagAndDuration); + } + } +} From f1b030adfffe64d9c17d92e2b146e730cf8dac54 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 14:24:09 -0400 Subject: [PATCH 03/70] Swap LRUCache for AggregateTable in Aggregator + route disable() clear Replace LRUCache with the AggregateTable added in the prior commit. The hot path in Drainer.accept becomes: AggregateMetric aggregate = aggregates.findOrInsert(snapshot); if (aggregate != null) { aggregate.recordOneDuration(snapshot.tagAndDuration); dirty = true; } else { healthMetrics.onStatsAggregateDropped(); } On the steady-state hit path the lookup is a 64-bit hash compute + bucket walk + matches(snapshot) -- no MetricKey allocation, no SERVICE_NAMES / SPAN_KINDS / PEER_TAGS_CACHE lookups. The canonical MetricKey is now built once per unique key at insert time, in MetricKeys.fromSnapshot. Behavioral change in the cap-overrun path ----------------------------------------- The old LRUCache evicted least-recently-used: at cap, a new insert would push out the oldest entry regardless of whether it was live or stale. AggregateTable instead scans for a hitCount==0 entry to recycle, and drops the new key if none exists. Practical impact: in the common case where the table holds a stable set of recurring keys, an unrelated burst of new keys is dropped (and reported via onStatsAggregateDropped) rather than evicting the established keys. The existing test that asserted "service0 evicted in favor of service10" is updated to assert the new semantics. The other cap-related test ("should not report dropped aggregate when evicted entry was already flushed") still passes unchanged: after report() clears all entries to hitCount=0, the next wave of inserts recycles them. Threading fix ------------- ConflatingMetricsAggregator.disable() used to call aggregator.clearAggregates() and inbox.clear() directly from the Sink's IO event thread, racing with the aggregator thread mid-write. The race was tolerable for LinkedHashMap; it is not for AggregateTable (chain corruption can NPE or loop). disable() now offers a ClearSignal to the inbox so the aggregator thread itself performs the table clear and the inbox.clear(). Adds one SignalItem subclass + one branch in Drainer.accept; preserves the single-writer invariant for AggregateTable end-to-end. Removed: LRUCache import, AggregateExpiry inner class, the static buildMetricKey / materializePeerTags / encodePeerTag helpers (now in MetricKeys). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/Aggregator.java | 120 ++++-------------- .../metrics/ConflatingMetricsAggregator.java | 7 +- .../trace/common/metrics/InboxItem.java | 11 ++ .../ConflatingMetricAggregatorTest.groovy | 11 +- 4 files changed, 49 insertions(+), 100 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index e632555cc21..d0262f328f6 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -1,26 +1,12 @@ package datadog.trace.common.metrics; -import static datadog.trace.api.Functions.UTF8_ENCODE; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE_ADDER; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SERVICE_NAMES; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SPAN_KINDS; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import datadog.trace.api.Pair; -import datadog.trace.api.cache.DDCache; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import datadog.trace.common.metrics.SignalItem.ClearSignal; import datadog.trace.common.metrics.SignalItem.StopSignal; import datadog.trace.core.monitor.HealthMetrics; -import datadog.trace.core.util.LRUCache; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import org.jctools.queues.MessagePassingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,8 +18,9 @@ final class Aggregator implements Runnable { private static final Logger log = LoggerFactory.getLogger(Aggregator.class); private final MessagePassingQueue inbox; - private final LRUCache aggregates; + private final AggregateTable aggregates; private final MetricWriter writer; + private final HealthMetrics healthMetrics; // the reporting interval controls how much history will be buffered // when the agent is unresponsive (only 10 pending requests will be // buffered by OkHttpSink) @@ -73,27 +60,10 @@ final class Aggregator implements Runnable { HealthMetrics healthMetrics) { this.writer = writer; this.inbox = inbox; - this.aggregates = - new LRUCache<>( - new AggregateExpiry(healthMetrics), maxAggregates * 4 / 3, 0.75f, maxAggregates); + this.aggregates = new AggregateTable(maxAggregates); this.reportingIntervalNanos = reportingIntervalTimeUnit.toNanos(reportingInterval); this.sleepMillis = sleepMillis; - } - - private static final class AggregateExpiry - implements LRUCache.ExpiryListener { - private final HealthMetrics healthMetrics; - - AggregateExpiry(HealthMetrics healthMetrics) { - this.healthMetrics = healthMetrics; - } - - @Override - public void accept(Map.Entry expired) { - if (expired.getValue().getHitCount() > 0) { - healthMetrics.onStatsAggregateDropped(); - } - } + this.healthMetrics = healthMetrics; } public void clearAggregates() { @@ -126,7 +96,13 @@ private final class Drainer implements MessagePassingQueue.Consumer { @Override public void accept(InboxItem item) { - if (item instanceof SignalItem) { + if (item == ClearSignal.CLEAR) { + if (!stopped) { + aggregates.clear(); + inbox.clear(); + } + ((SignalItem) item).complete(); + } else if (item instanceof SignalItem) { SignalItem signal = (SignalItem) item; if (!stopped) { report(wallClockTime(), signal); @@ -139,64 +115,31 @@ public void accept(InboxItem item) { } } else if (item instanceof SpanSnapshot && !stopped) { SpanSnapshot snapshot = (SpanSnapshot) item; - MetricKey key = buildMetricKey(snapshot); - AggregateMetric aggregate = aggregates.computeIfAbsent(key, k -> new AggregateMetric()); - aggregate.recordOneDuration(snapshot.tagAndDuration); - dirty = true; + AggregateMetric aggregate = aggregates.findOrInsert(snapshot); + if (aggregate != null) { + aggregate.recordOneDuration(snapshot.tagAndDuration); + dirty = true; + } else { + // table at cap with no stale entry available to evict + healthMetrics.onStatsAggregateDropped(); + } } } } - private static MetricKey buildMetricKey(SpanSnapshot s) { - return new MetricKey( - s.resourceName, - SERVICE_NAMES.computeIfAbsent(s.serviceName, UTF8_ENCODE), - s.operationName, - s.serviceNameSource, - s.spanType, - s.httpStatusCode, - s.synthetic, - s.traceRoot, - SPAN_KINDS.computeIfAbsent(s.spanKind, UTF8BytesString::create), - materializePeerTags(s.peerTagPairs), - s.httpMethod, - s.httpEndpoint, - s.grpcStatusCode); - } - - private static List materializePeerTags(String[] pairs) { - if (pairs == null || pairs.length == 0) { - return Collections.emptyList(); - } - if (pairs.length == 2) { - // single-entry fast path (matches the original singletonList shape for INTERNAL spans) - return Collections.singletonList(encodePeerTag(pairs[0], pairs[1])); - } - List tags = new ArrayList<>(pairs.length / 2); - for (int i = 0; i < pairs.length; i += 2) { - tags.add(encodePeerTag(pairs[i], pairs[i + 1])); - } - return tags; - } - - private static UTF8BytesString encodePeerTag(String name, String value) { - final Pair, Function> - cacheAndCreator = PEER_TAGS_CACHE.computeIfAbsent(name, PEER_TAGS_CACHE_ADDER); - return cacheAndCreator.getLeft().computeIfAbsent(value, cacheAndCreator.getRight()); - } - private void report(long when, SignalItem signal) { boolean skipped = true; if (dirty) { try { - expungeStaleAggregates(); + aggregates.expungeStaleAggregates(); if (!aggregates.isEmpty()) { skipped = false; writer.startBucket(aggregates.size(), when, reportingIntervalNanos); - for (Map.Entry aggregate : aggregates.entrySet()) { - writer.add(aggregate.getKey(), aggregate.getValue()); - aggregate.getValue().clear(); - } + aggregates.forEach( + (key, agg) -> { + writer.add(key, agg); + agg.clear(); + }); // note that this may do IO and block writer.finishBucket(); } @@ -212,17 +155,6 @@ private void report(long when, SignalItem signal) { } } - private void expungeStaleAggregates() { - Iterator> it = aggregates.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry pair = it.next(); - AggregateMetric metric = pair.getValue(); - if (metric.getHitCount() == 0) { - it.remove(); - } - } - } - private long wallClockTime() { return MILLISECONDS.toNanos(System.currentTimeMillis()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index 9ea77140113..79dcf991c10 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -8,6 +8,7 @@ import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG; import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG; +import static datadog.trace.common.metrics.SignalItem.ClearSignal.CLEAR; import static datadog.trace.common.metrics.SignalItem.ReportSignal.REPORT; import static datadog.trace.common.metrics.SignalItem.StopSignal.STOP; import static datadog.trace.util.AgentThreadFactory.AgentThread.METRICS_AGGREGATOR; @@ -418,8 +419,10 @@ private void disable() { features.discover(); if (!features.supportsMetrics()) { log.debug("Disabling metric reporting because an agent downgrade was detected"); - this.inbox.clear(); - this.aggregator.clearAggregates(); + // Route the clear through the inbox so the aggregator thread is the only writer. + // AggregateTable is not thread-safe; calling clearAggregates() directly from this thread + // would race with Drainer.accept on the aggregator thread. + inbox.offer(CLEAR); } } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/InboxItem.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/InboxItem.java index 7d66cad6a15..a0625be095b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/InboxItem.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/InboxItem.java @@ -28,4 +28,15 @@ private StopSignal() {} static final class ReportSignal extends SignalItem { static final ReportSignal REPORT = new ReportSignal(); } + + /** + * Posted from arbitrary threads (e.g. the Sink event thread during agent downgrade) so the + * aggregator thread is the one that actually performs the table reset. Keeps {@link + * AggregateTable} and {@code inbox.clear()} single-writer. + */ + static final class ClearSignal extends SignalItem { + static final ClearSignal CLEAR = new ClearSignal(); + + private ClearSignal() {} + } } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy index 962ad2ce892..dedd0bae75b 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy @@ -877,7 +877,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { aggregator.close() } - def "test least recently written to aggregate flushed when size limit exceeded"() { + def "new aggregates beyond size limit are dropped when no stale entries can be evicted"() { + // The table only evicts entries with hitCount == 0 to make room. When all entries are live + // (all have been recorded against), an over-cap insert drops the new key rather than evicting + // an established one. This protects the data we've already collected from a burst of new keys. setup: int maxAggregates = 10 MetricWriter writer = Mock(MetricWriter) @@ -901,10 +904,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { aggregator.report() def latchTriggered = latch.await(2, SECONDS) - then: "the first aggregate should be dropped but the rest reported" + then: "the established service0..service9 are reported; service10 is dropped" latchTriggered 1 * writer.startBucket(10, _, SECONDS.toNanos(reportingInterval)) - for (int i = 1; i < 11; ++i) { + for (int i = 0; i < 10; ++i) { 1 * writer.add(new MetricKey( "resource", "service" + i, @@ -925,7 +928,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { } 0 * writer.add(new MetricKey( "resource", - "service0", + "service10", "operation", null, "type", From 3738c85f75bb1cb88c05eb1d51a7d45fc9d353d1 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 15 May 2026 15:07:16 -0400 Subject: [PATCH 04/70] Eliminate MetricKey: inline its fields onto AggregateEntry MetricKey existed for two reasons -- the prior LRUCache key role (now handled by AggregateTable's Hashtable.Entry mechanics) and as the labels argument to MetricWriter.add. The first is gone; the second is the only thing keeping MetricKey alive. Fold its UTF8-encoded label fields onto AggregateEntry, change MetricWriter.add to take AggregateEntry directly, and delete MetricKey + MetricKeys. What AggregateEntry now holds ----------------------------- - 10 UTF8BytesString label fields (resource, service, operationName, serviceSource, type, spanKind, httpMethod, httpEndpoint, grpcStatusCode, and a List peerTags for serialization). - 3 primitives (httpStatusCode, synthetic, traceRoot). - AggregateMetric (the value being accumulated). - The raw String[] peerTagPairs is retained alongside the encoded peerTags -- matches() compares it positionally against the snapshot's pairs; the encoded form is only consumed by the writer. matches(SpanSnapshot) compares the entry's UTF8 forms to the snapshot's raw String / CharSequence fields via content-equality (UTF8BytesString.toString() returns the underlying String in O(1)). This closes a latent bug in the prior raw-vs-raw matches(): if one snapshot delivered a tag value as String and a later snapshot delivered the same content as UTF8BytesString, the old Objects.equals would return false and the table would split into two entries. Content-equality matching collapses them into one. Consolidated caches ------------------- The static UTF8 caches that used to live partly on MetricKey (RESOURCE_CACHE, OPERATION_CACHE, SERVICE_SOURCE_CACHE, TYPE_CACHE, KIND_CACHE, HTTP_METHOD_CACHE, HTTP_ENDPOINT_CACHE, GRPC_STATUS_CODE_CACHE, SERVICE_CACHE) and partly on ConflatingMetricsAggregator (SERVICE_NAMES, SPAN_KINDS, PEER_TAGS_CACHE) are all now on AggregateEntry. The split was duplicating work -- SERVICE_NAMES and SERVICE_CACHE both cached service-name to UTF8BytesString. One cache per field now. API change: MetricWriter.add ---------------------------- Was: add(MetricKey key, AggregateMetric aggregate) Now: add(AggregateEntry entry) The aggregate lives on the entry. Single-arg. SerializingMetricWriter reads the same UTF8 fields off AggregateEntry that it previously read off MetricKey; the wire format is byte-identical. Test impact ----------- AggregateEntry.of(...) takes the same 13 positional args new MetricKey(...) took, so test diffs are mostly mechanical: new MetricKey(args) -> AggregateEntry.of(args) writer.add(key, _) -> writer.add(entry) ValidatingSink in SerializingMetricWriterTest now iterates List directly. ConflatingMetricAggregatorTest's Spock matchers (~36 sites) rely on AggregateEntry.equals comparing the 13 label fields (not the aggregate) so the mock matches by labels regardless of the aggregate state at call time; post-invocation closures verify aggregate state. Benchmarks (2 forks x 5 iter x 15s) ----------------------------------- The change is consumer-thread only; producer publish() is unchanged. SimpleSpan bench: 3.123 +- 0.025 us/op (prior: 3.119 +- 0.018) DDSpan bench: 2.412 +- 0.022 us/op (prior: 2.463 +- 0.041) Both within noise -- the win is structural (one less class, one less allocation per miss, one fewer cache layer) rather than benchmarked. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 360 +++++++++++++++--- .../trace/common/metrics/AggregateTable.java | 16 +- .../trace/common/metrics/Aggregator.java | 6 +- .../metrics/ConflatingMetricsAggregator.java | 21 - .../trace/common/metrics/MetricKey.java | 178 --------- .../trace/common/metrics/MetricKeys.java | 65 ---- .../trace/common/metrics/MetricWriter.java | 6 +- .../metrics/SerializingMetricWriter.java | 37 +- .../trace/common/metrics/SpanSnapshot.java | 3 +- .../ConflatingMetricAggregatorTest.groovy | 264 ++++++------- .../SerializingMetricWriterTest.groovy | 333 ++++++---------- .../common/metrics/AggregateTableTest.java | 16 +- .../groovy/MetricsIntegrationTest.groovy | 17 +- 13 files changed, 609 insertions(+), 713 deletions(-) delete mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java delete mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 10e256620f5..e2fda9fde47 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -1,71 +1,176 @@ package datadog.trace.common.metrics; +import static datadog.trace.api.Functions.UTF8_ENCODE; +import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; + +import datadog.trace.api.Pair; +import datadog.trace.api.cache.DDCache; +import datadog.trace.api.cache.DDCaches; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Hashtable; import datadog.trace.util.LongHashingUtils; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Objects; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; /** - * Hashtable entry pairing the raw {@link SpanSnapshot} key fields with their canonical {@link - * MetricKey} (built once on miss) and the mutable {@link AggregateMetric}. + * Hashtable entry for the consumer-side aggregator. Holds the UTF8-encoded label fields (the data + * {@link SerializingMetricWriter} writes to the wire) plus the mutable {@link AggregateMetric}. + * + *

{@link #matches(SpanSnapshot)} compares the entry's stored UTF8 forms against the snapshot's + * raw {@code CharSequence}/{@code String}/{@code String[]} fields via content-equality, so {@code + * String} vs {@code UTF8BytesString} mixing on the same logical key collapses into one entry + * instead of splitting. * - *

Lookups compare the snapshot's raw fields against the entry's stored copies, so the consumer - * never has to build a {@link MetricKey} just to do a HashMap lookup. The {@code MetricKey} field - * is retained because the serializer ({@link MetricWriter#add}) needs it at report time. + *

The static UTF8 caches that used to live on {@code MetricKey} and {@code + * ConflatingMetricsAggregator} are consolidated here. */ final class AggregateEntry extends Hashtable.Entry { - final MetricKey key; - final AggregateMetric aggregate; - // Raw snapshot fields, used by matches(SpanSnapshot). Stored as captured at insert time; - // the canonical MetricKey above holds the UTF8BytesString-encoded forms. - private final CharSequence resourceName; - private final String serviceName; - private final CharSequence operationName; - private final CharSequence serviceNameSource; - private final CharSequence spanType; + // UTF8 caches consolidated from the previous MetricKey + ConflatingMetricsAggregator split. + private static final DDCache RESOURCE_CACHE = + DDCaches.newFixedSizeCache(32); + private static final DDCache SERVICE_CACHE = + DDCaches.newFixedSizeCache(32); + private static final DDCache OPERATION_CACHE = + DDCaches.newFixedSizeCache(64); + private static final DDCache SERVICE_SOURCE_CACHE = + DDCaches.newFixedSizeCache(16); + private static final DDCache TYPE_CACHE = DDCaches.newFixedSizeCache(8); + private static final DDCache SPAN_KIND_CACHE = + DDCaches.newFixedSizeCache(16); + private static final DDCache HTTP_METHOD_CACHE = + DDCaches.newFixedSizeCache(8); + private static final DDCache HTTP_ENDPOINT_CACHE = + DDCaches.newFixedSizeCache(32); + private static final DDCache GRPC_STATUS_CODE_CACHE = + DDCaches.newFixedSizeCache(32); + + /** + * Outer cache keyed by peer-tag name, with an inner per-name cache keyed by value. The inner + * cache produces the "name:value" encoded form the serializer writes. + */ + private static final DDCache< + String, Pair, Function>> + PEER_TAGS_CACHE = DDCaches.newFixedSizeCache(64); + + private static final Function< + String, Pair, Function>> + PEER_TAGS_CACHE_ADDER = + key -> + Pair.of( + DDCaches.newFixedSizeCache(512), + value -> UTF8BytesString.create(key + ":" + value)); + + private final UTF8BytesString resource; + private final UTF8BytesString service; + private final UTF8BytesString operationName; + private final UTF8BytesString serviceSource; // nullable + private final UTF8BytesString type; + private final UTF8BytesString spanKind; + private final UTF8BytesString httpMethod; // nullable + private final UTF8BytesString httpEndpoint; // nullable + private final UTF8BytesString grpcStatusCode; // nullable private final short httpStatusCode; private final boolean synthetic; private final boolean traceRoot; - private final String spanKind; - private final String[] peerTagPairs; - private final String httpMethod; - private final String httpEndpoint; - private final String grpcStatusCode; - - AggregateEntry(MetricKey key, SpanSnapshot s, AggregateMetric aggregate) { - super(hashOf(s)); - this.key = key; - this.aggregate = aggregate; - this.resourceName = s.resourceName; - this.serviceName = s.serviceName; - this.operationName = s.operationName; - this.serviceNameSource = s.serviceNameSource; - this.spanType = s.spanType; + + // Peer tags carried in two forms: raw String[] for matches() against the snapshot's pairs, + // and pre-encoded List ("name:value") for the serializer. + private final String[] peerTagPairsRaw; + private final List peerTags; + + final AggregateMetric aggregate; + + /** Hot-path constructor for the producer/consumer flow. Builds UTF8 fields via the caches. */ + private AggregateEntry(SpanSnapshot s, long keyHash, AggregateMetric aggregate) { + super(keyHash); + this.resource = canonicalize(RESOURCE_CACHE, s.resourceName); + this.service = SERVICE_CACHE.computeIfAbsent(s.serviceName, UTF8_ENCODE); + this.operationName = canonicalize(OPERATION_CACHE, s.operationName); + this.serviceSource = + s.serviceNameSource == null + ? null + : canonicalize(SERVICE_SOURCE_CACHE, s.serviceNameSource); + this.type = canonicalize(TYPE_CACHE, s.spanType); + this.spanKind = SPAN_KIND_CACHE.computeIfAbsent(s.spanKind, UTF8BytesString::create); + this.httpMethod = + s.httpMethod == null + ? null + : HTTP_METHOD_CACHE.computeIfAbsent(s.httpMethod, UTF8BytesString::create); + this.httpEndpoint = + s.httpEndpoint == null + ? null + : HTTP_ENDPOINT_CACHE.computeIfAbsent(s.httpEndpoint, UTF8BytesString::create); + this.grpcStatusCode = + s.grpcStatusCode == null + ? null + : GRPC_STATUS_CODE_CACHE.computeIfAbsent(s.grpcStatusCode, UTF8BytesString::create); this.httpStatusCode = s.httpStatusCode; this.synthetic = s.synthetic; this.traceRoot = s.traceRoot; - this.spanKind = s.spanKind; - this.peerTagPairs = s.peerTagPairs; - this.httpMethod = s.httpMethod; - this.httpEndpoint = s.httpEndpoint; - this.grpcStatusCode = s.grpcStatusCode; + this.peerTagPairsRaw = s.peerTagPairs; + this.peerTags = materializePeerTags(s.peerTagPairs); + this.aggregate = aggregate; + } + + /** Test-friendly factory mirroring the prior {@code new MetricKey(...)} positional args. */ + static AggregateEntry of( + CharSequence resource, + CharSequence service, + CharSequence operationName, + CharSequence serviceSource, + CharSequence type, + int httpStatusCode, + boolean synthetic, + boolean traceRoot, + CharSequence spanKind, + List peerTags, + CharSequence httpMethod, + CharSequence httpEndpoint, + CharSequence grpcStatusCode) { + String[] rawPairs = peerTagsToRawPairs(peerTags); + SpanSnapshot synthetic_snapshot = + new SpanSnapshot( + resource, + service == null ? null : service.toString(), + operationName, + serviceSource, + type, + (short) httpStatusCode, + synthetic, + traceRoot, + spanKind == null ? null : spanKind.toString(), + rawPairs, + httpMethod == null ? null : httpMethod.toString(), + httpEndpoint == null ? null : httpEndpoint.toString(), + grpcStatusCode == null ? null : grpcStatusCode.toString(), + 0L); + return new AggregateEntry( + synthetic_snapshot, hashOf(synthetic_snapshot), new AggregateMetric()); + } + + /** Construct from a snapshot at consumer-thread miss time. */ + static AggregateEntry forSnapshot(SpanSnapshot s, AggregateMetric aggregate) { + return new AggregateEntry(s, hashOf(s), aggregate); } boolean matches(SpanSnapshot s) { return httpStatusCode == s.httpStatusCode && synthetic == s.synthetic && traceRoot == s.traceRoot - && Objects.equals(resourceName, s.resourceName) - && Objects.equals(serviceName, s.serviceName) - && Objects.equals(operationName, s.operationName) - && Objects.equals(serviceNameSource, s.serviceNameSource) - && Objects.equals(spanType, s.spanType) - && Objects.equals(spanKind, s.spanKind) - && Arrays.equals(peerTagPairs, s.peerTagPairs) - && Objects.equals(httpMethod, s.httpMethod) - && Objects.equals(httpEndpoint, s.httpEndpoint) - && Objects.equals(grpcStatusCode, s.grpcStatusCode); + && contentEquals(resource, s.resourceName) + && stringContentEquals(service, s.serviceName) + && contentEquals(operationName, s.operationName) + && contentEquals(serviceSource, s.serviceNameSource) + && contentEquals(type, s.spanType) + && stringContentEquals(spanKind, s.spanKind) + && Arrays.equals(peerTagPairsRaw, s.peerTagPairs) + && stringContentEquals(httpMethod, s.httpMethod) + && stringContentEquals(httpEndpoint, s.httpEndpoint) + && stringContentEquals(grpcStatusCode, s.grpcStatusCode); } /** @@ -73,6 +178,9 @@ boolean matches(SpanSnapshot s) { * varargs / Object[] allocation, no autoboxing on primitive overloads. The constructor's * super({@code hashOf(s)}) call uses the same function so an entry built from a snapshot hashes * to the same bucket the snapshot itself looks up. + * + *

Hashes are content-stable across {@code String} / {@code UTF8BytesString}: {@link + * UTF8BytesString#hashCode()} returns the underlying {@code String}'s hash. */ static long hashOf(SpanSnapshot s) { long h = 0; @@ -95,4 +203,166 @@ static long hashOf(SpanSnapshot s) { h = LongHashingUtils.addToHash(h, s.grpcStatusCode); return h; } + + // Accessors for SerializingMetricWriter. + UTF8BytesString getResource() { + return resource; + } + + UTF8BytesString getService() { + return service; + } + + UTF8BytesString getOperationName() { + return operationName; + } + + UTF8BytesString getServiceSource() { + return serviceSource; + } + + UTF8BytesString getType() { + return type; + } + + UTF8BytesString getSpanKind() { + return spanKind; + } + + UTF8BytesString getHttpMethod() { + return httpMethod; + } + + UTF8BytesString getHttpEndpoint() { + return httpEndpoint; + } + + UTF8BytesString getGrpcStatusCode() { + return grpcStatusCode; + } + + int getHttpStatusCode() { + return httpStatusCode; + } + + boolean isSynthetics() { + return synthetic; + } + + boolean isTraceRoot() { + return traceRoot; + } + + List getPeerTags() { + return peerTags; + } + + /** + * Equality on the 13 label fields (not on the aggregate). Used only by test mock matchers; the + * {@link Hashtable} does its own bucketing via {@link #keyHash} + {@link #matches(SpanSnapshot)} + * and never calls {@code equals}. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof AggregateEntry)) return false; + AggregateEntry that = (AggregateEntry) o; + return httpStatusCode == that.httpStatusCode + && synthetic == that.synthetic + && traceRoot == that.traceRoot + && java.util.Objects.equals(resource, that.resource) + && java.util.Objects.equals(service, that.service) + && java.util.Objects.equals(operationName, that.operationName) + && java.util.Objects.equals(serviceSource, that.serviceSource) + && java.util.Objects.equals(type, that.type) + && java.util.Objects.equals(spanKind, that.spanKind) + && peerTags.equals(that.peerTags) + && java.util.Objects.equals(httpMethod, that.httpMethod) + && java.util.Objects.equals(httpEndpoint, that.httpEndpoint) + && java.util.Objects.equals(grpcStatusCode, that.grpcStatusCode); + } + + @Override + public int hashCode() { + return (int) keyHash; + } + + // ----- helpers ----- + + private static UTF8BytesString canonicalize( + DDCache cache, CharSequence charSeq) { + if (charSeq == null) { + return EMPTY; + } + if (charSeq instanceof UTF8BytesString) { + return (UTF8BytesString) charSeq; + } + return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); + } + + /** UTF8 vs raw CharSequence content-equality, no allocation in the common (String) case. */ + private static boolean contentEquals(UTF8BytesString a, CharSequence b) { + if (a == null) { + return b == null; + } + if (b == null) { + return false; + } + // UTF8BytesString.toString() returns the underlying String -- O(1), no allocation. + String aStr = a.toString(); + if (b instanceof String) { + return aStr.equals(b); + } + if (b instanceof UTF8BytesString) { + return aStr.equals(b.toString()); + } + return aStr.contentEquals(b); + } + + private static boolean stringContentEquals(UTF8BytesString a, String b) { + if (a == null) { + return b == null; + } + return b != null && a.toString().equals(b); + } + + private static List materializePeerTags(String[] pairs) { + if (pairs == null || pairs.length == 0) { + return Collections.emptyList(); + } + if (pairs.length == 2) { + return Collections.singletonList(encodePeerTag(pairs[0], pairs[1])); + } + List tags = new ArrayList<>(pairs.length / 2); + for (int i = 0; i < pairs.length; i += 2) { + tags.add(encodePeerTag(pairs[i], pairs[i + 1])); + } + return tags; + } + + private static UTF8BytesString encodePeerTag(String name, String value) { + final Pair, Function> + cacheAndCreator = PEER_TAGS_CACHE.computeIfAbsent(name, PEER_TAGS_CACHE_ADDER); + return cacheAndCreator.getLeft().computeIfAbsent(value, cacheAndCreator.getRight()); + } + + /** + * Inverse of {@link #materializePeerTags}: takes pre-encoded UTF8 peer tags and recovers the raw + * {@code [name0, value0, name1, value1, ...]} pairs. Used by the test factory {@link #of}, not by + * the hot path. + */ + private static String[] peerTagsToRawPairs(List peerTags) { + if (peerTags == null || peerTags.isEmpty()) { + return null; + } + String[] pairs = new String[peerTags.size() * 2]; + int i = 0; + for (UTF8BytesString peerTag : peerTags) { + String s = peerTag.toString(); + int colon = s.indexOf(':'); + pairs[i++] = colon < 0 ? s : s.substring(0, colon); + pairs[i++] = colon < 0 ? "" : s.substring(colon + 1); + } + return pairs; + } } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 98260a2e2b3..08300eab296 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -1,16 +1,16 @@ package datadog.trace.common.metrics; import datadog.trace.util.Hashtable; -import java.util.function.BiConsumer; +import java.util.function.Consumer; /** * Consumer-side {@link AggregateMetric} store, keyed on the raw fields of a {@link SpanSnapshot}. * *

Replaces the prior {@code LRUCache}. The win is on the * steady-state hit path: a snapshot lookup is a 64-bit hash compute + bucket walk + field-wise - * {@code matches}, with no {@link MetricKey} allocation and no UTF8 cache lookups. The canonical - * {@link MetricKey} (with UTF8-encoded forms) is only built once per unique key, at insert time, - * and lives on the {@link AggregateEntry}. + * {@code matches}, with no per-snapshot {@link AggregateEntry} allocation and no UTF8 cache + * lookups. The UTF8-encoded forms (formerly held on {@code MetricKey}) live on the {@link + * AggregateEntry} itself and are built once per unique key at insert time. * *

Not thread-safe. The aggregator thread is the sole writer; {@link #clear()} must be * routed through the inbox rather than called from arbitrary threads. @@ -53,8 +53,7 @@ AggregateMetric findOrInsert(SpanSnapshot snapshot) { if (size >= maxAggregates && !evictOneStale()) { return null; } - AggregateEntry entry = - new AggregateEntry(MetricKeys.fromSnapshot(snapshot), snapshot, new AggregateMetric()); + AggregateEntry entry = AggregateEntry.forSnapshot(snapshot, new AggregateMetric()); entry.setNext(buckets[bucketIndex]); buckets[bucketIndex] = entry; size++; @@ -88,11 +87,10 @@ private boolean evictOneStale() { return false; } - void forEach(BiConsumer consumer) { + void forEach(Consumer consumer) { for (int i = 0; i < buckets.length; i++) { for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { - AggregateEntry entry = (AggregateEntry) e; - consumer.accept(entry.key, entry.aggregate); + consumer.accept((AggregateEntry) e); } } } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index d0262f328f6..b4fc59d5a1d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -136,9 +136,9 @@ private void report(long when, SignalItem signal) { skipped = false; writer.startBucket(aggregates.size(), when, reportingIntervalNanos); aggregates.forEach( - (key, agg) -> { - writer.add(key, agg); - agg.clear(); + entry -> { + writer.add(entry); + entry.aggregate.clear(); }); // note that this may do IO and block writer.finishBucket(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index 79dcf991c10..c675fcb23c4 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -20,12 +20,8 @@ import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; -import datadog.trace.api.Pair; import datadog.trace.api.WellKnownTags; -import datadog.trace.api.cache.DDCache; -import datadog.trace.api.cache.DDCaches; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.common.metrics.SignalItem.ReportSignal; import datadog.trace.common.writer.ddagent.DDAgentApi; import datadog.trace.core.CoreSpan; @@ -40,7 +36,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import org.jctools.queues.MessagePassingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,22 +47,6 @@ public final class ConflatingMetricsAggregator implements MetricsAggregator, Eve private static final Map DEFAULT_HEADERS = Collections.singletonMap(DDAgentApi.DATADOG_META_TRACER_VERSION, DDTraceCoreInfo.VERSION); - static final DDCache SERVICE_NAMES = DDCaches.newFixedSizeCache(32); - - static final DDCache SPAN_KINDS = DDCaches.newFixedSizeCache(16); - static final DDCache< - String, Pair, Function>> - PEER_TAGS_CACHE = - DDCaches.newFixedSizeCache( - 64); // it can be unbounded since those values are returned by the agent and should be - // under control. 64 entries is enough in this case to contain all the peer tags. - static final Function< - String, Pair, Function>> - PEER_TAGS_CACHE_ADDER = - key -> - Pair.of( - DDCaches.newFixedSizeCache(512), - value -> UTF8BytesString.create(key + ":" + value)); private static final CharSequence SYNTHETICS_ORIGIN = "synthetics"; private static final SpanKindFilter METRICS_ELIGIBLE_KINDS = diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java deleted file mode 100644 index 9e2e2098d1f..00000000000 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKey.java +++ /dev/null @@ -1,178 +0,0 @@ -package datadog.trace.common.metrics; - -import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; - -import datadog.trace.api.cache.DDCache; -import datadog.trace.api.cache.DDCaches; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; -import datadog.trace.util.HashingUtils; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -/** The aggregation key for tracked metrics. */ -public final class MetricKey { - static final DDCache RESOURCE_CACHE = DDCaches.newFixedSizeCache(32); - static final DDCache SERVICE_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache SERVICE_SOURCE_CACHE = - DDCaches.newFixedSizeCache(16); - static final DDCache OPERATION_CACHE = DDCaches.newFixedSizeCache(64); - static final DDCache TYPE_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache KIND_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache HTTP_METHOD_CACHE = DDCaches.newFixedSizeCache(8); - static final DDCache HTTP_ENDPOINT_CACHE = - DDCaches.newFixedSizeCache(32); - static final DDCache GRPC_STATUS_CODE_CACHE = - DDCaches.newFixedSizeCache(32); - - private final UTF8BytesString resource; - private final UTF8BytesString service; - private final UTF8BytesString serviceSource; - private final UTF8BytesString operationName; - private final UTF8BytesString type; - private final int httpStatusCode; - private final boolean synthetics; - private final int hash; - private final boolean isTraceRoot; - private final UTF8BytesString spanKind; - private final List peerTags; - private final UTF8BytesString httpMethod; - private final UTF8BytesString httpEndpoint; - private final UTF8BytesString grpcStatusCode; - - public MetricKey( - CharSequence resource, - CharSequence service, - CharSequence operationName, - CharSequence serviceSource, - CharSequence type, - int httpStatusCode, - boolean synthetics, - boolean isTraceRoot, - CharSequence spanKind, - List peerTags, - CharSequence httpMethod, - CharSequence httpEndpoint, - CharSequence grpcStatusCode) { - this.resource = null == resource ? EMPTY : utf8(RESOURCE_CACHE, resource); - this.service = null == service ? EMPTY : utf8(SERVICE_CACHE, service); - this.serviceSource = null == serviceSource ? null : utf8(SERVICE_SOURCE_CACHE, serviceSource); - this.operationName = null == operationName ? EMPTY : utf8(OPERATION_CACHE, operationName); - this.type = null == type ? EMPTY : utf8(TYPE_CACHE, type); - this.httpStatusCode = httpStatusCode; - this.synthetics = synthetics; - this.isTraceRoot = isTraceRoot; - this.spanKind = null == spanKind ? EMPTY : utf8(KIND_CACHE, spanKind); - this.peerTags = peerTags == null ? Collections.emptyList() : peerTags; - this.httpMethod = httpMethod == null ? null : utf8(HTTP_METHOD_CACHE, httpMethod); - this.httpEndpoint = httpEndpoint == null ? null : utf8(HTTP_ENDPOINT_CACHE, httpEndpoint); - this.grpcStatusCode = - grpcStatusCode == null ? null : utf8(GRPC_STATUS_CODE_CACHE, grpcStatusCode); - - int tmpHash = 0; - tmpHash = HashingUtils.addToHash(tmpHash, this.isTraceRoot); - tmpHash = HashingUtils.addToHash(tmpHash, this.spanKind); - tmpHash = HashingUtils.addToHash(tmpHash, this.peerTags); - tmpHash = HashingUtils.addToHash(tmpHash, this.resource); - tmpHash = HashingUtils.addToHash(tmpHash, this.service); - tmpHash = HashingUtils.addToHash(tmpHash, this.operationName); - tmpHash = HashingUtils.addToHash(tmpHash, this.type); - tmpHash = HashingUtils.addToHash(tmpHash, this.httpStatusCode); - tmpHash = HashingUtils.addToHash(tmpHash, this.synthetics); - tmpHash = HashingUtils.addToHash(tmpHash, this.serviceSource); - tmpHash = HashingUtils.addToHash(tmpHash, this.httpEndpoint); - tmpHash = HashingUtils.addToHash(tmpHash, this.httpMethod); - tmpHash = HashingUtils.addToHash(tmpHash, this.grpcStatusCode); - this.hash = tmpHash; - } - - static UTF8BytesString utf8(DDCache cache, CharSequence charSeq) { - if (charSeq instanceof UTF8BytesString) { - return (UTF8BytesString) charSeq; - } else { - return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); - } - } - - public UTF8BytesString getResource() { - return resource; - } - - public UTF8BytesString getService() { - return service; - } - - public UTF8BytesString getServiceSource() { - return serviceSource; - } - - public UTF8BytesString getOperationName() { - return operationName; - } - - public UTF8BytesString getType() { - return type; - } - - public int getHttpStatusCode() { - return httpStatusCode; - } - - public boolean isSynthetics() { - return synthetics; - } - - public boolean isTraceRoot() { - return isTraceRoot; - } - - public UTF8BytesString getSpanKind() { - return spanKind; - } - - public List getPeerTags() { - return peerTags; - } - - public UTF8BytesString getHttpMethod() { - return httpMethod; - } - - public UTF8BytesString getHttpEndpoint() { - return httpEndpoint; - } - - public UTF8BytesString getGrpcStatusCode() { - return grpcStatusCode; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if ((o instanceof MetricKey)) { - MetricKey metricKey = (MetricKey) o; - return hash == metricKey.hash - && synthetics == metricKey.synthetics - && httpStatusCode == metricKey.httpStatusCode - && resource.equals(metricKey.resource) - && service.equals(metricKey.service) - && operationName.equals(metricKey.operationName) - && type.equals(metricKey.type) - && isTraceRoot == metricKey.isTraceRoot - && spanKind.equals(metricKey.spanKind) - && peerTags.equals(metricKey.peerTags) - && Objects.equals(serviceSource, metricKey.serviceSource) - && Objects.equals(httpMethod, metricKey.httpMethod) - && Objects.equals(httpEndpoint, metricKey.httpEndpoint) - && Objects.equals(grpcStatusCode, metricKey.grpcStatusCode); - } - return false; - } - - @Override - public int hashCode() { - return hash; - } -} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java deleted file mode 100644 index 2e03c3730d3..00000000000 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricKeys.java +++ /dev/null @@ -1,65 +0,0 @@ -package datadog.trace.common.metrics; - -import static datadog.trace.api.Functions.UTF8_ENCODE; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.PEER_TAGS_CACHE_ADDER; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SERVICE_NAMES; -import static datadog.trace.common.metrics.ConflatingMetricsAggregator.SPAN_KINDS; - -import datadog.trace.api.Pair; -import datadog.trace.api.cache.DDCache; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; - -/** - * Canonicalization helpers for {@link MetricKey}: applies the static {@link - * ConflatingMetricsAggregator#SERVICE_NAMES} / {@link ConflatingMetricsAggregator#SPAN_KINDS} / - * {@link ConflatingMetricsAggregator#PEER_TAGS_CACHE} caches to a {@link SpanSnapshot}. - * - *

Called only on a true miss in {@link AggregateTable}, so the CHM lookups inside the DDCaches - * happen once per unique key rather than once per snapshot. - */ -final class MetricKeys { - private MetricKeys() {} - - static MetricKey fromSnapshot(SpanSnapshot s) { - return new MetricKey( - s.resourceName, - SERVICE_NAMES.computeIfAbsent(s.serviceName, UTF8_ENCODE), - s.operationName, - s.serviceNameSource, - s.spanType, - s.httpStatusCode, - s.synthetic, - s.traceRoot, - SPAN_KINDS.computeIfAbsent(s.spanKind, UTF8BytesString::create), - materializePeerTags(s.peerTagPairs), - s.httpMethod, - s.httpEndpoint, - s.grpcStatusCode); - } - - private static List materializePeerTags(String[] pairs) { - if (pairs == null || pairs.length == 0) { - return Collections.emptyList(); - } - if (pairs.length == 2) { - // single-entry fast path (matches the original singletonList shape for INTERNAL spans) - return Collections.singletonList(encodePeerTag(pairs[0], pairs[1])); - } - List tags = new ArrayList<>(pairs.length / 2); - for (int i = 0; i < pairs.length; i += 2) { - tags.add(encodePeerTag(pairs[i], pairs[i + 1])); - } - return tags; - } - - private static UTF8BytesString encodePeerTag(String name, String value) { - final Pair, Function> - cacheAndCreator = PEER_TAGS_CACHE.computeIfAbsent(name, PEER_TAGS_CACHE_ADDER); - return cacheAndCreator.getLeft().computeIfAbsent(value, cacheAndCreator.getRight()); - } -} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java index fa26ed2e5db..c31825f6af8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java @@ -3,7 +3,11 @@ public interface MetricWriter { void startBucket(int metricCount, long start, long duration); - void add(MetricKey key, AggregateMetric aggregate); + /** + * Serialize one aggregate. The {@link AggregateEntry} carries both the label fields (resource, + * service, span.kind, peer tags, etc.) and the {@link AggregateMetric} counters being reported. + */ + void add(AggregateEntry entry); void finishBucket(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java index 0f84964e9db..ba6ae6c2699 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java @@ -142,12 +142,13 @@ public void startBucket(int metricCount, long start, long duration) { } @Override - public void add(MetricKey key, AggregateMetric aggregate) { + public void add(AggregateEntry entry) { + final AggregateMetric aggregate = entry.aggregate; // Calculate dynamic map size based on optional fields - final boolean hasHttpMethod = key.getHttpMethod() != null; - final boolean hasHttpEndpoint = key.getHttpEndpoint() != null; - final boolean hasServiceSource = key.getServiceSource() != null; - final boolean hasGrpcStatusCode = key.getGrpcStatusCode() != null; + final boolean hasHttpMethod = entry.getHttpMethod() != null; + final boolean hasHttpEndpoint = entry.getHttpEndpoint() != null; + final boolean hasServiceSource = entry.getServiceSource() != null; + final boolean hasGrpcStatusCode = entry.getGrpcStatusCode() != null; final int mapSize = 15 + (hasServiceSource ? 1 : 0) @@ -158,31 +159,31 @@ public void add(MetricKey key, AggregateMetric aggregate) { writer.startMap(mapSize); writer.writeUTF8(NAME); - writer.writeUTF8(key.getOperationName()); + writer.writeUTF8(entry.getOperationName()); writer.writeUTF8(SERVICE); - writer.writeUTF8(key.getService()); + writer.writeUTF8(entry.getService()); writer.writeUTF8(RESOURCE); - writer.writeUTF8(key.getResource()); + writer.writeUTF8(entry.getResource()); writer.writeUTF8(TYPE); - writer.writeUTF8(key.getType()); + writer.writeUTF8(entry.getType()); writer.writeUTF8(HTTP_STATUS_CODE); - writer.writeInt(key.getHttpStatusCode()); + writer.writeInt(entry.getHttpStatusCode()); writer.writeUTF8(SYNTHETICS); - writer.writeBoolean(key.isSynthetics()); + writer.writeBoolean(entry.isSynthetics()); writer.writeUTF8(IS_TRACE_ROOT); - writer.writeInt(key.isTraceRoot() ? TRISTATE_TRUE : TRISTATE_FALSE); + writer.writeInt(entry.isTraceRoot() ? TRISTATE_TRUE : TRISTATE_FALSE); writer.writeUTF8(SPAN_KIND); - writer.writeUTF8(key.getSpanKind()); + writer.writeUTF8(entry.getSpanKind()); writer.writeUTF8(PEER_TAGS); - final List peerTags = key.getPeerTags(); + final List peerTags = entry.getPeerTags(); writer.startArray(peerTags.size()); for (UTF8BytesString peerTag : peerTags) { @@ -191,24 +192,24 @@ public void add(MetricKey key, AggregateMetric aggregate) { if (hasServiceSource) { writer.writeUTF8(SERVICE_SOURCE); - writer.writeUTF8(key.getServiceSource()); + writer.writeUTF8(entry.getServiceSource()); } // Only include HTTPMethod if present if (hasHttpMethod) { writer.writeUTF8(HTTP_METHOD); - writer.writeUTF8(key.getHttpMethod()); + writer.writeUTF8(entry.getHttpMethod()); } // Only include HTTPEndpoint if present if (hasHttpEndpoint) { writer.writeUTF8(HTTP_ENDPOINT); - writer.writeUTF8(key.getHttpEndpoint()); + writer.writeUTF8(entry.getHttpEndpoint()); } // Only include GRPCStatusCode if present (rpc-type spans) if (hasGrpcStatusCode) { writer.writeUTF8(GRPC_STATUS_CODE); - writer.writeUTF8(key.getGrpcStatusCode()); + writer.writeUTF8(entry.getGrpcStatusCode()); } writer.writeUTF8(HITS); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java index 2816fad0411..b7f81712945 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java @@ -2,7 +2,8 @@ /** * Immutable per-span value posted from the producer to the aggregator thread. Carries the raw - * inputs the aggregator needs to build a {@link MetricKey} and update an {@link AggregateMetric}. + * inputs the aggregator needs to build an {@link AggregateEntry} and update its {@link + * AggregateMetric}. * *

All cache-canonicalization (service-name, span-kind, peer-tag string interning) happens on the * aggregator thread; the producer just shuffles references. diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy index dedd0bae75b..4dd0155443a 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy @@ -119,7 +119,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( null, "service", "operation", @@ -133,8 +133,8 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) >> { MetricKey key, AggregateMetric value -> - value.getHitCount() == 1 && value.getTopLevelCount() == 1 && value.getDuration() == 100 + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -165,7 +165,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -179,8 +179,8 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) >> { MetricKey key, AggregateMetric value -> - value.getHitCount() == 1 && value.getTopLevelCount() == 1 && value.getDuration() == 100 + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -217,7 +217,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered == statsComputed (statsComputed ? 1 : 0) * writer.startBucket(1, _, _) (statsComputed ? 1 : 0) * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -231,9 +231,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { httpMethod, httpEndpoint, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 0 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + } (statsComputed ? 1 : 0) * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -279,7 +279,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(2, _, _) 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -293,11 +293,11 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 0 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + } 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -311,9 +311,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 0 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -344,7 +344,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -358,9 +358,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 0 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -396,7 +396,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -410,9 +410,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getTopLevelCount() == topLevelCount && value.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == topLevelCount && e.aggregate.getDuration() == 100 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -455,7 +455,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.finishBucket() >> { latch.countDown() } 1 * writer.startBucket(2, _, SECONDS.toNanos(reportingInterval)) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -469,10 +469,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == count && value.getDuration() == count * duration - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration + } + 1 * writer.add(AggregateEntry.of( "resource2", "service2", "operation2", @@ -486,9 +486,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == count && value.getDuration() == count * duration * 2 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration * 2 + } cleanup: aggregator.close() @@ -526,7 +526,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should aggregate into single metric" latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -540,9 +540,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == count && value.getDuration() == count * duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration + } 1 * writer.finishBucket() >> { latch.countDown() } when: "publish spans with different endpoints" @@ -567,7 +567,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create separate metrics for each endpoint/method combination" latchTriggered2 1 * writer.startBucket(3, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -581,10 +581,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -598,10 +598,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/orders/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 2 - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -615,9 +615,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "POST", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 3 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 3 + } 1 * writer.finishBucket() >> { latch2.countDown() } cleanup: @@ -665,7 +665,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create 4 separate metrics" latchTriggered 1 * writer.startBucket(4, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -679,10 +679,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -696,10 +696,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "POST", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 2 - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -713,10 +713,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 3 - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 3 + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -730,9 +730,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/orders/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 4 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 4 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -769,7 +769,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create separate metric keys for spans with and without HTTP tags" latchTriggered 1 * writer.startBucket(2, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -783,10 +783,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -800,9 +800,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration * 2 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -837,7 +837,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create the different metric keys for spans with and without sources" latchTriggered 1 * writer.startBucket(2, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -851,10 +851,10 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 2 && value.getDuration() == 2 * duration - }) - 1 * writer.add(new MetricKey( + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 2 && e.aggregate.getDuration() == 2 * duration + } + 1 * writer.add(AggregateEntry.of( "resource", "service", "operation", @@ -868,9 +868,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -908,7 +908,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(10, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 10; ++i) { - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service" + i, "operation", @@ -922,11 +922,11 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) >> { MetricKey key, AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration } } - 0 * writer.add(new MetricKey( + 0 * writer.add(AggregateEntry.of( "resource", "service10", "operation", @@ -940,7 +940,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) + )) 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -1055,7 +1055,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service" + i, "operation", @@ -1069,9 +1069,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1090,7 +1090,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(4, _, SECONDS.toNanos(reportingInterval)) for (int i = 1; i < 5; ++i) { - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service" + i, "operation", @@ -1104,11 +1104,11 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } } - 0 * writer.add(new MetricKey( + 0 * writer.add(AggregateEntry.of( "resource", "service0", "operation", @@ -1122,7 +1122,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) + )) 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -1157,7 +1157,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service" + i, "operation", @@ -1171,9 +1171,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1183,7 +1183,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "aggregate not updated in cycle is not reported" 0 * writer.finishBucket() 0 * writer.startBucket(_, _, _) - 0 * writer.add(_, _) + 0 * writer.add(_) cleanup: aggregator.close() @@ -1216,7 +1216,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(1)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "resource", "service" + i, "operation", @@ -1230,9 +1230,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric value -> - value.getHitCount() == 1 && value.getDuration() == duration - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1383,7 +1383,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -1397,9 +1397,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 1 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -1438,7 +1438,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -1452,9 +1452,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 3 && aggregateMetric.getTopLevelCount() == 3 && aggregateMetric.getDuration() == 450 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 3 && e.aggregate.getTopLevelCount() == 3 && e.aggregate.getDuration() == 450 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -1493,7 +1493,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(3, _, _) 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -1507,11 +1507,11 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "GET", "/api/users/:id", null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 1 && aggregateMetric.getDuration() == 100 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + } 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -1525,11 +1525,11 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "POST", "/api/orders", null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 1 && aggregateMetric.getDuration() == 200 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 200 + } 1 * writer.add( - new MetricKey( + AggregateEntry.of( "resource", "service", "operation", @@ -1543,9 +1543,9 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), { AggregateMetric aggregateMetric -> - aggregateMetric.getHitCount() == 1 && aggregateMetric.getTopLevelCount() == 1 && aggregateMetric.getDuration() == 150 - }) + )) >> { AggregateEntry e -> + e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 150 + } 1 * writer.finishBucket() >> { latch.countDown() } cleanup: @@ -1581,7 +1581,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(3, _, _) - 1 * writer.add(new MetricKey( + 1 * writer.add(AggregateEntry.of( "grpc.service/Method", "service", "grpc.server", @@ -1595,8 +1595,8 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, "0" - ), _) - 1 * writer.add(new MetricKey( + )) + 1 * writer.add(AggregateEntry.of( "grpc.service/Method", "service", "grpc.server", @@ -1610,8 +1610,8 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, "5" - ), _) - 1 * writer.add(new MetricKey( + )) + 1 * writer.add(AggregateEntry.of( "GET /api", "service", "http.request", @@ -1625,7 +1625,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null, null - ), _) + )) 1 * writer.finishBucket() >> { latch.countDown() } cleanup: diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy index 3ff81de9851..08f0f7cbb92 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy @@ -7,7 +7,6 @@ import static java.util.concurrent.TimeUnit.SECONDS import datadog.metrics.api.Histograms import datadog.metrics.impl.DDSketchHistograms import datadog.trace.api.Config -import datadog.trace.api.Pair import datadog.trace.api.ProcessTags import datadog.trace.api.WellKnownTags import datadog.trace.api.git.CommitInfo @@ -26,6 +25,30 @@ class SerializingMetricWriterTest extends DDSpecification { Histograms.register(DDSketchHistograms.FACTORY) } + /** Build an {@link AggregateEntry} with a pre-recorded duration count. */ + private static AggregateEntry entry( + CharSequence resource, + CharSequence service, + CharSequence operationName, + CharSequence serviceSource, + CharSequence type, + int httpStatusCode, + boolean synthetic, + boolean traceRoot, + CharSequence spanKind, + List peerTags, + CharSequence httpMethod, + CharSequence httpEndpoint, + CharSequence grpcStatusCode, + int hitCount) { + AggregateEntry e = AggregateEntry.of( + resource, service, operationName, serviceSource, type, + httpStatusCode, synthetic, traceRoot, spanKind, peerTags, + httpMethod, httpEndpoint, grpcStatusCode) + e.aggregate.recordDurations(hitCount, new AtomicLongArray(1L)) + return e + } + def "should produce correct message #iterationIndex with process tags enabled #withProcessTags" () { setup: if (!withProcessTags) { @@ -40,8 +63,8 @@ class SerializingMetricWriterTest extends DDSpecification { when: writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) + for (AggregateEntry e : content) { + writer.add(e) } writer.finishBucket() @@ -55,88 +78,40 @@ class SerializingMetricWriterTest extends DDSpecification { where: content << [ [ - Pair.of( - new MetricKey( - "resource1", - "service1", - "operation1", - null, - "type", - 0, - false, - false, - "client", + entry( + "resource1", "service1", "operation1", null, "type", 0, + false, false, "client", [ UTF8BytesString.create("country:canada"), UTF8BytesString.create("georegion:amer"), UTF8BytesString.create("peer.service:remote-service") ], - null, - null, - null - ), - new AggregateMetric().recordDurations(10, new AtomicLongArray(1L)) - ), - Pair.of( - new MetricKey( - "resource2", - "service2", - "operation2", - null, - "type2", - 200, - true, - false, - "producer", + null, null, null, + 10), + entry( + "resource2", "service2", "operation2", null, "type2", 200, + true, false, "producer", [ UTF8BytesString.create("country:canada"), UTF8BytesString.create("georegion:amer"), UTF8BytesString.create("peer.service:remote-service") ], - null, - null, - null - ), - new AggregateMetric().recordDurations(9, new AtomicLongArray(1L)) - ), - Pair.of( - new MetricKey( - "GET /api/users/:id", - "web-service", - "http.request", - null, - "web", - 200, - false, - true, - "server", + null, null, null, + 9), + entry( + "GET /api/users/:id", "web-service", "http.request", null, "web", 200, + false, true, "server", [], - "GET", - "/api/users/:id", - null - ), - new AggregateMetric().recordDurations(5, new AtomicLongArray(1L)) - ) + null, null, null, + 5) ], (0..10000).collect({ i -> - Pair.of( - new MetricKey( - "resource" + i, - "service" + i, - "operation" + i, - null, - "type", - 0, - false, - false, - "producer", + entry( + "resource" + i, "service" + i, "operation" + i, null, "type", 0, + false, false, "producer", [UTF8BytesString.create("messaging.destination:dest" + i)], - null, - null, - null - ), - new AggregateMetric().recordDurations(10, new AtomicLongArray(1L)) - ) + null, null, null, + 10) }) ] withProcessTags << [true, false] @@ -148,22 +123,18 @@ class SerializingMetricWriterTest extends DDSpecification { long duration = SECONDS.toNanos(10) WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - // Create keys with different combinations of HTTP fields - def keyWithNoSource = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null) - def keyWithSource = new MetricKey("resource", "service", "operation", "source", "type", 200, false, false, "server", [], "POST", null, null) + def entryNoSource = entry("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null, 1) + def entryWithSource = entry("resource", "service", "operation", "source", "type", 200, false, false, "server", [], "POST", null, null, 1) - def content = [ - Pair.of(keyWithNoSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - ] + def content = [entryNoSource, entryWithSource] ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) when: writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) + for (AggregateEntry e : content) { + writer.add(e) } writer.finishBucket() @@ -177,34 +148,25 @@ class SerializingMetricWriterTest extends DDSpecification { long duration = SECONDS.toNanos(10) WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - // Create keys with different combinations of HTTP fields - def keyWithBoth = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null) - def keyWithMethodOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "POST", null,null) - def keyWithEndpointOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], null, "/api/orders",null) - def keyWithNeither = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "client", [], null, null, null) - - def content = [ - Pair.of(keyWithBoth, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithMethodOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithEndpointOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithNeither, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))) - ] + def entryWithBoth = entry("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null, 1) + def entryWithMethodOnly = entry("resource", "service", "operation", null, "type", 200, false, false, "server", [], "POST", null, null, 1) + def entryWithEndpointOnly = entry("resource", "service", "operation", null, "type", 200, false, false, "server", [], null, "/api/orders", null, 1) + def entryWithNeither = entry("resource", "service", "operation", null, "type", 200, false, false, "client", [], null, null, null, 1) + + def content = [entryWithBoth, entryWithMethodOnly, entryWithEndpointOnly, entryWithNeither] ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) when: writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) + for (AggregateEntry e : content) { + writer.add(e) } writer.finishBucket() then: sink.validatedInput() - // Test passes if validation in ValidatingSink succeeds - // ValidatingSink verifies that map size matches actual number of fields - // and that HTTPMethod/HTTPEndpoint are only present when non-empty } def "add git sha commit info when sha commit is #shaCommit"() { @@ -216,40 +178,63 @@ class SerializingMetricWriterTest extends DDSpecification { long duration = SECONDS.toNanos(10) WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - // Create keys with different combinations of HTTP fields - def key = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null) + def e = entry("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null, 1) - def content = [Pair.of(key, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))),] + def content = [e] ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128, gitInfoProvider) when: - writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) + for (AggregateEntry entryItem : content) { + writer.add(entryItem) } writer.finishBucket() then: - sink.validatedInput() where: shaCommit << [null, "123456"] } + def "GRPCStatusCode field is present in payload for rpc-type spans"() { + setup: + long startTime = MILLISECONDS.toNanos(System.currentTimeMillis()) + long duration = SECONDS.toNanos(10) + WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") + + def entryWithGrpc = entry("grpc.service/Method", "grpc-service", "grpc.server", null, "rpc", 0, false, false, "server", [], null, null, "OK", 1) + def entryWithGrpcError = entry("grpc.service/Method", "grpc-service", "grpc.server", null, "rpc", 0, false, false, "client", [], null, null, "NOT_FOUND", 1) + def entryWithoutGrpc = entry("resource", "service", "operation", null, "web", 200, false, false, "server", [], null, null, null, 1) + + def content = [entryWithGrpc, entryWithGrpcError, entryWithoutGrpc] + + ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) + SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) + + when: + writer.startBucket(content.size(), startTime, duration) + for (AggregateEntry e : content) { + writer.add(e) + } + writer.finishBucket() + + then: + sink.validatedInput() + } + static class ValidatingSink implements Sink { private final WellKnownTags wellKnownTags private final long startTimeNanos private final long duration private boolean validated = false - private List> content + private List content ValidatingSink(WellKnownTags wellKnownTags, long startTimeNanos, long duration, - List> content) { + List content) { this.wellKnownTags = wellKnownTags this.startTimeNanos = startTimeNanos this.duration = duration @@ -298,70 +283,69 @@ class SerializingMetricWriterTest extends DDSpecification { assert unpacker.unpackString() == "Stats" int statCount = unpacker.unpackArrayHeader() assert statCount == content.size() - for (Pair pair : content) { - MetricKey key = pair.getLeft() - AggregateMetric value = pair.getRight() + for (AggregateEntry entry : content) { + AggregateMetric value = entry.aggregate int metricMapSize = unpacker.unpackMapHeader() // Calculate expected map size based on optional fields - boolean hasHttpMethod = key.getHttpMethod() != null - boolean hasHttpEndpoint = key.getHttpEndpoint() != null - boolean hasServiceSource = key.getServiceSource() != null - boolean hasGrpcStatusCode = key.getGrpcStatusCode() != null + boolean hasHttpMethod = entry.getHttpMethod() != null + boolean hasHttpEndpoint = entry.getHttpEndpoint() != null + boolean hasServiceSource = entry.getServiceSource() != null + boolean hasGrpcStatusCode = entry.getGrpcStatusCode() != null int expectedMapSize = 15 + (hasServiceSource ? 1 : 0) + (hasHttpMethod ? 1 : 0) + (hasHttpEndpoint ? 1 : 0) + (hasGrpcStatusCode ? 1 : 0) assert metricMapSize == expectedMapSize int elementCount = 0 assert unpacker.unpackString() == "Name" - assert unpacker.unpackString() == key.getOperationName() as String + assert unpacker.unpackString() == entry.getOperationName() as String ++elementCount assert unpacker.unpackString() == "Service" - assert unpacker.unpackString() == key.getService() as String + assert unpacker.unpackString() == entry.getService() as String ++elementCount assert unpacker.unpackString() == "Resource" - assert unpacker.unpackString() == key.getResource() as String + assert unpacker.unpackString() == entry.getResource() as String ++elementCount assert unpacker.unpackString() == "Type" - assert unpacker.unpackString() == key.getType() as String + assert unpacker.unpackString() == entry.getType() as String ++elementCount assert unpacker.unpackString() == "HTTPStatusCode" - assert unpacker.unpackInt() == key.getHttpStatusCode() + assert unpacker.unpackInt() == entry.getHttpStatusCode() ++elementCount assert unpacker.unpackString() == "Synthetics" - assert unpacker.unpackBoolean() == key.isSynthetics() + assert unpacker.unpackBoolean() == entry.isSynthetics() ++elementCount assert unpacker.unpackString() == "IsTraceRoot" - assert unpacker.unpackInt() == (key.isTraceRoot() ? TriState.TRUE.serialValue : TriState.FALSE.serialValue) + assert unpacker.unpackInt() == (entry.isTraceRoot() ? TriState.TRUE.serialValue : TriState.FALSE.serialValue) ++elementCount assert unpacker.unpackString() == "SpanKind" - assert unpacker.unpackString() == key.getSpanKind() as String + assert unpacker.unpackString() == entry.getSpanKind() as String ++elementCount assert unpacker.unpackString() == "PeerTags" int peerTagsLength = unpacker.unpackArrayHeader() - assert peerTagsLength == key.getPeerTags().size() + assert peerTagsLength == entry.getPeerTags().size() for (int i = 0; i < peerTagsLength; i++) { def unpackedPeerTag = unpacker.unpackString() - assert unpackedPeerTag == key.getPeerTags()[i].toString() + assert unpackedPeerTag == entry.getPeerTags()[i].toString() } ++elementCount // Service source is only present when the service name has been overridden by the tracer if (hasServiceSource) { assert unpacker.unpackString() == "srv_src" - assert unpacker.unpackString() == key.getServiceSource().toString() + assert unpacker.unpackString() == entry.getServiceSource().toString() ++elementCount } // HTTPMethod and HTTPEndpoint are optional - only present if non-null if (hasHttpMethod) { assert unpacker.unpackString() == "HTTPMethod" - assert unpacker.unpackString() == key.getHttpMethod() as String + assert unpacker.unpackString() == entry.getHttpMethod() as String ++elementCount } if (hasHttpEndpoint) { assert unpacker.unpackString() == "HTTPEndpoint" - assert unpacker.unpackString() == key.getHttpEndpoint() as String + assert unpacker.unpackString() == entry.getHttpEndpoint() as String ++elementCount } if (hasGrpcStatusCode) { assert unpacker.unpackString() == "GRPCStatusCode" - assert unpacker.unpackString() == key.getGrpcStatusCode() as String + assert unpacker.unpackString() == entry.getGrpcStatusCode() as String ++elementCount } assert unpacker.unpackString() == "Hits" @@ -397,99 +381,4 @@ class SerializingMetricWriterTest extends DDSpecification { return validated } } - - def "ServiceSource optional in the payload"() { - setup: - long startTime = MILLISECONDS.toNanos(System.currentTimeMillis()) - long duration = SECONDS.toNanos(10) - WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - - // Create keys with different combinations of HTTP fields - def keyWithNoSource = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null) - def keyWithSource = new MetricKey("resource", "service", "operation", "source", "type", 200, false, false, "server", [], "POST", null, null) - - def content = [ - Pair.of(keyWithNoSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithSource, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - ] - - ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) - SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) - - when: - writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) - } - writer.finishBucket() - - then: - sink.validatedInput() - } - - def "GRPCStatusCode field is present in payload for rpc-type spans"() { - setup: - long startTime = MILLISECONDS.toNanos(System.currentTimeMillis()) - long duration = SECONDS.toNanos(10) - WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - - def keyWithGrpc = new MetricKey("grpc.service/Method", "grpc-service", "grpc.server", null, "rpc", 0, false, false, "server", [], null, null, "OK") - def keyWithGrpcError = new MetricKey("grpc.service/Method", "grpc-service", "grpc.server", null, "rpc", 0, false, false, "client", [], null, null, "NOT_FOUND") - def keyWithoutGrpc = new MetricKey("resource", "service", "operation", null, "web", 200, false, false, "server", [], null, null, null) - - def content = [ - Pair.of(keyWithGrpc, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithGrpcError, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithoutGrpc, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))) - ] - - ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) - SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) - - when: - writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) - } - writer.finishBucket() - - then: - sink.validatedInput() - } - - def "HTTPMethod and HTTPEndpoint fields are optional in payload"() { - setup: - long startTime = MILLISECONDS.toNanos(System.currentTimeMillis()) - long duration = SECONDS.toNanos(10) - WellKnownTags wellKnownTags = new WellKnownTags("runtimeid", "hostname", "env", "service", "version", "language") - - // Create keys with different combinations of HTTP fields - def keyWithBoth = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "GET", "/api/users", null) - def keyWithMethodOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], "POST", null, null) - def keyWithEndpointOnly = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "server", [], null, "/api/orders", null) - def keyWithNeither = new MetricKey("resource", "service", "operation", null, "type", 200, false, false, "client", [], null, null, null) - - def content = [ - Pair.of(keyWithBoth, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithMethodOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithEndpointOnly, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))), - Pair.of(keyWithNeither, new AggregateMetric().recordDurations(1, new AtomicLongArray(1L))) - ] - - ValidatingSink sink = new ValidatingSink(wellKnownTags, startTime, duration, content) - SerializingMetricWriter writer = new SerializingMetricWriter(wellKnownTags, sink, 128) - - when: - writer.startBucket(content.size(), startTime, duration) - for (Pair pair : content) { - writer.add(pair.getLeft(), pair.getRight()) - } - writer.finishBucket() - - then: - sink.validatedInput() - // Test passes if validation in ValidatingSink succeeds - // ValidatingSink verifies that map size matches actual number of fields - // and that HTTPMethod/HTTPEndpoint are only present when non-empty - } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java index 6c4839e4e4f..44f2b36cb6b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -148,7 +148,7 @@ void forEachVisitsEveryEntry() { table.findOrInsert(snapshot("c", "op", "client")).recordOneDuration(3L | ERROR_TAG); Map visited = new HashMap<>(); - table.forEach((key, agg) -> visited.put(key.getService().toString(), agg.getDuration())); + table.forEach(e -> visited.put(e.getService().toString(), e.aggregate.getDuration())); assertEquals(3, visited.size()); assertEquals(1L, visited.get("a")); @@ -172,17 +172,17 @@ void clearEmptiesTheTable() { } @Test - void canonicalMetricKeyIsBuiltOnInsert() { + void encodedLabelsAreBuiltOnInsert() { AggregateTable table = new AggregateTable(4); - List seen = new ArrayList<>(); + List seen = new ArrayList<>(); table.findOrInsert(snapshot("svc", "op", "client")); - table.forEach((key, agg) -> seen.add(key)); + table.forEach(seen::add); assertEquals(1, seen.size()); - MetricKey k = seen.get(0); - assertEquals("svc", k.getService().toString()); - assertEquals("op", k.getOperationName().toString()); - assertEquals("client", k.getSpanKind().toString()); + AggregateEntry e = seen.get(0); + assertEquals("svc", e.getService().toString()); + assertEquals("op", e.getOperationName().toString()); + assertEquals("client", e.getSpanKind().toString()); } // ---------- helpers ---------- diff --git a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy index 2972ffa2c18..81a476c67c8 100644 --- a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy +++ b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy @@ -8,9 +8,8 @@ import datadog.metrics.impl.DDSketchHistograms import datadog.trace.api.Config import datadog.trace.api.WellKnownTags import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString -import datadog.trace.common.metrics.AggregateMetric +import datadog.trace.common.metrics.AggregateEntry import datadog.trace.common.metrics.EventListener -import datadog.trace.common.metrics.MetricKey import datadog.trace.common.metrics.OkHttpSink import datadog.trace.common.metrics.SerializingMetricWriter import java.util.concurrent.CopyOnWriteArrayList @@ -39,14 +38,12 @@ class MetricsIntegrationTest extends AbstractTraceAgentTest { sink ) writer.startBucket(2, System.nanoTime(), SECONDS.toNanos(10)) - writer.add( - new MetricKey("resource1", "service1", "operation1", null, "sql", 0, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null), - new AggregateMetric().recordDurations(5, new AtomicLongArray(2, 1, 2, 250, 4, 5)) - ) - writer.add( - new MetricKey("resource2", "service2", "operation2", null, "web", 200, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null), - new AggregateMetric().recordDurations(10, new AtomicLongArray(1, 1, 200, 2, 3, 4, 5, 6, 7, 8, 9)) - ) + def entry1 = AggregateEntry.of("resource1", "service1", "operation1", null, "sql", 0, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) + entry1.aggregate.recordDurations(5, new AtomicLongArray(2, 1, 2, 250, 4, 5)) + writer.add(entry1) + def entry2 = AggregateEntry.of("resource2", "service2", "operation2", null, "web", 200, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) + entry2.aggregate.recordDurations(10, new AtomicLongArray(1, 1, 200, 2, 3, 4, 5, 6, 7, 8, 9)) + writer.add(entry2) writer.finishBucket() then: From 46a905567fa513f86b7d392a7dd7ad80f6e5c1d1 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 15:40:00 -0400 Subject: [PATCH 05/70] Add unit tests for Hashtable and LongHashingUtils LongHashingUtilsTest (14 cases): - hashCodeX null sentinel + non-null pass-through - all primitive hash() overloads match the boxed Java hashCodes - hash(Object...) 2/3/4/5-arg overloads match the chained addToHash formula they are documented to constant-fold to - addToHash(long, primitive) overloads match the Object-version - linear-accumulation invariant (31 * h + v) holds across a sequence - iterable / deprecated int[] / deprecated Object[] variants match chained addToHash - intHash treats null as 0 (observable via hash(null, "x")) HashtableTest (24 cases across 5 nested classes): - D1: insert/get/remove/insertOrReplace/clear/forEach, in-place value mutation, null-key handling, hash-collision chaining with disambig- uating equals, remove-from-collided-chain leaves siblings intact - D2: pair-key identity, remove(pair), insertOrReplace matches on both parts, forEach - Support: capacity rounds up to a power of two, bucketIndex stays in range across a wide hash sample, clear nulls every slot - BucketIterator: walks only matching-hash entries in a chain, throws NoSuchElementException when exhausted - MutatingBucketIterator: remove from head-of-chain unlinks, replace swaps the entry while preserving chain, remove() without prior next() throws IllegalStateException Tests live in internal-api/src/test/java/datadog/trace/util and use the already-present JUnit 5 setup. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/util/HashtableTest.java | 465 ++++++++++++++++++ .../trace/util/LongHashingUtilsTest.java | 160 ++++++ 2 files changed, 625 insertions(+) create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableTest.java create mode 100644 internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java new file mode 100644 index 00000000000..67c99c0d08d --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -0,0 +1,465 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.util.Hashtable.BucketIterator; +import datadog.trace.util.Hashtable.MutatingBucketIterator; +import datadog.trace.util.Hashtable.Support; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class HashtableTest { + + // ============ D1 ============ + + @Nested + class D1Tests { + + @Test + void emptyTableLookupReturnsNull() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get("missing")); + assertEquals(0, table.size()); + } + + @Test + void insertedEntryIsRetrievable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry e = new StringIntEntry("foo", 1); + table.insert(e); + assertEquals(1, table.size()); + assertSame(e, table.get("foo")); + } + + @Test + void multipleInsertsRetrievableSeparately() { + Hashtable.D1 table = new Hashtable.D1<>(16); + StringIntEntry a = new StringIntEntry("alpha", 1); + StringIntEntry b = new StringIntEntry("beta", 2); + StringIntEntry c = new StringIntEntry("gamma", 3); + table.insert(a); + table.insert(b); + table.insert(c); + assertEquals(3, table.size()); + assertSame(a, table.get("alpha")); + assertSame(b, table.get("beta")); + assertSame(c, table.get("gamma")); + } + + @Test + void inPlaceMutationVisibleViaSubsequentGet() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("counter", 0)); + for (int i = 0; i < 10; i++) { + StringIntEntry e = table.get("counter"); + e.value++; + } + assertEquals(10, table.get("counter").value); + } + + @Test + void removeUnlinksEntryAndDecrementsSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + assertEquals(2, table.size()); + + StringIntEntry removed = table.remove("a"); + assertNotNull(removed); + assertEquals("a", removed.key); + assertEquals(1, table.size()); + assertNull(table.get("a")); + assertNotNull(table.get("b")); + } + + @Test + void removeNonexistentReturnsNullAndDoesNotChangeSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + assertNull(table.remove("nope")); + assertEquals(1, table.size()); + } + + @Test + void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry first = new StringIntEntry("k", 1); + assertNull(table.insertOrReplace(first), "fresh insert returns null"); + assertEquals(1, table.size()); + + StringIntEntry second = new StringIntEntry("k", 2); + assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); + assertEquals(1, table.size()); + assertSame(second, table.get("k"), "new entry visible after replace"); + } + + @Test + void clearEmptiesTheTable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.clear(); + assertEquals(0, table.size()); + assertNull(table.get("a")); + // Reinsertion works after clear + table.insert(new StringIntEntry("a", 99)); + assertEquals(99, table.get("a").value); + } + + @Test + void forEachVisitsEveryInsertedEntry() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + Map seen = new HashMap<>(); + table.forEach(e -> seen.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(1, seen.get("a")); + assertEquals(2, seen.get("b")); + assertEquals(3, seen.get("c")); + } + + @Test + void nullKeyIsPermittedAndDistinctFromAbsent() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get(null)); + StringIntEntry nullKeyed = new StringIntEntry(null, 7); + table.insert(nullKeyed); + assertSame(nullKeyed, table.get(null)); + assertEquals(1, table.size()); + assertSame(nullKeyed, table.remove(null)); + assertEquals(0, table.size()); + } + + @Test + void hashCollisionsResolveByEquality() { + // Force two distinct keys with the same hashCode -- the chain must still distinguish them + // via matches(). + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); + table.insert(e1); + table.insert(e2); + assertEquals(2, table.size()); + assertSame(e1, table.get(k1)); + assertSame(e2, table.get(k2)); + } + + @Test + void hashCollisionsThenRemoveLeavesOtherIntact() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + table.remove(k2); + assertEquals(2, table.size()); + assertNotNull(table.get(k1)); + assertNull(table.get(k2)); + assertNotNull(table.get(k3)); + } + } + + // ============ D2 ============ + + @Nested + class D2Tests { + + @Test + void pairKeysParticipateInIdentity() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + PairEntry bb = new PairEntry("b", 1, 300); + table.insert(ab); + table.insert(ac); + table.insert(bb); + assertEquals(3, table.size()); + assertSame(ab, table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + assertSame(bb, table.get("b", 1)); + assertNull(table.get("a", 3)); + } + + @Test + void removePairUnlinks() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + table.insert(ab); + table.insert(ac); + assertSame(ab, table.remove("a", 1)); + assertEquals(1, table.size()); + assertNull(table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + } + + @Test + void insertOrReplaceMatchesOnBothKeys() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry first = new PairEntry("k", 7, 1); + assertNull(table.insertOrReplace(first)); + PairEntry second = new PairEntry("k", 7, 2); + assertSame(first, table.insertOrReplace(second)); + // Different second-key: should insert new, not replace + PairEntry third = new PairEntry("k", 8, 3); + assertNull(table.insertOrReplace(third)); + assertEquals(2, table.size()); + } + + @Test + void forEachVisitsBothPairs() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + } + + // ============ Support ============ + + @Nested + class SupportTests { + + @Test + void createRoundsCapacityUpToPowerOfTwo() { + // The Hashtable.D1 / D2 size() reflects entries, but the bucket array length is + // a power of two >= requestedCapacity. We can verify indirectly via bucketIndex masking. + Hashtable.Entry[] buckets = Support.create(5); + // Length must be a power of two >= 5 + int len = buckets.length; + assertTrue(len >= 5); + assertEquals(0, len & (len - 1), "length must be a power of two"); + } + + @Test + void bucketIndexIsBoundedByArrayLength() { + Hashtable.Entry[] buckets = Support.create(16); + for (long h : new long[] {0L, 1L, -1L, Long.MIN_VALUE, Long.MAX_VALUE, 12345L}) { + int idx = Support.bucketIndex(buckets, h); + assertTrue(idx >= 0 && idx < buckets.length, "bucketIndex out of range for hash " + h); + } + } + + @Test + void clearNullsAllBuckets() { + Hashtable.Entry[] buckets = Support.create(4); + buckets[0] = new StringIntEntry("x", 1); + buckets[1] = new StringIntEntry("y", 2); + Support.clear(buckets); + for (Hashtable.Entry b : buckets) { + assertNull(b); + } + } + } + + // ============ BucketIterator ============ + + @Nested + class BucketIteratorTests { + + @Test + void walksOnlyMatchingHash() { + // Build a bucket array with two entries that share a bucket but have different hashes. + // Use Hashtable.D1 to seed; then call Support.bucketIterator directly with the matching + // hash and verify it only returns the matching entry. + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. + BucketIterator it = + Support.bucketIterator(extractBuckets(table), 17L); + int count = 0; + while (it.hasNext()) { + assertNotNull(it.next()); + count++; + } + assertEquals(3, count); + } + + @Test + void exhaustedIteratorThrowsNoSuchElement() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("only", 1)); + long h = Hashtable.D1.Entry.hash("only"); + BucketIterator it = Support.bucketIterator(extractBuckets(table), h); + it.next(); + assertFalse(it.hasNext()); + assertThrows(NoSuchElementException.class, it::next); + } + } + + // ============ MutatingBucketIterator ============ + + @Nested + class MutatingBucketIteratorTests { + + @Test + void removeFromHeadOfChainUnlinks() { + // Make three entries with the same hash so they chain in one bucket + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + + MutatingBucketIterator it = + Support.mutatingBucketIterator(extractBuckets(table), 17L); + it.next(); // first match (head of chain in insertion-reverse order) + it.remove(); + // Two should remain + int remaining = 0; + while (it.hasNext()) { + it.next(); + remaining++; + } + assertEquals(2, remaining); + // And the table still finds the survivors via get(...) + // (which entry was the head depends on insertion order; we just verify count + that two + // of the three keys are still retrievable.) + int found = 0; + for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { + if (table.get(k) != null) found++; + } + assertEquals(2, found); + } + + @Test + void replaceSwapsEntryAndPreservesChain() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 1); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 2); + table.insert(e1); + table.insert(e2); + + MutatingBucketIterator it = + Support.mutatingBucketIterator(extractBuckets(table), 17L); + CollidingKeyEntry first = it.next(); + CollidingKeyEntry replacement = new CollidingKeyEntry(first.key, 999); + it.replace(replacement); + // Both entries still in the chain + assertNotNull(table.get(k1)); + assertNotNull(table.get(k2)); + // The replaced one now has value 999 + assertEquals(999, table.get(first.key).value); + } + + @Test + void removeWithoutNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + MutatingBucketIterator it = + Support.mutatingBucketIterator( + extractBuckets(table), Hashtable.D1.Entry.hash("a")); + assertThrows(IllegalStateException.class, it::remove); + } + } + + // ============ test helpers ============ + + /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ + private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { + try { + java.lang.reflect.Field f = Hashtable.D1.class.getDeclaredField("buckets"); + f.setAccessible(true); + return (Hashtable.Entry[]) f.get(table); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** Sort comparator used by tests that want deterministic visit order. */ + @SuppressWarnings("unused") + private static final Comparator BY_KEY = + Comparator.comparing(e -> e.key); + + private static final class StringIntEntry extends Hashtable.D1.Entry { + int value; + + StringIntEntry(String key, int value) { + super(key); + this.value = value; + } + } + + /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ + private static final class CollidingKey { + final String label; + final int hash; + + CollidingKey(String label, int hash) { + this.label = label; + this.hash = hash; + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CollidingKey)) return false; + CollidingKey that = (CollidingKey) o; + return hash == that.hash && label.equals(that.label); + } + + @Override + public String toString() { + return "CollidingKey(" + label + ", " + hash + ")"; + } + } + + private static final class CollidingKeyEntry extends Hashtable.D1.Entry { + int value; + + CollidingKeyEntry(CollidingKey key, int value) { + super(key); + this.value = value; + } + } + + private static final class PairEntry extends Hashtable.D2.Entry { + int value; + + PairEntry(String key1, Integer key2, int value) { + super(key1, key2); + this.value = value; + } + } + + // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning quiet. + @SuppressWarnings("unused") + private static final List UNUSED = new ArrayList<>(); +} diff --git a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java new file mode 100644 index 00000000000..d0053c75b42 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java @@ -0,0 +1,160 @@ +package datadog.trace.util; + +import static datadog.trace.util.LongHashingUtils.addToHash; +import static datadog.trace.util.LongHashingUtils.hash; +import static datadog.trace.util.LongHashingUtils.hashCodeX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.Arrays; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +class LongHashingUtilsTest { + + // ----- single-value overloads ----- + + @Test + void hashCodeXReturnsObjectHashCodeOrSentinelForNull() { + Object o = new Object(); + assertEquals(o.hashCode(), hashCodeX(o)); + assertEquals(Long.MIN_VALUE, hashCodeX(null)); + } + + @Test + void primitiveOverloadsMatchBoxedHashCodes() { + assertEquals(Boolean.hashCode(true), hash(true)); + assertEquals(Boolean.hashCode(false), hash(false)); + assertEquals(Character.hashCode('x'), hash('x')); + assertEquals(Byte.hashCode((byte) 42), hash((byte) 42)); + assertEquals(Short.hashCode((short) -7), hash((short) -7)); + assertEquals(Integer.hashCode(123456), hash(123456)); + assertEquals(123456L, hash(123456L)); + assertEquals(Float.hashCode(3.14f), hash(3.14f)); + assertEquals(Double.doubleToRawLongBits(2.71828), hash(2.71828)); + } + + // ----- multi-arg Object overloads vs chained addToHash ----- + + @Test + void twoArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + assertEquals(addToHash(addToHash(0L, a), b), hash(a, b)); + } + + @Test + void threeArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + assertEquals(addToHash(addToHash(addToHash(0L, a), b), c), hash(a, b, c)); + } + + @Test + void fourArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + Object d = 3.14; + assertEquals( + addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); + } + + @Test + void fiveArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + Object d = 3.14; + Object e = 'q'; + assertEquals( + addToHash(addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), e), + hash(a, b, c, d, e)); + } + + @Test + void multiArgHashHandlesNullsConsistentlyWithChainedAddToHash() { + assertEquals(addToHash(addToHash(0L, (Object) null), "x"), hash(null, "x")); + assertEquals(addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); + } + + @Test + void differentInputsProduceDifferentHashes() { + // Sanity: ordering matters, and distinct values produce distinct results in general. + assertNotEquals(hash("a", "b"), hash("b", "a")); + assertNotEquals(hash("a", "b", "c"), hash("a", "c", "b")); + } + + // ----- addToHash primitive overloads ----- + + @Test + void addToHashPrimitivesMatchObjectVersion() { + long seed = 100L; + assertEquals(addToHash(seed, Boolean.hashCode(true)), addToHash(seed, true)); + assertEquals(addToHash(seed, Character.hashCode('z')), addToHash(seed, 'z')); + assertEquals(addToHash(seed, Byte.hashCode((byte) 9)), addToHash(seed, (byte) 9)); + assertEquals(addToHash(seed, Short.hashCode((short) 5)), addToHash(seed, (short) 5)); + assertEquals(addToHash(seed, Long.hashCode(999_999L)), addToHash(seed, 999_999L)); + assertEquals(addToHash(seed, Float.hashCode(1.5f)), addToHash(seed, 1.5f)); + assertEquals(addToHash(seed, Double.hashCode(2.5d)), addToHash(seed, 2.5d)); + } + + @Test + void addToHashIsLinearAcrossSteps() { + // 31*h + v formula -- verify by accumulating an explicit sequence. + long expected = 0L; + for (int v : new int[] {1, 2, 3, 4, 5}) { + expected = 31L * expected + v; + } + long actual = 0L; + for (int v : new int[] {1, 2, 3, 4, 5}) { + actual = addToHash(actual, v); + } + assertEquals(expected, actual); + } + + // ----- iterable / array versions ----- + + @Test + void hashIterableMatchesChainedAddToHash() { + Iterable values = Arrays.asList("a", 1, true, null); + long expected = 0L; + for (Object o : values) { + expected = addToHash(expected, o); + } + assertEquals(expected, hash(values)); + } + + @Test + @SuppressWarnings("deprecation") + void deprecatedIntArrayHashMatchesChainedAddToHash() { + int[] hashes = new int[] {7, 13, 31, 1024}; + long expected = 0L; + for (int h : hashes) { + expected = addToHash(expected, h); + } + assertEquals(expected, hash(hashes)); + } + + @Test + @SuppressWarnings("deprecation") + void deprecatedObjectArrayHashMatchesChainedAddToHash() { + Object[] objs = new Object[] {"alpha", 7, null, true}; + long expected = 0L; + for (Object o : objs) { + expected = addToHash(expected, o); + } + assertEquals(expected, hash(objs)); + } + + // ----- intHash null behavior is observable via multi-arg overloads ----- + + @Test + void multiArgHashTreatsNullAsZero() { + // hash(Object,Object) feeds intHash(...) which returns 0 for null. + // Verify: hash(null, "x") == 31L*0 + "x".hashCode() + int xHash = Objects.hashCode("x"); + assertEquals(31L * 0 + xHash, hash(null, "x")); + } +} From 031dc8995489deab3e79556b05b269ba66ca185a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:19:35 -0400 Subject: [PATCH 06/70] Apply spotless formatting to Hashtable and LongHashingUtils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the new util/ files in line with google-java-format (tabs → spaces, line wrapping, javadoc list markup) so spotlessCheck passes in CI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 902 +++++++++--------- .../datadog/trace/util/LongHashingUtils.java | 8 +- .../datadog/trace/util/HashtableTest.java | 12 +- .../trace/util/LongHashingUtilsTest.java | 6 +- 4 files changed, 467 insertions(+), 461 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index d7f49dcae00..03dfbd7bf1c 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -7,31 +7,31 @@ import java.util.function.Consumer; /** - * Light weight simple Hashtable system that can be useful when HashMap would - * be unnecessarily heavy. - * - *
    Use cases include... - *
  • primitive keys - *
  • primitive values - *
  • multi-part keys + * Light weight simple Hashtable system that can be useful when HashMap would be unnecessarily + * heavy. + * + *
      + * Use cases include... + *
    • primitive keys + *
    • primitive values + *
    • multi-part keys *
    - * + * * Convenience classes are provided for lower key dimensions. - * - * For higher key dimensions, client code must implement its own class, - * but can still use the support class to ease the implementation complexity. + * + *

    For higher key dimensions, client code must implement its own class, but can still use the + * support class to ease the implementation complexity. */ public abstract class Hashtable { /** - * Internal base class for entries. Stores the precomputed 64-bit keyHash and - * the chain-next pointer used to link colliding entries within a single bucket. + * Internal base class for entries. Stores the precomputed 64-bit keyHash and the chain-next + * pointer used to link colliding entries within a single bucket. * - *

    Subclasses add the actual key field(s) and a {@code matches(...)} method - * tailored to their key arity. See {@link D1.Entry} and {@link D2.Entry}; for - * higher arities, client code can subclass this directly and use {@link Support} - * to drive the table mechanics. + *

    Subclasses add the actual key field(s) and a {@code matches(...)} method tailored to their + * key arity. See {@link D1.Entry} and {@link D2.Entry}; for higher arities, client code can + * subclass this directly and use {@link Support} to drive the table mechanics. */ - public static abstract class Entry { + public abstract static class Entry { public final long keyHash; Entry next = null; @@ -44,169 +44,172 @@ public final void setNext(TEntry next) { } @SuppressWarnings("unchecked") - public final TEntry next() { - return (TEntry)this.next; + public final TEntry next() { + return (TEntry) this.next; } } - + /** * Single-key open hash table with chaining. * - *

    The user supplies an {@link D1.Entry} subclass that carries the key and - * whatever value fields they want to mutate in place, then instantiates this - * class over that entry type. The main advantage over {@code HashMap} - * is that mutating an existing entry's value fields requires no allocation: - * call {@link #get} once and write directly to the returned entry's fields. - * For counter-style workloads this can be several times faster than - * {@code HashMap} and produces effectively zero GC pressure. + *

    The user supplies an {@link D1.Entry} subclass that carries the key and whatever value + * fields they want to mutate in place, then instantiates this class over that entry type. The + * main advantage over {@code HashMap} is that mutating an existing entry's value fields + * requires no allocation: call {@link #get} once and write directly to the returned entry's + * fields. For counter-style workloads this can be several times faster than {@code HashMap} and produces effectively zero GC pressure. * - *

    Capacity is fixed at construction. The table does not resize, so the - * caller is responsible for choosing a capacity appropriate to the working - * set. Actual bucket-array length is rounded up to the next power of two. + *

    Capacity is fixed at construction. The table does not resize, so the caller is responsible + * for choosing a capacity appropriate to the working set. Actual bucket-array length is rounded + * up to the next power of two. * - *

    Null keys are permitted; they collapse to a single bucket via the - * sentinel hash {@link Long#MIN_VALUE} defined in {@link D1.Entry#hash}. + *

    Null keys are permitted; they collapse to a single bucket via the sentinel hash {@link + * Long#MIN_VALUE} defined in {@link D1.Entry#hash}. * - *

    Not thread-safe. Concurrent access (including mixing reads with - * writes) requires external synchronization. + *

    Not thread-safe. Concurrent access (including mixing reads with writes) requires + * external synchronization. * * @param the key type * @param the user's {@link D1.Entry D1.Entry<K>} subclass */ public static final class D1> { - /** - * Abstract base for {@link D1} entries. Subclass to add value fields you - * wish to mutate in place after retrieving the entry via {@link D1#get}. - * - *

    The key is captured at construction and stored alongside its - * precomputed 64-bit hash. {@link #matches(Object)} uses - * {@link Objects#equals} by default; override if a different equality - * semantics is needed (e.g. reference equality for interned keys). - * - * @param the key type - */ - public static abstract class Entry extends Hashtable.Entry { - final K key; - - protected Entry(K key) { - super(hash(key)); - this.key = key; - } - - public boolean matches(Object key) { - return Objects.equals(this.key, key); - } - - public static long hash(Object key) { - return (key == null ) ? Long.MIN_VALUE : key.hashCode(); - } - } - - private final Hashtable.Entry[] buckets; - private int size; - - public D1(int capacity) { - this.buckets = Support.create(capacity); - this.size = 0; - } - - public int size() { - return this.size; - } - - @SuppressWarnings("unchecked") - public TEntry get(K key) { - long keyHash = D1.Entry.hash(key); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key)) return te; - } - } - return null; - } - - public TEntry remove(K key) { - long keyHash = D1.Entry.hash(key); - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(key)) { - iter.remove(); - this.size -= 1; - return curEntry; - } - } - - return null; - } - - public void insert(TEntry newEntry) { + /** + * Abstract base for {@link D1} entries. Subclass to add value fields you wish to mutate in + * place after retrieving the entry via {@link D1#get}. + * + *

    The key is captured at construction and stored alongside its precomputed 64-bit hash. + * {@link #matches(Object)} uses {@link Objects#equals} by default; override if a different + * equality semantics is needed (e.g. reference equality for interned keys). + * + * @param the key type + */ + public abstract static class Entry extends Hashtable.Entry { + final K key; + + protected Entry(K key) { + super(hash(key)); + this.key = key; + } + + public boolean matches(Object key) { + return Objects.equals(this.key, key); + } + + public static long hash(Object key) { + return (key == null) ? Long.MIN_VALUE : key.hashCode(); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D1(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K key) { + long keyHash = D1.Entry.hash(key); Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; + e != null; + e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key)) return te; + } + } + return null; + } + + public TEntry remove(K key) { + long keyHash = D1.Entry.hash(key); + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); Hashtable.Entry curHead = thisBuckets[bucketIndex]; newEntry.setNext(curHead); thisBuckets[bucketIndex] = newEntry; this.size += 1; - } - - public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(newEntry.key)) { - iter.replace(newEntry); - return curEntry; - } - } - - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - this.size += 1; - return null; - } - - public void clear() { - Support.clear(this.buckets); - this.size = 0; - } - - @SuppressWarnings("unchecked") - public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } - } + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } } /** * Two-key (composite-key) hash table with chaining. * - *

    The user supplies a {@link D2.Entry} subclass carrying both key parts - * and any value fields. Compared to {@code HashMap} this avoids the - * per-lookup {@code Pair} (or record) allocation: both key parts are passed - * directly through {@link #get}, {@link #remove}, {@link #insert}, and - * {@link #insertOrReplace}. Combined with in-place value mutation, this - * makes {@code D2} substantially less GC-intensive than the equivalent - * {@code HashMap} for counter-style workloads. + *

    The user supplies a {@link D2.Entry} subclass carrying both key parts and any value fields. + * Compared to {@code HashMap} this avoids the per-lookup {@code Pair} (or record) + * allocation: both key parts are passed directly through {@link #get}, {@link #remove}, {@link + * #insert}, and {@link #insertOrReplace}. Combined with in-place value mutation, this makes + * {@code D2} substantially less GC-intensive than the equivalent {@code HashMap} for + * counter-style workloads. * - *

    Capacity is fixed at construction; the table does not resize. Actual - * bucket-array length is rounded up to the next power of two. + *

    Capacity is fixed at construction; the table does not resize. Actual bucket-array length is + * rounded up to the next power of two. * - *

    Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; - * see {@link D2.Entry#hash(Object, Object)}. + *

    Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; see {@link + * D2.Entry#hash(Object, Object)}. * *

    Not thread-safe. * @@ -215,339 +218,340 @@ public void forEach(Consumer consumer) { * @param the user's {@link D2.Entry D2.Entry<K1, K2>} subclass */ public static final class D2> { - /** - * Abstract base for {@link D2} entries. Subclass to add value fields you - * wish to mutate in place. - * - *

    Both key parts are captured at construction and stored alongside their - * combined 64-bit hash. {@link #matches(Object, Object)} uses - * {@link Objects#equals} pairwise on the two parts. - * - * @param first key type - * @param second key type - */ - public static abstract class Entry extends Hashtable.Entry { - final K1 key1; - final K2 key2; - - protected Entry(K1 key1, K2 key2) { - super(hash(key1, key2)); - this.key1 = key1; - this.key2 = key2; - } - - public boolean matches(K1 key1, K2 key2) { - return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); - } - - public static long hash(Object key1, Object key2) { - return LongHashingUtils.hash(key1, key2); - } - } - - private final Hashtable.Entry[] buckets; - private int size; - - public D2(int capacity) { - this.buckets = Support.create(capacity); - this.size = 0; - } - - public int size() { - return this.size; - } - - @SuppressWarnings("unchecked") - public TEntry get(K1 key1, K2 key2) { - long keyHash = D2.Entry.hash(key1, key2); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key1, key2)) return te; - } - } - return null; - } - - public TEntry remove(K1 key1, K2 key2) { - long keyHash = D2.Entry.hash(key1, key2); - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(key1, key2)) { - iter.remove(); - this.size -= 1; - return curEntry; - } - } - - return null; - } - - public void insert(TEntry newEntry) { + /** + * Abstract base for {@link D2} entries. Subclass to add value fields you wish to mutate in + * place. + * + *

    Both key parts are captured at construction and stored alongside their combined 64-bit + * hash. {@link #matches(Object, Object)} uses {@link Objects#equals} pairwise on the two parts. + * + * @param first key type + * @param second key type + */ + public abstract static class Entry extends Hashtable.Entry { + final K1 key1; + final K2 key2; + + protected Entry(K1 key1, K2 key2) { + super(hash(key1, key2)); + this.key1 = key1; + this.key2 = key2; + } + + public boolean matches(K1 key1, K2 key2) { + return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); + } + + public static long hash(Object key1, Object key2) { + return LongHashingUtils.hash(key1, key2); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D2(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; + e != null; + e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key1, key2)) return te; + } + } + return null; + } + + public TEntry remove(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key1, key2)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); Hashtable.Entry curHead = thisBuckets[bucketIndex]; newEntry.setNext(curHead); thisBuckets[bucketIndex] = newEntry; this.size += 1; - } - - public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(newEntry.key1, newEntry.key2)) { - iter.replace(newEntry); - return curEntry; - } - } - - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - this.size += 1; - return null; - } - - public void clear() { - Support.clear(this.buckets); - this.size = 0; - } - - @SuppressWarnings("unchecked") - public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } - } + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key1, newEntry.key2)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } } /** * Internal building blocks for hash-table operations. * - *

    Used by {@link D1} and {@link D2}, and available to package code that - * wants to assemble its own higher-arity table (3+ key parts) without - * re-implementing the bucket-array mechanics. The typical recipe: + *

    Used by {@link D1} and {@link D2}, and available to package code that wants to assemble its + * own higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The + * typical recipe: * *

      - *
    • Subclass {@link Hashtable.Entry} directly, adding the key fields and - * a {@code matches(...)} method of your chosen arity. + *
    • Subclass {@link Hashtable.Entry} directly, adding the key fields and a {@code + * matches(...)} method of your chosen arity. *
    • Allocate a backing array with {@link #create(int)}. - *
    • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, - * {@link #bucketIterator(Hashtable.Entry[], long)} for read-only chain - * walks, and {@link #mutatingBucketIterator(Hashtable.Entry[], long)} - * when you also need {@code remove} / {@code replace}. + *
    • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, {@link + * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link + * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / + * {@code replace}. *
    • Clear with {@link #clear(Hashtable.Entry[])}. *
    * - *

    All bucket arrays produced by {@link #create(int)} have a power-of-two - * length, so {@link #bucketIndex(Object[], long)} can use a bit mask. + *

    All bucket arrays produced by {@link #create(int)} have a power-of-two length, so {@link + * #bucketIndex(Object[], long)} can use a bit mask. * - *

    Methods on this class are package-private; the class itself is public - * only so that its nested {@link BucketIterator} can be referenced by - * callers in other packages. + *

    Methods on this class are package-private; the class itself is public only so that its + * nested {@link BucketIterator} can be referenced by callers in other packages. */ public static final class Support { - public static final Hashtable.Entry[] create(int capacity) { - return new Entry[sizeFor(capacity)]; - } - - static final int sizeFor(int requestedCapacity) { - int pow; - for ( pow = 1; pow < requestedCapacity; pow *= 2 ); - return pow; - } - - public static final void clear(Hashtable.Entry[] buckets) { - Arrays.fill(buckets, null); - } - - public static final BucketIterator bucketIterator(Hashtable.Entry[] buckets, long keyHash) { - return new BucketIterator(buckets, keyHash); - } - - public static final MutatingBucketIterator mutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { - return new MutatingBucketIterator(buckets, keyHash); - } - - public static final int bucketIndex(Object[] buckets, long keyHash) { - return (int)(keyHash & buckets.length - 1); - } + public static final Hashtable.Entry[] create(int capacity) { + return new Entry[sizeFor(capacity)]; + } + + static final int sizeFor(int requestedCapacity) { + int pow; + for (pow = 1; pow < requestedCapacity; pow *= 2) + ; + return pow; + } + + public static final void clear(Hashtable.Entry[] buckets) { + Arrays.fill(buckets, null); + } + + public static final BucketIterator bucketIterator( + Hashtable.Entry[] buckets, long keyHash) { + return new BucketIterator(buckets, keyHash); + } + + public static final + MutatingBucketIterator mutatingBucketIterator( + Hashtable.Entry[] buckets, long keyHash) { + return new MutatingBucketIterator(buckets, keyHash); + } + + public static final int bucketIndex(Object[] buckets, long keyHash) { + return (int) (keyHash & buckets.length - 1); + } } - + /** - * Read-only iterator over entries in a single bucket whose {@code keyHash} - * matches a specific search hash. Cheaper than {@link MutatingBucketIterator} - * because it does not track the previous-node pointers required for - * splicing — use it when you only need to walk the chain. + * Read-only iterator over entries in a single bucket whose {@code keyHash} matches a specific + * search hash. Cheaper than {@link MutatingBucketIterator} because it does not track the + * previous-node pointers required for splicing — use it when you only need to walk the chain. * - *

    For {@code remove} or {@code replace} operations, use - * {@link MutatingBucketIterator} instead. + *

    For {@code remove} or {@code replace} operations, use {@link MutatingBucketIterator} + * instead. */ public static final class BucketIterator implements Iterator { - private final long keyHash; - private Hashtable.Entry nextEntry; - - BucketIterator(Hashtable.Entry[] buckets, long keyHash) { - this.keyHash = keyHash; - Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; - while (cur != null && cur.keyHash != keyHash) cur = cur.next; - this.nextEntry = cur; - } - - @Override - public boolean hasNext() { - return this.nextEntry != null; - } - - @Override - @SuppressWarnings("unchecked") - public TEntry next() { - Hashtable.Entry cur = this.nextEntry; - if (cur == null) throw new NoSuchElementException("no next!"); - - Hashtable.Entry advance = cur.next; - while (advance != null && advance.keyHash != keyHash) advance = advance.next; - this.nextEntry = advance; - - return (TEntry) cur; - } + private final long keyHash; + private Hashtable.Entry nextEntry; + + BucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.keyHash = keyHash; + Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; + while (cur != null && cur.keyHash != keyHash) cur = cur.next; + this.nextEntry = cur; + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry cur = this.nextEntry; + if (cur == null) throw new NoSuchElementException("no next!"); + + Hashtable.Entry advance = cur.next; + while (advance != null && advance.keyHash != keyHash) advance = advance.next; + this.nextEntry = advance; + + return (TEntry) cur; + } } /** - * Mutating iterator over entries in a single bucket whose {@code keyHash} - * matches a specific search hash. Supports {@link #remove()} and - * {@link #replace(Entry)} to splice the chain in place. + * Mutating iterator over entries in a single bucket whose {@code keyHash} matches a specific + * search hash. Supports {@link #remove()} and {@link #replace(Entry)} to splice the chain in + * place. * - *

    Carries previous-node pointers for the current entry and the next-match - * entry so that {@code remove} and {@code replace} can fix up the chain in - * O(1) without re-walking from the bucket head. After {@code remove} or - * {@code replace}, iteration may continue with another {@link #next()}. + *

    Carries previous-node pointers for the current entry and the next-match entry so that {@code + * remove} and {@code replace} can fix up the chain in O(1) without re-walking from the bucket + * head. After {@code remove} or {@code replace}, iteration may continue with another {@link + * #next()}. */ - public static final class MutatingBucketIterator implements Iterator { - private final long keyHash; - - private final Hashtable.Entry[] buckets; - - /** - * The entry prior to the last entry returned by next - * Used for mutating operations - */ - private Hashtable.Entry curPrevEntry; - - /** - * The entry that was last returned by next - */ - private Hashtable.Entry curEntry; - - /** - * The entry prior to the next entry - */ - private Hashtable.Entry nextPrevEntry; - - /** - * The next entry to be returned by next - */ - private Hashtable.Entry nextEntry; - - MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { - this.buckets = buckets; - this.keyHash = keyHash; - - int bucketIndex = Support.bucketIndex(buckets, keyHash); - Hashtable.Entry headEntry = this.buckets[bucketIndex]; - if ( headEntry == null ) { - this.nextEntry = null; - this.nextPrevEntry = null; - - this.curEntry = null; - this.curPrevEntry = null; - } else { - Hashtable.Entry prev, cur; - for ( prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next() ) { - if ( cur.keyHash == keyHash ) break; - } - this.nextPrevEntry = prev; - this.nextEntry = cur; - - this.curEntry = null; - this.curPrevEntry = null; - } - } - - @Override - public boolean hasNext() { - return (this.nextEntry != null); - } - - @Override - @SuppressWarnings("unchecked") - public TEntry next() { - Hashtable.Entry curEntry = this.nextEntry; - if ( curEntry == null ) throw new NoSuchElementException("no next!"); - - this.curEntry = curEntry; - this.curPrevEntry = this.nextPrevEntry; - - Hashtable.Entry prev, cur; - for ( prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next() ) { - if ( cur.keyHash == keyHash ) break; - } - this.nextPrevEntry = prev; - this.nextEntry = cur; - - return (TEntry) curEntry; - } - - @Override - public void remove() { - Hashtable.Entry oldCurEntry = this.curEntry; - if ( oldCurEntry == null ) throw new IllegalStateException(); + public static final class MutatingBucketIterator + implements Iterator { + private final long keyHash; + + private final Hashtable.Entry[] buckets; + + /** The entry prior to the last entry returned by next Used for mutating operations */ + private Hashtable.Entry curPrevEntry; + + /** The entry that was last returned by next */ + private Hashtable.Entry curEntry; + + /** The entry prior to the next entry */ + private Hashtable.Entry nextPrevEntry; + + /** The next entry to be returned by next */ + private Hashtable.Entry nextEntry; + + MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.buckets = buckets; + this.keyHash = keyHash; + + int bucketIndex = Support.bucketIndex(buckets, keyHash); + Hashtable.Entry headEntry = this.buckets[bucketIndex]; + if (headEntry == null) { + this.nextEntry = null; + this.nextPrevEntry = null; + + this.curEntry = null; + this.curPrevEntry = null; + } else { + Hashtable.Entry prev, cur; + for (prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next()) { + if (cur.keyHash == keyHash) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + this.curEntry = null; + this.curPrevEntry = null; + } + } + + @Override + public boolean hasNext() { + return (this.nextEntry != null); + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry curEntry = this.nextEntry; + if (curEntry == null) throw new NoSuchElementException("no next!"); + + this.curEntry = curEntry; + this.curPrevEntry = this.nextPrevEntry; + + Hashtable.Entry prev, cur; + for (prev = this.nextEntry, cur = this.nextEntry.next(); + cur != null; + prev = cur, cur = prev.next()) { + if (cur.keyHash == keyHash) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + return (TEntry) curEntry; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); this.setPrevNext(oldCurEntry.next()); // If the next match was directly after oldCurEntry, its predecessor is now // curPrevEntry (oldCurEntry was just unlinked from the chain). - if ( this.nextPrevEntry == oldCurEntry ) { + if (this.nextPrevEntry == oldCurEntry) { this.nextPrevEntry = this.curPrevEntry; } this.curEntry = null; - } - - public void replace(TEntry replacementEntry) { - Hashtable.Entry oldCurEntry = this.curEntry; - if ( oldCurEntry == null ) throw new IllegalStateException(); - - replacementEntry.setNext(oldCurEntry.next()); - this.setPrevNext(replacementEntry); - - // If the next match was directly after oldCurEntry, its predecessor is now - // the replacement entry (which took oldCurEntry's chain slot). - if ( this.nextPrevEntry == oldCurEntry ) { - this.nextPrevEntry = replacementEntry; - } - this.curEntry = replacementEntry; - } - - void setPrevNext(Hashtable.Entry nextEntry) { - if ( this.curPrevEntry == null ) { - Hashtable.Entry[] buckets = this.buckets; - buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; - } else { - this.curPrevEntry.setNext(nextEntry); - } - } + } + + public void replace(TEntry replacementEntry) { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); + + replacementEntry.setNext(oldCurEntry.next()); + this.setPrevNext(replacementEntry); + + // If the next match was directly after oldCurEntry, its predecessor is now + // the replacement entry (which took oldCurEntry's chain slot). + if (this.nextPrevEntry == oldCurEntry) { + this.nextPrevEntry = replacementEntry; + } + this.curEntry = replacementEntry; + } + + void setPrevNext(Hashtable.Entry nextEntry) { + if (this.curPrevEntry == null) { + Hashtable.Entry[] buckets = this.buckets; + buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; + } else { + this.curPrevEntry.setNext(nextEntry); + } + } } } diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index bc53bc4ecb6..ab8b18a4ca9 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -53,7 +53,7 @@ public static final long hash(int hash0, int hash1) { } private static final int intHash(Object obj) { - return obj == null ? 0 : obj.hashCode(); + return obj == null ? 0 : obj.hashCode(); } public static final long hash(Object obj0, Object obj1, Object obj2) { @@ -86,7 +86,11 @@ public static final long hash(int hash0, int hash1, int hash2, int hash3, int ha // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. - return 31L * 31L * 31L * 31L * hash0 + 31L * 31L * 31L * hash1 + 31L * 31L * hash2 + 31L * hash3 + hash4; + return 31L * 31L * 31L * 31L * hash0 + + 31L * 31L * 31L * hash1 + + 31L * 31L * hash2 + + 31L * hash3 + + hash4; } @Deprecated diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 67c99c0d08d..2d12d535178 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -294,8 +294,7 @@ void walksOnlyMatchingHash() { table.insert(new CollidingKeyEntry(k2, 2)); table.insert(new CollidingKeyEntry(k3, 3)); // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. - BucketIterator it = - Support.bucketIterator(extractBuckets(table), 17L); + BucketIterator it = Support.bucketIterator(extractBuckets(table), 17L); int count = 0; while (it.hasNext()) { assertNotNull(it.next()); @@ -380,8 +379,7 @@ void removeWithoutNextThrows() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("a", 1)); MutatingBucketIterator it = - Support.mutatingBucketIterator( - extractBuckets(table), Hashtable.D1.Entry.hash("a")); + Support.mutatingBucketIterator(extractBuckets(table), Hashtable.D1.Entry.hash("a")); assertThrows(IllegalStateException.class, it::remove); } } @@ -401,8 +399,7 @@ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { /** Sort comparator used by tests that want deterministic visit order. */ @SuppressWarnings("unused") - private static final Comparator BY_KEY = - Comparator.comparing(e -> e.key); + private static final Comparator BY_KEY = Comparator.comparing(e -> e.key); private static final class StringIntEntry extends Hashtable.D1.Entry { int value; @@ -459,7 +456,8 @@ private static final class PairEntry extends Hashtable.D2.Entry } } - // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning quiet. + // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning + // quiet. @SuppressWarnings("unused") private static final List UNUSED = new ArrayList<>(); } diff --git a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java index d0053c75b42..c0e0bebdda0 100644 --- a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java +++ b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java @@ -57,8 +57,7 @@ void fourArgHashMatchesChainedAddToHash() { Object b = 42; Object c = true; Object d = 3.14; - assertEquals( - addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); + assertEquals(addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); } @Test @@ -76,7 +75,8 @@ void fiveArgHashMatchesChainedAddToHash() { @Test void multiArgHashHandlesNullsConsistentlyWithChainedAddToHash() { assertEquals(addToHash(addToHash(0L, (Object) null), "x"), hash(null, "x")); - assertEquals(addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); + assertEquals( + addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); } @Test From f9e63b9ab627f8e901d9b9389e7e54b3b6f9b772 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:19:43 -0400 Subject: [PATCH 07/70] Add JMH benchmarks for Hashtable.D1 and D2 Compares Hashtable.D1 and Hashtable.D2 against equivalent HashMap usage for add, update, and iterate operations. Each benchmark thread owns its own map (Scope.Thread), but @Threads(8) is used so the allocation/GC pressure that Hashtable is designed to avoid surfaces in the throughput numbers. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableBenchmark.java | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java new file mode 100644 index 00000000000..bf25efba679 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java @@ -0,0 +1,290 @@ +package datadog.trace.util; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Compares {@link Hashtable.D1} and {@link Hashtable.D2} against equivalent {@link HashMap} usage + * for add, update, and iterate operations. + * + *

    Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count + * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main + * thing Hashtable is built to avoid. + * + *

      + *
    • add — clear the map then re-insert N fresh entries + * ({@code @OperationsPerInvocation(N_KEYS)}). Captures the steady-state cost of building up a + * map. + *
    • update — for an existing key, increment a counter. Hashtable does {@code get} + + * field mutation (no allocation); HashMap uses {@code merge(k, 1L, Long::sum)}, the idiomatic + * Java 8+ way, which still allocates a {@code Long} per call. + *
    • iterate — walk every entry and consume its key + value. + *
    + * + *

    The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path + * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. + */ +@Fork(2) +@Warmup(iterations = 2) +@Measurement(iterations = 3) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(MICROSECONDS) +@Threads(8) +public class HashtableBenchmark { + + static final int N_KEYS = 64; + static final int CAPACITY = 128; + + static final String[] SOURCE_K1 = new String[N_KEYS]; + static final Integer[] SOURCE_K2 = new Integer[N_KEYS]; + + static { + for (int i = 0; i < N_KEYS; ++i) { + SOURCE_K1[i] = "key-" + i; + SOURCE_K2[i] = i * 31 + 17; + } + } + + static final class D1Counter extends Hashtable.D1.Entry { + long count; + + D1Counter(String key) { + super(key); + } + } + + static final class D2Counter extends Hashtable.D2.Entry { + long count; + + D2Counter(String k1, Integer k2) { + super(k1, k2); + } + } + + /** Composite key for the HashMap baseline against D2. */ + static final class Key2 { + final String k1; + final Integer k2; + final int hash; + + Key2(String k1, Integer k2) { + this.k1 = k1; + this.k2 = k2; + this.hash = Objects.hash(k1, k2); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Key2)) return false; + Key2 other = (Key2) o; + return Objects.equals(k1, other.k1) && Objects.equals(k2, other.k2); + } + + @Override + public int hashCode() { + return hash; + } + } + + /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ + static final class BhD1Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D1Counter e) { + bh.consume(e.key); + bh.consume(e.count); + } + } + + static final class BhD2Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D2Counter e) { + bh.consume(e.key1); + bh.consume(e.key2); + bh.consume(e.count); + } + } + + @State(Scope.Thread) + public static class D1State { + Hashtable.D1 table; + HashMap hashMap; + String[] keys; + int cursor; + final BhD1Consumer consumer = new BhD1Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D1<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + keys = SOURCE_K1; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D1Counter(keys[i])); + hashMap.put(keys[i], 0L); + } + cursor = 0; + } + + String nextKey() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return keys[i]; + } + } + + @State(Scope.Thread) + public static class D2State { + Hashtable.D2 table; + HashMap hashMap; + String[] k1s; + Integer[] k2s; + int cursor; + final BhD2Consumer consumer = new BhD2Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D2<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + k1s = SOURCE_K1; + k2s = SOURCE_K2; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D2Counter(k1s[i], k2s[i])); + hashMap.put(new Key2(k1s[i], k2s[i]), 0L); + } + cursor = 0; + } + + int nextIndex() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return i; + } + } + + // ============================================================ + // D1 — single-key + // ============================================================ + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashtable(D1State s) { + Hashtable.D1 t = s.table; + String[] keys = s.keys; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D1Counter(keys[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashMap(D1State s) { + HashMap m = s.hashMap; + String[] keys = s.keys; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(keys[i], (long) i); + } + } + + @Benchmark + public long d1_update_hashtable(D1State s) { + D1Counter e = s.table.get(s.nextKey()); + return ++e.count; + } + + @Benchmark + public Long d1_update_hashMap(D1State s) { + return s.hashMap.merge(s.nextKey(), 1L, Long::sum); + } + + @Benchmark + public void d1_iterate_hashtable(D1State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d1_iterate_hashMap(D1State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } + + // ============================================================ + // D2 — two-key (composite) + // ============================================================ + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d2_add_hashtable(D2State s) { + Hashtable.D2 t = s.table; + String[] k1s = s.k1s; + Integer[] k2s = s.k2s; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D2Counter(k1s[i], k2s[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d2_add_hashMap(D2State s) { + HashMap m = s.hashMap; + String[] k1s = s.k1s; + Integer[] k2s = s.k2s; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(new Key2(k1s[i], k2s[i]), (long) i); + } + } + + @Benchmark + public long d2_update_hashtable(D2State s) { + int i = s.nextIndex(); + D2Counter e = s.table.get(s.k1s[i], s.k2s[i]); + return ++e.count; + } + + @Benchmark + public Long d2_update_hashMap(D2State s) { + int i = s.nextIndex(); + return s.hashMap.merge(new Key2(s.k1s[i], s.k2s[i]), 1L, Long::sum); + } + + @Benchmark + public void d2_iterate_hashtable(D2State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d2_iterate_hashMap(D2State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } +} From a534e4f4f4313a8d130501aa78b9d08a2e9e8eae Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:21:11 -0400 Subject: [PATCH 08/70] Add benchmark results to HashtableBenchmark header Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableBenchmark.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java index bf25efba679..46e483018e6 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java @@ -41,6 +41,33 @@ * *

    The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. + * + *

    Update is where Hashtable dominates: D1 is ~14x faster, D2 is ~26x faster, because the + * HashMap path allocates per call (a {@code Long}, plus a {@code Key2} for D2) and the resulting GC + * pressure throttles throughput under multiple threads. Add is roughly comparable for D1 + * (both allocate one entry per insert) and ~3x faster for D2 (Hashtable sidesteps the {@code Key2} + * allocation). Iterate is essentially a wash — both are bucket walks. + * MacBook M1 8 threads (Java 8) + * + * Benchmark Mode Cnt Score Error Units + * HashtableBenchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableBenchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * + * HashtableBenchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableBenchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * + * HashtableBenchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableBenchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * + * HashtableBenchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableBenchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us + * + * HashtableBenchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableBenchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us + * + * HashtableBenchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableBenchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * */ @Fork(2) @Warmup(iterations = 2) From ba66a365baa388196d84c3e3bd2606445ece47a4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 10:59:04 -0400 Subject: [PATCH 09/70] Address review feedback on Hashtable - Guard Support.sizeFor against overflow and use Integer.highestOneBit; reject capacities above 1 << 30 instead of looping forever. - Add braces around single-statement while bodies in BucketIterator. - Split HashtableBenchmark into HashtableD1Benchmark / HashtableD2Benchmark. - Add regression tests for Support.sizeFor bounds. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableD1Benchmark.java | 169 ++++++++++++++++++ ...nchmark.java => HashtableD2Benchmark.java} | 142 ++------------- .../java/datadog/trace/util/Hashtable.java | 25 ++- .../datadog/trace/util/HashtableTest.java | 27 +++ 4 files changed, 232 insertions(+), 131 deletions(-) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java rename internal-api/src/jmh/java/datadog/trace/util/{HashtableBenchmark.java => HashtableD2Benchmark.java} (55%) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java new file mode 100644 index 00000000000..16b95e089d5 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -0,0 +1,169 @@ +package datadog.trace.util; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Compares {@link Hashtable.D1} against equivalent {@link HashMap} usage for add, update, and + * iterate operations. + * + *

    Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count + * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main + * thing Hashtable is built to avoid. + * + *

      + *
    • add — clear the map then re-insert N fresh entries + * ({@code @OperationsPerInvocation(N_KEYS)}). Captures the steady-state cost of building up a + * map. + *
    • update — for an existing key, increment a counter. Hashtable does {@code get} + + * field mutation (no allocation); HashMap uses {@code merge(k, 1L, Long::sum)}, the idiomatic + * Java 8+ way, which still allocates a {@code Long} per call. + *
    • iterate — walk every entry and consume its key + value. + *
    + * + *

    Update is where Hashtable dominates: D1 is ~14x faster, because the HashMap path + * allocates per call (a {@code Long}) and the resulting GC pressure throttles throughput under + * multiple threads. Add is roughly comparable (both allocate one entry per insert). + * Iterate is essentially a wash — both are bucket walks. + * MacBook M1 8 threads (Java 8) + * + * Benchmark Mode Cnt Score Error Units + * HashtableD1Benchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableD1Benchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * + * HashtableD1Benchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableD1Benchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * + * HashtableD1Benchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableD1Benchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * + */ +@Fork(2) +@Warmup(iterations = 2) +@Measurement(iterations = 3) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(MICROSECONDS) +@Threads(8) +public class HashtableD1Benchmark { + + static final int N_KEYS = 64; + static final int CAPACITY = 128; + + static final String[] SOURCE_KEYS = new String[N_KEYS]; + + static { + for (int i = 0; i < N_KEYS; ++i) { + SOURCE_KEYS[i] = "key-" + i; + } + } + + static final class D1Counter extends Hashtable.D1.Entry { + long count; + + D1Counter(String key) { + super(key); + } + } + + /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ + static final class BhD1Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D1Counter e) { + bh.consume(e.key); + bh.consume(e.count); + } + } + + @State(Scope.Thread) + public static class D1State { + Hashtable.D1 table; + HashMap hashMap; + String[] keys; + int cursor; + final BhD1Consumer consumer = new BhD1Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D1<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + keys = SOURCE_KEYS; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D1Counter(keys[i])); + hashMap.put(keys[i], 0L); + } + cursor = 0; + } + + String nextKey() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return keys[i]; + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashtable(D1State s) { + Hashtable.D1 t = s.table; + String[] keys = s.keys; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D1Counter(keys[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashMap(D1State s) { + HashMap m = s.hashMap; + String[] keys = s.keys; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(keys[i], (long) i); + } + } + + @Benchmark + public long d1_update_hashtable(D1State s) { + D1Counter e = s.table.get(s.nextKey()); + return ++e.count; + } + + @Benchmark + public Long d1_update_hashMap(D1State s) { + return s.hashMap.merge(s.nextKey(), 1L, Long::sum); + } + + @Benchmark + public void d1_iterate_hashtable(D1State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d1_iterate_hashMap(D1State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } +} diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java similarity index 55% rename from internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java rename to internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 46e483018e6..5fd64ed9a75 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -22,8 +22,8 @@ import org.openjdk.jmh.infra.Blackhole; /** - * Compares {@link Hashtable.D1} and {@link Hashtable.D2} against equivalent {@link HashMap} usage - * for add, update, and iterate operations. + * Compares {@link Hashtable.D2} against equivalent {@link HashMap} usage for add, update, and + * iterate operations. * *

    Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main @@ -42,31 +42,21 @@ *

    The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. * - *

    Update is where Hashtable dominates: D1 is ~14x faster, D2 is ~26x faster, because the - * HashMap path allocates per call (a {@code Long}, plus a {@code Key2} for D2) and the resulting GC - * pressure throttles throughput under multiple threads. Add is roughly comparable for D1 - * (both allocate one entry per insert) and ~3x faster for D2 (Hashtable sidesteps the {@code Key2} - * allocation). Iterate is essentially a wash — both are bucket walks. + *

    Update is where Hashtable dominates: D2 is ~26x faster, because the HashMap path + * allocates per call (a {@code Long}, plus a {@code Key2}) and the resulting GC pressure throttles + * throughput under multiple threads. Add is ~3x faster for D2 (Hashtable sidesteps the + * {@code Key2} allocation). Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableBenchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us - * HashtableBenchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD2Benchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableD2Benchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us * - * HashtableBenchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us - * HashtableBenchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * HashtableD2Benchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableD2Benchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us * - * HashtableBenchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us - * HashtableBenchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us - * - * HashtableBenchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us - * HashtableBenchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us - * - * HashtableBenchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us - * HashtableBenchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us - * - * HashtableBenchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us - * HashtableBenchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * HashtableD2Benchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableD2Benchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us * */ @Fork(2) @@ -75,7 +65,7 @@ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(MICROSECONDS) @Threads(8) -public class HashtableBenchmark { +public class HashtableD2Benchmark { static final int N_KEYS = 64; static final int CAPACITY = 128; @@ -90,14 +80,6 @@ public class HashtableBenchmark { } } - static final class D1Counter extends Hashtable.D1.Entry { - long count; - - D1Counter(String key) { - super(key); - } - } - static final class D2Counter extends Hashtable.D2.Entry { long count; @@ -120,7 +102,9 @@ static final class Key2 { @Override public boolean equals(Object o) { - if (!(o instanceof Key2)) return false; + if (!(o instanceof Key2)) { + return false; + } Key2 other = (Key2) o; return Objects.equals(k1, other.k1) && Objects.equals(k2, other.k2); } @@ -132,16 +116,6 @@ public int hashCode() { } /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ - static final class BhD1Consumer implements Consumer { - Blackhole bh; - - @Override - public void accept(D1Counter e) { - bh.consume(e.key); - bh.consume(e.count); - } - } - static final class BhD2Consumer implements Consumer { Blackhole bh; @@ -153,33 +127,6 @@ public void accept(D2Counter e) { } } - @State(Scope.Thread) - public static class D1State { - Hashtable.D1 table; - HashMap hashMap; - String[] keys; - int cursor; - final BhD1Consumer consumer = new BhD1Consumer(); - - @Setup(Level.Iteration) - public void setUp() { - table = new Hashtable.D1<>(CAPACITY); - hashMap = new HashMap<>(CAPACITY); - keys = SOURCE_K1; - for (int i = 0; i < N_KEYS; ++i) { - table.insert(new D1Counter(keys[i])); - hashMap.put(keys[i], 0L); - } - cursor = 0; - } - - String nextKey() { - int i = cursor; - cursor = (i + 1) & (N_KEYS - 1); - return keys[i]; - } - } - @State(Scope.Thread) public static class D2State { Hashtable.D2 table; @@ -209,61 +156,6 @@ int nextIndex() { } } - // ============================================================ - // D1 — single-key - // ============================================================ - - @Benchmark - @OperationsPerInvocation(N_KEYS) - public void d1_add_hashtable(D1State s) { - Hashtable.D1 t = s.table; - String[] keys = s.keys; - t.clear(); - for (int i = 0; i < N_KEYS; ++i) { - t.insert(new D1Counter(keys[i])); - } - } - - @Benchmark - @OperationsPerInvocation(N_KEYS) - public void d1_add_hashMap(D1State s) { - HashMap m = s.hashMap; - String[] keys = s.keys; - m.clear(); - for (int i = 0; i < N_KEYS; ++i) { - m.put(keys[i], (long) i); - } - } - - @Benchmark - public long d1_update_hashtable(D1State s) { - D1Counter e = s.table.get(s.nextKey()); - return ++e.count; - } - - @Benchmark - public Long d1_update_hashMap(D1State s) { - return s.hashMap.merge(s.nextKey(), 1L, Long::sum); - } - - @Benchmark - public void d1_iterate_hashtable(D1State s, Blackhole bh) { - s.consumer.bh = bh; - s.table.forEach(s.consumer); - } - - @Benchmark - public void d1_iterate_hashMap(D1State s, Blackhole bh) { - for (Map.Entry entry : s.hashMap.entrySet()) { - bh.consume(entry.getKey()); - bh.consume(entry.getValue()); - } - } - - // ============================================================ - // D2 — two-key (composite) - // ============================================================ - @Benchmark @OperationsPerInvocation(N_KEYS) public void d2_add_hashtable(D2State s) { diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 03dfbd7bf1c..39dfaf6c7a4 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -371,11 +371,20 @@ public static final Hashtable.Entry[] create(int capacity) { return new Entry[sizeFor(capacity)]; } + static final int MAX_CAPACITY = 1 << 30; + static final int sizeFor(int requestedCapacity) { - int pow; - for (pow = 1; pow < requestedCapacity; pow *= 2) - ; - return pow; + if (requestedCapacity < 0) { + throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); + } + if (requestedCapacity > MAX_CAPACITY) { + throw new IllegalArgumentException( + "capacity exceeds maximum (" + MAX_CAPACITY + "): " + requestedCapacity); + } + if (requestedCapacity <= 1) { + return 1; + } + return Integer.highestOneBit(requestedCapacity - 1) << 1; } public static final void clear(Hashtable.Entry[] buckets) { @@ -413,7 +422,9 @@ public static final class BucketIterator implements Iterat BucketIterator(Hashtable.Entry[] buckets, long keyHash) { this.keyHash = keyHash; Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; - while (cur != null && cur.keyHash != keyHash) cur = cur.next; + while (cur != null && cur.keyHash != keyHash) { + cur = cur.next; + } this.nextEntry = cur; } @@ -429,7 +440,9 @@ public TEntry next() { if (cur == null) throw new NoSuchElementException("no next!"); Hashtable.Entry advance = cur.next; - while (advance != null && advance.keyHash != keyHash) advance = advance.next; + while (advance != null && advance.keyHash != keyHash) { + advance = advance.next; + } this.nextEntry = advance; return (TEntry) cur; diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 2d12d535178..b11a33a4322 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -255,6 +255,33 @@ void createRoundsCapacityUpToPowerOfTwo() { assertEquals(0, len & (len - 1), "length must be a power of two"); } + @Test + void sizeForReturnsAtLeastOne() { + assertEquals(1, Support.sizeFor(0)); + assertEquals(1, Support.sizeFor(1)); + } + + @Test + void sizeForRoundsUpToPowerOfTwo() { + assertEquals(2, Support.sizeFor(2)); + assertEquals(4, Support.sizeFor(3)); + assertEquals(4, Support.sizeFor(4)); + assertEquals(8, Support.sizeFor(5)); + assertEquals(1 << 30, Support.sizeFor(1 << 30)); + } + + @Test + void sizeForRejectsCapacityAboveMax() { + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor((1 << 30) + 1)); + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(Integer.MAX_VALUE)); + } + + @Test + void sizeForRejectsNegativeCapacity() { + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(-1)); + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(Integer.MIN_VALUE)); + } + @Test void bucketIndexIsBoundedByArrayLength() { Hashtable.Entry[] buckets = Support.create(16); From 310894134ffd9f5abfa7f829cb2012891061e85c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:19:43 -0400 Subject: [PATCH 10/70] Fix dropped argument in HashingUtils 5-arg Object hash The 5-arg Object overload was forwarding only obj0..obj3 to the int overload, silently dropping obj4. Also align LongHashingUtils.hash 3-arg signature with its 2/4/5-arg siblings (int parameters) and strengthen the 5-arg HashingUtilsTest to detect the missing-arg regression. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/main/java/datadog/trace/util/HashingUtils.java | 2 +- .../src/main/java/datadog/trace/util/LongHashingUtils.java | 2 +- .../src/test/java/datadog/trace/util/HashingUtilsTest.java | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/HashingUtils.java b/internal-api/src/main/java/datadog/trace/util/HashingUtils.java index 1522554836a..d975149f433 100644 --- a/internal-api/src/main/java/datadog/trace/util/HashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/HashingUtils.java @@ -79,7 +79,7 @@ public static final int hash(int hash0, int hash1, int hash2, int hash3) { } public static final int hash(Object obj0, Object obj1, Object obj2, Object obj3, Object obj4) { - return hash(hashCode(obj0), hashCode(obj1), hashCode(obj2), hashCode(obj3)); + return hash(hashCode(obj0), hashCode(obj1), hashCode(obj2), hashCode(obj3), hashCode(obj4)); } public static final int hash(int hash0, int hash1, int hash2, int hash3, int hash4) { diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index ab8b18a4ca9..c14b498cc9c 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -60,7 +60,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2) { return hash(intHash(obj0), intHash(obj1), intHash(obj2)); } - public static final long hash(long hash0, long hash1, long hash2) { + public static final long hash(int hash0, int hash1, int hash2) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. diff --git a/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java index 185d5a4f2e4..1f171852866 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java @@ -99,7 +99,7 @@ public void hash5() { String str3 = "foobar"; String str4 = "hello"; - assertNotEquals(0, HashingUtils.hash(str0, str1, str2, str3)); + assertNotEquals(0, HashingUtils.hash(str0, str1, str2, str3, str4)); String clone0 = clone(str0); String clone1 = clone(str1); @@ -110,6 +110,11 @@ public void hash5() { assertEquals( HashingUtils.hash(str0, str1, str2, str3, str4), HashingUtils.hash(clone0, clone1, clone2, clone3, clone4)); + + // The 5th argument must actually affect the hash (regression for a missing-arg bug). + assertNotEquals( + HashingUtils.hash(str0, str1, str2, str3, str4), + HashingUtils.hash(str0, str1, str2, str3, "different")); } @Test From 1415f12028493aad4d5476bb12b69c85148fa0a5 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:25:58 -0400 Subject: [PATCH 11/70] Address review feedback on Hashtable - Split D1Tests and D2Tests into HashtableD1Test and HashtableD2Test; extract shared test entry classes into HashtableTestEntries. - Reduce visibility of LongHashingUtils.hash(int...) chaining overloads to package-private; they are internal building blocks. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/util/LongHashingUtils.java | 8 +- .../datadog/trace/util/HashtableD1Test.java | 165 ++++++++++ .../datadog/trace/util/HashtableD2Test.java | 76 +++++ .../datadog/trace/util/HashtableTest.java | 296 +----------------- .../trace/util/HashtableTestEntries.java | 54 ++++ 5 files changed, 305 insertions(+), 294 deletions(-) create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index c14b498cc9c..9d1257a3f20 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -48,7 +48,7 @@ public static final long hash(Object obj0, Object obj1) { return hash(intHash(obj0), intHash(obj1)); } - public static final long hash(int hash0, int hash1) { + static final long hash(int hash0, int hash1) { return 31L * hash0 + hash1; } @@ -60,7 +60,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2) { return hash(intHash(obj0), intHash(obj1), intHash(obj2)); } - public static final long hash(int hash0, int hash1, int hash2) { + static final long hash(int hash0, int hash1, int hash2) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. @@ -71,7 +71,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3 return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3)); } - public static final long hash(int hash0, int hash1, int hash2, int hash3) { + static final long hash(int hash0, int hash1, int hash2, int hash3) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. @@ -82,7 +82,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3 return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3), intHash(obj4)); } - public static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { + static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java new file mode 100644 index 00000000000..10d8ad41976 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -0,0 +1,165 @@ +package datadog.trace.util; + +import static datadog.trace.util.HashtableTestEntries.CollidingKey; +import static datadog.trace.util.HashtableTestEntries.CollidingKeyEntry; +import static datadog.trace.util.HashtableTestEntries.StringIntEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class HashtableD1Test { + + @Test + void emptyTableLookupReturnsNull() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get("missing")); + assertEquals(0, table.size()); + } + + @Test + void insertedEntryIsRetrievable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry e = new StringIntEntry("foo", 1); + table.insert(e); + assertEquals(1, table.size()); + assertSame(e, table.get("foo")); + } + + @Test + void multipleInsertsRetrievableSeparately() { + Hashtable.D1 table = new Hashtable.D1<>(16); + StringIntEntry a = new StringIntEntry("alpha", 1); + StringIntEntry b = new StringIntEntry("beta", 2); + StringIntEntry c = new StringIntEntry("gamma", 3); + table.insert(a); + table.insert(b); + table.insert(c); + assertEquals(3, table.size()); + assertSame(a, table.get("alpha")); + assertSame(b, table.get("beta")); + assertSame(c, table.get("gamma")); + } + + @Test + void inPlaceMutationVisibleViaSubsequentGet() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("counter", 0)); + for (int i = 0; i < 10; i++) { + StringIntEntry e = table.get("counter"); + e.value++; + } + assertEquals(10, table.get("counter").value); + } + + @Test + void removeUnlinksEntryAndDecrementsSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + assertEquals(2, table.size()); + + StringIntEntry removed = table.remove("a"); + assertNotNull(removed); + assertEquals("a", removed.key); + assertEquals(1, table.size()); + assertNull(table.get("a")); + assertNotNull(table.get("b")); + } + + @Test + void removeNonexistentReturnsNullAndDoesNotChangeSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + assertNull(table.remove("nope")); + assertEquals(1, table.size()); + } + + @Test + void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry first = new StringIntEntry("k", 1); + assertNull(table.insertOrReplace(first), "fresh insert returns null"); + assertEquals(1, table.size()); + + StringIntEntry second = new StringIntEntry("k", 2); + assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); + assertEquals(1, table.size()); + assertSame(second, table.get("k"), "new entry visible after replace"); + } + + @Test + void clearEmptiesTheTable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.clear(); + assertEquals(0, table.size()); + assertNull(table.get("a")); + // Reinsertion works after clear + table.insert(new StringIntEntry("a", 99)); + assertEquals(99, table.get("a").value); + } + + @Test + void forEachVisitsEveryInsertedEntry() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + Map seen = new HashMap<>(); + table.forEach(e -> seen.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(1, seen.get("a")); + assertEquals(2, seen.get("b")); + assertEquals(3, seen.get("c")); + } + + @Test + void nullKeyIsPermittedAndDistinctFromAbsent() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get(null)); + StringIntEntry nullKeyed = new StringIntEntry(null, 7); + table.insert(nullKeyed); + assertSame(nullKeyed, table.get(null)); + assertEquals(1, table.size()); + assertSame(nullKeyed, table.remove(null)); + assertEquals(0, table.size()); + } + + @Test + void hashCollisionsResolveByEquality() { + // Force two distinct keys with the same hashCode -- the chain must still distinguish them + // via matches(). + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); + table.insert(e1); + table.insert(e2); + assertEquals(2, table.size()); + assertSame(e1, table.get(k1)); + assertSame(e2, table.get(k2)); + } + + @Test + void hashCollisionsThenRemoveLeavesOtherIntact() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + table.remove(k2); + assertEquals(2, table.size()); + assertNotNull(table.get(k1)); + assertNull(table.get(k2)); + assertNotNull(table.get(k3)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java new file mode 100644 index 00000000000..98c54b71c2c --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -0,0 +1,76 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class HashtableD2Test { + + @Test + void pairKeysParticipateInIdentity() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + PairEntry bb = new PairEntry("b", 1, 300); + table.insert(ab); + table.insert(ac); + table.insert(bb); + assertEquals(3, table.size()); + assertSame(ab, table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + assertSame(bb, table.get("b", 1)); + assertNull(table.get("a", 3)); + } + + @Test + void removePairUnlinks() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + table.insert(ab); + table.insert(ac); + assertSame(ab, table.remove("a", 1)); + assertEquals(1, table.size()); + assertNull(table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + } + + @Test + void insertOrReplaceMatchesOnBothKeys() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry first = new PairEntry("k", 7, 1); + assertNull(table.insertOrReplace(first)); + PairEntry second = new PairEntry("k", 7, 2); + assertSame(first, table.insertOrReplace(second)); + // Different second-key: should insert new, not replace + PairEntry third = new PairEntry("k", 8, 3); + assertNull(table.insertOrReplace(third)); + assertEquals(2, table.size()); + } + + @Test + void forEachVisitsBothPairs() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + + private static final class PairEntry extends Hashtable.D2.Entry { + int value; + + PairEntry(String key1, Integer key2, int value) { + super(key1, key2); + this.value = value; + } + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index b11a33a4322..553db03495b 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -1,244 +1,24 @@ package datadog.trace.util; +import static datadog.trace.util.HashtableTestEntries.CollidingKey; +import static datadog.trace.util.HashtableTestEntries.CollidingKeyEntry; +import static datadog.trace.util.HashtableTestEntries.StringIntEntry; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.util.Hashtable.BucketIterator; import datadog.trace.util.Hashtable.MutatingBucketIterator; import datadog.trace.util.Hashtable.Support; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import java.util.Set; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class HashtableTest { - // ============ D1 ============ - - @Nested - class D1Tests { - - @Test - void emptyTableLookupReturnsNull() { - Hashtable.D1 table = new Hashtable.D1<>(8); - assertNull(table.get("missing")); - assertEquals(0, table.size()); - } - - @Test - void insertedEntryIsRetrievable() { - Hashtable.D1 table = new Hashtable.D1<>(8); - StringIntEntry e = new StringIntEntry("foo", 1); - table.insert(e); - assertEquals(1, table.size()); - assertSame(e, table.get("foo")); - } - - @Test - void multipleInsertsRetrievableSeparately() { - Hashtable.D1 table = new Hashtable.D1<>(16); - StringIntEntry a = new StringIntEntry("alpha", 1); - StringIntEntry b = new StringIntEntry("beta", 2); - StringIntEntry c = new StringIntEntry("gamma", 3); - table.insert(a); - table.insert(b); - table.insert(c); - assertEquals(3, table.size()); - assertSame(a, table.get("alpha")); - assertSame(b, table.get("beta")); - assertSame(c, table.get("gamma")); - } - - @Test - void inPlaceMutationVisibleViaSubsequentGet() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("counter", 0)); - for (int i = 0; i < 10; i++) { - StringIntEntry e = table.get("counter"); - e.value++; - } - assertEquals(10, table.get("counter").value); - } - - @Test - void removeUnlinksEntryAndDecrementsSize() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - assertEquals(2, table.size()); - - StringIntEntry removed = table.remove("a"); - assertNotNull(removed); - assertEquals("a", removed.key); - assertEquals(1, table.size()); - assertNull(table.get("a")); - assertNotNull(table.get("b")); - } - - @Test - void removeNonexistentReturnsNullAndDoesNotChangeSize() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - assertNull(table.remove("nope")); - assertEquals(1, table.size()); - } - - @Test - void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { - Hashtable.D1 table = new Hashtable.D1<>(8); - StringIntEntry first = new StringIntEntry("k", 1); - assertNull(table.insertOrReplace(first), "fresh insert returns null"); - assertEquals(1, table.size()); - - StringIntEntry second = new StringIntEntry("k", 2); - assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); - assertEquals(1, table.size()); - assertSame(second, table.get("k"), "new entry visible after replace"); - } - - @Test - void clearEmptiesTheTable() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - table.clear(); - assertEquals(0, table.size()); - assertNull(table.get("a")); - // Reinsertion works after clear - table.insert(new StringIntEntry("a", 99)); - assertEquals(99, table.get("a").value); - } - - @Test - void forEachVisitsEveryInsertedEntry() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - table.insert(new StringIntEntry("c", 3)); - Map seen = new HashMap<>(); - table.forEach(e -> seen.put(e.key, e.value)); - assertEquals(3, seen.size()); - assertEquals(1, seen.get("a")); - assertEquals(2, seen.get("b")); - assertEquals(3, seen.get("c")); - } - - @Test - void nullKeyIsPermittedAndDistinctFromAbsent() { - Hashtable.D1 table = new Hashtable.D1<>(8); - assertNull(table.get(null)); - StringIntEntry nullKeyed = new StringIntEntry(null, 7); - table.insert(nullKeyed); - assertSame(nullKeyed, table.get(null)); - assertEquals(1, table.size()); - assertSame(nullKeyed, table.remove(null)); - assertEquals(0, table.size()); - } - - @Test - void hashCollisionsResolveByEquality() { - // Force two distinct keys with the same hashCode -- the chain must still distinguish them - // via matches(). - Hashtable.D1 table = new Hashtable.D1<>(4); - CollidingKey k1 = new CollidingKey("first", 17); - CollidingKey k2 = new CollidingKey("second", 17); - CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); - CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); - table.insert(e1); - table.insert(e2); - assertEquals(2, table.size()); - assertSame(e1, table.get(k1)); - assertSame(e2, table.get(k2)); - } - - @Test - void hashCollisionsThenRemoveLeavesOtherIntact() { - Hashtable.D1 table = new Hashtable.D1<>(4); - CollidingKey k1 = new CollidingKey("first", 17); - CollidingKey k2 = new CollidingKey("second", 17); - CollidingKey k3 = new CollidingKey("third", 17); - table.insert(new CollidingKeyEntry(k1, 1)); - table.insert(new CollidingKeyEntry(k2, 2)); - table.insert(new CollidingKeyEntry(k3, 3)); - table.remove(k2); - assertEquals(2, table.size()); - assertNotNull(table.get(k1)); - assertNull(table.get(k2)); - assertNotNull(table.get(k3)); - } - } - - // ============ D2 ============ - - @Nested - class D2Tests { - - @Test - void pairKeysParticipateInIdentity() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry ab = new PairEntry("a", 1, 100); - PairEntry ac = new PairEntry("a", 2, 200); - PairEntry bb = new PairEntry("b", 1, 300); - table.insert(ab); - table.insert(ac); - table.insert(bb); - assertEquals(3, table.size()); - assertSame(ab, table.get("a", 1)); - assertSame(ac, table.get("a", 2)); - assertSame(bb, table.get("b", 1)); - assertNull(table.get("a", 3)); - } - - @Test - void removePairUnlinks() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry ab = new PairEntry("a", 1, 100); - PairEntry ac = new PairEntry("a", 2, 200); - table.insert(ab); - table.insert(ac); - assertSame(ab, table.remove("a", 1)); - assertEquals(1, table.size()); - assertNull(table.get("a", 1)); - assertSame(ac, table.get("a", 2)); - } - - @Test - void insertOrReplaceMatchesOnBothKeys() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry first = new PairEntry("k", 7, 1); - assertNull(table.insertOrReplace(first)); - PairEntry second = new PairEntry("k", 7, 2); - assertSame(first, table.insertOrReplace(second)); - // Different second-key: should insert new, not replace - PairEntry third = new PairEntry("k", 8, 3); - assertNull(table.insertOrReplace(third)); - assertEquals(2, table.size()); - } - - @Test - void forEachVisitsBothPairs() { - Hashtable.D2 table = new Hashtable.D2<>(8); - table.insert(new PairEntry("a", 1, 100)); - table.insert(new PairEntry("b", 2, 200)); - Set seen = new HashSet<>(); - table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); - assertEquals(2, seen.size()); - assertTrue(seen.contains("a:1")); - assertTrue(seen.contains("b:2")); - } - } - // ============ Support ============ @Nested @@ -374,7 +154,9 @@ void removeFromHeadOfChainUnlinks() { // of the three keys are still retrievable.) int found = 0; for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { - if (table.get(k) != null) found++; + if (table.get(k) != null) { + found++; + } } assertEquals(2, found); } @@ -411,8 +193,6 @@ void removeWithoutNextThrows() { } } - // ============ test helpers ============ - /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { try { @@ -423,68 +203,4 @@ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { throw new RuntimeException(e); } } - - /** Sort comparator used by tests that want deterministic visit order. */ - @SuppressWarnings("unused") - private static final Comparator BY_KEY = Comparator.comparing(e -> e.key); - - private static final class StringIntEntry extends Hashtable.D1.Entry { - int value; - - StringIntEntry(String key, int value) { - super(key); - this.value = value; - } - } - - /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ - private static final class CollidingKey { - final String label; - final int hash; - - CollidingKey(String label, int hash) { - this.label = label; - this.hash = hash; - } - - @Override - public int hashCode() { - return hash; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CollidingKey)) return false; - CollidingKey that = (CollidingKey) o; - return hash == that.hash && label.equals(that.label); - } - - @Override - public String toString() { - return "CollidingKey(" + label + ", " + hash + ")"; - } - } - - private static final class CollidingKeyEntry extends Hashtable.D1.Entry { - int value; - - CollidingKeyEntry(CollidingKey key, int value) { - super(key); - this.value = value; - } - } - - private static final class PairEntry extends Hashtable.D2.Entry { - int value; - - PairEntry(String key1, Integer key2, int value) { - super(key1, key2); - this.value = value; - } - } - - // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning - // quiet. - @SuppressWarnings("unused") - private static final List UNUSED = new ArrayList<>(); } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java b/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java new file mode 100644 index 00000000000..e657028ee8b --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java @@ -0,0 +1,54 @@ +package datadog.trace.util; + +/** Shared test entry types for {@link HashtableTest}, {@link HashtableD1Test}, and friends. */ +final class HashtableTestEntries { + private HashtableTestEntries() {} + + static final class StringIntEntry extends Hashtable.D1.Entry { + int value; + + StringIntEntry(String key, int value) { + super(key); + this.value = value; + } + } + + /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ + static final class CollidingKey { + final String label; + final int hash; + + CollidingKey(String label, int hash) { + this.label = label; + this.hash = hash; + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CollidingKey)) { + return false; + } + CollidingKey that = (CollidingKey) o; + return hash == that.hash && label.equals(that.label); + } + + @Override + public String toString() { + return "CollidingKey(" + label + ", " + hash + ")"; + } + } + + static final class CollidingKeyEntry extends Hashtable.D1.Entry { + int value; + + CollidingKeyEntry(CollidingKey key, int value) { + super(key); + this.value = value; + } + } +} From b7cee2fee3dda3b668455a00649646cf9b1a6ef4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:32:57 -0400 Subject: [PATCH 12/70] Drop reflection in iterator tests via package-private D1.buckets The iterator tests need a populated Hashtable.Entry[] to drive Support.bucketIterator / mutatingBucketIterator. Relaxing D1.buckets from private to package-private lets the same-package tests read it directly, removing the reflection helper. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 2 +- .../datadog/trace/util/HashtableTest.java | 21 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 39dfaf6c7a4..e527ae45fcc 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -100,7 +100,7 @@ public static long hash(Object key) { } } - private final Hashtable.Entry[] buckets; + final Hashtable.Entry[] buckets; private int size; public D1(int capacity) { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 553db03495b..f78aec1c00f 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -101,7 +101,7 @@ void walksOnlyMatchingHash() { table.insert(new CollidingKeyEntry(k2, 2)); table.insert(new CollidingKeyEntry(k3, 3)); // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. - BucketIterator it = Support.bucketIterator(extractBuckets(table), 17L); + BucketIterator it = Support.bucketIterator(table.buckets, 17L); int count = 0; while (it.hasNext()) { assertNotNull(it.next()); @@ -115,7 +115,7 @@ void exhaustedIteratorThrowsNoSuchElement() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("only", 1)); long h = Hashtable.D1.Entry.hash("only"); - BucketIterator it = Support.bucketIterator(extractBuckets(table), h); + BucketIterator it = Support.bucketIterator(table.buckets, h); it.next(); assertFalse(it.hasNext()); assertThrows(NoSuchElementException.class, it::next); @@ -139,7 +139,7 @@ void removeFromHeadOfChainUnlinks() { table.insert(new CollidingKeyEntry(k3, 3)); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), 17L); + Support.mutatingBucketIterator(table.buckets, 17L); it.next(); // first match (head of chain in insertion-reverse order) it.remove(); // Two should remain @@ -172,7 +172,7 @@ void replaceSwapsEntryAndPreservesChain() { table.insert(e2); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), 17L); + Support.mutatingBucketIterator(table.buckets, 17L); CollidingKeyEntry first = it.next(); CollidingKeyEntry replacement = new CollidingKeyEntry(first.key, 999); it.replace(replacement); @@ -188,19 +188,8 @@ void removeWithoutNextThrows() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("a", 1)); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), Hashtable.D1.Entry.hash("a")); + Support.mutatingBucketIterator(table.buckets, Hashtable.D1.Entry.hash("a")); assertThrows(IllegalStateException.class, it::remove); } } - - /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ - private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { - try { - java.lang.reflect.Field f = Hashtable.D1.class.getDeclaredField("buckets"); - f.setAccessible(true); - return (Hashtable.Entry[]) f.get(table); - } catch (Exception e) { - throw new RuntimeException(e); - } - } } From 553ffb87edb1cc75ff96bafd7d154680ed0a34a9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:41:07 -0400 Subject: [PATCH 13/70] Fold AggregateMetric into AggregateEntry The label fields and the mutable counters/histograms are 1:1 with each entry; carrying them on a separate object meant one extra allocation per unique key plus an indirection on every hot-path update. Merging them puts the counters directly on AggregateEntry, drops the entry.aggregate hop, and consolidates ERROR_TAG / TOP_LEVEL_TAG onto the same class the consumer uses to decode them. AggregateTable.findOrInsert now returns AggregateEntry. Callers in Aggregator and SerializingMetricWriter updated. Migrated AggregateMetricTest.groovy to AggregateEntryTest.java per project policy. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 110 ++++++++++++++++-- .../trace/common/metrics/AggregateMetric.java | 103 ---------------- .../trace/common/metrics/AggregateTable.java | 33 +++--- .../trace/common/metrics/Aggregator.java | 8 +- .../metrics/ConflatingMetricsAggregator.java | 4 +- .../trace/common/metrics/MetricWriter.java | 2 +- .../metrics/SerializingMetricWriter.java | 13 +-- .../trace/common/metrics/SpanSnapshot.java | 4 +- .../common/metrics/AggregateMetricTest.groovy | 105 ----------------- .../ConflatingMetricAggregatorTest.groovy | 62 +++++----- .../SerializingMetricWriterTest.groovy | 11 +- .../common/metrics/AggregateEntryTest.java | 108 +++++++++++++++++ .../common/metrics/AggregateTableTest.java | 45 ++++--- 13 files changed, 299 insertions(+), 309 deletions(-) delete mode 100644 dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateMetric.java delete mode 100644 dd-trace-core/src/test/groovy/datadog/trace/common/metrics/AggregateMetricTest.groovy create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index e2fda9fde47..1cde9c0e68a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -3,21 +3,24 @@ import static datadog.trace.api.Functions.UTF8_ENCODE; import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; +import datadog.metrics.api.Histogram; import datadog.trace.api.Pair; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.util.Hashtable; import datadog.trace.util.LongHashingUtils; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.Function; /** - * Hashtable entry for the consumer-side aggregator. Holds the UTF8-encoded label fields (the data - * {@link SerializingMetricWriter} writes to the wire) plus the mutable {@link AggregateMetric}. + * Hashtable entry for the consumer-side aggregator. Holds the UTF8-encoded label fields that {@link + * SerializingMetricWriter} writes to the wire plus the mutable counter/histogram state for the key. * *

    {@link #matches(SpanSnapshot)} compares the entry's stored UTF8 forms against the snapshot's * raw {@code CharSequence}/{@code String}/{@code String[]} fields via content-equality, so {@code @@ -26,9 +29,19 @@ * *

    The static UTF8 caches that used to live on {@code MetricKey} and {@code * ConflatingMetricsAggregator} are consolidated here. + * + *

    Not thread-safe. Counter and histogram updates are performed by the single aggregator + * thread; producer threads tag durations via {@link #ERROR_TAG} / {@link #TOP_LEVEL_TAG} bits and + * hand them off through the snapshot inbox. */ +@SuppressFBWarnings( + value = {"AT_NONATOMIC_OPERATIONS_ON_SHARED_VARIABLE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}, + justification = "Explicitly not thread-safe. Accumulates counts and durations.") final class AggregateEntry extends Hashtable.Entry { + public static final long ERROR_TAG = 0x8000000000000000L; + public static final long TOP_LEVEL_TAG = 0x4000000000000000L; + // UTF8 caches consolidated from the previous MetricKey + ConflatingMetricsAggregator split. private static final DDCache RESOURCE_CACHE = DDCaches.newFixedSizeCache(32); @@ -82,10 +95,16 @@ final class AggregateEntry extends Hashtable.Entry { private final String[] peerTagPairsRaw; private final List peerTags; - final AggregateMetric aggregate; + // Mutable aggregate state -- single-thread (consumer/aggregator) writer. + private final Histogram okLatencies = Histogram.newHistogram(); + private final Histogram errorLatencies = Histogram.newHistogram(); + private int errorCount; + private int hitCount; + private int topLevelCount; + private long duration; /** Hot-path constructor for the producer/consumer flow. Builds UTF8 fields via the caches. */ - private AggregateEntry(SpanSnapshot s, long keyHash, AggregateMetric aggregate) { + private AggregateEntry(SpanSnapshot s, long keyHash) { super(keyHash); this.resource = canonicalize(RESOURCE_CACHE, s.resourceName); this.service = SERVICE_CACHE.computeIfAbsent(s.serviceName, UTF8_ENCODE); @@ -113,7 +132,6 @@ private AggregateEntry(SpanSnapshot s, long keyHash, AggregateMetric aggregate) this.traceRoot = s.traceRoot; this.peerTagPairsRaw = s.peerTagPairs; this.peerTags = materializePeerTags(s.peerTagPairs); - this.aggregate = aggregate; } /** Test-friendly factory mirroring the prior {@code new MetricKey(...)} positional args. */ @@ -148,13 +166,87 @@ static AggregateEntry of( httpEndpoint == null ? null : httpEndpoint.toString(), grpcStatusCode == null ? null : grpcStatusCode.toString(), 0L); - return new AggregateEntry( - synthetic_snapshot, hashOf(synthetic_snapshot), new AggregateMetric()); + return new AggregateEntry(synthetic_snapshot, hashOf(synthetic_snapshot)); } /** Construct from a snapshot at consumer-thread miss time. */ - static AggregateEntry forSnapshot(SpanSnapshot s, AggregateMetric aggregate) { - return new AggregateEntry(s, hashOf(s), aggregate); + static AggregateEntry forSnapshot(SpanSnapshot s) { + return new AggregateEntry(s, hashOf(s)); + } + + AggregateEntry recordDurations(int count, AtomicLongArray durations) { + this.hitCount += count; + for (int i = 0; i < count && i < durations.length(); ++i) { + long duration = durations.getAndSet(i, 0); + if ((duration & TOP_LEVEL_TAG) == TOP_LEVEL_TAG) { + duration ^= TOP_LEVEL_TAG; + ++topLevelCount; + } + if ((duration & ERROR_TAG) == ERROR_TAG) { + duration ^= ERROR_TAG; + errorLatencies.accept(duration); + ++errorCount; + } else { + okLatencies.accept(duration); + } + this.duration += duration; + } + return this; + } + + /** + * Records a single hit. {@code tagAndDuration} carries the duration nanos with optional {@link + * #ERROR_TAG} / {@link #TOP_LEVEL_TAG} bits OR-ed in. + */ + AggregateEntry recordOneDuration(long tagAndDuration) { + ++hitCount; + if ((tagAndDuration & TOP_LEVEL_TAG) == TOP_LEVEL_TAG) { + tagAndDuration ^= TOP_LEVEL_TAG; + ++topLevelCount; + } + if ((tagAndDuration & ERROR_TAG) == ERROR_TAG) { + tagAndDuration ^= ERROR_TAG; + errorLatencies.accept(tagAndDuration); + ++errorCount; + } else { + okLatencies.accept(tagAndDuration); + } + duration += tagAndDuration; + return this; + } + + int getErrorCount() { + return errorCount; + } + + int getHitCount() { + return hitCount; + } + + int getTopLevelCount() { + return topLevelCount; + } + + long getDuration() { + return duration; + } + + Histogram getOkLatencies() { + return okLatencies; + } + + Histogram getErrorLatencies() { + return errorLatencies; + } + + @SuppressFBWarnings("AT_NONATOMIC_64BIT_PRIMITIVE") + void clear() { + this.errorCount = 0; + this.hitCount = 0; + this.topLevelCount = 0; + this.duration = 0; + this.okLatencies.clear(); + this.errorLatencies.clear(); } boolean matches(SpanSnapshot s) { diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateMetric.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateMetric.java deleted file mode 100644 index dba66a5ab9c..00000000000 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateMetric.java +++ /dev/null @@ -1,103 +0,0 @@ -package datadog.trace.common.metrics; - -import datadog.metrics.api.Histogram; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.concurrent.atomic.AtomicLongArray; - -/** Not thread-safe. Accumulates counts and durations. */ -@SuppressFBWarnings( - value = {"AT_NONATOMIC_OPERATIONS_ON_SHARED_VARIABLE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}, - justification = "Explicitly not thread-safe. Accumulates counts and durations.") -public final class AggregateMetric { - - static final long ERROR_TAG = 0x8000000000000000L; - static final long TOP_LEVEL_TAG = 0x4000000000000000L; - - private final Histogram okLatencies; - private final Histogram errorLatencies; - private int errorCount; - private int hitCount; - private int topLevelCount; - private long duration; - - public AggregateMetric() { - okLatencies = Histogram.newHistogram(); - errorLatencies = Histogram.newHistogram(); - } - - public AggregateMetric recordDurations(int count, AtomicLongArray durations) { - this.hitCount += count; - for (int i = 0; i < count && i < durations.length(); ++i) { - long duration = durations.getAndSet(i, 0); - if ((duration & TOP_LEVEL_TAG) == TOP_LEVEL_TAG) { - duration ^= TOP_LEVEL_TAG; - ++topLevelCount; - } - if ((duration & ERROR_TAG) == ERROR_TAG) { - // then it's an error - duration ^= ERROR_TAG; - errorLatencies.accept(duration); - ++errorCount; - } else { - okLatencies.accept(duration); - } - this.duration += duration; - } - return this; - } - - /** - * Records a single hit. {@code tagAndDuration} carries the duration nanos with optional {@link - * #ERROR_TAG} / {@link #TOP_LEVEL_TAG} bits OR-ed in. - */ - public AggregateMetric recordOneDuration(long tagAndDuration) { - ++hitCount; - if ((tagAndDuration & TOP_LEVEL_TAG) == TOP_LEVEL_TAG) { - tagAndDuration ^= TOP_LEVEL_TAG; - ++topLevelCount; - } - if ((tagAndDuration & ERROR_TAG) == ERROR_TAG) { - tagAndDuration ^= ERROR_TAG; - errorLatencies.accept(tagAndDuration); - ++errorCount; - } else { - okLatencies.accept(tagAndDuration); - } - duration += tagAndDuration; - return this; - } - - public int getErrorCount() { - return errorCount; - } - - public int getHitCount() { - return hitCount; - } - - public int getTopLevelCount() { - return topLevelCount; - } - - public long getDuration() { - return duration; - } - - public Histogram getOkLatencies() { - return okLatencies; - } - - public Histogram getErrorLatencies() { - return errorLatencies; - } - - @SuppressFBWarnings("AT_NONATOMIC_64BIT_PRIMITIVE") - public void clear() { - this.errorCount = 0; - this.hitCount = 0; - this.topLevelCount = 0; - this.duration = 0; - this.okLatencies.clear(); - this.errorLatencies.clear(); - } -} diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 08300eab296..3bc3766227d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -4,13 +4,14 @@ import java.util.function.Consumer; /** - * Consumer-side {@link AggregateMetric} store, keyed on the raw fields of a {@link SpanSnapshot}. + * Consumer-side {@link AggregateEntry} store, keyed on the raw fields of a {@link SpanSnapshot}. * *

    Replaces the prior {@code LRUCache}. The win is on the * steady-state hit path: a snapshot lookup is a 64-bit hash compute + bucket walk + field-wise * {@code matches}, with no per-snapshot {@link AggregateEntry} allocation and no UTF8 cache - * lookups. The UTF8-encoded forms (formerly held on {@code MetricKey}) live on the {@link - * AggregateEntry} itself and are built once per unique key at insert time. + * lookups. The UTF8-encoded forms (formerly held on {@code MetricKey}) and the mutable counters + * (formerly held on {@code AggregateMetric}) both live on the {@link AggregateEntry} now, built + * once per unique key at insert time. * *

    Not thread-safe. The aggregator thread is the sole writer; {@link #clear()} must be * routed through the inbox rather than called from arbitrary threads. @@ -35,39 +36,39 @@ boolean isEmpty() { } /** - * Returns the {@link AggregateMetric} to update for {@code snapshot}, lazily creating an entry on - * miss. Returns {@code null} when the table is at capacity and no stale entry can be evicted -- - * the caller should drop the data point in that case. + * Returns the {@link AggregateEntry} to update for {@code snapshot}, lazily creating one on miss. + * Returns {@code null} when the table is at capacity and no stale entry can be evicted -- the + * caller should drop the data point in that case. */ - AggregateMetric findOrInsert(SpanSnapshot snapshot) { + AggregateEntry findOrInsert(SpanSnapshot snapshot) { long keyHash = AggregateEntry.hashOf(snapshot); int bucketIndex = Hashtable.Support.bucketIndex(buckets, keyHash); for (Hashtable.Entry e = buckets[bucketIndex]; e != null; e = e.next()) { if (e.keyHash == keyHash) { AggregateEntry candidate = (AggregateEntry) e; if (candidate.matches(snapshot)) { - return candidate.aggregate; + return candidate; } } } if (size >= maxAggregates && !evictOneStale()) { return null; } - AggregateEntry entry = AggregateEntry.forSnapshot(snapshot, new AggregateMetric()); + AggregateEntry entry = AggregateEntry.forSnapshot(snapshot); entry.setNext(buckets[bucketIndex]); buckets[bucketIndex] = entry; size++; - return entry.aggregate; + return entry; } - /** Unlink the first entry whose {@code AggregateMetric.getHitCount() == 0}. */ + /** Unlink the first entry whose {@code getHitCount() == 0}. */ private boolean evictOneStale() { for (int i = 0; i < buckets.length; i++) { Hashtable.Entry head = buckets[i]; if (head == null) { continue; } - if (((AggregateEntry) head).aggregate.getHitCount() == 0) { + if (((AggregateEntry) head).getHitCount() == 0) { buckets[i] = head.next(); size--; return true; @@ -75,7 +76,7 @@ private boolean evictOneStale() { Hashtable.Entry prev = head; Hashtable.Entry cur = head.next(); while (cur != null) { - if (((AggregateEntry) cur).aggregate.getHitCount() == 0) { + if (((AggregateEntry) cur).getHitCount() == 0) { prev.setNext(cur.next()); size--; return true; @@ -95,12 +96,12 @@ void forEach(Consumer consumer) { } } - /** Removes entries whose {@code AggregateMetric.getHitCount() == 0}. */ + /** Removes entries whose {@code getHitCount() == 0}. */ void expungeStaleAggregates() { for (int i = 0; i < buckets.length; i++) { // unlink leading stale entries Hashtable.Entry head = buckets[i]; - while (head != null && ((AggregateEntry) head).aggregate.getHitCount() == 0) { + while (head != null && ((AggregateEntry) head).getHitCount() == 0) { head = head.next(); size--; } @@ -112,7 +113,7 @@ void expungeStaleAggregates() { Hashtable.Entry prev = head; Hashtable.Entry cur = head.next(); while (cur != null) { - if (((AggregateEntry) cur).aggregate.getHitCount() == 0) { + if (((AggregateEntry) cur).getHitCount() == 0) { Hashtable.Entry skipped = cur.next(); prev.setNext(skipped); size--; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index b4fc59d5a1d..902d405db3a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -115,9 +115,9 @@ public void accept(InboxItem item) { } } else if (item instanceof SpanSnapshot && !stopped) { SpanSnapshot snapshot = (SpanSnapshot) item; - AggregateMetric aggregate = aggregates.findOrInsert(snapshot); - if (aggregate != null) { - aggregate.recordOneDuration(snapshot.tagAndDuration); + AggregateEntry entry = aggregates.findOrInsert(snapshot); + if (entry != null) { + entry.recordOneDuration(snapshot.tagAndDuration); dirty = true; } else { // table at cap with no stale entry available to evict @@ -138,7 +138,7 @@ private void report(long when, SignalItem signal) { aggregates.forEach( entry -> { writer.add(entry); - entry.aggregate.clear(); + entry.clear(); }); // note that this may do IO and block writer.finishBucket(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index c675fcb23c4..601f8cdb76b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -6,8 +6,8 @@ import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_ENDPOINT; import static datadog.trace.bootstrap.instrumentation.api.Tags.HTTP_METHOD; import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; -import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG; -import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG; +import static datadog.trace.common.metrics.AggregateEntry.ERROR_TAG; +import static datadog.trace.common.metrics.AggregateEntry.TOP_LEVEL_TAG; import static datadog.trace.common.metrics.SignalItem.ClearSignal.CLEAR; import static datadog.trace.common.metrics.SignalItem.ReportSignal.REPORT; import static datadog.trace.common.metrics.SignalItem.StopSignal.STOP; diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java index c31825f6af8..905ba498760 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/MetricWriter.java @@ -5,7 +5,7 @@ public interface MetricWriter { /** * Serialize one aggregate. The {@link AggregateEntry} carries both the label fields (resource, - * service, span.kind, peer tags, etc.) and the {@link AggregateMetric} counters being reported. + * service, span.kind, peer tags, etc.) and the counters being reported. */ void add(AggregateEntry entry); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java index ba6ae6c2699..7644ebaf044 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SerializingMetricWriter.java @@ -143,7 +143,6 @@ public void startBucket(int metricCount, long start, long duration) { @Override public void add(AggregateEntry entry) { - final AggregateMetric aggregate = entry.aggregate; // Calculate dynamic map size based on optional fields final boolean hasHttpMethod = entry.getHttpMethod() != null; final boolean hasHttpEndpoint = entry.getHttpEndpoint() != null; @@ -213,22 +212,22 @@ public void add(AggregateEntry entry) { } writer.writeUTF8(HITS); - writer.writeInt(aggregate.getHitCount()); + writer.writeInt(entry.getHitCount()); writer.writeUTF8(ERRORS); - writer.writeInt(aggregate.getErrorCount()); + writer.writeInt(entry.getErrorCount()); writer.writeUTF8(TOP_LEVEL_HITS); - writer.writeInt(aggregate.getTopLevelCount()); + writer.writeInt(entry.getTopLevelCount()); writer.writeUTF8(DURATION); - writer.writeLong(aggregate.getDuration()); + writer.writeLong(entry.getDuration()); writer.writeUTF8(OK_SUMMARY); - writer.writeBinary(aggregate.getOkLatencies().serialize()); + writer.writeBinary(entry.getOkLatencies().serialize()); writer.writeUTF8(ERROR_SUMMARY); - writer.writeBinary(aggregate.getErrorLatencies().serialize()); + writer.writeBinary(entry.getErrorLatencies().serialize()); } @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java index b7f81712945..df213797d5b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/SpanSnapshot.java @@ -2,8 +2,8 @@ /** * Immutable per-span value posted from the producer to the aggregator thread. Carries the raw - * inputs the aggregator needs to build an {@link AggregateEntry} and update its {@link - * AggregateMetric}. + * inputs the aggregator needs to look up or build an {@link AggregateEntry} and update its + * counters. * *

    All cache-canonicalization (service-name, span-kind, peer-tag string interning) happens on the * aggregator thread; the producer just shuffles references. diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/AggregateMetricTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/AggregateMetricTest.groovy deleted file mode 100644 index 140149d8324..00000000000 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/AggregateMetricTest.groovy +++ /dev/null @@ -1,105 +0,0 @@ -package datadog.trace.common.metrics - -import datadog.metrics.agent.AgentMeter -import datadog.metrics.impl.DDSketchHistograms -import datadog.metrics.impl.MonitoringImpl -import datadog.metrics.api.statsd.StatsDClient -import datadog.trace.test.util.DDSpecification - -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLongArray - -import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG -import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG - -class AggregateMetricTest extends DDSpecification { - - def setupSpec() { - // Initialize AgentMeter with monitoring - this is the standard mechanism used in production - def monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS) - AgentMeter.registerIfAbsent(StatsDClient.NO_OP, monitoring, DDSketchHistograms.FACTORY) - // Create a timer to trigger DDSketchHistograms loading and Factory registration - // This simulates what happens during CoreTracer initialization (traceWriteTimer) - monitoring.newTimer("test.init") - } - - def "record durations sums up to total"() { - given: - AggregateMetric aggregate = new AggregateMetric() - when: - aggregate.recordDurations(3, new AtomicLongArray(1, 2, 3)) - then: - aggregate.getDuration() == 6 - } - - def "total durations include errors"() { - given: - AggregateMetric aggregate = new AggregateMetric() - when: - aggregate.recordDurations(3, new AtomicLongArray(1, 2, 3)) - then: - aggregate.getDuration() == 6 - } - - def "clear"() { - given: - AggregateMetric aggregate = new AggregateMetric() - .recordDurations(3, new AtomicLongArray(5, ERROR_TAG | 6, TOP_LEVEL_TAG | 7)) - when: - aggregate.clear() - then: - aggregate.getDuration() == 0 - aggregate.getErrorCount() == 0 - aggregate.getTopLevelCount() == 0 - aggregate.getHitCount() == 0 - } - - def "recordOneDuration accumulates ok and error and top-level"() { - given: - AggregateMetric aggregate = new AggregateMetric() - .recordOneDuration(10L) - .recordOneDuration(10L | TOP_LEVEL_TAG) - .recordOneDuration(10L | ERROR_TAG) - - expect: - aggregate.getHitCount() == 3 - aggregate.getDuration() == 30 - aggregate.getErrorCount() == 1 - aggregate.getTopLevelCount() == 1 - } - - def "ignore trailing zeros"() { - given: - AggregateMetric aggregate = new AggregateMetric() - when: - aggregate.recordDurations(3, new AtomicLongArray(1, 2, 3, 0, 0, 0)) - then: - aggregate.getDuration() == 6 - aggregate.getHitCount() == 3 - aggregate.getErrorCount() == 0 - } - - def "hit count includes errors"() { - given: - AggregateMetric aggregate = new AggregateMetric() - when: - aggregate.recordDurations(3, new AtomicLongArray(1, 2, 3 | ERROR_TAG)) - then: - aggregate.getHitCount() == 3 - aggregate.getErrorCount() == 1 - } - - def "ok and error durations tracked separately"() { - given: - AggregateMetric aggregate = new AggregateMetric() - when: - aggregate.recordDurations(10, - new AtomicLongArray(1, 100 | ERROR_TAG, 2, 99 | ERROR_TAG, 3, - 98 | ERROR_TAG, 4, 97 | ERROR_TAG)) - then: - def errorLatencies = aggregate.getErrorLatencies() - def okLatencies = aggregate.getOkLatencies() - errorLatencies.getMaxValue() >= 99 - okLatencies.getMaxValue() <= 5 - } -} diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy index 4dd0155443a..3e58a8e68a6 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy @@ -134,7 +134,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -180,7 +180,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -232,7 +232,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { httpEndpoint, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } (statsComputed ? 1 : 0) * writer.finishBucket() >> { latch.countDown() } @@ -294,7 +294,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.add( AggregateEntry.of( @@ -312,7 +312,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -359,7 +359,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 0 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -411,7 +411,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == topLevelCount && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == topLevelCount && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -470,7 +470,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration + e.getHitCount() == count && e.getDuration() == count * duration } 1 * writer.add(AggregateEntry.of( "resource2", @@ -487,7 +487,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration * 2 + e.getHitCount() == count && e.getDuration() == count * duration * 2 } cleanup: @@ -541,7 +541,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == count && e.aggregate.getDuration() == count * duration + e.getHitCount() == count && e.getDuration() == count * duration } 1 * writer.finishBucket() >> { latch.countDown() } @@ -582,7 +582,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -599,7 +599,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.add(AggregateEntry.of( "resource", @@ -616,7 +616,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 3 + e.getHitCount() == 1 && e.getDuration() == duration * 3 } 1 * writer.finishBucket() >> { latch2.countDown() } @@ -680,7 +680,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -697,7 +697,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.add(AggregateEntry.of( "resource", @@ -714,7 +714,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 3 + e.getHitCount() == 1 && e.getDuration() == duration * 3 } 1 * writer.add(AggregateEntry.of( "resource", @@ -731,7 +731,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 4 + e.getHitCount() == 1 && e.getDuration() == duration * 4 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -784,7 +784,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -801,7 +801,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration * 2 + e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -852,7 +852,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 2 && e.aggregate.getDuration() == 2 * duration + e.getHitCount() == 2 && e.getDuration() == 2 * duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -869,7 +869,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.finishBucket() >> { latch.countDown() } @@ -923,7 +923,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } } 0 * writer.add(AggregateEntry.of( @@ -1070,7 +1070,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1105,7 +1105,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } } 0 * writer.add(AggregateEntry.of( @@ -1172,7 +1172,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1231,7 +1231,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getDuration() == duration + e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1398,7 +1398,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1453,7 +1453,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 3 && e.aggregate.getTopLevelCount() == 3 && e.aggregate.getDuration() == 450 + e.getHitCount() == 3 && e.getTopLevelCount() == 3 && e.getDuration() == 450 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1508,7 +1508,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 100 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.add( AggregateEntry.of( @@ -1526,7 +1526,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders", null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 200 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 200 } 1 * writer.add( AggregateEntry.of( @@ -1544,7 +1544,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.aggregate.getHitCount() == 1 && e.aggregate.getTopLevelCount() == 1 && e.aggregate.getDuration() == 150 + e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 150 } 1 * writer.finishBucket() >> { latch.countDown() } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy index 08f0f7cbb92..5e85c66557d 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy @@ -45,7 +45,7 @@ class SerializingMetricWriterTest extends DDSpecification { resource, service, operationName, serviceSource, type, httpStatusCode, synthetic, traceRoot, spanKind, peerTags, httpMethod, httpEndpoint, grpcStatusCode) - e.aggregate.recordDurations(hitCount, new AtomicLongArray(1L)) + e.recordDurations(hitCount, new AtomicLongArray(1L)) return e } @@ -284,7 +284,6 @@ class SerializingMetricWriterTest extends DDSpecification { int statCount = unpacker.unpackArrayHeader() assert statCount == content.size() for (AggregateEntry entry : content) { - AggregateMetric value = entry.aggregate int metricMapSize = unpacker.unpackMapHeader() // Calculate expected map size based on optional fields boolean hasHttpMethod = entry.getHttpMethod() != null @@ -349,16 +348,16 @@ class SerializingMetricWriterTest extends DDSpecification { ++elementCount } assert unpacker.unpackString() == "Hits" - assert unpacker.unpackInt() == value.getHitCount() + assert unpacker.unpackInt() == entry.getHitCount() ++elementCount assert unpacker.unpackString() == "Errors" - assert unpacker.unpackInt() == value.getErrorCount() + assert unpacker.unpackInt() == entry.getErrorCount() ++elementCount assert unpacker.unpackString() == "TopLevelHits" - assert unpacker.unpackInt() == value.getTopLevelCount() + assert unpacker.unpackInt() == entry.getTopLevelCount() ++elementCount assert unpacker.unpackString() == "Duration" - assert unpacker.unpackLong() == value.getDuration() + assert unpacker.unpackLong() == entry.getDuration() ++elementCount assert unpacker.unpackString() == "OkSummary" validateSketch(unpacker) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java new file mode 100644 index 00000000000..08362213969 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java @@ -0,0 +1,108 @@ +package datadog.trace.common.metrics; + +import static datadog.trace.common.metrics.AggregateEntry.ERROR_TAG; +import static datadog.trace.common.metrics.AggregateEntry.TOP_LEVEL_TAG; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.metrics.agent.AgentMeter; +import datadog.metrics.api.statsd.StatsDClient; +import datadog.metrics.impl.DDSketchHistograms; +import datadog.metrics.impl.MonitoringImpl; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLongArray; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class AggregateEntryTest { + + @BeforeAll + static void initAgentMeter() { + // recordOneDuration -> Histogram.accept needs AgentMeter to be initialized. + MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); + AgentMeter.registerIfAbsent(StatsDClient.NO_OP, monitoring, DDSketchHistograms.FACTORY); + monitoring.newTimer("test.init"); + } + + @Test + void recordDurationsSumsToTotal() { + AggregateEntry entry = newEntry(); + entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L})); + assertEquals(6, entry.getDuration()); + } + + @Test + void clearResetsAllCounters() { + AggregateEntry entry = newEntry(); + entry.recordDurations( + 3, new AtomicLongArray(new long[] {5L, ERROR_TAG | 6L, TOP_LEVEL_TAG | 7L})); + entry.clear(); + assertEquals(0, entry.getDuration()); + assertEquals(0, entry.getErrorCount()); + assertEquals(0, entry.getTopLevelCount()); + assertEquals(0, entry.getHitCount()); + } + + @Test + void recordOneDurationAccumulatesOkErrorAndTopLevel() { + AggregateEntry entry = newEntry(); + entry.recordOneDuration(10L); + entry.recordOneDuration(10L | TOP_LEVEL_TAG); + entry.recordOneDuration(10L | ERROR_TAG); + + assertEquals(3, entry.getHitCount()); + assertEquals(30, entry.getDuration()); + assertEquals(1, entry.getErrorCount()); + assertEquals(1, entry.getTopLevelCount()); + } + + @Test + void recordDurationsIgnoresTrailingZeros() { + AggregateEntry entry = newEntry(); + entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L, 0L, 0L, 0L})); + assertEquals(6, entry.getDuration()); + assertEquals(3, entry.getHitCount()); + assertEquals(0, entry.getErrorCount()); + } + + @Test + void hitCountIncludesErrors() { + AggregateEntry entry = newEntry(); + entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L | ERROR_TAG})); + assertEquals(3, entry.getHitCount()); + assertEquals(1, entry.getErrorCount()); + } + + @Test + void okAndErrorLatenciesTrackedSeparately() { + AggregateEntry entry = newEntry(); + entry.recordDurations( + 10, + new AtomicLongArray( + new long[] { + 1L, 100L | ERROR_TAG, 2L, 99L | ERROR_TAG, 3L, 98L | ERROR_TAG, 4L, 97L | ERROR_TAG + })); + assertTrue(entry.getErrorLatencies().getMaxValue() >= 99); + assertTrue(entry.getOkLatencies().getMaxValue() <= 5); + } + + private static AggregateEntry newEntry() { + SpanSnapshot snapshot = + new SpanSnapshot( + "resource", + "svc", + "op", + null, + "type", + (short) 200, + false, + true, + "client", + null, + null, + null, + null, + 0L); + return AggregateEntry.forSnapshot(snapshot); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java index 44f2b36cb6b..4af53f25c5b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -1,7 +1,7 @@ package datadog.trace.common.metrics; -import static datadog.trace.common.metrics.AggregateMetric.ERROR_TAG; -import static datadog.trace.common.metrics.AggregateMetric.TOP_LEVEL_TAG; +import static datadog.trace.common.metrics.AggregateEntry.ERROR_TAG; +import static datadog.trace.common.metrics.AggregateEntry.TOP_LEVEL_TAG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; @@ -25,8 +25,7 @@ class AggregateTableTest { @BeforeAll static void initAgentMeter() { - // AggregateMetric.recordOneDuration -> Histogram.accept needs AgentMeter to be initialized. - // Mirror what AggregateMetricTest does. + // AggregateEntry.recordOneDuration -> Histogram.accept needs AgentMeter to be initialized. MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS); AgentMeter.registerIfAbsent(StatsDClient.NO_OP, monitoring, DDSketchHistograms.FACTORY); monitoring.newTimer("test.init"); @@ -37,7 +36,7 @@ void insertOnMissReturnsNewAggregate() { AggregateTable table = new AggregateTable(8); SpanSnapshot s = snapshot("svc", "op", "client"); - AggregateMetric agg = table.findOrInsert(s); + AggregateEntry agg = table.findOrInsert(s); assertNotNull(agg); assertEquals(1, table.size()); @@ -50,8 +49,8 @@ void hitReturnsSameAggregateInstance() { SpanSnapshot s1 = snapshot("svc", "op", "client"); SpanSnapshot s2 = snapshot("svc", "op", "client"); - AggregateMetric first = table.findOrInsert(s1); - AggregateMetric second = table.findOrInsert(s2); + AggregateEntry first = table.findOrInsert(s1); + AggregateEntry second = table.findOrInsert(s2); assertSame(first, second); assertEquals(1, table.size()); @@ -61,8 +60,8 @@ void hitReturnsSameAggregateInstance() { void differentKindFieldsAreDistinct() { AggregateTable table = new AggregateTable(8); - AggregateMetric clientAgg = table.findOrInsert(snapshot("svc", "op", "client")); - AggregateMetric serverAgg = table.findOrInsert(snapshot("svc", "op", "server")); + AggregateEntry clientAgg = table.findOrInsert(snapshot("svc", "op", "client")); + AggregateEntry serverAgg = table.findOrInsert(snapshot("svc", "op", "server")); assertNotSame(clientAgg, serverAgg); assertEquals(2, table.size()); @@ -77,9 +76,9 @@ void peerTagPairsParticipateInIdentity() { builder("svc", "op", "client").peerTags("peer.hostname", "host-b").build(); SpanSnapshot noTags = builder("svc", "op", "client").build(); - AggregateMetric a = table.findOrInsert(withTags); - AggregateMetric b = table.findOrInsert(otherTags); - AggregateMetric c = table.findOrInsert(noTags); + AggregateEntry a = table.findOrInsert(withTags); + AggregateEntry b = table.findOrInsert(otherTags); + AggregateEntry c = table.findOrInsert(noTags); assertNotSame(a, b); assertNotSame(a, c); @@ -91,19 +90,19 @@ void peerTagPairsParticipateInIdentity() { void capOverrunEvictsStaleEntry() { AggregateTable table = new AggregateTable(2); - AggregateMetric stale = table.findOrInsert(snapshot("svc-a", "op", "client")); + AggregateEntry stale = table.findOrInsert(snapshot("svc-a", "op", "client")); // do not record on stale -> hitCount stays at 0 - AggregateMetric live = table.findOrInsert(snapshot("svc-b", "op", "client")); + AggregateEntry live = table.findOrInsert(snapshot("svc-b", "op", "client")); live.recordOneDuration(10L | TOP_LEVEL_TAG); // hitCount=1, not evictable // table is full (size=2). Inserting a third should evict the stale one and succeed. - AggregateMetric newcomer = table.findOrInsert(snapshot("svc-c", "op", "client")); + AggregateEntry newcomer = table.findOrInsert(snapshot("svc-c", "op", "client")); assertNotNull(newcomer); assertEquals(2, table.size()); // re-inserting the stale snapshot should miss now (it was evicted) and produce a fresh entry - AggregateMetric staleAgain = table.findOrInsert(snapshot("svc-a", "op", "client")); + AggregateEntry staleAgain = table.findOrInsert(snapshot("svc-a", "op", "client")); assertNotSame(stale, staleAgain); } @@ -111,12 +110,12 @@ void capOverrunEvictsStaleEntry() { void capOverrunWithNoStaleReturnsNull() { AggregateTable table = new AggregateTable(2); - AggregateMetric a = table.findOrInsert(snapshot("svc-a", "op", "client")); - AggregateMetric b = table.findOrInsert(snapshot("svc-b", "op", "client")); + AggregateEntry a = table.findOrInsert(snapshot("svc-a", "op", "client")); + AggregateEntry b = table.findOrInsert(snapshot("svc-b", "op", "client")); a.recordOneDuration(10L); b.recordOneDuration(20L); - AggregateMetric c = table.findOrInsert(snapshot("svc-c", "op", "client")); + AggregateEntry c = table.findOrInsert(snapshot("svc-c", "op", "client")); assertNull(c); assertEquals(2, table.size()); } @@ -125,10 +124,10 @@ void capOverrunWithNoStaleReturnsNull() { void expungeStaleAggregatesRemovesZeroHitsOnly() { AggregateTable table = new AggregateTable(16); - AggregateMetric live = table.findOrInsert(snapshot("svc-live", "op", "client")); + AggregateEntry live = table.findOrInsert(snapshot("svc-live", "op", "client")); live.recordOneDuration(10L); - AggregateMetric stale1 = table.findOrInsert(snapshot("svc-stale1", "op", "client")); - AggregateMetric stale2 = table.findOrInsert(snapshot("svc-stale2", "op", "client")); + AggregateEntry stale1 = table.findOrInsert(snapshot("svc-stale1", "op", "client")); + AggregateEntry stale2 = table.findOrInsert(snapshot("svc-stale2", "op", "client")); assertEquals(3, table.size()); assertEquals(0, stale1.getHitCount()); assertEquals(0, stale2.getHitCount()); @@ -148,7 +147,7 @@ void forEachVisitsEveryEntry() { table.findOrInsert(snapshot("c", "op", "client")).recordOneDuration(3L | ERROR_TAG); Map visited = new HashMap<>(); - table.forEach(e -> visited.put(e.getService().toString(), e.aggregate.getDuration())); + table.forEach(e -> visited.put(e.getService().toString(), e.getDuration())); assertEquals(3, visited.size()); assertEquals(1L, visited.get("a")); From 07ed635cdf4d25e13b3f5400c7ee225ec4990432 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:44:12 -0400 Subject: [PATCH 14/70] Avoid capturing lambda in Aggregator.report Add a context-passing forEach(T, BiConsumer) overload to AggregateTable, mirroring TagMap's pattern. Aggregator.report now hands the writer in as context to a static BiConsumer so no fresh Consumer is allocated each report cycle. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateTable.java | 14 ++++++++++++++ .../datadog/trace/common/metrics/Aggregator.java | 14 +++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 3bc3766227d..8b426985a68 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -1,6 +1,7 @@ package datadog.trace.common.metrics; import datadog.trace.util.Hashtable; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -96,6 +97,19 @@ void forEach(Consumer consumer) { } } + /** + * Context-passing forEach. Useful for callers that want to avoid a capturing-lambda allocation on + * each invocation -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) + * plus whatever side-band state it needs as {@code context}. + */ + void forEach(T context, BiConsumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + consumer.accept(context, (AggregateEntry) e); + } + } + } + /** Removes entries whose {@code getHitCount() == 0}. */ void expungeStaleAggregates() { for (int i = 0; i < buckets.length; i++) { diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index 902d405db3a..816b5463424 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -7,6 +7,7 @@ import datadog.trace.core.monitor.HealthMetrics; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import org.jctools.queues.MessagePassingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,6 +16,13 @@ final class Aggregator implements Runnable { private static final long DEFAULT_SLEEP_MILLIS = 10; + /** Non-capturing -- the writer arrives via the forEach context arg. */ + private static final BiConsumer WRITE_AND_CLEAR = + (writer, entry) -> { + writer.add(entry); + entry.clear(); + }; + private static final Logger log = LoggerFactory.getLogger(Aggregator.class); private final MessagePassingQueue inbox; @@ -135,11 +143,7 @@ private void report(long when, SignalItem signal) { if (!aggregates.isEmpty()) { skipped = false; writer.startBucket(aggregates.size(), when, reportingIntervalNanos); - aggregates.forEach( - entry -> { - writer.add(entry); - entry.clear(); - }); + aggregates.forEach(writer, WRITE_AND_CLEAR); // note that this may do IO and block writer.finishBucket(); } From df58ad76f6ac96fe7ad0560cc84a65caf2a50fde Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:49:03 -0400 Subject: [PATCH 15/70] Add context-passing forEach to Hashtable.D1 and D2 Mirrors the TagMap pattern: pairs the existing forEach(Consumer) with a forEach(T context, BiConsumer) overload so callers can hand side-band state to a non-capturing lambda and avoid the fresh-Consumer-per-call allocation. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 31 +++++++++++++++++++ .../datadog/trace/util/HashtableD1Test.java | 22 +++++++++++++ .../datadog/trace/util/HashtableD2Test.java | 12 +++++++ 3 files changed, 65 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index e527ae45fcc..f4c26f88d99 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -4,6 +4,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -193,6 +194,21 @@ public void forEach(Consumer consumer) { } } } + + /** + * Context-passing forEach. Useful for callers that want to avoid a capturing-lambda allocation + * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever + * side-band state it needs as {@code context}. + */ + @SuppressWarnings("unchecked") + public void forEach(T context, BiConsumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** @@ -340,6 +356,21 @@ public void forEach(Consumer consumer) { } } } + + /** + * Context-passing forEach. Useful for callers that want to avoid a capturing-lambda allocation + * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever + * side-band state it needs as {@code context}. + */ + @SuppressWarnings("unchecked") + public void forEach(T context, BiConsumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java index 10d8ad41976..11928bb4d5b 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -119,6 +119,28 @@ void forEachVisitsEveryInsertedEntry() { assertEquals(3, seen.get("c")); } + @Test + void forEachWithContextPassesContextToConsumer() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 10)); + table.insert(new StringIntEntry("b", 20)); + table.insert(new StringIntEntry("c", 30)); + Map seen = new HashMap<>(); + table.forEach(seen, (ctx, e) -> ctx.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(10, seen.get("a")); + assertEquals(20, seen.get("b")); + assertEquals(30, seen.get("c")); + } + + @Test + void forEachWithContextOnEmptyTableDoesNothing() { + Hashtable.D1 table = new Hashtable.D1<>(8); + Map seen = new HashMap<>(); + table.forEach(seen, (ctx, e) -> ctx.put(e.key, e.value)); + assertEquals(0, seen.size()); + } + @Test void nullKeyIsPermittedAndDistinctFromAbsent() { Hashtable.D1 table = new Hashtable.D1<>(8); diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java index 98c54b71c2c..59339fcd89e 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -65,6 +65,18 @@ void forEachVisitsBothPairs() { assertTrue(seen.contains("b:2")); } + @Test + void forEachWithContextPassesContextToConsumer() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(seen, (ctx, e) -> ctx.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + private static final class PairEntry extends Hashtable.D2.Entry { int value; From 9c6e95c161d99929ea33a2a5ea6a060b2422e66a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:58:43 -0400 Subject: [PATCH 16/70] Move forEach loop body to Support helper Factors the unchecked (TEntry) cast out of D1.forEach / D2.forEach (and the BiConsumer variants) into Support.forEach(buckets, ...). The cast now lives in one place, mirroring how Entry.next() handles it, and the D1/D2 methods become one-liners. Downstream higher-arity tables built on Support gain the same helper. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index f4c26f88d99..137118fc111 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -185,14 +185,8 @@ public void clear() { this.size = 0; } - @SuppressWarnings("unchecked") public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } + Support.forEach(this.buckets, consumer); } /** @@ -200,14 +194,8 @@ public void forEach(Consumer consumer) { * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever * side-band state it needs as {@code context}. */ - @SuppressWarnings("unchecked") public void forEach(T context, BiConsumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept(context, (TEntry) e); - } - } + Support.forEach(this.buckets, context, consumer); } } @@ -347,14 +335,8 @@ public void clear() { this.size = 0; } - @SuppressWarnings("unchecked") public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } + Support.forEach(this.buckets, consumer); } /** @@ -362,14 +344,8 @@ public void forEach(Consumer consumer) { * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever * side-band state it needs as {@code context}. */ - @SuppressWarnings("unchecked") public void forEach(T context, BiConsumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept(context, (TEntry) e); - } - } + Support.forEach(this.buckets, context, consumer); } } @@ -388,6 +364,8 @@ public void forEach(T context, BiConsumer consume * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / * {@code replace}. + *

  • Iterate every entry with {@link #forEach(Hashtable.Entry[], Consumer)} or its + * context-passing sibling. *
  • Clear with {@link #clear(Hashtable.Entry[])}. * * @@ -436,6 +414,36 @@ MutatingBucketIterator mutatingBucketIterator( public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + + /** + * Walks every entry in {@code buckets} and invokes {@code consumer} on it. The unchecked cast + * to {@code TEntry} lives here (mirroring {@link Entry#next()}) so callers don't have to + * sprinkle it across their own forEach loops. + */ + @SuppressWarnings("unchecked") + public static final void forEach( + Hashtable.Entry[] buckets, Consumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + + /** + * Context-passing variant of {@link #forEach(Hashtable.Entry[], Consumer)}. Pair a + * non-capturing {@link BiConsumer} (typically a {@code static final}) with side-band state + * passed as {@code context} to avoid a fresh-Consumer allocation each call. + */ + @SuppressWarnings("unchecked") + public static final void forEach( + Hashtable.Entry[] buckets, T context, BiConsumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** From 590ab4a37b87f6292c35dc0c3e1d94ebac58645e Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 14:01:38 -0400 Subject: [PATCH 17/70] Delegate AggregateTable.forEach to Support.forEach Now that Hashtable.Support exposes the parameterized forEach helpers, AggregateTable's own forEach methods can drop their duplicated loop body and the (AggregateEntry) cast. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/common/metrics/AggregateTable.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 8b426985a68..03df25849e0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -90,11 +90,7 @@ private boolean evictOneStale() { } void forEach(Consumer consumer) { - for (int i = 0; i < buckets.length; i++) { - for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { - consumer.accept((AggregateEntry) e); - } - } + Hashtable.Support.forEach(buckets, consumer); } /** @@ -103,11 +99,7 @@ void forEach(Consumer consumer) { * plus whatever side-band state it needs as {@code context}. */ void forEach(T context, BiConsumer consumer) { - for (int i = 0; i < buckets.length; i++) { - for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { - consumer.accept(context, (AggregateEntry) e); - } - } + Hashtable.Support.forEach(buckets, context, consumer); } /** Removes entries whose {@code getHitCount() == 0}. */ From 447ea33c72322fd1e155886871cf6cbcc2cb18bb Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 14:32:29 -0400 Subject: [PATCH 18/70] Move bucket-head cast to Support.bucket helper Adds Support.bucket(buckets, keyHash) which returns the bucket head already cast to the caller's concrete entry type. D1.get and D2.get now drop the raw-Entry intermediate variable and walk the chain via Entry.next() directly. The unchecked cast lives in one place, consistent with Entry.next() and Support.forEach. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 137118fc111..4945aed5a0f 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -113,16 +113,11 @@ public int size() { return this.size; } - @SuppressWarnings("unchecked") public TEntry get(K key) { long keyHash = D1.Entry.hash(key); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; - e != null; - e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key)) return te; + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key)) { + return te; } } return null; @@ -263,16 +258,11 @@ public int size() { return this.size; } - @SuppressWarnings("unchecked") public TEntry get(K1 key1, K2 key2) { long keyHash = D2.Entry.hash(key1, key2); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; - e != null; - e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key1, key2)) return te; + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key1, key2)) { + return te; } } return null; @@ -415,6 +405,17 @@ public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + /** + * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's + * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site + * doesn't need to thread a raw {@link Entry} variable through. + */ + @SuppressWarnings("unchecked") + public static final TEntry bucket( + Hashtable.Entry[] buckets, long keyHash) { + return (TEntry) buckets[bucketIndex(buckets, keyHash)]; + } + /** * Walks every entry in {@code buckets} and invokes {@code consumer} on it. The unchecked cast * to {@code TEntry} lives here (mirroring {@link Entry#next()}) so callers don't have to From dd5e13fa10b682864685d81ddce8fde0e1259a28 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 14:37:33 -0400 Subject: [PATCH 19/70] Use Support.bucket and type chain walks as AggregateEntry - findOrInsert: walks via Support.bucket(buckets, keyHash) instead of Hashtable.Entry + intermediate cast; bucketIndex is only computed on the miss path now. - evictOneStale / expungeStaleAggregates: chain variables typed as AggregateEntry from the head down, leveraging Entry.next()'s generic inference, so the per-iteration getHitCount() checks drop their (AggregateEntry) cast. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateTable.java | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 03df25849e0..8daf468e2a8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -43,19 +43,18 @@ boolean isEmpty() { */ AggregateEntry findOrInsert(SpanSnapshot snapshot) { long keyHash = AggregateEntry.hashOf(snapshot); - int bucketIndex = Hashtable.Support.bucketIndex(buckets, keyHash); - for (Hashtable.Entry e = buckets[bucketIndex]; e != null; e = e.next()) { - if (e.keyHash == keyHash) { - AggregateEntry candidate = (AggregateEntry) e; - if (candidate.matches(snapshot)) { - return candidate; - } + for (AggregateEntry candidate = Hashtable.Support.bucket(buckets, keyHash); + candidate != null; + candidate = candidate.next()) { + if (candidate.keyHash == keyHash && candidate.matches(snapshot)) { + return candidate; } } if (size >= maxAggregates && !evictOneStale()) { return null; } AggregateEntry entry = AggregateEntry.forSnapshot(snapshot); + int bucketIndex = Hashtable.Support.bucketIndex(buckets, keyHash); entry.setNext(buckets[bucketIndex]); buckets[bucketIndex] = entry; size++; @@ -65,19 +64,19 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { /** Unlink the first entry whose {@code getHitCount() == 0}. */ private boolean evictOneStale() { for (int i = 0; i < buckets.length; i++) { - Hashtable.Entry head = buckets[i]; + AggregateEntry head = (AggregateEntry) buckets[i]; if (head == null) { continue; } - if (((AggregateEntry) head).getHitCount() == 0) { + if (head.getHitCount() == 0) { buckets[i] = head.next(); size--; return true; } - Hashtable.Entry prev = head; - Hashtable.Entry cur = head.next(); + AggregateEntry prev = head; + AggregateEntry cur = head.next(); while (cur != null) { - if (((AggregateEntry) cur).getHitCount() == 0) { + if (cur.getHitCount() == 0) { prev.setNext(cur.next()); size--; return true; @@ -106,8 +105,8 @@ void forEach(T context, BiConsumer consumer) { void expungeStaleAggregates() { for (int i = 0; i < buckets.length; i++) { // unlink leading stale entries - Hashtable.Entry head = buckets[i]; - while (head != null && ((AggregateEntry) head).getHitCount() == 0) { + AggregateEntry head = (AggregateEntry) buckets[i]; + while (head != null && head.getHitCount() == 0) { head = head.next(); size--; } @@ -116,11 +115,11 @@ void expungeStaleAggregates() { continue; } // unlink stale entries in the chain - Hashtable.Entry prev = head; - Hashtable.Entry cur = head.next(); + AggregateEntry prev = head; + AggregateEntry cur = head.next(); while (cur != null) { - if (((AggregateEntry) cur).getHitCount() == 0) { - Hashtable.Entry skipped = cur.next(); + if (cur.getHitCount() == 0) { + AggregateEntry skipped = cur.next(); prev.setNext(skipped); size--; cur = skipped; From df7f98f95d26e2a0907e43f0b0b0e51e7beee9c0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 15:28:50 -0400 Subject: [PATCH 20/70] Drop d1_/d2_ prefix from per-table benchmark methods Holdover from when both lived in a shared HashtableBenchmark; redundant now that each lives in its own class. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableD1Benchmark.java | 26 +++++++++---------- .../trace/util/HashtableD2Benchmark.java | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index 16b95e089d5..f8ba7177e88 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -44,15 +44,15 @@ * Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableD1Benchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us - * HashtableD1Benchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD1Benchmark.add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableD1Benchmark.add_hashtable thrpt 6 198.710 ± 273.035 ops/us * - * HashtableD1Benchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us - * HashtableD1Benchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * HashtableD1Benchmark.update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableD1Benchmark.update_hashtable thrpt 6 1810.244 ± 44.645 ops/us * - * HashtableD1Benchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us - * HashtableD1Benchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * HashtableD1Benchmark.iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableD1Benchmark.iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us * */ @Fork(2) @@ -122,7 +122,7 @@ String nextKey() { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d1_add_hashtable(D1State s) { + public void add_hashtable(D1State s) { Hashtable.D1 t = s.table; String[] keys = s.keys; t.clear(); @@ -133,7 +133,7 @@ public void d1_add_hashtable(D1State s) { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d1_add_hashMap(D1State s) { + public void add_hashMap(D1State s) { HashMap m = s.hashMap; String[] keys = s.keys; m.clear(); @@ -143,24 +143,24 @@ public void d1_add_hashMap(D1State s) { } @Benchmark - public long d1_update_hashtable(D1State s) { + public long update_hashtable(D1State s) { D1Counter e = s.table.get(s.nextKey()); return ++e.count; } @Benchmark - public Long d1_update_hashMap(D1State s) { + public Long update_hashMap(D1State s) { return s.hashMap.merge(s.nextKey(), 1L, Long::sum); } @Benchmark - public void d1_iterate_hashtable(D1State s, Blackhole bh) { + public void iterate_hashtable(D1State s, Blackhole bh) { s.consumer.bh = bh; s.table.forEach(s.consumer); } @Benchmark - public void d1_iterate_hashMap(D1State s, Blackhole bh) { + public void iterate_hashMap(D1State s, Blackhole bh) { for (Map.Entry entry : s.hashMap.entrySet()) { bh.consume(entry.getKey()); bh.consume(entry.getValue()); diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 5fd64ed9a75..6f46a702005 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -48,15 +48,15 @@ * {@code Key2} allocation). Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableD2Benchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us - * HashtableD2Benchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD2Benchmark.add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableD2Benchmark.add_hashtable thrpt 6 216.813 ± 413.236 ops/us * - * HashtableD2Benchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us - * HashtableD2Benchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us + * HashtableD2Benchmark.update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableD2Benchmark.update_hashtable thrpt 6 1445.868 ± 157.705 ops/us * - * HashtableD2Benchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us - * HashtableD2Benchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * HashtableD2Benchmark.iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableD2Benchmark.iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us * */ @Fork(2) @@ -158,7 +158,7 @@ int nextIndex() { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d2_add_hashtable(D2State s) { + public void add_hashtable(D2State s) { Hashtable.D2 t = s.table; String[] k1s = s.k1s; Integer[] k2s = s.k2s; @@ -170,7 +170,7 @@ public void d2_add_hashtable(D2State s) { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d2_add_hashMap(D2State s) { + public void add_hashMap(D2State s) { HashMap m = s.hashMap; String[] k1s = s.k1s; Integer[] k2s = s.k2s; @@ -181,26 +181,26 @@ public void d2_add_hashMap(D2State s) { } @Benchmark - public long d2_update_hashtable(D2State s) { + public long update_hashtable(D2State s) { int i = s.nextIndex(); D2Counter e = s.table.get(s.k1s[i], s.k2s[i]); return ++e.count; } @Benchmark - public Long d2_update_hashMap(D2State s) { + public Long update_hashMap(D2State s) { int i = s.nextIndex(); return s.hashMap.merge(new Key2(s.k1s[i], s.k2s[i]), 1L, Long::sum); } @Benchmark - public void d2_iterate_hashtable(D2State s, Blackhole bh) { + public void iterate_hashtable(D2State s, Blackhole bh) { s.consumer.bh = bh; s.table.forEach(s.consumer); } @Benchmark - public void d2_iterate_hashMap(D2State s, Blackhole bh) { + public void iterate_hashMap(D2State s, Blackhole bh) { for (Map.Entry entry : s.hashMap.entrySet()) { bh.consume(entry.getKey()); bh.consume(entry.getValue()); From e6ecc16a3d7dc9054c753cd7a2348a9dcf2879ce Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 15:58:55 -0400 Subject: [PATCH 21/70] Add Hashtable.Support helpers: MAX_RATIO, insertHeadEntry, MutatingTableIterator Three consumer-facing helpers that callers building higher-arity tables on top of Hashtable.Support kept open-coding: - MAX_RATIO_NUMERATOR / _DENOMINATOR: the 4/3 multiplier for sizing a bucket array from a target working-set under a 75% load factor. - insertHeadEntry(buckets, bucketIndex, entry): the (setNext + array-store) pair for splicing a new entry at the head of a bucket chain. - MutatingTableIterator + Support.mutatingTableIterator(buckets): walks every entry in the table (not filtered by hash) with remove() support, for sweeps like eviction and expunge that aren't keyed to a specific hash. Sibling of MutatingBucketIterator. Tests cover the table-wide iterator at head-of-bucket and mid-chain removal, empty buckets between live entries, exhaustion, and remove-without-next. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 148 ++++++++++++++++- .../datadog/trace/util/HashtableTest.java | 153 ++++++++++++++++++ 2 files changed, 300 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 4945aed5a0f..bada7a8b98b 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -354,8 +354,11 @@ public void forEach(T context, BiConsumer consume * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / * {@code replace}. + *
  • Use {@link #insertHeadEntry(Hashtable.Entry[], int, Hashtable.Entry)} to splice a new + * entry as the head of a bucket chain. *
  • Iterate every entry with {@link #forEach(Hashtable.Entry[], Consumer)} or its - * context-passing sibling. + * context-passing sibling. For full-table sweeps with {@code remove}, use {@link + * #mutatingTableIterator(Hashtable.Entry[])}. *
  • Clear with {@link #clear(Hashtable.Entry[])}. * * @@ -372,6 +375,17 @@ public static final Hashtable.Entry[] create(int capacity) { static final int MAX_CAPACITY = 1 << 30; + /** + * Numerator/denominator pair for the inverse of a 75% load factor. Callers that size their + * bucket array from a target working-set size {@code n} should pass {@code n * + * MAX_RATIO_NUMERATOR / MAX_RATIO_DENOMINATOR} to {@link #create(int)} (or {@link + * #sizeFor(int)}) to leave ~25% headroom in the array. Kept as separate ints so callers can use + * integer arithmetic. + */ + public static final int MAX_RATIO_NUMERATOR = 4; + + public static final int MAX_RATIO_DENOMINATOR = 3; + static final int sizeFor(int requestedCapacity) { if (requestedCapacity < 0) { throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); @@ -401,10 +415,29 @@ MutatingBucketIterator mutatingBucketIterator( return new MutatingBucketIterator(buckets, keyHash); } + /** + * Returns a {@link MutatingTableIterator} over every entry in {@code buckets}. Useful for + * sweeps -- eviction, expunge -- that aren't keyed to a specific hash. + */ + public static final + MutatingTableIterator mutatingTableIterator(Hashtable.Entry[] buckets) { + return new MutatingTableIterator(buckets); + } + public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + /** + * Splices {@code entry} in as the new head of the chain at {@code bucketIndex}. Caller is + * responsible for size accounting -- this method only touches the chain pointers. + */ + public static final void insertHeadEntry( + Hashtable.Entry[] buckets, int bucketIndex, Hashtable.Entry entry) { + entry.setNext(buckets[bucketIndex]); + buckets[bucketIndex] = entry; + } + /** * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site @@ -607,4 +640,117 @@ void setPrevNext(Hashtable.Entry nextEntry) { } } } + + /** + * Mutating iterator over every entry in a bucket array, regardless of hash. Supports {@link + * #remove()} to unlink the entry last returned by {@link #next()}. + * + *

    Walks buckets in array order; within a bucket, walks the chain head-to-tail. After {@code + * remove}, iteration may continue with another {@link #next()}. + * + *

    Use this for sweeps -- eviction, expunge, full-table cleanup -- that aren't keyed to a + * specific hash. For per-bucket walks keyed to a search hash, use {@link MutatingBucketIterator}. + */ + public static final class MutatingTableIterator + implements Iterator { + private final Hashtable.Entry[] buckets; + + /** + * Index of the bucket holding {@link #nextEntry} (or holding {@link #curEntry} after remove). + */ + private int nextBucketIndex; + + /** + * Predecessor of {@link #nextEntry}, or {@code null} when {@code nextEntry} is the bucket head. + */ + private Hashtable.Entry nextPrevEntry; + + /** Next entry to be returned by {@link #next()}, or {@code null} if iteration is exhausted. */ + private Hashtable.Entry nextEntry; + + /** + * Bucket index that held the entry last returned by {@code next}; {@code -1} after {@code + * remove}. + */ + private int curBucketIndex = -1; + + /** + * Predecessor of the entry last returned by {@code next}, or {@code null} if it was the bucket + * head. + */ + private Hashtable.Entry curPrevEntry; + + /** + * Entry last returned by {@code next}; {@code null} before any call and after {@code remove}. + */ + private Hashtable.Entry curEntry; + + MutatingTableIterator(Hashtable.Entry[] buckets) { + this.buckets = buckets; + seekFromBucket(0); + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry e = this.nextEntry; + if (e == null) throw new NoSuchElementException("no next!"); + + this.curEntry = e; + this.curPrevEntry = this.nextPrevEntry; + this.curBucketIndex = this.nextBucketIndex; + + Hashtable.Entry n = e.next(); + if (n != null) { + this.nextPrevEntry = e; + this.nextEntry = n; + } else { + // walked off the end of this bucket; pick up at the next non-empty bucket + seekFromBucket(this.nextBucketIndex + 1); + } + return (TEntry) e; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); + + if (this.curPrevEntry == null) { + this.buckets[this.curBucketIndex] = oldCurEntry.next(); + } else { + this.curPrevEntry.setNext(oldCurEntry.next()); + } + // If the next entry was the immediate chain successor of oldCurEntry, its predecessor is + // now what came before oldCurEntry (oldCurEntry was just unlinked). + if (this.nextPrevEntry == oldCurEntry) { + this.nextPrevEntry = this.curPrevEntry; + } + this.curEntry = null; + } + + /** + * Advance {@code nextBucketIndex} / {@code nextEntry} to the first non-empty bucket >= {@code + * from}. + */ + private void seekFromBucket(int from) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = from; i < thisBuckets.length; i++) { + Hashtable.Entry head = thisBuckets[i]; + if (head != null) { + this.nextBucketIndex = i; + this.nextPrevEntry = null; + this.nextEntry = head; + return; + } + } + this.nextEntry = null; + this.nextPrevEntry = null; + } + } } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index f78aec1c00f..6fbf0cc752c 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -7,13 +7,17 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.util.Hashtable.BucketIterator; import datadog.trace.util.Hashtable.MutatingBucketIterator; +import datadog.trace.util.Hashtable.MutatingTableIterator; import datadog.trace.util.Hashtable.Support; +import java.util.HashSet; import java.util.NoSuchElementException; +import java.util.Set; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -81,6 +85,32 @@ void clearNullsAllBuckets() { assertNull(b); } } + + @Test + void maxRatioConstantsExpandTargetSize() { + // 75% load factor => bucket array sized at requestedSize * 4 / 3, rounded up to power of 2. + assertEquals(4, Support.MAX_RATIO_NUMERATOR); + assertEquals(3, Support.MAX_RATIO_DENOMINATOR); + int target = 12; + int sized = target * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR; + assertEquals(16, sized); + assertEquals(16, Support.sizeFor(sized)); + } + + @Test + void insertHeadEntrySplicesAsNewHead() { + Hashtable.Entry[] buckets = Support.create(4); + StringIntEntry a = new StringIntEntry("a", 1); + StringIntEntry b = new StringIntEntry("b", 2); + Support.insertHeadEntry(buckets, 0, a); + assertSame(a, buckets[0]); + assertNull(a.next()); + + Support.insertHeadEntry(buckets, 0, b); + assertSame(b, buckets[0]); + assertSame(a, b.next()); + assertNull(a.next()); + } } // ============ BucketIterator ============ @@ -192,4 +222,127 @@ void removeWithoutNextThrows() { assertThrows(IllegalStateException.class, it::remove); } } + + // ============ MutatingTableIterator ============ + + @Nested + class MutatingTableIteratorTests { + + @Test + void walksEveryEntryAcrossBuckets() { + Hashtable.D1 table = new Hashtable.D1<>(16); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + + Set seen = new HashSet<>(); + for (MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.hasNext(); ) { + seen.add(it.next().key); + } + assertEquals(3, seen.size()); + assertTrue(seen.contains("a")); + assertTrue(seen.contains("b")); + assertTrue(seen.contains("c")); + } + + @Test + void emptyTableIteratorIsExhausted() { + Hashtable.D1 table = new Hashtable.D1<>(8); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + assertFalse(it.hasNext()); + assertThrows(NoSuchElementException.class, it::next); + } + + @Test + void removeUnlinksBucketHead() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + + // The head of the chain is whichever was inserted last (insert prepends). + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + CollidingKeyEntry head = it.next(); + it.remove(); + + // Survivor still reachable via the table; removed one is not. + CollidingKey survivorKey = head.key.equals(k1) ? k2 : k1; + assertNotNull(table.get(survivorKey)); + assertNull(table.get(head.key)); + } + + @Test + void removeUnlinksMidChainEntry() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + + // Walk to the second entry, remove it. + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + CollidingKeyEntry victim = it.next(); + it.remove(); + + assertNull(table.get(victim.key)); + // The remaining two keys still resolve. + int remaining = 0; + for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { + if (table.get(k) != null) { + remaining++; + } + } + assertEquals(2, remaining); + + // Iteration can continue past a remove and yield the third entry. + assertTrue(it.hasNext()); + assertNotNull(it.next()); + assertFalse(it.hasNext()); + } + + @Test + void removeSkipsOverEmptyBuckets() { + // Three distinct keys that land in different buckets (low entry count vs large bucket array + // makes empty buckets between them very likely). Verify the iterator skips empties cleanly + // after a remove. + Hashtable.D1 table = new Hashtable.D1<>(64); + table.insert(new StringIntEntry("alpha", 1)); + table.insert(new StringIntEntry("beta", 2)); + table.insert(new StringIntEntry("gamma", 3)); + + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + it.remove(); + int remaining = 0; + while (it.hasNext()) { + it.next(); + remaining++; + } + assertEquals(2, remaining); + } + + @Test + void removeWithoutNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + assertThrows(IllegalStateException.class, it::remove); + } + + @Test + void removeTwiceWithoutInterveningNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + it.remove(); + assertThrows(IllegalStateException.class, it::remove); + } + } } From 96b40b8c7b3e4bb0d755ad73aa461e55166f14b3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:01:12 -0400 Subject: [PATCH 22/70] Simplify AggregateTable via new Hashtable.Support helpers - Constructor sizing now uses Support.MAX_RATIO_NUMERATOR / _DENOMINATOR instead of an open-coded * 4 / 3. - findOrInsert delegates the chain-head splice to Support.insertHeadEntry. - evictOneStale and expungeStaleAggregates both rewritten in terms of Support.mutatingTableIterator. Drops the bespoke head-vs-mid-chain branching that read as more complicated than the operation actually is. Net -28 lines in AggregateTable. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateTable.java | 70 ++++++------------- 1 file changed, 21 insertions(+), 49 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 8daf468e2a8..764b9700a2a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -1,6 +1,8 @@ package datadog.trace.common.metrics; import datadog.trace.util.Hashtable; +import datadog.trace.util.Hashtable.MutatingTableIterator; +import datadog.trace.util.Hashtable.Support; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -24,7 +26,10 @@ final class AggregateTable { private int size; AggregateTable(int maxAggregates) { - this.buckets = Hashtable.Support.create(maxAggregates * 4 / 3); + // ~25% headroom in the bucket array over the working-set target -- avoids the long-chain + // pathology at full capacity. + this.buckets = + Support.create(maxAggregates * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR); this.maxAggregates = maxAggregates; } @@ -43,7 +48,7 @@ boolean isEmpty() { */ AggregateEntry findOrInsert(SpanSnapshot snapshot) { long keyHash = AggregateEntry.hashOf(snapshot); - for (AggregateEntry candidate = Hashtable.Support.bucket(buckets, keyHash); + for (AggregateEntry candidate = Support.bucket(buckets, keyHash); candidate != null; candidate = candidate.next()) { if (candidate.keyHash == keyHash && candidate.matches(snapshot)) { @@ -54,42 +59,27 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { return null; } AggregateEntry entry = AggregateEntry.forSnapshot(snapshot); - int bucketIndex = Hashtable.Support.bucketIndex(buckets, keyHash); - entry.setNext(buckets[bucketIndex]); - buckets[bucketIndex] = entry; + Support.insertHeadEntry(buckets, Support.bucketIndex(buckets, keyHash), entry); size++; return entry; } /** Unlink the first entry whose {@code getHitCount() == 0}. */ private boolean evictOneStale() { - for (int i = 0; i < buckets.length; i++) { - AggregateEntry head = (AggregateEntry) buckets[i]; - if (head == null) { - continue; - } - if (head.getHitCount() == 0) { - buckets[i] = head.next(); + for (MutatingTableIterator it = Support.mutatingTableIterator(buckets); + it.hasNext(); ) { + AggregateEntry e = it.next(); + if (e.getHitCount() == 0) { + it.remove(); size--; return true; } - AggregateEntry prev = head; - AggregateEntry cur = head.next(); - while (cur != null) { - if (cur.getHitCount() == 0) { - prev.setNext(cur.next()); - size--; - return true; - } - prev = cur; - cur = cur.next(); - } } return false; } void forEach(Consumer consumer) { - Hashtable.Support.forEach(buckets, consumer); + Support.forEach(buckets, consumer); } /** @@ -98,41 +88,23 @@ void forEach(Consumer consumer) { * plus whatever side-band state it needs as {@code context}. */ void forEach(T context, BiConsumer consumer) { - Hashtable.Support.forEach(buckets, context, consumer); + Support.forEach(buckets, context, consumer); } /** Removes entries whose {@code getHitCount() == 0}. */ void expungeStaleAggregates() { - for (int i = 0; i < buckets.length; i++) { - // unlink leading stale entries - AggregateEntry head = (AggregateEntry) buckets[i]; - while (head != null && head.getHitCount() == 0) { - head = head.next(); + for (MutatingTableIterator it = Support.mutatingTableIterator(buckets); + it.hasNext(); ) { + AggregateEntry e = it.next(); + if (e.getHitCount() == 0) { + it.remove(); size--; } - buckets[i] = head; - if (head == null) { - continue; - } - // unlink stale entries in the chain - AggregateEntry prev = head; - AggregateEntry cur = head.next(); - while (cur != null) { - if (cur.getHitCount() == 0) { - AggregateEntry skipped = cur.next(); - prev.setNext(skipped); - size--; - cur = skipped; - } else { - prev = cur; - cur = cur.next(); - } - } } } void clear() { - Hashtable.Support.clear(buckets); + Support.clear(buckets); size = 0; } } From 55ca20482304e1b4ceba2c8cb674a6ee1db0a4f3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:12:50 -0400 Subject: [PATCH 23/70] Swap MAX_RATIO numerator/denominator pair for a single float + scaled create() Replace Support.MAX_RATIO_NUMERATOR / _DENOMINATOR with a single float MAX_RATIO constant, and add a Support.create(int, float) overload that takes a scale factor. Callers now write Support.create(n, MAX_RATIO) instead of stitching together the int arithmetic at the call site. The scaled size is truncated (not ceiled) before going through sizeFor. sizeFor already rounds up to the next power of two, so truncation just absorbs float fuzz that would otherwise push a result like 12 * 4/3 = 16.0000005f past 16 and double the bucket array size for no reason. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 27 +++++++++++++------ .../datadog/trace/util/HashtableTest.java | 21 +++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index bada7a8b98b..9e9ecb1c61a 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -373,18 +373,29 @@ public static final Hashtable.Entry[] create(int capacity) { return new Entry[sizeFor(capacity)]; } + /** + * Variant of {@link #create(int)} that scales the requested working-set size before sizing the + * bucket array. Pair with {@link #MAX_RATIO} (or similar) to leave headroom over the working + * set for a desired load factor. + * + *

    The scaled size is truncated to {@code int} before going through {@link #sizeFor(int)}. + * Truncation rather than {@code ceil} is intentional: {@code sizeFor} rounds up to the next + * power of two anyway, so the fractional part would only matter when float fuzz pushes the + * result across a power-of-two boundary -- {@code ceil} would then double the array size for no + * reason (e.g. {@code 12 * 4/3 = 16.0...0005f -> ceil 17 -> sizeFor 32}). + */ + public static final Hashtable.Entry[] create(int requestedSize, float scale) { + return new Entry[sizeFor((int) (requestedSize * scale))]; + } + static final int MAX_CAPACITY = 1 << 30; /** - * Numerator/denominator pair for the inverse of a 75% load factor. Callers that size their - * bucket array from a target working-set size {@code n} should pass {@code n * - * MAX_RATIO_NUMERATOR / MAX_RATIO_DENOMINATOR} to {@link #create(int)} (or {@link - * #sizeFor(int)}) to leave ~25% headroom in the array. Kept as separate ints so callers can use - * integer arithmetic. + * Inverse of a 75% load factor. Callers that size their bucket array from a target working-set + * size {@code n} should pass {@code create(n, MAX_RATIO)} (or {@code sizeFor((int) Math.ceil(n + * * MAX_RATIO))}) to leave ~25% headroom in the array. */ - public static final int MAX_RATIO_NUMERATOR = 4; - - public static final int MAX_RATIO_DENOMINATOR = 3; + public static final float MAX_RATIO = 4.0f / 3.0f; static final int sizeFor(int requestedCapacity) { if (requestedCapacity < 0) { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 6fbf0cc752c..2992279be6d 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -87,14 +87,19 @@ void clearNullsAllBuckets() { } @Test - void maxRatioConstantsExpandTargetSize() { - // 75% load factor => bucket array sized at requestedSize * 4 / 3, rounded up to power of 2. - assertEquals(4, Support.MAX_RATIO_NUMERATOR); - assertEquals(3, Support.MAX_RATIO_DENOMINATOR); - int target = 12; - int sized = target * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR; - assertEquals(16, sized); - assertEquals(16, Support.sizeFor(sized)); + void maxRatioScalesTargetForLoadFactor() { + // 75% load factor => bucket array sized at requestedSize * 4/3, rounded up to power of 2. + // 12 * (4/3) = 16 entries, rounded up to power-of-2 length = 16. + assertEquals(4.0f / 3.0f, Support.MAX_RATIO); + Hashtable.Entry[] buckets = Support.create(12, Support.MAX_RATIO); + assertEquals(16, buckets.length); + } + + @Test + void createWithScaleRoundsUpToPowerOfTwo() { + // 7 * 1.5 = 10.5 -> (int) 10 -> sizeFor rounds up to next power-of-two = 16 + Hashtable.Entry[] buckets = Support.create(7, 1.5f); + assertEquals(16, buckets.length); } @Test From 192de0cd27278f342ade6f9e2ef848560841b408 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:15:32 -0400 Subject: [PATCH 24/70] Address second-round review on AggregateTable / Aggregator - AggregateTable: switch to Support.create(maxAggregates, Support.MAX_RATIO) now that the load-factor scaling is a Support concern. - AggregateTable: replace open-coded "keyHash == X && matches(s)" with a new AggregateEntry.matches(long keyHash, SpanSnapshot) overload that bundles the hash gate. - AggregateTable: rename local iterator var "it" -> "iter". - Aggregator: drop WRITE_AND_CLEAR static field, inline as a non-capturing lambda; the JIT reuses non-capturing lambdas, no need for the static until a profile says otherwise. - Aggregator: comment the ClearSignal branch with the thread-safety rationale (single-writer invariant for AggregateTable). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 9 ++++++++ .../trace/common/metrics/AggregateTable.java | 21 +++++++++---------- .../trace/common/metrics/Aggregator.java | 19 +++++++++-------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 1cde9c0e68a..d7a50f67eeb 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -265,6 +265,15 @@ && stringContentEquals(httpEndpoint, s.httpEndpoint) && stringContentEquals(grpcStatusCode, s.grpcStatusCode); } + /** + * Pre-checks {@link #keyHash} against {@code keyHash} before delegating to {@link + * #matches(SpanSnapshot)}. The hash check is cheap and rules out most mismatches without touching + * the field-by-field comparison. + */ + boolean matches(long keyHash, SpanSnapshot s) { + return this.keyHash == keyHash && matches(s); + } + /** * Computes the 64-bit lookup hash for a {@link SpanSnapshot}. Chained per-field calls -- no * varargs / Object[] allocation, no autoboxing on primitive overloads. The constructor's diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 764b9700a2a..2b9b4c26452 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -28,8 +28,7 @@ final class AggregateTable { AggregateTable(int maxAggregates) { // ~25% headroom in the bucket array over the working-set target -- avoids the long-chain // pathology at full capacity. - this.buckets = - Support.create(maxAggregates * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR); + this.buckets = Support.create(maxAggregates, Support.MAX_RATIO); this.maxAggregates = maxAggregates; } @@ -51,7 +50,7 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { for (AggregateEntry candidate = Support.bucket(buckets, keyHash); candidate != null; candidate = candidate.next()) { - if (candidate.keyHash == keyHash && candidate.matches(snapshot)) { + if (candidate.matches(keyHash, snapshot)) { return candidate; } } @@ -66,11 +65,11 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { /** Unlink the first entry whose {@code getHitCount() == 0}. */ private boolean evictOneStale() { - for (MutatingTableIterator it = Support.mutatingTableIterator(buckets); - it.hasNext(); ) { - AggregateEntry e = it.next(); + for (MutatingTableIterator iter = Support.mutatingTableIterator(buckets); + iter.hasNext(); ) { + AggregateEntry e = iter.next(); if (e.getHitCount() == 0) { - it.remove(); + iter.remove(); size--; return true; } @@ -93,11 +92,11 @@ void forEach(T context, BiConsumer consumer) { /** Removes entries whose {@code getHitCount() == 0}. */ void expungeStaleAggregates() { - for (MutatingTableIterator it = Support.mutatingTableIterator(buckets); - it.hasNext(); ) { - AggregateEntry e = it.next(); + for (MutatingTableIterator iter = Support.mutatingTableIterator(buckets); + iter.hasNext(); ) { + AggregateEntry e = iter.next(); if (e.getHitCount() == 0) { - it.remove(); + iter.remove(); size--; } } diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index 816b5463424..f24ca23018d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -7,7 +7,6 @@ import datadog.trace.core.monitor.HealthMetrics; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; import org.jctools.queues.MessagePassingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,13 +15,6 @@ final class Aggregator implements Runnable { private static final long DEFAULT_SLEEP_MILLIS = 10; - /** Non-capturing -- the writer arrives via the forEach context arg. */ - private static final BiConsumer WRITE_AND_CLEAR = - (writer, entry) -> { - writer.add(entry); - entry.clear(); - }; - private static final Logger log = LoggerFactory.getLogger(Aggregator.class); private final MessagePassingQueue inbox; @@ -105,6 +97,10 @@ private final class Drainer implements MessagePassingQueue.Consumer { @Override public void accept(InboxItem item) { if (item == ClearSignal.CLEAR) { + // ClearSignal is routed through the inbox (rather than letting the caller mutate + // AggregateTable directly) so the aggregator thread stays the sole writer. AggregateTable + // is not thread-safe; a direct clear() from e.g. the OkHttpSink callback thread would + // race with Drainer.accept on this thread. if (!stopped) { aggregates.clear(); inbox.clear(); @@ -143,7 +139,12 @@ private void report(long when, SignalItem signal) { if (!aggregates.isEmpty()) { skipped = false; writer.startBucket(aggregates.size(), when, reportingIntervalNanos); - aggregates.forEach(writer, WRITE_AND_CLEAR); + aggregates.forEach( + writer, + (w, entry) -> { + w.add(entry); + entry.clear(); + }); // note that this may do IO and block writer.finishBucket(); } From 4bac439666851ecab0d8c7c14353015aa648aa8b Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:23:02 -0400 Subject: [PATCH 25/70] Tighten Hashtable docs + rename MAX_CAPACITY to MAX_BUCKETS Five small cleanups from a design re-review pass: 1. Support javadoc: drop the stale "methods are package-private" sentence; most of them were made public in earlier commits for higher-arity callers. Also drop the "nested BucketIterator" framing (iterators are peers of Support inside Hashtable, not nested inside Support). 2. MAX_RATIO javadoc: drop the Math.ceil recommendation; create(int, float) deliberately truncates and is the canonical pathway. 3. Document the null-hash treatment on D1.Entry.hash and D2.Entry.hash so the behavior difference is explicit: D1 uses Long.MIN_VALUE as a sentinel that's collision-free against any int-valued hashCode(); D2 has no such sentinel and relies on matches() to resolve null/null vs hash-0 collisions. 4. Rename Support.MAX_CAPACITY -> MAX_BUCKETS and sizeFor's parameter to requestedSize. The cap is on the bucket-array length, not entry count; the new name reflects that. Error messages updated to match. 5. Drop the `abstract` modifier on Hashtable in favor of `final` with a private constructor. Nothing actually subclasses Hashtable -- the abstract was a namespace device that read as "intended for extension." Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 73 +++++++++++++------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 9e9ecb1c61a..b6cff2bc493 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -22,8 +22,13 @@ * *

    For higher key dimensions, client code must implement its own class, but can still use the * support class to ease the implementation complexity. + * + *

    This outer class is a pure namespace -- it can't be instantiated. The actual table types are + * {@link D1}, {@link D2}, and (for higher-arity callers) {@link Support}-driven custom tables. */ -public abstract class Hashtable { +public final class Hashtable { + private Hashtable() {} + /** * Internal base class for entries. Stores the precomputed 64-bit keyHash and the chain-next * pointer used to link colliding entries within a single bucket. @@ -96,6 +101,14 @@ public boolean matches(Object key) { return Objects.equals(this.key, key); } + /** + * Returns the 64-bit lookup hash for {@code key}. Null keys map to {@link Long#MIN_VALUE} so + * that they don't collide with a real key that hashes to 0 (e.g. {@code + * Integer.hashCode(0)}). The {@code Long.MIN_VALUE} sentinel is safe against any {@code + * int}-valued {@code hashCode()} since those widen to a long in the range {@code + * [Integer.MIN_VALUE, Integer.MAX_VALUE]}; real-key collisions in chains are resolved by + * {@link #matches(Object)}. + */ public static long hash(Object key) { return (key == null) ? Long.MIN_VALUE : key.hashCode(); } @@ -241,6 +254,13 @@ public boolean matches(K1 key1, K2 key2) { return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); } + /** + * Returns the 64-bit lookup hash combining both key parts via {@link + * LongHashingUtils#hash(Object, Object)}. Null parts contribute {@code 0} (not a sentinel, + * unlike {@link D1.Entry#hash(Object)}): the combined hash can collide with real-key + * combinations whose chained hash equals {@code hash(0, 0) = 0} or similar values. {@link + * #matches(Object, Object)} resolves any such collision. + */ public static long hash(Object key1, Object key2) { return LongHashingUtils.hash(key1, key2); } @@ -340,16 +360,17 @@ public void forEach(T context, BiConsumer consume } /** - * Internal building blocks for hash-table operations. + * Building blocks for hash-table operations. * - *

    Used by {@link D1} and {@link D2}, and available to package code that wants to assemble its - * own higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The + *

    Used by {@link D1} and {@link D2}, and available to callers that want to assemble their own + * higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The * typical recipe: * *

      *
    • Subclass {@link Hashtable.Entry} directly, adding the key fields and a {@code * matches(...)} method of your chosen arity. - *
    • Allocate a backing array with {@link #create(int)}. + *
    • Allocate a backing array with {@link #create(int)} or {@link #create(int, float)} (the + * latter scales for a target load factor; see {@link #MAX_RATIO}). *
    • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, {@link * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / @@ -362,21 +383,22 @@ public void forEach(T context, BiConsumer consume *
    • Clear with {@link #clear(Hashtable.Entry[])}. *
    * - *

    All bucket arrays produced by {@link #create(int)} have a power-of-two length, so {@link + *

    All bucket arrays produced by {@code create} have a power-of-two length, so {@link * #bucketIndex(Object[], long)} can use a bit mask. - * - *

    Methods on this class are package-private; the class itself is public only so that its - * nested {@link BucketIterator} can be referenced by callers in other packages. */ public static final class Support { - public static final Hashtable.Entry[] create(int capacity) { - return new Entry[sizeFor(capacity)]; + /** + * Allocates a bucket array sized to hold {@code requestedSize} entries. Returned length is + * {@code requestedSize} rounded up to the next power of two (capped at {@link #MAX_BUCKETS}). + */ + public static final Hashtable.Entry[] create(int requestedSize) { + return new Entry[sizeFor(requestedSize)]; } /** * Variant of {@link #create(int)} that scales the requested working-set size before sizing the - * bucket array. Pair with {@link #MAX_RATIO} (or similar) to leave headroom over the working - * set for a desired load factor. + * bucket array. Pair with {@link #MAX_RATIO} to leave headroom over the working set for a + * desired load factor; the canonical call is {@code create(n, MAX_RATIO)}. * *

    The scaled size is truncated to {@code int} before going through {@link #sizeFor(int)}. * Truncation rather than {@code ceil} is intentional: {@code sizeFor} rounds up to the next @@ -388,27 +410,32 @@ public static final Hashtable.Entry[] create(int requestedSize, float scale) { return new Entry[sizeFor((int) (requestedSize * scale))]; } - static final int MAX_CAPACITY = 1 << 30; + /** Upper bound on the bucket array length returned by {@link #sizeFor(int)}. */ + static final int MAX_BUCKETS = 1 << 30; /** * Inverse of a 75% load factor. Callers that size their bucket array from a target working-set - * size {@code n} should pass {@code create(n, MAX_RATIO)} (or {@code sizeFor((int) Math.ceil(n - * * MAX_RATIO))}) to leave ~25% headroom in the array. + * size {@code n} should pass {@code create(n, MAX_RATIO)} to leave ~25% headroom in the array. */ public static final float MAX_RATIO = 4.0f / 3.0f; - static final int sizeFor(int requestedCapacity) { - if (requestedCapacity < 0) { - throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); + /** + * Rounds {@code requestedSize} up to the next power of two, capped at {@link #MAX_BUCKETS}. + * Throws {@link IllegalArgumentException} for negative inputs or inputs above the cap. Returns + * the bucket-array length to allocate. + */ + static final int sizeFor(int requestedSize) { + if (requestedSize < 0) { + throw new IllegalArgumentException("requestedSize must be non-negative: " + requestedSize); } - if (requestedCapacity > MAX_CAPACITY) { + if (requestedSize > MAX_BUCKETS) { throw new IllegalArgumentException( - "capacity exceeds maximum (" + MAX_CAPACITY + "): " + requestedCapacity); + "requestedSize exceeds maximum bucket count (" + MAX_BUCKETS + "): " + requestedSize); } - if (requestedCapacity <= 1) { + if (requestedSize <= 1) { return 1; } - return Integer.highestOneBit(requestedCapacity - 1) << 1; + return Integer.highestOneBit(requestedSize - 1) << 1; } public static final void clear(Hashtable.Entry[] buckets) { From de289a05fa22689c37007a4f0d75a448869bca88 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:25:52 -0400 Subject: [PATCH 26/70] Dedupe chain-head splice in D1/D2 via keyHash insertHeadEntry overload - Add Support.insertHeadEntry(buckets, long keyHash, entry) overload that derives the bucket index itself. Callers that already have a hash but not the index (the common case) now avoid the redundant bucketIndex(...) hop. - D1.insert, D1.insertOrReplace, D2.insert, D2.insertOrReplace: use the new overload, drop the (thisBuckets local, bucketIndex compute, setNext, store) sequence at each call site. - D2.buckets: drop the `private` modifier to match D1.buckets. Both are package-private so iterator tests in the same package can drive Support.bucketIterator against the table's bucket array. Added a short comment on both fields documenting the rationale. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index b6cff2bc493..8db5bee6f14 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -114,6 +114,8 @@ public static long hash(Object key) { } } + // Package-private so iterator tests in the same package can drive Support.bucketIterator and + // friends directly against the table's bucket array. final Hashtable.Entry[] buckets; private int size; @@ -155,19 +157,11 @@ public TEntry remove(K key) { } public void insert(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; } public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { @@ -179,11 +173,7 @@ public TEntry insertOrReplace(TEntry newEntry) { } } - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; return null; } @@ -266,7 +256,8 @@ public static long hash(Object key1, Object key2) { } } - private final Hashtable.Entry[] buckets; + // Package-private to match D1.buckets -- available for iterator tests in the same package. + final Hashtable.Entry[] buckets; private int size; public D2(int capacity) { @@ -307,19 +298,11 @@ public TEntry remove(K1 key1, K2 key2) { } public void insert(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; } public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { @@ -331,11 +314,7 @@ public TEntry insertOrReplace(TEntry newEntry) { } } - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; return null; } @@ -476,6 +455,17 @@ public static final void insertHeadEntry( buckets[bucketIndex] = entry; } + /** + * Convenience overload of {@link #insertHeadEntry(Hashtable.Entry[], int, Hashtable.Entry)} + * that derives the bucket index from {@code keyHash}. Use this when the caller has the hash but + * not the index; if the index has already been computed for another reason, prefer the + * int-taking overload to avoid the redundant mask. + */ + public static final void insertHeadEntry( + Hashtable.Entry[] buckets, long keyHash, Hashtable.Entry entry) { + insertHeadEntry(buckets, bucketIndex(buckets, keyHash), entry); + } + /** * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site From 2dd65ed2ca2cd9f7225c4f7671d5e44cf999831b Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:31:37 -0400 Subject: [PATCH 27/70] Tighten Entry.next encapsulation; doc hasNext; add D1/D2 getOrCreate Three follow-ups from the design review: - Make Hashtable.Entry.next private. All same-package readers (BucketIterator) already had a next() accessor; the leftover direct field reads now route through it. Closes the "mixed encapsulation" gap where some readers used the accessor and same-package ones reached for the field. - BucketIterator and MutatingBucketIterator now document that chain-walk work happens in next() (and the constructor for the first match); hasNext() is an O(1) field read. - Add D1.getOrCreate(K, Function) and D2.getOrCreate(K1, K2, BiFunction). Both reuse the lookup hash for the insert on miss, avoiding the double-hash that "get; if null then insert" callers would otherwise pay. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 58 +++++++++++++++++-- .../datadog/trace/util/HashtableD1Test.java | 48 +++++++++++++++ .../datadog/trace/util/HashtableD2Test.java | 41 +++++++++++++ 3 files changed, 143 insertions(+), 4 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 8db5bee6f14..9d9063ae8a8 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -5,7 +5,9 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; /** * Light weight simple Hashtable system that can be useful when HashMap would be unnecessarily @@ -39,7 +41,7 @@ private Hashtable() {} */ public abstract static class Entry { public final long keyHash; - Entry next = null; + private Entry next = null; protected Entry(long keyHash) { this.keyHash = keyHash; @@ -178,6 +180,29 @@ public TEntry insertOrReplace(TEntry newEntry) { return null; } + /** + * Returns the entry for {@code key}, building one via {@code creator} if absent. Computes the + * hash once and reuses it for both the lookup and (on miss) the insert -- avoids the + * double-hash that "{@code get}; if null then {@code insert}" would incur. + * + *

    The {@code creator} is expected to build an entry whose {@code keyHash} equals {@link + * Entry#hash(Object) D1.Entry.hash(key)} -- typically by passing {@code key} to a constructor + * that calls {@code super(key)}. A mismatched hash will leave the new entry inserted at a + * bucket that future {@link #get} calls won't probe. + */ + public TEntry getOrCreate(K key, Function creator) { + long keyHash = D1.Entry.hash(key); + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key)) { + return te; + } + } + TEntry newEntry = creator.apply(key); + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); + this.size += 1; + return newEntry; + } + public void clear() { Support.clear(this.buckets); this.size = 0; @@ -319,6 +344,25 @@ public TEntry insertOrReplace(TEntry newEntry) { return null; } + /** + * Two-key analogue of {@link D1#getOrCreate}. Computes the combined hash once and reuses it for + * both lookup and (on miss) insert. The {@code creator} is expected to build an entry whose + * {@code keyHash} equals {@link Entry#hash(Object, Object) D2.Entry.hash(key1, key2)}. + */ + public TEntry getOrCreate( + K1 key1, K2 key2, BiFunction creator) { + long keyHash = D2.Entry.hash(key1, key2); + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key1, key2)) { + return te; + } + } + TEntry newEntry = creator.apply(key1, key2); + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); + this.size += 1; + return newEntry; + } + public void clear() { Support.clear(this.buckets); this.size = 0; @@ -515,6 +559,9 @@ public static final void forEach( * *

    For {@code remove} or {@code replace} operations, use {@link MutatingBucketIterator} * instead. + * + *

    The chain-walk work to find the next-match entry happens in {@link #next()} (and in the + * constructor for the first match); {@link #hasNext()} is an O(1) field read. */ public static final class BucketIterator implements Iterator { private final long keyHash; @@ -524,7 +571,7 @@ public static final class BucketIterator implements Iterat this.keyHash = keyHash; Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; while (cur != null && cur.keyHash != keyHash) { - cur = cur.next; + cur = cur.next(); } this.nextEntry = cur; } @@ -540,9 +587,9 @@ public TEntry next() { Hashtable.Entry cur = this.nextEntry; if (cur == null) throw new NoSuchElementException("no next!"); - Hashtable.Entry advance = cur.next; + Hashtable.Entry advance = cur.next(); while (advance != null && advance.keyHash != keyHash) { - advance = advance.next; + advance = advance.next(); } this.nextEntry = advance; @@ -559,6 +606,9 @@ public TEntry next() { * remove} and {@code replace} can fix up the chain in O(1) without re-walking from the bucket * head. After {@code remove} or {@code replace}, iteration may continue with another {@link * #next()}. + * + *

    The chain-walk work to find the next-match entry happens in {@link #next()} (and in the + * constructor for the first match); {@link #hasNext()} is an O(1) field read. */ public static final class MutatingBucketIterator implements Iterator { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java index 11928bb4d5b..11cf93fc1dd 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -184,4 +184,52 @@ void hashCollisionsThenRemoveLeavesOtherIntact() { assertNull(table.get(k2)); assertNotNull(table.get(k3)); } + + @Test + void getOrCreateOnMissBuildsEntryViaCreator() { + Hashtable.D1 table = new Hashtable.D1<>(8); + int[] createCount = {0}; + StringIntEntry created = + table.getOrCreate( + "foo", + k -> { + createCount[0]++; + return new StringIntEntry(k, 42); + }); + assertNotNull(created); + assertEquals("foo", created.key); + assertEquals(42, created.value); + assertEquals(1, table.size()); + assertEquals(1, createCount[0]); + assertSame(created, table.get("foo")); + } + + @Test + void getOrCreateOnHitSkipsCreator() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry seeded = new StringIntEntry("foo", 1); + table.insert(seeded); + int[] createCount = {0}; + StringIntEntry got = + table.getOrCreate( + "foo", + k -> { + createCount[0]++; + return new StringIntEntry(k, 999); + }); + assertSame(seeded, got); + assertEquals(1, table.size()); + assertEquals(0, createCount[0]); + } + + @Test + void getOrCreateNullKeyIsPermitted() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry created = table.getOrCreate(null, k -> new StringIntEntry(k, 7)); + assertNotNull(created); + assertNull(created.key); + assertEquals(7, created.value); + assertSame(created, table.getOrCreate(null, k -> new StringIntEntry(k, 999))); + assertEquals(1, table.size()); + } } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java index 59339fcd89e..edcb0ad9f74 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -1,6 +1,7 @@ package datadog.trace.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -77,6 +78,46 @@ void forEachWithContextPassesContextToConsumer() { assertTrue(seen.contains("b:2")); } + @Test + void getOrCreateOnMissBuildsEntryViaCreator() { + Hashtable.D2 table = new Hashtable.D2<>(8); + int[] createCount = {0}; + PairEntry created = + table.getOrCreate( + "a", + 1, + (k1, k2) -> { + createCount[0]++; + return new PairEntry(k1, k2, 100); + }); + assertNotNull(created); + assertEquals("a", created.key1); + assertEquals(Integer.valueOf(1), created.key2); + assertEquals(100, created.value); + assertEquals(1, table.size()); + assertEquals(1, createCount[0]); + assertSame(created, table.get("a", 1)); + } + + @Test + void getOrCreateOnHitSkipsCreator() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry seeded = new PairEntry("a", 1, 100); + table.insert(seeded); + int[] createCount = {0}; + PairEntry got = + table.getOrCreate( + "a", + 1, + (k1, k2) -> { + createCount[0]++; + return new PairEntry(k1, k2, 999); + }); + assertSame(seeded, got); + assertEquals(1, table.size()); + assertEquals(0, createCount[0]); + } + private static final class PairEntry extends Hashtable.D2.Entry { int value; From 6a9063b20b935ad487701800f3573ad12f89cda5 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:35:39 -0400 Subject: [PATCH 28/70] Use keyHash insertHeadEntry overload in AggregateTable.findOrInsert Picks up the Support.insertHeadEntry(buckets, long keyHash, entry) overload added on the util-hashtable branch; saves the redundant Support.bucketIndex(buckets, keyHash) hop at the call site. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../main/java/datadog/trace/common/metrics/AggregateTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 2b9b4c26452..1d37a2156c8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -58,7 +58,7 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { return null; } AggregateEntry entry = AggregateEntry.forSnapshot(snapshot); - Support.insertHeadEntry(buckets, Support.bucketIndex(buckets, keyHash), entry); + Support.insertHeadEntry(buckets, keyHash, entry); size++; return entry; } From dbb17025e323351a208d06b94ecec23007e12c5d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 17:07:58 -0400 Subject: [PATCH 29/70] Replace // nullable comments with @Nullable annotations on AggregateEntry Use javax.annotation.Nullable (the codebase's convention -- see DDSpan, TagInterceptor, ScopeContext, etc.) on the four nullable label fields (serviceSource, httpMethod, httpEndpoint, grpcStatusCode), their getters, and the corresponding parameters of AggregateEntry.of. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index d7a50f67eeb..f7c4270ed51 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.Function; +import javax.annotation.Nullable; /** * Hashtable entry for the consumer-side aggregator. Holds the UTF8-encoded label fields that {@link @@ -80,12 +81,12 @@ final class AggregateEntry extends Hashtable.Entry { private final UTF8BytesString resource; private final UTF8BytesString service; private final UTF8BytesString operationName; - private final UTF8BytesString serviceSource; // nullable + @Nullable private final UTF8BytesString serviceSource; private final UTF8BytesString type; private final UTF8BytesString spanKind; - private final UTF8BytesString httpMethod; // nullable - private final UTF8BytesString httpEndpoint; // nullable - private final UTF8BytesString grpcStatusCode; // nullable + @Nullable private final UTF8BytesString httpMethod; + @Nullable private final UTF8BytesString httpEndpoint; + @Nullable private final UTF8BytesString grpcStatusCode; private final short httpStatusCode; private final boolean synthetic; private final boolean traceRoot; @@ -139,16 +140,16 @@ static AggregateEntry of( CharSequence resource, CharSequence service, CharSequence operationName, - CharSequence serviceSource, + @Nullable CharSequence serviceSource, CharSequence type, int httpStatusCode, boolean synthetic, boolean traceRoot, CharSequence spanKind, - List peerTags, - CharSequence httpMethod, - CharSequence httpEndpoint, - CharSequence grpcStatusCode) { + @Nullable List peerTags, + @Nullable CharSequence httpMethod, + @Nullable CharSequence httpEndpoint, + @Nullable CharSequence grpcStatusCode) { String[] rawPairs = peerTagsToRawPairs(peerTags); SpanSnapshot synthetic_snapshot = new SpanSnapshot( @@ -318,6 +319,7 @@ UTF8BytesString getOperationName() { return operationName; } + @Nullable UTF8BytesString getServiceSource() { return serviceSource; } @@ -330,14 +332,17 @@ UTF8BytesString getSpanKind() { return spanKind; } + @Nullable UTF8BytesString getHttpMethod() { return httpMethod; } + @Nullable UTF8BytesString getHttpEndpoint() { return httpEndpoint; } + @Nullable UTF8BytesString getGrpcStatusCode() { return grpcStatusCode; } From 545e74c898c01e5b74eedcd18925385ab999caa9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 17:09:16 -0400 Subject: [PATCH 30/70] Drop redundant load-factor comment from AggregateTable ctor Support.MAX_RATIO and the scaled create(int, float) overload already convey the 75% load-factor intent at the call site -- the inline comment was duplicating their self-documentation. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../main/java/datadog/trace/common/metrics/AggregateTable.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 1d37a2156c8..91a601fd5f0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -26,8 +26,6 @@ final class AggregateTable { private int size; AggregateTable(int maxAggregates) { - // ~25% headroom in the bucket array over the working-set target -- avoids the long-chain - // pathology at full capacity. this.buckets = Support.create(maxAggregates, Support.MAX_RATIO); this.maxAggregates = maxAggregates; } From 9983a590c45ab186cf3281ff83294c713fcc6099 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 17:15:45 -0400 Subject: [PATCH 31/70] Import java.util.Objects in AggregateEntry instead of fully qualifying Style nit -- the equals() method had eight fully-qualified references to java.util.Objects.equals; add the import and drop the qualifier. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index f7c4270ed51..4f9fe41437d 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -15,6 +15,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.Function; import javax.annotation.Nullable; @@ -376,16 +377,16 @@ public boolean equals(Object o) { return httpStatusCode == that.httpStatusCode && synthetic == that.synthetic && traceRoot == that.traceRoot - && java.util.Objects.equals(resource, that.resource) - && java.util.Objects.equals(service, that.service) - && java.util.Objects.equals(operationName, that.operationName) - && java.util.Objects.equals(serviceSource, that.serviceSource) - && java.util.Objects.equals(type, that.type) - && java.util.Objects.equals(spanKind, that.spanKind) + && Objects.equals(resource, that.resource) + && Objects.equals(service, that.service) + && Objects.equals(operationName, that.operationName) + && Objects.equals(serviceSource, that.serviceSource) + && Objects.equals(type, that.type) + && Objects.equals(spanKind, that.spanKind) && peerTags.equals(that.peerTags) - && java.util.Objects.equals(httpMethod, that.httpMethod) - && java.util.Objects.equals(httpEndpoint, that.httpEndpoint) - && java.util.Objects.equals(grpcStatusCode, that.grpcStatusCode); + && Objects.equals(httpMethod, that.httpMethod) + && Objects.equals(httpEndpoint, that.httpEndpoint) + && Objects.equals(grpcStatusCode, that.grpcStatusCode); } @Override From d2e4477f78dd4d288de7ea4f495534eb4f9d2c79 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 17:17:06 -0400 Subject: [PATCH 32/70] Document evictOneStale cost and disable() best-effort offer Two design-review trade-offs that won't change in this PR but should be explicit at the call sites: - AggregateTable.evictOneStale: O(N) scan per call (vs LRUCache's O(1)), acceptable because the new policy drops the *new* key on cap-overrun rather than evicting an established one -- so eviction is expected to be rare. Cursor-caching is the future optimization if a workload runs persistently at cap. - ConflatingMetricsAggregator.disable: single inbox.offer(CLEAR) is best-effort. If the inbox is full the clear is dropped, but the system self-heals (supportsMetrics() is already false, the next report-sink-rejection retries disable). Worst case is one extra cycle of stale data, not a leak. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/common/metrics/AggregateTable.java | 12 +++++++++++- .../common/metrics/ConflatingMetricsAggregator.java | 7 +++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 91a601fd5f0..2255ca1cdf8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -61,7 +61,17 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { return entry; } - /** Unlink the first entry whose {@code getHitCount() == 0}. */ + /** + * Unlinks the first entry whose {@code getHitCount() == 0}. + * + *

    O(N) per call -- scans buckets in array order from the start every time. That's a regression + * from the prior {@code LRUCache}'s O(1) LRU eviction, but the semantic change is deliberate: at + * cap with all entries live, we drop the new key (and report it via {@code + * onStatsAggregateDropped}) rather than evicting an established key. The expectation is that the + * cap is sized to the steady-state working set, so eviction is rare; if a future workload runs + * persistently at cap, this is the place to consider caching a cursor across calls so the scan + * resumes where it left off. + */ private boolean evictOneStale() { for (MutatingTableIterator iter = Support.mutatingTableIterator(buckets); iter.hasNext(); ) { diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index 601f8cdb76b..0996e630c70 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -401,6 +401,13 @@ private void disable() { // Route the clear through the inbox so the aggregator thread is the only writer. // AggregateTable is not thread-safe; calling clearAggregates() directly from this thread // would race with Drainer.accept on the aggregator thread. + // + // Best-effort single offer rather than the retry-loop pattern in report(). If the inbox is + // full at downgrade time the clear is dropped, but the system self-heals: features.discover() + // already flipped supportsMetrics() false, so producer publish() calls now skip the inbox; + // the aggregator drains existing snapshots and ships them on the next report cycle; the + // sink rejects that payload and fires DOWNGRADED again, which retries disable() against a + // now-empty inbox. Worst case: one extra reporting cycle of stale data. inbox.offer(CLEAR); } } From 66ec7f66275716bd5b9732bf3156313cb056fd50 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 20 May 2026 13:58:28 -0400 Subject: [PATCH 33/70] Hashtable: add missing braces and detach removed/replaced entries Addresses PR #11409 review comments: - #3267164119 / #3267165525: wrap every single-line if/break body in braces (7 sites across BucketIterator, MutatingBucketIterator, and the full-table Iterator). - #3275947761 / #3275948108 (sarahchen6): null out the removed/replaced entry's next pointer after splicing it out of the chain in MutatingBucketIterator.remove / .replace. Applied the same fix to the full-table Iterator.remove for consistency. Rationale: detaching prevents accidental traversal through a removed entry via a stale reference and lets the GC reclaim a chain tail that the removed entry was the last referrer to. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 9d9063ae8a8..8f40e4609bc 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -585,7 +585,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry cur = this.nextEntry; - if (cur == null) throw new NoSuchElementException("no next!"); + if (cur == null) { + throw new NoSuchElementException("no next!"); + } Hashtable.Entry advance = cur.next(); while (advance != null && advance.keyHash != keyHash) { @@ -643,7 +645,9 @@ public static final class MutatingBucketIterator } else { Hashtable.Entry prev, cur; for (prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next()) { - if (cur.keyHash == keyHash) break; + if (cur.keyHash == keyHash) { + break; + } } this.nextPrevEntry = prev; this.nextEntry = cur; @@ -662,7 +666,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry curEntry = this.nextEntry; - if (curEntry == null) throw new NoSuchElementException("no next!"); + if (curEntry == null) { + throw new NoSuchElementException("no next!"); + } this.curEntry = curEntry; this.curPrevEntry = this.nextPrevEntry; @@ -671,7 +677,9 @@ public TEntry next() { for (prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next()) { - if (cur.keyHash == keyHash) break; + if (cur.keyHash == keyHash) { + break; + } } this.nextPrevEntry = prev; this.nextEntry = cur; @@ -682,9 +690,15 @@ public TEntry next() { @Override public void remove() { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } - this.setPrevNext(oldCurEntry.next()); + Hashtable.Entry oldNext = oldCurEntry.next(); + this.setPrevNext(oldNext); + // Detach the removed entry from the chain so stale references can't traverse back into + // the live chain and so a now-unreachable tail can be reclaimed by GC. + oldCurEntry.setNext(null); // If the next match was directly after oldCurEntry, its predecessor is now // curPrevEntry (oldCurEntry was just unlinked from the chain). @@ -696,10 +710,15 @@ public void remove() { public void replace(TEntry replacementEntry) { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } - replacementEntry.setNext(oldCurEntry.next()); + Hashtable.Entry oldNext = oldCurEntry.next(); + replacementEntry.setNext(oldNext); this.setPrevNext(replacementEntry); + // Detach the replaced entry from the chain; the replacement now owns the chain slot. + oldCurEntry.setNext(null); // If the next match was directly after oldCurEntry, its predecessor is now // the replacement entry (which took oldCurEntry's chain slot). @@ -777,7 +796,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry e = this.nextEntry; - if (e == null) throw new NoSuchElementException("no next!"); + if (e == null) { + throw new NoSuchElementException("no next!"); + } this.curEntry = e; this.curPrevEntry = this.nextPrevEntry; @@ -797,13 +818,20 @@ public TEntry next() { @Override public void remove() { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } + Hashtable.Entry oldNext = oldCurEntry.next(); if (this.curPrevEntry == null) { - this.buckets[this.curBucketIndex] = oldCurEntry.next(); + this.buckets[this.curBucketIndex] = oldNext; } else { - this.curPrevEntry.setNext(oldCurEntry.next()); + this.curPrevEntry.setNext(oldNext); } + // Detach the removed entry from the chain so stale references can't traverse back into + // the live chain and so a now-unreachable tail can be reclaimed by GC. + oldCurEntry.setNext(null); + // If the next entry was the immediate chain successor of oldCurEntry, its predecessor is // now what came before oldCurEntry (oldCurEntry was just unlinked). if (this.nextPrevEntry == oldCurEntry) { From 10956b244c7559f6bab964cd081437ee2b5a6ae9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 15:14:07 -0400 Subject: [PATCH 34/70] Add Hashtable and LongHashingUtils to datadog.trace.util Two general-purpose utilities used by the client-side stats aggregator work (PR #11382 and follow-ups), extracted into their own change so the metrics-specific PRs can build on a smaller, reviewable foundation. - Hashtable: a generic open-addressed-ish bucket table abstraction keyed by a 64-bit hash, with a public abstract Entry type so client code can subclass it for higher-arity keys. The metrics aggregator uses it to back its AggregateTable. - LongHashingUtils: chained 64-bit hash combiners with primitive overloads (boolean, short, int, long, Object). Used in place of varargs combiners to avoid Object[] allocation and boxing on the hot path. No callers within internal-api itself yet -- the metrics aggregator PR will introduce the first usages. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 553 ++++++++++++++++++ .../datadog/trace/util/LongHashingUtils.java | 158 +++++ 2 files changed, 711 insertions(+) create mode 100644 internal-api/src/main/java/datadog/trace/util/Hashtable.java create mode 100644 internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java new file mode 100644 index 00000000000..d7f49dcae00 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -0,0 +1,553 @@ +package datadog.trace.util; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Light weight simple Hashtable system that can be useful when HashMap would + * be unnecessarily heavy. + * + *

      Use cases include... + *
    • primitive keys + *
    • primitive values + *
    • multi-part keys + *
    + * + * Convenience classes are provided for lower key dimensions. + * + * For higher key dimensions, client code must implement its own class, + * but can still use the support class to ease the implementation complexity. + */ +public abstract class Hashtable { + /** + * Internal base class for entries. Stores the precomputed 64-bit keyHash and + * the chain-next pointer used to link colliding entries within a single bucket. + * + *

    Subclasses add the actual key field(s) and a {@code matches(...)} method + * tailored to their key arity. See {@link D1.Entry} and {@link D2.Entry}; for + * higher arities, client code can subclass this directly and use {@link Support} + * to drive the table mechanics. + */ + public static abstract class Entry { + public final long keyHash; + Entry next = null; + + protected Entry(long keyHash) { + this.keyHash = keyHash; + } + + public final void setNext(TEntry next) { + this.next = next; + } + + @SuppressWarnings("unchecked") + public final TEntry next() { + return (TEntry)this.next; + } + } + + /** + * Single-key open hash table with chaining. + * + *

    The user supplies an {@link D1.Entry} subclass that carries the key and + * whatever value fields they want to mutate in place, then instantiates this + * class over that entry type. The main advantage over {@code HashMap} + * is that mutating an existing entry's value fields requires no allocation: + * call {@link #get} once and write directly to the returned entry's fields. + * For counter-style workloads this can be several times faster than + * {@code HashMap} and produces effectively zero GC pressure. + * + *

    Capacity is fixed at construction. The table does not resize, so the + * caller is responsible for choosing a capacity appropriate to the working + * set. Actual bucket-array length is rounded up to the next power of two. + * + *

    Null keys are permitted; they collapse to a single bucket via the + * sentinel hash {@link Long#MIN_VALUE} defined in {@link D1.Entry#hash}. + * + *

    Not thread-safe. Concurrent access (including mixing reads with + * writes) requires external synchronization. + * + * @param the key type + * @param the user's {@link D1.Entry D1.Entry<K>} subclass + */ + public static final class D1> { + /** + * Abstract base for {@link D1} entries. Subclass to add value fields you + * wish to mutate in place after retrieving the entry via {@link D1#get}. + * + *

    The key is captured at construction and stored alongside its + * precomputed 64-bit hash. {@link #matches(Object)} uses + * {@link Objects#equals} by default; override if a different equality + * semantics is needed (e.g. reference equality for interned keys). + * + * @param the key type + */ + public static abstract class Entry extends Hashtable.Entry { + final K key; + + protected Entry(K key) { + super(hash(key)); + this.key = key; + } + + public boolean matches(Object key) { + return Objects.equals(this.key, key); + } + + public static long hash(Object key) { + return (key == null ) ? Long.MIN_VALUE : key.hashCode(); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D1(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K key) { + long keyHash = D1.Entry.hash(key); + Hashtable.Entry[] thisBuckets = this.buckets; + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key)) return te; + } + } + return null; + } + + public TEntry remove(K key) { + long keyHash = D1.Entry.hash(key); + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + + this.size += 1; + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + } + + /** + * Two-key (composite-key) hash table with chaining. + * + *

    The user supplies a {@link D2.Entry} subclass carrying both key parts + * and any value fields. Compared to {@code HashMap} this avoids the + * per-lookup {@code Pair} (or record) allocation: both key parts are passed + * directly through {@link #get}, {@link #remove}, {@link #insert}, and + * {@link #insertOrReplace}. Combined with in-place value mutation, this + * makes {@code D2} substantially less GC-intensive than the equivalent + * {@code HashMap} for counter-style workloads. + * + *

    Capacity is fixed at construction; the table does not resize. Actual + * bucket-array length is rounded up to the next power of two. + * + *

    Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; + * see {@link D2.Entry#hash(Object, Object)}. + * + *

    Not thread-safe. + * + * @param first key type + * @param second key type + * @param the user's {@link D2.Entry D2.Entry<K1, K2>} subclass + */ + public static final class D2> { + /** + * Abstract base for {@link D2} entries. Subclass to add value fields you + * wish to mutate in place. + * + *

    Both key parts are captured at construction and stored alongside their + * combined 64-bit hash. {@link #matches(Object, Object)} uses + * {@link Objects#equals} pairwise on the two parts. + * + * @param first key type + * @param second key type + */ + public static abstract class Entry extends Hashtable.Entry { + final K1 key1; + final K2 key2; + + protected Entry(K1 key1, K2 key2) { + super(hash(key1, key2)); + this.key1 = key1; + this.key2 = key2; + } + + public boolean matches(K1 key1, K2 key2) { + return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); + } + + public static long hash(Object key1, Object key2) { + return LongHashingUtils.hash(key1, key2); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D2(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + Hashtable.Entry[] thisBuckets = this.buckets; + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key1, key2)) return te; + } + } + return null; + } + + public TEntry remove(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key1, key2)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + + this.size += 1; + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key1, newEntry.key2)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + } + + /** + * Internal building blocks for hash-table operations. + * + *

    Used by {@link D1} and {@link D2}, and available to package code that + * wants to assemble its own higher-arity table (3+ key parts) without + * re-implementing the bucket-array mechanics. The typical recipe: + * + *

      + *
    • Subclass {@link Hashtable.Entry} directly, adding the key fields and + * a {@code matches(...)} method of your chosen arity. + *
    • Allocate a backing array with {@link #create(int)}. + *
    • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, + * {@link #bucketIterator(Hashtable.Entry[], long)} for read-only chain + * walks, and {@link #mutatingBucketIterator(Hashtable.Entry[], long)} + * when you also need {@code remove} / {@code replace}. + *
    • Clear with {@link #clear(Hashtable.Entry[])}. + *
    + * + *

    All bucket arrays produced by {@link #create(int)} have a power-of-two + * length, so {@link #bucketIndex(Object[], long)} can use a bit mask. + * + *

    Methods on this class are package-private; the class itself is public + * only so that its nested {@link BucketIterator} can be referenced by + * callers in other packages. + */ + public static final class Support { + public static final Hashtable.Entry[] create(int capacity) { + return new Entry[sizeFor(capacity)]; + } + + static final int sizeFor(int requestedCapacity) { + int pow; + for ( pow = 1; pow < requestedCapacity; pow *= 2 ); + return pow; + } + + public static final void clear(Hashtable.Entry[] buckets) { + Arrays.fill(buckets, null); + } + + public static final BucketIterator bucketIterator(Hashtable.Entry[] buckets, long keyHash) { + return new BucketIterator(buckets, keyHash); + } + + public static final MutatingBucketIterator mutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + return new MutatingBucketIterator(buckets, keyHash); + } + + public static final int bucketIndex(Object[] buckets, long keyHash) { + return (int)(keyHash & buckets.length - 1); + } + } + + /** + * Read-only iterator over entries in a single bucket whose {@code keyHash} + * matches a specific search hash. Cheaper than {@link MutatingBucketIterator} + * because it does not track the previous-node pointers required for + * splicing — use it when you only need to walk the chain. + * + *

    For {@code remove} or {@code replace} operations, use + * {@link MutatingBucketIterator} instead. + */ + public static final class BucketIterator implements Iterator { + private final long keyHash; + private Hashtable.Entry nextEntry; + + BucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.keyHash = keyHash; + Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; + while (cur != null && cur.keyHash != keyHash) cur = cur.next; + this.nextEntry = cur; + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry cur = this.nextEntry; + if (cur == null) throw new NoSuchElementException("no next!"); + + Hashtable.Entry advance = cur.next; + while (advance != null && advance.keyHash != keyHash) advance = advance.next; + this.nextEntry = advance; + + return (TEntry) cur; + } + } + + /** + * Mutating iterator over entries in a single bucket whose {@code keyHash} + * matches a specific search hash. Supports {@link #remove()} and + * {@link #replace(Entry)} to splice the chain in place. + * + *

    Carries previous-node pointers for the current entry and the next-match + * entry so that {@code remove} and {@code replace} can fix up the chain in + * O(1) without re-walking from the bucket head. After {@code remove} or + * {@code replace}, iteration may continue with another {@link #next()}. + */ + public static final class MutatingBucketIterator implements Iterator { + private final long keyHash; + + private final Hashtable.Entry[] buckets; + + /** + * The entry prior to the last entry returned by next + * Used for mutating operations + */ + private Hashtable.Entry curPrevEntry; + + /** + * The entry that was last returned by next + */ + private Hashtable.Entry curEntry; + + /** + * The entry prior to the next entry + */ + private Hashtable.Entry nextPrevEntry; + + /** + * The next entry to be returned by next + */ + private Hashtable.Entry nextEntry; + + MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.buckets = buckets; + this.keyHash = keyHash; + + int bucketIndex = Support.bucketIndex(buckets, keyHash); + Hashtable.Entry headEntry = this.buckets[bucketIndex]; + if ( headEntry == null ) { + this.nextEntry = null; + this.nextPrevEntry = null; + + this.curEntry = null; + this.curPrevEntry = null; + } else { + Hashtable.Entry prev, cur; + for ( prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next() ) { + if ( cur.keyHash == keyHash ) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + this.curEntry = null; + this.curPrevEntry = null; + } + } + + @Override + public boolean hasNext() { + return (this.nextEntry != null); + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry curEntry = this.nextEntry; + if ( curEntry == null ) throw new NoSuchElementException("no next!"); + + this.curEntry = curEntry; + this.curPrevEntry = this.nextPrevEntry; + + Hashtable.Entry prev, cur; + for ( prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next() ) { + if ( cur.keyHash == keyHash ) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + return (TEntry) curEntry; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if ( oldCurEntry == null ) throw new IllegalStateException(); + + this.setPrevNext(oldCurEntry.next()); + + // If the next match was directly after oldCurEntry, its predecessor is now + // curPrevEntry (oldCurEntry was just unlinked from the chain). + if ( this.nextPrevEntry == oldCurEntry ) { + this.nextPrevEntry = this.curPrevEntry; + } + this.curEntry = null; + } + + public void replace(TEntry replacementEntry) { + Hashtable.Entry oldCurEntry = this.curEntry; + if ( oldCurEntry == null ) throw new IllegalStateException(); + + replacementEntry.setNext(oldCurEntry.next()); + this.setPrevNext(replacementEntry); + + // If the next match was directly after oldCurEntry, its predecessor is now + // the replacement entry (which took oldCurEntry's chain slot). + if ( this.nextPrevEntry == oldCurEntry ) { + this.nextPrevEntry = replacementEntry; + } + this.curEntry = replacementEntry; + } + + void setPrevNext(Hashtable.Entry nextEntry) { + if ( this.curPrevEntry == null ) { + Hashtable.Entry[] buckets = this.buckets; + buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; + } else { + this.curPrevEntry.setNext(nextEntry); + } + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java new file mode 100644 index 00000000000..bc53bc4ecb6 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -0,0 +1,158 @@ +package datadog.trace.util; + +/** + * This class is intended to be a drop-in replacement for the hashing portions of java.util.Objects. + * This class provides more convenience methods for hashing primitives and includes overrides for + * hash that take many argument lengths to avoid var-args allocation. + */ +public final class LongHashingUtils { + private LongHashingUtils() {} + + public static final long hashCodeX(Object obj) { + return obj == null ? Long.MIN_VALUE : obj.hashCode(); + } + + public static final long hash(boolean value) { + return Boolean.hashCode(value); + } + + public static final long hash(char value) { + return Character.hashCode(value); + } + + public static final long hash(byte value) { + return Byte.hashCode(value); + } + + public static final long hash(short value) { + return Short.hashCode(value); + } + + public static final long hash(int value) { + return Integer.hashCode(value); + } + + public static final long hash(long value) { + return value; + } + + public static final long hash(float value) { + return Float.hashCode(value); + } + + public static final long hash(double value) { + return Double.doubleToRawLongBits(value); + } + + public static final long hash(Object obj0, Object obj1) { + return hash(intHash(obj0), intHash(obj1)); + } + + public static final long hash(int hash0, int hash1) { + return 31L * hash0 + hash1; + } + + private static final int intHash(Object obj) { + return obj == null ? 0 : obj.hashCode(); + } + + public static final long hash(Object obj0, Object obj1, Object obj2) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2)); + } + + public static final long hash(long hash0, long hash1, long hash2) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * hash0 + 31L * hash1 + hash2; + } + + public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3)); + } + + public static final long hash(int hash0, int hash1, int hash2, int hash3) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * 31L * hash0 + 31L * 31L * hash1 + 31L * hash2 + hash3; + } + + public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3, Object obj4) { + return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3), intHash(obj4)); + } + + public static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { + // DQH - Micro-optimizing, 31L * 31L will constant fold + // Since there are multiple execution ports for load & store, + // this will make good use of the core. + return 31L * 31L * 31L * 31L * hash0 + 31L * 31L * 31L * hash1 + 31L * 31L * hash2 + 31L * hash3 + hash4; + } + + @Deprecated + public static final long hash(int[] hashes) { + long result = 0; + for (int hash : hashes) { + result = addToHash(result, hash); + } + return result; + } + + public static final long addToHash(long hash, int value) { + return 31L * hash + value; + } + + public static final long addToHash(long hash, Object obj) { + return addToHash(hash, intHash(obj)); + } + + public static final long addToHash(long hash, boolean value) { + return addToHash(hash, Boolean.hashCode(value)); + } + + public static final long addToHash(long hash, char value) { + return addToHash(hash, Character.hashCode(value)); + } + + public static final long addToHash(long hash, byte value) { + return addToHash(hash, Byte.hashCode(value)); + } + + public static final long addToHash(long hash, short value) { + return addToHash(hash, Short.hashCode(value)); + } + + public static final long addToHash(long hash, long value) { + return addToHash(hash, Long.hashCode(value)); + } + + public static final long addToHash(long hash, float value) { + return addToHash(hash, Float.hashCode(value)); + } + + public static final long addToHash(long hash, double value) { + return addToHash(hash, Double.hashCode(value)); + } + + public static final long hash(Iterable objs) { + long result = 0; + for (Object obj : objs) { + result = addToHash(result, obj); + } + return result; + } + + /** + * Calling this var-arg version can result in large amounts of allocation (see HashingBenchmark) + * Rather than calliing this method, add another override of hash that handles a larger number of + * arguments or use calls to addToHash. + */ + @Deprecated + public static final long hash(Object[] objs) { + long result = 0; + for (Object obj : objs) { + result = addToHash(result, obj); + } + return result; + } +} From 035dc095597b34eeec54cc889b401c204031bec4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 15:40:00 -0400 Subject: [PATCH 35/70] Add unit tests for Hashtable and LongHashingUtils LongHashingUtilsTest (14 cases): - hashCodeX null sentinel + non-null pass-through - all primitive hash() overloads match the boxed Java hashCodes - hash(Object...) 2/3/4/5-arg overloads match the chained addToHash formula they are documented to constant-fold to - addToHash(long, primitive) overloads match the Object-version - linear-accumulation invariant (31 * h + v) holds across a sequence - iterable / deprecated int[] / deprecated Object[] variants match chained addToHash - intHash treats null as 0 (observable via hash(null, "x")) HashtableTest (24 cases across 5 nested classes): - D1: insert/get/remove/insertOrReplace/clear/forEach, in-place value mutation, null-key handling, hash-collision chaining with disambig- uating equals, remove-from-collided-chain leaves siblings intact - D2: pair-key identity, remove(pair), insertOrReplace matches on both parts, forEach - Support: capacity rounds up to a power of two, bucketIndex stays in range across a wide hash sample, clear nulls every slot - BucketIterator: walks only matching-hash entries in a chain, throws NoSuchElementException when exhausted - MutatingBucketIterator: remove from head-of-chain unlinks, replace swaps the entry while preserving chain, remove() without prior next() throws IllegalStateException Tests live in internal-api/src/test/java/datadog/trace/util and use the already-present JUnit 5 setup. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/util/HashtableTest.java | 465 ++++++++++++++++++ .../trace/util/LongHashingUtilsTest.java | 160 ++++++ 2 files changed, 625 insertions(+) create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableTest.java create mode 100644 internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java new file mode 100644 index 00000000000..67c99c0d08d --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -0,0 +1,465 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.util.Hashtable.BucketIterator; +import datadog.trace.util.Hashtable.MutatingBucketIterator; +import datadog.trace.util.Hashtable.Support; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class HashtableTest { + + // ============ D1 ============ + + @Nested + class D1Tests { + + @Test + void emptyTableLookupReturnsNull() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get("missing")); + assertEquals(0, table.size()); + } + + @Test + void insertedEntryIsRetrievable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry e = new StringIntEntry("foo", 1); + table.insert(e); + assertEquals(1, table.size()); + assertSame(e, table.get("foo")); + } + + @Test + void multipleInsertsRetrievableSeparately() { + Hashtable.D1 table = new Hashtable.D1<>(16); + StringIntEntry a = new StringIntEntry("alpha", 1); + StringIntEntry b = new StringIntEntry("beta", 2); + StringIntEntry c = new StringIntEntry("gamma", 3); + table.insert(a); + table.insert(b); + table.insert(c); + assertEquals(3, table.size()); + assertSame(a, table.get("alpha")); + assertSame(b, table.get("beta")); + assertSame(c, table.get("gamma")); + } + + @Test + void inPlaceMutationVisibleViaSubsequentGet() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("counter", 0)); + for (int i = 0; i < 10; i++) { + StringIntEntry e = table.get("counter"); + e.value++; + } + assertEquals(10, table.get("counter").value); + } + + @Test + void removeUnlinksEntryAndDecrementsSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + assertEquals(2, table.size()); + + StringIntEntry removed = table.remove("a"); + assertNotNull(removed); + assertEquals("a", removed.key); + assertEquals(1, table.size()); + assertNull(table.get("a")); + assertNotNull(table.get("b")); + } + + @Test + void removeNonexistentReturnsNullAndDoesNotChangeSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + assertNull(table.remove("nope")); + assertEquals(1, table.size()); + } + + @Test + void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry first = new StringIntEntry("k", 1); + assertNull(table.insertOrReplace(first), "fresh insert returns null"); + assertEquals(1, table.size()); + + StringIntEntry second = new StringIntEntry("k", 2); + assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); + assertEquals(1, table.size()); + assertSame(second, table.get("k"), "new entry visible after replace"); + } + + @Test + void clearEmptiesTheTable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.clear(); + assertEquals(0, table.size()); + assertNull(table.get("a")); + // Reinsertion works after clear + table.insert(new StringIntEntry("a", 99)); + assertEquals(99, table.get("a").value); + } + + @Test + void forEachVisitsEveryInsertedEntry() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + Map seen = new HashMap<>(); + table.forEach(e -> seen.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(1, seen.get("a")); + assertEquals(2, seen.get("b")); + assertEquals(3, seen.get("c")); + } + + @Test + void nullKeyIsPermittedAndDistinctFromAbsent() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get(null)); + StringIntEntry nullKeyed = new StringIntEntry(null, 7); + table.insert(nullKeyed); + assertSame(nullKeyed, table.get(null)); + assertEquals(1, table.size()); + assertSame(nullKeyed, table.remove(null)); + assertEquals(0, table.size()); + } + + @Test + void hashCollisionsResolveByEquality() { + // Force two distinct keys with the same hashCode -- the chain must still distinguish them + // via matches(). + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); + table.insert(e1); + table.insert(e2); + assertEquals(2, table.size()); + assertSame(e1, table.get(k1)); + assertSame(e2, table.get(k2)); + } + + @Test + void hashCollisionsThenRemoveLeavesOtherIntact() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + table.remove(k2); + assertEquals(2, table.size()); + assertNotNull(table.get(k1)); + assertNull(table.get(k2)); + assertNotNull(table.get(k3)); + } + } + + // ============ D2 ============ + + @Nested + class D2Tests { + + @Test + void pairKeysParticipateInIdentity() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + PairEntry bb = new PairEntry("b", 1, 300); + table.insert(ab); + table.insert(ac); + table.insert(bb); + assertEquals(3, table.size()); + assertSame(ab, table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + assertSame(bb, table.get("b", 1)); + assertNull(table.get("a", 3)); + } + + @Test + void removePairUnlinks() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + table.insert(ab); + table.insert(ac); + assertSame(ab, table.remove("a", 1)); + assertEquals(1, table.size()); + assertNull(table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + } + + @Test + void insertOrReplaceMatchesOnBothKeys() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry first = new PairEntry("k", 7, 1); + assertNull(table.insertOrReplace(first)); + PairEntry second = new PairEntry("k", 7, 2); + assertSame(first, table.insertOrReplace(second)); + // Different second-key: should insert new, not replace + PairEntry third = new PairEntry("k", 8, 3); + assertNull(table.insertOrReplace(third)); + assertEquals(2, table.size()); + } + + @Test + void forEachVisitsBothPairs() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + } + + // ============ Support ============ + + @Nested + class SupportTests { + + @Test + void createRoundsCapacityUpToPowerOfTwo() { + // The Hashtable.D1 / D2 size() reflects entries, but the bucket array length is + // a power of two >= requestedCapacity. We can verify indirectly via bucketIndex masking. + Hashtable.Entry[] buckets = Support.create(5); + // Length must be a power of two >= 5 + int len = buckets.length; + assertTrue(len >= 5); + assertEquals(0, len & (len - 1), "length must be a power of two"); + } + + @Test + void bucketIndexIsBoundedByArrayLength() { + Hashtable.Entry[] buckets = Support.create(16); + for (long h : new long[] {0L, 1L, -1L, Long.MIN_VALUE, Long.MAX_VALUE, 12345L}) { + int idx = Support.bucketIndex(buckets, h); + assertTrue(idx >= 0 && idx < buckets.length, "bucketIndex out of range for hash " + h); + } + } + + @Test + void clearNullsAllBuckets() { + Hashtable.Entry[] buckets = Support.create(4); + buckets[0] = new StringIntEntry("x", 1); + buckets[1] = new StringIntEntry("y", 2); + Support.clear(buckets); + for (Hashtable.Entry b : buckets) { + assertNull(b); + } + } + } + + // ============ BucketIterator ============ + + @Nested + class BucketIteratorTests { + + @Test + void walksOnlyMatchingHash() { + // Build a bucket array with two entries that share a bucket but have different hashes. + // Use Hashtable.D1 to seed; then call Support.bucketIterator directly with the matching + // hash and verify it only returns the matching entry. + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. + BucketIterator it = + Support.bucketIterator(extractBuckets(table), 17L); + int count = 0; + while (it.hasNext()) { + assertNotNull(it.next()); + count++; + } + assertEquals(3, count); + } + + @Test + void exhaustedIteratorThrowsNoSuchElement() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("only", 1)); + long h = Hashtable.D1.Entry.hash("only"); + BucketIterator it = Support.bucketIterator(extractBuckets(table), h); + it.next(); + assertFalse(it.hasNext()); + assertThrows(NoSuchElementException.class, it::next); + } + } + + // ============ MutatingBucketIterator ============ + + @Nested + class MutatingBucketIteratorTests { + + @Test + void removeFromHeadOfChainUnlinks() { + // Make three entries with the same hash so they chain in one bucket + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + + MutatingBucketIterator it = + Support.mutatingBucketIterator(extractBuckets(table), 17L); + it.next(); // first match (head of chain in insertion-reverse order) + it.remove(); + // Two should remain + int remaining = 0; + while (it.hasNext()) { + it.next(); + remaining++; + } + assertEquals(2, remaining); + // And the table still finds the survivors via get(...) + // (which entry was the head depends on insertion order; we just verify count + that two + // of the three keys are still retrievable.) + int found = 0; + for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { + if (table.get(k) != null) found++; + } + assertEquals(2, found); + } + + @Test + void replaceSwapsEntryAndPreservesChain() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 1); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 2); + table.insert(e1); + table.insert(e2); + + MutatingBucketIterator it = + Support.mutatingBucketIterator(extractBuckets(table), 17L); + CollidingKeyEntry first = it.next(); + CollidingKeyEntry replacement = new CollidingKeyEntry(first.key, 999); + it.replace(replacement); + // Both entries still in the chain + assertNotNull(table.get(k1)); + assertNotNull(table.get(k2)); + // The replaced one now has value 999 + assertEquals(999, table.get(first.key).value); + } + + @Test + void removeWithoutNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + MutatingBucketIterator it = + Support.mutatingBucketIterator( + extractBuckets(table), Hashtable.D1.Entry.hash("a")); + assertThrows(IllegalStateException.class, it::remove); + } + } + + // ============ test helpers ============ + + /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ + private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { + try { + java.lang.reflect.Field f = Hashtable.D1.class.getDeclaredField("buckets"); + f.setAccessible(true); + return (Hashtable.Entry[]) f.get(table); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** Sort comparator used by tests that want deterministic visit order. */ + @SuppressWarnings("unused") + private static final Comparator BY_KEY = + Comparator.comparing(e -> e.key); + + private static final class StringIntEntry extends Hashtable.D1.Entry { + int value; + + StringIntEntry(String key, int value) { + super(key); + this.value = value; + } + } + + /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ + private static final class CollidingKey { + final String label; + final int hash; + + CollidingKey(String label, int hash) { + this.label = label; + this.hash = hash; + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CollidingKey)) return false; + CollidingKey that = (CollidingKey) o; + return hash == that.hash && label.equals(that.label); + } + + @Override + public String toString() { + return "CollidingKey(" + label + ", " + hash + ")"; + } + } + + private static final class CollidingKeyEntry extends Hashtable.D1.Entry { + int value; + + CollidingKeyEntry(CollidingKey key, int value) { + super(key); + this.value = value; + } + } + + private static final class PairEntry extends Hashtable.D2.Entry { + int value; + + PairEntry(String key1, Integer key2, int value) { + super(key1, key2); + this.value = value; + } + } + + // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning quiet. + @SuppressWarnings("unused") + private static final List UNUSED = new ArrayList<>(); +} diff --git a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java new file mode 100644 index 00000000000..d0053c75b42 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java @@ -0,0 +1,160 @@ +package datadog.trace.util; + +import static datadog.trace.util.LongHashingUtils.addToHash; +import static datadog.trace.util.LongHashingUtils.hash; +import static datadog.trace.util.LongHashingUtils.hashCodeX; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.Arrays; +import java.util.Objects; +import org.junit.jupiter.api.Test; + +class LongHashingUtilsTest { + + // ----- single-value overloads ----- + + @Test + void hashCodeXReturnsObjectHashCodeOrSentinelForNull() { + Object o = new Object(); + assertEquals(o.hashCode(), hashCodeX(o)); + assertEquals(Long.MIN_VALUE, hashCodeX(null)); + } + + @Test + void primitiveOverloadsMatchBoxedHashCodes() { + assertEquals(Boolean.hashCode(true), hash(true)); + assertEquals(Boolean.hashCode(false), hash(false)); + assertEquals(Character.hashCode('x'), hash('x')); + assertEquals(Byte.hashCode((byte) 42), hash((byte) 42)); + assertEquals(Short.hashCode((short) -7), hash((short) -7)); + assertEquals(Integer.hashCode(123456), hash(123456)); + assertEquals(123456L, hash(123456L)); + assertEquals(Float.hashCode(3.14f), hash(3.14f)); + assertEquals(Double.doubleToRawLongBits(2.71828), hash(2.71828)); + } + + // ----- multi-arg Object overloads vs chained addToHash ----- + + @Test + void twoArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + assertEquals(addToHash(addToHash(0L, a), b), hash(a, b)); + } + + @Test + void threeArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + assertEquals(addToHash(addToHash(addToHash(0L, a), b), c), hash(a, b, c)); + } + + @Test + void fourArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + Object d = 3.14; + assertEquals( + addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); + } + + @Test + void fiveArgHashMatchesChainedAddToHash() { + Object a = "alpha"; + Object b = 42; + Object c = true; + Object d = 3.14; + Object e = 'q'; + assertEquals( + addToHash(addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), e), + hash(a, b, c, d, e)); + } + + @Test + void multiArgHashHandlesNullsConsistentlyWithChainedAddToHash() { + assertEquals(addToHash(addToHash(0L, (Object) null), "x"), hash(null, "x")); + assertEquals(addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); + } + + @Test + void differentInputsProduceDifferentHashes() { + // Sanity: ordering matters, and distinct values produce distinct results in general. + assertNotEquals(hash("a", "b"), hash("b", "a")); + assertNotEquals(hash("a", "b", "c"), hash("a", "c", "b")); + } + + // ----- addToHash primitive overloads ----- + + @Test + void addToHashPrimitivesMatchObjectVersion() { + long seed = 100L; + assertEquals(addToHash(seed, Boolean.hashCode(true)), addToHash(seed, true)); + assertEquals(addToHash(seed, Character.hashCode('z')), addToHash(seed, 'z')); + assertEquals(addToHash(seed, Byte.hashCode((byte) 9)), addToHash(seed, (byte) 9)); + assertEquals(addToHash(seed, Short.hashCode((short) 5)), addToHash(seed, (short) 5)); + assertEquals(addToHash(seed, Long.hashCode(999_999L)), addToHash(seed, 999_999L)); + assertEquals(addToHash(seed, Float.hashCode(1.5f)), addToHash(seed, 1.5f)); + assertEquals(addToHash(seed, Double.hashCode(2.5d)), addToHash(seed, 2.5d)); + } + + @Test + void addToHashIsLinearAcrossSteps() { + // 31*h + v formula -- verify by accumulating an explicit sequence. + long expected = 0L; + for (int v : new int[] {1, 2, 3, 4, 5}) { + expected = 31L * expected + v; + } + long actual = 0L; + for (int v : new int[] {1, 2, 3, 4, 5}) { + actual = addToHash(actual, v); + } + assertEquals(expected, actual); + } + + // ----- iterable / array versions ----- + + @Test + void hashIterableMatchesChainedAddToHash() { + Iterable values = Arrays.asList("a", 1, true, null); + long expected = 0L; + for (Object o : values) { + expected = addToHash(expected, o); + } + assertEquals(expected, hash(values)); + } + + @Test + @SuppressWarnings("deprecation") + void deprecatedIntArrayHashMatchesChainedAddToHash() { + int[] hashes = new int[] {7, 13, 31, 1024}; + long expected = 0L; + for (int h : hashes) { + expected = addToHash(expected, h); + } + assertEquals(expected, hash(hashes)); + } + + @Test + @SuppressWarnings("deprecation") + void deprecatedObjectArrayHashMatchesChainedAddToHash() { + Object[] objs = new Object[] {"alpha", 7, null, true}; + long expected = 0L; + for (Object o : objs) { + expected = addToHash(expected, o); + } + assertEquals(expected, hash(objs)); + } + + // ----- intHash null behavior is observable via multi-arg overloads ----- + + @Test + void multiArgHashTreatsNullAsZero() { + // hash(Object,Object) feeds intHash(...) which returns 0 for null. + // Verify: hash(null, "x") == 31L*0 + "x".hashCode() + int xHash = Objects.hashCode("x"); + assertEquals(31L * 0 + xHash, hash(null, "x")); + } +} From 7728b603f37cf23b13d04b771565dff089519e0c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:19:35 -0400 Subject: [PATCH 36/70] Apply spotless formatting to Hashtable and LongHashingUtils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the new util/ files in line with google-java-format (tabs → spaces, line wrapping, javadoc list markup) so spotlessCheck passes in CI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 902 +++++++++--------- .../datadog/trace/util/LongHashingUtils.java | 8 +- .../datadog/trace/util/HashtableTest.java | 12 +- .../trace/util/LongHashingUtilsTest.java | 6 +- 4 files changed, 467 insertions(+), 461 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index d7f49dcae00..03dfbd7bf1c 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -7,31 +7,31 @@ import java.util.function.Consumer; /** - * Light weight simple Hashtable system that can be useful when HashMap would - * be unnecessarily heavy. - * - *
      Use cases include... - *
    • primitive keys - *
    • primitive values - *
    • multi-part keys + * Light weight simple Hashtable system that can be useful when HashMap would be unnecessarily + * heavy. + * + *
        + * Use cases include... + *
      • primitive keys + *
      • primitive values + *
      • multi-part keys *
      - * + * * Convenience classes are provided for lower key dimensions. - * - * For higher key dimensions, client code must implement its own class, - * but can still use the support class to ease the implementation complexity. + * + *

      For higher key dimensions, client code must implement its own class, but can still use the + * support class to ease the implementation complexity. */ public abstract class Hashtable { /** - * Internal base class for entries. Stores the precomputed 64-bit keyHash and - * the chain-next pointer used to link colliding entries within a single bucket. + * Internal base class for entries. Stores the precomputed 64-bit keyHash and the chain-next + * pointer used to link colliding entries within a single bucket. * - *

      Subclasses add the actual key field(s) and a {@code matches(...)} method - * tailored to their key arity. See {@link D1.Entry} and {@link D2.Entry}; for - * higher arities, client code can subclass this directly and use {@link Support} - * to drive the table mechanics. + *

      Subclasses add the actual key field(s) and a {@code matches(...)} method tailored to their + * key arity. See {@link D1.Entry} and {@link D2.Entry}; for higher arities, client code can + * subclass this directly and use {@link Support} to drive the table mechanics. */ - public static abstract class Entry { + public abstract static class Entry { public final long keyHash; Entry next = null; @@ -44,169 +44,172 @@ public final void setNext(TEntry next) { } @SuppressWarnings("unchecked") - public final TEntry next() { - return (TEntry)this.next; + public final TEntry next() { + return (TEntry) this.next; } } - + /** * Single-key open hash table with chaining. * - *

      The user supplies an {@link D1.Entry} subclass that carries the key and - * whatever value fields they want to mutate in place, then instantiates this - * class over that entry type. The main advantage over {@code HashMap} - * is that mutating an existing entry's value fields requires no allocation: - * call {@link #get} once and write directly to the returned entry's fields. - * For counter-style workloads this can be several times faster than - * {@code HashMap} and produces effectively zero GC pressure. + *

      The user supplies an {@link D1.Entry} subclass that carries the key and whatever value + * fields they want to mutate in place, then instantiates this class over that entry type. The + * main advantage over {@code HashMap} is that mutating an existing entry's value fields + * requires no allocation: call {@link #get} once and write directly to the returned entry's + * fields. For counter-style workloads this can be several times faster than {@code HashMap} and produces effectively zero GC pressure. * - *

      Capacity is fixed at construction. The table does not resize, so the - * caller is responsible for choosing a capacity appropriate to the working - * set. Actual bucket-array length is rounded up to the next power of two. + *

      Capacity is fixed at construction. The table does not resize, so the caller is responsible + * for choosing a capacity appropriate to the working set. Actual bucket-array length is rounded + * up to the next power of two. * - *

      Null keys are permitted; they collapse to a single bucket via the - * sentinel hash {@link Long#MIN_VALUE} defined in {@link D1.Entry#hash}. + *

      Null keys are permitted; they collapse to a single bucket via the sentinel hash {@link + * Long#MIN_VALUE} defined in {@link D1.Entry#hash}. * - *

      Not thread-safe. Concurrent access (including mixing reads with - * writes) requires external synchronization. + *

      Not thread-safe. Concurrent access (including mixing reads with writes) requires + * external synchronization. * * @param the key type * @param the user's {@link D1.Entry D1.Entry<K>} subclass */ public static final class D1> { - /** - * Abstract base for {@link D1} entries. Subclass to add value fields you - * wish to mutate in place after retrieving the entry via {@link D1#get}. - * - *

      The key is captured at construction and stored alongside its - * precomputed 64-bit hash. {@link #matches(Object)} uses - * {@link Objects#equals} by default; override if a different equality - * semantics is needed (e.g. reference equality for interned keys). - * - * @param the key type - */ - public static abstract class Entry extends Hashtable.Entry { - final K key; - - protected Entry(K key) { - super(hash(key)); - this.key = key; - } - - public boolean matches(Object key) { - return Objects.equals(this.key, key); - } - - public static long hash(Object key) { - return (key == null ) ? Long.MIN_VALUE : key.hashCode(); - } - } - - private final Hashtable.Entry[] buckets; - private int size; - - public D1(int capacity) { - this.buckets = Support.create(capacity); - this.size = 0; - } - - public int size() { - return this.size; - } - - @SuppressWarnings("unchecked") - public TEntry get(K key) { - long keyHash = D1.Entry.hash(key); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key)) return te; - } - } - return null; - } - - public TEntry remove(K key) { - long keyHash = D1.Entry.hash(key); - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(key)) { - iter.remove(); - this.size -= 1; - return curEntry; - } - } - - return null; - } - - public void insert(TEntry newEntry) { + /** + * Abstract base for {@link D1} entries. Subclass to add value fields you wish to mutate in + * place after retrieving the entry via {@link D1#get}. + * + *

      The key is captured at construction and stored alongside its precomputed 64-bit hash. + * {@link #matches(Object)} uses {@link Objects#equals} by default; override if a different + * equality semantics is needed (e.g. reference equality for interned keys). + * + * @param the key type + */ + public abstract static class Entry extends Hashtable.Entry { + final K key; + + protected Entry(K key) { + super(hash(key)); + this.key = key; + } + + public boolean matches(Object key) { + return Objects.equals(this.key, key); + } + + public static long hash(Object key) { + return (key == null) ? Long.MIN_VALUE : key.hashCode(); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D1(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K key) { + long keyHash = D1.Entry.hash(key); Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; + e != null; + e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key)) return te; + } + } + return null; + } + + public TEntry remove(K key) { + long keyHash = D1.Entry.hash(key); + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); Hashtable.Entry curHead = thisBuckets[bucketIndex]; newEntry.setNext(curHead); thisBuckets[bucketIndex] = newEntry; this.size += 1; - } - - public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(newEntry.key)) { - iter.replace(newEntry); - return curEntry; - } - } - - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - this.size += 1; - return null; - } - - public void clear() { - Support.clear(this.buckets); - this.size = 0; - } - - @SuppressWarnings("unchecked") - public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } - } + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } } /** * Two-key (composite-key) hash table with chaining. * - *

      The user supplies a {@link D2.Entry} subclass carrying both key parts - * and any value fields. Compared to {@code HashMap} this avoids the - * per-lookup {@code Pair} (or record) allocation: both key parts are passed - * directly through {@link #get}, {@link #remove}, {@link #insert}, and - * {@link #insertOrReplace}. Combined with in-place value mutation, this - * makes {@code D2} substantially less GC-intensive than the equivalent - * {@code HashMap} for counter-style workloads. + *

      The user supplies a {@link D2.Entry} subclass carrying both key parts and any value fields. + * Compared to {@code HashMap} this avoids the per-lookup {@code Pair} (or record) + * allocation: both key parts are passed directly through {@link #get}, {@link #remove}, {@link + * #insert}, and {@link #insertOrReplace}. Combined with in-place value mutation, this makes + * {@code D2} substantially less GC-intensive than the equivalent {@code HashMap} for + * counter-style workloads. * - *

      Capacity is fixed at construction; the table does not resize. Actual - * bucket-array length is rounded up to the next power of two. + *

      Capacity is fixed at construction; the table does not resize. Actual bucket-array length is + * rounded up to the next power of two. * - *

      Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; - * see {@link D2.Entry#hash(Object, Object)}. + *

      Key parts are combined into a 64-bit hash via {@link LongHashingUtils}; see {@link + * D2.Entry#hash(Object, Object)}. * *

      Not thread-safe. * @@ -215,339 +218,340 @@ public void forEach(Consumer consumer) { * @param the user's {@link D2.Entry D2.Entry<K1, K2>} subclass */ public static final class D2> { - /** - * Abstract base for {@link D2} entries. Subclass to add value fields you - * wish to mutate in place. - * - *

      Both key parts are captured at construction and stored alongside their - * combined 64-bit hash. {@link #matches(Object, Object)} uses - * {@link Objects#equals} pairwise on the two parts. - * - * @param first key type - * @param second key type - */ - public static abstract class Entry extends Hashtable.Entry { - final K1 key1; - final K2 key2; - - protected Entry(K1 key1, K2 key2) { - super(hash(key1, key2)); - this.key1 = key1; - this.key2 = key2; - } - - public boolean matches(K1 key1, K2 key2) { - return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); - } - - public static long hash(Object key1, Object key2) { - return LongHashingUtils.hash(key1, key2); - } - } - - private final Hashtable.Entry[] buckets; - private int size; - - public D2(int capacity) { - this.buckets = Support.create(capacity); - this.size = 0; - } - - public int size() { - return this.size; - } - - @SuppressWarnings("unchecked") - public TEntry get(K1 key1, K2 key2) { - long keyHash = D2.Entry.hash(key1, key2); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; e != null; e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key1, key2)) return te; - } - } - return null; - } - - public TEntry remove(K1 key1, K2 key2) { - long keyHash = D2.Entry.hash(key1, key2); - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(key1, key2)) { - iter.remove(); - this.size -= 1; - return curEntry; - } - } - - return null; - } - - public void insert(TEntry newEntry) { + /** + * Abstract base for {@link D2} entries. Subclass to add value fields you wish to mutate in + * place. + * + *

      Both key parts are captured at construction and stored alongside their combined 64-bit + * hash. {@link #matches(Object, Object)} uses {@link Objects#equals} pairwise on the two parts. + * + * @param first key type + * @param second key type + */ + public abstract static class Entry extends Hashtable.Entry { + final K1 key1; + final K2 key2; + + protected Entry(K1 key1, K2 key2) { + super(hash(key1, key2)); + this.key1 = key1; + this.key2 = key2; + } + + public boolean matches(K1 key1, K2 key2) { + return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); + } + + public static long hash(Object key1, Object key2) { + return LongHashingUtils.hash(key1, key2); + } + } + + private final Hashtable.Entry[] buckets; + private int size; + + public D2(int capacity) { + this.buckets = Support.create(capacity); + this.size = 0; + } + + public int size() { + return this.size; + } + + @SuppressWarnings("unchecked") + public TEntry get(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; + e != null; + e = e.next) { + if (e.keyHash == keyHash) { + TEntry te = (TEntry) e; + if (te.matches(key1, key2)) return te; + } + } + return null; + } + + public TEntry remove(K1 key1, K2 key2) { + long keyHash = D2.Entry.hash(key1, key2); + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(key1, key2)) { + iter.remove(); + this.size -= 1; + return curEntry; + } + } + + return null; + } + + public void insert(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); Hashtable.Entry curHead = thisBuckets[bucketIndex]; newEntry.setNext(curHead); thisBuckets[bucketIndex] = newEntry; this.size += 1; - } - - public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { - TEntry curEntry = iter.next(); - - if (curEntry.matches(newEntry.key1, newEntry.key2)) { - iter.replace(newEntry); - return curEntry; - } - } - - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - this.size += 1; - return null; - } - - public void clear() { - Support.clear(this.buckets); - this.size = 0; - } - - @SuppressWarnings("unchecked") - public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } - } + } + + public TEntry insertOrReplace(TEntry newEntry) { + Hashtable.Entry[] thisBuckets = this.buckets; + + for (MutatingBucketIterator iter = + Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); + iter.hasNext(); ) { + TEntry curEntry = iter.next(); + + if (curEntry.matches(newEntry.key1, newEntry.key2)) { + iter.replace(newEntry); + return curEntry; + } + } + + int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); + + Hashtable.Entry curHead = thisBuckets[bucketIndex]; + newEntry.setNext(curHead); + thisBuckets[bucketIndex] = newEntry; + this.size += 1; + return null; + } + + public void clear() { + Support.clear(this.buckets); + this.size = 0; + } + + @SuppressWarnings("unchecked") + public void forEach(Consumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } } /** * Internal building blocks for hash-table operations. * - *

      Used by {@link D1} and {@link D2}, and available to package code that - * wants to assemble its own higher-arity table (3+ key parts) without - * re-implementing the bucket-array mechanics. The typical recipe: + *

      Used by {@link D1} and {@link D2}, and available to package code that wants to assemble its + * own higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The + * typical recipe: * *

        - *
      • Subclass {@link Hashtable.Entry} directly, adding the key fields and - * a {@code matches(...)} method of your chosen arity. + *
      • Subclass {@link Hashtable.Entry} directly, adding the key fields and a {@code + * matches(...)} method of your chosen arity. *
      • Allocate a backing array with {@link #create(int)}. - *
      • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, - * {@link #bucketIterator(Hashtable.Entry[], long)} for read-only chain - * walks, and {@link #mutatingBucketIterator(Hashtable.Entry[], long)} - * when you also need {@code remove} / {@code replace}. + *
      • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, {@link + * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link + * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / + * {@code replace}. *
      • Clear with {@link #clear(Hashtable.Entry[])}. *
      * - *

      All bucket arrays produced by {@link #create(int)} have a power-of-two - * length, so {@link #bucketIndex(Object[], long)} can use a bit mask. + *

      All bucket arrays produced by {@link #create(int)} have a power-of-two length, so {@link + * #bucketIndex(Object[], long)} can use a bit mask. * - *

      Methods on this class are package-private; the class itself is public - * only so that its nested {@link BucketIterator} can be referenced by - * callers in other packages. + *

      Methods on this class are package-private; the class itself is public only so that its + * nested {@link BucketIterator} can be referenced by callers in other packages. */ public static final class Support { - public static final Hashtable.Entry[] create(int capacity) { - return new Entry[sizeFor(capacity)]; - } - - static final int sizeFor(int requestedCapacity) { - int pow; - for ( pow = 1; pow < requestedCapacity; pow *= 2 ); - return pow; - } - - public static final void clear(Hashtable.Entry[] buckets) { - Arrays.fill(buckets, null); - } - - public static final BucketIterator bucketIterator(Hashtable.Entry[] buckets, long keyHash) { - return new BucketIterator(buckets, keyHash); - } - - public static final MutatingBucketIterator mutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { - return new MutatingBucketIterator(buckets, keyHash); - } - - public static final int bucketIndex(Object[] buckets, long keyHash) { - return (int)(keyHash & buckets.length - 1); - } + public static final Hashtable.Entry[] create(int capacity) { + return new Entry[sizeFor(capacity)]; + } + + static final int sizeFor(int requestedCapacity) { + int pow; + for (pow = 1; pow < requestedCapacity; pow *= 2) + ; + return pow; + } + + public static final void clear(Hashtable.Entry[] buckets) { + Arrays.fill(buckets, null); + } + + public static final BucketIterator bucketIterator( + Hashtable.Entry[] buckets, long keyHash) { + return new BucketIterator(buckets, keyHash); + } + + public static final + MutatingBucketIterator mutatingBucketIterator( + Hashtable.Entry[] buckets, long keyHash) { + return new MutatingBucketIterator(buckets, keyHash); + } + + public static final int bucketIndex(Object[] buckets, long keyHash) { + return (int) (keyHash & buckets.length - 1); + } } - + /** - * Read-only iterator over entries in a single bucket whose {@code keyHash} - * matches a specific search hash. Cheaper than {@link MutatingBucketIterator} - * because it does not track the previous-node pointers required for - * splicing — use it when you only need to walk the chain. + * Read-only iterator over entries in a single bucket whose {@code keyHash} matches a specific + * search hash. Cheaper than {@link MutatingBucketIterator} because it does not track the + * previous-node pointers required for splicing — use it when you only need to walk the chain. * - *

      For {@code remove} or {@code replace} operations, use - * {@link MutatingBucketIterator} instead. + *

      For {@code remove} or {@code replace} operations, use {@link MutatingBucketIterator} + * instead. */ public static final class BucketIterator implements Iterator { - private final long keyHash; - private Hashtable.Entry nextEntry; - - BucketIterator(Hashtable.Entry[] buckets, long keyHash) { - this.keyHash = keyHash; - Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; - while (cur != null && cur.keyHash != keyHash) cur = cur.next; - this.nextEntry = cur; - } - - @Override - public boolean hasNext() { - return this.nextEntry != null; - } - - @Override - @SuppressWarnings("unchecked") - public TEntry next() { - Hashtable.Entry cur = this.nextEntry; - if (cur == null) throw new NoSuchElementException("no next!"); - - Hashtable.Entry advance = cur.next; - while (advance != null && advance.keyHash != keyHash) advance = advance.next; - this.nextEntry = advance; - - return (TEntry) cur; - } + private final long keyHash; + private Hashtable.Entry nextEntry; + + BucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.keyHash = keyHash; + Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; + while (cur != null && cur.keyHash != keyHash) cur = cur.next; + this.nextEntry = cur; + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry cur = this.nextEntry; + if (cur == null) throw new NoSuchElementException("no next!"); + + Hashtable.Entry advance = cur.next; + while (advance != null && advance.keyHash != keyHash) advance = advance.next; + this.nextEntry = advance; + + return (TEntry) cur; + } } /** - * Mutating iterator over entries in a single bucket whose {@code keyHash} - * matches a specific search hash. Supports {@link #remove()} and - * {@link #replace(Entry)} to splice the chain in place. + * Mutating iterator over entries in a single bucket whose {@code keyHash} matches a specific + * search hash. Supports {@link #remove()} and {@link #replace(Entry)} to splice the chain in + * place. * - *

      Carries previous-node pointers for the current entry and the next-match - * entry so that {@code remove} and {@code replace} can fix up the chain in - * O(1) without re-walking from the bucket head. After {@code remove} or - * {@code replace}, iteration may continue with another {@link #next()}. + *

      Carries previous-node pointers for the current entry and the next-match entry so that {@code + * remove} and {@code replace} can fix up the chain in O(1) without re-walking from the bucket + * head. After {@code remove} or {@code replace}, iteration may continue with another {@link + * #next()}. */ - public static final class MutatingBucketIterator implements Iterator { - private final long keyHash; - - private final Hashtable.Entry[] buckets; - - /** - * The entry prior to the last entry returned by next - * Used for mutating operations - */ - private Hashtable.Entry curPrevEntry; - - /** - * The entry that was last returned by next - */ - private Hashtable.Entry curEntry; - - /** - * The entry prior to the next entry - */ - private Hashtable.Entry nextPrevEntry; - - /** - * The next entry to be returned by next - */ - private Hashtable.Entry nextEntry; - - MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { - this.buckets = buckets; - this.keyHash = keyHash; - - int bucketIndex = Support.bucketIndex(buckets, keyHash); - Hashtable.Entry headEntry = this.buckets[bucketIndex]; - if ( headEntry == null ) { - this.nextEntry = null; - this.nextPrevEntry = null; - - this.curEntry = null; - this.curPrevEntry = null; - } else { - Hashtable.Entry prev, cur; - for ( prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next() ) { - if ( cur.keyHash == keyHash ) break; - } - this.nextPrevEntry = prev; - this.nextEntry = cur; - - this.curEntry = null; - this.curPrevEntry = null; - } - } - - @Override - public boolean hasNext() { - return (this.nextEntry != null); - } - - @Override - @SuppressWarnings("unchecked") - public TEntry next() { - Hashtable.Entry curEntry = this.nextEntry; - if ( curEntry == null ) throw new NoSuchElementException("no next!"); - - this.curEntry = curEntry; - this.curPrevEntry = this.nextPrevEntry; - - Hashtable.Entry prev, cur; - for ( prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next() ) { - if ( cur.keyHash == keyHash ) break; - } - this.nextPrevEntry = prev; - this.nextEntry = cur; - - return (TEntry) curEntry; - } - - @Override - public void remove() { - Hashtable.Entry oldCurEntry = this.curEntry; - if ( oldCurEntry == null ) throw new IllegalStateException(); + public static final class MutatingBucketIterator + implements Iterator { + private final long keyHash; + + private final Hashtable.Entry[] buckets; + + /** The entry prior to the last entry returned by next Used for mutating operations */ + private Hashtable.Entry curPrevEntry; + + /** The entry that was last returned by next */ + private Hashtable.Entry curEntry; + + /** The entry prior to the next entry */ + private Hashtable.Entry nextPrevEntry; + + /** The next entry to be returned by next */ + private Hashtable.Entry nextEntry; + + MutatingBucketIterator(Hashtable.Entry[] buckets, long keyHash) { + this.buckets = buckets; + this.keyHash = keyHash; + + int bucketIndex = Support.bucketIndex(buckets, keyHash); + Hashtable.Entry headEntry = this.buckets[bucketIndex]; + if (headEntry == null) { + this.nextEntry = null; + this.nextPrevEntry = null; + + this.curEntry = null; + this.curPrevEntry = null; + } else { + Hashtable.Entry prev, cur; + for (prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next()) { + if (cur.keyHash == keyHash) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + this.curEntry = null; + this.curPrevEntry = null; + } + } + + @Override + public boolean hasNext() { + return (this.nextEntry != null); + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry curEntry = this.nextEntry; + if (curEntry == null) throw new NoSuchElementException("no next!"); + + this.curEntry = curEntry; + this.curPrevEntry = this.nextPrevEntry; + + Hashtable.Entry prev, cur; + for (prev = this.nextEntry, cur = this.nextEntry.next(); + cur != null; + prev = cur, cur = prev.next()) { + if (cur.keyHash == keyHash) break; + } + this.nextPrevEntry = prev; + this.nextEntry = cur; + + return (TEntry) curEntry; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); this.setPrevNext(oldCurEntry.next()); // If the next match was directly after oldCurEntry, its predecessor is now // curPrevEntry (oldCurEntry was just unlinked from the chain). - if ( this.nextPrevEntry == oldCurEntry ) { + if (this.nextPrevEntry == oldCurEntry) { this.nextPrevEntry = this.curPrevEntry; } this.curEntry = null; - } - - public void replace(TEntry replacementEntry) { - Hashtable.Entry oldCurEntry = this.curEntry; - if ( oldCurEntry == null ) throw new IllegalStateException(); - - replacementEntry.setNext(oldCurEntry.next()); - this.setPrevNext(replacementEntry); - - // If the next match was directly after oldCurEntry, its predecessor is now - // the replacement entry (which took oldCurEntry's chain slot). - if ( this.nextPrevEntry == oldCurEntry ) { - this.nextPrevEntry = replacementEntry; - } - this.curEntry = replacementEntry; - } - - void setPrevNext(Hashtable.Entry nextEntry) { - if ( this.curPrevEntry == null ) { - Hashtable.Entry[] buckets = this.buckets; - buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; - } else { - this.curPrevEntry.setNext(nextEntry); - } - } + } + + public void replace(TEntry replacementEntry) { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); + + replacementEntry.setNext(oldCurEntry.next()); + this.setPrevNext(replacementEntry); + + // If the next match was directly after oldCurEntry, its predecessor is now + // the replacement entry (which took oldCurEntry's chain slot). + if (this.nextPrevEntry == oldCurEntry) { + this.nextPrevEntry = replacementEntry; + } + this.curEntry = replacementEntry; + } + + void setPrevNext(Hashtable.Entry nextEntry) { + if (this.curPrevEntry == null) { + Hashtable.Entry[] buckets = this.buckets; + buckets[Support.bucketIndex(buckets, this.keyHash)] = nextEntry; + } else { + this.curPrevEntry.setNext(nextEntry); + } + } } } diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index bc53bc4ecb6..ab8b18a4ca9 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -53,7 +53,7 @@ public static final long hash(int hash0, int hash1) { } private static final int intHash(Object obj) { - return obj == null ? 0 : obj.hashCode(); + return obj == null ? 0 : obj.hashCode(); } public static final long hash(Object obj0, Object obj1, Object obj2) { @@ -86,7 +86,11 @@ public static final long hash(int hash0, int hash1, int hash2, int hash3, int ha // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. - return 31L * 31L * 31L * 31L * hash0 + 31L * 31L * 31L * hash1 + 31L * 31L * hash2 + 31L * hash3 + hash4; + return 31L * 31L * 31L * 31L * hash0 + + 31L * 31L * 31L * hash1 + + 31L * 31L * hash2 + + 31L * hash3 + + hash4; } @Deprecated diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 67c99c0d08d..2d12d535178 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -294,8 +294,7 @@ void walksOnlyMatchingHash() { table.insert(new CollidingKeyEntry(k2, 2)); table.insert(new CollidingKeyEntry(k3, 3)); // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. - BucketIterator it = - Support.bucketIterator(extractBuckets(table), 17L); + BucketIterator it = Support.bucketIterator(extractBuckets(table), 17L); int count = 0; while (it.hasNext()) { assertNotNull(it.next()); @@ -380,8 +379,7 @@ void removeWithoutNextThrows() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("a", 1)); MutatingBucketIterator it = - Support.mutatingBucketIterator( - extractBuckets(table), Hashtable.D1.Entry.hash("a")); + Support.mutatingBucketIterator(extractBuckets(table), Hashtable.D1.Entry.hash("a")); assertThrows(IllegalStateException.class, it::remove); } } @@ -401,8 +399,7 @@ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { /** Sort comparator used by tests that want deterministic visit order. */ @SuppressWarnings("unused") - private static final Comparator BY_KEY = - Comparator.comparing(e -> e.key); + private static final Comparator BY_KEY = Comparator.comparing(e -> e.key); private static final class StringIntEntry extends Hashtable.D1.Entry { int value; @@ -459,7 +456,8 @@ private static final class PairEntry extends Hashtable.D2.Entry } } - // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning quiet. + // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning + // quiet. @SuppressWarnings("unused") private static final List UNUSED = new ArrayList<>(); } diff --git a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java index d0053c75b42..c0e0bebdda0 100644 --- a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java +++ b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java @@ -57,8 +57,7 @@ void fourArgHashMatchesChainedAddToHash() { Object b = 42; Object c = true; Object d = 3.14; - assertEquals( - addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); + assertEquals(addToHash(addToHash(addToHash(addToHash(0L, a), b), c), d), hash(a, b, c, d)); } @Test @@ -76,7 +75,8 @@ void fiveArgHashMatchesChainedAddToHash() { @Test void multiArgHashHandlesNullsConsistentlyWithChainedAddToHash() { assertEquals(addToHash(addToHash(0L, (Object) null), "x"), hash(null, "x")); - assertEquals(addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); + assertEquals( + addToHash(addToHash(addToHash(0L, "x"), (Object) null), "y"), hash("x", null, "y")); } @Test From 8cd2d86ba467dbbc2b7859ff4941479e4386ec3f Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:19:43 -0400 Subject: [PATCH 37/70] Add JMH benchmarks for Hashtable.D1 and D2 Compares Hashtable.D1 and Hashtable.D2 against equivalent HashMap usage for add, update, and iterate operations. Each benchmark thread owns its own map (Scope.Thread), but @Threads(8) is used so the allocation/GC pressure that Hashtable is designed to avoid surfaces in the throughput numbers. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableBenchmark.java | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java new file mode 100644 index 00000000000..bf25efba679 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java @@ -0,0 +1,290 @@ +package datadog.trace.util; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Compares {@link Hashtable.D1} and {@link Hashtable.D2} against equivalent {@link HashMap} usage + * for add, update, and iterate operations. + * + *

      Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count + * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main + * thing Hashtable is built to avoid. + * + *

        + *
      • add — clear the map then re-insert N fresh entries + * ({@code @OperationsPerInvocation(N_KEYS)}). Captures the steady-state cost of building up a + * map. + *
      • update — for an existing key, increment a counter. Hashtable does {@code get} + + * field mutation (no allocation); HashMap uses {@code merge(k, 1L, Long::sum)}, the idiomatic + * Java 8+ way, which still allocates a {@code Long} per call. + *
      • iterate — walk every entry and consume its key + value. + *
      + * + *

      The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path + * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. + */ +@Fork(2) +@Warmup(iterations = 2) +@Measurement(iterations = 3) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(MICROSECONDS) +@Threads(8) +public class HashtableBenchmark { + + static final int N_KEYS = 64; + static final int CAPACITY = 128; + + static final String[] SOURCE_K1 = new String[N_KEYS]; + static final Integer[] SOURCE_K2 = new Integer[N_KEYS]; + + static { + for (int i = 0; i < N_KEYS; ++i) { + SOURCE_K1[i] = "key-" + i; + SOURCE_K2[i] = i * 31 + 17; + } + } + + static final class D1Counter extends Hashtable.D1.Entry { + long count; + + D1Counter(String key) { + super(key); + } + } + + static final class D2Counter extends Hashtable.D2.Entry { + long count; + + D2Counter(String k1, Integer k2) { + super(k1, k2); + } + } + + /** Composite key for the HashMap baseline against D2. */ + static final class Key2 { + final String k1; + final Integer k2; + final int hash; + + Key2(String k1, Integer k2) { + this.k1 = k1; + this.k2 = k2; + this.hash = Objects.hash(k1, k2); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Key2)) return false; + Key2 other = (Key2) o; + return Objects.equals(k1, other.k1) && Objects.equals(k2, other.k2); + } + + @Override + public int hashCode() { + return hash; + } + } + + /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ + static final class BhD1Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D1Counter e) { + bh.consume(e.key); + bh.consume(e.count); + } + } + + static final class BhD2Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D2Counter e) { + bh.consume(e.key1); + bh.consume(e.key2); + bh.consume(e.count); + } + } + + @State(Scope.Thread) + public static class D1State { + Hashtable.D1 table; + HashMap hashMap; + String[] keys; + int cursor; + final BhD1Consumer consumer = new BhD1Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D1<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + keys = SOURCE_K1; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D1Counter(keys[i])); + hashMap.put(keys[i], 0L); + } + cursor = 0; + } + + String nextKey() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return keys[i]; + } + } + + @State(Scope.Thread) + public static class D2State { + Hashtable.D2 table; + HashMap hashMap; + String[] k1s; + Integer[] k2s; + int cursor; + final BhD2Consumer consumer = new BhD2Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D2<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + k1s = SOURCE_K1; + k2s = SOURCE_K2; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D2Counter(k1s[i], k2s[i])); + hashMap.put(new Key2(k1s[i], k2s[i]), 0L); + } + cursor = 0; + } + + int nextIndex() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return i; + } + } + + // ============================================================ + // D1 — single-key + // ============================================================ + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashtable(D1State s) { + Hashtable.D1 t = s.table; + String[] keys = s.keys; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D1Counter(keys[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashMap(D1State s) { + HashMap m = s.hashMap; + String[] keys = s.keys; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(keys[i], (long) i); + } + } + + @Benchmark + public long d1_update_hashtable(D1State s) { + D1Counter e = s.table.get(s.nextKey()); + return ++e.count; + } + + @Benchmark + public Long d1_update_hashMap(D1State s) { + return s.hashMap.merge(s.nextKey(), 1L, Long::sum); + } + + @Benchmark + public void d1_iterate_hashtable(D1State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d1_iterate_hashMap(D1State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } + + // ============================================================ + // D2 — two-key (composite) + // ============================================================ + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d2_add_hashtable(D2State s) { + Hashtable.D2 t = s.table; + String[] k1s = s.k1s; + Integer[] k2s = s.k2s; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D2Counter(k1s[i], k2s[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d2_add_hashMap(D2State s) { + HashMap m = s.hashMap; + String[] k1s = s.k1s; + Integer[] k2s = s.k2s; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(new Key2(k1s[i], k2s[i]), (long) i); + } + } + + @Benchmark + public long d2_update_hashtable(D2State s) { + int i = s.nextIndex(); + D2Counter e = s.table.get(s.k1s[i], s.k2s[i]); + return ++e.count; + } + + @Benchmark + public Long d2_update_hashMap(D2State s) { + int i = s.nextIndex(); + return s.hashMap.merge(new Key2(s.k1s[i], s.k2s[i]), 1L, Long::sum); + } + + @Benchmark + public void d2_iterate_hashtable(D2State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d2_iterate_hashMap(D2State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } +} From c689ef968552fc34399e9382d162cd56b7676467 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 18 May 2026 16:21:11 -0400 Subject: [PATCH 38/70] Add benchmark results to HashtableBenchmark header Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableBenchmark.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java index bf25efba679..46e483018e6 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java @@ -41,6 +41,33 @@ * *

      The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. + * + *

      Update is where Hashtable dominates: D1 is ~14x faster, D2 is ~26x faster, because the + * HashMap path allocates per call (a {@code Long}, plus a {@code Key2} for D2) and the resulting GC + * pressure throttles throughput under multiple threads. Add is roughly comparable for D1 + * (both allocate one entry per insert) and ~3x faster for D2 (Hashtable sidesteps the {@code Key2} + * allocation). Iterate is essentially a wash — both are bucket walks. + * MacBook M1 8 threads (Java 8) + * + * Benchmark Mode Cnt Score Error Units + * HashtableBenchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableBenchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * + * HashtableBenchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableBenchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * + * HashtableBenchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableBenchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * + * HashtableBenchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableBenchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us + * + * HashtableBenchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableBenchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us + * + * HashtableBenchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableBenchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * */ @Fork(2) @Warmup(iterations = 2) From 75790eb371b6401186f88ad1c6e16a197d6672a0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 10:59:04 -0400 Subject: [PATCH 39/70] Address review feedback on Hashtable - Guard Support.sizeFor against overflow and use Integer.highestOneBit; reject capacities above 1 << 30 instead of looping forever. - Add braces around single-statement while bodies in BucketIterator. - Split HashtableBenchmark into HashtableD1Benchmark / HashtableD2Benchmark. - Add regression tests for Support.sizeFor bounds. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableD1Benchmark.java | 169 ++++++++++++++++++ ...nchmark.java => HashtableD2Benchmark.java} | 142 ++------------- .../java/datadog/trace/util/Hashtable.java | 25 ++- .../datadog/trace/util/HashtableTest.java | 27 +++ 4 files changed, 232 insertions(+), 131 deletions(-) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java rename internal-api/src/jmh/java/datadog/trace/util/{HashtableBenchmark.java => HashtableD2Benchmark.java} (55%) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java new file mode 100644 index 00000000000..16b95e089d5 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -0,0 +1,169 @@ +package datadog.trace.util; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Compares {@link Hashtable.D1} against equivalent {@link HashMap} usage for add, update, and + * iterate operations. + * + *

      Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count + * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main + * thing Hashtable is built to avoid. + * + *

        + *
      • add — clear the map then re-insert N fresh entries + * ({@code @OperationsPerInvocation(N_KEYS)}). Captures the steady-state cost of building up a + * map. + *
      • update — for an existing key, increment a counter. Hashtable does {@code get} + + * field mutation (no allocation); HashMap uses {@code merge(k, 1L, Long::sum)}, the idiomatic + * Java 8+ way, which still allocates a {@code Long} per call. + *
      • iterate — walk every entry and consume its key + value. + *
      + * + *

      Update is where Hashtable dominates: D1 is ~14x faster, because the HashMap path + * allocates per call (a {@code Long}) and the resulting GC pressure throttles throughput under + * multiple threads. Add is roughly comparable (both allocate one entry per insert). + * Iterate is essentially a wash — both are bucket walks. + * MacBook M1 8 threads (Java 8) + * + * Benchmark Mode Cnt Score Error Units + * HashtableD1Benchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableD1Benchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * + * HashtableD1Benchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableD1Benchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * + * HashtableD1Benchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableD1Benchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * + */ +@Fork(2) +@Warmup(iterations = 2) +@Measurement(iterations = 3) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(MICROSECONDS) +@Threads(8) +public class HashtableD1Benchmark { + + static final int N_KEYS = 64; + static final int CAPACITY = 128; + + static final String[] SOURCE_KEYS = new String[N_KEYS]; + + static { + for (int i = 0; i < N_KEYS; ++i) { + SOURCE_KEYS[i] = "key-" + i; + } + } + + static final class D1Counter extends Hashtable.D1.Entry { + long count; + + D1Counter(String key) { + super(key); + } + } + + /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ + static final class BhD1Consumer implements Consumer { + Blackhole bh; + + @Override + public void accept(D1Counter e) { + bh.consume(e.key); + bh.consume(e.count); + } + } + + @State(Scope.Thread) + public static class D1State { + Hashtable.D1 table; + HashMap hashMap; + String[] keys; + int cursor; + final BhD1Consumer consumer = new BhD1Consumer(); + + @Setup(Level.Iteration) + public void setUp() { + table = new Hashtable.D1<>(CAPACITY); + hashMap = new HashMap<>(CAPACITY); + keys = SOURCE_KEYS; + for (int i = 0; i < N_KEYS; ++i) { + table.insert(new D1Counter(keys[i])); + hashMap.put(keys[i], 0L); + } + cursor = 0; + } + + String nextKey() { + int i = cursor; + cursor = (i + 1) & (N_KEYS - 1); + return keys[i]; + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashtable(D1State s) { + Hashtable.D1 t = s.table; + String[] keys = s.keys; + t.clear(); + for (int i = 0; i < N_KEYS; ++i) { + t.insert(new D1Counter(keys[i])); + } + } + + @Benchmark + @OperationsPerInvocation(N_KEYS) + public void d1_add_hashMap(D1State s) { + HashMap m = s.hashMap; + String[] keys = s.keys; + m.clear(); + for (int i = 0; i < N_KEYS; ++i) { + m.put(keys[i], (long) i); + } + } + + @Benchmark + public long d1_update_hashtable(D1State s) { + D1Counter e = s.table.get(s.nextKey()); + return ++e.count; + } + + @Benchmark + public Long d1_update_hashMap(D1State s) { + return s.hashMap.merge(s.nextKey(), 1L, Long::sum); + } + + @Benchmark + public void d1_iterate_hashtable(D1State s, Blackhole bh) { + s.consumer.bh = bh; + s.table.forEach(s.consumer); + } + + @Benchmark + public void d1_iterate_hashMap(D1State s, Blackhole bh) { + for (Map.Entry entry : s.hashMap.entrySet()) { + bh.consume(entry.getKey()); + bh.consume(entry.getValue()); + } + } +} diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java similarity index 55% rename from internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java rename to internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 46e483018e6..5fd64ed9a75 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -22,8 +22,8 @@ import org.openjdk.jmh.infra.Blackhole; /** - * Compares {@link Hashtable.D1} and {@link Hashtable.D2} against equivalent {@link HashMap} usage - * for add, update, and iterate operations. + * Compares {@link Hashtable.D2} against equivalent {@link HashMap} usage for add, update, and + * iterate operations. * *

      Each benchmark thread owns its own map ({@link Scope#Thread}), but a non-trivial thread count * is used so allocation/GC pressure surfaces in the throughput numbers — that pressure is the main @@ -42,31 +42,21 @@ *

      The D2 variants additionally pay for a composite-key wrapper allocation in the HashMap path * (Java has no built-in tuple-as-key) — D2 sidesteps it by taking both key parts directly. * - *

      Update is where Hashtable dominates: D1 is ~14x faster, D2 is ~26x faster, because the - * HashMap path allocates per call (a {@code Long}, plus a {@code Key2} for D2) and the resulting GC - * pressure throttles throughput under multiple threads. Add is roughly comparable for D1 - * (both allocate one entry per insert) and ~3x faster for D2 (Hashtable sidesteps the {@code Key2} - * allocation). Iterate is essentially a wash — both are bucket walks. + *

      Update is where Hashtable dominates: D2 is ~26x faster, because the HashMap path + * allocates per call (a {@code Long}, plus a {@code Key2}) and the resulting GC pressure throttles + * throughput under multiple threads. Add is ~3x faster for D2 (Hashtable sidesteps the + * {@code Key2} allocation). Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableBenchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us - * HashtableBenchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD2Benchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableD2Benchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us * - * HashtableBenchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us - * HashtableBenchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * HashtableD2Benchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableD2Benchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us * - * HashtableBenchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us - * HashtableBenchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us - * - * HashtableBenchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us - * HashtableBenchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us - * - * HashtableBenchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us - * HashtableBenchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us - * - * HashtableBenchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us - * HashtableBenchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * HashtableD2Benchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableD2Benchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us * */ @Fork(2) @@ -75,7 +65,7 @@ @BenchmarkMode(Mode.Throughput) @OutputTimeUnit(MICROSECONDS) @Threads(8) -public class HashtableBenchmark { +public class HashtableD2Benchmark { static final int N_KEYS = 64; static final int CAPACITY = 128; @@ -90,14 +80,6 @@ public class HashtableBenchmark { } } - static final class D1Counter extends Hashtable.D1.Entry { - long count; - - D1Counter(String key) { - super(key); - } - } - static final class D2Counter extends Hashtable.D2.Entry { long count; @@ -120,7 +102,9 @@ static final class Key2 { @Override public boolean equals(Object o) { - if (!(o instanceof Key2)) return false; + if (!(o instanceof Key2)) { + return false; + } Key2 other = (Key2) o; return Objects.equals(k1, other.k1) && Objects.equals(k2, other.k2); } @@ -132,16 +116,6 @@ public int hashCode() { } /** Reusable iteration consumer — avoids per-call lambda capture allocation. */ - static final class BhD1Consumer implements Consumer { - Blackhole bh; - - @Override - public void accept(D1Counter e) { - bh.consume(e.key); - bh.consume(e.count); - } - } - static final class BhD2Consumer implements Consumer { Blackhole bh; @@ -153,33 +127,6 @@ public void accept(D2Counter e) { } } - @State(Scope.Thread) - public static class D1State { - Hashtable.D1 table; - HashMap hashMap; - String[] keys; - int cursor; - final BhD1Consumer consumer = new BhD1Consumer(); - - @Setup(Level.Iteration) - public void setUp() { - table = new Hashtable.D1<>(CAPACITY); - hashMap = new HashMap<>(CAPACITY); - keys = SOURCE_K1; - for (int i = 0; i < N_KEYS; ++i) { - table.insert(new D1Counter(keys[i])); - hashMap.put(keys[i], 0L); - } - cursor = 0; - } - - String nextKey() { - int i = cursor; - cursor = (i + 1) & (N_KEYS - 1); - return keys[i]; - } - } - @State(Scope.Thread) public static class D2State { Hashtable.D2 table; @@ -209,61 +156,6 @@ int nextIndex() { } } - // ============================================================ - // D1 — single-key - // ============================================================ - - @Benchmark - @OperationsPerInvocation(N_KEYS) - public void d1_add_hashtable(D1State s) { - Hashtable.D1 t = s.table; - String[] keys = s.keys; - t.clear(); - for (int i = 0; i < N_KEYS; ++i) { - t.insert(new D1Counter(keys[i])); - } - } - - @Benchmark - @OperationsPerInvocation(N_KEYS) - public void d1_add_hashMap(D1State s) { - HashMap m = s.hashMap; - String[] keys = s.keys; - m.clear(); - for (int i = 0; i < N_KEYS; ++i) { - m.put(keys[i], (long) i); - } - } - - @Benchmark - public long d1_update_hashtable(D1State s) { - D1Counter e = s.table.get(s.nextKey()); - return ++e.count; - } - - @Benchmark - public Long d1_update_hashMap(D1State s) { - return s.hashMap.merge(s.nextKey(), 1L, Long::sum); - } - - @Benchmark - public void d1_iterate_hashtable(D1State s, Blackhole bh) { - s.consumer.bh = bh; - s.table.forEach(s.consumer); - } - - @Benchmark - public void d1_iterate_hashMap(D1State s, Blackhole bh) { - for (Map.Entry entry : s.hashMap.entrySet()) { - bh.consume(entry.getKey()); - bh.consume(entry.getValue()); - } - } - - // ============================================================ - // D2 — two-key (composite) - // ============================================================ - @Benchmark @OperationsPerInvocation(N_KEYS) public void d2_add_hashtable(D2State s) { diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 03dfbd7bf1c..39dfaf6c7a4 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -371,11 +371,20 @@ public static final Hashtable.Entry[] create(int capacity) { return new Entry[sizeFor(capacity)]; } + static final int MAX_CAPACITY = 1 << 30; + static final int sizeFor(int requestedCapacity) { - int pow; - for (pow = 1; pow < requestedCapacity; pow *= 2) - ; - return pow; + if (requestedCapacity < 0) { + throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); + } + if (requestedCapacity > MAX_CAPACITY) { + throw new IllegalArgumentException( + "capacity exceeds maximum (" + MAX_CAPACITY + "): " + requestedCapacity); + } + if (requestedCapacity <= 1) { + return 1; + } + return Integer.highestOneBit(requestedCapacity - 1) << 1; } public static final void clear(Hashtable.Entry[] buckets) { @@ -413,7 +422,9 @@ public static final class BucketIterator implements Iterat BucketIterator(Hashtable.Entry[] buckets, long keyHash) { this.keyHash = keyHash; Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; - while (cur != null && cur.keyHash != keyHash) cur = cur.next; + while (cur != null && cur.keyHash != keyHash) { + cur = cur.next; + } this.nextEntry = cur; } @@ -429,7 +440,9 @@ public TEntry next() { if (cur == null) throw new NoSuchElementException("no next!"); Hashtable.Entry advance = cur.next; - while (advance != null && advance.keyHash != keyHash) advance = advance.next; + while (advance != null && advance.keyHash != keyHash) { + advance = advance.next; + } this.nextEntry = advance; return (TEntry) cur; diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 2d12d535178..b11a33a4322 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -255,6 +255,33 @@ void createRoundsCapacityUpToPowerOfTwo() { assertEquals(0, len & (len - 1), "length must be a power of two"); } + @Test + void sizeForReturnsAtLeastOne() { + assertEquals(1, Support.sizeFor(0)); + assertEquals(1, Support.sizeFor(1)); + } + + @Test + void sizeForRoundsUpToPowerOfTwo() { + assertEquals(2, Support.sizeFor(2)); + assertEquals(4, Support.sizeFor(3)); + assertEquals(4, Support.sizeFor(4)); + assertEquals(8, Support.sizeFor(5)); + assertEquals(1 << 30, Support.sizeFor(1 << 30)); + } + + @Test + void sizeForRejectsCapacityAboveMax() { + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor((1 << 30) + 1)); + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(Integer.MAX_VALUE)); + } + + @Test + void sizeForRejectsNegativeCapacity() { + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(-1)); + assertThrows(IllegalArgumentException.class, () -> Support.sizeFor(Integer.MIN_VALUE)); + } + @Test void bucketIndexIsBoundedByArrayLength() { Hashtable.Entry[] buckets = Support.create(16); From 6056ff7b71abe33d82417529b390bb6cf4b82a26 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:19:43 -0400 Subject: [PATCH 40/70] Fix dropped argument in HashingUtils 5-arg Object hash The 5-arg Object overload was forwarding only obj0..obj3 to the int overload, silently dropping obj4. Also align LongHashingUtils.hash 3-arg signature with its 2/4/5-arg siblings (int parameters) and strengthen the 5-arg HashingUtilsTest to detect the missing-arg regression. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/main/java/datadog/trace/util/HashingUtils.java | 2 +- .../src/main/java/datadog/trace/util/LongHashingUtils.java | 2 +- .../src/test/java/datadog/trace/util/HashingUtilsTest.java | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/HashingUtils.java b/internal-api/src/main/java/datadog/trace/util/HashingUtils.java index 1522554836a..d975149f433 100644 --- a/internal-api/src/main/java/datadog/trace/util/HashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/HashingUtils.java @@ -79,7 +79,7 @@ public static final int hash(int hash0, int hash1, int hash2, int hash3) { } public static final int hash(Object obj0, Object obj1, Object obj2, Object obj3, Object obj4) { - return hash(hashCode(obj0), hashCode(obj1), hashCode(obj2), hashCode(obj3)); + return hash(hashCode(obj0), hashCode(obj1), hashCode(obj2), hashCode(obj3), hashCode(obj4)); } public static final int hash(int hash0, int hash1, int hash2, int hash3, int hash4) { diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index ab8b18a4ca9..c14b498cc9c 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -60,7 +60,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2) { return hash(intHash(obj0), intHash(obj1), intHash(obj2)); } - public static final long hash(long hash0, long hash1, long hash2) { + public static final long hash(int hash0, int hash1, int hash2) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. diff --git a/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java index 185d5a4f2e4..1f171852866 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashingUtilsTest.java @@ -99,7 +99,7 @@ public void hash5() { String str3 = "foobar"; String str4 = "hello"; - assertNotEquals(0, HashingUtils.hash(str0, str1, str2, str3)); + assertNotEquals(0, HashingUtils.hash(str0, str1, str2, str3, str4)); String clone0 = clone(str0); String clone1 = clone(str1); @@ -110,6 +110,11 @@ public void hash5() { assertEquals( HashingUtils.hash(str0, str1, str2, str3, str4), HashingUtils.hash(clone0, clone1, clone2, clone3, clone4)); + + // The 5th argument must actually affect the hash (regression for a missing-arg bug). + assertNotEquals( + HashingUtils.hash(str0, str1, str2, str3, str4), + HashingUtils.hash(str0, str1, str2, str3, "different")); } @Test From da55021b68b779d86346372ba65828d01fb4f4a8 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:25:58 -0400 Subject: [PATCH 41/70] Address review feedback on Hashtable - Split D1Tests and D2Tests into HashtableD1Test and HashtableD2Test; extract shared test entry classes into HashtableTestEntries. - Reduce visibility of LongHashingUtils.hash(int...) chaining overloads to package-private; they are internal building blocks. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../datadog/trace/util/LongHashingUtils.java | 8 +- .../datadog/trace/util/HashtableD1Test.java | 165 ++++++++++ .../datadog/trace/util/HashtableD2Test.java | 76 +++++ .../datadog/trace/util/HashtableTest.java | 296 +----------------- .../trace/util/HashtableTestEntries.java | 54 ++++ 5 files changed, 305 insertions(+), 294 deletions(-) create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java create mode 100644 internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index c14b498cc9c..9d1257a3f20 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -48,7 +48,7 @@ public static final long hash(Object obj0, Object obj1) { return hash(intHash(obj0), intHash(obj1)); } - public static final long hash(int hash0, int hash1) { + static final long hash(int hash0, int hash1) { return 31L * hash0 + hash1; } @@ -60,7 +60,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2) { return hash(intHash(obj0), intHash(obj1), intHash(obj2)); } - public static final long hash(int hash0, int hash1, int hash2) { + static final long hash(int hash0, int hash1, int hash2) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. @@ -71,7 +71,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3 return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3)); } - public static final long hash(int hash0, int hash1, int hash2, int hash3) { + static final long hash(int hash0, int hash1, int hash2, int hash3) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. @@ -82,7 +82,7 @@ public static final long hash(Object obj0, Object obj1, Object obj2, Object obj3 return hash(intHash(obj0), intHash(obj1), intHash(obj2), intHash(obj3), intHash(obj4)); } - public static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { + static final long hash(int hash0, int hash1, int hash2, int hash3, int hash4) { // DQH - Micro-optimizing, 31L * 31L will constant fold // Since there are multiple execution ports for load & store, // this will make good use of the core. diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java new file mode 100644 index 00000000000..10d8ad41976 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -0,0 +1,165 @@ +package datadog.trace.util; + +import static datadog.trace.util.HashtableTestEntries.CollidingKey; +import static datadog.trace.util.HashtableTestEntries.CollidingKeyEntry; +import static datadog.trace.util.HashtableTestEntries.StringIntEntry; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class HashtableD1Test { + + @Test + void emptyTableLookupReturnsNull() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get("missing")); + assertEquals(0, table.size()); + } + + @Test + void insertedEntryIsRetrievable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry e = new StringIntEntry("foo", 1); + table.insert(e); + assertEquals(1, table.size()); + assertSame(e, table.get("foo")); + } + + @Test + void multipleInsertsRetrievableSeparately() { + Hashtable.D1 table = new Hashtable.D1<>(16); + StringIntEntry a = new StringIntEntry("alpha", 1); + StringIntEntry b = new StringIntEntry("beta", 2); + StringIntEntry c = new StringIntEntry("gamma", 3); + table.insert(a); + table.insert(b); + table.insert(c); + assertEquals(3, table.size()); + assertSame(a, table.get("alpha")); + assertSame(b, table.get("beta")); + assertSame(c, table.get("gamma")); + } + + @Test + void inPlaceMutationVisibleViaSubsequentGet() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("counter", 0)); + for (int i = 0; i < 10; i++) { + StringIntEntry e = table.get("counter"); + e.value++; + } + assertEquals(10, table.get("counter").value); + } + + @Test + void removeUnlinksEntryAndDecrementsSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + assertEquals(2, table.size()); + + StringIntEntry removed = table.remove("a"); + assertNotNull(removed); + assertEquals("a", removed.key); + assertEquals(1, table.size()); + assertNull(table.get("a")); + assertNotNull(table.get("b")); + } + + @Test + void removeNonexistentReturnsNullAndDoesNotChangeSize() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + assertNull(table.remove("nope")); + assertEquals(1, table.size()); + } + + @Test + void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry first = new StringIntEntry("k", 1); + assertNull(table.insertOrReplace(first), "fresh insert returns null"); + assertEquals(1, table.size()); + + StringIntEntry second = new StringIntEntry("k", 2); + assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); + assertEquals(1, table.size()); + assertSame(second, table.get("k"), "new entry visible after replace"); + } + + @Test + void clearEmptiesTheTable() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.clear(); + assertEquals(0, table.size()); + assertNull(table.get("a")); + // Reinsertion works after clear + table.insert(new StringIntEntry("a", 99)); + assertEquals(99, table.get("a").value); + } + + @Test + void forEachVisitsEveryInsertedEntry() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + Map seen = new HashMap<>(); + table.forEach(e -> seen.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(1, seen.get("a")); + assertEquals(2, seen.get("b")); + assertEquals(3, seen.get("c")); + } + + @Test + void nullKeyIsPermittedAndDistinctFromAbsent() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertNull(table.get(null)); + StringIntEntry nullKeyed = new StringIntEntry(null, 7); + table.insert(nullKeyed); + assertSame(nullKeyed, table.get(null)); + assertEquals(1, table.size()); + assertSame(nullKeyed, table.remove(null)); + assertEquals(0, table.size()); + } + + @Test + void hashCollisionsResolveByEquality() { + // Force two distinct keys with the same hashCode -- the chain must still distinguish them + // via matches(). + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); + CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); + table.insert(e1); + table.insert(e2); + assertEquals(2, table.size()); + assertSame(e1, table.get(k1)); + assertSame(e2, table.get(k2)); + } + + @Test + void hashCollisionsThenRemoveLeavesOtherIntact() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + table.remove(k2); + assertEquals(2, table.size()); + assertNotNull(table.get(k1)); + assertNull(table.get(k2)); + assertNotNull(table.get(k3)); + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java new file mode 100644 index 00000000000..98c54b71c2c --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -0,0 +1,76 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class HashtableD2Test { + + @Test + void pairKeysParticipateInIdentity() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + PairEntry bb = new PairEntry("b", 1, 300); + table.insert(ab); + table.insert(ac); + table.insert(bb); + assertEquals(3, table.size()); + assertSame(ab, table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + assertSame(bb, table.get("b", 1)); + assertNull(table.get("a", 3)); + } + + @Test + void removePairUnlinks() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry ab = new PairEntry("a", 1, 100); + PairEntry ac = new PairEntry("a", 2, 200); + table.insert(ab); + table.insert(ac); + assertSame(ab, table.remove("a", 1)); + assertEquals(1, table.size()); + assertNull(table.get("a", 1)); + assertSame(ac, table.get("a", 2)); + } + + @Test + void insertOrReplaceMatchesOnBothKeys() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry first = new PairEntry("k", 7, 1); + assertNull(table.insertOrReplace(first)); + PairEntry second = new PairEntry("k", 7, 2); + assertSame(first, table.insertOrReplace(second)); + // Different second-key: should insert new, not replace + PairEntry third = new PairEntry("k", 8, 3); + assertNull(table.insertOrReplace(third)); + assertEquals(2, table.size()); + } + + @Test + void forEachVisitsBothPairs() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + + private static final class PairEntry extends Hashtable.D2.Entry { + int value; + + PairEntry(String key1, Integer key2, int value) { + super(key1, key2); + this.value = value; + } + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index b11a33a4322..553db03495b 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -1,244 +1,24 @@ package datadog.trace.util; +import static datadog.trace.util.HashtableTestEntries.CollidingKey; +import static datadog.trace.util.HashtableTestEntries.CollidingKeyEntry; +import static datadog.trace.util.HashtableTestEntries.StringIntEntry; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.util.Hashtable.BucketIterator; import datadog.trace.util.Hashtable.MutatingBucketIterator; import datadog.trace.util.Hashtable.Support; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.NoSuchElementException; -import java.util.Set; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class HashtableTest { - // ============ D1 ============ - - @Nested - class D1Tests { - - @Test - void emptyTableLookupReturnsNull() { - Hashtable.D1 table = new Hashtable.D1<>(8); - assertNull(table.get("missing")); - assertEquals(0, table.size()); - } - - @Test - void insertedEntryIsRetrievable() { - Hashtable.D1 table = new Hashtable.D1<>(8); - StringIntEntry e = new StringIntEntry("foo", 1); - table.insert(e); - assertEquals(1, table.size()); - assertSame(e, table.get("foo")); - } - - @Test - void multipleInsertsRetrievableSeparately() { - Hashtable.D1 table = new Hashtable.D1<>(16); - StringIntEntry a = new StringIntEntry("alpha", 1); - StringIntEntry b = new StringIntEntry("beta", 2); - StringIntEntry c = new StringIntEntry("gamma", 3); - table.insert(a); - table.insert(b); - table.insert(c); - assertEquals(3, table.size()); - assertSame(a, table.get("alpha")); - assertSame(b, table.get("beta")); - assertSame(c, table.get("gamma")); - } - - @Test - void inPlaceMutationVisibleViaSubsequentGet() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("counter", 0)); - for (int i = 0; i < 10; i++) { - StringIntEntry e = table.get("counter"); - e.value++; - } - assertEquals(10, table.get("counter").value); - } - - @Test - void removeUnlinksEntryAndDecrementsSize() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - assertEquals(2, table.size()); - - StringIntEntry removed = table.remove("a"); - assertNotNull(removed); - assertEquals("a", removed.key); - assertEquals(1, table.size()); - assertNull(table.get("a")); - assertNotNull(table.get("b")); - } - - @Test - void removeNonexistentReturnsNullAndDoesNotChangeSize() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - assertNull(table.remove("nope")); - assertEquals(1, table.size()); - } - - @Test - void insertOrReplaceReturnsPriorEntryOrNullOnInsert() { - Hashtable.D1 table = new Hashtable.D1<>(8); - StringIntEntry first = new StringIntEntry("k", 1); - assertNull(table.insertOrReplace(first), "fresh insert returns null"); - assertEquals(1, table.size()); - - StringIntEntry second = new StringIntEntry("k", 2); - assertSame(first, table.insertOrReplace(second), "replace returns the prior entry"); - assertEquals(1, table.size()); - assertSame(second, table.get("k"), "new entry visible after replace"); - } - - @Test - void clearEmptiesTheTable() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - table.clear(); - assertEquals(0, table.size()); - assertNull(table.get("a")); - // Reinsertion works after clear - table.insert(new StringIntEntry("a", 99)); - assertEquals(99, table.get("a").value); - } - - @Test - void forEachVisitsEveryInsertedEntry() { - Hashtable.D1 table = new Hashtable.D1<>(8); - table.insert(new StringIntEntry("a", 1)); - table.insert(new StringIntEntry("b", 2)); - table.insert(new StringIntEntry("c", 3)); - Map seen = new HashMap<>(); - table.forEach(e -> seen.put(e.key, e.value)); - assertEquals(3, seen.size()); - assertEquals(1, seen.get("a")); - assertEquals(2, seen.get("b")); - assertEquals(3, seen.get("c")); - } - - @Test - void nullKeyIsPermittedAndDistinctFromAbsent() { - Hashtable.D1 table = new Hashtable.D1<>(8); - assertNull(table.get(null)); - StringIntEntry nullKeyed = new StringIntEntry(null, 7); - table.insert(nullKeyed); - assertSame(nullKeyed, table.get(null)); - assertEquals(1, table.size()); - assertSame(nullKeyed, table.remove(null)); - assertEquals(0, table.size()); - } - - @Test - void hashCollisionsResolveByEquality() { - // Force two distinct keys with the same hashCode -- the chain must still distinguish them - // via matches(). - Hashtable.D1 table = new Hashtable.D1<>(4); - CollidingKey k1 = new CollidingKey("first", 17); - CollidingKey k2 = new CollidingKey("second", 17); - CollidingKeyEntry e1 = new CollidingKeyEntry(k1, 100); - CollidingKeyEntry e2 = new CollidingKeyEntry(k2, 200); - table.insert(e1); - table.insert(e2); - assertEquals(2, table.size()); - assertSame(e1, table.get(k1)); - assertSame(e2, table.get(k2)); - } - - @Test - void hashCollisionsThenRemoveLeavesOtherIntact() { - Hashtable.D1 table = new Hashtable.D1<>(4); - CollidingKey k1 = new CollidingKey("first", 17); - CollidingKey k2 = new CollidingKey("second", 17); - CollidingKey k3 = new CollidingKey("third", 17); - table.insert(new CollidingKeyEntry(k1, 1)); - table.insert(new CollidingKeyEntry(k2, 2)); - table.insert(new CollidingKeyEntry(k3, 3)); - table.remove(k2); - assertEquals(2, table.size()); - assertNotNull(table.get(k1)); - assertNull(table.get(k2)); - assertNotNull(table.get(k3)); - } - } - - // ============ D2 ============ - - @Nested - class D2Tests { - - @Test - void pairKeysParticipateInIdentity() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry ab = new PairEntry("a", 1, 100); - PairEntry ac = new PairEntry("a", 2, 200); - PairEntry bb = new PairEntry("b", 1, 300); - table.insert(ab); - table.insert(ac); - table.insert(bb); - assertEquals(3, table.size()); - assertSame(ab, table.get("a", 1)); - assertSame(ac, table.get("a", 2)); - assertSame(bb, table.get("b", 1)); - assertNull(table.get("a", 3)); - } - - @Test - void removePairUnlinks() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry ab = new PairEntry("a", 1, 100); - PairEntry ac = new PairEntry("a", 2, 200); - table.insert(ab); - table.insert(ac); - assertSame(ab, table.remove("a", 1)); - assertEquals(1, table.size()); - assertNull(table.get("a", 1)); - assertSame(ac, table.get("a", 2)); - } - - @Test - void insertOrReplaceMatchesOnBothKeys() { - Hashtable.D2 table = new Hashtable.D2<>(8); - PairEntry first = new PairEntry("k", 7, 1); - assertNull(table.insertOrReplace(first)); - PairEntry second = new PairEntry("k", 7, 2); - assertSame(first, table.insertOrReplace(second)); - // Different second-key: should insert new, not replace - PairEntry third = new PairEntry("k", 8, 3); - assertNull(table.insertOrReplace(third)); - assertEquals(2, table.size()); - } - - @Test - void forEachVisitsBothPairs() { - Hashtable.D2 table = new Hashtable.D2<>(8); - table.insert(new PairEntry("a", 1, 100)); - table.insert(new PairEntry("b", 2, 200)); - Set seen = new HashSet<>(); - table.forEach(e -> seen.add(e.key1 + ":" + e.key2)); - assertEquals(2, seen.size()); - assertTrue(seen.contains("a:1")); - assertTrue(seen.contains("b:2")); - } - } - // ============ Support ============ @Nested @@ -374,7 +154,9 @@ void removeFromHeadOfChainUnlinks() { // of the three keys are still retrievable.) int found = 0; for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { - if (table.get(k) != null) found++; + if (table.get(k) != null) { + found++; + } } assertEquals(2, found); } @@ -411,8 +193,6 @@ void removeWithoutNextThrows() { } } - // ============ test helpers ============ - /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { try { @@ -423,68 +203,4 @@ private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { throw new RuntimeException(e); } } - - /** Sort comparator used by tests that want deterministic visit order. */ - @SuppressWarnings("unused") - private static final Comparator BY_KEY = Comparator.comparing(e -> e.key); - - private static final class StringIntEntry extends Hashtable.D1.Entry { - int value; - - StringIntEntry(String key, int value) { - super(key); - this.value = value; - } - } - - /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ - private static final class CollidingKey { - final String label; - final int hash; - - CollidingKey(String label, int hash) { - this.label = label; - this.hash = hash; - } - - @Override - public int hashCode() { - return hash; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CollidingKey)) return false; - CollidingKey that = (CollidingKey) o; - return hash == that.hash && label.equals(that.label); - } - - @Override - public String toString() { - return "CollidingKey(" + label + ", " + hash + ")"; - } - } - - private static final class CollidingKeyEntry extends Hashtable.D1.Entry { - int value; - - CollidingKeyEntry(CollidingKey key, int value) { - super(key); - this.value = value; - } - } - - private static final class PairEntry extends Hashtable.D2.Entry { - int value; - - PairEntry(String key1, Integer key2, int value) { - super(key1, key2); - this.value = value; - } - } - - // Imports kept narrow but List is referenced in test helpers below; this keeps the import warning - // quiet. - @SuppressWarnings("unused") - private static final List UNUSED = new ArrayList<>(); } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java b/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java new file mode 100644 index 00000000000..e657028ee8b --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTestEntries.java @@ -0,0 +1,54 @@ +package datadog.trace.util; + +/** Shared test entry types for {@link HashtableTest}, {@link HashtableD1Test}, and friends. */ +final class HashtableTestEntries { + private HashtableTestEntries() {} + + static final class StringIntEntry extends Hashtable.D1.Entry { + int value; + + StringIntEntry(String key, int value) { + super(key); + this.value = value; + } + } + + /** Key whose hashCode is fully controllable, to force chain collisions deterministically. */ + static final class CollidingKey { + final String label; + final int hash; + + CollidingKey(String label, int hash) { + this.label = label; + this.hash = hash; + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CollidingKey)) { + return false; + } + CollidingKey that = (CollidingKey) o; + return hash == that.hash && label.equals(that.label); + } + + @Override + public String toString() { + return "CollidingKey(" + label + ", " + hash + ")"; + } + } + + static final class CollidingKeyEntry extends Hashtable.D1.Entry { + int value; + + CollidingKeyEntry(CollidingKey key, int value) { + super(key); + this.value = value; + } + } +} From 8b8b0887586195bf4afbb172ebee2830d02a0090 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 11:32:57 -0400 Subject: [PATCH 42/70] Drop reflection in iterator tests via package-private D1.buckets The iterator tests need a populated Hashtable.Entry[] to drive Support.bucketIterator / mutatingBucketIterator. Relaxing D1.buckets from private to package-private lets the same-package tests read it directly, removing the reflection helper. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 2 +- .../datadog/trace/util/HashtableTest.java | 21 +++++-------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 39dfaf6c7a4..e527ae45fcc 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -100,7 +100,7 @@ public static long hash(Object key) { } } - private final Hashtable.Entry[] buckets; + final Hashtable.Entry[] buckets; private int size; public D1(int capacity) { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 553db03495b..f78aec1c00f 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -101,7 +101,7 @@ void walksOnlyMatchingHash() { table.insert(new CollidingKeyEntry(k2, 2)); table.insert(new CollidingKeyEntry(k3, 3)); // All three share the same hash (17), so a bucket iterator over hash=17 yields all three. - BucketIterator it = Support.bucketIterator(extractBuckets(table), 17L); + BucketIterator it = Support.bucketIterator(table.buckets, 17L); int count = 0; while (it.hasNext()) { assertNotNull(it.next()); @@ -115,7 +115,7 @@ void exhaustedIteratorThrowsNoSuchElement() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("only", 1)); long h = Hashtable.D1.Entry.hash("only"); - BucketIterator it = Support.bucketIterator(extractBuckets(table), h); + BucketIterator it = Support.bucketIterator(table.buckets, h); it.next(); assertFalse(it.hasNext()); assertThrows(NoSuchElementException.class, it::next); @@ -139,7 +139,7 @@ void removeFromHeadOfChainUnlinks() { table.insert(new CollidingKeyEntry(k3, 3)); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), 17L); + Support.mutatingBucketIterator(table.buckets, 17L); it.next(); // first match (head of chain in insertion-reverse order) it.remove(); // Two should remain @@ -172,7 +172,7 @@ void replaceSwapsEntryAndPreservesChain() { table.insert(e2); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), 17L); + Support.mutatingBucketIterator(table.buckets, 17L); CollidingKeyEntry first = it.next(); CollidingKeyEntry replacement = new CollidingKeyEntry(first.key, 999); it.replace(replacement); @@ -188,19 +188,8 @@ void removeWithoutNextThrows() { Hashtable.D1 table = new Hashtable.D1<>(4); table.insert(new StringIntEntry("a", 1)); MutatingBucketIterator it = - Support.mutatingBucketIterator(extractBuckets(table), Hashtable.D1.Entry.hash("a")); + Support.mutatingBucketIterator(table.buckets, Hashtable.D1.Entry.hash("a")); assertThrows(IllegalStateException.class, it::remove); } } - - /** Reach into a D1 table's bucket array via reflection -- only needed by iterator tests. */ - private static Hashtable.Entry[] extractBuckets(Hashtable.D1 table) { - try { - java.lang.reflect.Field f = Hashtable.D1.class.getDeclaredField("buckets"); - f.setAccessible(true); - return (Hashtable.Entry[]) f.get(table); - } catch (Exception e) { - throw new RuntimeException(e); - } - } } From 0fde7cd142638afaeebf51023f47297d45889073 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:49:03 -0400 Subject: [PATCH 43/70] Add context-passing forEach to Hashtable.D1 and D2 Mirrors the TagMap pattern: pairs the existing forEach(Consumer) with a forEach(T context, BiConsumer) overload so callers can hand side-band state to a non-capturing lambda and avoid the fresh-Consumer-per-call allocation. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 31 +++++++++++++++++++ .../datadog/trace/util/HashtableD1Test.java | 22 +++++++++++++ .../datadog/trace/util/HashtableD2Test.java | 12 +++++++ 3 files changed, 65 insertions(+) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index e527ae45fcc..f4c26f88d99 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -4,6 +4,7 @@ import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.function.BiConsumer; import java.util.function.Consumer; /** @@ -193,6 +194,21 @@ public void forEach(Consumer consumer) { } } } + + /** + * Context-passing forEach. Useful for callers that want to avoid a capturing-lambda allocation + * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever + * side-band state it needs as {@code context}. + */ + @SuppressWarnings("unchecked") + public void forEach(T context, BiConsumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** @@ -340,6 +356,21 @@ public void forEach(Consumer consumer) { } } } + + /** + * Context-passing forEach. Useful for callers that want to avoid a capturing-lambda allocation + * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever + * side-band state it needs as {@code context}. + */ + @SuppressWarnings("unchecked") + public void forEach(T context, BiConsumer consumer) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = 0; i < thisBuckets.length; i++) { + for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java index 10d8ad41976..11928bb4d5b 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -119,6 +119,28 @@ void forEachVisitsEveryInsertedEntry() { assertEquals(3, seen.get("c")); } + @Test + void forEachWithContextPassesContextToConsumer() { + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 10)); + table.insert(new StringIntEntry("b", 20)); + table.insert(new StringIntEntry("c", 30)); + Map seen = new HashMap<>(); + table.forEach(seen, (ctx, e) -> ctx.put(e.key, e.value)); + assertEquals(3, seen.size()); + assertEquals(10, seen.get("a")); + assertEquals(20, seen.get("b")); + assertEquals(30, seen.get("c")); + } + + @Test + void forEachWithContextOnEmptyTableDoesNothing() { + Hashtable.D1 table = new Hashtable.D1<>(8); + Map seen = new HashMap<>(); + table.forEach(seen, (ctx, e) -> ctx.put(e.key, e.value)); + assertEquals(0, seen.size()); + } + @Test void nullKeyIsPermittedAndDistinctFromAbsent() { Hashtable.D1 table = new Hashtable.D1<>(8); diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java index 98c54b71c2c..59339fcd89e 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -65,6 +65,18 @@ void forEachVisitsBothPairs() { assertTrue(seen.contains("b:2")); } + @Test + void forEachWithContextPassesContextToConsumer() { + Hashtable.D2 table = new Hashtable.D2<>(8); + table.insert(new PairEntry("a", 1, 100)); + table.insert(new PairEntry("b", 2, 200)); + Set seen = new HashSet<>(); + table.forEach(seen, (ctx, e) -> ctx.add(e.key1 + ":" + e.key2)); + assertEquals(2, seen.size()); + assertTrue(seen.contains("a:1")); + assertTrue(seen.contains("b:2")); + } + private static final class PairEntry extends Hashtable.D2.Entry { int value; From 6d6c2e05772b10542668888d92e682c996135c32 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 13:58:43 -0400 Subject: [PATCH 44/70] Move forEach loop body to Support helper Factors the unchecked (TEntry) cast out of D1.forEach / D2.forEach (and the BiConsumer variants) into Support.forEach(buckets, ...). The cast now lives in one place, mirroring how Entry.next() handles it, and the D1/D2 methods become one-liners. Downstream higher-arity tables built on Support gain the same helper. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 64 +++++++++++-------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index f4c26f88d99..137118fc111 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -185,14 +185,8 @@ public void clear() { this.size = 0; } - @SuppressWarnings("unchecked") public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } + Support.forEach(this.buckets, consumer); } /** @@ -200,14 +194,8 @@ public void forEach(Consumer consumer) { * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever * side-band state it needs as {@code context}. */ - @SuppressWarnings("unchecked") public void forEach(T context, BiConsumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept(context, (TEntry) e); - } - } + Support.forEach(this.buckets, context, consumer); } } @@ -347,14 +335,8 @@ public void clear() { this.size = 0; } - @SuppressWarnings("unchecked") public void forEach(Consumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept((TEntry) e); - } - } + Support.forEach(this.buckets, consumer); } /** @@ -362,14 +344,8 @@ public void forEach(Consumer consumer) { * -- pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus whatever * side-band state it needs as {@code context}. */ - @SuppressWarnings("unchecked") public void forEach(T context, BiConsumer consumer) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = 0; i < thisBuckets.length; i++) { - for (Hashtable.Entry e = thisBuckets[i]; e != null; e = e.next()) { - consumer.accept(context, (TEntry) e); - } - } + Support.forEach(this.buckets, context, consumer); } } @@ -388,6 +364,8 @@ public void forEach(T context, BiConsumer consume * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / * {@code replace}. + *
    • Iterate every entry with {@link #forEach(Hashtable.Entry[], Consumer)} or its + * context-passing sibling. *
    • Clear with {@link #clear(Hashtable.Entry[])}. * * @@ -436,6 +414,36 @@ MutatingBucketIterator mutatingBucketIterator( public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + + /** + * Walks every entry in {@code buckets} and invokes {@code consumer} on it. The unchecked cast + * to {@code TEntry} lives here (mirroring {@link Entry#next()}) so callers don't have to + * sprinkle it across their own forEach loops. + */ + @SuppressWarnings("unchecked") + public static final void forEach( + Hashtable.Entry[] buckets, Consumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + consumer.accept((TEntry) e); + } + } + } + + /** + * Context-passing variant of {@link #forEach(Hashtable.Entry[], Consumer)}. Pair a + * non-capturing {@link BiConsumer} (typically a {@code static final}) with side-band state + * passed as {@code context} to avoid a fresh-Consumer allocation each call. + */ + @SuppressWarnings("unchecked") + public static final void forEach( + Hashtable.Entry[] buckets, T context, BiConsumer consumer) { + for (int i = 0; i < buckets.length; i++) { + for (Hashtable.Entry e = buckets[i]; e != null; e = e.next()) { + consumer.accept(context, (TEntry) e); + } + } + } } /** From 268de2b7d9cdc76eefb79b90ab39857d2487072e Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 14:32:29 -0400 Subject: [PATCH 45/70] Move bucket-head cast to Support.bucket helper Adds Support.bucket(buckets, keyHash) which returns the bucket head already cast to the caller's concrete entry type. D1.get and D2.get now drop the raw-Entry intermediate variable and walk the chain via Entry.next() directly. The unchecked cast lives in one place, consistent with Entry.next() and Support.forEach. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 137118fc111..4945aed5a0f 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -113,16 +113,11 @@ public int size() { return this.size; } - @SuppressWarnings("unchecked") public TEntry get(K key) { long keyHash = D1.Entry.hash(key); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; - e != null; - e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key)) return te; + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key)) { + return te; } } return null; @@ -263,16 +258,11 @@ public int size() { return this.size; } - @SuppressWarnings("unchecked") public TEntry get(K1 key1, K2 key2) { long keyHash = D2.Entry.hash(key1, key2); - Hashtable.Entry[] thisBuckets = this.buckets; - for (Hashtable.Entry e = thisBuckets[Support.bucketIndex(thisBuckets, keyHash)]; - e != null; - e = e.next) { - if (e.keyHash == keyHash) { - TEntry te = (TEntry) e; - if (te.matches(key1, key2)) return te; + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key1, key2)) { + return te; } } return null; @@ -415,6 +405,17 @@ public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + /** + * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's + * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site + * doesn't need to thread a raw {@link Entry} variable through. + */ + @SuppressWarnings("unchecked") + public static final TEntry bucket( + Hashtable.Entry[] buckets, long keyHash) { + return (TEntry) buckets[bucketIndex(buckets, keyHash)]; + } + /** * Walks every entry in {@code buckets} and invokes {@code consumer} on it. The unchecked cast * to {@code TEntry} lives here (mirroring {@link Entry#next()}) so callers don't have to From 93813b9515e5fded85423ca7ff5da7b83629767c Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 15:28:50 -0400 Subject: [PATCH 46/70] Drop d1_/d2_ prefix from per-table benchmark methods Holdover from when both lived in a shared HashtableBenchmark; redundant now that each lives in its own class. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/util/HashtableD1Benchmark.java | 26 +++++++++---------- .../trace/util/HashtableD2Benchmark.java | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java index 16b95e089d5..f8ba7177e88 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD1Benchmark.java @@ -44,15 +44,15 @@ * Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableD1Benchmark.d1_add_hashMap thrpt 6 187.883 ± 189.858 ops/us - * HashtableD1Benchmark.d1_add_hashtable thrpt 6 198.710 ± 273.035 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD1Benchmark.add_hashMap thrpt 6 187.883 ± 189.858 ops/us + * HashtableD1Benchmark.add_hashtable thrpt 6 198.710 ± 273.035 ops/us * - * HashtableD1Benchmark.d1_update_hashMap thrpt 6 127.392 ± 87.482 ops/us - * HashtableD1Benchmark.d1_update_hashtable thrpt 6 1810.244 ± 44.645 ops/us + * HashtableD1Benchmark.update_hashMap thrpt 6 127.392 ± 87.482 ops/us + * HashtableD1Benchmark.update_hashtable thrpt 6 1810.244 ± 44.645 ops/us * - * HashtableD1Benchmark.d1_iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us - * HashtableD1Benchmark.d1_iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us + * HashtableD1Benchmark.iterate_hashMap thrpt 6 20.043 ± 0.752 ops/us + * HashtableD1Benchmark.iterate_hashtable thrpt 6 22.208 ± 0.956 ops/us * */ @Fork(2) @@ -122,7 +122,7 @@ String nextKey() { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d1_add_hashtable(D1State s) { + public void add_hashtable(D1State s) { Hashtable.D1 t = s.table; String[] keys = s.keys; t.clear(); @@ -133,7 +133,7 @@ public void d1_add_hashtable(D1State s) { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d1_add_hashMap(D1State s) { + public void add_hashMap(D1State s) { HashMap m = s.hashMap; String[] keys = s.keys; m.clear(); @@ -143,24 +143,24 @@ public void d1_add_hashMap(D1State s) { } @Benchmark - public long d1_update_hashtable(D1State s) { + public long update_hashtable(D1State s) { D1Counter e = s.table.get(s.nextKey()); return ++e.count; } @Benchmark - public Long d1_update_hashMap(D1State s) { + public Long update_hashMap(D1State s) { return s.hashMap.merge(s.nextKey(), 1L, Long::sum); } @Benchmark - public void d1_iterate_hashtable(D1State s, Blackhole bh) { + public void iterate_hashtable(D1State s, Blackhole bh) { s.consumer.bh = bh; s.table.forEach(s.consumer); } @Benchmark - public void d1_iterate_hashMap(D1State s, Blackhole bh) { + public void iterate_hashMap(D1State s, Blackhole bh) { for (Map.Entry entry : s.hashMap.entrySet()) { bh.consume(entry.getKey()); bh.consume(entry.getValue()); diff --git a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java index 5fd64ed9a75..6f46a702005 100644 --- a/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/util/HashtableD2Benchmark.java @@ -48,15 +48,15 @@ * {@code Key2} allocation). Iterate is essentially a wash — both are bucket walks. * MacBook M1 8 threads (Java 8) * - * Benchmark Mode Cnt Score Error Units - * HashtableD2Benchmark.d2_add_hashMap thrpt 6 77.082 ± 72.278 ops/us - * HashtableD2Benchmark.d2_add_hashtable thrpt 6 216.813 ± 413.236 ops/us + * Benchmark Mode Cnt Score Error Units + * HashtableD2Benchmark.add_hashMap thrpt 6 77.082 ± 72.278 ops/us + * HashtableD2Benchmark.add_hashtable thrpt 6 216.813 ± 413.236 ops/us * - * HashtableD2Benchmark.d2_update_hashMap thrpt 6 56.077 ± 23.716 ops/us - * HashtableD2Benchmark.d2_update_hashtable thrpt 6 1445.868 ± 157.705 ops/us + * HashtableD2Benchmark.update_hashMap thrpt 6 56.077 ± 23.716 ops/us + * HashtableD2Benchmark.update_hashtable thrpt 6 1445.868 ± 157.705 ops/us * - * HashtableD2Benchmark.d2_iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us - * HashtableD2Benchmark.d2_iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us + * HashtableD2Benchmark.iterate_hashMap thrpt 6 19.508 ± 0.760 ops/us + * HashtableD2Benchmark.iterate_hashtable thrpt 6 16.968 ± 0.371 ops/us * */ @Fork(2) @@ -158,7 +158,7 @@ int nextIndex() { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d2_add_hashtable(D2State s) { + public void add_hashtable(D2State s) { Hashtable.D2 t = s.table; String[] k1s = s.k1s; Integer[] k2s = s.k2s; @@ -170,7 +170,7 @@ public void d2_add_hashtable(D2State s) { @Benchmark @OperationsPerInvocation(N_KEYS) - public void d2_add_hashMap(D2State s) { + public void add_hashMap(D2State s) { HashMap m = s.hashMap; String[] k1s = s.k1s; Integer[] k2s = s.k2s; @@ -181,26 +181,26 @@ public void d2_add_hashMap(D2State s) { } @Benchmark - public long d2_update_hashtable(D2State s) { + public long update_hashtable(D2State s) { int i = s.nextIndex(); D2Counter e = s.table.get(s.k1s[i], s.k2s[i]); return ++e.count; } @Benchmark - public Long d2_update_hashMap(D2State s) { + public Long update_hashMap(D2State s) { int i = s.nextIndex(); return s.hashMap.merge(new Key2(s.k1s[i], s.k2s[i]), 1L, Long::sum); } @Benchmark - public void d2_iterate_hashtable(D2State s, Blackhole bh) { + public void iterate_hashtable(D2State s, Blackhole bh) { s.consumer.bh = bh; s.table.forEach(s.consumer); } @Benchmark - public void d2_iterate_hashMap(D2State s, Blackhole bh) { + public void iterate_hashMap(D2State s, Blackhole bh) { for (Map.Entry entry : s.hashMap.entrySet()) { bh.consume(entry.getKey()); bh.consume(entry.getValue()); From 11a58bff54b35430cba602650b0a1e2147f0075b Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 15:58:55 -0400 Subject: [PATCH 47/70] Add Hashtable.Support helpers: MAX_RATIO, insertHeadEntry, MutatingTableIterator Three consumer-facing helpers that callers building higher-arity tables on top of Hashtable.Support kept open-coding: - MAX_RATIO_NUMERATOR / _DENOMINATOR: the 4/3 multiplier for sizing a bucket array from a target working-set under a 75% load factor. - insertHeadEntry(buckets, bucketIndex, entry): the (setNext + array-store) pair for splicing a new entry at the head of a bucket chain. - MutatingTableIterator + Support.mutatingTableIterator(buckets): walks every entry in the table (not filtered by hash) with remove() support, for sweeps like eviction and expunge that aren't keyed to a specific hash. Sibling of MutatingBucketIterator. Tests cover the table-wide iterator at head-of-bucket and mid-chain removal, empty buckets between live entries, exhaustion, and remove-without-next. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 148 ++++++++++++++++- .../datadog/trace/util/HashtableTest.java | 153 ++++++++++++++++++ 2 files changed, 300 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 4945aed5a0f..bada7a8b98b 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -354,8 +354,11 @@ public void forEach(T context, BiConsumer consume * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / * {@code replace}. + *
    • Use {@link #insertHeadEntry(Hashtable.Entry[], int, Hashtable.Entry)} to splice a new + * entry as the head of a bucket chain. *
    • Iterate every entry with {@link #forEach(Hashtable.Entry[], Consumer)} or its - * context-passing sibling. + * context-passing sibling. For full-table sweeps with {@code remove}, use {@link + * #mutatingTableIterator(Hashtable.Entry[])}. *
    • Clear with {@link #clear(Hashtable.Entry[])}. * * @@ -372,6 +375,17 @@ public static final Hashtable.Entry[] create(int capacity) { static final int MAX_CAPACITY = 1 << 30; + /** + * Numerator/denominator pair for the inverse of a 75% load factor. Callers that size their + * bucket array from a target working-set size {@code n} should pass {@code n * + * MAX_RATIO_NUMERATOR / MAX_RATIO_DENOMINATOR} to {@link #create(int)} (or {@link + * #sizeFor(int)}) to leave ~25% headroom in the array. Kept as separate ints so callers can use + * integer arithmetic. + */ + public static final int MAX_RATIO_NUMERATOR = 4; + + public static final int MAX_RATIO_DENOMINATOR = 3; + static final int sizeFor(int requestedCapacity) { if (requestedCapacity < 0) { throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); @@ -401,10 +415,29 @@ MutatingBucketIterator mutatingBucketIterator( return new MutatingBucketIterator(buckets, keyHash); } + /** + * Returns a {@link MutatingTableIterator} over every entry in {@code buckets}. Useful for + * sweeps -- eviction, expunge -- that aren't keyed to a specific hash. + */ + public static final + MutatingTableIterator mutatingTableIterator(Hashtable.Entry[] buckets) { + return new MutatingTableIterator(buckets); + } + public static final int bucketIndex(Object[] buckets, long keyHash) { return (int) (keyHash & buckets.length - 1); } + /** + * Splices {@code entry} in as the new head of the chain at {@code bucketIndex}. Caller is + * responsible for size accounting -- this method only touches the chain pointers. + */ + public static final void insertHeadEntry( + Hashtable.Entry[] buckets, int bucketIndex, Hashtable.Entry entry) { + entry.setNext(buckets[bucketIndex]); + buckets[bucketIndex] = entry; + } + /** * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site @@ -607,4 +640,117 @@ void setPrevNext(Hashtable.Entry nextEntry) { } } } + + /** + * Mutating iterator over every entry in a bucket array, regardless of hash. Supports {@link + * #remove()} to unlink the entry last returned by {@link #next()}. + * + *

      Walks buckets in array order; within a bucket, walks the chain head-to-tail. After {@code + * remove}, iteration may continue with another {@link #next()}. + * + *

      Use this for sweeps -- eviction, expunge, full-table cleanup -- that aren't keyed to a + * specific hash. For per-bucket walks keyed to a search hash, use {@link MutatingBucketIterator}. + */ + public static final class MutatingTableIterator + implements Iterator { + private final Hashtable.Entry[] buckets; + + /** + * Index of the bucket holding {@link #nextEntry} (or holding {@link #curEntry} after remove). + */ + private int nextBucketIndex; + + /** + * Predecessor of {@link #nextEntry}, or {@code null} when {@code nextEntry} is the bucket head. + */ + private Hashtable.Entry nextPrevEntry; + + /** Next entry to be returned by {@link #next()}, or {@code null} if iteration is exhausted. */ + private Hashtable.Entry nextEntry; + + /** + * Bucket index that held the entry last returned by {@code next}; {@code -1} after {@code + * remove}. + */ + private int curBucketIndex = -1; + + /** + * Predecessor of the entry last returned by {@code next}, or {@code null} if it was the bucket + * head. + */ + private Hashtable.Entry curPrevEntry; + + /** + * Entry last returned by {@code next}; {@code null} before any call and after {@code remove}. + */ + private Hashtable.Entry curEntry; + + MutatingTableIterator(Hashtable.Entry[] buckets) { + this.buckets = buckets; + seekFromBucket(0); + } + + @Override + public boolean hasNext() { + return this.nextEntry != null; + } + + @Override + @SuppressWarnings("unchecked") + public TEntry next() { + Hashtable.Entry e = this.nextEntry; + if (e == null) throw new NoSuchElementException("no next!"); + + this.curEntry = e; + this.curPrevEntry = this.nextPrevEntry; + this.curBucketIndex = this.nextBucketIndex; + + Hashtable.Entry n = e.next(); + if (n != null) { + this.nextPrevEntry = e; + this.nextEntry = n; + } else { + // walked off the end of this bucket; pick up at the next non-empty bucket + seekFromBucket(this.nextBucketIndex + 1); + } + return (TEntry) e; + } + + @Override + public void remove() { + Hashtable.Entry oldCurEntry = this.curEntry; + if (oldCurEntry == null) throw new IllegalStateException(); + + if (this.curPrevEntry == null) { + this.buckets[this.curBucketIndex] = oldCurEntry.next(); + } else { + this.curPrevEntry.setNext(oldCurEntry.next()); + } + // If the next entry was the immediate chain successor of oldCurEntry, its predecessor is + // now what came before oldCurEntry (oldCurEntry was just unlinked). + if (this.nextPrevEntry == oldCurEntry) { + this.nextPrevEntry = this.curPrevEntry; + } + this.curEntry = null; + } + + /** + * Advance {@code nextBucketIndex} / {@code nextEntry} to the first non-empty bucket >= {@code + * from}. + */ + private void seekFromBucket(int from) { + Hashtable.Entry[] thisBuckets = this.buckets; + for (int i = from; i < thisBuckets.length; i++) { + Hashtable.Entry head = thisBuckets[i]; + if (head != null) { + this.nextBucketIndex = i; + this.nextPrevEntry = null; + this.nextEntry = head; + return; + } + } + this.nextEntry = null; + this.nextPrevEntry = null; + } + } } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index f78aec1c00f..6fbf0cc752c 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -7,13 +7,17 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.util.Hashtable.BucketIterator; import datadog.trace.util.Hashtable.MutatingBucketIterator; +import datadog.trace.util.Hashtable.MutatingTableIterator; import datadog.trace.util.Hashtable.Support; +import java.util.HashSet; import java.util.NoSuchElementException; +import java.util.Set; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -81,6 +85,32 @@ void clearNullsAllBuckets() { assertNull(b); } } + + @Test + void maxRatioConstantsExpandTargetSize() { + // 75% load factor => bucket array sized at requestedSize * 4 / 3, rounded up to power of 2. + assertEquals(4, Support.MAX_RATIO_NUMERATOR); + assertEquals(3, Support.MAX_RATIO_DENOMINATOR); + int target = 12; + int sized = target * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR; + assertEquals(16, sized); + assertEquals(16, Support.sizeFor(sized)); + } + + @Test + void insertHeadEntrySplicesAsNewHead() { + Hashtable.Entry[] buckets = Support.create(4); + StringIntEntry a = new StringIntEntry("a", 1); + StringIntEntry b = new StringIntEntry("b", 2); + Support.insertHeadEntry(buckets, 0, a); + assertSame(a, buckets[0]); + assertNull(a.next()); + + Support.insertHeadEntry(buckets, 0, b); + assertSame(b, buckets[0]); + assertSame(a, b.next()); + assertNull(a.next()); + } } // ============ BucketIterator ============ @@ -192,4 +222,127 @@ void removeWithoutNextThrows() { assertThrows(IllegalStateException.class, it::remove); } } + + // ============ MutatingTableIterator ============ + + @Nested + class MutatingTableIteratorTests { + + @Test + void walksEveryEntryAcrossBuckets() { + Hashtable.D1 table = new Hashtable.D1<>(16); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + table.insert(new StringIntEntry("c", 3)); + + Set seen = new HashSet<>(); + for (MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.hasNext(); ) { + seen.add(it.next().key); + } + assertEquals(3, seen.size()); + assertTrue(seen.contains("a")); + assertTrue(seen.contains("b")); + assertTrue(seen.contains("c")); + } + + @Test + void emptyTableIteratorIsExhausted() { + Hashtable.D1 table = new Hashtable.D1<>(8); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + assertFalse(it.hasNext()); + assertThrows(NoSuchElementException.class, it::next); + } + + @Test + void removeUnlinksBucketHead() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + + // The head of the chain is whichever was inserted last (insert prepends). + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + CollidingKeyEntry head = it.next(); + it.remove(); + + // Survivor still reachable via the table; removed one is not. + CollidingKey survivorKey = head.key.equals(k1) ? k2 : k1; + assertNotNull(table.get(survivorKey)); + assertNull(table.get(head.key)); + } + + @Test + void removeUnlinksMidChainEntry() { + Hashtable.D1 table = new Hashtable.D1<>(4); + CollidingKey k1 = new CollidingKey("first", 17); + CollidingKey k2 = new CollidingKey("second", 17); + CollidingKey k3 = new CollidingKey("third", 17); + table.insert(new CollidingKeyEntry(k1, 1)); + table.insert(new CollidingKeyEntry(k2, 2)); + table.insert(new CollidingKeyEntry(k3, 3)); + + // Walk to the second entry, remove it. + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + CollidingKeyEntry victim = it.next(); + it.remove(); + + assertNull(table.get(victim.key)); + // The remaining two keys still resolve. + int remaining = 0; + for (CollidingKey k : new CollidingKey[] {k1, k2, k3}) { + if (table.get(k) != null) { + remaining++; + } + } + assertEquals(2, remaining); + + // Iteration can continue past a remove and yield the third entry. + assertTrue(it.hasNext()); + assertNotNull(it.next()); + assertFalse(it.hasNext()); + } + + @Test + void removeSkipsOverEmptyBuckets() { + // Three distinct keys that land in different buckets (low entry count vs large bucket array + // makes empty buckets between them very likely). Verify the iterator skips empties cleanly + // after a remove. + Hashtable.D1 table = new Hashtable.D1<>(64); + table.insert(new StringIntEntry("alpha", 1)); + table.insert(new StringIntEntry("beta", 2)); + table.insert(new StringIntEntry("gamma", 3)); + + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + it.remove(); + int remaining = 0; + while (it.hasNext()) { + it.next(); + remaining++; + } + assertEquals(2, remaining); + } + + @Test + void removeWithoutNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + assertThrows(IllegalStateException.class, it::remove); + } + + @Test + void removeTwiceWithoutInterveningNextThrows() { + Hashtable.D1 table = new Hashtable.D1<>(4); + table.insert(new StringIntEntry("a", 1)); + table.insert(new StringIntEntry("b", 2)); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + it.next(); + it.remove(); + assertThrows(IllegalStateException.class, it::remove); + } + } } From 8f1828d6eb9ef199e81426dcfc2294358ed4b9bd Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:12:50 -0400 Subject: [PATCH 48/70] Swap MAX_RATIO numerator/denominator pair for a single float + scaled create() Replace Support.MAX_RATIO_NUMERATOR / _DENOMINATOR with a single float MAX_RATIO constant, and add a Support.create(int, float) overload that takes a scale factor. Callers now write Support.create(n, MAX_RATIO) instead of stitching together the int arithmetic at the call site. The scaled size is truncated (not ceiled) before going through sizeFor. sizeFor already rounds up to the next power of two, so truncation just absorbs float fuzz that would otherwise push a result like 12 * 4/3 = 16.0000005f past 16 and double the bucket array size for no reason. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 27 +++++++++++++------ .../datadog/trace/util/HashtableTest.java | 21 +++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index bada7a8b98b..9e9ecb1c61a 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -373,18 +373,29 @@ public static final Hashtable.Entry[] create(int capacity) { return new Entry[sizeFor(capacity)]; } + /** + * Variant of {@link #create(int)} that scales the requested working-set size before sizing the + * bucket array. Pair with {@link #MAX_RATIO} (or similar) to leave headroom over the working + * set for a desired load factor. + * + *

      The scaled size is truncated to {@code int} before going through {@link #sizeFor(int)}. + * Truncation rather than {@code ceil} is intentional: {@code sizeFor} rounds up to the next + * power of two anyway, so the fractional part would only matter when float fuzz pushes the + * result across a power-of-two boundary -- {@code ceil} would then double the array size for no + * reason (e.g. {@code 12 * 4/3 = 16.0...0005f -> ceil 17 -> sizeFor 32}). + */ + public static final Hashtable.Entry[] create(int requestedSize, float scale) { + return new Entry[sizeFor((int) (requestedSize * scale))]; + } + static final int MAX_CAPACITY = 1 << 30; /** - * Numerator/denominator pair for the inverse of a 75% load factor. Callers that size their - * bucket array from a target working-set size {@code n} should pass {@code n * - * MAX_RATIO_NUMERATOR / MAX_RATIO_DENOMINATOR} to {@link #create(int)} (or {@link - * #sizeFor(int)}) to leave ~25% headroom in the array. Kept as separate ints so callers can use - * integer arithmetic. + * Inverse of a 75% load factor. Callers that size their bucket array from a target working-set + * size {@code n} should pass {@code create(n, MAX_RATIO)} (or {@code sizeFor((int) Math.ceil(n + * * MAX_RATIO))}) to leave ~25% headroom in the array. */ - public static final int MAX_RATIO_NUMERATOR = 4; - - public static final int MAX_RATIO_DENOMINATOR = 3; + public static final float MAX_RATIO = 4.0f / 3.0f; static final int sizeFor(int requestedCapacity) { if (requestedCapacity < 0) { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 6fbf0cc752c..2992279be6d 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -87,14 +87,19 @@ void clearNullsAllBuckets() { } @Test - void maxRatioConstantsExpandTargetSize() { - // 75% load factor => bucket array sized at requestedSize * 4 / 3, rounded up to power of 2. - assertEquals(4, Support.MAX_RATIO_NUMERATOR); - assertEquals(3, Support.MAX_RATIO_DENOMINATOR); - int target = 12; - int sized = target * Support.MAX_RATIO_NUMERATOR / Support.MAX_RATIO_DENOMINATOR; - assertEquals(16, sized); - assertEquals(16, Support.sizeFor(sized)); + void maxRatioScalesTargetForLoadFactor() { + // 75% load factor => bucket array sized at requestedSize * 4/3, rounded up to power of 2. + // 12 * (4/3) = 16 entries, rounded up to power-of-2 length = 16. + assertEquals(4.0f / 3.0f, Support.MAX_RATIO); + Hashtable.Entry[] buckets = Support.create(12, Support.MAX_RATIO); + assertEquals(16, buckets.length); + } + + @Test + void createWithScaleRoundsUpToPowerOfTwo() { + // 7 * 1.5 = 10.5 -> (int) 10 -> sizeFor rounds up to next power-of-two = 16 + Hashtable.Entry[] buckets = Support.create(7, 1.5f); + assertEquals(16, buckets.length); } @Test From c0d3e263aa0f406c2bdd23352d54fd510f2a56d2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:23:02 -0400 Subject: [PATCH 49/70] Tighten Hashtable docs + rename MAX_CAPACITY to MAX_BUCKETS Five small cleanups from a design re-review pass: 1. Support javadoc: drop the stale "methods are package-private" sentence; most of them were made public in earlier commits for higher-arity callers. Also drop the "nested BucketIterator" framing (iterators are peers of Support inside Hashtable, not nested inside Support). 2. MAX_RATIO javadoc: drop the Math.ceil recommendation; create(int, float) deliberately truncates and is the canonical pathway. 3. Document the null-hash treatment on D1.Entry.hash and D2.Entry.hash so the behavior difference is explicit: D1 uses Long.MIN_VALUE as a sentinel that's collision-free against any int-valued hashCode(); D2 has no such sentinel and relies on matches() to resolve null/null vs hash-0 collisions. 4. Rename Support.MAX_CAPACITY -> MAX_BUCKETS and sizeFor's parameter to requestedSize. The cap is on the bucket-array length, not entry count; the new name reflects that. Error messages updated to match. 5. Drop the `abstract` modifier on Hashtable in favor of `final` with a private constructor. Nothing actually subclasses Hashtable -- the abstract was a namespace device that read as "intended for extension." Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 73 +++++++++++++------ 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 9e9ecb1c61a..b6cff2bc493 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -22,8 +22,13 @@ * *

      For higher key dimensions, client code must implement its own class, but can still use the * support class to ease the implementation complexity. + * + *

      This outer class is a pure namespace -- it can't be instantiated. The actual table types are + * {@link D1}, {@link D2}, and (for higher-arity callers) {@link Support}-driven custom tables. */ -public abstract class Hashtable { +public final class Hashtable { + private Hashtable() {} + /** * Internal base class for entries. Stores the precomputed 64-bit keyHash and the chain-next * pointer used to link colliding entries within a single bucket. @@ -96,6 +101,14 @@ public boolean matches(Object key) { return Objects.equals(this.key, key); } + /** + * Returns the 64-bit lookup hash for {@code key}. Null keys map to {@link Long#MIN_VALUE} so + * that they don't collide with a real key that hashes to 0 (e.g. {@code + * Integer.hashCode(0)}). The {@code Long.MIN_VALUE} sentinel is safe against any {@code + * int}-valued {@code hashCode()} since those widen to a long in the range {@code + * [Integer.MIN_VALUE, Integer.MAX_VALUE]}; real-key collisions in chains are resolved by + * {@link #matches(Object)}. + */ public static long hash(Object key) { return (key == null) ? Long.MIN_VALUE : key.hashCode(); } @@ -241,6 +254,13 @@ public boolean matches(K1 key1, K2 key2) { return Objects.equals(this.key1, key1) && Objects.equals(this.key2, key2); } + /** + * Returns the 64-bit lookup hash combining both key parts via {@link + * LongHashingUtils#hash(Object, Object)}. Null parts contribute {@code 0} (not a sentinel, + * unlike {@link D1.Entry#hash(Object)}): the combined hash can collide with real-key + * combinations whose chained hash equals {@code hash(0, 0) = 0} or similar values. {@link + * #matches(Object, Object)} resolves any such collision. + */ public static long hash(Object key1, Object key2) { return LongHashingUtils.hash(key1, key2); } @@ -340,16 +360,17 @@ public void forEach(T context, BiConsumer consume } /** - * Internal building blocks for hash-table operations. + * Building blocks for hash-table operations. * - *

      Used by {@link D1} and {@link D2}, and available to package code that wants to assemble its - * own higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The + *

      Used by {@link D1} and {@link D2}, and available to callers that want to assemble their own + * higher-arity table (3+ key parts) without re-implementing the bucket-array mechanics. The * typical recipe: * *

        *
      • Subclass {@link Hashtable.Entry} directly, adding the key fields and a {@code * matches(...)} method of your chosen arity. - *
      • Allocate a backing array with {@link #create(int)}. + *
      • Allocate a backing array with {@link #create(int)} or {@link #create(int, float)} (the + * latter scales for a target load factor; see {@link #MAX_RATIO}). *
      • Use {@link #bucketIndex(Object[], long)} for the bucket lookup, {@link * #bucketIterator(Hashtable.Entry[], long)} for read-only chain walks, and {@link * #mutatingBucketIterator(Hashtable.Entry[], long)} when you also need {@code remove} / @@ -362,21 +383,22 @@ public void forEach(T context, BiConsumer consume *
      • Clear with {@link #clear(Hashtable.Entry[])}. *
      * - *

      All bucket arrays produced by {@link #create(int)} have a power-of-two length, so {@link + *

      All bucket arrays produced by {@code create} have a power-of-two length, so {@link * #bucketIndex(Object[], long)} can use a bit mask. - * - *

      Methods on this class are package-private; the class itself is public only so that its - * nested {@link BucketIterator} can be referenced by callers in other packages. */ public static final class Support { - public static final Hashtable.Entry[] create(int capacity) { - return new Entry[sizeFor(capacity)]; + /** + * Allocates a bucket array sized to hold {@code requestedSize} entries. Returned length is + * {@code requestedSize} rounded up to the next power of two (capped at {@link #MAX_BUCKETS}). + */ + public static final Hashtable.Entry[] create(int requestedSize) { + return new Entry[sizeFor(requestedSize)]; } /** * Variant of {@link #create(int)} that scales the requested working-set size before sizing the - * bucket array. Pair with {@link #MAX_RATIO} (or similar) to leave headroom over the working - * set for a desired load factor. + * bucket array. Pair with {@link #MAX_RATIO} to leave headroom over the working set for a + * desired load factor; the canonical call is {@code create(n, MAX_RATIO)}. * *

      The scaled size is truncated to {@code int} before going through {@link #sizeFor(int)}. * Truncation rather than {@code ceil} is intentional: {@code sizeFor} rounds up to the next @@ -388,27 +410,32 @@ public static final Hashtable.Entry[] create(int requestedSize, float scale) { return new Entry[sizeFor((int) (requestedSize * scale))]; } - static final int MAX_CAPACITY = 1 << 30; + /** Upper bound on the bucket array length returned by {@link #sizeFor(int)}. */ + static final int MAX_BUCKETS = 1 << 30; /** * Inverse of a 75% load factor. Callers that size their bucket array from a target working-set - * size {@code n} should pass {@code create(n, MAX_RATIO)} (or {@code sizeFor((int) Math.ceil(n - * * MAX_RATIO))}) to leave ~25% headroom in the array. + * size {@code n} should pass {@code create(n, MAX_RATIO)} to leave ~25% headroom in the array. */ public static final float MAX_RATIO = 4.0f / 3.0f; - static final int sizeFor(int requestedCapacity) { - if (requestedCapacity < 0) { - throw new IllegalArgumentException("capacity must be non-negative: " + requestedCapacity); + /** + * Rounds {@code requestedSize} up to the next power of two, capped at {@link #MAX_BUCKETS}. + * Throws {@link IllegalArgumentException} for negative inputs or inputs above the cap. Returns + * the bucket-array length to allocate. + */ + static final int sizeFor(int requestedSize) { + if (requestedSize < 0) { + throw new IllegalArgumentException("requestedSize must be non-negative: " + requestedSize); } - if (requestedCapacity > MAX_CAPACITY) { + if (requestedSize > MAX_BUCKETS) { throw new IllegalArgumentException( - "capacity exceeds maximum (" + MAX_CAPACITY + "): " + requestedCapacity); + "requestedSize exceeds maximum bucket count (" + MAX_BUCKETS + "): " + requestedSize); } - if (requestedCapacity <= 1) { + if (requestedSize <= 1) { return 1; } - return Integer.highestOneBit(requestedCapacity - 1) << 1; + return Integer.highestOneBit(requestedSize - 1) << 1; } public static final void clear(Hashtable.Entry[] buckets) { From a0978bac3ede5a2da47f8fbac1ffc019781d34f5 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:25:52 -0400 Subject: [PATCH 50/70] Dedupe chain-head splice in D1/D2 via keyHash insertHeadEntry overload - Add Support.insertHeadEntry(buckets, long keyHash, entry) overload that derives the bucket index itself. Callers that already have a hash but not the index (the common case) now avoid the redundant bucketIndex(...) hop. - D1.insert, D1.insertOrReplace, D2.insert, D2.insertOrReplace: use the new overload, drop the (thisBuckets local, bucketIndex compute, setNext, store) sequence at each call site. - D2.buckets: drop the `private` modifier to match D1.buckets. Both are package-private so iterator tests in the same package can drive Support.bucketIterator against the table's bucket array. Added a short comment on both fields documenting the rationale. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 48 ++++++++----------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index b6cff2bc493..8db5bee6f14 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -114,6 +114,8 @@ public static long hash(Object key) { } } + // Package-private so iterator tests in the same package can drive Support.bucketIterator and + // friends directly against the table's bucket array. final Hashtable.Entry[] buckets; private int size; @@ -155,19 +157,11 @@ public TEntry remove(K key) { } public void insert(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; } public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { @@ -179,11 +173,7 @@ public TEntry insertOrReplace(TEntry newEntry) { } } - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; return null; } @@ -266,7 +256,8 @@ public static long hash(Object key1, Object key2) { } } - private final Hashtable.Entry[] buckets; + // Package-private to match D1.buckets -- available for iterator tests in the same package. + final Hashtable.Entry[] buckets; private int size; public D2(int capacity) { @@ -307,19 +298,11 @@ public TEntry remove(K1 key1, K2 key2) { } public void insert(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; - + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; } public TEntry insertOrReplace(TEntry newEntry) { - Hashtable.Entry[] thisBuckets = this.buckets; - for (MutatingBucketIterator iter = Support.mutatingBucketIterator(this.buckets, newEntry.keyHash); iter.hasNext(); ) { @@ -331,11 +314,7 @@ public TEntry insertOrReplace(TEntry newEntry) { } } - int bucketIndex = Support.bucketIndex(thisBuckets, newEntry.keyHash); - - Hashtable.Entry curHead = thisBuckets[bucketIndex]; - newEntry.setNext(curHead); - thisBuckets[bucketIndex] = newEntry; + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); this.size += 1; return null; } @@ -476,6 +455,17 @@ public static final void insertHeadEntry( buckets[bucketIndex] = entry; } + /** + * Convenience overload of {@link #insertHeadEntry(Hashtable.Entry[], int, Hashtable.Entry)} + * that derives the bucket index from {@code keyHash}. Use this when the caller has the hash but + * not the index; if the index has already been computed for another reason, prefer the + * int-taking overload to avoid the redundant mask. + */ + public static final void insertHeadEntry( + Hashtable.Entry[] buckets, long keyHash, Hashtable.Entry entry) { + insertHeadEntry(buckets, bucketIndex(buckets, keyHash), entry); + } + /** * Returns the head entry of the bucket that {@code keyHash} maps to, cast to the caller's * concrete entry type. The unchecked cast lives here so the chain-walk loop at the call site From e604a8f78d1b0cf1e11ddf724c88414c65c1a198 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 19 May 2026 16:31:37 -0400 Subject: [PATCH 51/70] Tighten Entry.next encapsulation; doc hasNext; add D1/D2 getOrCreate Three follow-ups from the design review: - Make Hashtable.Entry.next private. All same-package readers (BucketIterator) already had a next() accessor; the leftover direct field reads now route through it. Closes the "mixed encapsulation" gap where some readers used the accessor and same-package ones reached for the field. - BucketIterator and MutatingBucketIterator now document that chain-walk work happens in next() (and the constructor for the first match); hasNext() is an O(1) field read. - Add D1.getOrCreate(K, Function) and D2.getOrCreate(K1, K2, BiFunction). Both reuse the lookup hash for the insert on miss, avoiding the double-hash that "get; if null then insert" callers would otherwise pay. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 58 +++++++++++++++++-- .../datadog/trace/util/HashtableD1Test.java | 48 +++++++++++++++ .../datadog/trace/util/HashtableD2Test.java | 41 +++++++++++++ 3 files changed, 143 insertions(+), 4 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 8db5bee6f14..9d9063ae8a8 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -5,7 +5,9 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.function.BiConsumer; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; /** * Light weight simple Hashtable system that can be useful when HashMap would be unnecessarily @@ -39,7 +41,7 @@ private Hashtable() {} */ public abstract static class Entry { public final long keyHash; - Entry next = null; + private Entry next = null; protected Entry(long keyHash) { this.keyHash = keyHash; @@ -178,6 +180,29 @@ public TEntry insertOrReplace(TEntry newEntry) { return null; } + /** + * Returns the entry for {@code key}, building one via {@code creator} if absent. Computes the + * hash once and reuses it for both the lookup and (on miss) the insert -- avoids the + * double-hash that "{@code get}; if null then {@code insert}" would incur. + * + *

      The {@code creator} is expected to build an entry whose {@code keyHash} equals {@link + * Entry#hash(Object) D1.Entry.hash(key)} -- typically by passing {@code key} to a constructor + * that calls {@code super(key)}. A mismatched hash will leave the new entry inserted at a + * bucket that future {@link #get} calls won't probe. + */ + public TEntry getOrCreate(K key, Function creator) { + long keyHash = D1.Entry.hash(key); + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key)) { + return te; + } + } + TEntry newEntry = creator.apply(key); + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); + this.size += 1; + return newEntry; + } + public void clear() { Support.clear(this.buckets); this.size = 0; @@ -319,6 +344,25 @@ public TEntry insertOrReplace(TEntry newEntry) { return null; } + /** + * Two-key analogue of {@link D1#getOrCreate}. Computes the combined hash once and reuses it for + * both lookup and (on miss) insert. The {@code creator} is expected to build an entry whose + * {@code keyHash} equals {@link Entry#hash(Object, Object) D2.Entry.hash(key1, key2)}. + */ + public TEntry getOrCreate( + K1 key1, K2 key2, BiFunction creator) { + long keyHash = D2.Entry.hash(key1, key2); + for (TEntry te = Support.bucket(this.buckets, keyHash); te != null; te = te.next()) { + if (te.keyHash == keyHash && te.matches(key1, key2)) { + return te; + } + } + TEntry newEntry = creator.apply(key1, key2); + Support.insertHeadEntry(this.buckets, newEntry.keyHash, newEntry); + this.size += 1; + return newEntry; + } + public void clear() { Support.clear(this.buckets); this.size = 0; @@ -515,6 +559,9 @@ public static final void forEach( * *

      For {@code remove} or {@code replace} operations, use {@link MutatingBucketIterator} * instead. + * + *

      The chain-walk work to find the next-match entry happens in {@link #next()} (and in the + * constructor for the first match); {@link #hasNext()} is an O(1) field read. */ public static final class BucketIterator implements Iterator { private final long keyHash; @@ -524,7 +571,7 @@ public static final class BucketIterator implements Iterat this.keyHash = keyHash; Hashtable.Entry cur = buckets[Support.bucketIndex(buckets, keyHash)]; while (cur != null && cur.keyHash != keyHash) { - cur = cur.next; + cur = cur.next(); } this.nextEntry = cur; } @@ -540,9 +587,9 @@ public TEntry next() { Hashtable.Entry cur = this.nextEntry; if (cur == null) throw new NoSuchElementException("no next!"); - Hashtable.Entry advance = cur.next; + Hashtable.Entry advance = cur.next(); while (advance != null && advance.keyHash != keyHash) { - advance = advance.next; + advance = advance.next(); } this.nextEntry = advance; @@ -559,6 +606,9 @@ public TEntry next() { * remove} and {@code replace} can fix up the chain in O(1) without re-walking from the bucket * head. After {@code remove} or {@code replace}, iteration may continue with another {@link * #next()}. + * + *

      The chain-walk work to find the next-match entry happens in {@link #next()} (and in the + * constructor for the first match); {@link #hasNext()} is an O(1) field read. */ public static final class MutatingBucketIterator implements Iterator { diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java index 11928bb4d5b..11cf93fc1dd 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD1Test.java @@ -184,4 +184,52 @@ void hashCollisionsThenRemoveLeavesOtherIntact() { assertNull(table.get(k2)); assertNotNull(table.get(k3)); } + + @Test + void getOrCreateOnMissBuildsEntryViaCreator() { + Hashtable.D1 table = new Hashtable.D1<>(8); + int[] createCount = {0}; + StringIntEntry created = + table.getOrCreate( + "foo", + k -> { + createCount[0]++; + return new StringIntEntry(k, 42); + }); + assertNotNull(created); + assertEquals("foo", created.key); + assertEquals(42, created.value); + assertEquals(1, table.size()); + assertEquals(1, createCount[0]); + assertSame(created, table.get("foo")); + } + + @Test + void getOrCreateOnHitSkipsCreator() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry seeded = new StringIntEntry("foo", 1); + table.insert(seeded); + int[] createCount = {0}; + StringIntEntry got = + table.getOrCreate( + "foo", + k -> { + createCount[0]++; + return new StringIntEntry(k, 999); + }); + assertSame(seeded, got); + assertEquals(1, table.size()); + assertEquals(0, createCount[0]); + } + + @Test + void getOrCreateNullKeyIsPermitted() { + Hashtable.D1 table = new Hashtable.D1<>(8); + StringIntEntry created = table.getOrCreate(null, k -> new StringIntEntry(k, 7)); + assertNotNull(created); + assertNull(created.key); + assertEquals(7, created.value); + assertSame(created, table.getOrCreate(null, k -> new StringIntEntry(k, 999))); + assertEquals(1, table.size()); + } } diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java index 59339fcd89e..edcb0ad9f74 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableD2Test.java @@ -1,6 +1,7 @@ package datadog.trace.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -77,6 +78,46 @@ void forEachWithContextPassesContextToConsumer() { assertTrue(seen.contains("b:2")); } + @Test + void getOrCreateOnMissBuildsEntryViaCreator() { + Hashtable.D2 table = new Hashtable.D2<>(8); + int[] createCount = {0}; + PairEntry created = + table.getOrCreate( + "a", + 1, + (k1, k2) -> { + createCount[0]++; + return new PairEntry(k1, k2, 100); + }); + assertNotNull(created); + assertEquals("a", created.key1); + assertEquals(Integer.valueOf(1), created.key2); + assertEquals(100, created.value); + assertEquals(1, table.size()); + assertEquals(1, createCount[0]); + assertSame(created, table.get("a", 1)); + } + + @Test + void getOrCreateOnHitSkipsCreator() { + Hashtable.D2 table = new Hashtable.D2<>(8); + PairEntry seeded = new PairEntry("a", 1, 100); + table.insert(seeded); + int[] createCount = {0}; + PairEntry got = + table.getOrCreate( + "a", + 1, + (k1, k2) -> { + createCount[0]++; + return new PairEntry(k1, k2, 999); + }); + assertSame(seeded, got); + assertEquals(1, table.size()); + assertEquals(0, createCount[0]); + } + private static final class PairEntry extends Hashtable.D2.Entry { int value; From e2642cdf1f05a785641008cff56fe14ffbdad4da Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 20 May 2026 13:58:28 -0400 Subject: [PATCH 52/70] Hashtable: add missing braces and detach removed/replaced entries Addresses PR #11409 review comments: - #3267164119 / #3267165525: wrap every single-line if/break body in braces (7 sites across BucketIterator, MutatingBucketIterator, and the full-table Iterator). - #3275947761 / #3275948108 (sarahchen6): null out the removed/replaced entry's next pointer after splicing it out of the chain in MutatingBucketIterator.remove / .replace. Applied the same fix to the full-table Iterator.remove for consistency. Rationale: detaching prevents accidental traversal through a removed entry via a stale reference and lets the GC reclaim a chain tail that the removed entry was the last referrer to. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/util/Hashtable.java | 52 ++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 9d9063ae8a8..8f40e4609bc 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -585,7 +585,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry cur = this.nextEntry; - if (cur == null) throw new NoSuchElementException("no next!"); + if (cur == null) { + throw new NoSuchElementException("no next!"); + } Hashtable.Entry advance = cur.next(); while (advance != null && advance.keyHash != keyHash) { @@ -643,7 +645,9 @@ public static final class MutatingBucketIterator } else { Hashtable.Entry prev, cur; for (prev = null, cur = headEntry; cur != null; prev = cur, cur = cur.next()) { - if (cur.keyHash == keyHash) break; + if (cur.keyHash == keyHash) { + break; + } } this.nextPrevEntry = prev; this.nextEntry = cur; @@ -662,7 +666,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry curEntry = this.nextEntry; - if (curEntry == null) throw new NoSuchElementException("no next!"); + if (curEntry == null) { + throw new NoSuchElementException("no next!"); + } this.curEntry = curEntry; this.curPrevEntry = this.nextPrevEntry; @@ -671,7 +677,9 @@ public TEntry next() { for (prev = this.nextEntry, cur = this.nextEntry.next(); cur != null; prev = cur, cur = prev.next()) { - if (cur.keyHash == keyHash) break; + if (cur.keyHash == keyHash) { + break; + } } this.nextPrevEntry = prev; this.nextEntry = cur; @@ -682,9 +690,15 @@ public TEntry next() { @Override public void remove() { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } - this.setPrevNext(oldCurEntry.next()); + Hashtable.Entry oldNext = oldCurEntry.next(); + this.setPrevNext(oldNext); + // Detach the removed entry from the chain so stale references can't traverse back into + // the live chain and so a now-unreachable tail can be reclaimed by GC. + oldCurEntry.setNext(null); // If the next match was directly after oldCurEntry, its predecessor is now // curPrevEntry (oldCurEntry was just unlinked from the chain). @@ -696,10 +710,15 @@ public void remove() { public void replace(TEntry replacementEntry) { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } - replacementEntry.setNext(oldCurEntry.next()); + Hashtable.Entry oldNext = oldCurEntry.next(); + replacementEntry.setNext(oldNext); this.setPrevNext(replacementEntry); + // Detach the replaced entry from the chain; the replacement now owns the chain slot. + oldCurEntry.setNext(null); // If the next match was directly after oldCurEntry, its predecessor is now // the replacement entry (which took oldCurEntry's chain slot). @@ -777,7 +796,9 @@ public boolean hasNext() { @SuppressWarnings("unchecked") public TEntry next() { Hashtable.Entry e = this.nextEntry; - if (e == null) throw new NoSuchElementException("no next!"); + if (e == null) { + throw new NoSuchElementException("no next!"); + } this.curEntry = e; this.curPrevEntry = this.nextPrevEntry; @@ -797,13 +818,20 @@ public TEntry next() { @Override public void remove() { Hashtable.Entry oldCurEntry = this.curEntry; - if (oldCurEntry == null) throw new IllegalStateException(); + if (oldCurEntry == null) { + throw new IllegalStateException(); + } + Hashtable.Entry oldNext = oldCurEntry.next(); if (this.curPrevEntry == null) { - this.buckets[this.curBucketIndex] = oldCurEntry.next(); + this.buckets[this.curBucketIndex] = oldNext; } else { - this.curPrevEntry.setNext(oldCurEntry.next()); + this.curPrevEntry.setNext(oldNext); } + // Detach the removed entry from the chain so stale references can't traverse back into + // the live chain and so a now-unreachable tail can be reclaimed by GC. + oldCurEntry.setNext(null); + // If the next entry was the immediate chain successor of oldCurEntry, its predecessor is // now what came before oldCurEntry (oldCurEntry was just unlinked). if (this.nextPrevEntry == oldCurEntry) { From 585ca56cc17575ee33f63c02be9bf36b9cb896a1 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 20 May 2026 14:19:06 -0400 Subject: [PATCH 53/70] Rename LongHashingUtils.hashCodeX(Object) to hash(Object) for API consistency Addresses PR #11409 review comment #3276167001. The method parallels the primitive hash(boolean) / hash(int) / hash(long) / ... family, so naming it hash(Object) -- with null collapsing to Long.MIN_VALUE as a sentinel distinct from any real hashCode -- matches the rest of the public surface. Test call sites that pass a literal null now disambiguate against hash(int[]) / hash(Object[]) / hash(Iterable) via an (Object) cast. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/main/java/datadog/trace/util/LongHashingUtils.java | 2 +- .../test/java/datadog/trace/util/LongHashingUtilsTest.java | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java index 9d1257a3f20..88104baa8d8 100644 --- a/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java +++ b/internal-api/src/main/java/datadog/trace/util/LongHashingUtils.java @@ -8,7 +8,7 @@ public final class LongHashingUtils { private LongHashingUtils() {} - public static final long hashCodeX(Object obj) { + public static final long hash(Object obj) { return obj == null ? Long.MIN_VALUE : obj.hashCode(); } diff --git a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java index c0e0bebdda0..795c182df18 100644 --- a/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java +++ b/internal-api/src/test/java/datadog/trace/util/LongHashingUtilsTest.java @@ -2,7 +2,6 @@ import static datadog.trace.util.LongHashingUtils.addToHash; import static datadog.trace.util.LongHashingUtils.hash; -import static datadog.trace.util.LongHashingUtils.hashCodeX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -15,10 +14,10 @@ class LongHashingUtilsTest { // ----- single-value overloads ----- @Test - void hashCodeXReturnsObjectHashCodeOrSentinelForNull() { + void hashOfObjectReturnsHashCodeOrSentinelForNull() { Object o = new Object(); - assertEquals(o.hashCode(), hashCodeX(o)); - assertEquals(Long.MIN_VALUE, hashCodeX(null)); + assertEquals(o.hashCode(), hash(o)); + assertEquals(Long.MIN_VALUE, hash((Object) null)); } @Test From 9391c4800c8a57052e1ded25d1ad44a6015789b8 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 11:47:35 -0400 Subject: [PATCH 54/70] Use writer.finishBucket() count in bootstrap test for cascade compatibility The verify(writer).add(MetricKey, AggregateMetric) signature is unique to #11381; downstream branches use AggregateEntry. Switching to verify(writer, times(2)).finishBucket() keeps the same behavioral guarantee (both cycles flushed) across the stack. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../metrics/ConflatingMetricsAggregatorBootstrapTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java index b8b46a31298..76347e505c0 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java @@ -193,10 +193,10 @@ void reconcileSurvivesTimestampBumpWhenTagsUnchanged() throws Exception { aggregator.report(); assertTrue(cycle2.await(2, SECONDS)); - // Both cycles flushed: writer.add was invoked twice (once per cycle). The schema kept - // producing the same MetricKey across cycles -- if the schema had been broken by the - // timestamp bump, no buckets would have flushed. - verify(writer, times(2)).add(any(MetricKey.class), any(AggregateMetric.class)); + // Both cycles flushed (both latches counted down via writer.finishBucket). The schema kept + // producing buckets across the timestamp bumps; if the schema had been broken by the + // bump-in-place path, the second cycle's flush would not have happened. + verify(writer, times(2)).finishBucket(); // Bootstrap (1) + two reconciles (2) -- each reconcile saw a timestamp mismatch and went // through the deep compare, calling peerTags() once = 3 total. verify(features, times(3)).peerTags(); From df3b31d8f8074c741ef784ead4296be0965f3704 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 14:52:40 -0400 Subject: [PATCH 55/70] Adapt reconcileSwapsSchemaWhenTagSetChanges to AggregateEntry shape #11382 collapses MetricWriter.add(MetricKey, AggregateMetric) into add(AggregateEntry). Re-target the captor and accessors on this branch so the test compiles and the same end-to-end peer-tag verification holds. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...flatingMetricsAggregatorBootstrapTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java index aea44e3682f..060da2ba9b6 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorBootstrapTest.java @@ -218,9 +218,9 @@ void reconcileSurvivesTimestampBumpWhenTagsUnchanged() throws Exception { void reconcileSwapsSchemaWhenTagSetChanges() throws Exception { // The reconcile slow-path's swap branch: discovery refreshes the timestamp AND the tag set // grows. Cached schema is rebuilt and the volatile reference points at the new schema. - // Verification is end-to-end -- we look at the MetricKey the writer receives. Pre-swap the - // span snapshot was pinned to the old schema so only peer.hostname appears; post-swap a new - // publish reads the new schema and the next flush carries both peer tags. + // Verification is end-to-end -- we look at the AggregateEntry the writer receives. Pre-swap + // the span snapshot was pinned to the old schema so only peer.hostname appears; post-swap a + // new publish reads the new schema and the next flush carries both peer tags. HealthMetrics healthMetrics = mock(HealthMetrics.class); MetricWriter writer = mock(MetricWriter.class); Sink sink = mock(Sink.class); @@ -267,8 +267,8 @@ void reconcileSwapsSchemaWhenTagSetChanges() throws Exception { .finishBucket(); // Publish 1: snapshot pinned to the original {peer.hostname} schema. cycle 1's reconcile - // will swap the cached schema BEFORE the flush, but this snapshot is already pinned so its - // MetricKey will still carry only peer.hostname. + // will swap the cached schema BEFORE the flush, but this snapshot is already pinned so the + // resulting AggregateEntry will still carry only peer.hostname. aggregator.publish( Collections.>singletonList(peerAggregationSpanWithBothPeerTags())); aggregator.report(); @@ -281,20 +281,20 @@ void reconcileSwapsSchemaWhenTagSetChanges() throws Exception { aggregator.report(); assertTrue(cycle2.await(2, SECONDS)); - // Capture every (MetricKey, AggregateMetric) the writer saw across both cycles. Pre-swap - // snapshot has 1 peer tag, post-swap has 2. - ArgumentCaptor keyCaptor = ArgumentCaptor.forClass(MetricKey.class); - verify(writer, times(2)).add(keyCaptor.capture(), any(AggregateMetric.class)); - List keys = keyCaptor.getAllValues(); + // Capture every AggregateEntry the writer saw across both cycles. Pre-swap snapshot has 1 + // peer tag, post-swap has 2. + ArgumentCaptor entryCaptor = ArgumentCaptor.forClass(AggregateEntry.class); + verify(writer, times(2)).add(entryCaptor.capture()); + List entries = entryCaptor.getAllValues(); assertEquals( Collections.singletonList(UTF8BytesString.create("peer.hostname:localhost")), - keys.get(0).getPeerTags(), + entries.get(0).getPeerTags(), "pre-swap snapshot should encode only peer.hostname"); assertEquals( Arrays.asList( UTF8BytesString.create("peer.hostname:localhost"), UTF8BytesString.create("peer.service:billing")), - keys.get(1).getPeerTags(), + entries.get(1).getPeerTags(), "post-swap snapshot should encode both peer.hostname and peer.service"); // Bootstrap (1) + cycle 1 slow-path (1) -- cycle 2 is fast-path so doesn't reach peerTags(). From a6066929452663b69e1ca6a7130d275bfafd62c4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 15:19:48 -0400 Subject: [PATCH 56/70] Fix MetricsIntegrationTest entry recording call site AggregateEntry consolidated MetricKey + AggregateMetric so recordDurations lives directly on AggregateEntry now. The previous entry1.aggregate. recordDurations(...) form compiles under Groovy's dynamic dispatch but would throw MissingPropertyException at runtime since there is no `aggregate` property. Resolves chatgpt-codex-connector's review comment. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/traceAgentTest/groovy/MetricsIntegrationTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy index 81a476c67c8..4883543cf68 100644 --- a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy +++ b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy @@ -39,10 +39,10 @@ class MetricsIntegrationTest extends AbstractTraceAgentTest { ) writer.startBucket(2, System.nanoTime(), SECONDS.toNanos(10)) def entry1 = AggregateEntry.of("resource1", "service1", "operation1", null, "sql", 0, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) - entry1.aggregate.recordDurations(5, new AtomicLongArray(2, 1, 2, 250, 4, 5)) + entry1.recordDurations(5, new AtomicLongArray(2, 1, 2, 250, 4, 5)) writer.add(entry1) def entry2 = AggregateEntry.of("resource2", "service2", "operation2", null, "web", 200, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) - entry2.aggregate.recordDurations(10, new AtomicLongArray(1, 1, 200, 2, 3, 4, 5, 6, 7, 8, 9)) + entry2.recordDurations(10, new AtomicLongArray(1, 1, 200, 2, 3, 4, 5, 6, 7, 8, 9)) writer.add(entry2) writer.finishBucket() From 913e7d754bfc658810c51057d9c42d4f85081236 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 15:28:26 -0400 Subject: [PATCH 57/70] Make ConflatingMetricAggregatorTest counter checks actually verify The `1 * writer.add(value) >> { closure }` pattern treats the closure as a stubbed return value -- Spock evaluates it but discards the result, so `e.getHitCount() == X && ...` was a silent no-op across 31 occurrences. Wrapping the expression in `assert` makes Groovy's power-assert throw on mismatch, which Spock surfaces as a real failure. Resolves chatgpt-codex-connector's review comment. All 41 tests still pass, so the previously-unverified assertions happened to hold. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../ConflatingMetricAggregatorTest.groovy | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy index 3d75e43a88e..0fa1ed2a2a2 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy @@ -134,7 +134,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -180,7 +180,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -232,7 +232,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { httpEndpoint, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } (statsComputed ? 1 : 0) * writer.finishBucket() >> { latch.countDown() } @@ -309,7 +309,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.add( AggregateEntry.of( @@ -327,7 +327,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 2 * writer.finishBucket() >> { latch1.countDown(); latch2.countDown() } @@ -374,7 +374,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -426,7 +426,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == topLevelCount && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == topLevelCount && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -485,7 +485,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == count && e.getDuration() == count * duration + assert e.getHitCount() == count && e.getDuration() == count * duration } 1 * writer.add(AggregateEntry.of( "resource2", @@ -502,7 +502,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == count && e.getDuration() == count * duration * 2 + assert e.getHitCount() == count && e.getDuration() == count * duration * 2 } cleanup: @@ -556,7 +556,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == count && e.getDuration() == count * duration + assert e.getHitCount() == count && e.getDuration() == count * duration } 1 * writer.finishBucket() >> { latch.countDown() } @@ -597,7 +597,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -614,7 +614,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 2 + assert e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.add(AggregateEntry.of( "resource", @@ -631,7 +631,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 3 + assert e.getHitCount() == 1 && e.getDuration() == duration * 3 } 1 * writer.finishBucket() >> { latch2.countDown() } @@ -695,7 +695,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -712,7 +712,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 2 + assert e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.add(AggregateEntry.of( "resource", @@ -729,7 +729,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 3 + assert e.getHitCount() == 1 && e.getDuration() == duration * 3 } 1 * writer.add(AggregateEntry.of( "resource", @@ -746,7 +746,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 4 + assert e.getHitCount() == 1 && e.getDuration() == duration * 4 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -799,7 +799,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -816,7 +816,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration * 2 + assert e.getHitCount() == 1 && e.getDuration() == duration * 2 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -867,7 +867,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 2 && e.getDuration() == 2 * duration + assert e.getHitCount() == 2 && e.getDuration() == 2 * duration } 1 * writer.add(AggregateEntry.of( "resource", @@ -884,7 +884,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } 1 * writer.finishBucket() >> { latch.countDown() } @@ -938,7 +938,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } } 0 * writer.add(AggregateEntry.of( @@ -1085,7 +1085,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1120,7 +1120,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } } 0 * writer.add(AggregateEntry.of( @@ -1187,7 +1187,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1246,7 +1246,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getDuration() == duration + assert e.getHitCount() == 1 && e.getDuration() == duration } } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1413,7 +1413,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1468,7 +1468,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 3 && e.getTopLevelCount() == 3 && e.getDuration() == 450 + assert e.getHitCount() == 3 && e.getTopLevelCount() == 3 && e.getDuration() == 450 } 1 * writer.finishBucket() >> { latch.countDown() } @@ -1523,7 +1523,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/users/:id", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.add( AggregateEntry.of( @@ -1541,7 +1541,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { "/api/orders", null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 200 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 200 } 1 * writer.add( AggregateEntry.of( @@ -1559,7 +1559,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, null )) >> { AggregateEntry e -> - e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 150 + assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 150 } 1 * writer.finishBucket() >> { latch.countDown() } From 2dcea9a9c3273870c6ab9bca17c98087027b70c3 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 15:41:58 -0400 Subject: [PATCH 58/70] Drop dead recordDurations(int, AtomicLongArray) batch API This method was a vestige of master's Batch design where multiple producer threads wrote into an AtomicLongArray slot concurrently and the aggregator drained ~64 durations per Batch in one call. The new producer/consumer split publishes one SpanSnapshot per span, so production only ever calls recordOneDuration(long). Migrate the three remaining callers (AggregateEntryTest, SerializingMetricWriterTest, MetricsIntegrationTest) to a loop of recordOneDuration(long) calls, then delete the batched method and its AtomicLongArray imports. Drops the recordDurationsIgnoresTrailingZeros test -- that behavior was a specific quirk of the batched API (count parameter shorter than the array length) and doesn't apply to recordOneDuration. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 21 ----------- .../SerializingMetricWriterTest.groovy | 3 +- .../common/metrics/AggregateEntryTest.java | 37 ++++++++----------- .../groovy/MetricsIntegrationTest.groovy | 5 +-- 4 files changed, 19 insertions(+), 47 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 90d41ff7bdc..cd1d7083e05 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -16,7 +16,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicLongArray; import java.util.function.Function; import javax.annotation.Nullable; @@ -199,26 +198,6 @@ static AggregateEntry forSnapshot(SpanSnapshot s) { return new AggregateEntry(s, hashOf(s)); } - AggregateEntry recordDurations(int count, AtomicLongArray durations) { - this.hitCount += count; - for (int i = 0; i < count && i < durations.length(); ++i) { - long duration = durations.getAndSet(i, 0); - if ((duration & TOP_LEVEL_TAG) == TOP_LEVEL_TAG) { - duration ^= TOP_LEVEL_TAG; - ++topLevelCount; - } - if ((duration & ERROR_TAG) == ERROR_TAG) { - duration ^= ERROR_TAG; - errorLatencies.accept(duration); - ++errorCount; - } else { - okLatencies.accept(duration); - } - this.duration += duration; - } - return this; - } - /** * Records a single hit. {@code tagAndDuration} carries the duration nanos with optional {@link * #ERROR_TAG} / {@link #TOP_LEVEL_TAG} bits OR-ed in. diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy index 5e85c66557d..752cea028d1 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy @@ -15,7 +15,6 @@ import datadog.trace.api.git.GitInfoProvider import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString import datadog.trace.test.util.DDSpecification import java.nio.ByteBuffer -import java.util.concurrent.atomic.AtomicLongArray import org.msgpack.core.MessagePack import org.msgpack.core.MessageUnpacker @@ -45,7 +44,7 @@ class SerializingMetricWriterTest extends DDSpecification { resource, service, operationName, serviceSource, type, httpStatusCode, synthetic, traceRoot, spanKind, peerTags, httpMethod, httpEndpoint, grpcStatusCode) - e.recordDurations(hitCount, new AtomicLongArray(1L)) + hitCount.times { e.recordOneDuration(1L) } return e } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java index 7b3a8a1f398..578f3b753b8 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java @@ -10,7 +10,6 @@ import datadog.metrics.impl.DDSketchHistograms; import datadog.metrics.impl.MonitoringImpl; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLongArray; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -25,17 +24,20 @@ static void initAgentMeter() { } @Test - void recordDurationsSumsToTotal() { + void recordOneDurationSumsToTotal() { AggregateEntry entry = newEntry(); - entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L})); + entry.recordOneDuration(1L); + entry.recordOneDuration(2L); + entry.recordOneDuration(3L); assertEquals(6, entry.getDuration()); } @Test void clearResetsAllCounters() { AggregateEntry entry = newEntry(); - entry.recordDurations( - 3, new AtomicLongArray(new long[] {5L, ERROR_TAG | 6L, TOP_LEVEL_TAG | 7L})); + entry.recordOneDuration(5L); + entry.recordOneDuration(ERROR_TAG | 6L); + entry.recordOneDuration(TOP_LEVEL_TAG | 7L); entry.clear(); assertEquals(0, entry.getDuration()); assertEquals(0, entry.getErrorCount()); @@ -56,19 +58,12 @@ void recordOneDurationAccumulatesOkErrorAndTopLevel() { assertEquals(1, entry.getTopLevelCount()); } - @Test - void recordDurationsIgnoresTrailingZeros() { - AggregateEntry entry = newEntry(); - entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L, 0L, 0L, 0L})); - assertEquals(6, entry.getDuration()); - assertEquals(3, entry.getHitCount()); - assertEquals(0, entry.getErrorCount()); - } - @Test void hitCountIncludesErrors() { AggregateEntry entry = newEntry(); - entry.recordDurations(3, new AtomicLongArray(new long[] {1L, 2L, 3L | ERROR_TAG})); + entry.recordOneDuration(1L); + entry.recordOneDuration(2L); + entry.recordOneDuration(3L | ERROR_TAG); assertEquals(3, entry.getHitCount()); assertEquals(1, entry.getErrorCount()); } @@ -76,12 +71,12 @@ void hitCountIncludesErrors() { @Test void okAndErrorLatenciesTrackedSeparately() { AggregateEntry entry = newEntry(); - entry.recordDurations( - 10, - new AtomicLongArray( - new long[] { - 1L, 100L | ERROR_TAG, 2L, 99L | ERROR_TAG, 3L, 98L | ERROR_TAG, 4L, 97L | ERROR_TAG - })); + long[] durations = { + 1L, 100L | ERROR_TAG, 2L, 99L | ERROR_TAG, 3L, 98L | ERROR_TAG, 4L, 97L | ERROR_TAG + }; + for (long d : durations) { + entry.recordOneDuration(d); + } assertTrue(entry.getErrorLatencies().getMaxValue() >= 99); assertTrue(entry.getOkLatencies().getMaxValue() <= 5); } diff --git a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy index 4883543cf68..7afacc179cc 100644 --- a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy +++ b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy @@ -14,7 +14,6 @@ import datadog.trace.common.metrics.OkHttpSink import datadog.trace.common.metrics.SerializingMetricWriter import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch -import java.util.concurrent.atomic.AtomicLongArray import okhttp3.HttpUrl class MetricsIntegrationTest extends AbstractTraceAgentTest { @@ -39,10 +38,10 @@ class MetricsIntegrationTest extends AbstractTraceAgentTest { ) writer.startBucket(2, System.nanoTime(), SECONDS.toNanos(10)) def entry1 = AggregateEntry.of("resource1", "service1", "operation1", null, "sql", 0, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) - entry1.recordDurations(5, new AtomicLongArray(2, 1, 2, 250, 4, 5)) + [2, 1, 2, 250, 4].each { entry1.recordOneDuration(it as long) } writer.add(entry1) def entry2 = AggregateEntry.of("resource2", "service2", "operation2", null, "web", 200, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) - entry2.recordDurations(10, new AtomicLongArray(1, 1, 200, 2, 3, 4, 5, 6, 7, 8, 9)) + [1, 1, 200, 2, 3, 4, 5, 6, 7, 8].each { entry2.recordOneDuration(it as long) } writer.add(entry2) writer.finishBucket() From 50b06e59c4212109efa036a3e2b6b8565ba49019 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 15:42:34 -0400 Subject: [PATCH 59/70] Warn about colon split in AggregateEntry.of test factory The factory recovers (name, value) pairs from pre-encoded "name:value" strings by splitting at the FIRST colon. Test-only, but worth being explicit so callers don't hand it a peer-tag value containing a colon (URLs, IPv6, service:env) and get a silently wrong (name, value) pair. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../java/datadog/trace/common/metrics/AggregateEntry.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index cd1d7083e05..4755b26c1b2 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -143,6 +143,12 @@ private AggregateEntry(SpanSnapshot s, long keyHash) { * Test-friendly factory mirroring the prior {@code new MetricKey(...)} positional args. Accepts a * pre-encoded {@code List} of {@code "name:value"} peer tags and recovers the * parallel-array {@code (names, values)} form by splitting on the {@code ':'} delimiter. + * + *

      Test-only. The split is at the first {@code ':'}, so peer-tag values + * containing a colon (URLs, IPv6 addresses, {@code service:env} patterns) will be silently + * misparsed and the recovered (name, value) pair will be wrong. Keep test data colon-free in + * peer-tag values, or wire production-style snapshots through {@link #forSnapshot(SpanSnapshot)} + * instead. */ static AggregateEntry of( CharSequence resource, From b0f21bf891ed3c23961836264e1845dc46fe1b26 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 21 May 2026 15:56:38 -0400 Subject: [PATCH 60/70] Add coverage for disable() -> ClearSignal threading path The bundled fix in this PR routes the agent-downgrade clear through the inbox so the aggregator thread stays the sole writer to AggregateTable. Prior to this test, there was no regression coverage for that routing. The test fires DOWNGRADED from the test thread (production-like OkHttpSink callback path), waits for the immediate no-flush window, then publishes a marker span with a distinct resource name. The subsequent report's writer.add captor must see only the marker -- if CLEAR didn't actually wipe the original entry, the original "resource" would still be present and the assertion would catch it. Cannot directly verify thread identity of the clear from inside this test (CLEAR's inbox.clear() drops any latch signal we'd queue behind it), so this is an observable-contract test rather than a strict thread-id test. Still catches both the missing-clear regression and the bucket-chain-corruption regression that the original threading race could produce. Co-Authored-By: Claude Opus 4.7 (1M context) --- ...onflatingMetricsAggregatorDisableTest.java | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java new file mode 100644 index 00000000000..72ac8e6ff42 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java @@ -0,0 +1,187 @@ +package datadog.trace.common.metrics; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.after; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreSpan; +import datadog.trace.core.SpanKindFilter; +import datadog.trace.core.monitor.HealthMetrics; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** + * Coverage for the {@code disable() -> ClearSignal.CLEAR} threading routing introduced in this PR. + * + *

      The bundled fix routes the agent-downgrade clear through the inbox so the aggregator thread + * stays the sole writer to {@link AggregateTable} (which is not thread-safe). The behavioral + * contract this test pins: + * + *

        + *
      • {@code onEvent(DOWNGRADED)} can fire from a non-aggregator thread (in production, the + * OkHttpSink callback thread). + *
      • By the time the next report cycle reconciles peer-tag schema on the aggregator thread, the + * {@code AggregateTable} has been cleared -- {@code CLEAR} arrived in the FIFO inbox before + * the {@code REPORT} signal triggered by {@code aggregator.report()}. + *
      • The aggregator therefore flushes nothing on that next report cycle: no {@code startBucket}, + * no {@code add}, no {@code finishBucket}. + *
      + * + *

      The test would fail if {@code disable()} reverted to mutating {@code AggregateTable} directly + * (the pre-fix path) only via races -- not deterministically -- so the assertions here are about + * the observable end-to-end shape rather than thread identity. + */ +class ConflatingMetricsAggregatorDisableTest { + + @Test + void downgradeRoutesClearThroughInboxBeforeNextReport() throws Exception { + HealthMetrics healthMetrics = mock(HealthMetrics.class); + MetricWriter writer = mock(MetricWriter.class); + Sink sink = mock(Sink.class); + DDAgentFeaturesDiscovery features = mock(DDAgentFeaturesDiscovery.class); + when(features.supportsMetrics()).thenReturn(true); + when(features.peerTags()).thenReturn(Collections.emptySet()); + when(features.getLastTimeDiscovered()).thenReturn(1L); + + ConflatingMetricsAggregator aggregator = + new ConflatingMetricsAggregator( + Collections.emptySet(), + features, + healthMetrics, + sink, + writer, + /* maxAggregates */ 16, + /* queueSize */ 64, + /* reportingInterval */ 10, + SECONDS, + /* includeEndpointInMetrics */ false); + aggregator.start(); + try { + // Baseline: publish a span, run a report, verify the table flushes normally. This gives + // us a clean post-first-report state with the aggregator's reconcile already having fired + // once on the aggregator thread. + CountDownLatch firstFlush = new CountDownLatch(1); + org.mockito.Mockito.doAnswer( + invocation -> { + firstFlush.countDown(); + return null; + }) + .when(writer) + .finishBucket(); + + aggregator.publish(Collections.>singletonList(metricsEligibleSpan())); + aggregator.report(); + assertTrue(firstFlush.await(2, SECONDS)); + + // Reset writer-side mock interactions so the post-disable verify() blocks below only see + // what happens after the downgrade. features mock keeps accumulating call counts -- we use + // those counts as a latch on aggregator-thread reconcile timing. + reset(writer); + + // Flip the discovery state. disable()'s first action is features.discover() followed by a + // features.supportsMetrics() check; returning false here selects the clear path. + when(features.supportsMetrics()).thenReturn(false); + + // Fire DOWNGRADED on the test thread. This is the production scenario where the OkHttpSink + // callback thread triggers onEvent. disable() offers ClearSignal.CLEAR to the inbox but + // does not (and must not) mutate AggregateTable directly here. + aggregator.onEvent(EventListener.EventType.DOWNGRADED, ""); + + // First: verify nothing flushes immediately after disable. We can't pin reconcile-on-the- + // aggregator-thread as a latch here because CLEAR's inbox.clear() drops any REPORT we'd + // queue behind it -- so we just wait a window for any flush attempt to materialize. + verify(writer, after(500).never()).startBucket(anyInt(), anyLong(), anyLong()); + + // Stronger contract: prove the table is actually empty after CLEAR by re-enabling metrics + // and publishing a *marker* span with a distinct resource name. The next report should + // flush exactly one entry -- the marker -- with the original "resource" gone. If disable() + // had failed to clear the table (or had cleared it from the wrong thread and corrupted + // bucket chains), this assertion would catch it. + when(features.supportsMetrics()).thenReturn(true); + CountDownLatch postClearFlush = new CountDownLatch(1); + org.mockito.Mockito.doAnswer( + invocation -> { + postClearFlush.countDown(); + return null; + }) + .when(writer) + .finishBucket(); + aggregator.publish(Collections.>singletonList(markerSpan())); + aggregator.report(); + assertTrue(postClearFlush.await(2, SECONDS)); + + ArgumentCaptor entryCaptor = ArgumentCaptor.forClass(AggregateEntry.class); + verify(writer, times(1)).add(entryCaptor.capture()); + assertEquals( + "marker-resource", + entryCaptor.getValue().getResource().toString(), + "post-CLEAR bucket should contain only the marker -- the original entry was wiped"); + } finally { + aggregator.close(); + } + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private static CoreSpan metricsEligibleSpan() { + CoreSpan span = mock(CoreSpan.class); + when(span.isMeasured()).thenReturn(false); + when(span.isTopLevel()).thenReturn(true); + // Return true for any SpanKindFilter so peerTagSchemaFor enters the bootstrap path on the + // first publish. We want that bootstrap to fire (it's what makes features.getLastTimeDiscovered + // observable), even though peerTags() returns emptySet here and the resulting schema has + // size 0. + when(span.isKind(any(SpanKindFilter.class))).thenReturn(true); + when(span.getLongRunningVersion()).thenReturn(0); + when(span.getDurationNano()).thenReturn(100L); + when(span.getError()).thenReturn(0); + when(span.getResourceName()).thenReturn("resource"); + when(span.getServiceName()).thenReturn("svc"); + when(span.getOperationName()).thenReturn("op"); + when(span.getServiceNameSource()).thenReturn(null); + when(span.getType()).thenReturn("web"); + when(span.getHttpStatusCode()).thenReturn((short) 200); + when(span.getParentId()).thenReturn(0L); + when(span.getOrigin()).thenReturn(null); + when(span.unsafeGetTag(eq(Tags.SPAN_KIND), any(CharSequence.class))).thenReturn("client"); + return span; + } + + /** + * Distinct from {@link #metricsEligibleSpan()} via the resource name: post-CLEAR the writer + * should see "marker-resource", proving the original "resource" entry is gone from the table. + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static CoreSpan markerSpan() { + CoreSpan span = mock(CoreSpan.class); + when(span.isMeasured()).thenReturn(false); + when(span.isTopLevel()).thenReturn(true); + when(span.isKind(any(SpanKindFilter.class))).thenReturn(true); + when(span.getLongRunningVersion()).thenReturn(0); + when(span.getDurationNano()).thenReturn(100L); + when(span.getError()).thenReturn(0); + when(span.getResourceName()).thenReturn("marker-resource"); + when(span.getServiceName()).thenReturn("svc"); + when(span.getOperationName()).thenReturn("op"); + when(span.getServiceNameSource()).thenReturn(null); + when(span.getType()).thenReturn("web"); + when(span.getHttpStatusCode()).thenReturn((short) 200); + when(span.getParentId()).thenReturn(0L); + when(span.getOrigin()).thenReturn(null); + when(span.unsafeGetTag(eq(Tags.SPAN_KIND), any(CharSequence.class))).thenReturn("client"); + return span; + } +} From 5f73c2deb6dc02c06baca3ddc6f10bb3b5957925 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 08:42:24 -0400 Subject: [PATCH 61/70] Fix duplicate-entry bug for null-fielded SpanSnapshots The constructor canonicalizes null fields through canonicalize() which returns UTF8BytesString.EMPTY for null inputs (or a cached UTF8BytesString("") for empty-string inputs). But matches() compared those entries against subsequent snapshots via contentEquals(...) / stringContentEquals(...), which treated non-null UTF8BytesString vs null CharSequence as inequal. Result: two snapshots with the same null-valued resource/operation/ type/serviceSource hashed to the same bucket (intHash(null) == 0 == "".hashCode()), but matches() returned false on the EMPTY-vs-null field comparison, so the second snapshot inserted a *duplicate* entry into the table. Same path for empty-string vs null. Unify the semantics: null and length-zero are treated as equivalent on either side of contentEquals/stringContentEquals. The hash already agreed (intHash(null) == "".hashCode() == 0), so this restores the matches() contract to match the existing hash contract. Adds AggregateTableTest.nullAndEmptyOptionalFieldsCollapseToOneEntry to pin the contract: two null-fielded and one empty-string-fielded snapshot must all hit the same entry. Test would have failed before the fix (a duplicate insert) but the existing 10 cases still pass. Resolves sarahchen6's review comment on AggregateEntry.java:113 and amarziali's related concern on AggregateEntry.java:114. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 21 +++++++--- .../common/metrics/AggregateTableTest.java | 41 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 4755b26c1b2..f0a26c5d5b3 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -422,13 +422,21 @@ private static UTF8BytesString canonicalize( return cache.computeIfAbsent(charSeq.toString(), UTF8BytesString::create); } - /** UTF8 vs raw CharSequence content-equality, no allocation in the common (String) case. */ + /** + * UTF8 vs raw CharSequence content-equality, no allocation in the common (String) case. + * + *

      Treats {@code null} and empty (length 0) as equivalent on either side. This matches the + * canonicalization semantics: {@link #canonicalize} maps a {@code null} input to {@link + * UTF8BytesString#EMPTY}, so an entry built from a snapshot with a null field needs to match a + * subsequent snapshot whose field is still null. {@code intHash(null) == 0 == "".hashCode()}, so + * the hash already agrees with this view. + */ private static boolean contentEquals(UTF8BytesString a, CharSequence b) { if (a == null) { - return b == null; + return b == null || b.length() == 0; } if (b == null) { - return false; + return a.length() == 0; } // UTF8BytesString.toString() returns the underlying String -- O(1), no allocation. String aStr = a.toString(); @@ -443,9 +451,12 @@ private static boolean contentEquals(UTF8BytesString a, CharSequence b) { private static boolean stringContentEquals(UTF8BytesString a, String b) { if (a == null) { - return b == null; + return b == null || b.isEmpty(); + } + if (b == null) { + return a.length() == 0; } - return b != null && a.toString().equals(b); + return a.toString().equals(b); } /** diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java index 8e108902789..b5f22bd185d 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -184,6 +184,47 @@ void encodedLabelsAreBuiltOnInsert() { assertEquals("client", e.getSpanKind().toString()); } + @Test + void nullAndEmptyOptionalFieldsCollapseToOneEntry() { + // Regression: canonicalize() maps null -> EMPTY (or to a cache.computeIfAbsent("") entry for + // ""), but the prior contentEquals impl treated `non-null vs null` as not-equal -- so a second + // snapshot with the same null fields hashed to the same bucket but failed matches(), causing a + // spurious duplicate insert. The fix unifies null and length-zero on both sides of + // contentEquals/stringContentEquals. + AggregateTable table = new AggregateTable(8); + + SpanSnapshot snapNull = nullableSnapshot(null, null, null, null); + SpanSnapshot snapEmpty = nullableSnapshot("", "", "", ""); + + AggregateEntry first = table.findOrInsert(snapNull); + AggregateEntry secondNull = table.findOrInsert(nullableSnapshot(null, null, null, null)); + AggregateEntry forEmpty = table.findOrInsert(snapEmpty); + + assertSame(first, secondNull, "two null-fielded snapshots must hit the same entry"); + assertSame(first, forEmpty, "null- and empty-fielded snapshots must hit the same entry"); + assertEquals(1, table.size()); + } + + private static SpanSnapshot nullableSnapshot( + String resource, String operation, String type, String serviceNameSource) { + return new SpanSnapshot( + resource, + "svc", + operation, + serviceNameSource, + type, + (short) 200, + false, + true, + "client", + null, + null, + null, + null, + null, + 0L); + } + // ---------- helpers ---------- private static SpanSnapshot snapshot(String service, String operation, String spanKind) { From 9dddf0aac66f673533b1ab88620c1bd4ed77fe03 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 08:49:43 -0400 Subject: [PATCH 62/70] Clear dirty flag in ClearSignal handler After CLEAR runs the table is empty but dirty would still carry over from any prior SpanSnapshot insert. The next report() would see dirty=true, expunge no-op the empty table, find isEmpty(), and log "skipped metrics reporting because no points have changed" -- same observable outcome, but resetting dirty here keeps the invariant "dirty implies there's data to flush" honest. Resolves amarziali's review comment on Aggregator.java:121. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../main/java/datadog/trace/common/metrics/Aggregator.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index c6f407f382c..f1d74ee0f28 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -117,6 +117,11 @@ public void accept(InboxItem item) { if (!stopped) { aggregates.clear(); inbox.clear(); + // Clear dirty too -- without this, the next report() would see dirty=true, run + // expungeStaleAggregates against the (now-empty) table, find isEmpty()=true, and skip + // the flush anyway. Same observable outcome, but resetting here keeps the invariant + // "dirty implies there's data to flush" honest. + dirty = false; } ((SignalItem) item).complete(); } else if (item instanceof SignalItem) { From 80778c4e87c848d8d03aba0aa3b1de1649c13ec0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 08:53:43 -0400 Subject: [PATCH 63/70] Drop conditional null-skip from peer-tag hashing Previously hashOf wrapped the peer-tag contribution in `if (s.peerTagSchema != null && s.peerTagValues != null)`. That meant two snapshots with different null arrangements (schema-null vs values-null) collapsed to the same hash, getting resolved only by the field-by-field matches() fallback at the bucket walk -- wasteful, and the asymmetry hurt hash quality generally. Replace with unconditional contributions: - PeerTagSchema now overrides hashCode() to be content-based on names (lazy + cached, benign-race pattern matching UTF8BytesString / utf8Bytes elsewhere). addToHash(h, schema) routes through that. - For the String[] values, pass Arrays.hashCode(values) through the int overload -- Object[].hashCode() is identity-based by default, so we have to compute content hash explicitly. Null arrays hash to 0 via Arrays.hashCode's contract. Null inputs on either side now hash to 0 distinctly from any real schema or non-empty values array, so all four null combinations are distinguishable. Same final hash for content-equal inputs across schema replacements (the reconcile path), which preserves the entry- hit invariant after the aggregator rebuilds the schema. Resolves amarziali's review comment on AggregateEntry.java:309 and dougqh's suggestion on AggregateEntry.java:310. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 17 +++++------- .../trace/common/metrics/PeerTagSchema.java | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index f0a26c5d5b3..4531955799e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -306,16 +306,13 @@ static long hashOf(SpanSnapshot s) { h = LongHashingUtils.addToHash(h, s.synthetic); h = LongHashingUtils.addToHash(h, s.traceRoot); h = LongHashingUtils.addToHash(h, s.spanKind); - if (s.peerTagSchema != null && s.peerTagValues != null) { - String[] names = s.peerTagSchema.names; - String[] values = s.peerTagValues; - for (int i = 0; i < names.length; i++) { - if (values[i] != null) { - h = LongHashingUtils.addToHash(h, names[i]); - h = LongHashingUtils.addToHash(h, values[i]); - } - } - } + // Always mix in both the schema's content hash and the values' content hash, unconditionally + // (no null-skip). PeerTagSchema overrides hashCode() to be content-based on names; we use + // Arrays.hashCode for the String[] values since the default Object[].hashCode is identity- + // based, not content-based. Null inputs hash to 0 for both, distinct from any real schema's + // hash or any non-empty values array. + h = LongHashingUtils.addToHash(h, s.peerTagSchema); + h = LongHashingUtils.addToHash(h, Arrays.hashCode(s.peerTagValues)); h = LongHashingUtils.addToHash(h, s.httpMethod); h = LongHashingUtils.addToHash(h, s.httpEndpoint); h = LongHashingUtils.addToHash(h, s.grpcStatusCode); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java index 87a0b955f5f..5af81d929c0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java @@ -3,6 +3,7 @@ import static datadog.trace.api.DDTags.BASE_SERVICE; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import java.util.Arrays; import java.util.Set; /** @@ -53,6 +54,15 @@ final class PeerTagSchema { */ long lastTimeDiscovered; + /** + * Lazily computed content hash of {@link #names}, used as the bucket-distinguishing contribution + * when {@link AggregateEntry#hashOf} hashes a snapshot's peer-tag schema. Benign race pattern: a + * concurrent first-time read may recompute the value, but {@link Arrays#hashCode(Object[])} on + * the same content array is deterministic so the recomputed value matches. {@code int} writes are + * atomic per JLS. + */ + private int cachedHashCode; + private PeerTagSchema(String[] names, long lastTimeDiscovered) { this.names = names; this.lastTimeDiscovered = lastTimeDiscovered; @@ -93,4 +103,20 @@ boolean hasSameTagsAs(Set other) { int size() { return names.length; } + + /** + * Content-based hash of {@link #names}. Used by {@link AggregateEntry#hashOf} to incorporate the + * schema identity into a snapshot's lookup hash. Distinct schemas with the same names hash to the + * same value so an entry built under one schema instance still matches a snapshot pinned to a + * content-equal replacement (e.g. after reconcile rebuilds the schema). + */ + @Override + public int hashCode() { + int h = cachedHashCode; + if (h == 0) { + h = Arrays.hashCode(names); + cachedHashCode = h; + } + return h; + } } From 21e75452d66cb40da2732d4f94050d739d50e386 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 08:56:36 -0400 Subject: [PATCH 64/70] Delete dead Aggregator.clearAggregates() Once the ClearSignal routing replaced the direct disable()-to-table mutation, clearAggregates() lost all its call sites -- no production code, no test code. Worse, leaving it public invited future callers to bypass the ClearSignal contract and race against Drainer.accept on the aggregator thread. Drop the method outright. Update the inline comment in ConflatingMetricsAggregator.disable() to not name the deleted method. Resolves amarziali's review comment on Aggregator.java:82. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../main/java/datadog/trace/common/metrics/Aggregator.java | 4 ---- .../trace/common/metrics/ConflatingMetricsAggregator.java | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index f1d74ee0f28..5bfcf157ba7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -79,10 +79,6 @@ final class Aggregator implements Runnable { this.onReportCycle = onReportCycle; } - public void clearAggregates() { - this.aggregates.clear(); - } - @Override public void run() { Thread currentThread = Thread.currentThread(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java index 0151b4ce2f3..a8328319b3e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/ConflatingMetricsAggregator.java @@ -512,8 +512,8 @@ private void disable() { if (!features.supportsMetrics()) { log.debug("Disabling metric reporting because an agent downgrade was detected"); // Route the clear through the inbox so the aggregator thread is the only writer. - // AggregateTable is not thread-safe; calling clearAggregates() directly from this thread - // would race with Drainer.accept on the aggregator thread. + // AggregateTable is not thread-safe; mutating it directly from this thread would race + // with Drainer.accept on the aggregator thread. // // Best-effort single offer rather than the retry-loop pattern in report(). If the inbox is // full at downgrade time the clear is dropped, but the system self-heals: features.discover() From 877d95c4a9e42d37c06719907b2f7b7968b4b4e0 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 09:21:09 -0400 Subject: [PATCH 65/70] Cursor-resume eviction in AggregateTable via half-open MutatingTableIterator Previously AggregateTable.evictOneStale walked the bucket array from bucket 0 on every call. Under sustained cap pressure with mostly-hot entries clustered in low buckets, every eviction re-scanned the same hot prefix before finding a cold entry. amarziali's review concern. Add a cursor: after a successful eviction, remember the bucket where it landed. The next call resumes from there. Worst case for a single call is still O(N) when nearly every entry is hot, but a sustained eviction stream amortizes to O(1) per call -- the hot prefix is never re-scanned more than twice across N evictions. Implemented as two iterators driving [cursor, length) then [0, cursor), which required a small Hashtable.Support API addition: - New `mutatingTableIterator(buckets, startBucket, endBucket)` overload for walking a half-open bucket range. The existing zero-arg overload is kept; it now delegates to the new ctor with [0, buckets.length). - New `MutatingTableIterator.currentBucket()` accessor exposing the bucket index of the entry last returned by next() (or -1 before any next/after a remove). AggregateTable saves this as the new cursor. - The empty-range case (startBucket == endBucket) yields an immediately-exhausted iterator -- this is what makes the wrap-around pass [0, cursor) naturally produce nothing when cursor == 0, so the two-pass driver in evictOneStale needs no special case. Tests: - 4 new HashtableTest cases covering the half-open API, empty ranges, out-of-range bounds, and currentBucket() behavior before/after next. - 2 new AggregateTableTest cases: backToBackEvictionsAllSucceed (drives 3x capacity worth of cap-overrun inserts; each must succeed, which only holds if the cursor advances correctly) and clearResetsCursorForSubsequentEvictions (clear() also resets the cursor so subsequent eviction passes start from bucket 0). Resolves amarziali's review comment on AggregateTable.java:75. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateTable.java | 39 ++++++++---- .../common/metrics/AggregateTableTest.java | 44 ++++++++++++++ .../java/datadog/trace/util/Hashtable.java | 57 ++++++++++++++++-- .../datadog/trace/util/HashtableTest.java | 59 +++++++++++++++++++ 4 files changed, 183 insertions(+), 16 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java index 2255ca1cdf8..ffa6924f0ea 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateTable.java @@ -25,6 +25,13 @@ final class AggregateTable { private final int maxAggregates; private int size; + /** + * Bucket index where the last {@link #evictOneStale} successfully removed an entry. The next call + * resumes from this bucket so a fast-evicting workload doesn't repeatedly re-walk the same hot + * entries clustered near bucket 0. Reset to {@code 0} by {@link #clear}. + */ + private int evictCursor; + AggregateTable(int maxAggregates) { this.buckets = Support.create(maxAggregates, Support.MAX_RATIO); this.maxAggregates = maxAggregates; @@ -62,23 +69,34 @@ AggregateEntry findOrInsert(SpanSnapshot snapshot) { } /** - * Unlinks the first entry whose {@code getHitCount() == 0}. + * Unlinks the first entry whose {@code getHitCount() == 0}, resuming the scan from {@link + * #evictCursor} so back-to-back evictions amortize to O(1) per call. Worst case for a single call + * is still O(N) when nearly every entry is hot, but a sustained eviction stream never re-scans + * the hot prefix more than twice across N evictions. * - *

      O(N) per call -- scans buckets in array order from the start every time. That's a regression - * from the prior {@code LRUCache}'s O(1) LRU eviction, but the semantic change is deliberate: at - * cap with all entries live, we drop the new key (and report it via {@code - * onStatsAggregateDropped}) rather than evicting an established key. The expectation is that the - * cap is sized to the steady-state working set, so eviction is rare; if a future workload runs - * persistently at cap, this is the place to consider caching a cursor across calls so the scan - * resumes where it left off. + *

      The semantic intent: at cap with all entries live, drop the new key (reported via {@code + * onStatsAggregateDropped}) rather than evicting an established one. Cap is sized to the + * steady-state working set, so eviction is rare; this cursor optimization handles the + * pathological "persistently at cap" case. */ private boolean evictOneStale() { - for (MutatingTableIterator iter = Support.mutatingTableIterator(buckets); - iter.hasNext(); ) { + // Two passes -- [cursor, length) then [0, cursor) -- using the half-open-range iterator. The + // second pass is naturally empty when cursor==0, so no extra check needed. + return evictOneStaleInRange(evictCursor, buckets.length) + || evictOneStaleInRange(0, evictCursor); + } + + /** Scans {@code [startBucket, endBucket)} for the first stale entry and unlinks it. */ + private boolean evictOneStaleInRange(int startBucket, int endBucket) { + MutatingTableIterator iter = + Support.mutatingTableIterator(buckets, startBucket, endBucket); + while (iter.hasNext()) { AggregateEntry e = iter.next(); if (e.getHitCount() == 0) { + int bucket = iter.currentBucket(); iter.remove(); size--; + evictCursor = bucket; return true; } } @@ -113,5 +131,6 @@ void expungeStaleAggregates() { void clear() { Support.clear(buckets); size = 0; + evictCursor = 0; } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java index b5f22bd185d..12c9fd1de09 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -106,6 +106,50 @@ void capOverrunEvictsStaleEntry() { assertNotSame(stale, staleAgain); } + @Test + void backToBackEvictionsAllSucceed() { + // Cursor amortization regression: cap the table, fill with stale entries, then force a + // sequence of cap-overrun inserts. Each insert must succeed (evicting one stale entry and + // inserting one new). The cursor field is internal, but if it were ever wedged (e.g. + // pointing past the end of buckets, or not advancing after a successful eviction), some + // later insert would fail to find a stale entry. Drives ~3x the capacity worth of inserts to + // give wrap-around plenty of chances to misbehave. + AggregateTable table = new AggregateTable(8); + for (int i = 0; i < 8; i++) { + table.findOrInsert(snapshot("init-" + i, "op", "client")); + } + for (int i = 0; i < 32; i++) { + AggregateEntry inserted = table.findOrInsert(snapshot("post-" + i, "op", "client")); + assertNotNull( + inserted, "insert #" + i + " should evict a stale entry and succeed (table full)"); + } + assertEquals(8, table.size()); + } + + @Test + void clearResetsCursorForSubsequentEvictions() { + // The cursor must reset to 0 on clear so a re-filled table doesn't start eviction at a + // stale bucket index. Verified indirectly: clear and re-fill, then force an eviction; the + // newcomer must successfully take a slot (which only works if a stale entry was found). + AggregateTable table = new AggregateTable(4); + + // Fill, age, evict once -- cursor lands at some non-zero bucket + for (int i = 0; i < 4; i++) { + table.findOrInsert(snapshot("warm-" + i, "op", "client")); + } + table.findOrInsert(snapshot("evict-trigger", "op", "client")); + + table.clear(); + assertEquals(0, table.size()); + + // Re-fill, age, force eviction -- should still find a stale entry from bucket 0 onward + for (int i = 0; i < 4; i++) { + table.findOrInsert(snapshot("fresh-" + i, "op", "client")); + } + AggregateEntry newcomer = table.findOrInsert(snapshot("post-clear", "op", "client")); + assertNotNull(newcomer, "post-clear cap-overrun insert must succeed via cursor-reset evict"); + } + @Test void capOverrunWithNoStaleReturnsNull() { AggregateTable table = new AggregateTable(2); diff --git a/internal-api/src/main/java/datadog/trace/util/Hashtable.java b/internal-api/src/main/java/datadog/trace/util/Hashtable.java index 8f40e4609bc..ff3202c1f33 100644 --- a/internal-api/src/main/java/datadog/trace/util/Hashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/Hashtable.java @@ -482,7 +482,24 @@ MutatingBucketIterator mutatingBucketIterator( */ public static final MutatingTableIterator mutatingTableIterator(Hashtable.Entry[] buckets) { - return new MutatingTableIterator(buckets); + return new MutatingTableIterator(buckets, 0, buckets.length); + } + + /** + * Variant of {@link #mutatingTableIterator(Hashtable.Entry[])} that walks only the half-open + * bucket range {@code [startBucket, endBucket)}. Useful for resumable sweeps -- e.g. cursor- + * based eviction in {@code AggregateTable} -- where one call drives {@code [cursor, length)} + * and a wrap-around call drives {@code [0, cursor)}. The iterator does not wrap around + * within a single instance; callers compose two iterators when wrap-around is desired. An empty + * range ({@code startBucket == endBucket}) produces an immediately exhausted iterator. + * + * @param startBucket inclusive lower bound; must be in {@code [0, buckets.length]}. + * @param endBucket exclusive upper bound; must be in {@code [startBucket, buckets.length]}. + */ + public static final + MutatingTableIterator mutatingTableIterator( + Hashtable.Entry[] buckets, int startBucket, int endBucket) { + return new MutatingTableIterator(buckets, startBucket, endBucket); } public static final int bucketIndex(Object[] buckets, long keyHash) { @@ -752,6 +769,9 @@ public static final class MutatingTableIterator implements Iterator { private final Hashtable.Entry[] buckets; + /** Exclusive upper bound for bucket indices visited by this iterator. */ + private final int endBucket; + /** * Index of the bucket holding {@link #nextEntry} (or holding {@link #curEntry} after remove). */ @@ -782,9 +802,34 @@ public static final class MutatingTableIterator */ private Hashtable.Entry curEntry; - MutatingTableIterator(Hashtable.Entry[] buckets) { + MutatingTableIterator(Hashtable.Entry[] buckets, int startBucket, int endBucket) { this.buckets = buckets; - seekFromBucket(0); + if (startBucket < 0 || startBucket > buckets.length) { + throw new IndexOutOfBoundsException( + "startBucket " + startBucket + " out of range [0, " + buckets.length + "]"); + } + if (endBucket < startBucket || endBucket > buckets.length) { + throw new IndexOutOfBoundsException( + "endBucket " + + endBucket + + " out of range [" + + startBucket + + ", " + + buckets.length + + "]"); + } + this.endBucket = endBucket; + seekFromBucket(startBucket); + } + + /** + * Bucket index of the entry last returned by {@link #next()}, or {@code -1} if {@code next} has + * not yet been called or the most recent call was {@link #remove()}. Useful for callers driving + * a cursor — e.g. resumable eviction sweeps that want to remember where the last successful + * removal landed. + */ + public int currentBucket() { + return this.curBucketIndex; } @Override @@ -841,12 +886,12 @@ public void remove() { } /** - * Advance {@code nextBucketIndex} / {@code nextEntry} to the first non-empty bucket >= {@code - * from}. + * Advance {@code nextBucketIndex} / {@code nextEntry} to the first non-empty bucket {@code >= + * from} within {@code [0, endBucket)}. */ private void seekFromBucket(int from) { Hashtable.Entry[] thisBuckets = this.buckets; - for (int i = from; i < thisBuckets.length; i++) { + for (int i = from; i < this.endBucket; i++) { Hashtable.Entry head = thisBuckets[i]; if (head != null) { this.nextBucketIndex = i; diff --git a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java index 2992279be6d..953453ca3aa 100644 --- a/internal-api/src/test/java/datadog/trace/util/HashtableTest.java +++ b/internal-api/src/test/java/datadog/trace/util/HashtableTest.java @@ -349,5 +349,64 @@ void removeTwiceWithoutInterveningNextThrows() { it.remove(); assertThrows(IllegalStateException.class, it::remove); } + + @Test + void halfOpenRangeOmitsBucketsOutsideTheRange() { + // CollidingKey lets us pin entries to specific buckets via controlled hashCode. 16-slot + // table -> bucketIndex = hash & 15. Place entries in buckets 0, 5, and 10; iterate + // [5, 10) -- should see only bucket 5. + Hashtable.D1 table = new Hashtable.D1<>(16); + table.insert(new CollidingKeyEntry(new CollidingKey("b0", 0), 1)); + table.insert(new CollidingKeyEntry(new CollidingKey("b5", 5), 2)); + table.insert(new CollidingKeyEntry(new CollidingKey("b10", 10), 3)); + + Set seen = new HashSet<>(); + for (MutatingTableIterator it = + Support.mutatingTableIterator(table.buckets, 5, 10); + it.hasNext(); ) { + seen.add(it.next().key.label); + } + assertEquals(1, seen.size()); + assertTrue(seen.contains("b5")); + } + + @Test + void emptyHalfOpenRangeIsExhausted() { + // start == end -> immediately-exhausted iterator. Important: this is the wrap-around + // pass [0, cursor) when cursor == 0 in resumable sweeps. + Hashtable.D1 table = new Hashtable.D1<>(8); + table.insert(new StringIntEntry("a", 1)); + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets, 0, 0); + assertFalse(it.hasNext()); + } + + @Test + void rangeBoundsOutOfOrderThrows() { + Hashtable.D1 table = new Hashtable.D1<>(8); + assertThrows( + IndexOutOfBoundsException.class, + () -> Support.mutatingTableIterator(table.buckets, -1, 4)); + assertThrows( + IndexOutOfBoundsException.class, + () -> Support.mutatingTableIterator(table.buckets, 4, 2)); // end < start + assertThrows( + IndexOutOfBoundsException.class, + () -> + Support.mutatingTableIterator( + table.buckets, 0, table.buckets.length + 1)); // end > len + } + + @Test + void currentBucketReportsLandingIndex() { + // Pin one entry to a known bucket and check currentBucket() after next() reports that + // bucket. Before any next() (or after remove()), currentBucket() returns -1. + Hashtable.D1 table = new Hashtable.D1<>(16); + table.insert(new CollidingKeyEntry(new CollidingKey("b3", 3), 1)); + + MutatingTableIterator it = Support.mutatingTableIterator(table.buckets); + assertEquals(-1, it.currentBucket(), "before any next() currentBucket should be -1"); + it.next(); + assertEquals(3, it.currentBucket(), "currentBucket should report the entry's bucket"); + } } } From e2f2585a097b09ed21ac665034e1ee1bd088c7d9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 09:27:28 -0400 Subject: [PATCH 66/70] Move AggregateEntry.of() test factory out of production class dougqh's review comment on AggregateEntry.java:153 asked to keep test code out of the production class. Move the factory to a new AggregateEntries helper in src/test/java/datadog/trace/common/metrics. Same package so it can call the package-private forSnapshot(); delegating to forSnapshot also means no need to widen the AggregateEntry constructor visibility. The 37 src/test/groovy call sites get a mechanical rewrite of AggregateEntry.of(...) -> AggregateEntries.of(...) (36 in ConflatingMetricAggregatorTest, 1 in SerializingMetricWriterTest). src/traceAgentTest is a separate source set without compile-time visibility into src/test, so its 2 MetricsIntegrationTest.groovy call sites can't use AggregateEntries. Migrated those to construct a SpanSnapshot inline + call AggregateEntry.forSnapshot(snapshot). Groovy's permissive package-private access makes this work from the default package the integration test currently sits in. Resolves dougqh's review comment on AggregateEntry.java:153. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 60 --------------- .../ConflatingMetricAggregatorTest.groovy | 72 +++++++++--------- .../SerializingMetricWriterTest.groovy | 2 +- .../common/metrics/AggregateEntries.java | 76 +++++++++++++++++++ .../groovy/MetricsIntegrationTest.groovy | 14 +++- 5 files changed, 125 insertions(+), 99 deletions(-) create mode 100644 dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntries.java diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 4531955799e..9a2a71dc825 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -139,66 +139,6 @@ private AggregateEntry(SpanSnapshot s, long keyHash) { this.peerTags = materializePeerTags(this.peerTagNames, this.peerTagValues); } - /** - * Test-friendly factory mirroring the prior {@code new MetricKey(...)} positional args. Accepts a - * pre-encoded {@code List} of {@code "name:value"} peer tags and recovers the - * parallel-array {@code (names, values)} form by splitting on the {@code ':'} delimiter. - * - *

      Test-only. The split is at the first {@code ':'}, so peer-tag values - * containing a colon (URLs, IPv6 addresses, {@code service:env} patterns) will be silently - * misparsed and the recovered (name, value) pair will be wrong. Keep test data colon-free in - * peer-tag values, or wire production-style snapshots through {@link #forSnapshot(SpanSnapshot)} - * instead. - */ - static AggregateEntry of( - CharSequence resource, - CharSequence service, - CharSequence operationName, - @Nullable CharSequence serviceSource, - CharSequence type, - int httpStatusCode, - boolean synthetic, - boolean traceRoot, - CharSequence spanKind, - @Nullable List peerTags, - @Nullable CharSequence httpMethod, - @Nullable CharSequence httpEndpoint, - @Nullable CharSequence grpcStatusCode) { - PeerTagSchema schema = null; - String[] values = null; - if (peerTags != null && !peerTags.isEmpty()) { - String[] names = new String[peerTags.size()]; - values = new String[peerTags.size()]; - int i = 0; - for (UTF8BytesString t : peerTags) { - String s = t.toString(); - int colon = s.indexOf(':'); - names[i] = colon < 0 ? s : s.substring(0, colon); - values[i] = colon < 0 ? "" : s.substring(colon + 1); - i++; - } - schema = PeerTagSchema.testSchema(names); - } - SpanSnapshot synthetic_snapshot = - new SpanSnapshot( - resource, - service == null ? null : service.toString(), - operationName, - serviceSource, - type, - (short) httpStatusCode, - synthetic, - traceRoot, - spanKind == null ? null : spanKind.toString(), - schema, - values, - httpMethod == null ? null : httpMethod.toString(), - httpEndpoint == null ? null : httpEndpoint.toString(), - grpcStatusCode == null ? null : grpcStatusCode.toString(), - 0L); - return new AggregateEntry(synthetic_snapshot, hashOf(synthetic_snapshot)); - } - /** Construct from a snapshot at consumer-thread miss time. */ static AggregateEntry forSnapshot(SpanSnapshot s) { return new AggregateEntry(s, hashOf(s)); diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy index 0fa1ed2a2a2..9c5bfbec5e9 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/ConflatingMetricAggregatorTest.groovy @@ -119,7 +119,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( null, "service", "operation", @@ -165,7 +165,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -217,7 +217,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered == statsComputed (statsComputed ? 1 : 0) * writer.startBucket(1, _, _) (statsComputed ? 1 : 0) * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -294,7 +294,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { cycle1Triggered cycle2Triggered 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -312,7 +312,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { assert e.getHitCount() == 1 && e.getTopLevelCount() == 0 && e.getDuration() == 100 } 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -359,7 +359,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -411,7 +411,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -470,7 +470,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.finishBucket() >> { latch.countDown() } 1 * writer.startBucket(2, _, SECONDS.toNanos(reportingInterval)) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -487,7 +487,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == count && e.getDuration() == count * duration } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource2", "service2", "operation2", @@ -541,7 +541,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should aggregate into single metric" latchTriggered 1 * writer.startBucket(1, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -582,7 +582,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create separate metrics for each endpoint/method combination" latchTriggered2 1 * writer.startBucket(3, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -599,7 +599,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -616,7 +616,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration * 2 } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -680,7 +680,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create 4 separate metrics" latchTriggered 1 * writer.startBucket(4, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -697,7 +697,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -714,7 +714,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration * 2 } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -731,7 +731,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration * 3 } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -784,7 +784,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create separate metric keys for spans with and without HTTP tags" latchTriggered 1 * writer.startBucket(2, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -801,7 +801,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 1 && e.getDuration() == duration } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -852,7 +852,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: "should create the different metric keys for spans with and without sources" latchTriggered 1 * writer.startBucket(2, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -869,7 +869,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { )) >> { AggregateEntry e -> assert e.getHitCount() == 2 && e.getDuration() == 2 * duration } - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service", "operation", @@ -923,7 +923,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(10, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 10; ++i) { - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service" + i, "operation", @@ -941,7 +941,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { assert e.getHitCount() == 1 && e.getDuration() == duration } } - 0 * writer.add(AggregateEntry.of( + 0 * writer.add(AggregateEntries.of( "resource", "service10", "operation", @@ -1070,7 +1070,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service" + i, "operation", @@ -1105,7 +1105,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(4, _, SECONDS.toNanos(reportingInterval)) for (int i = 1; i < 5; ++i) { - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service" + i, "operation", @@ -1123,7 +1123,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { assert e.getHitCount() == 1 && e.getDuration() == duration } } - 0 * writer.add(AggregateEntry.of( + 0 * writer.add(AggregateEntries.of( "resource", "service0", "operation", @@ -1172,7 +1172,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(reportingInterval)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service" + i, "operation", @@ -1231,7 +1231,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(5, _, SECONDS.toNanos(1)) for (int i = 0; i < 5; ++i) { - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "resource", "service" + i, "operation", @@ -1398,7 +1398,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -1453,7 +1453,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(1, _, _) 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -1508,7 +1508,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { latchTriggered 1 * writer.startBucket(3, _, _) 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -1526,7 +1526,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 100 } 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -1544,7 +1544,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { assert e.getHitCount() == 1 && e.getTopLevelCount() == 1 && e.getDuration() == 200 } 1 * writer.add( - AggregateEntry.of( + AggregateEntries.of( "resource", "service", "operation", @@ -1596,7 +1596,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { then: latchTriggered 1 * writer.startBucket(3, _, _) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "grpc.service/Method", "service", "grpc.server", @@ -1611,7 +1611,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, "0" )) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "grpc.service/Method", "service", "grpc.server", @@ -1626,7 +1626,7 @@ class ConflatingMetricAggregatorTest extends DDSpecification { null, "5" )) - 1 * writer.add(AggregateEntry.of( + 1 * writer.add(AggregateEntries.of( "GET /api", "service", "http.request", diff --git a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy index 752cea028d1..03605dc5273 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/common/metrics/SerializingMetricWriterTest.groovy @@ -40,7 +40,7 @@ class SerializingMetricWriterTest extends DDSpecification { CharSequence httpEndpoint, CharSequence grpcStatusCode, int hitCount) { - AggregateEntry e = AggregateEntry.of( + AggregateEntry e = AggregateEntries.of( resource, service, operationName, serviceSource, type, httpStatusCode, synthetic, traceRoot, spanKind, peerTags, httpMethod, httpEndpoint, grpcStatusCode) diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntries.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntries.java new file mode 100644 index 00000000000..1208d88402a --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntries.java @@ -0,0 +1,76 @@ +package datadog.trace.common.metrics; + +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Test-side factories for {@link AggregateEntry}. Lives in {@code src/test} so the production class + * stays free of test-only API; same {@code datadog.trace.common.metrics} package so this helper can + * reach {@link AggregateEntry#forSnapshot(SpanSnapshot)} and the package-private {@link + * SpanSnapshot} constructor. + */ +public final class AggregateEntries { + private AggregateEntries() {} + + /** + * Builds an {@link AggregateEntry} from the same positional shape the prior {@code new + * MetricKey(...)} took. Accepts a pre-encoded {@code List} of {@code + * "name:value"} peer tags and recovers the parallel-array {@code (names, values)} form by + * splitting on the {@code ':'} delimiter. + * + *

      Test-only. The split is at the first {@code ':'}, so peer-tag values + * containing a colon (URLs, IPv6 addresses, {@code service:env} patterns) will be silently + * misparsed and the recovered (name, value) pair will be wrong. Keep test data colon-free in + * peer-tag values, or wire a production-style snapshot through {@link + * AggregateEntry#forSnapshot(SpanSnapshot)} directly instead. + */ + public static AggregateEntry of( + CharSequence resource, + CharSequence service, + CharSequence operationName, + @Nullable CharSequence serviceSource, + CharSequence type, + int httpStatusCode, + boolean synthetic, + boolean traceRoot, + CharSequence spanKind, + @Nullable List peerTags, + @Nullable CharSequence httpMethod, + @Nullable CharSequence httpEndpoint, + @Nullable CharSequence grpcStatusCode) { + PeerTagSchema schema = null; + String[] values = null; + if (peerTags != null && !peerTags.isEmpty()) { + String[] names = new String[peerTags.size()]; + values = new String[peerTags.size()]; + int i = 0; + for (UTF8BytesString t : peerTags) { + String s = t.toString(); + int colon = s.indexOf(':'); + names[i] = colon < 0 ? s : s.substring(0, colon); + values[i] = colon < 0 ? "" : s.substring(colon + 1); + i++; + } + schema = PeerTagSchema.testSchema(names); + } + SpanSnapshot syntheticSnapshot = + new SpanSnapshot( + resource, + service == null ? null : service.toString(), + operationName, + serviceSource, + type, + (short) httpStatusCode, + synthetic, + traceRoot, + spanKind == null ? null : spanKind.toString(), + schema, + values, + httpMethod == null ? null : httpMethod.toString(), + httpEndpoint == null ? null : httpEndpoint.toString(), + grpcStatusCode == null ? null : grpcStatusCode.toString(), + 0L); + return AggregateEntry.forSnapshot(syntheticSnapshot); + } +} diff --git a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy index 7afacc179cc..3cc703603e1 100644 --- a/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy +++ b/dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy @@ -11,7 +11,9 @@ import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString import datadog.trace.common.metrics.AggregateEntry import datadog.trace.common.metrics.EventListener import datadog.trace.common.metrics.OkHttpSink +import datadog.trace.common.metrics.PeerTagSchema import datadog.trace.common.metrics.SerializingMetricWriter +import datadog.trace.common.metrics.SpanSnapshot import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import okhttp3.HttpUrl @@ -37,10 +39,18 @@ class MetricsIntegrationTest extends AbstractTraceAgentTest { sink ) writer.startBucket(2, System.nanoTime(), SECONDS.toNanos(10)) - def entry1 = AggregateEntry.of("resource1", "service1", "operation1", null, "sql", 0, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) + // Build entries via SpanSnapshot directly: the test factory lives in src/test/java but this + // is the separate traceAgentTest source set, so we can't see it. Both entries use one peer + // tag (grault:quux) -> schema names=["grault"], values=["quux"]. + PeerTagSchema schema = PeerTagSchema.testSchema(["grault"] as String[]) + def entry1 = AggregateEntry.forSnapshot(new SpanSnapshot( + "resource1", "service1", "operation1", null, "sql", (short) 0, + false, true, "xyzzy", schema, ["quux"] as String[], null, null, null, 0L)) [2, 1, 2, 250, 4].each { entry1.recordOneDuration(it as long) } writer.add(entry1) - def entry2 = AggregateEntry.of("resource2", "service2", "operation2", null, "web", 200, false, true, "xyzzy", [UTF8BytesString.create("grault:quux")], null, null, null) + def entry2 = AggregateEntry.forSnapshot(new SpanSnapshot( + "resource2", "service2", "operation2", null, "web", (short) 200, + false, true, "xyzzy", schema, ["quux"] as String[], null, null, null, 0L)) [1, 1, 200, 2, 3, 4, 5, 6, 7, 8].each { entry2.recordOneDuration(it as long) } writer.add(entry2) writer.finishBucket() From 2536aa2e7619f7472c905c974329e5d1bba62672 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 09:33:49 -0400 Subject: [PATCH 67/70] Fix AggregateEntry equals/hashCode contract violation equals compared the pre-encoded peerTags List while hashCode (via hashOf) mixes in the raw peerTagSchema + values arrays. Two entries built from different schema layouts can collapse to the same encoded form -- e.g. tag "b" at index 1 in schema {a,b} with values {null,"x"} produces the same encoded ["b:x"] as schema {b,c} with values {"x",null}. equals returned true; hashCodes differed. Hashcode contract violated. Switch equals to compare the raw peerTagNames + peerTagValues arrays, mirroring matches(SpanSnapshot) and hashOf(SpanSnapshot). The production lookup path (AggregateTable.findOrInsert) already uses those, so this just brings equals in line with the rest of the class. Adds two regression tests on AggregateEntryTest: - equalsConsistentWithHashCodeAcrossDifferentSchemaLayouts: the failing-case shape above. Pre-fix, the encoded-list equals returned true while hashCodes differed; now equals returns false and the hashCodes differ in agreement. - equalEntriesHaveEqualHashCodes: positive case -- two entries from identical snapshots must equal and share hashCode. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 12 +++- .../common/metrics/AggregateEntryTest.java | 61 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 9a2a71dc825..8eb42340b30 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -320,6 +320,15 @@ List getPeerTags() { * Equality on the 13 label fields (not on the aggregate). Used only by test mock matchers; the * {@link Hashtable} does its own bucketing via {@link #keyHash} + {@link #matches(SpanSnapshot)} * and never calls {@code equals}. + * + *

      Peer tags are compared via the raw parallel arrays ({@code peerTagNames} and {@code + * peerTagValues}) rather than the pre-encoded {@code peerTags} list, so the equality contract + * stays consistent with {@link #hashCode()} (which goes through {@link #hashOf} -- driven off the + * raw arrays via {@link PeerTagSchema#hashCode} and {@link java.util.Arrays#hashCode}). Comparing + * the encoded list would let two entries with different raw layouts collapse to the same encoded + * form (e.g. tag {@code "b"} at index 1 in schema A vs index 0 in schema B, with matching values) + * and produce {@code equals=true} alongside different {@code hashCode}s -- violating the hashCode + * contract. */ @Override public boolean equals(Object o) { @@ -335,7 +344,8 @@ public boolean equals(Object o) { && Objects.equals(serviceSource, that.serviceSource) && Objects.equals(type, that.type) && Objects.equals(spanKind, that.spanKind) - && peerTags.equals(that.peerTags) + && Arrays.equals(peerTagNames, that.peerTagNames) + && Arrays.equals(peerTagValues, that.peerTagValues) && Objects.equals(httpMethod, that.httpMethod) && Objects.equals(httpEndpoint, that.httpEndpoint) && Objects.equals(grpcStatusCode, that.grpcStatusCode); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java index 578f3b753b8..42f2a15610e 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateEntryTest.java @@ -3,6 +3,7 @@ import static datadog.trace.common.metrics.AggregateEntry.ERROR_TAG; import static datadog.trace.common.metrics.AggregateEntry.TOP_LEVEL_TAG; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.metrics.agent.AgentMeter; @@ -81,6 +82,66 @@ void okAndErrorLatenciesTrackedSeparately() { assertTrue(entry.getOkLatencies().getMaxValue() <= 5); } + @Test + void equalsConsistentWithHashCodeAcrossDifferentSchemaLayouts() { + // Regression: equals() compared the pre-encoded peerTags list, but hashCode (via hashOf) + // mixes in the raw schema names + values arrays. Two entries built from different schema + // layouts could collapse to the same encoded peerTags ("b:x") while their raw arrays differ + // -- equals returned true but hashCodes differed, violating the hashCode contract. Now + // equals compares the raw arrays directly, mirroring matches()/hashOf(). + // + // Build two entries that exercise that exact shape: + // A: schema ["a","b"], values [null,"x"] -> encoded ["b:x"] + // B: schema ["b","c"], values ["x",null] -> encoded ["b:x"] + AggregateEntry a = + AggregateEntry.forSnapshot( + snapshotWithPeerTags(new String[] {"a", "b"}, new String[] {null, "x"})); + AggregateEntry b = + AggregateEntry.forSnapshot( + snapshotWithPeerTags(new String[] {"b", "c"}, new String[] {"x", null})); + + // Sanity: same encoded peer tags, despite different raw layout. + assertEquals(a.getPeerTags(), b.getPeerTags()); + + // Different raw layouts -> entries must not be equal. + assertNotEquals(a, b); + // And different hashCodes (matching the inequality). + assertNotEquals(a.hashCode(), b.hashCode()); + } + + @Test + void equalEntriesHaveEqualHashCodes() { + // Positive case: two entries built from identical snapshots must equal AND share hashCode. + AggregateEntry a = + AggregateEntry.forSnapshot( + snapshotWithPeerTags(new String[] {"a", "b"}, new String[] {null, "x"})); + AggregateEntry b = + AggregateEntry.forSnapshot( + snapshotWithPeerTags(new String[] {"a", "b"}, new String[] {null, "x"})); + + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + } + + private static SpanSnapshot snapshotWithPeerTags(String[] names, String[] values) { + return new SpanSnapshot( + "resource", + "svc", + "op", + null, + "type", + (short) 200, + false, + true, + "client", + PeerTagSchema.testSchema(names), + values, + null, + null, + null, + 0L); + } + private static AggregateEntry newEntry() { SpanSnapshot snapshot = new SpanSnapshot( From c0449a3677cc43f7aa66d5c794846573a4ae22c9 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 09:39:12 -0400 Subject: [PATCH 68/70] Don't trample queued STOP in ClearSignal handler Prior CLEAR handler called inbox.clear() as belt-and-suspenders cleanup of in-flight snapshots. That would also erase any STOP signal queued behind CLEAR -- a real concern in disable() -> close() sequences, where the trampled STOP leaves the aggregator thread spinning until thread.join's timeout. sarahchen6 surfaced this from a Codex pass on the CLEAR logic; dougqh confirmed it's worth fixing. The CLEAR handler now clears only the aggregates table. Queued snapshots will drain naturally into the just-cleared table -- but since features.supportsMetrics() is already false by the time CLEAR was offered, producers have stopped publishing; the inbox drains and empties on its own. Worst case: one extra reporting cycle of wasted work on stale snapshots that the agent rejects, which triggers another DOWNGRADED -> disable() -> CLEAR. Self-healing, same as before. Adds ConflatingMetricsAggregatorDisableTest.clearDoesNotTrampleQueuedStopSignal: publish a snapshot, fire DOWNGRADED, call close(); the test bounds close() with its own 2s timeout and asserts the thread exits within it. Pre-fix this would have hung out THREAD_JOIN_TIMEOUT_MS; post-fix it returns in milliseconds. Resolves sarahchen6/Codex's CLEAR-trampling-STOP review comment. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/Aggregator.java | 11 ++++- ...onflatingMetricsAggregatorDisableTest.java | 49 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java index 5bfcf157ba7..d809d452522 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/Aggregator.java @@ -110,9 +110,18 @@ public void accept(InboxItem item) { // AggregateTable directly) so the aggregator thread stays the sole writer. AggregateTable // is not thread-safe; a direct clear() from e.g. the OkHttpSink callback thread would // race with Drainer.accept on this thread. + // + // We deliberately do NOT call inbox.clear() here. Doing so would erase any queued STOP + // (or REPORT) signals that happen to sit behind CLEAR -- a real concern when a + // downgrade is followed quickly by close(), where the trampled STOP leaves the + // aggregator thread spinning until thread.join times out. features.supportsMetrics() is + // already false by the time CLEAR was offered, so producers have stopped publishing; + // any in-flight snapshots will drain naturally into the just-cleared table, get + // re-aggregated, and flushed on the next report -- where the agent rejects them again, + // triggering another DOWNGRADED -> disable() -> CLEAR cycle. Worst case: one extra + // reporting cycle of wasted work, which we accept for the safety of preserving STOP. if (!stopped) { aggregates.clear(); - inbox.clear(); // Clear dirty too -- without this, the next report() would see dirty=true, run // expungeStaleAggregates against the (now-empty) table, find isEmpty()=true, and skip // the flush anyway. Same observable outcome, but resetting here keeps the invariant diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java index 72ac8e6ff42..369b16e0c92 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/ConflatingMetricsAggregatorDisableTest.java @@ -135,6 +135,55 @@ void downgradeRoutesClearThroughInboxBeforeNextReport() throws Exception { } } + @Test + void clearDoesNotTrampleQueuedStopSignal() throws Exception { + // Regression: prior CLEAR handler called inbox.clear(), which would erase any STOP signal + // queued behind it. close() then waited out thread.join's timeout because Drainer never saw + // the STOP and `stopped` was never set. Now the CLEAR handler clears only the aggregates + // table; queued signals (STOP, REPORT) survive and get processed normally. + HealthMetrics healthMetrics = mock(HealthMetrics.class); + MetricWriter writer = mock(MetricWriter.class); + Sink sink = mock(Sink.class); + DDAgentFeaturesDiscovery features = mock(DDAgentFeaturesDiscovery.class); + when(features.supportsMetrics()).thenReturn(true); + when(features.peerTags()).thenReturn(Collections.emptySet()); + when(features.getLastTimeDiscovered()).thenReturn(1L); + + ConflatingMetricsAggregator aggregator = + new ConflatingMetricsAggregator( + Collections.emptySet(), + features, + healthMetrics, + sink, + writer, + /* maxAggregates */ 16, + /* queueSize */ 64, + /* reportingInterval */ 10, + SECONDS, + /* includeEndpointInMetrics */ false); + aggregator.start(); + + // Force at least one snapshot into the inbox so the aggregator has something to drain. + aggregator.publish(Collections.>singletonList(metricsEligibleSpan())); + + // Fire DOWNGRADED on this thread. disable() flips supportsMetrics() to false and offers + // CLEAR. Then immediately call close() which offers STOP. If CLEAR's handler clears the + // inbox, STOP gets trampled and close() hangs until the join timeout. + when(features.supportsMetrics()).thenReturn(false); + aggregator.onEvent(EventListener.EventType.DOWNGRADED, ""); + + // close() is synchronous; bound it ourselves rather than trusting THREAD_JOIN_TIMEOUT_MS. + long deadlineNanos = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(2); + Thread closer = new Thread(aggregator::close, "test-closer"); + closer.start(); + while (closer.isAlive() && System.nanoTime() < deadlineNanos) { + closer.join(50); + } + assertTrue( + !closer.isAlive(), + "close() must return promptly -- if CLEAR trampled STOP, this hangs out the join timeout"); + } + @SuppressWarnings({"rawtypes", "unchecked"}) private static CoreSpan metricsEligibleSpan() { CoreSpan span = mock(CoreSpan.class); From be134317442f1cec60eb647f4b8de5d61613770d Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 09:43:53 -0400 Subject: [PATCH 69/70] Implement PeerTagSchema.equals symmetric with hashCode The prior commit added a content-based hashCode() but left equals falling back to Object.equals (reference identity). That violates the hashCode contract for any caller that compares two distinct schema instances built from the same tag list -- e.g. before/after a reconcile rebuilds the cached schema with an unchanged tag set. equals() now mirrors hashCode(): content-equal on names. The reconcile- timing field lastTimeDiscovered is intentionally excluded from both -- it's bookkeeping for the aggregator's discovery-version compare, not part of schema identity. Tests: - equalsIsContentBasedOnNames -- same names, two instances, equal + matching hashCode. - equalsIgnoresLastTimeDiscovered -- pins that the bookkeeping field doesn't leak into identity. - equalsDistinguishesByOrder -- names is positional (pairs with SpanSnapshot.peerTagValues by index), so reordered schemas are not interchangeable. - equalsHandlesNullAndOtherTypes -- contract corners. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/PeerTagSchema.java | 16 ++++++++ .../common/metrics/PeerTagSchemaTest.java | 39 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java index 5af81d929c0..aae606dafa5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/PeerTagSchema.java @@ -119,4 +119,20 @@ public int hashCode() { } return h; } + + /** + * Content equality on {@link #names}. {@link #lastTimeDiscovered} is intentionally excluded: it + * is a reconcile-timing field, not part of the schema's identity. Two schemas built from the same + * tag list at different discovery snapshots represent the same schema. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PeerTagSchema)) { + return false; + } + return Arrays.equals(names, ((PeerTagSchema) o).names); + } } diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/PeerTagSchemaTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/PeerTagSchemaTest.java index 6b9f557d046..279df4f0384 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/PeerTagSchemaTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/PeerTagSchemaTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; @@ -84,4 +85,42 @@ void hasSameTagsAsHandlesEmpty() { assertTrue(empty.hasSameTagsAs(Collections.emptySet())); assertFalse(empty.hasSameTagsAs(Collections.singleton("peer.hostname"))); } + + @Test + void equalsIsContentBasedOnNames() { + PeerTagSchema a = PeerTagSchema.testSchema(new String[] {"peer.hostname", "peer.service"}); + PeerTagSchema b = PeerTagSchema.testSchema(new String[] {"peer.hostname", "peer.service"}); + + assertEquals(a, b); + assertEquals(b, a); + assertEquals(a.hashCode(), b.hashCode()); + } + + @Test + void equalsIgnoresLastTimeDiscovered() { + // lastTimeDiscovered is a reconcile-timing field, not part of schema identity. + PeerTagSchema early = PeerTagSchema.of(Collections.singleton("peer.hostname"), 100L); + PeerTagSchema late = PeerTagSchema.of(Collections.singleton("peer.hostname"), 999L); + + assertEquals(early, late); + assertEquals(early.hashCode(), late.hashCode()); + } + + @Test + void equalsDistinguishesByOrder() { + // names is positional -- the array index pairs with SpanSnapshot.peerTagValues. Schemas with + // the same tags in different positions are NOT interchangeable. + PeerTagSchema ab = PeerTagSchema.testSchema(new String[] {"a", "b"}); + PeerTagSchema ba = PeerTagSchema.testSchema(new String[] {"b", "a"}); + + assertNotEquals(ab, ba); + } + + @Test + void equalsHandlesNullAndOtherTypes() { + PeerTagSchema schema = PeerTagSchema.testSchema(new String[] {"peer.hostname"}); + + assertNotEquals(schema, null); + assertNotEquals(schema, "peer.hostname"); + } } From d1749389b9ddea27ab6c972a25b0b5b054c18495 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 22 May 2026 10:12:58 -0400 Subject: [PATCH 70/70] Route service and spanKind through canonicalize for null-safety AggregateEntry's constructor canonicalized resource, operationName, type, and serviceSource (mapping null -> EMPTY via the canonicalize helper) but called SERVICE_CACHE.computeIfAbsent / SPAN_KIND_CACHE .computeIfAbsent directly for service and spanKind. Inputs of null would NPE on the cache call. Production paths never pass null for these -- DDSpan always supplies a service, and the producer defaults spanKind to "" via unsafeGetTag(SPAN_KIND, (CharSequence) "") -- so this is a latent- defense fix, not a live bug. But the matches/contentEquals logic already treats null and length-zero as equal on both sides, and every other label field in the constructor defends via canonicalize. Two unprotected outliers are an inconsistency that bites the next person who reaches for a new code path. Drops the Functions.UTF8_ENCODE import (its sole use was the service cache line) -- canonicalize internally creates the UTF8BytesString. Test: AggregateTableTest.nullServiceAndSpanKindDoNotNpeAndCollapseWithEmpty publishes (null, null), (null, null) again, and ("", ""); asserts a single entry results and that getService()/getSpanKind() are length-0. Without the fix, the first publish would have NPE'd at the .computeIfAbsent call. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../trace/common/metrics/AggregateEntry.java | 5 +-- .../common/metrics/AggregateTableTest.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java index 8eb42340b30..b493696c52b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/metrics/AggregateEntry.java @@ -1,6 +1,5 @@ package datadog.trace.common.metrics; -import static datadog.trace.api.Functions.UTF8_ENCODE; import static datadog.trace.bootstrap.instrumentation.api.UTF8BytesString.EMPTY; import datadog.metrics.api.Histogram; @@ -111,14 +110,14 @@ final class AggregateEntry extends Hashtable.Entry { private AggregateEntry(SpanSnapshot s, long keyHash) { super(keyHash); this.resource = canonicalize(RESOURCE_CACHE, s.resourceName); - this.service = SERVICE_CACHE.computeIfAbsent(s.serviceName, UTF8_ENCODE); + this.service = canonicalize(SERVICE_CACHE, s.serviceName); this.operationName = canonicalize(OPERATION_CACHE, s.operationName); this.serviceSource = s.serviceNameSource == null ? null : canonicalize(SERVICE_SOURCE_CACHE, s.serviceNameSource); this.type = canonicalize(TYPE_CACHE, s.spanType); - this.spanKind = SPAN_KIND_CACHE.computeIfAbsent(s.spanKind, UTF8BytesString::create); + this.spanKind = canonicalize(SPAN_KIND_CACHE, s.spanKind); this.httpMethod = s.httpMethod == null ? null diff --git a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java index 12c9fd1de09..42a5b98db39 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/metrics/AggregateTableTest.java @@ -249,6 +249,49 @@ void nullAndEmptyOptionalFieldsCollapseToOneEntry() { assertEquals(1, table.size()); } + @Test + void nullServiceAndSpanKindDoNotNpeAndCollapseWithEmpty() { + // Regression: serviceName and spanKind used to bypass canonicalize() and call + // cache.computeIfAbsent directly, which would NPE on a null input. Production paths never + // pass null for these (DDSpan always supplies a service; producer defaults spanKind to ""), + // but the matches/contentEquals logic already treats null-and-empty as equal, so the + // constructor should be consistent. This pins both null-safety and null-equals-empty + // behavior for the two fields that recently moved through canonicalize(). + AggregateTable table = new AggregateTable(8); + + SpanSnapshot allNulls = nullServiceKindSnapshot(null, null); + SpanSnapshot allEmpty = nullServiceKindSnapshot("", ""); + + AggregateEntry first = table.findOrInsert(allNulls); + AggregateEntry secondNull = table.findOrInsert(nullServiceKindSnapshot(null, null)); + AggregateEntry forEmpty = table.findOrInsert(allEmpty); + + assertSame(first, secondNull, "two null-service/-kind snapshots must hit the same entry"); + assertSame(first, forEmpty, "null- and empty-service/-kind snapshots must hit the same entry"); + assertEquals(1, table.size()); + assertEquals(0, first.getService().length(), "null serviceName should canonicalize to EMPTY"); + assertEquals(0, first.getSpanKind().length(), "null spanKind should canonicalize to EMPTY"); + } + + private static SpanSnapshot nullServiceKindSnapshot(String service, String spanKind) { + return new SpanSnapshot( + "resource", + service, + "op", + null, + "web", + (short) 200, + false, + true, + spanKind, + null, + null, + null, + null, + null, + 0L); + } + private static SpanSnapshot nullableSnapshot( String resource, String operation, String type, String serviceNameSource) { return new SpanSnapshot(