diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index 5109e01b..8860fbd6 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -37,7 +37,7 @@ public interface ILdAiConfigTracker /// /// Records at most once per Tracker; further calls are ignored. /// the duration in milliseconds - public void TrackDuration(float durationMs); + public void TrackDuration(double durationMs); /// /// Wraps a callable operation, measures its wall-clock duration, and records the result via diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index ab6c0d75..70a2182e 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.Diagnostics; using System.IO; +using System.Linq; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; @@ -47,6 +49,10 @@ public class LdAiConfigTracker : ILdAiConfigTracker private StrongBox _feedback; private StrongBox _trackedSuccess; // true = success, false = error + // Accumulates tool keys from TrackToolCall calls. Guarded by a lock since tool calls + // have no at-most-once semantics and may be called concurrently. + private readonly List _toolCallKeys = new(); + // Lazy caches the encoded token so repeated reads avoid re-encoding the immutable // payload. All resumption-token inputs are readonly for a tracker's lifetime. private readonly Lazy _resumptionToken; @@ -134,16 +140,29 @@ private string BuildResumptionToken() } /// - public MetricSummary Summary => new MetricSummary( - _durationMs?.Value, - _feedback?.Value, - _tokens?.Value, - _trackedSuccess?.Value, - _timeToFirstTokenMs?.Value - ); + public MetricSummary Summary + { + get + { + IReadOnlyList toolCalls; + lock (_toolCallKeys) + { + toolCalls = _toolCallKeys.Count > 0 ? _toolCallKeys.ToImmutableArray() : null; + } + return new MetricSummary( + _durationMs?.Value, + _feedback?.Value, + _tokens?.Value, + _trackedSuccess?.Value, + _timeToFirstTokenMs?.Value, + toolCalls, + ResumptionToken + ); + } + } /// - public void TrackDuration(float durationMs) + public void TrackDuration(double durationMs) { if (Interlocked.CompareExchange(ref _durationMs, new StrongBox(durationMs), null) != null) @@ -166,7 +185,7 @@ public async Task TrackDurationOf(Func> operation) finally { sw.Stop(); - TrackDuration((float)sw.Elapsed.TotalMilliseconds); + TrackDuration(sw.Elapsed.TotalMilliseconds); } } @@ -255,16 +274,32 @@ public async Task TrackMetricsOf(Func metricsExtractor, Func } catch (Exception) { + sw.Stop(); + TrackDuration(sw.Elapsed.TotalMilliseconds); TrackError(); throw; } - finally + + // Capture elapsed immediately so a slow extractor doesn't inflate the metric. + sw.Stop(); + var operationElapsedMs = sw.Elapsed.TotalMilliseconds; + + // Extractor failure: track duration but NOT error — the AI operation itself succeeded. + // Matches Java's LDAIConfigTrackerImpl.trackMetricsOf behavior. + AiMetrics metrics; + try { - sw.Stop(); - TrackDuration((float)sw.Elapsed.TotalMilliseconds); + metrics = metricsExtractor(result); + } + catch (Exception) + { + TrackDuration(operationElapsedMs); + throw; } - var metrics = metricsExtractor(result); + // Honor an explicit duration override from the caller; fall back to the measured value. + TrackDuration(metrics.DurationMs ?? operationElapsedMs); + if (metrics.Success) { TrackSuccess(); @@ -279,6 +314,11 @@ public async Task TrackMetricsOf(Func metricsExtractor, Func TrackTokens(metrics.Tokens.Value); } + if (metrics.ToolCalls?.Count > 0) + { + TrackToolCalls(metrics.ToolCalls); + } + return result; } @@ -359,6 +399,7 @@ public void TrackJudgeResult(JudgeResult result) /// public void TrackToolCall(string toolKey) { + lock (_toolCallKeys) { _toolCallKeys.Add(toolKey); } var data = MergeTrackData("toolKey", LdValue.Of(toolKey)); _client.Track(ToolCall, _context, data, 1); } diff --git a/pkgs/sdk/server-ai/src/Tracking/AiMetrics.cs b/pkgs/sdk/server-ai/src/Tracking/AiMetrics.cs index 75dd6605..2bb05396 100644 --- a/pkgs/sdk/server-ai/src/Tracking/AiMetrics.cs +++ b/pkgs/sdk/server-ai/src/Tracking/AiMetrics.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; + namespace LaunchDarkly.Sdk.Server.Ai.Tracking; /// @@ -16,14 +18,30 @@ public sealed record AiMetrics /// public readonly Usage? Tokens; + /// + /// Optional list of tool keys invoked during the operation. + /// + public readonly IReadOnlyList ToolCalls; + + /// + /// Optional duration override in milliseconds. When set, this value is used instead + /// of the stopwatch measurement in TrackMetricsOf. + /// + public readonly double? DurationMs; + /// /// Constructs an value. /// /// whether the operation succeeded /// optional token usage - public AiMetrics(bool success, Usage? tokens = null) + /// optional list of tool keys invoked + /// optional duration override in milliseconds + public AiMetrics(bool success, Usage? tokens = null, + IReadOnlyList toolCalls = null, double? durationMs = null) { Success = success; Tokens = tokens; + ToolCalls = toolCalls; + DurationMs = durationMs; } } diff --git a/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs b/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs index bfe79cf2..63cf09e7 100644 --- a/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs +++ b/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; + namespace LaunchDarkly.Sdk.Server.Ai.Tracking; /// @@ -8,10 +10,14 @@ namespace LaunchDarkly.Sdk.Server.Ai.Tracking; /// the token usage /// whether the generation was successful /// the time to first token in milliseconds +/// the tool keys invoked during the operation, or null if none were tracked +/// the resumption token for cross-process continuation public record struct MetricSummary( double? DurationMs, Feedback? Feedback, Usage? Tokens, bool? Success, - double? TimeToFirstTokenMs + double? TimeToFirstTokenMs, + IReadOnlyList ToolCalls = null, + string ResumptionToken = null ); diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 3750a9c7..0bd7793a 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -113,7 +113,7 @@ public void TrackDataIncludesRunId() var tracker = config.CreateTracker(); - tracker.TrackDuration(1.0f); + tracker.TrackDuration(1.0); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, It.Is(d => d.Get("runId").Type == LdValueType.String && @@ -130,7 +130,7 @@ public void CanTrackDuration() var config = BuildConfig(mockClient.Object, flagKey, context); var tracker = config.CreateTracker(); - tracker.TrackDuration(1.0f); + tracker.TrackDuration(1.0); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } @@ -368,8 +368,8 @@ public void DuplicateTrackDurationIsIgnored() var tracker = config.CreateTracker(); - tracker.TrackDuration(1.0f); - tracker.TrackDuration(2.0f); + tracker.TrackDuration(1.0); + tracker.TrackDuration(2.0); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, It.IsAny(), It.IsAny()), Times.Once); @@ -583,8 +583,8 @@ public void ConfigCreateTrackerReturnsNewTrackerWithFreshRunId() var tracker1 = config.CreateTracker(); var tracker2 = config.CreateTracker(); - tracker1.TrackDuration(10.0f); - tracker2.TrackDuration(20.0f); + tracker1.TrackDuration(10.0); + tracker2.TrackDuration(20.0); string runId1 = null; string runId2 = null; @@ -615,11 +615,11 @@ public void ConfigCreateTrackerReturnsTrackerWithIndependentTrackingState() // Track duration on the first tracker (exhausts at-most-once) var tracker1 = config.CreateTracker(); - tracker1.TrackDuration(1.0f); + tracker1.TrackDuration(1.0); // A second tracker has fresh tracking state var tracker2 = config.CreateTracker(); - tracker2.TrackDuration(2.0f); + tracker2.TrackDuration(2.0); // Both calls should have gone through mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, @@ -868,7 +868,7 @@ public void SummaryReflectsTrackedMetrics() Assert.Null(summary.TimeToFirstTokenMs); // Track some metrics - tracker.TrackDuration(100.5f); + tracker.TrackDuration(100.5); tracker.TrackTimeToFirstToken(10.2f); tracker.TrackFeedback(Feedback.Positive); tracker.TrackSuccess(); @@ -978,8 +978,8 @@ public void TrackDurationIsAtMostOnceUnderConcurrency() var tracker = config.CreateTracker(); var barrier = new Barrier(2); - var t1 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0f); }); - var t2 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0f); }); + var t1 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0); }); + var t2 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0); }); Task.WaitAll(t1, t2); var calls = mockClient.Invocations.Count(inv => @@ -1202,5 +1202,188 @@ public void DeprecatedShims_StillCallable() mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, It.IsAny(), It.IsAny()), Times.Once); } + + // ── AiMetrics new fields ──────────────────────────────────────────── + + [Fact] + public void AiMetrics_BackwardCompat_OldConstructorStillWorks() + { + var tokens = new Usage(10, 4, 6); + var metrics = new AiMetrics(true, tokens); + + Assert.True(metrics.Success); + Assert.Equal(tokens, metrics.Tokens); + Assert.Null(metrics.ToolCalls); + Assert.Null(metrics.DurationMs); + } + + [Fact] + public void AiMetrics_NewFields_PopulatedCorrectly() + { + var tokens = new Usage(10, 4, 6); + var toolCalls = new[] { "search", "fetch" }; + var metrics = new AiMetrics(true, tokens, toolCalls: toolCalls, durationMs: 42.0); + + Assert.True(metrics.Success); + Assert.Equal(tokens, metrics.Tokens); + Assert.Equal(toolCalls, metrics.ToolCalls); + Assert.Equal(42.0, metrics.DurationMs); + } + + // ── TrackMetricsOf tool calls and duration override ───────────────── + + [Fact] + public async Task TrackMetricsOf_AutoTracksToolCalls() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + await tracker.TrackMetricsOf( + _ => new AiMetrics(true, toolCalls: new[] { "search", "fetch" }), + () => Task.FromResult("response")); + + mockClient.Verify(x => x.Track( + "$ld:ai:tool_call", + context, + It.Is(d => d.Get("toolKey").AsString == "search"), + 1.0), Times.Once); + mockClient.Verify(x => x.Track( + "$ld:ai:tool_call", + context, + It.Is(d => d.Get("toolKey").AsString == "fetch"), + 1.0), Times.Once); + } + + [Fact] + public async Task TrackMetricsOf_DurationMsOverride() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + await tracker.TrackMetricsOf( + _ => new AiMetrics(true, durationMs: 999.0), + () => Task.FromResult("response")); + + mockClient.Verify(x => x.Track( + "$ld:ai:duration:total", + context, + It.IsAny(), + 999.0), Times.Once); + } + + [Fact] + public async Task TrackMetricsOf_ExtractorException_TracksDurationButNotError() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + await Assert.ThrowsAsync(async () => + await tracker.TrackMetricsOf( + _ => throw new InvalidOperationException("extractor boom"), + () => Task.FromResult("response"))); + + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, + It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task TrackMetricsOf_DurationMsNull_UsesStopwatch() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + await tracker.TrackMetricsOf( + _ => new AiMetrics(true), + () => Task.FromResult("response")); + + mockClient.Verify(x => x.Track( + "$ld:ai:duration:total", + context, + It.IsAny(), + It.Is(ms => ms >= 0)), Times.Once); + } + + // ── Summary ToolCalls and ResumptionToken ─────────────────────────── + + [Fact] + public void Summary_IncludesToolCalls() + { + var mockClient = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns((ILogger)null); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + tracker.TrackToolCall("search"); + tracker.TrackToolCall("fetch"); + + var summary = tracker.Summary; + Assert.NotNull(summary.ToolCalls); + Assert.Equal(2, summary.ToolCalls.Count); + Assert.Equal("search", summary.ToolCalls[0]); + Assert.Equal("fetch", summary.ToolCalls[1]); + } + + [Fact] + public void Summary_ToolCallsNullWhenNoneTracked() + { + var mockClient = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns((ILogger)null); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + Assert.Null(tracker.Summary.ToolCalls); + } + + [Fact] + public void Summary_IncludesResumptionToken() + { + var mockClient = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns((ILogger)null); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + var summary = tracker.Summary; + Assert.NotNull(summary.ResumptionToken); + Assert.Equal(tracker.ResumptionToken, summary.ResumptionToken); + } + + [Fact] + public void Summary_ToolCallsIsSnapshot_NotLiveView() + { + var mockClient = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns((ILogger)null); + var context = Context.New("key"); + var config = BuildConfig(mockClient.Object, "flag", context); + var tracker = config.CreateTracker(); + + tracker.TrackToolCall("search"); + var snapshot = tracker.Summary; + + tracker.TrackToolCall("fetch"); + + Assert.Equal(1, snapshot.ToolCalls.Count); + Assert.Equal("search", snapshot.ToolCalls[0]); + } } }