diff --git a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs index e6eee49d..5e4e3943 100644 --- a/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs +++ b/pkgs/sdk/server-ai/src/Config/ConfigFactory.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; using Mustache; @@ -21,11 +22,14 @@ internal sealed class ConfigFactory private readonly ILaunchDarklyClient _client; private readonly ILogger _logger; + private readonly Func _runnerFactory; - public ConfigFactory(ILaunchDarklyClient client, ILogger logger) + public ConfigFactory(ILaunchDarklyClient client, ILogger logger, + Func runnerFactory = null) { _client = client; _logger = logger; + _runnerFactory = runnerFactory; } public LdAiCompletionConfig BuildCompletionConfig( @@ -44,7 +48,7 @@ public LdAiCompletionConfig BuildCompletionConfig( _logger.Error( "AI Config '{0}': variation result is not an object (got {1}); using caller's default.", key, ldValue.Type); - return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); + return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate); } var (enabled, variationKey, version, mode) = ParseMeta(ldValue); @@ -54,7 +58,7 @@ public LdAiCompletionConfig BuildCompletionConfig( _logger.Warn( "AI Config mode mismatch for {0}: expected {1}, got {2}. Returning caller's default.", key, LdAiCompletionConfig.Mode, mode); - return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); + return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate); } var model = ParseModel(ldValue.Get("model")); @@ -64,6 +68,7 @@ public LdAiCompletionConfig BuildCompletionConfig( : ParseMessages(ldValue.Get("messages")); var tools = ParseTools(ldValue.Get("tools")); var judgeConfiguration = ParseJudgeConfiguration(ldValue.Get("judgeConfiguration")); + var evaluator = BuildEvaluator(judgeConfiguration, context, variables); return new LdAiCompletionConfig( key, @@ -75,7 +80,8 @@ public LdAiCompletionConfig BuildCompletionConfig( judgeConfiguration, model, provider, - trackerFactory); + trackerFactory, + evaluator); } private LdAiCompletionConfig BuildCompletionFromDefault( @@ -83,6 +89,8 @@ private LdAiCompletionConfig BuildCompletionFromDefault( LdAiCompletionConfigDefault defaultValue, IReadOnlyDictionary mergedVars, Func trackerFactory, + Context context, + IReadOnlyDictionary variables = null, bool interpolate = true) { // Caller-supplied default messages can contain Mustache templates too; interpolate @@ -90,6 +98,7 @@ private LdAiCompletionConfig BuildCompletionFromDefault( var messages = interpolate ? InterpolateMessages(defaultValue.Messages, mergedVars, key) : (defaultValue.Messages ?? new List()); + var evaluator = BuildEvaluator(defaultValue.JudgeConfiguration, context, variables); return new LdAiCompletionConfig( key, defaultValue.Enabled ?? true, @@ -100,7 +109,8 @@ private LdAiCompletionConfig BuildCompletionFromDefault( defaultValue.JudgeConfiguration, defaultValue.Model, defaultValue.Provider, - trackerFactory); + trackerFactory, + evaluator); } public LdAiAgentConfig BuildAgentConfig( @@ -120,7 +130,7 @@ public LdAiAgentConfig BuildAgentConfig( _logger.Error( "AI Config '{0}': variation result is not an object (got {1}); using caller's default.", key, ldValue.Type); - return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); + return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate, graphKey); } var (enabled, variationKey, version, mode) = ParseMeta(ldValue); @@ -130,7 +140,7 @@ public LdAiAgentConfig BuildAgentConfig( _logger.Warn( "AI Config mode mismatch for {0}: expected {1}, got {2}. Returning caller's default.", key, LdAiAgentConfig.Mode, mode); - return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate); + return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate, graphKey); } var model = ParseModel(ldValue.Get("model")); @@ -140,6 +150,7 @@ public LdAiAgentConfig BuildAgentConfig( ? InterpolateInstructions(ParseInstructions(ldValue.Get("instructions")), mergedVars, key) : ParseInstructions(ldValue.Get("instructions")); var judgeConfiguration = ParseJudgeConfiguration(ldValue.Get("judgeConfiguration")); + var evaluator = BuildEvaluator(judgeConfiguration, context, variables, graphKey); return new LdAiAgentConfig( key, @@ -151,7 +162,8 @@ public LdAiAgentConfig BuildAgentConfig( model, provider, judgeConfiguration, - trackerFactory); + trackerFactory, + evaluator); } internal LdAiAgentConfig BuildAgentFromDefault( @@ -159,11 +171,15 @@ internal LdAiAgentConfig BuildAgentFromDefault( LdAiAgentConfigDefault defaultValue, IReadOnlyDictionary mergedVars, Func trackerFactory, - bool interpolate = true) + Context context, + IReadOnlyDictionary variables = null, + bool interpolate = true, + string graphKey = null) { var instructions = interpolate ? InterpolateInstructions(defaultValue.Instructions, mergedVars, key) : defaultValue.Instructions; + var evaluator = BuildEvaluator(defaultValue.JudgeConfiguration, context, variables, graphKey); return new LdAiAgentConfig( key, defaultValue.Enabled ?? true, @@ -174,7 +190,8 @@ internal LdAiAgentConfig BuildAgentFromDefault( defaultValue.Model, defaultValue.Provider, defaultValue.JudgeConfiguration, - trackerFactory); + trackerFactory, + evaluator); } public LdAiJudgeConfig BuildJudgeConfig( @@ -183,10 +200,11 @@ public LdAiJudgeConfig BuildJudgeConfig( Context context, LdAiJudgeConfigDefault defaultValue, IReadOnlyDictionary variables, - bool interpolate = true) + bool interpolate = true, + string graphKey = null) { var mergedVars = interpolate ? MergeVariables(variables, context) : null; - var trackerFactory = TrackerFactoryFor(context); + var trackerFactory = TrackerFactoryFor(context, graphKey); if (ldValue.Type != LdValueType.Object) { @@ -247,6 +265,49 @@ private LdAiJudgeConfig BuildJudgeFromDefault( trackerFactory); } + private Evaluator BuildEvaluator(LdAiConfigTypes.JudgeConfiguration judgeConfiguration, Context context, + IReadOnlyDictionary variables = null, string graphKey = null) + { + if (_runnerFactory == null || judgeConfiguration == null || judgeConfiguration.Judges.Count == 0) + { + return Evaluator.Noop(); + } + + var judges = new Dictionary(); + foreach (var judgeEntry in judgeConfiguration.Judges) + { + try + { + var defaultValue = LdAiJudgeConfigDefault.Disabled; + var ldValue = _client.JsonVariation(judgeEntry.Key, context, defaultValue.ToLdValue()); + var judgeConfig = BuildJudgeConfig(judgeEntry.Key, ldValue, context, defaultValue, variables, graphKey: graphKey); + + if (!judgeConfig.Enabled) + { + _logger?.Debug("Judge '{0}' is disabled; skipping.", judgeEntry.Key); + continue; + } + + var runner = _runnerFactory(judgeConfig); + if (runner == null) + { + _logger?.Warn("Runner factory returned null for judge '{0}'; skipping.", judgeEntry.Key); + continue; + } + + judges[judgeEntry.Key] = new Judge(judgeConfig, runner, _logger); + } + catch (Exception ex) + { + _logger?.Warn("Failed to initialize judge '{0}': {1}", judgeEntry.Key, ex.Message); + } + } + + var filteredConfig = new LdAiConfigTypes.JudgeConfiguration( + judgeConfiguration.Judges.Where(j => judges.ContainsKey(j.Key)).ToList()); + return new Evaluator(judges, filteredConfig, _logger); + } + private string InterpolateInstructions( string instructions, IReadOnlyDictionary mergedVars, @@ -401,9 +462,13 @@ internal static LdAiConfigTypes.JudgeConfiguration ParseJudgeConfiguration(LdVal { var j = judgesArray.Get(i); if (j.Type != LdValueType.Object) continue; + var samplingRateValue = j.Get("samplingRate"); + double? samplingRate = samplingRateValue.Type == LdValueType.Number + ? samplingRateValue.AsDouble + : (double?)null; entries.Add(new LdAiConfigTypes.JudgeConfiguration.Judge( j.Get("key").AsString ?? "", - j.Get("samplingRate").AsDouble)); + samplingRate)); } return new LdAiConfigTypes.JudgeConfiguration(entries); } diff --git a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs index 152f021f..7d6c5faf 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; namespace LaunchDarkly.Sdk.Server.Ai.Config; @@ -46,8 +47,9 @@ internal LdAiAgentConfig( LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, LdAiConfigTypes.JudgeConfiguration judgeConfiguration, - Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + Func trackerFactory, + Evaluator evaluator = null) + : base(key, enabled, variationKey, version, model, provider, trackerFactory, evaluator) { Instructions = instructions; Tools = tools?.ToImmutableDictionary() ?? ImmutableDictionary.Empty; diff --git a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfigDefault.cs b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfigDefault.cs index ae6a2e57..8b59d3b5 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiAgentConfigDefault.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiAgentConfigDefault.cs @@ -192,11 +192,15 @@ internal LdValue ToLdValue() root["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary { { "judges", LdValue.ArrayFrom(JudgeConfiguration.Judges.Select(j => - LdValue.ObjectFrom(new Dictionary { - { "key", LdValue.Of(j.Key) }, - { "samplingRate", LdValue.Of(j.SamplingRate) } - }))) + var judgeFields = new Dictionary + { + { "key", LdValue.Of(j.Key) } + }; + if (j.SamplingRate.HasValue) + judgeFields["samplingRate"] = LdValue.Of(j.SamplingRate.Value); + return LdValue.ObjectFrom(judgeFields); + })) } }); } diff --git a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs index 92b48da7..5cdca287 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; namespace LaunchDarkly.Sdk.Server.Ai.Config; @@ -42,8 +43,9 @@ internal LdAiCompletionConfig(string key, bool enabled, string variationKey, int IEnumerable messages, IReadOnlyDictionary tools, LdAiConfigTypes.JudgeConfiguration judgeConfiguration, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, - Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + Func trackerFactory, + Evaluator evaluator = null) + : base(key, enabled, variationKey, version, model, provider, trackerFactory, evaluator) { Messages = messages?.ToList() ?? new List(); Tools = tools ?? ImmutableDictionary.Empty; diff --git a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfigDefault.cs b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfigDefault.cs index 1907cf4d..a79724ab 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfigDefault.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiCompletionConfigDefault.cs @@ -198,11 +198,15 @@ internal LdValue ToLdValue() root["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary { { "judges", LdValue.ArrayFrom(JudgeConfiguration.Judges.Select(j => - LdValue.ObjectFrom(new Dictionary { - { "key", LdValue.Of(j.Key) }, - { "samplingRate", LdValue.Of(j.SamplingRate) } - }))) + var judgeFields = new Dictionary + { + { "key", LdValue.Of(j.Key) } + }; + if (j.SamplingRate.HasValue) + judgeFields["samplingRate"] = LdValue.Of(j.SamplingRate.Value); + return LdValue.ObjectFrom(judgeFields); + })) } }); } diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs index 6a7bee19..53a59ab4 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; namespace LaunchDarkly.Sdk.Server.Ai.Config; @@ -40,6 +41,12 @@ public abstract class LdAiConfig /// internal int Version { get; } + /// + /// The evaluator attached to this config. Always non-null; use + /// when no real evaluation is configured. + /// + public Evaluator Evaluator { get; } + /// /// Factory that produces a tracker for the config. The factory is mode-agnostic — it /// operates only on the shared fields (, , @@ -66,7 +73,8 @@ private protected LdAiConfig( int version, LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, - Func trackerFactory) + Func trackerFactory, + Evaluator evaluator = null) { Key = key; Enabled = enabled; @@ -75,5 +83,6 @@ private protected LdAiConfig( Model = model ?? new LdAiConfigTypes.ModelConfig("", new Dictionary(), new Dictionary()); Provider = provider ?? new LdAiConfigTypes.ProviderConfig(""); _trackerFactory = trackerFactory ?? throw new ArgumentNullException(nameof(trackerFactory)); + Evaluator = evaluator ?? Evals.Evaluator.Noop(); } } diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs index a4278ab7..551e90a8 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs @@ -161,10 +161,11 @@ public sealed class Judge /// /// The fraction of requests that this judge evaluates, in the range [0, 1]. + /// When null, the judge evaluates 100% of requests. /// - public double SamplingRate { get; } + public double? SamplingRate { get; } - internal Judge(string key, double samplingRate) + internal Judge(string key, double? samplingRate) { Key = key; SamplingRate = samplingRate; diff --git a/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs index de32cb9d..155921e6 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; namespace LaunchDarkly.Sdk.Server.Ai.Config; @@ -41,7 +42,7 @@ internal LdAiJudgeConfig( LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider, Func trackerFactory) - : base(key, enabled, variationKey, version, model, provider, trackerFactory) + : base(key, enabled, variationKey, version, model, provider, trackerFactory, Evaluator.Noop()) { Messages = messages?.ToList() ?? new List(); EvaluationMetricKey = evaluationMetricKey; diff --git a/pkgs/sdk/server-ai/src/Evals/Evaluator.cs b/pkgs/sdk/server-ai/src/Evals/Evaluator.cs new file mode 100644 index 00000000..d86878b5 --- /dev/null +++ b/pkgs/sdk/server-ai/src/Evals/Evaluator.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; + +namespace LaunchDarkly.Sdk.Server.Ai.Evals; + +/// +/// Runs one or more instances against a single input/output pair and +/// collects their results. +/// +/// +/// +/// The evaluator does NOT call TrackJudgeResult — that is the responsibility of the +/// managed type (Plan D). Use when evaluation is not required. +/// +/// +public sealed class Evaluator +{ + private static readonly Evaluator _noop = new Evaluator(); + + private readonly IReadOnlyDictionary _judges; + private readonly LdAiConfigTypes.JudgeConfiguration _judgeConfiguration; + private readonly ILogger _logger; + private readonly bool _isNoop; + private readonly Random _random; + + private Evaluator() + { + _isNoop = true; + _judges = new Dictionary(); + _judgeConfiguration = null; + _logger = null; + } + + /// + /// Constructs an . + /// + /// a dictionary mapping judge keys to their instances + /// the judge configuration describing which judges to run and at + /// what sampling rate + /// optional logger for missing-judge warnings + public Evaluator( + IReadOnlyDictionary judges, + LdAiConfigTypes.JudgeConfiguration judgeConfiguration, + ILogger logger = null) + { + _judges = judges ?? new Dictionary(); + _judgeConfiguration = judgeConfiguration; + _logger = logger; + _isNoop = false; + _random = new Random(); + } + + /// + /// Returns an that performs no evaluation. + /// on the returned instance resolves immediately to an empty list without invoking any judges + /// or emitting any warnings. + /// + public static Evaluator Noop() => _noop; + + /// + /// Runs all configured judges against the given input/output pair. + /// + /// the original input (e.g. user prompt) + /// the model response to evaluate + /// a list of values, one per judge that was found and + /// executed; judges that are missing from the judges dictionary are skipped and + /// produce no entry + public async Task> EvaluateAsync(string input, string output) + { + if (_isNoop || _judgeConfiguration == null || _judgeConfiguration.Judges.Count == 0) + { + return new List(); + } + + var results = new List(_judgeConfiguration.Judges.Count); + foreach (var judgeEntry in _judgeConfiguration.Judges) + { + if (!_judges.TryGetValue(judgeEntry.Key, out var judge)) + { + _logger?.Warn("Evaluator: judge '{0}' not found; skipping", judgeEntry.Key); + continue; + } + + var result = await judge.EvaluateAsync(input, output, judgeEntry.SamplingRate, _random); + results.Add(result); + } + + return results; + } +} diff --git a/pkgs/sdk/server-ai/src/Evals/Judge.cs b/pkgs/sdk/server-ai/src/Evals/Judge.cs new file mode 100644 index 00000000..1595b0db --- /dev/null +++ b/pkgs/sdk/server-ai/src/Evals/Judge.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; + +namespace LaunchDarkly.Sdk.Server.Ai.Evals; + +/// +/// Executes a judge evaluation against a model provider using a configured +/// and an . +/// +public sealed class Judge +{ + private static readonly IReadOnlyDictionary EvaluationSchema = + new Dictionary + { + ["type"] = "object", + ["properties"] = new Dictionary + { + ["score"] = new Dictionary { ["type"] = "number" }, + ["reasoning"] = new Dictionary { ["type"] = "string" } + }, + ["required"] = new[] { "score", "reasoning" }, + ["additionalProperties"] = false + }; + + /// The judge's AI config. + public LdAiJudgeConfig Config { get; } + + /// The runner used to invoke the model. + public IRunner Runner { get; } + + private readonly ILogger _logger; + + /// + /// Constructs a . + /// + /// the judge AI config + /// the model runner + /// optional logger for warnings + public Judge(LdAiJudgeConfig config, IRunner runner, ILogger logger = null) + { + Config = config ?? throw new ArgumentNullException(nameof(config)); + Runner = runner ?? throw new ArgumentNullException(nameof(runner)); + _logger = logger; + } + + /// + /// Evaluates the given input/output pair using the judge's model. + /// + /// the original input (e.g. user prompt or message history) + /// the model response to evaluate + /// when provided, the fraction of requests to actually evaluate; + /// if a random draw exceeds this value the evaluation is skipped + /// optional Random instance for sampling; when null a new instance is created + /// a describing the evaluation outcome + public async Task EvaluateAsync(string input, string output, + double? samplingRate = null, Random random = null) + { + if (string.IsNullOrWhiteSpace(Config.EvaluationMetricKey)) + { + _logger?.Warn("Judge '{0}': missing evaluation metric key", Config.Key); + return new JudgeResult(sampled: true, success: false, judgeConfigKey: Config.Key, + errorMessage: "Judge configuration is missing required evaluation metric key"); + } + + var effectiveRate = samplingRate.HasValue ? NormalizeSamplingRate(samplingRate.Value) : 1.0; + if ((random ?? new Random()).NextDouble() > effectiveRate) + { + return new JudgeResult(sampled: false, judgeConfigKey: Config.Key); + } + + var formatted = $"MESSAGE HISTORY:\n{input}\n\nRESPONSE TO EVALUATE:\n{output}"; + var tracker = Config.CreateTracker(); + + RunnerResult result; + try + { + result = await tracker.TrackMetricsOf( + r => r.Metrics, + () => Runner.RunAsync(formatted, EvaluationSchema)); + } + catch (Exception ex) + { + return new JudgeResult(Config.EvaluationMetricKey, 0, + sampled: true, success: false, judgeConfigKey: Config.Key, + errorMessage: ex.Message); + } + + double score = 0; + string reasoning = null; + bool scoreExtracted = false; + + if (result?.Parsed != null) + { + if (result.Parsed.TryGetValue("score", out var rawScore)) + { + try + { + if (rawScore is System.Text.Json.JsonElement je && + je.ValueKind == System.Text.Json.JsonValueKind.Number) + { + score = je.GetDouble(); + scoreExtracted = true; + } + else + { + score = Convert.ToDouble(rawScore); + scoreExtracted = true; + } + } + catch { /* handled below */ } + } + + if (result.Parsed.TryGetValue("reasoning", out var rawReasoning)) + { + if (rawReasoning is string s) + { + reasoning = s; + } + else + { + _logger?.Warn( + "Judge '{0}': reasoning field is not a string; ignoring", Config.Key); + } + } + } + + if (!scoreExtracted) + { + var noScoreMsg = "Runner did not return a valid score"; + _logger?.Warn("Judge '{0}': {1}", Config.Key, noScoreMsg); + return new JudgeResult(Config.EvaluationMetricKey, 0, + sampled: true, success: false, judgeConfigKey: Config.Key, + errorMessage: noScoreMsg); + } + + if (double.IsNaN(score) || double.IsInfinity(score) || score < 0.0 || score > 1.0) + { + var msg = $"Score {score} is out of range [0, 1]"; + _logger?.Warn("Judge '{0}': {1}", Config.Key, msg); + return new JudgeResult(Config.EvaluationMetricKey, 0, + sampled: true, success: false, judgeConfigKey: Config.Key, + errorMessage: msg); + } + + return new JudgeResult(Config.EvaluationMetricKey, score, + sampled: true, success: true, judgeConfigKey: Config.Key, + reasoning: reasoning); + } + + /// + /// Evaluates the given conversation messages and runner result using the judge's model. + /// Messages are formatted as "role: content\n..." and the runner result's + /// is used as the output to evaluate. + /// + /// the conversation messages + /// the runner result whose content is to be evaluated + /// optional sampling rate; see + /// optional Random instance for sampling; forwarded to + /// a describing the evaluation outcome + public Task EvaluateMessagesAsync( + IReadOnlyList messages, + RunnerResult runnerResult, + double? samplingRate = null, + Random random = null) + { + var formattedMessages = messages == null + ? string.Empty + : string.Join("\n", messages.Select(m => $"{m.Role.ToString().ToLowerInvariant()}: {m.Content}")); + + return EvaluateAsync(formattedMessages, runnerResult?.Content ?? string.Empty, samplingRate, random); + } + + private static double NormalizeSamplingRate(double rate) + { + if (double.IsNaN(rate) || double.IsInfinity(rate)) return 1.0; + if (rate < 0.0) return 0.0; + if (rate > 1.0) return 1.0; + return rate; + } +} diff --git a/pkgs/sdk/server-ai/src/Evals/RunnerResult.cs b/pkgs/sdk/server-ai/src/Evals/RunnerResult.cs new file mode 100644 index 00000000..01bab35c --- /dev/null +++ b/pkgs/sdk/server-ai/src/Evals/RunnerResult.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using LaunchDarkly.Sdk.Server.Ai.Tracking; + +namespace LaunchDarkly.Sdk.Server.Ai.Evals; + +/// +/// The result of a model invocation performed by an . +/// +public sealed record RunnerResult( + /// + /// The response text from the model provider. + /// + string Content, + + /// + /// Success and token metrics for the invocation. + /// + AiMetrics Metrics, + + /// + /// The unmodified provider response. Optional; intended for advanced callers that need + /// access to provider-specific fields. + /// + object Raw = null, + + /// + /// Structured output parsed from the response when outputType was provided to + /// RunAsync. null when no output schema was requested. + /// + IReadOnlyDictionary Parsed = null +); diff --git a/pkgs/sdk/server-ai/src/Interfaces/IRunner.cs b/pkgs/sdk/server-ai/src/Interfaces/IRunner.cs new file mode 100644 index 00000000..1a6a185c --- /dev/null +++ b/pkgs/sdk/server-ai/src/Interfaces/IRunner.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using LaunchDarkly.Sdk.Server.Ai.Evals; + +namespace LaunchDarkly.Sdk.Server.Ai.Interfaces; + +/// +/// Represents a runner that can execute a prompt against a model provider and return a result. +/// +public interface IRunner +{ + /// + /// Executes the given input against the model provider. + /// + /// the prompt text to send to the model + /// optional JSON schema for structured output; when null the runner + /// returns free-form text + /// the result of the model invocation + Task RunAsync(string input, + IReadOnlyDictionary outputType = null); +} diff --git a/pkgs/sdk/server-ai/src/LdAiClient.cs b/pkgs/sdk/server-ai/src/LdAiClient.cs index 61fcd5ff..a079b66d 100644 --- a/pkgs/sdk/server-ai/src/LdAiClient.cs +++ b/pkgs/sdk/server-ai/src/LdAiClient.cs @@ -4,6 +4,7 @@ using System.Linq; using LaunchDarkly.Sdk.Server.Ai.Adapters; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Graph; using LaunchDarkly.Sdk.Server.Ai.Interfaces; @@ -41,10 +42,14 @@ public sealed class LdAiClient : ILdAiClient, ILdAiGraphClient /// /// /// an object satisfying , such as an - public LdAiClient(ILaunchDarklyClient client) + /// optional factory used to create an for each + /// judge when building an ; when null all configs receive + /// + public LdAiClient(ILaunchDarklyClient client, + Func runnerFactory = null) { _client = client ?? throw new ArgumentNullException(nameof(client)); - _factory = new ConfigFactory(_client, _client.GetLogger()); + _factory = new ConfigFactory(_client, _client.GetLogger(), runnerFactory); _client.Track( TrackSdkInfo, diff --git a/pkgs/sdk/server-ai/src/Tracking/JudgeResult.cs b/pkgs/sdk/server-ai/src/Tracking/JudgeResult.cs index d2f17539..8765460c 100644 --- a/pkgs/sdk/server-ai/src/Tracking/JudgeResult.cs +++ b/pkgs/sdk/server-ai/src/Tracking/JudgeResult.cs @@ -31,25 +31,41 @@ public sealed record JudgeResult /// public readonly string JudgeConfigKey; + /// + /// Error message when the evaluation failed. Present only when is false. + /// + public readonly string ErrorMessage; + + /// + /// The judge's reasoning behind the score, if provided by the runner. + /// + public readonly string Reasoning; + /// /// Constructs a . /// - /// the LaunchDarkly metric key - /// the numeric score - /// whether sampled; defaults to true - /// whether successful; defaults to true + /// the LaunchDarkly metric key; optional per spec + /// the numeric score; optional per spec + /// whether sampled; defaults to false + /// whether successful; defaults to false /// optional judge config key + /// optional error message when success is false + /// optional reasoning behind the score public JudgeResult( - string metricKey, - double score, - bool sampled = true, - bool success = true, - string judgeConfigKey = null) + string metricKey = null, + double score = 0.0, + bool sampled = false, + bool success = false, + string judgeConfigKey = null, + string errorMessage = null, + string reasoning = null) { MetricKey = metricKey; Score = score; Sampled = sampled; Success = success; JudgeConfigKey = judgeConfigKey; + ErrorMessage = errorMessage; + Reasoning = reasoning; } } diff --git a/pkgs/sdk/server-ai/test/EvaluatorTest.cs b/pkgs/sdk/server-ai/test/EvaluatorTest.cs new file mode 100644 index 00000000..4a0ebcfb --- /dev/null +++ b/pkgs/sdk/server-ai/test/EvaluatorTest.cs @@ -0,0 +1,281 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Evals; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; +using Moq; +using Xunit; + +namespace LaunchDarkly.Sdk.Server.Ai; + +public class EvaluatorTest +{ + private static LdAiJudgeConfig MakeJudgeConfig(string key, string metricKey, + Mock mockTracker) + { + return new LdAiJudgeConfig( + key, + enabled: true, + variationKey: "v1", + version: 1, + messages: new List(), + evaluationMetricKey: metricKey, + model: null, + provider: null, + trackerFactory: _ => mockTracker.Object); + } + + private static Mock MakeTrackerWithResult(RunnerResult result) + { + var mockTracker = new Mock(); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, System.Func>>( + (_, op) => op()); + return mockTracker; + } + + private static Mock MockRunner(RunnerResult result) + { + var mock = new Mock(); + mock.Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(result); + return mock; + } + + [Fact] + public async Task Noop_ReturnsEmptyListImmediately() + { + var evaluator = Evaluator.Noop(); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Empty(results); + } + + [Fact] + public async Task Noop_DoesNotInvokeAnyJudges() + { + var mockRunner = new Mock(); + var evaluator = Evaluator.Noop(); + + await evaluator.EvaluateAsync("input", "output"); + + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + } + + [Fact] + public async Task Noop_DoesNotLogWarnings() + { + var mockLogger = new Mock(); + var evaluator = Evaluator.Noop(); + + await evaluator.EvaluateAsync("input", "output"); + + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_RunsAllConfiguredJudges() + { + var runnerResult1 = new RunnerResult("ok1", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.8 }); + var runnerResult2 = new RunnerResult("ok2", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.6 }); + + var tracker1 = MakeTrackerWithResult(runnerResult1); + var tracker2 = MakeTrackerWithResult(runnerResult2); + + var judgeConfig1 = MakeJudgeConfig("judge-1", "metric-1", tracker1); + var judgeConfig2 = MakeJudgeConfig("judge-2", "metric-2", tracker2); + + var judges = new Dictionary + { + ["judge-1"] = new Judge(judgeConfig1, MockRunner(runnerResult1).Object), + ["judge-2"] = new Judge(judgeConfig2, MockRunner(runnerResult2).Object) + }; + + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-1", samplingRate: 1.0), + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-2", samplingRate: 1.0) + }); + + var evaluator = new Evaluator(judges, judgeConfiguration); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Equal(2, results.Count); + Assert.Contains(results, r => r.MetricKey == "metric-1" && r.Score == 0.8); + Assert.Contains(results, r => r.MetricKey == "metric-2" && r.Score == 0.6); + } + + [Fact] + public async Task EvaluateAsync_MissingJudge_LogsWarningAndSkips() + { + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.9 }); + var tracker = MakeTrackerWithResult(runnerResult); + var judgeConfig = MakeJudgeConfig("judge-found", "metric-found", tracker); + + var judges = new Dictionary + { + ["judge-found"] = new Judge(judgeConfig, MockRunner(runnerResult).Object) + }; + + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-found", samplingRate: 1.0), + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-missing", samplingRate: 1.0) + }); + + var mockLogger = new Mock(); + var evaluator = new Evaluator(judges, judgeConfiguration, mockLogger.Object); + var results = await evaluator.EvaluateAsync("input", "output"); + + // Only the found judge produces a result; the missing one does not + Assert.Single(results); + Assert.Equal("metric-found", results[0].MetricKey); + + mockLogger.Verify(x => x.Warn( + It.Is(s => s.Contains("judge-missing") || s.Contains("not found")), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_MissingJudge_ProducesNoEntryForMissingJudge() + { + var judges = new Dictionary(); + + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-missing", samplingRate: 1.0) + }); + + var mockLogger = new Mock(); + var evaluator = new Evaluator(judges, judgeConfiguration, mockLogger.Object); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Empty(results); + } + + [Fact] + public async Task EvaluateAsync_DoesNotCallTrackJudgeResult() + { + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + var mockTracker = MakeTrackerWithResult(runnerResult); + var judgeConfig = MakeJudgeConfig("judge-1", "metric-1", mockTracker); + var judges = new Dictionary + { + ["judge-1"] = new Judge(judgeConfig, MockRunner(runnerResult).Object) + }; + + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-1", samplingRate: 1.0) + }); + + var evaluator = new Evaluator(judges, judgeConfiguration); + await evaluator.EvaluateAsync("input", "output"); + + mockTracker.Verify(x => x.TrackJudgeResult(It.IsAny()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_NullJudgeConfiguration_ReturnsEmpty() + { + var evaluator = new Evaluator(new Dictionary(), judgeConfiguration: null); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Empty(results); + } + + [Fact] + public async Task EvaluateAsync_EmptyJudgeConfiguration_ReturnsEmpty() + { + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List()); + + var evaluator = new Evaluator(new Dictionary(), judgeConfiguration); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Empty(results); + } + + [Fact] + public async Task EvaluateAsync_NullSamplingRate_AlwaysEvaluates() + { + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.9 }); + var tracker = MakeTrackerWithResult(runnerResult); + var mockRunner = MockRunner(runnerResult); + var judgeConfig = MakeJudgeConfig("judge-1", "metric-1", tracker); + var judges = new Dictionary + { + ["judge-1"] = new Judge(judgeConfig, mockRunner.Object) + }; + + // samplingRate omitted (null) — should default to 1.0 and always run + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-1", samplingRate: null) + }); + + var evaluator = new Evaluator(judges, judgeConfiguration); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Single(results); + Assert.True(results[0].Success); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_ExplicitZeroSamplingRate_AlwaysSkips() + { + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.9 }); + var tracker = MakeTrackerWithResult(runnerResult); + var mockRunner = MockRunner(runnerResult); + var judgeConfig = MakeJudgeConfig("judge-1", "metric-1", tracker); + var judges = new Dictionary + { + ["judge-1"] = new Judge(judgeConfig, mockRunner.Object) + }; + + // samplingRate explicitly 0.0 — judge should be skipped every time + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge("judge-1", samplingRate: 0.0) + }); + + var evaluator = new Evaluator(judges, judgeConfiguration); + var results = await evaluator.EvaluateAsync("input", "output"); + + Assert.Single(results); + Assert.False(results[0].Sampled); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_SkippedJudge_DoesNotWarnAtEvalTime() + { + // Simulate fix 4: the judgeConfiguration passed to the Evaluator already has the + // skipped judge filtered out, so no "not found" warning should ever fire at eval time. + var judgeConfiguration = new LdAiConfigTypes.JudgeConfiguration( + new List()); + + var mockLogger = new Mock(); + var evaluator = new Evaluator(new Dictionary(), judgeConfiguration, mockLogger.Object); + await evaluator.EvaluateAsync("input", "output"); + + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Never); + } +} diff --git a/pkgs/sdk/server-ai/test/JudgeTest.cs b/pkgs/sdk/server-ai/test/JudgeTest.cs new file mode 100644 index 00000000..045c06a4 --- /dev/null +++ b/pkgs/sdk/server-ai/test/JudgeTest.cs @@ -0,0 +1,644 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Evals; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; +using Moq; +using Xunit; + +namespace LaunchDarkly.Sdk.Server.Ai; + +public class JudgeTest +{ + private static LdAiJudgeConfig MakeJudgeConfig( + Mock mockTracker, + string key = "my-judge", + string metricKey = "$ld:ai:judge:test") + { + return new LdAiJudgeConfig( + key, + enabled: true, + variationKey: "v1", + version: 1, + messages: new List(), + evaluationMetricKey: metricKey, + model: null, + provider: null, + trackerFactory: _ => mockTracker.Object); + } + + private static void SetupTrackMetricsOf(Mock mockTracker, RunnerResult toReturn) + { + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, Func>>( + (_, op) => op()); + } + + private static Mock MockRunner(RunnerResult result) + { + var mock = new Mock(); + mock.Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(result); + return mock; + } + + [Fact] + public async Task EvaluateAsync_SuccessfulEvaluation_ReturnsScoreAndReasoning() + { + var mockTracker = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary + { + ["score"] = 0.8, + ["reasoning"] = "Very relevant response" + }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("user input", "model output"); + + Assert.Equal("$ld:ai:judge:test", result.MetricKey); + Assert.Equal(0.8, result.Score); + Assert.True(result.Sampled); + Assert.True(result.Success); + Assert.Equal("my-judge", result.JudgeConfigKey); + Assert.Equal("Very relevant response", result.Reasoning); + Assert.Null(result.ErrorMessage); + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_SamplingRateExceeded_ReturnsSampledFalse() + { + var mockTracker = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var mockRunner = new Mock(); + var judge = new Judge(config, mockRunner.Object); + + // samplingRate = 0 means no evaluations ever pass + var result = await judge.EvaluateAsync("input", "output", samplingRate: 0.0); + + Assert.False(result.Sampled); + Assert.False(result.Success); + Assert.Equal(0.0, result.Score); + // Runner must not be called when sampling rejects + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + mockTracker.Verify(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_SamplingRateOne_AlwaysEvaluates() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + // samplingRate = 1.0: random double is always < 1.0, so always runs + var result = await judge.EvaluateAsync("input", "output", samplingRate: 1.0); + + Assert.True(result.Sampled); + Assert.True(result.Success); + } + + [Fact] + public async Task EvaluateAsync_FormatsInputCorrectly() + { + var mockTracker = new Mock(); + string capturedInput = null; + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner + .Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .Callback>((input, _) => capturedInput = input) + .ReturnsAsync(runnerResult); + + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, mockRunner.Object); + + await judge.EvaluateAsync("the user message", "the model response"); + + Assert.Equal( + "MESSAGE HISTORY:\nthe user message\n\nRESPONSE TO EVALUATE:\nthe model response", + capturedInput); + } + + [Fact] + public async Task EvaluateAsync_RunnerThrows_ReturnsSuccessFalseWithErrorMessage() + { + var mockTracker = new Mock(); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner + .Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ThrowsAsync(new InvalidOperationException("provider unavailable")); + + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, mockRunner.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.True(result.Sampled); + Assert.Equal(0.0, result.Score); + Assert.Equal("provider unavailable", result.ErrorMessage); + } + + [Fact] + public async Task EvaluateAsync_ScoreOutOfRange_LogsWarningAndReturnsSuccessFalse() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 1.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.Equal(0.0, result.Score); + Assert.Contains("out of range", result.ErrorMessage); + mockLogger.Verify(x => x.Warn( + It.IsAny(), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_NegativeScore_LogsWarningAndReturnsSuccessFalse() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = -0.1 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.Contains("out of range", result.ErrorMessage); + mockLogger.Verify(x => x.Warn( + It.IsAny(), + It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + [InlineData(double.NegativeInfinity)] + public async Task EvaluateAsync_NaNOrInfinityScore_LogsWarningAndReturnsSuccessFalse(double badScore) + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = badScore }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.Equal(0.0, result.Score); + Assert.NotNull(result.ErrorMessage); + mockLogger.Verify(x => x.Warn( + It.IsAny(), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_NonStringReasoning_LogsWarningAndSetsNull() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary + { + ["score"] = 0.7, + ["reasoning"] = 42 // not a string + }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(0.7, result.Score); + Assert.Null(result.Reasoning); + mockLogger.Verify(x => x.Warn( + It.Is(s => s.Contains("not a string")), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_DoesNotCallTrackJudgeResult() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + await judge.EvaluateAsync("input", "output"); + + mockTracker.Verify(x => x.TrackJudgeResult(It.IsAny()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_CreatesPerEvaluationTracker() + { + int trackerCreateCount = 0; + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + + var config = new LdAiJudgeConfig( + "my-judge", + enabled: true, + variationKey: "v1", + version: 1, + messages: new List(), + evaluationMetricKey: "metric", + model: null, + provider: null, + trackerFactory: _ => + { + trackerCreateCount++; + return mockTracker.Object; + }); + + var judge = new Judge(config, MockRunner(runnerResult).Object); + await judge.EvaluateAsync("input1", "output1"); + await judge.EvaluateAsync("input2", "output2"); + + Assert.Equal(2, trackerCreateCount); + } + + [Fact] + public async Task EvaluateAsync_WrapsRunnerInTrackMetricsOf() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + await judge.EvaluateAsync("input", "output"); + + mockTracker.Verify(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>()), Times.Once); + } + + [Fact] + public async Task EvaluateMessagesAsync_FormatsMessagesAndDelegates() + { + var mockTracker = new Mock(); + string capturedInput = null; + var runnerResult = new RunnerResult( + Content: "the response", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.6 }); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner + .Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .Callback>((input, _) => capturedInput = input) + .ReturnsAsync(runnerResult); + + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, mockRunner.Object); + var messages = new List + { + new LdAiConfigTypes.Message("Hello", LdAiConfigTypes.Role.User), + new LdAiConfigTypes.Message("Hi there", LdAiConfigTypes.Role.Assistant) + }; + + var result = await judge.EvaluateMessagesAsync(messages, runnerResult); + + Assert.True(result.Success); + // Messages formatted as "role: content\nrole: content" + Assert.Contains("user: Hello", capturedInput); + Assert.Contains("assistant: Hi there", capturedInput); + // Output is runnerResult.Content + Assert.Contains("the response", capturedInput); + } + + [Fact] + public async Task EvaluateAsync_MissingEvaluationMetricKey_ReturnsErrorResult() + { + var mockTracker = new Mock(); + var mockRunner = new Mock(); + var mockLogger = new Mock(); + + var config = MakeJudgeConfig(mockTracker, metricKey: null); + var judge = new Judge(config, mockRunner.Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Sampled); + Assert.False(result.Success); + Assert.Equal("my-judge", result.JudgeConfigKey); + Assert.Equal("Judge configuration is missing required evaluation metric key", result.ErrorMessage); + Assert.Null(result.MetricKey); + // Runner must not be called when metric key is missing + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + mockLogger.Verify(x => x.Warn( + It.Is(s => s.Contains("missing evaluation metric key")), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_NaNSamplingRate_TreatedAsFullRateAndEvaluates() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.5 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + // NaN normalizes to 1.0, so random > 1.0 is always false → always evaluates + var result = await judge.EvaluateAsync("input", "output", samplingRate: double.NaN); + + Assert.True(result.Sampled); + Assert.True(result.Success); + } + + [Fact] + public async Task EvaluateAsync_NegativeSamplingRate_TreatedAsZeroAndSkips() + { + var mockTracker = new Mock(); + var mockRunner = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, mockRunner.Object); + + // Negative normalizes to 0.0, so random > 0.0 is almost always true → always skips + var result = await judge.EvaluateAsync("input", "output", samplingRate: -0.5); + + Assert.False(result.Sampled); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + } + + [Fact] + public async Task EvaluateAsync_AboveOneSamplingRate_TreatedAsFullRateAndEvaluates() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.7 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + // >1 normalizes to 1.0, so random > 1.0 is always false → always evaluates + var result = await judge.EvaluateAsync("input", "output", samplingRate: 2.0); + + Assert.True(result.Sampled); + Assert.True(result.Success); + } + + [Fact] + public async Task EvaluateAsync_ScoreAsInt_ExtractsCorrectly() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = (int)1 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(1.0, result.Score); + } + + [Fact] + public async Task EvaluateAsync_ScoreAsLong_ExtractsCorrectly() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = (long)1 }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(1.0, result.Score); + } + + [Fact] + public async Task EvaluateAsync_ScoreAsFloat_ExtractsCorrectly() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = (float)0.5f }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(0.5, result.Score, precision: 5); + } + + [Fact] + public async Task EvaluateAsync_ScoreAsDecimal_ExtractsCorrectly() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = (decimal)0.7m }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(0.7, result.Score, precision: 5); + } + + [Fact] + public async Task EvaluateAsync_ScoreAsJsonElement_ExtractsCorrectly() + { + var mockTracker = new Mock(); + var jsonDoc = JsonDocument.Parse("{\"score\": 0.85}"); + var jsonElement = jsonDoc.RootElement.GetProperty("score"); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = jsonElement }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.True(result.Success); + Assert.Equal(0.85, result.Score, precision: 5); + } + + [Fact] + public async Task EvaluateAsync_NullParsed_ReturnsSuccessFalse() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: null); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.True(result.Sampled); + Assert.Equal(0.0, result.Score); + Assert.NotNull(result.ErrorMessage); + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_MissingScoreKey_ReturnsSuccessFalse() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["reasoning"] = "no score here" }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.True(result.Sampled); + Assert.Equal(0.0, result.Score); + Assert.NotNull(result.ErrorMessage); + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_NonNumericScore_ReturnsSuccessFalse() + { + var mockTracker = new Mock(); + var runnerResult = new RunnerResult( + Content: "ok", + Metrics: new AiMetrics(true), + Parsed: new Dictionary { ["score"] = "abc" }); + SetupTrackMetricsOf(mockTracker, runnerResult); + var mockLogger = new Mock(); + var config = MakeJudgeConfig(mockTracker); + var judge = new Judge(config, MockRunner(runnerResult).Object, mockLogger.Object); + + var result = await judge.EvaluateAsync("input", "output"); + + Assert.False(result.Success); + Assert.True(result.Sampled); + Assert.Equal(0.0, result.Score); + Assert.NotNull(result.ErrorMessage); + mockLogger.Verify(x => x.Warn(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task EvaluateAsync_WithSeededRandom_SamplingDecisionIsDeterministic() + { + // Use a seeded Random so the sampling decision is predictable without + // hardcoding a platform-specific float: draw from a fresh same-seed instance + // to know what the judge will draw, then choose a rate that will skip it. + const int seed = 1; + var expectedDraw = new Random(seed).NextDouble(); + var rateThatSkips = expectedDraw / 2.0; // draw > rate → skipped + + var mockTracker = new Mock(); + var runnerResult = new RunnerResult("ok", new AiMetrics(true)); + var config = MakeJudgeConfig(mockTracker); + var mockRunner = MockRunner(runnerResult); + var judge = new Judge(config, mockRunner.Object); + + var result = await judge.EvaluateAsync("input", "output", rateThatSkips, new Random(seed)); + + Assert.False(result.Sampled); + mockRunner.Verify( + x => x.RunAsync(It.IsAny(), It.IsAny>()), + Times.Never); + } +} diff --git a/pkgs/sdk/server-ai/test/LdAiAgentConfigTest.cs b/pkgs/sdk/server-ai/test/LdAiAgentConfigTest.cs index 3ba3098e..61dd4457 100644 --- a/pkgs/sdk/server-ai/test/LdAiAgentConfigTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiAgentConfigTest.cs @@ -1,6 +1,9 @@ using System.Collections.Generic; +using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; using Moq; using Xunit; @@ -300,4 +303,177 @@ public void BuildAgentConfig_InstructionsInterpolated() Assert.Equal("Hello Alice, you specialize in astronomy", result.Instructions); } + + [Fact] + public void Evaluator_JudgeMessages_InterpolatedWithCallerVariables() + { + // Regression test: before the fix, BuildEvaluator called BuildJudgeConfig with + // variables=null, so judge message templates only received ldctx. Caller-supplied + // prompt variables were silently dropped. + const string judgeKey = "my-judge"; + const string agentJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "agent"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {}, + "evaluationMetricKey": "$ld:ai:judge:test", + "messages": [ + {"content": "Evaluate for topic: {{topic}}", "role": "system"} + ] + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => x.JsonVariation("agent-foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(agentJson)); + mockClient.Setup(x => x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + LdAiJudgeConfig capturedJudgeConfig = null; + var client = new LdAiClient(mockClient.Object, runnerFactory: jc => + { + capturedJudgeConfig = jc; + return new Mock().Object; + }); + + var variables = new Dictionary { ["topic"] = "safety" }; + client.AgentConfig("agent-foo", Context.New("user"), variables: variables); + + Assert.NotNull(capturedJudgeConfig); + Assert.Single(capturedJudgeConfig.Messages); + Assert.Equal("Evaluate for topic: safety", capturedJudgeConfig.Messages[0].Content); + } + + [Fact] + public async Task Evaluator_IsReal_WhenDefaultPathTriggered() + { + // When the flag returns a non-object value, BuildAgentConfig falls through to + // BuildAgentFromDefault. After fix 3, that path now calls BuildEvaluator with + // the default's JudgeConfiguration, so a real evaluator is wired up. + const string judgeKey = "my-judge"; + const string judgeMetricKey = "$ld:ai:judge:test"; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {"name": "judge-model"}, + "provider": {"name": "openai"}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + // Return null (non-object) for the agent flag → forces the default path + mockClient.Setup(x => x.JsonVariation("agent-foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Null); + mockClient.Setup(x => x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + var mockTracker = new Mock(); + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.65 }); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, System.Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner.Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(runnerResult); + + var defaultConfig = LdAiAgentConfigDefault.New() + .SetJudgeConfiguration(new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge(judgeKey, samplingRate: 1.0) + })) + .Build(); + + var client = new LdAiClient(mockClient.Object, runnerFactory: _ => mockRunner.Object); + var agentConfig = client.AgentConfig("agent-foo", Context.New("user"), defaultConfig); + + Assert.NotNull(agentConfig.Evaluator); + + var evalResults = await agentConfig.Evaluator.EvaluateAsync("user input", "model output"); + + Assert.Single(evalResults); + Assert.Equal(judgeMetricKey, evalResults[0].MetricKey); + Assert.Equal(0.65, evalResults[0].Score); + Assert.True(evalResults[0].Success); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Once); + } + + [Fact] + public void GraphAgent_JudgeTrackerIncludesGraphKey() + { + const string judgeKey = "my-judge"; + const string agentJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "agent"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + LdAiJudgeConfig capturedJudgeConfig = null; + var factory = new ConfigFactory(mockClient.Object, mockLogger.Object, + runnerFactory: jc => + { + capturedJudgeConfig = jc; + return new Mock().Object; + }); + + var context = Context.New("user"); + factory.BuildAgentConfig( + "agent-foo", + LdValue.Parse(agentJson), + context, + LdAiAgentConfigDefault.Disabled, + variables: null, + graphKey: "my-graph"); + + Assert.NotNull(capturedJudgeConfig); + + // Fire a tracking event via the judge's tracker — it should include graphKey + var tracker = capturedJudgeConfig.CreateTracker(); + tracker.TrackSuccess(); + + mockClient.Verify(c => c.Track( + "$ld:ai:generation:success", + context, + It.Is(v => v.Get("graphKey").AsString == "my-graph"), + It.IsAny()), Times.Once); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiCompletionConfigTest.cs b/pkgs/sdk/server-ai/test/LdAiCompletionConfigTest.cs index fcf954a7..ccce09ed 100644 --- a/pkgs/sdk/server-ai/test/LdAiCompletionConfigTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiCompletionConfigTest.cs @@ -1,6 +1,10 @@ +using System.Collections.Generic; using System.Reflection; +using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.Evals; using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; using Moq; using Xunit; @@ -79,4 +83,306 @@ public void CreateTrackerIsNonNullWhenMessageTemplateIsMalformed() tracker.TrackSuccess(); mockClient.Verify(x => x.Track("$ld:ai:generation:success", It.IsAny(), It.IsAny(), 1.0f), Times.Once); } + + [Fact] + public void Evaluator_IsNoop_WhenNoRunnerFactory() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => + x.JsonVariation("foo", It.IsAny(), It.IsAny())).Returns(LdValue.Null); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + // No runnerFactory supplied → Evaluator should be Noop + var client = new LdAiClient(mockClient.Object); + var result = client.CompletionConfig("foo", Context.New("key")); + + Assert.NotNull(result.Evaluator); + // Noop evaluator returns empty list immediately + var evalResults = result.Evaluator.EvaluateAsync("input", "output").GetAwaiter().GetResult(); + Assert.Empty(evalResults); + } + + [Fact] + public async Task Evaluator_IsReal_WhenRunnerFactoryProvided() + { + const string judgeKey = "my-judge"; + const string judgeMetricKey = "$ld:ai:judge:test"; + const string completionJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "completion"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {"name": "judge-model"}, + "provider": {"name": "openai"}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => + x.JsonVariation("foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(completionJson)); + mockClient.Setup(x => + x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + // Runner factory returns a runner that always succeeds with score 0.9 + var mockTracker = new Mock(); + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.9 }); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, System.Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner.Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(runnerResult); + + var client = new LdAiClient(mockClient.Object, + runnerFactory: judgeConfig => + { + // Tracker factory on the judge config must be wired up from the client + return mockRunner.Object; + }); + + // Override the tracker on the judge config by hooking the mockClient + mockClient.Setup(x => + x.Track(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())); + + var completionConfig = client.CompletionConfig("foo", Context.New("user")); + + Assert.NotNull(completionConfig.Evaluator); + + // The evaluator is not Noop — it should actually use the runner + // We verify this by checking the runner gets called during evaluation + // (we can't easily distinguish Noop vs real without running it, so run it) + var evalResults = await completionConfig.Evaluator.EvaluateAsync("user input", "model output"); + + Assert.Single(evalResults); + Assert.Equal(judgeMetricKey, evalResults[0].MetricKey); + Assert.Equal(0.9, evalResults[0].Score); + Assert.True(evalResults[0].Success); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Once); + } + + [Fact] + public async Task Evaluator_SkipsJudge_WhenRunnerFactoryReturnsNull() + { + const string completionJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "completion"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => x.JsonVariation("foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(completionJson)); + mockClient.Setup(x => x.JsonVariation("my-judge", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + // Runner factory returns null → judge must be skipped, not throw + var client = new LdAiClient(mockClient.Object, runnerFactory: _ => null); + var completionConfig = client.CompletionConfig("foo", Context.New("user")); + + // Evaluator runs without throwing, producing an empty result (judge was skipped) + var results = await completionConfig.Evaluator.EvaluateAsync("input", "output"); + Assert.Empty(results); + + mockLogger.Verify(x => x.Warn( + It.Is(s => s.Contains("null") && s.Contains("skipping")), + It.IsAny()), Times.Once); + } + + [Fact] + public async Task Evaluator_SkipsJudge_WhenJudgeConfigIsDisabled() + { + const string completionJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "completion"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": false, "mode": "judge"}, + "model": {}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => x.JsonVariation("foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(completionJson)); + mockClient.Setup(x => x.JsonVariation("my-judge", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + var mockRunner = new Mock(); + var client = new LdAiClient(mockClient.Object, runnerFactory: _ => mockRunner.Object); + var completionConfig = client.CompletionConfig("foo", Context.New("user")); + + // Evaluator runs without throwing; disabled judge is skipped → empty results + var results = await completionConfig.Evaluator.EvaluateAsync("input", "output"); + Assert.Empty(results); + + // The runner must never be invoked for a disabled judge + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Never); + + mockLogger.Verify(x => x.Debug( + It.Is(s => s.Contains("disabled") && s.Contains("skipping")), + It.IsAny()), Times.Once); + mockLogger.Verify(x => x.Warn( + It.Is(s => s.Contains("disabled") && s.Contains("skipping")), + It.IsAny()), Times.Never); + } + + [Fact] + public void Evaluator_JudgeMessages_InterpolatedWithCallerVariables() + { + // Regression test: before the fix, BuildEvaluator called BuildJudgeConfig with + // variables=null, so judge message templates only received ldctx. Caller-supplied + // prompt variables were silently dropped. + const string judgeKey = "my-judge"; + const string completionJson = """ + { + "_ldMeta": {"variationKey": "v1", "enabled": true, "mode": "completion"}, + "model": {}, + "judgeConfiguration": { + "judges": [ + {"key": "my-judge", "samplingRate": 1.0} + ] + } + } + """; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {}, + "evaluationMetricKey": "$ld:ai:judge:test", + "messages": [ + {"content": "Evaluate for topic: {{topic}}", "role": "system"} + ] + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + mockClient.Setup(x => x.JsonVariation("foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(completionJson)); + mockClient.Setup(x => x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + LdAiJudgeConfig capturedJudgeConfig = null; + var client = new LdAiClient(mockClient.Object, runnerFactory: jc => + { + capturedJudgeConfig = jc; + return new Mock().Object; + }); + + var variables = new Dictionary { ["topic"] = "safety" }; + client.CompletionConfig("foo", Context.New("user"), variables: variables); + + Assert.NotNull(capturedJudgeConfig); + Assert.Single(capturedJudgeConfig.Messages); + Assert.Equal("Evaluate for topic: safety", capturedJudgeConfig.Messages[0].Content); + } + + [Fact] + public async Task Evaluator_IsReal_WhenDefaultPathTriggered() + { + // When the flag returns a non-object value, BuildCompletionConfig falls through to + // BuildCompletionFromDefault. After fix 3, that path now calls BuildEvaluator with + // the default's JudgeConfiguration, so a real evaluator is wired up. + const string judgeKey = "my-judge"; + const string judgeMetricKey = "$ld:ai:judge:test"; + const string judgeJson = """ + { + "_ldMeta": {"variationKey": "j1", "enabled": true, "mode": "judge"}, + "model": {"name": "judge-model"}, + "provider": {"name": "openai"}, + "evaluationMetricKey": "$ld:ai:judge:test" + } + """; + + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + // Return null (non-object) for the completion flag → forces the default path + mockClient.Setup(x => x.JsonVariation("foo", It.IsAny(), It.IsAny())) + .Returns(LdValue.Null); + mockClient.Setup(x => x.JsonVariation(judgeKey, It.IsAny(), It.IsAny())) + .Returns(LdValue.Parse(judgeJson)); + + var mockTracker = new Mock(); + var runnerResult = new RunnerResult("ok", new AiMetrics(true), + Parsed: new Dictionary { ["score"] = 0.75 }); + mockTracker + .Setup(x => x.TrackMetricsOf( + It.IsAny>(), + It.IsAny>>())) + .Returns, System.Func>>( + (_, op) => op()); + + var mockRunner = new Mock(); + mockRunner.Setup(x => x.RunAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(runnerResult); + + var judgeConfig = LdAiCompletionConfigDefault.New() + .SetJudgeConfiguration(new LdAiConfigTypes.JudgeConfiguration( + new List + { + new LdAiConfigTypes.JudgeConfiguration.Judge(judgeKey, samplingRate: 1.0) + })) + .Build(); + + var client = new LdAiClient(mockClient.Object, runnerFactory: _ => mockRunner.Object); + var completionConfig = client.CompletionConfig("foo", Context.New("user"), judgeConfig); + + Assert.NotNull(completionConfig.Evaluator); + + var evalResults = await completionConfig.Evaluator.EvaluateAsync("user input", "model output"); + + Assert.Single(evalResults); + Assert.Equal(judgeMetricKey, evalResults[0].MetricKey); + Assert.Equal(0.75, evalResults[0].Score); + Assert.True(evalResults[0].Success); + mockRunner.Verify(x => x.RunAsync(It.IsAny(), It.IsAny>()), Times.Once); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 3750a9c7..f7645d3b 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -1124,6 +1124,8 @@ public void TrackJudgeResult_SuccessPath_EmitsCorrectEvent() tracker.TrackJudgeResult(new JudgeResult( metricKey: "$ld:ai:judge:relevance", score: 0.85, + sampled: true, + success: true, judgeConfigKey: "my-judge")); mockClient.Verify(x => x.Track( diff --git a/pkgs/sdk/server-ai/test/LdAiJudgeConfigTest.cs b/pkgs/sdk/server-ai/test/LdAiJudgeConfigTest.cs index 94a618bb..6baf03ff 100644 --- a/pkgs/sdk/server-ai/test/LdAiJudgeConfigTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiJudgeConfigTest.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.Interfaces; +using LaunchDarkly.Sdk.Server.Ai.Tracking; using Moq; using Xunit; @@ -162,6 +163,61 @@ public void BuildJudgeConfig_NonObjectVariation_ReturnsCallerDefault() ), Times.Once); } + [Fact] + public void JudgeResult_BackwardCompat_OldConstructorStillWorks() + { + // Five-argument form (no ErrorMessage/Reasoning) must still compile and work. + var result = new JudgeResult("my-metric", 0.75, sampled: true, success: true, judgeConfigKey: "j1"); + + Assert.Equal("my-metric", result.MetricKey); + Assert.Equal(0.75, result.Score); + Assert.True(result.Sampled); + Assert.True(result.Success); + Assert.Equal("j1", result.JudgeConfigKey); + Assert.Null(result.ErrorMessage); + Assert.Null(result.Reasoning); + } + + [Fact] + public void JudgeResult_ErrorMessageRoundTrip() + { + var result = new JudgeResult("metric", 0, success: false, errorMessage: "provider down"); + + Assert.False(result.Success); + Assert.Equal("provider down", result.ErrorMessage); + Assert.Null(result.Reasoning); + } + + [Fact] + public void JudgeResult_ReasoningRoundTrip() + { + var result = new JudgeResult("metric", 0.9, sampled: true, success: true, reasoning: "Very coherent"); + + Assert.True(result.Success); + Assert.Equal("Very coherent", result.Reasoning); + Assert.Null(result.ErrorMessage); + } + + [Fact] + public void JudgeResult_AllFieldsSet() + { + var result = new JudgeResult( + "metric", 0.5, + sampled: true, + success: false, + judgeConfigKey: "jk", + errorMessage: "bad input", + reasoning: "n/a"); + + Assert.Equal("metric", result.MetricKey); + Assert.Equal(0.5, result.Score); + Assert.True(result.Sampled); + Assert.False(result.Success); + Assert.Equal("jk", result.JudgeConfigKey); + Assert.Equal("bad input", result.ErrorMessage); + Assert.Equal("n/a", result.Reasoning); + } + [Fact] public void DefaultTypes_BehaviorAndToLdValue() {