From 819d2b5eef58c458ceefaa0c055b72f33ba940b3 Mon Sep 17 00:00:00 2001 From: Matt McCarthy Date: Mon, 6 Jul 2026 17:37:52 -0500 Subject: [PATCH] feat: wire judge evaluators via AIRunnerProvider in LDAIClientImpl --- .../sdk/server/ai/AIRunnerProvider.java | 23 +++ .../sdk/server/ai/LDAIClientImpl.java | 74 ++++++++- .../sdk/server/ai/LDAIClientImplTest.java | 144 ++++++++++++++++++ 3 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/AIRunnerProvider.java diff --git a/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/AIRunnerProvider.java b/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/AIRunnerProvider.java new file mode 100644 index 00000000..a7fd8b9f --- /dev/null +++ b/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/AIRunnerProvider.java @@ -0,0 +1,23 @@ +package com.launchdarkly.sdk.server.ai; + +/** + * Supplies a {@link Runner} for a given judge AI Config. + *

+ * Implement this interface and pass it to + * {@link LDAIClientImpl#LDAIClientImpl(com.launchdarkly.sdk.server.interfaces.LDClientInterface, + * com.launchdarkly.logging.LDLogger, AIRunnerProvider)} to enable online evaluation (judges). + * The client calls {@link #create} once per judge key when building an {@link Evaluator} for a + * completion or agent config that carries a {@code judgeConfiguration}. + *

+ * Return {@code null} to skip a particular judge. Implementations should be thread-safe. + */ +@FunctionalInterface +public interface AIRunnerProvider { + /** + * Creates a {@link Runner} for the given judge AI Config. + * + * @param judgeConfig the judge AI Config for which a runner is needed; never {@code null} + * @return a runner to use for this judge, or {@code null} to skip this judge + */ + Runner create(AIJudgeConfig judgeConfig); +} diff --git a/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java b/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java index b6df8bed..a50520d5 100644 --- a/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java +++ b/lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java @@ -5,6 +5,7 @@ import com.launchdarkly.sdk.LDContext; import com.launchdarkly.sdk.LDValue; import com.launchdarkly.sdk.LDValueType; +import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.JudgeConfiguration; import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Message; import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Mode; import com.launchdarkly.sdk.server.ai.internal.AgentGraphFlagValue; @@ -62,6 +63,7 @@ public final class LDAIClientImpl implements LDAIClient { private final LDClientInterface client; private final LDLogger logger; private final Interpolator interpolator; + private final AIRunnerProvider runnerProvider; /** * Creates an AI client wrapping the given base client, using a default logger. @@ -69,7 +71,7 @@ public final class LDAIClientImpl implements LDAIClient { * @param client an initialized server-side {@code LDClient}; must not be {@code null} */ public LDAIClientImpl(LDClientInterface client) { - this(client, Loggers.defaultLogger()); + this(client, Loggers.defaultLogger(), null); } /** @@ -79,9 +81,25 @@ public LDAIClientImpl(LDClientInterface client) { * @param logger the logger to use for warnings; must not be {@code null} */ public LDAIClientImpl(LDClientInterface client, LDLogger logger) { + this(client, logger, null); + } + + /** + * Creates an AI client wrapping the given base client, logger, and runner provider. + *

+ * The {@code runnerProvider} is called once per judge key when building an {@link Evaluator} for + * a completion or agent config that carries a {@code judgeConfiguration}. Pass {@code null} to + * disable online evaluation (equivalent to the two-argument constructor). + * + * @param client an initialized server-side {@code LDClient}; must not be {@code null} + * @param logger the logger to use for warnings; must not be {@code null} + * @param runnerProvider supplies a {@link Runner} per judge config; may be {@code null} + */ + public LDAIClientImpl(LDClientInterface client, LDLogger logger, AIRunnerProvider runnerProvider) { this.client = Objects.requireNonNull(client, "client"); this.logger = Objects.requireNonNull(logger, "logger"); this.interpolator = new Interpolator(); + this.runnerProvider = runnerProvider; LDValue info = LDValue.buildObject() .put("aiSdkName", AISdkInfo.NAME) @@ -237,7 +255,7 @@ private AIConfig buildConfig( parsed.getJudgeConfiguration(), parsed.getTools(), factory, - Evaluator.noop()); + buildEvaluator(parsed.getJudgeConfiguration(), context, variables)); case JUDGE: return new AIJudgeConfig( key, @@ -258,7 +276,7 @@ private AIConfig buildConfig( parsed.getJudgeConfiguration(), parsed.getTools(), factory, - Evaluator.noop()); + buildEvaluator(parsed.getJudgeConfiguration(), context, variables)); } } @@ -297,7 +315,7 @@ private AIConfig buildConfigFromDefault( agent.getJudgeConfiguration(), agent.getTools(), factory, - Evaluator.noop()); + buildEvaluator(agent.getJudgeConfiguration(), context, variables)); } case JUDGE: { AIJudgeConfigDefault judge = (AIJudgeConfigDefault) defaultValue; @@ -322,7 +340,7 @@ private AIConfig buildConfigFromDefault( completion.getJudgeConfiguration(), completion.getTools(), factory, - Evaluator.noop()); + buildEvaluator(completion.getJudgeConfiguration(), context, variables)); } } } @@ -458,6 +476,52 @@ public LDAIConfigTracker createTracker(String resumptionToken, LDContext context return LDAIConfigTrackerImpl.fromResumptionToken(resumptionToken, client, context, logger); } + /** + * Builds a real {@link Evaluator} from the judge configuration, or returns {@link Evaluator#noop()} + * when no runner provider is configured, the judge configuration is absent/empty, or all judge + * instances fail to construct. + */ + private Evaluator buildEvaluator( + JudgeConfiguration judgeConfig, LDContext context, Map variables) { + if (runnerProvider == null || judgeConfig == null || judgeConfig.getJudges().isEmpty()) { + return Evaluator.noop(); + } + Map judges = new LinkedHashMap<>(); + for (JudgeConfiguration.Judge entry : judgeConfig.getJudges()) { + Judge judge = createJudgeInstance(entry.getKey(), context, variables); + if (judge != null) { + judges.put(entry.getKey(), judge); + } + } + return judges.isEmpty() ? Evaluator.noop() : new Evaluator(judges, judgeConfig, logger); + } + + /** + * Fetches the judge AI Config for the given key and constructs a {@link Judge} using the + * configured runner provider. Returns {@code null} (and logs) when the judge config is disabled, + * the runner provider returns {@code null}, or any exception is thrown — so one bad judge never + * prevents the parent config from being built. + */ + private Judge createJudgeInstance(String key, LDContext context, Map variables) { + try { + AIJudgeConfig cfg = (AIJudgeConfig) evaluate( + key, context, AIJudgeConfigDefault.disabled(), Mode.JUDGE, variables); + if (!cfg.isEnabled()) { + logger.info("Judge configuration is disabled: {}", key); + return null; + } + Runner runner = runnerProvider.create(cfg); + if (runner == null) { + logger.warn("Runner provider returned null for judge: {}", key); + return null; + } + return new Judge(cfg, runner, logger); + } catch (Exception e) { + logger.warn("Failed to create judge instance for key {}: {}", key, e.getMessage()); + return null; + } + } + private List interpolateMessages( List messages, Map variables, LDContext context) { if (messages == null) { diff --git a/lib/sdk/server-ai/src/test/java/com/launchdarkly/sdk/server/ai/LDAIClientImplTest.java b/lib/sdk/server-ai/src/test/java/com/launchdarkly/sdk/server/ai/LDAIClientImplTest.java index e72816a6..34fd5e02 100644 --- a/lib/sdk/server-ai/src/test/java/com/launchdarkly/sdk/server/ai/LDAIClientImplTest.java +++ b/lib/sdk/server-ai/src/test/java/com/launchdarkly/sdk/server/ai/LDAIClientImplTest.java @@ -7,8 +7,10 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyString; @@ -494,4 +496,146 @@ public void createGraphTrackerThrowsForMalformedToken() { private static java.util.LinkedHashMap newLinkedHashMap() { return new java.util.LinkedHashMap<>(); } + + // ---- Evaluator-building (runner provider) --------------------------------- + + private static LDValue completionFlagWithJudge(String judgeKey) { + return LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"}," + + "\"judgeConfiguration\":{\"judges\":[{\"key\":\"" + judgeKey + "\",\"samplingRate\":1.0}]}}"); + } + + private static LDValue completionFlagWithJudges(String... judgeKeys) { + StringBuilder sb = new StringBuilder( + "{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"}," + + "\"judgeConfiguration\":{\"judges\":["); + for (int i = 0; i < judgeKeys.length; i++) { + if (i > 0) sb.append(","); + sb.append("{\"key\":\"").append(judgeKeys[i]).append("\",\"samplingRate\":1.0}"); + } + sb.append("]}}"); + return LDValue.parse(sb.toString()); + } + + private static LDValue judgeFlagValue(boolean enabled) { + return LDValue.parse("{\"_ldMeta\":{\"enabled\":" + enabled + ",\"mode\":\"judge\"}," + + "\"evaluationMetricKey\":\"relevance\"}"); + } + + private List infoMessages() { + return logCapture.getMessages().stream() + .filter(m -> m.getLevel() == LDLogLevel.INFO) + .map(LogCapture.Message::getText) + .collect(Collectors.toList()); + } + + @Test + public void completionConfigWithJudgeConfigAndProviderBuildsRealEvaluator() { + Runner mockRunner = mock(Runner.class); + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mockRunner); + + when(client.jsonValueVariation(eq("comp-key"), any(), any())) + .thenReturn(completionFlagWithJudge("judge-key")); + when(client.jsonValueVariation(eq("judge-key"), any(), any())) + .thenReturn(judgeFlagValue(true)); + + AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null); + + assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop())))); + } + + @Test + public void completionConfigWithNoJudgesYieldsNoopEvaluator() { + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mock(Runner.class)); + when(client.jsonValueVariation(anyString(), any(), any())) + .thenReturn(LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"}}")); + + AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null); + + assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop()))); + } + + @Test + public void completionConfigWithNullRunnerProviderYieldsNoopEvaluator() { + // Legacy two-arg constructor → no provider → noop evaluator even with judgeConfiguration. + when(client.jsonValueVariation(anyString(), any(), any())) + .thenReturn(completionFlagWithJudge("judge-key")); + + AICompletionConfig config = ai.completionConfig("comp-key", context, null, null); + + assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop()))); + } + + @Test + public void disabledJudgeConfigIsFilteredAndEvaluatorIsNoop() { + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mock(Runner.class)); + + when(client.jsonValueVariation(eq("comp-key"), any(), any())) + .thenReturn(completionFlagWithJudge("judge-key")); + when(client.jsonValueVariation(eq("judge-key"), any(), any())) + .thenReturn(judgeFlagValue(false)); + + AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null); + + assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop()))); + assertThat(infoMessages(), hasSize(1)); + assertThat(infoMessages().get(0), containsString("disabled")); + } + + @Test + public void throwingRunnerProviderSkipsThatJudgeButKeepsOthers() { + Runner mockRunner = mock(Runner.class); + AIRunnerProvider provider = cfg -> { + if ("bad-judge".equals(cfg.getKey())) { + throw new RuntimeException("provider error"); + } + return mockRunner; + }; + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, provider); + + when(client.jsonValueVariation(eq("comp-key"), any(), any())) + .thenReturn(completionFlagWithJudges("bad-judge", "good-judge")); + when(client.jsonValueVariation(eq("bad-judge"), any(), any())) + .thenReturn(judgeFlagValue(true)); + when(client.jsonValueVariation(eq("good-judge"), any(), any())) + .thenReturn(judgeFlagValue(true)); + + AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null); + + // good-judge survived; evaluator is non-noop + assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop())))); + assertThat(warnings(), hasSize(1)); + assertThat(warnings().get(0), containsString("bad-judge")); + } + + @Test + public void nullRunnerFromProviderSkipsThatJudge() { + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> null); + + when(client.jsonValueVariation(eq("comp-key"), any(), any())) + .thenReturn(completionFlagWithJudge("judge-key")); + when(client.jsonValueVariation(eq("judge-key"), any(), any())) + .thenReturn(judgeFlagValue(true)); + + AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null); + + assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop()))); + assertThat(warnings(), hasSize(1)); + assertThat(warnings().get(0), containsString("judge-key")); + } + + @Test + public void agentConfigWithJudgeConfigAndProviderBuildsRealEvaluator() { + Runner mockRunner = mock(Runner.class); + LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mockRunner); + + when(client.jsonValueVariation(eq("agent-key"), any(), any())) + .thenReturn(LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"agent\"}," + + "\"judgeConfiguration\":{\"judges\":[{\"key\":\"judge-key\",\"samplingRate\":1.0}]}}")); + when(client.jsonValueVariation(eq("judge-key"), any(), any())) + .thenReturn(judgeFlagValue(true)); + + AIAgentConfig config = aiWithProvider.agentConfig("agent-key", context, null, null); + + assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop())))); + } }