feat: Wire judge evaluators via AIRunnerProvider in LDAIClientImpl

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/AIRunnerProvider.java

@@ -0,0 +1,23 @@
+package com.launchdarkly.sdk.server.ai;
+
+/**
+ * Supplies a {@link Runner} for a given judge AI Config.
+ * <p>
+ * Implement this interface and pass it to
+ * {@link LDAIClientImpl#LDAIClientImpl(com.launchdarkly.sdk.server.interfaces.LDClientInterface,
+ * com.launchdarkly.logging.LDLogger, AIRunnerProvider)} to enable online evaluation (judges).
+ * The client calls {@link #create} once per judge key when building an {@link Evaluator} for a
+ * completion or agent config that carries a {@code judgeConfiguration}.
+ * <p>
+ * Return {@code null} to skip a particular judge. Implementations should be thread-safe.
+ */
+@FunctionalInterface
+public interface AIRunnerProvider {
+  /**
+   * Creates a {@link Runner} for the given judge AI Config.
+   *
+   * @param judgeConfig the judge AI Config for which a runner is needed; never {@code null}
+   * @return a runner to use for this judge, or {@code null} to skip this judge
+   */
+  Runner create(AIJudgeConfig judgeConfig);
+}

lib/sdk/server-ai/src/main/java/com/launchdarkly/sdk/server/ai/LDAIClientImpl.java

@@ -5,6 +5,7 @@
 import com.launchdarkly.sdk.LDContext;
 import com.launchdarkly.sdk.LDValue;
 import com.launchdarkly.sdk.LDValueType;
+import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.JudgeConfiguration;
 import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Message;
 import com.launchdarkly.sdk.server.ai.datamodel.LDAIConfigTypes.Mode;
 import com.launchdarkly.sdk.server.ai.internal.AgentGraphFlagValue;
@@ -62,14 +63,15 @@ public final class LDAIClientImpl implements LDAIClient {
   private final LDClientInterface client;
   private final LDLogger logger;
   private final Interpolator interpolator;
+  private final AIRunnerProvider runnerProvider;
 
   /**
    * Creates an AI client wrapping the given base client, using a default logger.
    *
    * @param client an initialized server-side {@code LDClient}; must not be {@code null}
    */
   public LDAIClientImpl(LDClientInterface client) {
-    this(client, Loggers.defaultLogger());
+    this(client, Loggers.defaultLogger(), null);
   }
 
   /**
@@ -79,9 +81,25 @@ public LDAIClientImpl(LDClientInterface client) {
    * @param logger the logger to use for warnings; must not be {@code null}
    */
   public LDAIClientImpl(LDClientInterface client, LDLogger logger) {
+    this(client, logger, null);
+  }
+
+  /**
+   * Creates an AI client wrapping the given base client, logger, and runner provider.
+   * <p>
+   * The {@code runnerProvider} is called once per judge key when building an {@link Evaluator} for
+   * a completion or agent config that carries a {@code judgeConfiguration}. Pass {@code null} to
+   * disable online evaluation (equivalent to the two-argument constructor).
+   *
+   * @param client an initialized server-side {@code LDClient}; must not be {@code null}
+   * @param logger the logger to use for warnings; must not be {@code null}
+   * @param runnerProvider supplies a {@link Runner} per judge config; may be {@code null}
+   */
+  public LDAIClientImpl(LDClientInterface client, LDLogger logger, AIRunnerProvider runnerProvider) {
     this.client = Objects.requireNonNull(client, "client");
     this.logger = Objects.requireNonNull(logger, "logger");
     this.interpolator = new Interpolator();
+    this.runnerProvider = runnerProvider;
 
     LDValue info = LDValue.buildObject()
         .put("aiSdkName", AISdkInfo.NAME)
@@ -237,7 +255,7 @@ private AIConfig buildConfig(
             parsed.getJudgeConfiguration(),
             parsed.getTools(),
             factory,
-            Evaluator.noop());
+            buildEvaluator(parsed.getJudgeConfiguration(), context, variables));
       case JUDGE:
         return new AIJudgeConfig(
             key,
@@ -258,7 +276,7 @@ private AIConfig buildConfig(
             parsed.getJudgeConfiguration(),
             parsed.getTools(),
             factory,
-            Evaluator.noop());
+            buildEvaluator(parsed.getJudgeConfiguration(), context, variables));
     }
   }
 
@@ -297,7 +315,7 @@ private AIConfig buildConfigFromDefault(
             agent.getJudgeConfiguration(),
             agent.getTools(),
             factory,
-            Evaluator.noop());
+            buildEvaluator(agent.getJudgeConfiguration(), context, variables));
       }
       case JUDGE: {
         AIJudgeConfigDefault judge = (AIJudgeConfigDefault) defaultValue;
@@ -322,7 +340,7 @@ private AIConfig buildConfigFromDefault(
             completion.getJudgeConfiguration(),
             completion.getTools(),
             factory,
-            Evaluator.noop());
+            buildEvaluator(completion.getJudgeConfiguration(), context, variables));
       }
     }
   }
@@ -458,6 +476,52 @@ public LDAIConfigTracker createTracker(String resumptionToken, LDContext context
     return LDAIConfigTrackerImpl.fromResumptionToken(resumptionToken, client, context, logger);
   }
 
+  /**
+   * Builds a real {@link Evaluator} from the judge configuration, or returns {@link Evaluator#noop()}
+   * when no runner provider is configured, the judge configuration is absent/empty, or all judge
+   * instances fail to construct.
+   */
+  private Evaluator buildEvaluator(
+      JudgeConfiguration judgeConfig, LDContext context, Map<String, Object> variables) {
+    if (runnerProvider == null || judgeConfig == null || judgeConfig.getJudges().isEmpty()) {
+      return Evaluator.noop();
+    }
+    Map<String, Judge> judges = new LinkedHashMap<>();
+    for (JudgeConfiguration.Judge entry : judgeConfig.getJudges()) {
+      Judge judge = createJudgeInstance(entry.getKey(), context, variables);
+      if (judge != null) {
+        judges.put(entry.getKey(), judge);
+      }
+    }
+    return judges.isEmpty() ? Evaluator.noop() : new Evaluator(judges, judgeConfig, logger);
+  }
+
+  /**
+   * Fetches the judge AI Config for the given key and constructs a {@link Judge} using the
+   * configured runner provider. Returns {@code null} (and logs) when the judge config is disabled,
+   * the runner provider returns {@code null}, or any exception is thrown — so one bad judge never
+   * prevents the parent config from being built.
+   */
+  private Judge createJudgeInstance(String key, LDContext context, Map<String, Object> variables) {
+    try {
+      AIJudgeConfig cfg = (AIJudgeConfig) evaluate(
+          key, context, AIJudgeConfigDefault.disabled(), Mode.JUDGE, variables);
+      if (!cfg.isEnabled()) {
+        logger.info("Judge configuration is disabled: {}", key);
+        return null;
+      }
+      Runner runner = runnerProvider.create(cfg);
+      if (runner == null) {
+        logger.warn("Runner provider returned null for judge: {}", key);
+        return null;
+      }
+      return new Judge(cfg, runner, logger);
+    } catch (Exception e) {
+      logger.warn("Failed to create judge instance for key {}: {}", key, e.getMessage());
+      return null;
+    }
+  }
+
   private List<Message> interpolateMessages(
       List<Message> messages, Map<String, Object> variables, LDContext context) {
     if (messages == null) {

lib/sdk/server-ai/src/test/java/com/launchdarkly/sdk/server/ai/LDAIClientImplTest.java

@@ -7,8 +7,10 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyDouble;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -494,4 +496,146 @@ public void createGraphTrackerThrowsForMalformedToken() {
   private static <K, V> java.util.LinkedHashMap<K, V> newLinkedHashMap() {
     return new java.util.LinkedHashMap<>();
   }
+
+  // ---- Evaluator-building (runner provider) ---------------------------------
+
+  private static LDValue completionFlagWithJudge(String judgeKey) {
+    return LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"},"
+        + "\"judgeConfiguration\":{\"judges\":[{\"key\":\"" + judgeKey + "\",\"samplingRate\":1.0}]}}");
+  }
+
+  private static LDValue completionFlagWithJudges(String... judgeKeys) {
+    StringBuilder sb = new StringBuilder(
+        "{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"},"
+        + "\"judgeConfiguration\":{\"judges\":[");
+    for (int i = 0; i < judgeKeys.length; i++) {
+      if (i > 0) sb.append(",");
+      sb.append("{\"key\":\"").append(judgeKeys[i]).append("\",\"samplingRate\":1.0}");
+    }
+    sb.append("]}}");
+    return LDValue.parse(sb.toString());
+  }
+
+  private static LDValue judgeFlagValue(boolean enabled) {
+    return LDValue.parse("{\"_ldMeta\":{\"enabled\":" + enabled + ",\"mode\":\"judge\"},"
+        + "\"evaluationMetricKey\":\"relevance\"}");
+  }
+
+  private List<String> infoMessages() {
+    return logCapture.getMessages().stream()
+        .filter(m -> m.getLevel() == LDLogLevel.INFO)
+        .map(LogCapture.Message::getText)
+        .collect(Collectors.toList());
+  }
+
+  @Test
+  public void completionConfigWithJudgeConfigAndProviderBuildsRealEvaluator() {
+    Runner mockRunner = mock(Runner.class);
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mockRunner);
+
+    when(client.jsonValueVariation(eq("comp-key"), any(), any()))
+        .thenReturn(completionFlagWithJudge("judge-key"));
+    when(client.jsonValueVariation(eq("judge-key"), any(), any()))
+        .thenReturn(judgeFlagValue(true));
+
+    AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop()))));
+  }
+
+  @Test
+  public void completionConfigWithNoJudgesYieldsNoopEvaluator() {
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mock(Runner.class));
+    when(client.jsonValueVariation(anyString(), any(), any()))
+        .thenReturn(LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"completion\"}}"));
+
+    AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop())));
+  }
+
+  @Test
+  public void completionConfigWithNullRunnerProviderYieldsNoopEvaluator() {
+    // Legacy two-arg constructor → no provider → noop evaluator even with judgeConfiguration.
+    when(client.jsonValueVariation(anyString(), any(), any()))
+        .thenReturn(completionFlagWithJudge("judge-key"));
+
+    AICompletionConfig config = ai.completionConfig("comp-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop())));
+  }
+
+  @Test
+  public void disabledJudgeConfigIsFilteredAndEvaluatorIsNoop() {
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mock(Runner.class));
+
+    when(client.jsonValueVariation(eq("comp-key"), any(), any()))
+        .thenReturn(completionFlagWithJudge("judge-key"));
+    when(client.jsonValueVariation(eq("judge-key"), any(), any()))
+        .thenReturn(judgeFlagValue(false));
+
+    AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop())));
+    assertThat(infoMessages(), hasSize(1));
+    assertThat(infoMessages().get(0), containsString("disabled"));
+  }
+
+  @Test
+  public void throwingRunnerProviderSkipsThatJudgeButKeepsOthers() {
+    Runner mockRunner = mock(Runner.class);
+    AIRunnerProvider provider = cfg -> {
+      if ("bad-judge".equals(cfg.getKey())) {
+        throw new RuntimeException("provider error");
+      }
+      return mockRunner;
+    };
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, provider);
+
+    when(client.jsonValueVariation(eq("comp-key"), any(), any()))
+        .thenReturn(completionFlagWithJudges("bad-judge", "good-judge"));
+    when(client.jsonValueVariation(eq("bad-judge"), any(), any()))
+        .thenReturn(judgeFlagValue(true));
+    when(client.jsonValueVariation(eq("good-judge"), any(), any()))
+        .thenReturn(judgeFlagValue(true));
+
+    AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null);
+
+    // good-judge survived; evaluator is non-noop
+    assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop()))));
+    assertThat(warnings(), hasSize(1));
+    assertThat(warnings().get(0), containsString("bad-judge"));
+  }
+
+  @Test
+  public void nullRunnerFromProviderSkipsThatJudge() {
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> null);
+
+    when(client.jsonValueVariation(eq("comp-key"), any(), any()))
+        .thenReturn(completionFlagWithJudge("judge-key"));
+    when(client.jsonValueVariation(eq("judge-key"), any(), any()))
+        .thenReturn(judgeFlagValue(true));
+
+    AICompletionConfig config = aiWithProvider.completionConfig("comp-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(sameInstance(Evaluator.noop())));
+    assertThat(warnings(), hasSize(1));
+    assertThat(warnings().get(0), containsString("judge-key"));
+  }
+
+  @Test
+  public void agentConfigWithJudgeConfigAndProviderBuildsRealEvaluator() {
+    Runner mockRunner = mock(Runner.class);
+    LDAIClientImpl aiWithProvider = new LDAIClientImpl(client, logger, cfg -> mockRunner);
+
+    when(client.jsonValueVariation(eq("agent-key"), any(), any()))
+        .thenReturn(LDValue.parse("{\"_ldMeta\":{\"enabled\":true,\"mode\":\"agent\"},"
+            + "\"judgeConfiguration\":{\"judges\":[{\"key\":\"judge-key\",\"samplingRate\":1.0}]}}"));
+    when(client.jsonValueVariation(eq("judge-key"), any(), any()))
+        .thenReturn(judgeFlagValue(true));
+
+    AIAgentConfig config = aiWithProvider.agentConfig("agent-key", context, null, null);
+
+    assertThat(config.getEvaluator(), is(not(sameInstance(Evaluator.noop()))));
+  }
 }