From 5a7dc09b630863270db51a4765bdee8913ba36e9 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 20 Oct 2025 20:22:47 -0600 Subject: [PATCH 1/3] api outline --- .../OpenAIInstrumentationExample.java | 16 ++-- .../examples/PromptFetchingExample.java | 48 ++++++++++ src/main/java/dev/braintrust/Braintrust.java | 91 +++++++++++++++++++ .../openai/BraintrustOpenAI.java | 6 ++ .../braintrust/prompt/BraintrustPrompt.java | 5 + .../prompt/BraintrustPromptLoader.java | 42 +++++++++ 6 files changed, 201 insertions(+), 7 deletions(-) create mode 100644 examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java create mode 100644 src/main/java/dev/braintrust/Braintrust.java create mode 100644 src/main/java/dev/braintrust/prompt/BraintrustPrompt.java create mode 100644 src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java diff --git a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java index 33f04ab5..cde20784 100644 --- a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java +++ b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java @@ -5,9 +5,8 @@ import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; import com.openai.models.chat.completions.ChatCompletionStreamOptions; -import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.Braintrust; import dev.braintrust.instrumentation.openai.BraintrustOpenAI; -import dev.braintrust.trace.BraintrustTracing; /** Basic OTel + OpenAI instrumentation example */ public class OpenAIInstrumentationExample { @@ -16,12 +15,15 @@ public static void main(String[] args) throws Exception { System.err.println( "\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n"); } - var braintrustConfig = BraintrustConfig.fromEnvironment(); - var openTelemetry = BraintrustTracing.of(braintrustConfig, true); - var tracer = BraintrustTracing.getTracer(openTelemetry); + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); - var rootSpan = tracer.spanBuilder("openai-java-instrumentation-example").startSpan(); + var rootSpan = + openTelemetry + .getTracer("my-instrumentation") + .spanBuilder("openai-java-instrumentation-example") + .startSpan(); try (var ignored = rootSpan.makeCurrent()) { chatCompletionsExample(openAIClient); // chatCompletionsStreamingExample(openAIClient); @@ -29,7 +31,7 @@ public static void main(String[] args) throws Exception { rootSpan.end(); } var url = - braintrustConfig.fetchProjectURI() + braintrust.projectUri() + "/logs?r=%s&s=%s" .formatted( rootSpan.getSpanContext().getTraceId(), diff --git a/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java b/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java new file mode 100644 index 00000000..fc608096 --- /dev/null +++ b/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java @@ -0,0 +1,48 @@ +package dev.braintrust.examples; + +import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.buildChatCompletionsPrompt; +import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.wrapOpenAI; + +import com.openai.client.OpenAIClient; +import com.openai.client.okhttp.OpenAIOkHttpClient; +import dev.braintrust.Braintrust; +import dev.braintrust.prompt.BraintrustPromptLoader.PromptLoadRequest; + +public class PromptFetchingExample { + public static void main(String... args) { + if (null == System.getenv("OPENAI_API_KEY")) { + System.err.println( + "\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n"); + } + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + + OpenAIClient openAIClient = wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); + + { // simple example + var prompt = braintrust.promptLoader().load("my-prompt-slug"); + var response = + openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt)); + System.out.println("got response: %s".formatted(response)); + } + { // more complex prompt options + var prompt = + braintrust + .promptLoader() + .load( + PromptLoadRequest.builder() + .projectName("my-project") + .promptSlug("my-prompt-slug") + .version("5878bd218351fb8e") + .defaults("model", "gpt-3.5-turbo") + .build()); + var response = + openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt)); + System.out.println("got response: %s".formatted(response)); + } + + System.out.println( + "\n\n Example complete! View your data in Braintrust: %s\n" + .formatted(braintrust.projectUri() + "/logs")); + } +} diff --git a/src/main/java/dev/braintrust/Braintrust.java b/src/main/java/dev/braintrust/Braintrust.java new file mode 100644 index 00000000..50d5d538 --- /dev/null +++ b/src/main/java/dev/braintrust/Braintrust.java @@ -0,0 +1,91 @@ +package dev.braintrust; + +import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.prompt.BraintrustPromptLoader; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.logs.SdkLoggerProviderBuilder; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import java.net.URI; +import javax.annotation.Nonnull; +import lombok.Getter; +import lombok.experimental.Accessors; + +/** + * Main entry point for the Braintrust SDK. + * + *

This class provides access to all Braintrust functionality. Most users will interact with a + * singleton instance via {@link #get()}, though you can create independent instances if needed. + * + *

The Braintrust instance also provides methods for enabling Braintrust in open telemetry + * builders. + * + *

Additionally, vendor-specific instrumentation or functionality is provided by {@code + * Braintrust}. E.g. {@code BraintrustOpenAI}, {@code BraintrustAnthropic}, etc. + * + * @see #get() + * @see BraintrustConfig + * @see #openTelemetryCreate() + * @see #openTelemetryEnable(SdkTracerProviderBuilder, SdkLoggerProviderBuilder, + * SdkMeterProviderBuilder) + */ +public class Braintrust { + /** + * get or create the global braintrust instance. Most users will want to use this method to + * access the Braintrust SDK. + */ + public static Braintrust get() { + throw new RuntimeException("TODO"); + } + + /** get or create the global braintrust instance from the given config */ + public static Braintrust get(BraintrustConfig config) { + throw new RuntimeException("TODO"); + } + + /** Create a new Braintrust instance from the given config */ + public static Braintrust of(BraintrustConfig config) { + throw new RuntimeException("TODO"); + } + + @Getter + @Accessors(fluent = true) + private final BraintrustConfig config; + + @Getter + @Accessors(fluent = true) + private final BraintrustApiClient apiClient; + + @Getter + @Accessors(fluent = true) + private final BraintrustPromptLoader promptLoader; + + private Braintrust( + BraintrustConfig config, + BraintrustApiClient apiClient, + BraintrustPromptLoader promptLoader) { + this.config = config; + this.apiClient = apiClient; + this.promptLoader = promptLoader; + } + + public URI projectUri() { + throw new RuntimeException("TODO"); + } + + public OpenTelemetry openTelemetryCreate() { + throw new RuntimeException("TODO"); + } + + public OpenTelemetry openTelemetryCreate(boolean registerGlobal) { + throw new RuntimeException("TODO"); + } + + public void openTelemetryEnable( + @Nonnull SdkTracerProviderBuilder tracerProviderBuilder, + @Nonnull SdkLoggerProviderBuilder loggerProviderBuilder, + @Nonnull SdkMeterProviderBuilder meterProviderBuilder) { + throw new RuntimeException("TODO"); + } +} diff --git a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java index 39c01a69..cf15493f 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java @@ -1,7 +1,9 @@ package dev.braintrust.instrumentation.openai; import com.openai.client.OpenAIClient; +import com.openai.models.chat.completions.ChatCompletionCreateParams; import dev.braintrust.instrumentation.openai.otel.OpenAITelemetry; +import dev.braintrust.prompt.BraintrustPrompt; import io.opentelemetry.api.OpenTelemetry; /** Braintrust OpenAI client instrumentation. */ @@ -21,4 +23,8 @@ public static OpenAIClient wrapOpenAI(OpenTelemetry openTelemetry, OpenAIClient .wrap(openAIClient); } } + + public static ChatCompletionCreateParams buildChatCompletionsPrompt(BraintrustPrompt prompt) { + throw new RuntimeException("TODO"); + } } diff --git a/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java new file mode 100644 index 00000000..a68fca3e --- /dev/null +++ b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java @@ -0,0 +1,5 @@ +package dev.braintrust.prompt; + +public class BraintrustPrompt { + // TODO +} diff --git a/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java b/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java new file mode 100644 index 00000000..896153fa --- /dev/null +++ b/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java @@ -0,0 +1,42 @@ +package dev.braintrust.prompt; + +import dev.braintrust.config.BraintrustConfig; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.Builder; + +public class BraintrustPromptLoader { + public static BraintrustPromptLoader of(BraintrustConfig config) { + throw new RuntimeException("TODO"); + } + + public BraintrustPrompt load(String promptSlug) { + PromptLoadRequest.builder().promptSlug(promptSlug); + throw new RuntimeException("TODO"); + } + + public BraintrustPrompt load(PromptLoadRequest promptLoadRequest) { + throw new RuntimeException("TODO"); + } + + @Builder + public static class PromptLoadRequest { + private @Nonnull String promptSlug; + private @Nullable String projectName; + private @Nullable String version; + private @Nonnull @Builder.Default Map defaults = Map.of(); + + public static class PromptLoadRequestBuilder { + public PromptLoadRequestBuilder defaults(String... keyValuePairDefaults) { + throw new RuntimeException("TODO"); + } + } + } + + // TODO: what default values are allowed? Do we want to use an enum or is that too constrictive? + public static class PromptLoadDefaultKeys { + public static String MODEL = "model"; + public static String TEMPERATURE = "temperature"; + } +} From 8edc0a1f1b74ebe972b2b68b8443103dad7a6fb5 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Tue, 21 Oct 2025 19:06:53 -0600 Subject: [PATCH 2/3] prompt fetching impl --- README.md | 13 +- build.gradle | 2 +- .../AnthropicInstrumentationExample.java | 11 +- .../examples/CustomOpenTelemetryExample.java | 9 +- .../examples/ExperimentExample.java | 13 +- .../OpenAIInstrumentationExample.java | 1 + .../examples/PromptFetchingExample.java | 25 ++- .../examples/SimpleOpenTelemetryExample.java | 11 +- src/main/java/dev/braintrust/Braintrust.java | 70 +++++- .../dev/braintrust/{trace => }/SDKMain.java | 2 +- .../braintrust/api/BraintrustApiClient.java | 129 +++++++++++ .../openai/BraintrustOpenAI.java | 22 +- .../braintrust/prompt/BraintrustPrompt.java | 112 ++++++++- .../prompt/BraintrustPromptLoader.java | 60 ++++- .../braintrust/trace/BraintrustTracing.java | 13 +- .../java/dev/braintrust/BraintrustTest.java | 167 ++++++++++++++ .../openai/BraintrustOpenAITest.java | 69 +++++- .../prompt/BraintrustPromptLoaderTest.java | 182 +++++++++++++++ .../prompt/BraintrustPromptTest.java | 212 ++++++++++++++++++ 19 files changed, 1053 insertions(+), 70 deletions(-) rename src/main/java/dev/braintrust/{trace => }/SDKMain.java (97%) create mode 100644 src/test/java/dev/braintrust/BraintrustTest.java create mode 100644 src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java create mode 100644 src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java diff --git a/README.md b/README.md index 4a31ff56..5540eab5 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ dependencies { ### Evals ```java -var config = BraintrustConfig.fromEnvironment(); -var openTelemetry = BraintrustTracing.of(config, true); +var braintrust = Braintrust.get(); +var openTelemetry = braintrust.openTelemetryCreate(); var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); Function getFoodType = @@ -49,11 +49,8 @@ Function getFoodType = return response.choices().get(0).message().content().orElse("").toLowerCase(); }; -var eval = - Eval.builder() +var eval = braintrust.evalBuilder() .name("java-eval-x-" + System.currentTimeMillis()) - .tracer(BraintrustTracing.getTracer(openTelemetry)) - .config(config) .cases( EvalCase.of("asparagus", "vegetable"), EvalCase.of("banana", "fruit")) @@ -73,8 +70,8 @@ System.out.println("\n\n" + result.createReportString()); ### OpenAI Tracing ```java -var braintrustConfig = BraintrustConfig.fromEnvironment(); -var openTelemetry = BraintrustTracing.of(braintrustConfig, true); +var braintrust = Braintrust.get(); +var openTelemetry = braintrust.openTelemetryCreate(); OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); var request = diff --git a/build.gradle b/build.gradle index 3f268476..2270a0a3 100644 --- a/build.gradle +++ b/build.gradle @@ -219,7 +219,7 @@ jar { 'Implementation-Title': 'Braintrust Java SDK', 'Implementation-Version': version, 'Implementation-Vendor': 'Braintrust', - 'Main-Class': 'dev.braintrust.trace.SDKMain' + 'Main-Class': 'dev.braintrust.SDKMain' ) } } diff --git a/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java index 94018fbe..2f162672 100644 --- a/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java +++ b/examples/src/main/java/dev/braintrust/examples/AnthropicInstrumentationExample.java @@ -4,9 +4,8 @@ import com.anthropic.client.okhttp.AnthropicOkHttpClient; import com.anthropic.models.messages.MessageCreateParams; import com.anthropic.models.messages.Model; -import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.Braintrust; import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic; -import dev.braintrust.trace.BraintrustTracing; /** Basic OTel + Anthropic instrumentation example */ public class AnthropicInstrumentationExample { @@ -18,9 +17,9 @@ public static void main(String[] args) throws Exception { + " fail.\n"); } - var braintrustConfig = BraintrustConfig.fromEnvironment(); - var openTelemetry = BraintrustTracing.of(braintrustConfig, true); - var tracer = BraintrustTracing.getTracer(openTelemetry); + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + var tracer = openTelemetry.getTracer("my-instrumentation"); // Wrap Anthropic client with Braintrust instrumentation AnthropicClient anthropicClient = @@ -35,7 +34,7 @@ public static void main(String[] args) throws Exception { } var url = - braintrustConfig.fetchProjectURI() + braintrust.projectUri() + "/logs?r=%s&s=%s" .formatted( rootSpan.getSpanContext().getTraceId(), diff --git a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java index 388d0fa1..cea869b5 100644 --- a/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/CustomOpenTelemetryExample.java @@ -1,7 +1,6 @@ package dev.braintrust.examples; -import dev.braintrust.config.BraintrustConfig; -import dev.braintrust.trace.BraintrustTracing; +import dev.braintrust.Braintrust; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter; import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; @@ -51,8 +50,8 @@ public static void main(String[] args) throws Exception { // NOTE: there are many ways to set up otel builders, etc. // The important line is here: call enable with your otel builders and braintrust will // export open telemetry data in addition to your existing setup - var braintrustConfig = BraintrustConfig.fromEnvironment(); - BraintrustTracing.enable(braintrustConfig, tracerBuilder, loggerBuilder, meterBuilder); + var braintrust = Braintrust.get(); + braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder); var openTelemetry = OpenTelemetrySdk.builder() @@ -74,7 +73,7 @@ public static void main(String[] args) throws Exception { span.end(); } var url = - braintrustConfig.fetchProjectURI() + braintrust.projectUri() + "/logs?r=%s&s=%s" .formatted( span.getSpanContext().getTraceId(), diff --git a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java index b343ddd1..cfd9a1e8 100644 --- a/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java +++ b/examples/src/main/java/dev/braintrust/examples/ExperimentExample.java @@ -3,18 +3,16 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; -import dev.braintrust.config.BraintrustConfig; -import dev.braintrust.eval.Eval; +import dev.braintrust.Braintrust; import dev.braintrust.eval.EvalCase; import dev.braintrust.eval.Scorer; import dev.braintrust.instrumentation.openai.BraintrustOpenAI; -import dev.braintrust.trace.BraintrustTracing; import java.util.function.Function; public class ExperimentExample { public static void main(String[] args) throws Exception { - var config = BraintrustConfig.fromEnvironment(); - var openTelemetry = BraintrustTracing.of(config, true); + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); Function getFoodType = @@ -32,13 +30,12 @@ public static void main(String[] args) throws Exception { }; var eval = - Eval.builder() + braintrust + .evalBuilder() .name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a // constant, additional runs // will append new cases to // the same experiment - .tracer(BraintrustTracing.getTracer(openTelemetry)) - .config(config) .cases( EvalCase.of("strawberry", "fruit"), EvalCase.of("asparagus", "vegetable"), diff --git a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java index cde20784..4e7a403a 100644 --- a/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java +++ b/examples/src/main/java/dev/braintrust/examples/OpenAIInstrumentationExample.java @@ -10,6 +10,7 @@ /** Basic OTel + OpenAI instrumentation example */ public class OpenAIInstrumentationExample { + public static void main(String[] args) throws Exception { if (null == System.getenv("OPENAI_API_KEY")) { System.err.println( diff --git a/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java b/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java index fc608096..4e2f657d 100644 --- a/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java +++ b/examples/src/main/java/dev/braintrust/examples/PromptFetchingExample.java @@ -7,6 +7,7 @@ import com.openai.client.okhttp.OpenAIOkHttpClient; import dev.braintrust.Braintrust; import dev.braintrust.prompt.BraintrustPromptLoader.PromptLoadRequest; +import java.util.Map; public class PromptFetchingExample { public static void main(String... args) { @@ -20,10 +21,15 @@ public static void main(String... args) { OpenAIClient openAIClient = wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv()); { // simple example - var prompt = braintrust.promptLoader().load("my-prompt-slug"); + var prompt = braintrust.promptLoader().load("kind-greeter-69d2"); var response = - openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt)); - System.out.println("got response: %s".formatted(response)); + openAIClient + .chat() + .completions() + .create( + buildChatCompletionsPrompt( + prompt, Map.of("name", "Sam Malone"))); + System.out.println("got response: %s".formatted(response.choices().get(0).message())); } { // more complex prompt options var prompt = @@ -31,13 +37,18 @@ public static void main(String... args) { .promptLoader() .load( PromptLoadRequest.builder() - .projectName("my-project") - .promptSlug("my-prompt-slug") - .version("5878bd218351fb8e") + .projectName("andrew-misc") + .promptSlug("unkind-greeter-fd4c") + .version("cbbc711da9f7d445") .defaults("model", "gpt-3.5-turbo") .build()); var response = - openAIClient.chat().completions().create(buildChatCompletionsPrompt(prompt)); + openAIClient + .chat() + .completions() + .create( + buildChatCompletionsPrompt( + prompt, Map.of("name", "Frasier Crane"))); System.out.println("got response: %s".formatted(response)); } diff --git a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java index b1a6c502..172adc62 100644 --- a/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java +++ b/examples/src/main/java/dev/braintrust/examples/SimpleOpenTelemetryExample.java @@ -1,13 +1,12 @@ package dev.braintrust.examples; -import dev.braintrust.config.BraintrustConfig; -import dev.braintrust.trace.BraintrustTracing; +import dev.braintrust.Braintrust; public class SimpleOpenTelemetryExample { public static void main(String[] args) throws Exception { - var braintrustConfig = BraintrustConfig.fromEnvironment(); - var openTelemetry = BraintrustTracing.quickstart(); - var tracer = BraintrustTracing.getTracer(openTelemetry); + var braintrust = Braintrust.get(); + var openTelemetry = braintrust.openTelemetryCreate(); + var tracer = openTelemetry.getTracer("my-instrumentation"); var span = tracer.spanBuilder("hello-java").startSpan(); try (var ignored = span.makeCurrent()) { @@ -18,7 +17,7 @@ public static void main(String[] args) throws Exception { span.end(); } var url = - braintrustConfig.fetchProjectURI() + braintrust.projectUri() + "/logs?r=%s&s=%s" .formatted( span.getSpanContext().getTraceId(), diff --git a/src/main/java/dev/braintrust/Braintrust.java b/src/main/java/dev/braintrust/Braintrust.java index 50d5d538..0e5ea61b 100644 --- a/src/main/java/dev/braintrust/Braintrust.java +++ b/src/main/java/dev/braintrust/Braintrust.java @@ -2,15 +2,19 @@ import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.eval.Eval; import dev.braintrust.prompt.BraintrustPromptLoader; +import dev.braintrust.trace.BraintrustTracing; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.sdk.logs.SdkLoggerProviderBuilder; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import java.net.URI; +import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nonnull; import lombok.Getter; import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; /** * Main entry point for the Braintrust SDK. @@ -30,23 +34,43 @@ * @see #openTelemetryEnable(SdkTracerProviderBuilder, SdkLoggerProviderBuilder, * SdkMeterProviderBuilder) */ +@Slf4j public class Braintrust { + private static final String SDK_VERSION = SDKMain.loadVersionFromProperties(); + private static final AtomicReference instance = new AtomicReference<>(); + /** * get or create the global braintrust instance. Most users will want to use this method to * access the Braintrust SDK. */ public static Braintrust get() { - throw new RuntimeException("TODO"); + var current = instance.get(); + if (null == current) { + return get(BraintrustConfig.fromEnvironment()); + } else { + return current; + } } /** get or create the global braintrust instance from the given config */ public static Braintrust get(BraintrustConfig config) { - throw new RuntimeException("TODO"); + var current = instance.get(); + if (null == current) { + var success = instance.compareAndSet(null, of(config)); + if (success) { + log.info("initialized global Braintrust sdk {}", SDK_VERSION); + } + return instance.get(); + } else { + return current; + } } /** Create a new Braintrust instance from the given config */ public static Braintrust of(BraintrustConfig config) { - throw new RuntimeException("TODO"); + BraintrustApiClient apiClient = BraintrustApiClient.of(config); + BraintrustPromptLoader promptLoader = BraintrustPromptLoader.of(config, apiClient); + return new Braintrust(config, apiClient, promptLoader); } @Getter @@ -71,21 +95,53 @@ private Braintrust( } public URI projectUri() { - throw new RuntimeException("TODO"); + // TODO cache? + return config.fetchProjectURI(); } + /** + * Quick start method that sets up global OpenTelemetry with this Braintrust.
+ *
+ * If you're looking for more options for configuring Braintrust/OpenTelemetry, consult the + * `enable` method. + */ public OpenTelemetry openTelemetryCreate() { - throw new RuntimeException("TODO"); + return openTelemetryCreate(true); } + /** + * Quick start method that sets up OpenTelemetry with this Braintrust.
+ *
+ * If you're looking for more options for configuring Braintrust and OpenTelemetry, consult the + * `enable` method. + */ public OpenTelemetry openTelemetryCreate(boolean registerGlobal) { - throw new RuntimeException("TODO"); + return BraintrustTracing.of(this.config, registerGlobal); } + /** + * Add braintrust to existing open telemetry builders
+ *
+ * This method provides the most options for configuring Braintrust and OpenTelemetry. If you're + * looking for a more user-friendly setup, consult the `openTelemetryCreate` methods.
+ *
+ * NOTE: if your otel setup does not have any particular builder, pass an instance of the + * default provider builder. E.g. `SdkMeterProvider.builder()`
+ *
+ * NOTE: This method should only be invoked once. Enabling Braintrust multiple times is + * unsupported and may lead to undesired behavior + */ public void openTelemetryEnable( @Nonnull SdkTracerProviderBuilder tracerProviderBuilder, @Nonnull SdkLoggerProviderBuilder loggerProviderBuilder, @Nonnull SdkMeterProviderBuilder meterProviderBuilder) { - throw new RuntimeException("TODO"); + BraintrustTracing.enable( + this.config, tracerProviderBuilder, loggerProviderBuilder, meterProviderBuilder); + } + + /** Create a new eval builder */ + public Eval.Builder evalBuilder() { + return (Eval.Builder) + Eval.builder().config(this.config).apiClient(this.apiClient); } } diff --git a/src/main/java/dev/braintrust/trace/SDKMain.java b/src/main/java/dev/braintrust/SDKMain.java similarity index 97% rename from src/main/java/dev/braintrust/trace/SDKMain.java rename to src/main/java/dev/braintrust/SDKMain.java index 143ff878..c1b8308e 100644 --- a/src/main/java/dev/braintrust/trace/SDKMain.java +++ b/src/main/java/dev/braintrust/SDKMain.java @@ -1,4 +1,4 @@ -package dev.braintrust.trace; +package dev.braintrust; import java.util.Properties; diff --git a/src/main/java/dev/braintrust/api/BraintrustApiClient.java b/src/main/java/dev/braintrust/api/BraintrustApiClient.java index 1650373d..74d22911 100644 --- a/src/main/java/dev/braintrust/api/BraintrustApiClient.java +++ b/src/main/java/dev/braintrust/api/BraintrustApiClient.java @@ -17,6 +17,8 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import lombok.extern.slf4j.Slf4j; /** @@ -39,6 +41,10 @@ public interface BraintrustApiClient { /** Get project and org info for the given project ID */ Optional getProjectAndOrgInfo(String projectId); + /** Get a prompt by slug and optional version */ + Optional getPrompt( + @Nonnull String projectName, @Nonnull String slug, @Nullable String version); + static BraintrustApiClient of(BraintrustConfig config) { return new HttpImpl(config); } @@ -139,6 +145,55 @@ public Optional getProjectAndOrgInfo(String projectI return Optional.of(new OrganizationAndProjectInfo(orgInfo, project)); } + @Override + public Optional getPrompt( + @Nonnull String projectName, @Nonnull String slug, @Nullable String version) { + Objects.requireNonNull(projectName, slug); + try { + var uriBuilder = new StringBuilder(config.apiUrl() + "/v1/prompt?"); + + if (!slug.isEmpty()) { + uriBuilder.append("slug=").append(slug); + } + + if (!projectName.isEmpty()) { + if (uriBuilder.charAt(uriBuilder.length() - 1) != '?') { + uriBuilder.append("&"); + } + uriBuilder.append("project_name=").append(projectName); + } + + if (version != null && !version.isEmpty()) { + if (uriBuilder.charAt(uriBuilder.length() - 1) != '?') { + uriBuilder.append("&"); + } + uriBuilder.append("version=").append(version); + } + + PromptListResponse response = + getAsync( + uriBuilder.toString().replace(config.apiUrl(), ""), + PromptListResponse.class) + .get(); + + if (response.objects() == null || response.objects().isEmpty()) { + return Optional.empty(); + } + + if (response.objects().size() > 1) { + throw new ApiException( + "Multiple objects found for slug: " + + slug + + ", projectName: " + + projectName); + } + + return Optional.of(response.objects().get(0)); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + private CompletableFuture getAsync(String path, Class responseType) { var request = HttpRequest.newBuilder() @@ -235,11 +290,19 @@ class InMemoryImpl implements BraintrustApiClient { private final List organizationAndProjectInfos; private final Set experiments = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final List prompts = new ArrayList<>(); public InMemoryImpl(OrganizationAndProjectInfo... organizationAndProjectInfos) { this.organizationAndProjectInfos = List.of(organizationAndProjectInfos); } + public InMemoryImpl( + List organizationAndProjectInfos, + List prompts) { + this.organizationAndProjectInfos = organizationAndProjectInfos; + this.prompts.addAll(prompts); + } + @Override public Project getOrCreateProject(String projectName) { // Find existing project by name @@ -293,6 +356,55 @@ public Optional getProjectAndOrgInfo(String projectI .filter(orgAndProject -> orgAndProject.project().id().equals(projectId)) .findFirst(); } + + @Override + public Optional getPrompt( + @Nonnull String projectName, @Nonnull String slug, @Nullable String version) { + Objects.requireNonNull(projectName, slug); + List matchingPrompts = + prompts.stream() + .filter( + prompt -> { + // Filter by slug if provided + if (slug != null && !slug.isEmpty()) { + if (!prompt.slug().equals(slug)) { + return false; + } + } + + // Filter by project name if provided + if (projectName != null && !projectName.isEmpty()) { + // Find project by name and check if ID matches + Project project = getOrCreateProject(projectName); + if (!prompt.projectId().equals(project.id())) { + return false; + } + } + + // Filter by version if provided + // Note: Version filtering would require additional metadata + // on Prompt + // For now, we'll skip this as Prompt doesn't have a + // version field + + return true; + }) + .toList(); + + if (matchingPrompts.isEmpty()) { + return Optional.empty(); + } + + if (matchingPrompts.size() > 1) { + throw new ApiException( + "Multiple objects found for slug: " + + slug + + ", projectName: " + + projectName); + } + + return Optional.of(matchingPrompts.get(0)); + } } // Request/Response DTOs @@ -362,4 +474,21 @@ record LoginRequest(String token) {} record LoginResponse(List orgInfo) {} record OrganizationAndProjectInfo(OrganizationInfo orgInfo, Project project) {} + + // Prompt models + record PromptData(Object prompt, Object options) {} + + record Prompt( + String id, + String projectId, + String orgId, + String name, + String slug, + Optional description, + String created, + PromptData promptData, + Optional> tags, + Optional metadata) {} + + record PromptListResponse(List objects) {} } diff --git a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java index cf15493f..957d6a31 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java @@ -1,13 +1,18 @@ package dev.braintrust.instrumentation.openai; import com.openai.client.OpenAIClient; +import com.openai.core.ObjectMappers; import com.openai.models.chat.completions.ChatCompletionCreateParams; import dev.braintrust.instrumentation.openai.otel.OpenAITelemetry; import dev.braintrust.prompt.BraintrustPrompt; import io.opentelemetry.api.OpenTelemetry; +import java.util.HashMap; +import java.util.Map; +import lombok.SneakyThrows; /** Braintrust OpenAI client instrumentation. */ public class BraintrustOpenAI { + /** Instrument openai client with braintrust traces */ public static OpenAIClient wrapOpenAI(OpenTelemetry openTelemetry, OpenAIClient openAIClient) { if ("true".equalsIgnoreCase(System.getenv("BRAINTRUST_X_OTEL_LOGS"))) { @@ -24,7 +29,20 @@ public static OpenAIClient wrapOpenAI(OpenTelemetry openTelemetry, OpenAIClient } } - public static ChatCompletionCreateParams buildChatCompletionsPrompt(BraintrustPrompt prompt) { - throw new RuntimeException("TODO"); + @SneakyThrows + public static ChatCompletionCreateParams buildChatCompletionsPrompt( + BraintrustPrompt prompt, Map parameters) { + var promptMap = new HashMap<>(prompt.getOptions()); + promptMap.put("messages", prompt.renderMessages(parameters)); + var promptJson = ObjectMappers.jsonMapper().writeValueAsString(promptMap); + + var body = + ObjectMappers.jsonMapper() + .readValue(promptJson, ChatCompletionCreateParams.Body.class); + + return ChatCompletionCreateParams.builder() + .body(body) + .additionalBodyProperties(Map.of()) + .build(); } } diff --git a/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java index a68fca3e..b292af11 100644 --- a/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java +++ b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java @@ -1,5 +1,115 @@ package dev.braintrust.prompt; +import dev.braintrust.api.BraintrustApiClient; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + public class BraintrustPrompt { - // TODO + private static final Pattern MUSTACHE_PATTERN = Pattern.compile("\\{\\{([^}]+)\\}\\}"); + + private final BraintrustApiClient.Prompt apiPrompt; + private final Map defaults; + + public BraintrustPrompt(BraintrustApiClient.Prompt apiPrompt) { + this(apiPrompt, Map.of()); + } + + public BraintrustPrompt(BraintrustApiClient.Prompt apiPrompt, Map defaults) { + this.apiPrompt = apiPrompt; + this.defaults = defaults; + } + + public List> renderMessages(Map parameters) { + // get promptData->prompt->messages + Map promptData = (Map) apiPrompt.promptData().prompt(); + List> messages = (List>) promptData.get("messages"); + + if (messages == null) { + throw new RuntimeException("No messages found in prompt data"); + } + + Set usedParameters = new HashSet<>(); + List> renderedMessages = new ArrayList<>(); + + for (Map message : messages) { + Map renderedMessage = new HashMap<>(message); + String content = (String) message.get("content"); + + if (content != null) { + String renderedContent = renderTemplate(content, parameters, usedParameters); + renderedMessage.put("content", renderedContent); + } + + renderedMessages.add(renderedMessage); + } + + // Check if all parameters were used + Set unusedParameters = new HashSet<>(parameters.keySet()); + unusedParameters.removeAll(usedParameters); + if (!unusedParameters.isEmpty()) { + throw new RuntimeException("Unused parameters: " + unusedParameters); + } + + return renderedMessages; + } + + public Map getOptions() { + // get map in promptData->options and merge with promptData->options->params + Map options = (Map) apiPrompt.promptData().options(); + + if (options == null) { + return Map.of(); + } + + Map result = new HashMap<>(); + + // Add all top-level options except "params" + for (Map.Entry entry : options.entrySet()) { + if (!"params".equals(entry.getKey())) { + result.put(entry.getKey(), entry.getValue()); + } + } + + // Merge in the params + Map params = (Map) options.get("params"); + if (params != null) { + result.putAll(params); + } + + // Apply defaults for any values not already set + for (Map.Entry defaultEntry : this.defaults.entrySet()) { + if (!result.containsKey(defaultEntry.getKey())) { + result.put(defaultEntry.getKey(), defaultEntry.getValue()); + } + } + + return result; + } + + private String renderTemplate( + String template, Map parameters, Set usedParameters) { + Matcher matcher = MUSTACHE_PATTERN.matcher(template); + StringBuffer result = new StringBuffer(); + + while (matcher.find()) { + String paramName = matcher.group(1); + String paramValue = parameters.get(paramName); + + if (paramValue == null) { + throw new RuntimeException("Missing parameter: " + paramName); + } + + usedParameters.add(paramName); + matcher.appendReplacement(result, Matcher.quoteReplacement(paramValue)); + } + + matcher.appendTail(result); + return result.toString(); + } } diff --git a/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java b/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java index 896153fa..00bdc1c8 100644 --- a/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java +++ b/src/main/java/dev/braintrust/prompt/BraintrustPromptLoader.java @@ -1,23 +1,48 @@ package dev.braintrust.prompt; +import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.config.BraintrustConfig; +import java.util.LinkedHashMap; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; +/** Load LLM objects from the Braintrust API */ public class BraintrustPromptLoader { - public static BraintrustPromptLoader of(BraintrustConfig config) { - throw new RuntimeException("TODO"); + private final BraintrustConfig config; + private final BraintrustApiClient client; + + private BraintrustPromptLoader(BraintrustConfig config, BraintrustApiClient client) { + this.config = config; + this.client = client; + } + + public static BraintrustPromptLoader of(BraintrustConfig config, BraintrustApiClient client) { + return new BraintrustPromptLoader(config, client); } public BraintrustPrompt load(String promptSlug) { - PromptLoadRequest.builder().promptSlug(promptSlug); - throw new RuntimeException("TODO"); + PromptLoadRequest request = PromptLoadRequest.builder().promptSlug(promptSlug).build(); + return load(request); } public BraintrustPrompt load(PromptLoadRequest promptLoadRequest) { - throw new RuntimeException("TODO"); + var projectName = promptLoadRequest.projectName; + if (null == projectName) { + // TODO: fall back to project ID if appropriate + projectName = config.defaultProjectName().orElseThrow(); + } + // Request the prompt from the Braintrust API + var promptOpt = + client.getPrompt( + projectName, promptLoadRequest.promptSlug, promptLoadRequest.version); + var prompt = + promptOpt.orElseThrow( + () -> + new RuntimeException( + "Prompt not found: " + promptLoadRequest.promptSlug)); + return new BraintrustPrompt(prompt, promptLoadRequest.defaults); } @Builder @@ -25,18 +50,31 @@ public static class PromptLoadRequest { private @Nonnull String promptSlug; private @Nullable String projectName; private @Nullable String version; - private @Nonnull @Builder.Default Map defaults = Map.of(); + @Builder.Default private @Nonnull Map defaults = Map.of(); public static class PromptLoadRequestBuilder { public PromptLoadRequestBuilder defaults(String... keyValuePairDefaults) { - throw new RuntimeException("TODO"); + this.defaults$value = keyValueListToMap(keyValuePairDefaults); + this.defaults$set = true; + return this; } } } - // TODO: what default values are allowed? Do we want to use an enum or is that too constrictive? - public static class PromptLoadDefaultKeys { - public static String MODEL = "model"; - public static String TEMPERATURE = "temperature"; + @SuppressWarnings("unchecked") + private static Map keyValueListToMap(T... keyValueList) { + if (keyValueList.length % 2 != 0) { + throw new IllegalArgumentException( + "keyValueList must contain an even number of elements (key-value pairs)"); + } + + Map map = new LinkedHashMap<>(); + for (int i = 0; i < keyValueList.length; i += 2) { + T key = keyValueList[i]; + T value = keyValueList[i + 1]; + map.put(key, value); + } + + return Map.copyOf(map); } } diff --git a/src/main/java/dev/braintrust/trace/BraintrustTracing.java b/src/main/java/dev/braintrust/trace/BraintrustTracing.java index bcf013bb..0a8259d4 100644 --- a/src/main/java/dev/braintrust/trace/BraintrustTracing.java +++ b/src/main/java/dev/braintrust/trace/BraintrustTracing.java @@ -17,6 +17,7 @@ import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; import java.time.Duration; +import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -31,7 +32,7 @@ public final class BraintrustTracing { public static final String PARENT_KEY = "braintrust.parent"; static final String OTEL_SERVICE_NAME = "braintrust-app"; static final String INSTRUMENTATION_NAME = "braintrust-java"; - static final String INSTRUMENTATION_VERSION = SDKMain.loadVersionFromProperties(); + static final String INSTRUMENTATION_VERSION = loadVersionFromProperties(); /** * Quick start method that sets up global OpenTelemetry with Braintrust defaults.
@@ -166,5 +167,15 @@ private static String sdkInfoLogMessage() { System.getProperty("java.vm.name")); } + static String loadVersionFromProperties() { + try (var is = BraintrustTracing.class.getResourceAsStream("/braintrust.properties")) { + var props = new Properties(); + props.load(is); + return props.getProperty("sdk.version"); + } catch (Exception e) { + throw new RuntimeException("unable to determine sdk version", e); + } + } + private BraintrustTracing() {} } diff --git a/src/test/java/dev/braintrust/BraintrustTest.java b/src/test/java/dev/braintrust/BraintrustTest.java new file mode 100644 index 00000000..55e1f28f --- /dev/null +++ b/src/test/java/dev/braintrust/BraintrustTest.java @@ -0,0 +1,167 @@ +package dev.braintrust; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.config.BraintrustConfig; +import io.opentelemetry.api.GlobalOpenTelemetry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class BraintrustTest { + + @BeforeEach + void setUp() { + // Reset global OpenTelemetry before each test + GlobalOpenTelemetry.resetForTest(); + } + + @AfterEach + void tearDown() { + // Clean up after tests + GlobalOpenTelemetry.resetForTest(); + } + + @Test + void testOfCreatesNewInstance() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust = Braintrust.of(config); + + assertNotNull(braintrust); + assertNotNull(braintrust.config()); + assertNotNull(braintrust.apiClient()); + assertNotNull(braintrust.promptLoader()); + assertEquals(config, braintrust.config()); + } + + @Test + void testGetCreatesGlobalInstance() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust1 = Braintrust.get(config); + var braintrust2 = Braintrust.get(); + + // Should return the same instance + assertSame(braintrust1, braintrust2); + } + + @Test + void testOpenTelemetryCreate() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust = Braintrust.of(config); + + var openTelemetry = braintrust.openTelemetryCreate(false); // Don't register global + assertNotNull(openTelemetry); + assertNotNull(openTelemetry.getTracer("test")); + } + + @Test + void testOpenTelemetryCreateRegistersGlobal() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust = Braintrust.of(config); + + var openTelemetry = braintrust.openTelemetryCreate(true); // Register global + assertNotNull(openTelemetry); + + // Verify it was registered globally + var globalOtel = GlobalOpenTelemetry.get(); + assertNotNull(globalOtel); + assertNotNull(globalOtel.getTracer("test")); + } + + @Test + void testOpenTelemetryCreateDefaultRegistersGlobal() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust = Braintrust.of(config); + + var openTelemetry = braintrust.openTelemetryCreate(); // Default should register global + assertNotNull(openTelemetry); + + // Verify it was registered globally + var globalOtel = GlobalOpenTelemetry.get(); + assertNotNull(globalOtel); + } + + @Test + void testEvalBuilder() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_ID", + "test-project-id", + "BRAINTRUST_API_KEY", + "test-key"); + var braintrust = Braintrust.of(config); + + var evalBuilder = braintrust.evalBuilder(); + assertNotNull(evalBuilder); + } + + @Test + void testMultipleOfCallsCreateIndependentInstances() { + var config1 = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "project-1", + "BRAINTRUST_API_KEY", + "test-key"); + var config2 = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "project-2", + "BRAINTRUST_API_KEY", + "test-key"); + + var braintrust1 = Braintrust.of(config1); + var braintrust2 = Braintrust.of(config2); + + // Should be different instances + assertNotSame(braintrust1, braintrust2); + assertNotEquals( + braintrust1.config().defaultProjectName().get(), + braintrust2.config().defaultProjectName().get()); + } + + @Test + void testGetReturnsSameInstanceAfterInitialization() { + var config = + BraintrustConfig.of( + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project", + "BRAINTRUST_API_KEY", + "test-key"); + + // First call initializes + var braintrust1 = Braintrust.get(config); + + // Subsequent calls return same instance + var braintrust2 = Braintrust.get(); + var braintrust3 = Braintrust.get(config); + + assertSame(braintrust1, braintrust2); + assertSame(braintrust1, braintrust3); + } +} diff --git a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java index 79262de7..70d2349c 100644 --- a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java @@ -10,19 +10,19 @@ import com.openai.client.OpenAIClient; import com.openai.client.okhttp.OpenAIOkHttpClient; import com.openai.models.ChatModel; -import com.openai.models.chat.completions.ChatCompletionContentPart; -import com.openai.models.chat.completions.ChatCompletionContentPartImage; -import com.openai.models.chat.completions.ChatCompletionContentPartText; -import com.openai.models.chat.completions.ChatCompletionCreateParams; -import com.openai.models.chat.completions.ChatCompletionStreamOptions; -import com.openai.models.chat.completions.ChatCompletionUserMessageParam; +import com.openai.models.chat.completions.*; +import dev.braintrust.api.BraintrustApiClient; import dev.braintrust.config.BraintrustConfig; +import dev.braintrust.prompt.BraintrustPrompt; import dev.braintrust.trace.Base64Attachment; import dev.braintrust.trace.BraintrustTracing; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.sdk.OpenTelemetrySdk; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeEach; @@ -455,4 +455,61 @@ void testWrapOpenAiWithImageAttachment() { + " shows the Eiffel Tower in Paris, France.\"}],\"finish_reason\":\"stop\"}]", outputJson); } + + @Test + @SneakyThrows + void testBuildChatCompletionsPrompt() { + Map promptContent = + Map.of( + "type", + "chat", + "messages", + List.of( + Map.of( + "role", "system", + "content", + "You are a kind chatbot who briefly greets people"), + Map.of( + "role", "user", + "content", "What's up my friend? My name is {{name}}"))); + + Map options = + Map.of( + "model", + "gpt-4o-mini", + "params", + Map.of("temperature", 0.1, "max_tokens", 102)); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(promptContent, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "test-project-id", + "test-org-id", + "kind-greeter", + "kind-greeter-test", + Optional.of("Test prompt"), + "2025-10-21T21:35:18.287Z", + promptData, + Optional.empty(), + Optional.empty()); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + Map parameters = Map.of("name", "Alice"); + ChatCompletionCreateParams renderedParams = + BraintrustOpenAI.buildChatCompletionsPrompt(prompt, parameters); + + assertEquals( + ChatCompletionCreateParams.builder() + .model(ChatModel.GPT_4O_MINI) + .temperature(0.1) + .maxTokens(102L) + .addSystemMessage("You are a kind chatbot who briefly greets people") + .addUserMessage("What's up my friend? My name is Alice") + .build(), + renderedParams); + } } diff --git a/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java b/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java new file mode 100644 index 00000000..21718774 --- /dev/null +++ b/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java @@ -0,0 +1,182 @@ +package dev.braintrust.prompt; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.api.BraintrustApiClient; +import dev.braintrust.config.BraintrustConfig; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class BraintrustPromptLoaderTest { + + @Test + void testLoadPromptBySlug() { + // Create test data + BraintrustApiClient.OrganizationInfo orgInfo = + new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); + BraintrustApiClient.Project project = + new BraintrustApiClient.Project( + "proj-456", "test-project", "org-123", "2025-01-01", "2025-01-01"); + BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = + new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); + + // Create a test prompt + BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); + + // Create in-memory API client with the test prompt + BraintrustApiClient apiClient = + new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); + + // Create config + BraintrustConfig config = + BraintrustConfig.of( + "BRAINTRUST_API_KEY", + "doesntmatter", + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project"); + + // Create loader + BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); + + // Load the prompt + BraintrustPrompt prompt = loader.load("kind-greeter"); + + // Verify the prompt was loaded correctly + assertNotNull(prompt); + + // Test rendering + Map parameters = Map.of("name", "Bob"); + List> renderedMessages = prompt.renderMessages(parameters); + + assertEquals(2, renderedMessages.size()); + assertEquals("What's up my friend? My name is Bob", renderedMessages.get(1).get("content")); + } + + @Test + void testLoadPromptWithDefaults() { + // Create test data + BraintrustApiClient.OrganizationInfo orgInfo = + new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); + BraintrustApiClient.Project project = + new BraintrustApiClient.Project( + "proj-456", "test-project", "org-123", "2025-01-01", "2025-01-01"); + BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = + new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); + + // Create a test prompt + BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); + + // Create in-memory API client with the test prompt + BraintrustApiClient apiClient = + new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); + + // Create config + BraintrustConfig config = + BraintrustConfig.of( + "BRAINTRUST_API_KEY", + "doesntmatter", + "BRAINTRUST_DEFAULT_PROJECT_NAME", + "test-project"); + + // Create loader + BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); + + // Load the prompt with defaults + BraintrustPrompt prompt = + loader.load( + BraintrustPromptLoader.PromptLoadRequest.builder() + .promptSlug("kind-greeter") + .defaults("max_tokens", "2000", "top_p", "0.95") + .build()); + + // Verify defaults are applied + Map options = prompt.getOptions(); + assertEquals("2000", options.get("max_tokens")); + assertEquals("0.95", options.get("top_p")); + + // Verify original options are preserved + assertEquals("gpt-4o-mini", options.get("model")); + assertEquals(0, options.get("temperature")); + } + + @Test + void testLoadPromptWithProjectName() { + // Create test data + BraintrustApiClient.OrganizationInfo orgInfo = + new BraintrustApiClient.OrganizationInfo("org-123", "Test Org"); + BraintrustApiClient.Project project = + new BraintrustApiClient.Project( + "proj-456", "my-project", "org-123", "2025-01-01", "2025-01-01"); + BraintrustApiClient.OrganizationAndProjectInfo orgAndProject = + new BraintrustApiClient.OrganizationAndProjectInfo(orgInfo, project); + + // Create a test prompt + BraintrustApiClient.Prompt testPrompt = createTestPrompt(project.id()); + + // Create in-memory API client with the test prompt + BraintrustApiClient apiClient = + new BraintrustApiClient.InMemoryImpl(List.of(orgAndProject), List.of(testPrompt)); + + // Create config without default project name + BraintrustConfig config = BraintrustConfig.of("BRAINTRUST_API_KEY", "test-key"); + + // Create loader + BraintrustPromptLoader loader = BraintrustPromptLoader.of(config, apiClient); + + // Load the prompt with explicit project name + BraintrustPrompt prompt = + loader.load( + BraintrustPromptLoader.PromptLoadRequest.builder() + .promptSlug("kind-greeter") + .projectName("my-project") + .build()); + + // Verify the prompt was loaded correctly + assertNotNull(prompt); + } + + private BraintrustApiClient.Prompt createTestPrompt(String projectId) { + // Create the prompt data structure matching the example JSON + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", + "You are a kind chatbot who briefly greets people"), + Map.of( + "role", "user", + "content", "What's up my friend? My name is {{name}}"))); + + Map options = + Map.of( + "model", "gpt-4o-mini", + "params", + Map.of( + "use_cache", + true, + "temperature", + 0, + "response_format", + Map.of("type", "text")), + "position", "0|hzzzzz:"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + return new BraintrustApiClient.Prompt( + "e2a4fb20-e97e-4e8a-be07-b226d55047b2", + projectId, + "e8d257dd-944c-479a-9916-40a9fa09f120", + "kind-greeter", + "kind-greeter", + Optional.of("A very good boi"), + "2025-10-21T21:35:18.287Z", + promptData, + Optional.empty(), + Optional.empty()); + } +} diff --git a/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java b/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java new file mode 100644 index 00000000..0a70ddb6 --- /dev/null +++ b/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java @@ -0,0 +1,212 @@ +package dev.braintrust.prompt; + +import static org.junit.jupiter.api.Assertions.*; + +import dev.braintrust.api.BraintrustApiClient; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class BraintrustPromptTest { + + @Test + void testRenderMessagesWithParameters() { + // Create a test prompt object + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Render messages with parameters + Map parameters = Map.of("name", "Alice"); + List> renderedMessages = prompt.renderMessages(parameters); + + // Verify the messages were rendered correctly + assertEquals(2, renderedMessages.size()); + + Map systemMessage = renderedMessages.get(0); + assertEquals("system", systemMessage.get("role")); + assertEquals( + "You are a kind chatbot who briefly greets people", systemMessage.get("content")); + + Map userMessage = renderedMessages.get(1); + assertEquals("user", userMessage.get("role")); + assertEquals("What's up my friend? My name is Alice", userMessage.get("content")); + } + + @Test + void testRenderMessagesWithMissingParameter() { + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Try to render without providing the required parameter + Map parameters = Map.of(); + + RuntimeException exception = + assertThrows( + RuntimeException.class, + () -> { + prompt.renderMessages(parameters); + }); + + assertTrue(exception.getMessage().contains("Missing parameter: name")); + } + + @Test + void testRenderMessagesWithUnusedParameter() { + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Provide extra parameters that aren't used + Map parameters = Map.of("name", "Alice", "unused", "value"); + + RuntimeException exception = + assertThrows( + RuntimeException.class, + () -> { + prompt.renderMessages(parameters); + }); + + assertTrue(exception.getMessage().contains("Unused parameters")); + assertTrue(exception.getMessage().contains("unused")); + } + + @Test + void testGetOptions() { + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + Map options = prompt.getOptions(); + + // Verify that top-level options and params are merged + assertEquals("gpt-4o-mini", options.get("model")); + assertEquals(true, options.get("use_cache")); + assertEquals(0, options.get("temperature")); + + Map responseFormat = (Map) options.get("response_format"); + assertNotNull(responseFormat); + assertEquals("text", responseFormat.get("type")); + + // Verify that "params" itself is not in the result + assertFalse(options.containsKey("params")); + + // Verify that "position" (a top-level option) is included + assertEquals("0|hzzzzz:", options.get("position")); + } + + @Test + void testGetOptionsWithDefaults() { + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + + // Create a prompt with defaults + Map defaults = + Map.of( + "max_tokens", "1000", + "temperature", + "0.7", // This should be ignored as temperature is already set to 0 + "top_p", "0.9"); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject, defaults); + + Map options = prompt.getOptions(); + + // Verify that defaults are applied only when not already set + assertEquals("1000", options.get("max_tokens")); // Applied from defaults + assertEquals("0.9", options.get("top_p")); // Applied from defaults + assertEquals( + 0, + options.get( + "temperature")); // NOT overridden by defaults (original value preserved) + + // Verify original options are still present + assertEquals("gpt-4o-mini", options.get("model")); + assertEquals(true, options.get("use_cache")); + } + + @Test + void testRenderMessagesWithMalformedMustache() { + // Create a prompt with malformed mustache syntax + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", "You are a helpful assistant"), + Map.of( + "role", "user", + "content", "Hello {{ whatever. This should not match."))); + + Map options = Map.of("model", "gpt-4o-mini"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Render with empty parameters - malformed mustache should be ignored + Map parameters = Map.of(); + List> renderedMessages = prompt.renderMessages(parameters); + + // Verify the malformed mustache is left as-is (not treated as a parameter) + assertEquals(2, renderedMessages.size()); + assertEquals( + "Hello {{ whatever. This should not match.", + renderedMessages.get(1).get("content")); + } + + private BraintrustApiClient.Prompt createTestPrompt() { + // Create the prompt data structure matching the example JSON + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", + "You are a kind chatbot who briefly greets people"), + Map.of( + "role", "user", + "content", "What's up my friend? My name is {{name}}"))); + + Map options = + Map.of( + "model", "gpt-4o-mini", + "params", + Map.of( + "use_cache", + true, + "temperature", + 0, + "response_format", + Map.of("type", "text")), + "position", "0|hzzzzz:"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + return new BraintrustApiClient.Prompt( + "e2a4fb20-e97e-4e8a-be07-b226d55047b2", + "e8d257dd-944c-479a-9916-40a9fa09f120", + "5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e", + "kind-greeter", + "kind-greeter-69d2", + Optional.of("A very good boi"), + "2025-10-21T21:35:18.287Z", + promptData, + Optional.empty(), + Optional.empty()); + } +} From 7eab143f6d37a045e157c5827040cbfbb013b519 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Thu, 23 Oct 2025 13:10:17 -0600 Subject: [PATCH 3/3] use proper mustache library to render prompt templates --- build.gradle | 2 + .../openai/BraintrustOpenAI.java | 2 +- .../braintrust/prompt/BraintrustPrompt.java | 55 +-- .../openai/BraintrustOpenAITest.java | 2 +- .../prompt/BraintrustPromptLoaderTest.java | 2 +- .../prompt/BraintrustPromptTest.java | 382 ++++++++++++++---- 6 files changed, 336 insertions(+), 109 deletions(-) diff --git a/build.gradle b/build.gradle index 2270a0a3..fd63e259 100644 --- a/build.gradle +++ b/build.gradle @@ -66,6 +66,8 @@ dependencies { implementation 'org.apache.commons:commons-lang3:3.14.0' implementation 'com.google.code.findbugs:jsr305:3.0.2' // for @Nullable annotations + implementation "com.github.spullara.mustache.java:compiler:0.9.14" + testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}" testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" diff --git a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java index 957d6a31..e9d4cf16 100644 --- a/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java +++ b/src/main/java/dev/braintrust/instrumentation/openai/BraintrustOpenAI.java @@ -31,7 +31,7 @@ public static OpenAIClient wrapOpenAI(OpenTelemetry openTelemetry, OpenAIClient @SneakyThrows public static ChatCompletionCreateParams buildChatCompletionsPrompt( - BraintrustPrompt prompt, Map parameters) { + BraintrustPrompt prompt, Map parameters) { var promptMap = new HashMap<>(prompt.getOptions()); promptMap.put("messages", prompt.renderMessages(parameters)); var promptJson = ObjectMappers.jsonMapper().writeValueAsString(promptMap); diff --git a/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java index b292af11..56e2ffcd 100644 --- a/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java +++ b/src/main/java/dev/braintrust/prompt/BraintrustPrompt.java @@ -1,18 +1,17 @@ package dev.braintrust.prompt; +import com.github.mustachejava.DefaultMustacheFactory; +import com.github.mustachejava.Mustache; +import com.github.mustachejava.MustacheException; import dev.braintrust.api.BraintrustApiClient; +import java.io.StringReader; +import java.io.StringWriter; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class BraintrustPrompt { - private static final Pattern MUSTACHE_PATTERN = Pattern.compile("\\{\\{([^}]+)\\}\\}"); - private final BraintrustApiClient.Prompt apiPrompt; private final Map defaults; @@ -25,7 +24,7 @@ public BraintrustPrompt(BraintrustApiClient.Prompt apiPrompt, Map> renderMessages(Map parameters) { + public List> renderMessages(Map parameters) { // get promptData->prompt->messages Map promptData = (Map) apiPrompt.promptData().prompt(); List> messages = (List>) promptData.get("messages"); @@ -34,7 +33,6 @@ public List> renderMessages(Map parameters) throw new RuntimeException("No messages found in prompt data"); } - Set usedParameters = new HashSet<>(); List> renderedMessages = new ArrayList<>(); for (Map message : messages) { @@ -42,20 +40,13 @@ public List> renderMessages(Map parameters) String content = (String) message.get("content"); if (content != null) { - String renderedContent = renderTemplate(content, parameters, usedParameters); + String renderedContent = renderTemplate(content, parameters); renderedMessage.put("content", renderedContent); } renderedMessages.add(renderedMessage); } - // Check if all parameters were used - Set unusedParameters = new HashSet<>(parameters.keySet()); - unusedParameters.removeAll(usedParameters); - if (!unusedParameters.isEmpty()) { - throw new RuntimeException("Unused parameters: " + unusedParameters); - } - return renderedMessages; } @@ -92,24 +83,22 @@ public Map getOptions() { return result; } - private String renderTemplate( - String template, Map parameters, Set usedParameters) { - Matcher matcher = MUSTACHE_PATTERN.matcher(template); - StringBuffer result = new StringBuffer(); - - while (matcher.find()) { - String paramName = matcher.group(1); - String paramValue = parameters.get(paramName); - - if (paramValue == null) { - throw new RuntimeException("Missing parameter: " + paramName); + private String renderTemplate(String template, Map parameters) { + try { + DefaultMustacheFactory factory = new DefaultMustacheFactory(); + Mustache mustache = factory.compile(new StringReader(template), "template"); + StringWriter writer = new StringWriter(); + mustache.execute(writer, parameters); + writer.flush(); + return writer.toString(); + } catch (MustacheException e) { + // If the template is malformed, just return it as-is + return template; + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; } - - usedParameters.add(paramName); - matcher.appendReplacement(result, Matcher.quoteReplacement(paramValue)); + throw new RuntimeException("Failed to render template", e); } - - matcher.appendTail(result); - return result.toString(); } } diff --git a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java index 70d2349c..f3b21952 100644 --- a/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java +++ b/src/test/java/dev/braintrust/instrumentation/openai/BraintrustOpenAITest.java @@ -498,7 +498,7 @@ void testBuildChatCompletionsPrompt() { BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - Map parameters = Map.of("name", "Alice"); + Map parameters = Map.of("name", "Alice"); ChatCompletionCreateParams renderedParams = BraintrustOpenAI.buildChatCompletionsPrompt(prompt, parameters); diff --git a/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java b/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java index 21718774..96fa790f 100644 --- a/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java +++ b/src/test/java/dev/braintrust/prompt/BraintrustPromptLoaderTest.java @@ -47,7 +47,7 @@ void testLoadPromptBySlug() { assertNotNull(prompt); // Test rendering - Map parameters = Map.of("name", "Bob"); + Map parameters = Map.of("name", "Bob"); List> renderedMessages = prompt.renderMessages(parameters); assertEquals(2, renderedMessages.size()); diff --git a/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java b/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java index 0a70ddb6..17a61adc 100644 --- a/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java +++ b/src/test/java/dev/braintrust/prompt/BraintrustPromptTest.java @@ -10,6 +10,79 @@ public class BraintrustPromptTest { + @Test + void testGetOptionsWithDefaults() { + BraintrustApiClient.Prompt promptObject = createTestPrompt(); + + // Create a prompt with defaults + Map defaults = + Map.of( + "max_tokens", "1000", + "temperature", + "0.7", // This should be ignored as temperature is already set to 0 + "top_p", "0.9"); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject, defaults); + + Map options = prompt.getOptions(); + + // Verify that defaults are applied only when not already set + assertEquals("1000", options.get("max_tokens")); // Applied from defaults + assertEquals("0.9", options.get("top_p")); // Applied from defaults + assertEquals( + 0, + options.get( + "temperature")); // NOT overridden by defaults (original value preserved) + + // Verify original options are still present + assertEquals("gpt-4o-mini", options.get("model")); + assertEquals(true, options.get("use_cache")); + } + + @Test + void testRenderMessagesWithMalformedMustache() { + // Create a prompt with malformed mustache syntax + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", "You are a helpful assistant"), + Map.of( + "role", "user", + "content", "Hello {{ whatever. This should not match."))); + + Map options = Map.of("model", "gpt-4o-mini"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Render with empty parameters - malformed mustache should be ignored + Map parameters = Map.of(); + List> renderedMessages = prompt.renderMessages(parameters); + + // Verify the malformed mustache is left as-is (not treated as a parameter) + assertEquals(2, renderedMessages.size()); + assertEquals( + "Hello {{ whatever. This should not match.", + renderedMessages.get(1).get("content")); + } + @Test void testRenderMessagesWithParameters() { // Create a test prompt object @@ -18,7 +91,7 @@ void testRenderMessagesWithParameters() { BraintrustPrompt prompt = new BraintrustPrompt(promptObject); // Render messages with parameters - Map parameters = Map.of("name", "Alice"); + Map parameters = Map.of("name", "Alice"); List> renderedMessages = prompt.renderMessages(parameters); // Verify the messages were rendered correctly @@ -35,106 +108,270 @@ void testRenderMessagesWithParameters() { } @Test - void testRenderMessagesWithMissingParameter() { - BraintrustApiClient.Prompt promptObject = createTestPrompt(); + void testRenderMessagesWithList() { + // Create a prompt that uses Mustache list iteration + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", "You are a helpful assistant"), + Map.of( + "role", + "user", + "content", + "Here are the items:\n" + + "{{#items}}- {{name}}: {{description}}\n" + + "{{/items}}"))); + + Map options = Map.of("model", "gpt-4o-mini"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - // Try to render without providing the required parameter - Map parameters = Map.of(); + // Render with list parameters + Map parameters = + Map.of( + "items", + List.of( + Map.of("name", "Apple", "description", "A red fruit"), + Map.of("name", "Banana", "description", "A yellow fruit"), + Map.of("name", "Cherry", "description", "A small red fruit"))); - RuntimeException exception = - assertThrows( - RuntimeException.class, - () -> { - prompt.renderMessages(parameters); - }); + List> renderedMessages = prompt.renderMessages(parameters); - assertTrue(exception.getMessage().contains("Missing parameter: name")); + assertEquals(2, renderedMessages.size()); + String expectedContent = + "Here are the items:\n" + + "- Apple: A red fruit\n" + + "- Banana: A yellow fruit\n" + + "- Cherry: A small red fruit\n"; + assertEquals(expectedContent, renderedMessages.get(1).get("content")); } @Test - void testRenderMessagesWithUnusedParameter() { - BraintrustApiClient.Prompt promptObject = createTestPrompt(); - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + void testRenderMessagesWithEmptyList() { + // Create a prompt that uses Mustache list iteration + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", + "user", + "content", + "Items: {{#items}}{{name}} {{/items}}{{^items}}No items" + + " found{{/items}}"))); + + Map options = Map.of("model", "gpt-4o-mini"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); - // Provide extra parameters that aren't used - Map parameters = Map.of("name", "Alice", "unused", "value"); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - RuntimeException exception = - assertThrows( - RuntimeException.class, - () -> { - prompt.renderMessages(parameters); - }); + // Render with empty list + Map parameters = Map.of("items", List.of()); + List> renderedMessages = prompt.renderMessages(parameters); - assertTrue(exception.getMessage().contains("Unused parameters")); - assertTrue(exception.getMessage().contains("unused")); + assertEquals(1, renderedMessages.size()); + assertEquals("Items: No items found", renderedMessages.get(0).get("content")); } @Test - void testGetOptions() { - BraintrustApiClient.Prompt promptObject = createTestPrompt(); - BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + void testRenderMessagesWithConditional() { + // Create a prompt that uses Mustache conditionals + Map messages = + Map.of( + "messages", + List.of( + Map.of( + "role", "system", + "content", "You are a helpful assistant"), + Map.of( + "role", + "user", + "content", + "Hello {{name}}!{{#isAdmin}} You have admin" + + " privileges.{{/isAdmin}}{{^isAdmin}} You are a" + + " regular user.{{/isAdmin}}"))); - Map options = prompt.getOptions(); + Map options = Map.of("model", "gpt-4o-mini"); - // Verify that top-level options and params are merged - assertEquals("gpt-4o-mini", options.get("model")); - assertEquals(true, options.get("use_cache")); - assertEquals(0, options.get("temperature")); + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); - Map responseFormat = (Map) options.get("response_format"); - assertNotNull(responseFormat); - assertEquals("text", responseFormat.get("type")); + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - // Verify that "params" itself is not in the result - assertFalse(options.containsKey("params")); + // Test with admin user + Map adminParameters = Map.of("name", "Alice", "isAdmin", true); + List> adminMessages = prompt.renderMessages(adminParameters); + assertEquals( + "Hello Alice! You have admin privileges.", adminMessages.get(1).get("content")); - // Verify that "position" (a top-level option) is included - assertEquals("0|hzzzzz:", options.get("position")); + // Test with regular user + Map regularParameters = Map.of("name", "Bob", "isAdmin", false); + List> regularMessages = prompt.renderMessages(regularParameters); + assertEquals("Hello Bob! You are a regular user.", regularMessages.get(1).get("content")); } @Test - void testGetOptionsWithDefaults() { - BraintrustApiClient.Prompt promptObject = createTestPrompt(); - - // Create a prompt with defaults - Map defaults = + void testRenderMessagesWithNestedObjects() { + // Create a prompt that uses nested object properties + Map messages = Map.of( - "max_tokens", "1000", - "temperature", - "0.7", // This should be ignored as temperature is already set to 0 - "top_p", "0.9"); - BraintrustPrompt prompt = new BraintrustPrompt(promptObject, defaults); + "messages", + List.of( + Map.of( + "role", + "user", + "content", + "User: {{user.firstName}} {{user.lastName}}\n" + + "Email: {{user.contact.email}}\n" + + "Phone: {{user.contact.phone}}"))); - Map options = prompt.getOptions(); + Map options = Map.of("model", "gpt-4o-mini"); - // Verify that defaults are applied only when not already set - assertEquals("1000", options.get("max_tokens")); // Applied from defaults - assertEquals("0.9", options.get("top_p")); // Applied from defaults - assertEquals( - 0, - options.get( - "temperature")); // NOT overridden by defaults (original value preserved) + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); - // Verify original options are still present - assertEquals("gpt-4o-mini", options.get("model")); - assertEquals(true, options.get("use_cache")); + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Render with nested object + Map parameters = + Map.of( + "user", + Map.of( + "firstName", + "John", + "lastName", + "Doe", + "contact", + Map.of("email", "john@example.com", "phone", "555-1234"))); + + List> renderedMessages = prompt.renderMessages(parameters); + + assertEquals(1, renderedMessages.size()); + String expectedContent = "User: John Doe\nEmail: john@example.com\nPhone: " + "555-1234"; + assertEquals(expectedContent, renderedMessages.get(0).get("content")); } @Test - void testRenderMessagesWithMalformedMustache() { - // Create a prompt with malformed mustache syntax + void testRenderMessagesWithInvertedSection() { + // Create a prompt that uses inverted sections (renders when value is false/empty) Map messages = Map.of( "messages", List.of( Map.of( - "role", "system", - "content", "You are a helpful assistant"), + "role", + "user", + "content", + "{{#hasError}}Error:" + + " {{errorMessage}}{{/hasError}}{{^hasError}}All" + + " systems operational{{/hasError}}"))); + + Map options = Map.of("model", "gpt-4o-mini"); + + BraintrustApiClient.PromptData promptData = + new BraintrustApiClient.PromptData(messages, options); + + BraintrustApiClient.Prompt promptObject = + new BraintrustApiClient.Prompt( + "test-id", + "proj-id", + "org-id", + "test-prompt", + "test-slug", + Optional.empty(), + "2025-01-01T00:00:00Z", + promptData, + Optional.empty(), + Optional.empty()); + + BraintrustPrompt prompt = new BraintrustPrompt(promptObject); + + // Test without error + Map noErrorParams = Map.of("hasError", false); + List> noErrorMessages = prompt.renderMessages(noErrorParams); + assertEquals("All systems operational", noErrorMessages.get(0).get("content")); + + // Test with error + Map errorParams = + Map.of("hasError", true, "errorMessage", "Database connection failed"); + List> errorMessages = prompt.renderMessages(errorParams); + assertEquals("Error: Database connection failed", errorMessages.get(0).get("content")); + } + + @Test + void testRenderMessagesWithComplexTypes() { + // Test that non-string types are properly rendered + Map messages = + Map.of( + "messages", + List.of( Map.of( - "role", "user", - "content", "Hello {{ whatever. This should not match."))); + "role", + "user", + "content", + "Count: {{count}}\n" + + "Price: ${{price}}\n" + + "Enabled: {{enabled}}"))); Map options = Map.of("model", "gpt-4o-mini"); @@ -156,15 +393,14 @@ void testRenderMessagesWithMalformedMustache() { BraintrustPrompt prompt = new BraintrustPrompt(promptObject); - // Render with empty parameters - malformed mustache should be ignored - Map parameters = Map.of(); + // Test with various data types + Map parameters = Map.of("count", 42, "price", 19.99, "enabled", true); + List> renderedMessages = prompt.renderMessages(parameters); - // Verify the malformed mustache is left as-is (not treated as a parameter) - assertEquals(2, renderedMessages.size()); + assertEquals(1, renderedMessages.size()); assertEquals( - "Hello {{ whatever. This should not match.", - renderedMessages.get(1).get("content")); + "Count: 42\nPrice: $19.99\nEnabled: true", renderedMessages.get(0).get("content")); } private BraintrustApiClient.Prompt createTestPrompt() {