Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 5 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ dependencies {
### Evals

```java
var config = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.of(config, true);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());

Function<String, String> getFoodType =
Expand All @@ -49,11 +49,8 @@ Function<String, String> getFoodType =
return response.choices().get(0).message().content().orElse("").toLowerCase();
};

var eval =
Eval.<String, String>builder()
var eval = braintrust.<String, String>evalBuilder()
.name("java-eval-x-" + System.currentTimeMillis())
.tracer(BraintrustTracing.getTracer(openTelemetry))
.config(config)
.cases(
EvalCase.of("asparagus", "vegetable"),
EvalCase.of("banana", "fruit"))
Expand All @@ -73,8 +70,8 @@ System.out.println("\n\n" + result.createReportString());
### OpenAI Tracing

```java
var braintrustConfig = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
OpenAIClient openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());

var request =
Expand Down
4 changes: 3 additions & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ dependencies {
implementation 'org.apache.commons:commons-lang3:3.14.0'
implementation 'com.google.code.findbugs:jsr305:3.0.2' // for @Nullable annotations

implementation "com.github.spullara.mustache.java:compiler:0.9.14"

testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}"
testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}"
testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"
Expand Down Expand Up @@ -219,7 +221,7 @@ jar {
'Implementation-Title': 'Braintrust Java SDK',
'Implementation-Version': version,
'Implementation-Vendor': 'Braintrust',
'Main-Class': 'dev.braintrust.trace.SDKMain'
'Main-Class': 'dev.braintrust.SDKMain'
)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@
import com.anthropic.client.okhttp.AnthropicOkHttpClient;
import com.anthropic.models.messages.MessageCreateParams;
import com.anthropic.models.messages.Model;
import dev.braintrust.config.BraintrustConfig;
import dev.braintrust.Braintrust;
import dev.braintrust.instrumentation.anthropic.BraintrustAnthropic;
import dev.braintrust.trace.BraintrustTracing;

/** Basic OTel + Anthropic instrumentation example */
public class AnthropicInstrumentationExample {
Expand All @@ -18,9 +17,9 @@ public static void main(String[] args) throws Exception {
+ " fail.\n");
}

var braintrustConfig = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
var tracer = BraintrustTracing.getTracer(openTelemetry);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
var tracer = openTelemetry.getTracer("my-instrumentation");

// Wrap Anthropic client with Braintrust instrumentation
AnthropicClient anthropicClient =
Expand All @@ -35,7 +34,7 @@ public static void main(String[] args) throws Exception {
}

var url =
braintrustConfig.fetchProjectURI()
braintrust.projectUri()
+ "/logs?r=%s&s=%s"
.formatted(
rootSpan.getSpanContext().getTraceId(),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package dev.braintrust.examples;

import dev.braintrust.config.BraintrustConfig;
import dev.braintrust.trace.BraintrustTracing;
import dev.braintrust.Braintrust;
import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
Expand Down Expand Up @@ -51,8 +50,8 @@ public static void main(String[] args) throws Exception {
// NOTE: there are many ways to set up otel builders, etc.
// The important line is here: call enable with your otel builders and braintrust will
// export open telemetry data in addition to your existing setup
var braintrustConfig = BraintrustConfig.fromEnvironment();
BraintrustTracing.enable(braintrustConfig, tracerBuilder, loggerBuilder, meterBuilder);
var braintrust = Braintrust.get();
braintrust.openTelemetryEnable(tracerBuilder, loggerBuilder, meterBuilder);

var openTelemetry =
OpenTelemetrySdk.builder()
Expand All @@ -74,7 +73,7 @@ public static void main(String[] args) throws Exception {
span.end();
}
var url =
braintrustConfig.fetchProjectURI()
braintrust.projectUri()
+ "/logs?r=%s&s=%s"
.formatted(
span.getSpanContext().getTraceId(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,16 @@
import com.openai.client.okhttp.OpenAIOkHttpClient;
import com.openai.models.ChatModel;
import com.openai.models.chat.completions.ChatCompletionCreateParams;
import dev.braintrust.config.BraintrustConfig;
import dev.braintrust.eval.Eval;
import dev.braintrust.Braintrust;
import dev.braintrust.eval.EvalCase;
import dev.braintrust.eval.Scorer;
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
import dev.braintrust.trace.BraintrustTracing;
import java.util.function.Function;

public class ExperimentExample {
public static void main(String[] args) throws Exception {
var config = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.of(config, true);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());

Function<String, String> getFoodType =
Expand All @@ -32,13 +30,12 @@ public static void main(String[] args) throws Exception {
};

var eval =
Eval.<String, String>builder()
braintrust
.<String, String>evalBuilder()
.name("java-eval-x-" + System.currentTimeMillis()) // NOTE: if you use a
// constant, additional runs
// will append new cases to
// the same experiment
.tracer(BraintrustTracing.getTracer(openTelemetry))
.config(config)
.cases(
EvalCase.of("strawberry", "fruit"),
EvalCase.of("asparagus", "vegetable"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,31 +5,34 @@
import com.openai.models.ChatModel;
import com.openai.models.chat.completions.ChatCompletionCreateParams;
import com.openai.models.chat.completions.ChatCompletionStreamOptions;
import dev.braintrust.config.BraintrustConfig;
import dev.braintrust.Braintrust;
import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
import dev.braintrust.trace.BraintrustTracing;

/** Basic OTel + OpenAI instrumentation example */
public class OpenAIInstrumentationExample {

public static void main(String[] args) throws Exception {
if (null == System.getenv("OPENAI_API_KEY")) {
System.err.println(
"\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n");
}
var braintrustConfig = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.of(braintrustConfig, true);
var tracer = BraintrustTracing.getTracer(openTelemetry);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
OpenAIClient openAIClient =
BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
var rootSpan = tracer.spanBuilder("openai-java-instrumentation-example").startSpan();
var rootSpan =
openTelemetry
.getTracer("my-instrumentation")
.spanBuilder("openai-java-instrumentation-example")
.startSpan();
try (var ignored = rootSpan.makeCurrent()) {
chatCompletionsExample(openAIClient);
// chatCompletionsStreamingExample(openAIClient);
} finally {
rootSpan.end();
}
var url =
braintrustConfig.fetchProjectURI()
braintrust.projectUri()
+ "/logs?r=%s&s=%s"
.formatted(
rootSpan.getSpanContext().getTraceId(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package dev.braintrust.examples;

import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.buildChatCompletionsPrompt;
import static dev.braintrust.instrumentation.openai.BraintrustOpenAI.wrapOpenAI;

import com.openai.client.OpenAIClient;
import com.openai.client.okhttp.OpenAIOkHttpClient;
import dev.braintrust.Braintrust;
import dev.braintrust.prompt.BraintrustPromptLoader.PromptLoadRequest;
import java.util.Map;

public class PromptFetchingExample {
public static void main(String... args) {
if (null == System.getenv("OPENAI_API_KEY")) {
System.err.println(
"\nWARNING envar OPEN_AI_API_KEY not found. This example will likely fail.\n");
}
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();

OpenAIClient openAIClient = wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());

{ // simple example
var prompt = braintrust.promptLoader().load("kind-greeter-69d2");
var response =
openAIClient
.chat()
.completions()
.create(
buildChatCompletionsPrompt(
prompt, Map.of("name", "Sam Malone")));
System.out.println("got response: %s".formatted(response.choices().get(0).message()));
}
{ // more complex prompt options
var prompt =
braintrust
.promptLoader()
.load(
PromptLoadRequest.builder()
.projectName("andrew-misc")
.promptSlug("unkind-greeter-fd4c")
.version("cbbc711da9f7d445")
.defaults("model", "gpt-3.5-turbo")
.build());
var response =
openAIClient
.chat()
.completions()
.create(
buildChatCompletionsPrompt(
prompt, Map.of("name", "Frasier Crane")));
System.out.println("got response: %s".formatted(response));
}

System.out.println(
"\n\n Example complete! View your data in Braintrust: %s\n"
.formatted(braintrust.projectUri() + "/logs"));
}
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package dev.braintrust.examples;

import dev.braintrust.config.BraintrustConfig;
import dev.braintrust.trace.BraintrustTracing;
import dev.braintrust.Braintrust;

public class SimpleOpenTelemetryExample {
public static void main(String[] args) throws Exception {
var braintrustConfig = BraintrustConfig.fromEnvironment();
var openTelemetry = BraintrustTracing.quickstart();
var tracer = BraintrustTracing.getTracer(openTelemetry);
var braintrust = Braintrust.get();
var openTelemetry = braintrust.openTelemetryCreate();
var tracer = openTelemetry.getTracer("my-instrumentation");

var span = tracer.spanBuilder("hello-java").startSpan();
try (var ignored = span.makeCurrent()) {
Expand All @@ -18,7 +17,7 @@ public static void main(String[] args) throws Exception {
span.end();
}
var url =
braintrustConfig.fetchProjectURI()
braintrust.projectUri()
+ "/logs?r=%s&s=%s"
.formatted(
span.getSpanContext().getTraceId(),
Expand Down
Loading