Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -392,10 +392,7 @@ private static void tagOpenAIResponse(
metrics.put("completion_tokens", usage.get("completion_tokens"));
if (usage.has("total_tokens")) metrics.put("tokens", usage.get("total_tokens"));
if (usage.has("prompt_tokens_details")) {
JsonNode details = usage.get("prompt_tokens_details");
if (details.has("cached_tokens")) {
metrics.put("prompt_cached_tokens", details.get("cached_tokens"));
}
addOpenAICacheMetrics(metrics, usage.get("prompt_tokens_details"));
}
// Reasoning tokens (Chat Completions API)
if (usage.has("completion_tokens_details")) {
Expand All @@ -414,10 +411,7 @@ private static void tagOpenAIResponse(
usage.get("input_tokens").asLong() + usage.get("output_tokens").asLong());
}
if (usage.has("input_tokens_details")) {
JsonNode details = usage.get("input_tokens_details");
if (details.has("cached_tokens")) {
metrics.put("prompt_cached_tokens", details.get("cached_tokens"));
}
addOpenAICacheMetrics(metrics, usage.get("input_tokens_details"));
}
// Reasoning tokens (Responses API)
if (usage.has("output_tokens_details")) {
Expand All @@ -433,6 +427,26 @@ private static void tagOpenAIResponse(
}
}

private static void addOpenAICacheMetrics(Map<String, Object> metrics, JsonNode details) {
if (details.hasNonNull("cached_tokens")) {
metrics.put("prompt_cached_tokens", details.get("cached_tokens"));
}
JsonNode writes = details.path("cache_write_tokens_details");
boolean has30m = writes.hasNonNull("ttl_30m_tokens");
boolean has12h = writes.hasNonNull("ttl_12h_tokens");
if (has30m || has12h) {
// The TTL buckets replace the aggregate, and are already included in input tokens.
if (has30m) {
metrics.put("prompt_cache_creation_30m_tokens", writes.get("ttl_30m_tokens"));
}
if (has12h) {
metrics.put("prompt_cache_creation_12h_tokens", writes.get("ttl_12h_tokens"));
}
} else if (details.hasNonNull("cache_write_tokens")) {
metrics.put("prompt_cache_creation_tokens", details.get("cache_write_tokens"));
}
}

private static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}";

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,96 @@ private SpanData tagOpenAIResponse(String responseBody) {
return byName(exporter.getFinishedSpanItems(), "llm");
}

@Test
void tagsOpenAIMixedCacheTtlsWithoutDoubleCounting() {
for (String detailsKey : List.of("input_tokens_details", "prompt_tokens_details")) {
exporter.reset();
SpanData span =
tagOpenAIResponse(
"""
{
"usage": {
"prompt_tokens": 12500,
"completion_tokens": 420,
"total_tokens": 12920,
"%s": {
"cached_tokens": 2048,
"cache_write_tokens": 9600,
"cache_write_tokens_details": {
"ttl_30m_tokens": 1600,
"ttl_12h_tokens": 8000
}
}
}
}
"""
.formatted(detailsKey));
assertEquals(
json(
"""
{"prompt_tokens":12500,"completion_tokens":420,"tokens":12920,
"prompt_cached_tokens":2048,"prompt_cache_creation_30m_tokens":1600,
"prompt_cache_creation_12h_tokens":8000}
"""),
json(span.getAttributes().get(METRICS)));
}
}

@Test
void preservesOpenAIZeroAndPartialCacheSplits() {
SpanData span =
tagOpenAIResponse(
"""
{"usage":{"input_tokens":20,"output_tokens":3,"input_tokens_details":{
"cached_tokens":0,"cache_write_tokens":0,
"cache_write_tokens_details":{"ttl_30m_tokens":0,"ttl_12h_tokens":null}
}}}
""");
assertEquals(
json(
"""
{"prompt_tokens":20,"completion_tokens":3,"tokens":23,
"prompt_cached_tokens":0,"prompt_cache_creation_30m_tokens":0}
"""),
json(span.getAttributes().get(METRICS)));
}

@Test
void fallsBackToReportedOpenAIAggregateWithoutTtlSplit() {
SpanData span =
tagOpenAIResponse(
"""
{"usage":{"input_tokens":200,"output_tokens":3,"input_tokens_details":{
"cache_write_tokens":160,"cache_write_tokens_details":null
}}}
""");
assertEquals(
json(
"""
{"prompt_tokens":200,"completion_tokens":3,"tokens":203,
"prompt_cache_creation_tokens":160}
"""),
json(span.getAttributes().get(METRICS)));
}

@Test
void doesNotInventOpenAICacheWritesForOlderModels() {
SpanData span =
tagOpenAIResponse(
"""
{"usage":{"input_tokens":200,"output_tokens":3,"input_tokens_details":{
"cached_tokens":128
}}}
""");
assertEquals(
json(
"""
{"prompt_tokens":200,"completion_tokens":3,"tokens":203,
"prompt_cached_tokens":128}
"""),
json(span.getAttributes().get(METRICS)));
}

@Test
void tagsOpenAIAudioTranscriptionOutput() {
SpanData span = tagOpenAIResponse("{\"text\":\"Hello from the recording.\"}");
Expand Down
2 changes: 1 addition & 1 deletion btx/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ dependencies {
testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1'

// OpenAI SDK
testImplementation 'com.openai:openai-java:2.8.1'
testImplementation 'com.openai:openai-java:2.15.0'

// Anthropic SDK
testImplementation 'com.anthropic:anthropic-java:2.10.0'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,19 @@ public final class TokenAccountingSpec {
private static final String CACHE_CREATE = "prompt_cache_creation_tokens";
private static final String CACHE_CREATE_5M = "prompt_cache_creation_5m_tokens";
private static final String CACHE_CREATE_1H = "prompt_cache_creation_1h_tokens";
private static final String CACHE_CREATE_30M = "prompt_cache_creation_30m_tokens";
private static final String CACHE_CREATE_12H = "prompt_cache_creation_12h_tokens";

/** Detail metrics that must not exceed {@code prompt_tokens}. */
private static final List<String> PROMPT_SUBSET_METRICS =
List.of(CACHED, CACHE_CREATE, CACHE_CREATE_5M, CACHE_CREATE_1H, "prompt_audio_tokens");
List.of(
CACHED,
CACHE_CREATE,
CACHE_CREATE_5M,
CACHE_CREATE_1H,
CACHE_CREATE_30M,
CACHE_CREATE_12H,
"prompt_audio_tokens");

/** Detail metrics that must not exceed {@code completion_tokens}. */
private static final List<String> COMPLETION_SUBSET_METRICS =
Expand Down Expand Up @@ -245,7 +254,10 @@ private static String spanName(Map<String, Object> span) {
private static long effectiveCacheCreationTokens(Map<String, Long> counts) {
long aggregate = counts.getOrDefault(CACHE_CREATE, 0L);
long split =
counts.getOrDefault(CACHE_CREATE_5M, 0L) + counts.getOrDefault(CACHE_CREATE_1H, 0L);
counts.getOrDefault(CACHE_CREATE_5M, 0L)
+ counts.getOrDefault(CACHE_CREATE_1H, 0L)
+ counts.getOrDefault(CACHE_CREATE_30M, 0L)
+ counts.getOrDefault(CACHE_CREATE_12H, 0L);
return Math.max(aggregate, split);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,31 @@ void detailWithoutParentIsClean() {
@Nested
class CacheRollIn {

@Test
void openAiTtlWritesAndReadsMustFitWithinPrompt() {
var valid =
metrics(
"prompt_tokens", 12500,
"prompt_cached_tokens", 2048,
"prompt_cache_creation_30m_tokens", 1600,
"prompt_cache_creation_12h_tokens", 8000);
assertClean(valid);
valid.put("prompt_tokens", 11000);
assertViolation(valid, "exceeds prompt_tokens");
}

@Test
void openAiTtlCountsMustBeNonNegativeIntegers() {
assertViolation(
metrics("prompt_cache_creation_30m_tokens", -1),
"prompt_cache_creation_30m_tokens",
"non-negative");
assertViolation(
metrics("prompt_cache_creation_12h_tokens", 1.5),
"prompt_cache_creation_12h_tokens",
"integer");
}

/**
* The regression this whole helper exists for: Anthropic and Bedrock report their native
* input count exclusive of cache tokens, so copying it into prompt_tokens as-is leaves the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ public String provider() {

@Override
public boolean supports(LlmSpanSpec spec) {
// LangChain4j 1.19 exposes legacy promptCacheRetention, but cannot send
// prompt_cache_options or explicit breakpoints for OpenAI's new TTLs.
if (spec.requests().stream().anyMatch(r -> r.containsKey("prompt_cache_options"))) {
return false;
}
return "/v1/chat/completions".equals(spec.endpoint())
|| "/v1/responses".equals(spec.endpoint());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,20 @@ private void executeResponses(
}
var params = paramsBuilder.build();

Response response = client(ctx).responses().create(params);
Response response;
if (Boolean.TRUE.equals(request.get("stream"))) {
try (var stream = client(ctx).responses().createStreaming(params)) {
response =
stream.stream()
.flatMap(event -> event.completed().stream())
.map(event -> event.response())
.reduce((previous, current) -> current)
.orElseThrow(
() -> new IllegalStateException("Response did not complete"));
}
} else {
response = client(ctx).responses().create(params);
}

// Accumulate this turn's input + output into history for the next turn
history.addAll(thisInput);
Expand Down
2 changes: 1 addition & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ org.gradle.daemon=true
org.gradle.warning.mode=summary

# braintrust-spec git ref (SHA or tag) used by btx tests
braintrustSpecRef=c3c80aa77cc722164dc6748400853b9a5b2512be
braintrustSpecRef=bf985fb4d63e78ca04fa883537768b7f7e12e907

# braintrust-openapi commit SHA used by braintrust-api
braintrustOpenApiRef=4cc0615694512d6e77fce6bff2c70d1531208daa
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading