Skip to content

Commit 32fbcce

Browse files
committed
squash bot discovered issues
fixes #101 fixes #69 also added new specs for these cases: braintrustdata/braintrust-spec#69
1 parent 83dd628 commit 32fbcce

48 files changed

Lines changed: 1633 additions & 69 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CONTRIBUTING.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,33 @@ Because the SDK is new and under active development, third-party contribution be
2121

2222
See [AGENTS.md](./AGENTS.md) for best practices developing, testing, and releasing the SDK.
2323

24+
### Cross-language instrumentation specs
25+
26+
Canonical span input/output and token metrics belong in
27+
[braintrust-spec](https://github.com/braintrustdata/braintrust-spec), not duplicated
28+
provider unit tests. Keep Java-only adapter boundaries and deterministic edge cases
29+
(such as absent usage fields or integer overflow) in unit tests.
30+
31+
BTX fetches the revision pinned by `braintrustSpecRef` in `gradle.properties`.
32+
To develop against a separate spec worktree without changing that pin:
33+
34+
```bash
35+
export BTX_SPEC_ROOT=/path/to/braintrust-spec/test/llm_span
36+
VCR_MODE=off ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
37+
VCR_MODE=record ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
38+
VCR_MODE=replay ./gradlew :btx:test -Pbtx.spec.filter=openai/responses_,google/thinking
39+
```
40+
41+
The Responses cases exercise string input and instructions through the direct
42+
OpenAI SDK, plus equivalent chat messages through LangChain4j. The Google thinking
43+
case checks positive reasoning counts and inclusive completion-token accounting.
44+
The runner implements the shared `is_google_usage_metadata_normalized` predicate;
45+
it must not treat that predicate as a generic non-null assertion.
46+
47+
Before merging, pin a remotely fetchable spec revision, unset `BTX_SPEC_ROOT`, and
48+
run `./gradlew :btx:test` in replay mode to verify the default consumption path.
49+
Record cassettes through VCR; never edit them by hand.
50+
2451
### CodeQL result caching
2552

2653
CI installs the recommended CodeQL bundle before looking up cached SARIF results.

‎braintrust-sdk/instrumentation/genai_1_18_0/src/main/java/com/google/genai/BraintrustApiClient.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,9 +152,18 @@ private void tagSpan(
152152
if (usage.containsKey("promptTokenCount")) {
153153
metrics.put("prompt_tokens", (Number) usage.get("promptTokenCount"));
154154
}
155-
if (usage.containsKey("candidatesTokenCount")) {
155+
Number candidates = (Number) usage.get("candidatesTokenCount");
156+
Number thoughts = (Number) usage.get("thoughtsTokenCount");
157+
// Gemini reports candidates separately from thoughts; completion includes both.
158+
// A missing component contributes zero only when the other is reported.
159+
if (candidates != null || thoughts != null) {
156160
metrics.put(
157-
"completion_tokens", (Number) usage.get("candidatesTokenCount"));
161+
"completion_tokens",
162+
(candidates == null ? 0L : candidates.longValue())
163+
+ (thoughts == null ? 0L : thoughts.longValue()));
164+
}
165+
if (thoughts != null) {
166+
metrics.put("completion_reasoning_tokens", thoughts);
158167
}
159168
if (usage.containsKey("totalTokenCount")) {
160169
metrics.put("tokens", (Number) usage.get("totalTokenCount"));

‎braintrust-sdk/instrumentation/genai_1_18_0/src/test/java/com/google/genai/BraintrustApiClientTest.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.junit.jupiter.api.BeforeEach;
2727
import org.junit.jupiter.api.Test;
2828
import org.junit.jupiter.params.ParameterizedTest;
29+
import org.junit.jupiter.params.provider.CsvSource;
2930
import org.junit.jupiter.params.provider.ValueSource;
3031

3132
/** Transport-level coverage without credentials, network access, or recorded batch jobs. */
@@ -227,6 +228,41 @@ void llmEndpointsAreTagged(String endpoint) throws Exception {
227228
}
228229
}
229230

231+
@ParameterizedTest
232+
@CsvSource(
233+
delimiter = '|',
234+
value = {
235+
"{\"promptTokenCount\":8,\"candidatesTokenCount\":7,\"totalTokenCount\":15}"
236+
+ "|{\"prompt_tokens\":8,\"completion_tokens\":7,\"tokens\":15}",
237+
"{\"candidatesTokenCount\":7,\"thoughtsTokenCount\":0}"
238+
+ "|{\"completion_tokens\":7,\"completion_reasoning_tokens\":0}",
239+
"{\"promptTokenCount\":8,\"thoughtsTokenCount\":12,\"totalTokenCount\":20}"
240+
+ "|{\"prompt_tokens\":8,\"completion_tokens\":12,\"completion_reasoning_tokens\":12,\"tokens\":20}",
241+
"{}|{}",
242+
"{\"candidatesTokenCount\":null,\"thoughtsTokenCount\":null}|{}",
243+
"{\"candidatesTokenCount\":2147483647,\"thoughtsTokenCount\":1}"
244+
+ "|{\"completion_tokens\":2147483648,\"completion_reasoning_tokens\":1}"
245+
})
246+
void normalizesPartialAndBoundaryUsage(String usage, String expected) throws Exception {
247+
delegate.response = "{\"candidates\":[],\"usageMetadata\":" + usage + "}";
248+
try (ApiResponse response =
249+
invoke("post", "models/gemini:generateContent", "{}", false, false)) {
250+
response.getBody().string();
251+
assertEquals(JSON.readTree(expected), JSON.readTree(attribute("braintrust.metrics")));
252+
}
253+
}
254+
255+
@ParameterizedTest
256+
@ValueSource(strings = {"{}", "{\"usageMetadata\":null}"})
257+
void missingUsageDoesNotInventMetrics(String output) throws Exception {
258+
delegate.response = output;
259+
try (ApiResponse response =
260+
invoke("post", "models/gemini:generateContent", "{}", false, false)) {
261+
response.getBody().string();
262+
assertNull(attribute("braintrust.metrics"));
263+
}
264+
}
265+
230266
@Test
231267
void asynchronousNonLlmSpansRetainParentAndEndOnlyWhenCompleted() throws Exception {
232268
for (boolean bytes : List.of(false, true)) {

‎braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -322,8 +322,15 @@ private static void tagOpenAIRequest(
322322
metadata.put("model", requestJson.get("model").asText());
323323
}
324324
putGenerationParameters(metadata, requestJson);
325-
// Chat completions API uses "messages"; Responses API uses "input"
326-
if (requestJson.has("messages")) {
325+
// Only Responses requests normalize input and instructions into message entries.
326+
if ("POST".equals(method)
327+
&& !pathSegments.isEmpty()
328+
&& "responses".equals(pathSegments.get(pathSegments.size() - 1))) {
329+
JsonNode input = openAIResponsesInput(requestJson);
330+
if (input != null) {
331+
span.setAttribute("braintrust.input_json", toJson(input));
332+
}
333+
} else if (requestJson.has("messages")) {
327334
span.setAttribute("braintrust.input_json", toJson(requestJson.get("messages")));
328335
} else if (requestJson.has("input") && requestJson.get("input").isArray()) {
329336
span.setAttribute("braintrust.input_json", toJson(requestJson.get("input")));
@@ -333,6 +340,29 @@ private static void tagOpenAIRequest(
333340
span.setAttribute("braintrust.metadata", toJson(metadata));
334341
}
335342

343+
@Nullable
344+
private static JsonNode openAIResponsesInput(JsonNode requestJson) {
345+
JsonNode input = requestJson.get("input");
346+
JsonNode instructions = requestJson.get("instructions");
347+
boolean hasInstructions = instructions != null && instructions.isTextual();
348+
boolean hasTextInput = input != null && input.isTextual();
349+
if (!hasInstructions && !hasTextInput) {
350+
return input != null && input.isArray() ? input : null;
351+
}
352+
353+
ArrayNode messages = BraintrustJsonMapper.get().createArrayNode();
354+
if (hasInstructions) {
355+
messages.addObject().put("role", "system").set("content", instructions);
356+
}
357+
if (hasTextInput) {
358+
messages.addObject().put("role", "user").set("content", input);
359+
} else if (input != null && input.isArray()) {
360+
// Reuse heterogeneous entries, including tool calls/results and multimodal content.
361+
messages.addAll((ArrayNode) input);
362+
}
363+
return messages;
364+
}
365+
336366
@SneakyThrows
337367
private static void tagOpenAIResponse(
338368
Span span, JsonNode responseJson, @Nullable Long timeToFirstTokenNanoseconds) {
@@ -1160,8 +1190,8 @@ private static JsonNode maybeParseJsonString(JsonNode value) {
11601190
/**
11611191
* Request fields that are conversation content, not generation parameters. Each is already
11621192
* captured as the span's input, so copying it into metadata would duplicate a potentially large
1163-
* payload. {@code instructions} (the Responses API's system prompt) belongs in the span input
1164-
* too — tracked separately — so it is withheld here rather than landing in metadata.
1193+
* payload. {@code instructions} (the Responses API's system prompt) is prepended to the span
1194+
* input as a system message, rather than landing in metadata.
11651195
*
11661196
* <p>The list spans every endpoint reachable through a wrapped client, not just chat: {@code
11671197
* prompt} carries the user's text on legacy OpenAI completions, Anthropic's legacy {@code

‎braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java‎

Lines changed: 182 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -390,16 +390,192 @@ void clientToolUseAndPlainTextEmitNothing() {
390390
* braintrust.input_json} attribute (null when none was set).
391391
*/
392392
private String inputJsonForRequest(String provider, String requestBody) {
393+
return inputJsonForRequest(provider, List.of("v1", "messages"), "POST", requestBody);
394+
}
395+
396+
private String inputJsonForRequest(
397+
String provider, List<String> pathSegments, String method, String requestBody) {
393398
Span span = tracer.spanBuilder("llm").startSpan();
394399
InstrumentationSemConv.tagLLMSpanRequest(
395-
span,
396-
provider,
397-
"https://api.anthropic.com",
398-
List.of("v1", "messages"),
400+
span, provider, "https://api.example.com", pathSegments, method, requestBody);
401+
span.end();
402+
var spans = exporter.getFinishedSpanItems();
403+
return spans.get(spans.size() - 1).getAttributes().get(INPUT_JSON);
404+
}
405+
406+
private String responsesInput(String requestBody) {
407+
return inputJsonForRequest(
408+
InstrumentationSemConv.PROVIDER_NAME_OPENAI,
409+
List.of("v1", "responses"),
399410
"POST",
400411
requestBody);
401-
span.end();
402-
return exporter.getFinishedSpanItems().get(0).getAttributes().get(INPUT_JSON);
412+
}
413+
414+
@Test
415+
void openAiResponsesRecognizesDeploymentPrefixedEndpoint() {
416+
String body =
417+
"""
418+
{"instructions": "be terse", "input": "hello"}
419+
""";
420+
assertEquals(
421+
json(
422+
"""
423+
[{"role": "system", "content": "be terse"},
424+
{"role": "user", "content": "hello"}]
425+
"""),
426+
json(
427+
inputJsonForRequest(
428+
InstrumentationSemConv.PROVIDER_NAME_OPENAI,
429+
List.of("openai", "deployments", "my-model", "responses"),
430+
"POST",
431+
body)));
432+
}
433+
434+
@Test
435+
void openAiResponsesArrayPreservesToolAndMultimodalEntriesAfterInstructions() {
436+
String body =
437+
"""
438+
{
439+
"instructions": "be terse",
440+
"input": [
441+
{"role": "user", "content": [
442+
{"type": "input_text", "text": "describe this"},
443+
{"type": "input_image", "image_url": "https://example.com/image.png"}
444+
]},
445+
{"type": "function_call", "id": "fc_1", "call_id": "call_1",
446+
"name": "get_weather", "arguments": "{\\"city\\":\\"SF\\"}"},
447+
{"type": "function_call_output", "call_id": "call_1", "output": "sunny"},
448+
{"type": "item_reference", "id": "msg_previous"}
449+
]
450+
}
451+
""";
452+
JsonNode originalInput = json(body).get("input");
453+
JsonNode captured = json(responsesInput(body));
454+
assertEquals(originalInput.size() + 1, captured.size());
455+
assertEquals(
456+
json(
457+
"""
458+
{"role": "system", "content": "be terse"}
459+
"""),
460+
captured.get(0));
461+
for (int i = 0; i < originalInput.size(); i++) {
462+
assertEquals(originalInput.get(i), captured.get(i + 1));
463+
}
464+
465+
var withoutInstructions = json(body).deepCopy();
466+
((com.fasterxml.jackson.databind.node.ObjectNode) withoutInstructions)
467+
.remove("instructions");
468+
assertEquals(originalInput, json(responsesInput(withoutInstructions.toString())));
469+
}
470+
471+
@Test
472+
void openAiResponsesInstructionsOnlyContinuationOmitsMissingOrNullInput() {
473+
for (String body :
474+
List.of(
475+
"""
476+
{"instructions": "continue tersely", "previous_response_id": "resp_1"}
477+
""",
478+
"""
479+
{"instructions": "continue tersely", "previous_response_id": "resp_1",
480+
"input": null}
481+
""")) {
482+
assertEquals(
483+
json(
484+
"""
485+
[{"role": "system", "content": "continue tersely"}]
486+
"""),
487+
json(responsesInput(body)));
488+
}
489+
}
490+
491+
@Test
492+
void openAiResponsesAbsentOrNullInstructionsDoNotAddSystemMessage() {
493+
JsonNode expected =
494+
json(
495+
"""
496+
[{"role": "user", "content": "hello"}]
497+
""");
498+
for (String body :
499+
List.of(
500+
"""
501+
{"input": "hello"}
502+
""",
503+
"""
504+
{"instructions": null, "input": "hello"}
505+
""",
506+
"""
507+
{"input": [{"role": "user", "content": "hello"}]}
508+
""",
509+
"""
510+
{"instructions": null, "input": [{"role": "user", "content": "hello"}]}
511+
""")) {
512+
assertEquals(expected, json(responsesInput(body)));
513+
}
514+
for (String body :
515+
List.of(
516+
"{}",
517+
"{\"instructions\": null}",
518+
"{\"input\": null}",
519+
"{\"instructions\": null, \"input\": null}")) {
520+
assertNull(responsesInput(body), body);
521+
}
522+
}
523+
524+
@Test
525+
void openAiResponsesEmptyContentIsPreserved() {
526+
assertEquals(json("[]"), json(responsesInput("{\"input\": []}")));
527+
assertEquals(
528+
json(
529+
"""
530+
[{"role": "system", "content": ""}, {"role": "user", "content": ""}]
531+
"""),
532+
json(responsesInput("{\"instructions\": \"\", \"input\": \"\"}")));
533+
}
534+
535+
@Test
536+
void openAiNonResponsesEndpointsDoNotNormalizeInputOrInstructions() {
537+
for (List<String> path :
538+
List.of(
539+
List.<String>of(),
540+
List.of("v1", "embeddings"),
541+
List.of("v1", "moderations"),
542+
List.of("v1", "audio", "speech"),
543+
List.of("v1", "chat", "completions"),
544+
List.of("v1", "responses", "resp_1"),
545+
List.of("responses", "batches"),
546+
List.of("v1", "not-responses"))) {
547+
assertEquals(
548+
json("[\"hello\", \"world\"]"),
549+
json(
550+
inputJsonForRequest(
551+
InstrumentationSemConv.PROVIDER_NAME_OPENAI,
552+
path,
553+
"POST",
554+
"""
555+
{"instructions": "be terse", "input": ["hello", "world"]}
556+
""")),
557+
path.toString());
558+
}
559+
assertNull(
560+
inputJsonForRequest(
561+
InstrumentationSemConv.PROVIDER_NAME_OPENAI,
562+
List.of("v1", "responses"),
563+
"GET",
564+
"{\"instructions\": \"be terse\", \"input\": \"hello\"}"));
565+
assertEquals(
566+
json(
567+
"""
568+
[{"role": "user", "content": "chat message"}]
569+
"""),
570+
json(
571+
inputJsonForRequest(
572+
InstrumentationSemConv.PROVIDER_NAME_OPENAI,
573+
List.of("v1", "chat", "completions"),
574+
"POST",
575+
"""
576+
{"messages": [{"role": "user", "content": "chat message"}],
577+
"instructions": "be terse", "input": "hello"}
578+
""")));
403579
}
404580

405581
@Test

‎btx/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ dependencies {
116116
test {
117117
useJUnitPlatform()
118118
workingDir = rootProject.projectDir
119+
inputs.file(rootProject.file('scripts/spec-fetch.sh'))
119120

120121
// Hand the isolated client classpaths to the test JVM (resolved lazily so configuration
121122
// time doesn't force dependency resolution).
@@ -146,6 +147,7 @@ tasks.register('fetchSpec', Exec) {
146147
description = 'Fetches braintrust-spec YAML files for the pinned ref.'
147148

148149
inputs.property('specRef', specRef)
150+
inputs.file(rootProject.file('scripts/spec-fetch.sh'))
149151
outputs.dir(specOutputDir)
150152

151153
def outDir = specOutputDir.get().asFile

0 commit comments

Comments
 (0)