From c5f9ed4bc5baeaec2090f77c7429d4936e98b540 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:03:13 +0200 Subject: [PATCH 01/10] test(node): Port Anthropic integration tests to span streaming Co-Authored-By: GPT-6 --- .../anthropic/instrument-span-streaming.mjs | 11 - .../anthropic/instrument-with-options.mjs | 8 - .../tracing/anthropic/instrument-with-pii.mjs | 8 - .../suites/tracing/anthropic/instrument.mjs | 8 - .../suites/tracing/anthropic/test.ts | 204 ++++++++---------- .../v0.129/instrument-default-lifecycle.mjs | 10 - .../tracing/anthropic/v0.129/instrument.mjs | 8 - .../suites/tracing/anthropic/v0.129/test.ts | 5 +- 8 files changed, 88 insertions(+), 174 deletions(-) delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs delete mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs deleted file mode 100644 index 657bed0a3a8c..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-span-streaming.mjs +++ /dev/null @@ -1,11 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: true, outputs: true } }, - transport: loggingTransport, - traceLifecycle: 'stream', -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs index 1046a4b66191..286186a48620 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-options.mjs @@ -2,7 +2,6 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, @@ -14,11 +13,4 @@ Sentry.init({ recordOutputs: true, }), ], - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs index fc8f4e266829..1c507ba84b9c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument-with-pii.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: true, outputs: true } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs index 9b15a0c635e8..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 937ae389ca30..903190f7115c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -17,6 +17,7 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; @@ -33,15 +34,13 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(6); - const nonStreamingSpans = container.items.filter( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(6); + const nonStreamingSpans = spans.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse', ); expect(nonStreamingSpans).toHaveLength(1); @@ -52,7 +51,7 @@ describe('Anthropic integration', () => { expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); } - const streamingSpan = container.items.find( + const streamingSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', ); expect(streamingSpan).toBeDefined(); @@ -60,7 +59,7 @@ describe('Anthropic integration', () => { expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - const rawResponseSpans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); expect(rawResponseSpans).toHaveLength(4); for (const span of rawResponseSpans) { expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); @@ -86,24 +85,22 @@ describe('Anthropic integration', () => { const runner = createRunner(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan).toBeDefined(); expect(errorSpan!.status).toBe('error'); - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); @@ -121,13 +118,13 @@ describe('Anthropic integration', () => { const runner = createRunner(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); @@ -148,7 +145,7 @@ describe('Anthropic integration', () => { expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - const errorSpan = container.items.find( + const errorSpan = spans.find( span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', ); expect(errorSpan).toBeDefined(); @@ -158,9 +155,7 @@ describe('Anthropic integration', () => { // TODO: messages.stream() should produce its own distinct gen_ai span, but it // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); @@ -185,13 +180,13 @@ describe('Anthropic integration', () => { const runner = createRunner(); await runner - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); @@ -199,14 +194,12 @@ describe('Anthropic integration', () => { expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - const errorSpan = container.items.find(span => span.name === 'chat error-model'); + const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan).toBeDefined(); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - const streamingSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123', - ); + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); @@ -219,22 +212,16 @@ describe('Anthropic integration', () => { }); }); - const EXPECTED_STREAM_SPANS_PII_FALSE = { - transaction: 'main', - }; - - const EXPECTED_STREAM_SPANS_PII_TRUE = { - transaction: 'main', - }; - createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { await createRunner() - .expect({ transaction: EXPECTED_STREAM_SPANS_PII_FALSE }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const requestStreamSpans = container.items.filter( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(requestStreamSpans).toHaveLength(2); @@ -258,7 +245,7 @@ describe('Anthropic integration', () => { expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); expect(messagesStreamSpan).toBeDefined(); @@ -283,11 +270,13 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { await createRunner() - .expect({ transaction: EXPECTED_STREAM_SPANS_PII_TRUE }) .expect({ span: container => { - expect(container.items).toHaveLength(3); - const requestStreamSpans = container.items.filter( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(requestStreamSpans).toHaveLength(2); @@ -299,7 +288,7 @@ describe('Anthropic integration', () => { expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); } - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); expect(messagesStreamSpan).toBeDefined(); @@ -333,20 +322,18 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => { test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - const nestedSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested', - ); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); expect(nestedSpan).toBeDefined(); expect(nestedSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); // The helper's own internal `create` delegation must be deduped: exactly one span // for the streamed response, not a duplicate child span. - const streamingSpans = container.items.filter( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', - ); + const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); expect(streamingSpans).toHaveLength(1); }, }) @@ -363,13 +350,13 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .expect({ - transaction: {}, - }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + const segment = container.items.find(span => span.is_segment); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] messages.create with tools — available tools + tool calls recorded with PII expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); @@ -392,13 +379,13 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .expect({ - transaction: {}, - }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - for (const span of container.items) { + const segment = container.items.find(span => span.is_segment); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + for (const span of spans) { expect(span.name).toBe('chat claude-3-haiku-20240307'); expect(span.status).toBe('ok'); expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); @@ -409,11 +396,11 @@ describe('Anthropic integration', () => { } // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. - const createStreamSpan = container.items.find( + const createStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); expect(createStreamSpan).toBeDefined(); - const messagesStreamSpan = container.items.find( + const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); expect(messagesStreamSpan).toBeDefined(); @@ -425,10 +412,6 @@ describe('Anthropic integration', () => { }); // Additional error scenarios - Streaming errors - const EXPECTED_STREAM_ERROR_SPANS = { - transaction: 'main', - }; - createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { await createRunner() @@ -436,11 +419,13 @@ describe('Anthropic integration', () => { // being raised as an unhandled rejection, so the instrumentation captures it. This test only // asserts the spans. .ignore('event') - .expect({ transaction: EXPECTED_STREAM_ERROR_SPANS }) .expect({ span: container => { - expect(container.items).toHaveLength(4); - const createInitErrorSpan = container.items.find( + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(4); + const createInitErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, @@ -449,7 +434,7 @@ describe('Anthropic integration', () => { expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); expect(createInitErrorSpan!.status).toBe('error'); - const streamInitErrorSpan = container.items.find( + const streamInitErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, @@ -458,7 +443,7 @@ describe('Anthropic integration', () => { expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); expect(streamInitErrorSpan!.status).toBe('error'); - const createMidwayErrorSpan = container.items.find( + const createMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', ); expect(createMidwayErrorSpan).toBeDefined(); @@ -467,7 +452,7 @@ describe('Anthropic integration', () => { expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); - const streamMidwayErrorSpan = container.items.find( + const streamMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', ); expect(streamMidwayErrorSpan).toBeDefined(); @@ -483,17 +468,19 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors correctly', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { - expect(container.items).toHaveLength(2); - const invalidFormatSpan = container.items.find(span => span.name === 'chat invalid-format'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); expect(invalidFormatSpan).toBeDefined(); expect(invalidFormatSpan!.status).toBe('error'); expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); expect(invalidFormatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - const toolSuccessSpan = container.items.find(span => span.name === 'chat claude-3-haiku-20240307'); + const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); expect(toolSuccessSpan).toBeDefined(); expect(toolSuccessSpan!.status).toBe('ok'); expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); @@ -512,15 +499,15 @@ describe('Anthropic integration', () => { test('extracts system instructions from messages', async () => { const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() - .expect({ - transaction: { - transaction: 'main', - }, - }) .expect({ span: container => { - expect(container.items).toHaveLength(1); - const [firstSpan] = container.items; + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; // [0] messages.create — system instructions extracted into dedicated attribute expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); @@ -532,32 +519,6 @@ describe('Anthropic integration', () => { }, ); - createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-span-streaming.mjs', (createRunner, test) => { - test('creates anthropic related spans with span streaming enabled', async () => { - await createRunner() - .expect({ - span: container => { - const completionSpan = container.items.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123', - ); - expect(completionSpan).toBeDefined(); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the capital of France?"}]', - ); - expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); - }, - }) - .start() - .completed(); - }); - }); - createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error-shaped responses returned as data', async () => { await createRunner() @@ -576,7 +537,12 @@ describe('Anthropic integration', () => { }, }, }) - .expect({ transaction: { transaction: 'main' } }) + .expect({ + span: container => { + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); + }, + }) .start() .completed(); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs deleted file mode 100644 index cc192fb89834..000000000000 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument-default-lifecycle.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import * as Sentry from '@sentry/node'; -import { loggingTransport } from '@sentry-internal/node-integration-tests'; - -Sentry.init({ - dsn: 'https://public@dsn.ingest.sentry.io/1337', - release: '1.0', - tracesSampleRate: 1.0, - dataCollection: { genAI: { inputs: false, outputs: false } }, - transport: loggingTransport, -}); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs index 9b15a0c635e8..cc192fb89834 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/instrument.mjs @@ -2,17 +2,9 @@ import * as Sentry from '@sentry/node'; import { loggingTransport } from '@sentry-internal/node-integration-tests'; Sentry.init({ - traceLifecycle: 'static', dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, dataCollection: { genAI: { inputs: false, outputs: false } }, transport: loggingTransport, - beforeSendTransaction: event => { - // Filter out mock express server transactions - if (event.transaction.includes('/anthropic/v1/')) { - return null; - } - return event; - }, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 04f9cf75d98a..68cf6f9d6223 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -18,9 +18,10 @@ describe('Anthropic integration (0.129)', () => { (createRunner, test) => { test('emits one span for messages.stream(), not a second one for its internal create', async () => { await createRunner() - .expect({ transaction: { transaction: 'main' } }) .expect({ span: container => { + const segment = container.items.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); const streamingSpans = container.items.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', ); @@ -48,7 +49,7 @@ describe('Anthropic integration (0.129)', () => { createEsmAndCjsTests( __dirname, 'scenario-beta-stream-helpers.mjs', - 'instrument-default-lifecycle.mjs', + 'instrument.mjs', (createRunner, test) => { test('emits one span each for beta.messages.stream() and the eager streaming tool runner', async () => { await createRunner() From 255ccf7c02b0251bba1c881a67bc172cd013c809 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:24:54 +0200 Subject: [PATCH 02/10] test(node): Remove redundant anthropic assertions Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 903190f7115c..723f956522b6 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -54,7 +54,6 @@ describe('Anthropic integration', () => { const streamingSpan = spans.find( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', ); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -92,16 +91,13 @@ describe('Anthropic integration', () => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); const errorSpan = spans.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); expect(errorSpan!.status).toBe('error'); const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -125,7 +121,6 @@ describe('Anthropic integration', () => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); @@ -148,7 +143,6 @@ describe('Anthropic integration', () => { const errorSpan = spans.find( span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', ); - expect(errorSpan).toBeDefined(); expect(errorSpan!.name).toBe('chat error-model'); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); @@ -156,7 +150,6 @@ describe('Anthropic integration', () => { // TODO: messages.stream() should produce its own distinct gen_ai span, but it // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); @@ -187,7 +180,6 @@ describe('Anthropic integration', () => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan).toBeDefined(); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); @@ -195,12 +187,10 @@ describe('Anthropic integration', () => { expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); const errorSpan = spans.find(span => span.name === 'chat error-model'); - expect(errorSpan).toBeDefined(); expect(errorSpan!.status).toBe('error'); expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan).toBeDefined(); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(streamingSpan!.status).toBe('ok'); expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -236,7 +226,6 @@ describe('Anthropic integration', () => { const detailedStreamSpan = requestStreamSpans.find( span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', ); - expect(detailedStreamSpan).toBeDefined(); expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); @@ -248,7 +237,6 @@ describe('Anthropic integration', () => { const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(messagesStreamSpan).toBeDefined(); expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(messagesStreamSpan!.status).toBe('ok'); expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); @@ -291,7 +279,6 @@ describe('Anthropic integration', () => { const messagesStreamSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(messagesStreamSpan).toBeDefined(); expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(messagesStreamSpan!.status).toBe('ok'); expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -328,7 +315,6 @@ describe('Anthropic integration', () => { expect(segment).toBeDefined(); const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); - expect(nestedSpan).toBeDefined(); expect(nestedSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); // The helper's own internal `create` delegation must be deduped: exactly one span @@ -430,7 +416,6 @@ describe('Anthropic integration', () => { span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, ); - expect(createInitErrorSpan).toBeDefined(); expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); expect(createInitErrorSpan!.status).toBe('error'); @@ -439,14 +424,12 @@ describe('Anthropic integration', () => { span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, ); - expect(streamInitErrorSpan).toBeDefined(); expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); expect(streamInitErrorSpan!.status).toBe('error'); const createMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', ); - expect(createMidwayErrorSpan).toBeDefined(); expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); @@ -455,7 +438,6 @@ describe('Anthropic integration', () => { const streamMidwayErrorSpan = spans.find( span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', ); - expect(streamMidwayErrorSpan).toBeDefined(); expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); }, @@ -475,13 +457,11 @@ describe('Anthropic integration', () => { const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(2); const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); - expect(invalidFormatSpan).toBeDefined(); expect(invalidFormatSpan!.status).toBe('error'); expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); expect(invalidFormatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); - expect(toolSuccessSpan).toBeDefined(); expect(toolSuccessSpan!.status).toBe('ok'); expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); }, From 03bbcb8c4eaaa296a1844150db173413aa588a54 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:52:30 +0200 Subject: [PATCH 03/10] test(node): Collect anthropic spans across envelopes Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 107 +++++++++++++----- .../suites/tracing/anthropic/v0.129/test.ts | 15 ++- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 723f956522b6..1b0331934044 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES, @@ -33,12 +34,16 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(6); const nonStreamingSpans = spans.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse', @@ -81,14 +86,17 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording disabled', async () => { - const runner = createRunner(); + const allSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); @@ -111,14 +119,17 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording enabled', async () => { - const runner = createRunner(); + const allSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); @@ -170,14 +181,17 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates anthropic related spans with custom options', async () => { - const runner = createRunner(); + const allSpans: SerializedStreamedSpanContainer['items'] = []; + + const runner = createRunner().unordered(); await runner .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); @@ -204,12 +218,16 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, @@ -257,12 +275,16 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(3); const requestStreamSpans = spans.filter( span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, @@ -308,12 +330,16 @@ describe('Anthropic integration', () => { // helper span is still the active span). Regression test for over-suppression. createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => { test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); expect(nestedSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); @@ -331,16 +357,20 @@ describe('Anthropic integration', () => { // Non-streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('non-streaming sets available tools and tool calls with PII', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get the weather by city","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]'; await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(1); const [firstSpan] = spans; @@ -360,16 +390,20 @@ describe('Anthropic integration', () => { // Streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-stream-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streaming sets available tools and tool calls with PII', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]'; await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(2); for (const span of spans) { expect(span.name).toBe('chat claude-3-haiku-20240307'); @@ -400,16 +434,20 @@ describe('Anthropic integration', () => { // Additional error scenarios - Streaming errors createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() // Stream errors surface via the MessageStream `error` event; attaching that listener stops it // being raised as an unhandled rejection, so the instrumentation captures it. This test only // asserts the spans. .ignore('event') .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(4); const createInitErrorSpan = spans.find( span => @@ -449,12 +487,16 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors correctly', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(2); const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); expect(invalidFormatSpan!.status).toBe('error'); @@ -477,15 +519,17 @@ describe('Anthropic integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const spans = container.items.filter( - span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', - ); + const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(1); const [firstSpan] = spans; @@ -501,6 +545,8 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error-shaped responses returned as data', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() // The API returns the error as data on a 200 response, never as a thrown error to the caller, // so the instrumentation intentionally captures it as an event. @@ -519,7 +565,8 @@ describe('Anthropic integration', () => { }) .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); }, }) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 68cf6f9d6223..a71328a6e7b3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -1,3 +1,4 @@ +import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING } from '@sentry/conventions/attributes'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -17,12 +18,16 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span for messages.stream(), not a second one for its internal create', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { - const segment = container.items.find(span => span.is_segment && span.name === 'main'); + allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); - const streamingSpans = container.items.filter( + const streamingSpans = allSpans.filter( span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', ); expect(streamingSpans).toHaveLength(1); @@ -52,11 +57,15 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span each for beta.messages.stream() and the eager streaming tool runner', async () => { + const allSpans: SerializedStreamedSpanContainer['items'] = []; + await createRunner() + .unordered() .expect({ span: container => { + allSpans.push(...container.items); for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { - const spans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); + const spans = allSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); expect(spans, id).toHaveLength(1); expect(spans[0]!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); } From 550ecdf2de4416af39c26e3ef9998ea95c5c1dbc Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 11:55:28 +0200 Subject: [PATCH 04/10] test(node): Match the intended Anthropic root spans Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 1b0331934044..f89bd9492d04 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -368,7 +368,7 @@ describe('Anthropic integration', () => { .expect({ span: container => { allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(1); @@ -401,7 +401,7 @@ describe('Anthropic integration', () => { .expect({ span: container => { allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); expect(spans).toHaveLength(2); @@ -545,8 +545,6 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error-shaped responses returned as data', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() // The API returns the error as data on a 200 response, never as a thrown error to the caller, // so the instrumentation intentionally captures it as an event. @@ -565,8 +563,7 @@ describe('Anthropic integration', () => { }) .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + const segment = container.items.find(span => span.is_segment && span.name === 'main'); expect(segment).toBeDefined(); }, }) From aaf387f1fcef1af4b68d31fa745727c85d11c3ab Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:06:36 +0200 Subject: [PATCH 05/10] test(node): Use the SENTRY_ORIGIN convention in anthropic tests Co-Authored-By: GPT-6 --- .../node-integration-tests/suites/tracing/anthropic/test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index f89bd9492d04..0b12fff8572d 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -149,7 +149,7 @@ describe('Anthropic integration', () => { expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes['sentry.origin'].value).toBe('auto.ai.anthropic'); + expect(completionSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.anthropic'); const errorSpan = spans.find( span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', From b4f3b2a0e1db0faf36744727afeab34c4e7d2add Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:10:22 +0200 Subject: [PATCH 06/10] test(node): Wait for Anthropic beta helpers to finish before counting spans Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/v0.129/test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index a71328a6e7b3..e282169c7773 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -64,6 +64,8 @@ describe('Anthropic integration (0.129)', () => { .expect({ span: container => { allSpans.push(...container.items); + const segment = allSpans.find(span => span.is_segment && span.name === 'main'); + expect(segment).toBeDefined(); for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { const spans = allSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); expect(spans, id).toHaveLength(1); From f637a470c47aa063133547b4008def2479189c08 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 6 Oct 2026 12:12:34 +0200 Subject: [PATCH 07/10] test(node): Use convention constants for anthropic span attributes Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 15 ++++++++------- .../suites/tracing/anthropic/v0.129/test.ts | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 0b12fff8572d..1dfb37b9092f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -18,6 +18,7 @@ import { GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, SENTRY_ORIGIN, } from '@sentry/conventions/attributes'; import { GEN_AI_REQUEST_STREAM_ATTRIBUTE } from '../../../../../packages/server-utils/src/ai/core/gen-ai-attributes'; @@ -148,7 +149,7 @@ describe('Anthropic integration', () => { expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(completionSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.anthropic'); const errorSpan = spans.find( @@ -196,13 +197,13 @@ describe('Anthropic integration', () => { const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); const errorSpan = spans.find(span => span.name === 'chat error-model'); expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); @@ -341,7 +342,7 @@ describe('Anthropic integration', () => { expect(segment).toBeDefined(); const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); - expect(nestedSpan.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); // The helper's own internal `create` delegation must be deduped: exactly one span // for the streamed response, not a duplicate child span. @@ -377,7 +378,7 @@ describe('Anthropic integration', () => { // [0] messages.create with tools — available tools + tool calls recorded with PII expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); }, @@ -408,7 +409,7 @@ describe('Anthropic integration', () => { for (const span of spans) { expect(span.name).toBe('chat claude-3-haiku-20240307'); expect(span.status).toBe('ok'); - expect(span.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); @@ -501,7 +502,7 @@ describe('Anthropic integration', () => { const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); expect(invalidFormatSpan!.status).toBe('error'); expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); - expect(invalidFormatSpan!.attributes['sentry.op'].value).toBe('gen_ai.chat'); + expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); expect(toolSuccessSpan!.status).toBe('ok'); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index e282169c7773..23f9a1a7aa4a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -1,5 +1,5 @@ import type { SerializedStreamedSpanContainer } from '@sentry/core'; -import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING } from '@sentry/conventions/attributes'; +import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING, SENTRY_OP } from '@sentry/conventions/attributes'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -31,7 +31,7 @@ describe('Anthropic integration (0.129)', () => { span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', ); expect(streamingSpans).toHaveLength(1); - expect(streamingSpans[0]!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ type: 'boolean', value: true, @@ -69,7 +69,7 @@ describe('Anthropic integration (0.129)', () => { for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { const spans = allSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); expect(spans, id).toHaveLength(1); - expect(spans[0]!.attributes['sentry.op']).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); } }, }) From dfa3f3082756b23bd1c409451747ebd1f99fe8fd Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 09:31:30 +0200 Subject: [PATCH 08/10] test(node): Report Anthropic assertions outside envelope matching Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 640 +++++++++--------- .../suites/tracing/anthropic/v0.129/test.ts | 44 +- 2 files changed, 349 insertions(+), 335 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 1dfb37b9092f..c8409b491b51 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -1,4 +1,4 @@ -import type { SerializedStreamedSpanContainer } from '@sentry/core'; +import type { Event, SerializedStreamedSpanContainer } from '@sentry/core'; import { afterAll, describe, expect } from 'vitest'; import { GEN_AI_INPUT_MESSAGES, @@ -35,281 +35,292 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(6); - const nonStreamingSpans = spans.filter( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse', - ); - expect(nonStreamingSpans).toHaveLength(1); - for (const span of nonStreamingSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - } - - const streamingSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', - ); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - - const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); - expect(rawResponseSpans).toHaveLength(4); - for (const span of rawResponseSpans) { - expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - } - expect(rawResponseSpans.map(span => span.attributes[GEN_AI_REQUEST_MODEL].value)).toEqual([ - 'claude-3-haiku-20240307', - 'claude-3-haiku-20240307', - 'claude-2', - 'claude-3-haiku-20240307', - ]); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(6); + const nonStreamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse'); + expect(nonStreamingSpans).toHaveLength(1); + for (const span of nonStreamingSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + } + + const streamingSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', + ); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + + const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + expect(rawResponseSpans).toHaveLength(4); + for (const span of rawResponseSpans) { + expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + } + expect(rawResponseSpans.map(span => span.attributes[GEN_AI_REQUEST_MODEL].value)).toEqual([ + 'claude-3-haiku-20240307', + 'claude-3-haiku-20240307', + 'claude-2', + 'claude-3-haiku-20240307', + ]); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording disabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const runner = createRunner().unordered(); await runner .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - - const errorSpan = spans.find(span => span.name === 'chat error-model'); - expect(errorSpan!.status).toBe('error'); - - const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('creates anthropic related spans with genAI recording enabled', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const runner = createRunner().unordered(); await runner .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); - expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_mock123'); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from Anthropic mock!'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( - '[{"role":"user","content":"What is the capital of France?"}]', - ); - expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.anthropic'); - - const errorSpan = spans.find( - span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', - ); - expect(errorSpan!.name).toBe('chat error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); - - // TODO: messages.stream() should produce its own distinct gen_ai span, but it - // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. - const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(streamingSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(streamingSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); - expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + expect(completionSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(completionSpan!.attributes[GEN_AI_REQUEST_MAX_TOKENS].value).toBe(100); + expect(completionSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(completionSpan!.attributes[GEN_AI_REQUEST_TEMPERATURE].value).toBe(0.7); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_mock123'); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from Anthropic mock!'); + expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES].value).toBe( + '[{"role":"user","content":"What is the capital of France?"}]', + ); + expect(completionSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(completionSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(completionSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(completionSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.anthropic'); + + const errorSpan = spans.find( + span => span.attributes[GEN_AI_INPUT_MESSAGES]?.value === '[{"role":"user","content":"This will fail"}]', + ); + expect(errorSpan!.name).toBe('chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('error-model'); + + // TODO: messages.stream() should produce its own distinct gen_ai span, but it + // currently does not (pre-existing bug). Once fixed, add an additional indexed span assertion. + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(streamingSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(streamingSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); + expect(streamingSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(streamingSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(streamingSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); }); }); createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-options.mjs', (createRunner, test) => { test('creates anthropic related spans with custom options', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedEvent: Event | undefined; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const runner = createRunner().unordered(); await runner .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); - expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(completionSpan!.status).toBe('ok'); - expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); - expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); - - const errorSpan = spans.find(span => span.name === 'chat error-model'); - expect(errorSpan!.status).toBe('error'); - expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - - const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; + }, + }) + .expect({ + event: event => { + receivedEvent = event; }, }) - .expect({ event: EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ event_id: expect.any(String), ...EXPECTED_STREAM_EVENT_HANDLER_MESSAGE }); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const completionSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_mock123'); + expect(completionSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(completionSpan!.status).toBe('ok'); + expect(completionSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(completionSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toBeDefined(); + expect(completionSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeDefined(); + + const errorSpan = spans.find(span => span.name === 'chat error-model'); + expect(errorSpan!.status).toBe('error'); + expect(errorSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + const streamingSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream123'); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); }); }); createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const requestStreamSpans = spans.filter( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(requestStreamSpans).toHaveLength(2); - for (const span of requestStreamSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); - } - - const detailedStreamSpan = requestStreamSpans.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', - ); - expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(detailedStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - - const messagesStreamSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(messagesStreamSpan!.status).toBe('ok'); - expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(messagesStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(messagesStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); + expect(requestStreamSpans).toHaveLength(2); + for (const span of requestStreamSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); + } + + const detailedStreamSpan = requestStreamSpans.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', + ); + expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(detailedStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); + expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(messagesStreamSpan!.status).toBe('ok'); + expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(messagesStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(messagesStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); }); }); createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const requestStreamSpans = spans.filter( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(requestStreamSpans).toHaveLength(2); - for (const span of requestStreamSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); - } - - const messagesStreamSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(messagesStreamSpan!.status).toBe('ok'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); + expect(requestStreamSpans).toHaveLength(2); + for (const span of requestStreamSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); + } + + const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); + expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(messagesStreamSpan!.status).toBe('ok'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); }); }); @@ -331,34 +342,34 @@ describe('Anthropic integration', () => { // helper span is still the active span). Regression test for over-suppression. createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => { test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); - expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - - // The helper's own internal `create` delegation must be deduped: exactly one span - // for the streamed response, not a duplicate child span. - const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); - expect(streamingSpans).toHaveLength(1); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); + expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + // The helper's own internal `create` delegation must be deduped: exactly one span + // for the streamed response, not a duplicate child span. + const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); + expect(streamingSpans).toHaveLength(1); }); }); // Non-streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('non-streaming sets available tools and tool calls with PII', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get the weather by city","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; @@ -368,30 +379,30 @@ describe('Anthropic integration', () => { .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(1); - const [firstSpan] = spans; - - // [0] messages.create with tools — available tools + tool calls recorded with PII - expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] messages.create with tools — available tools + tool calls recorded with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); }); }); // Streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-stream-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streaming sets available tools and tool calls with PII', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; @@ -401,41 +412,37 @@ describe('Anthropic integration', () => { .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(2); - for (const span of spans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); - expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); - expect(span.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); - } - - // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. - const createStreamSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(createStreamSpan).toBeDefined(); - const messagesStreamSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(messagesStreamSpan).toBeDefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + for (const span of spans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); + expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); + expect(span.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); + } + + // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. + const createStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); + expect(createStreamSpan).toBeDefined(); + const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); + expect(messagesStreamSpan).toBeDefined(); }); }); // Additional error scenarios - Streaming errors createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() @@ -445,72 +452,72 @@ describe('Anthropic integration', () => { .ignore('event') .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(4); - const createInitErrorSpan = spans.find( - span => - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && - span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); - expect(createInitErrorSpan!.status).toBe('error'); - - const streamInitErrorSpan = spans.find( - span => - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && - span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); - expect(streamInitErrorSpan!.status).toBe('error'); - - const createMidwayErrorSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', - ); - expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); - expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); - - const streamMidwayErrorSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', - ); - expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); - expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(4); + const createInitErrorSpan = spans.find( + span => + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && + span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); + expect(createInitErrorSpan!.status).toBe('error'); + + const streamInitErrorSpan = spans.find( + span => + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && + span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); + expect(streamInitErrorSpan!.status).toBe('error'); + + const createMidwayErrorSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', + ); + expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); + expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); + + const streamMidwayErrorSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', + ); + expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); + expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); }); }); createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors correctly', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(2); - const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); - expect(invalidFormatSpan!.status).toBe('error'); - expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); - expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - - const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); - expect(toolSuccessSpan!.status).toBe('ok'); - expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); + expect(invalidFormatSpan!.status).toBe('error'); + expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); + expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); + expect(toolSuccessSpan!.status).toBe('ok'); + expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); }); }); @@ -520,46 +527,41 @@ describe('Anthropic integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const spans = allSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(1); - const [firstSpan] = spans; - - // [0] messages.create — system instructions extracted into dedicated attribute - expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] messages.create — system instructions extracted into dedicated attribute + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); }); }, ); createEsmAndCjsTests(__dirname, 'scenario-response-error.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error-shaped responses returned as data', async () => { + let receivedEvent: Event | undefined; + await createRunner() // The API returns the error as data on a 200 response, never as a thrown error to the caller, // so the instrumentation intentionally captures it as an event. .unordered() .expect({ - event: { - exception: { - values: [ - { - value: 'Overloaded', - mechanism: { type: 'auto.ai.anthropic.anthropic_error', handled: false }, - }, - ], - }, + event: event => { + receivedEvent = event; }, }) .expect({ @@ -570,6 +572,18 @@ describe('Anthropic integration', () => { }) .start() .completed(); + + expect(receivedEvent).toMatchObject({ + event_id: expect.any(String), + exception: { + values: [ + { + value: 'Overloaded', + mechanism: { type: 'auto.ai.anthropic.anthropic_error', handled: false }, + }, + ], + }, + }); }); }); }); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 23f9a1a7aa4a..73e2452ebd2f 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -18,28 +18,28 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span for messages.stream(), not a second one for its internal create', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - const streamingSpans = allSpans.filter( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', - ); - expect(streamingSpans).toHaveLength(1); - expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + const streamingSpans = receivedSpans.filter( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', + ); + expect(streamingSpans).toHaveLength(1); + expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); }); }, { @@ -57,24 +57,24 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span each for beta.messages.stream() and the eager streaming tool runner', async () => { - const allSpans: SerializedStreamedSpanContainer['items'] = []; + let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() .unordered() .expect({ span: container => { - allSpans.push(...container.items); - const segment = allSpans.find(span => span.is_segment && span.name === 'main'); - expect(segment).toBeDefined(); - for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { - const spans = allSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); - expect(spans, id).toHaveLength(1); - expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } + expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); + receivedSpans = container.items; }, }) .start() .completed(); + + for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { + const spans = receivedSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); + expect(spans, id).toHaveLength(1); + expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }); }, { From 20d8f15474b251a94f963ecd30faeeab6a84d8b5 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:28:49 +0200 Subject: [PATCH 09/10] test(node): Remove unnecessary Anthropic unordered matching Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 9 --------- .../suites/tracing/anthropic/v0.129/test.ts | 2 -- 2 files changed, 11 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index c8409b491b51..43e5302c680a 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -38,7 +38,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -241,7 +240,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -294,7 +292,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -345,7 +342,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -376,7 +372,6 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_1","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -409,7 +404,6 @@ describe('Anthropic integration', () => { const EXPECTED_TOOL_CALLS_JSON = '[{"type":"tool_use","id":"tool_weather_2","name":"weather","input":{"city":"Paris"}}]'; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -445,7 +439,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() // Stream errors surface via the MessageStream `error` event; attaching that listener stops it // being raised as an unhandled rejection, so the instrumentation captures it. This test only // asserts the spans. @@ -498,7 +491,6 @@ describe('Anthropic integration', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -531,7 +523,6 @@ describe('Anthropic integration', () => { const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index 73e2452ebd2f..db49a71dd097 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -21,7 +21,6 @@ describe('Anthropic integration (0.129)', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); @@ -60,7 +59,6 @@ describe('Anthropic integration (0.129)', () => { let receivedSpans: SerializedStreamedSpanContainer['items'] = []; await createRunner() - .unordered() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); From 139269aad4d6c76b61ff5a957a3034690e68bd04 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Wed, 7 Oct 2026 14:36:38 +0200 Subject: [PATCH 10/10] test(node): Simplify ordered Anthropic span assertions Co-Authored-By: GPT-6 --- .../suites/tracing/anthropic/test.ts | 384 +++++++++--------- .../suites/tracing/anthropic/v0.129/test.ts | 37 +- 2 files changed, 196 insertions(+), 225 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index 43e5302c680a..ad9ee334d927 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -35,50 +35,48 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-with-response.mjs', 'instrument.mjs', (createRunner, test) => { test('preserves .withResponse() and .asResponse() for non-streaming and streaming', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(6); + const nonStreamingSpans = spans.filter( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse', + ); + expect(nonStreamingSpans).toHaveLength(1); + for (const span of nonStreamingSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + } + + const streamingSpan = spans.find( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', + ); + expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(streamingSpan!.status).toBe('ok'); + expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + + const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); + expect(rawResponseSpans).toHaveLength(4); + for (const span of rawResponseSpans) { + expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + } + expect(rawResponseSpans.map(span => span.attributes[GEN_AI_REQUEST_MODEL].value)).toEqual([ + 'claude-3-haiku-20240307', + 'claude-3-haiku-20240307', + 'claude-2', + 'claude-3-haiku-20240307', + ]); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(6); - const nonStreamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_withresponse'); - expect(nonStreamingSpans).toHaveLength(1); - for (const span of nonStreamingSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(span.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - } - - const streamingSpan = spans.find( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_withresponse', - ); - expect(streamingSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(streamingSpan!.status).toBe('ok'); - expect(streamingSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - - const rawResponseSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID] === undefined); - expect(rawResponseSpans).toHaveLength(4); - for (const span of rawResponseSpans) { - expect(span.name).toBe(`chat ${span.attributes[GEN_AI_REQUEST_MODEL].value}`); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); - } - expect(rawResponseSpans.map(span => span.attributes[GEN_AI_REQUEST_MODEL].value)).toEqual([ - 'claude-3-haiku-20240307', - 'claude-3-haiku-20240307', - 'claude-2', - 'claude-3-haiku-20240307', - ]); }); }); @@ -237,87 +235,87 @@ describe('Anthropic integration', () => { createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument.mjs', (createRunner, test) => { test('streams produce spans with token usage and metadata (PII false)', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(requestStreamSpans).toHaveLength(2); + for (const span of requestStreamSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); + } + + const detailedStreamSpan = requestStreamSpans.find( + span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', + ); + expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(detailedStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); + + const messagesStreamSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(messagesStreamSpan!.status).toBe('ok'); + expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); + expect(messagesStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); + expect(messagesStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); + expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const requestStreamSpans = spans.filter(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); - expect(requestStreamSpans).toHaveLength(2); - for (const span of requestStreamSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); - } - - const detailedStreamSpan = requestStreamSpans.find( - span => span.attributes[GEN_AI_RESPONSE_FINISH_REASONS]?.value === '["end_turn"]', - ); - expect(detailedStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(detailedStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(detailedStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(detailedStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(detailedStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); - - const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); - expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(messagesStreamSpan!.status).toBe('ok'); - expect(messagesStreamSpan!.attributes[GEN_AI_PROVIDER_NAME].value).toBe('anthropic'); - expect(messagesStreamSpan!.attributes[GEN_AI_OPERATION_NAME].value).toBe('chat'); - expect(messagesStreamSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('claude-3-haiku-20240307'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_ID].value).toBe('msg_stream_1'); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(10); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(15); - expect(messagesStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(25); }); }); createEsmAndCjsTests(__dirname, 'scenario-stream.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streams record response text when PII true', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(3); + const requestStreamSpans = spans.filter( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(requestStreamSpans).toHaveLength(2); + for (const span of requestStreamSpans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); + } + + const messagesStreamSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(messagesStreamSpan!.status).toBe('ok'); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(3); - const requestStreamSpans = spans.filter(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); - expect(requestStreamSpans).toHaveLength(2); - for (const span of requestStreamSpans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); - } - - const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); - expect(messagesStreamSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(messagesStreamSpan!.status).toBe('ok'); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(messagesStreamSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('Hello from stream!'); }); }); @@ -339,34 +337,28 @@ describe('Anthropic integration', () => { // helper span is still the active span). Regression test for over-suppression. createEsmAndCjsTests(__dirname, 'scenario-stream-nested-create.mjs', 'instrument.mjs', (createRunner, test) => { test('traces a create() invoked from a stream event handler (dedup does not over-suppress)', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); + expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + // The helper's own internal `create` delegation must be deduped: exactly one span + // for the streamed response, not a duplicate child span. + const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); + expect(streamingSpans).toHaveLength(1); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - const nestedSpan = spans.find(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_nested'); - expect(nestedSpan.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - - // The helper's own internal `create` delegation must be deduped: exactly one span - // for the streamed response, not a duplicate child span. - const streamingSpans = spans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1'); - expect(streamingSpans).toHaveLength(1); }); }); // Non-streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('non-streaming sets available tools and tool calls with PII', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get the weather by city","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; const EXPECTED_TOOL_CALLS_JSON = @@ -375,30 +367,26 @@ describe('Anthropic integration', () => { .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] messages.create with tools — available tools + tool calls recorded with PII + expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); + expect(firstSpan!.status).toBe('ok'); + expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); + expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(1); - const [firstSpan] = spans; - - // [0] messages.create with tools — available tools + tool calls recorded with PII - expect(firstSpan!.name).toBe('chat claude-3-haiku-20240307'); - expect(firstSpan!.status).toBe('ok'); - expect(firstSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(firstSpan!.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); - expect(firstSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); }); }); // Streaming tool calls + available tools (PII true) createEsmAndCjsTests(__dirname, 'scenario-stream-tools.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('streaming sets available tools and tool calls with PII', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - const EXPECTED_TOOLS_JSON = '[{"name":"weather","description":"Get weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}]'; const EXPECTED_TOOL_CALLS_JSON = @@ -407,37 +395,37 @@ describe('Anthropic integration', () => { .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + for (const span of spans) { + expect(span.name).toBe('chat claude-3-haiku-20240307'); + expect(span.status).toBe('ok'); + expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); + expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); + expect(span.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); + } + + // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. + const createStreamSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(createStreamSpan).toBeDefined(); + const messagesStreamSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(messagesStreamSpan).toBeDefined(); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(2); - for (const span of spans) { - expect(span.name).toBe('chat claude-3-haiku-20240307'); - expect(span.status).toBe('ok'); - expect(span.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - expect(span.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(span.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toBe('["tool_use"]'); - expect(span.attributes[GEN_AI_TOOL_DEFINITIONS].value).toBe(EXPECTED_TOOLS_JSON); - expect(span.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toBe(EXPECTED_TOOL_CALLS_JSON); - } - - // messages.create({ stream: true }) carries the request stream param; messages.stream() does not. - const createStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true); - expect(createStreamSpan).toBeDefined(); - const messagesStreamSpan = spans.find(span => span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined); - expect(messagesStreamSpan).toBeDefined(); }); }); // Additional error scenarios - Streaming errors createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles streaming errors correctly', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() // Stream errors surface via the MessageStream `error` event; attaching that listener stops it // being raised as an unhandled rejection, so the instrumentation captures it. This test only @@ -446,70 +434,64 @@ describe('Anthropic integration', () => { .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(4); + const createInitErrorSpan = spans.find( + span => + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && + span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, + ); + expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); + expect(createInitErrorSpan!.status).toBe('error'); + + const streamInitErrorSpan = spans.find( + span => + span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && + span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, + ); + expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); + expect(streamInitErrorSpan!.status).toBe('error'); + + const createMidwayErrorSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', + ); + expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); + expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); + expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); + expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); + + const streamMidwayErrorSpan = spans.find( + span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', + ); + expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); + expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(4); - const createInitErrorSpan = spans.find( - span => - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && - span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]?.value === true, - ); - expect(createInitErrorSpan!.name).toBe('chat error-stream-init'); - expect(createInitErrorSpan!.status).toBe('error'); - - const streamInitErrorSpan = spans.find( - span => - span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-init' && - span.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] === undefined, - ); - expect(streamInitErrorSpan!.name).toBe('chat error-stream-init'); - expect(streamInitErrorSpan!.status).toBe('error'); - - const createMidwayErrorSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'ok', - ); - expect(createMidwayErrorSpan!.name).toBe('chat error-stream-midway'); - expect(createMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE].value).toBe(true); - expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true); - expect(createMidwayErrorSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe('This stream will '); - - const streamMidwayErrorSpan = spans.find( - span => span.attributes[GEN_AI_REQUEST_MODEL]?.value === 'error-stream-midway' && span.status === 'error', - ); - expect(streamMidwayErrorSpan!.name).toBe('chat error-stream-midway'); - expect(streamMidwayErrorSpan!.attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE]).toBeUndefined(); }); }); createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { test('handles tool errors correctly', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); + expect(spans).toHaveLength(2); + const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); + expect(invalidFormatSpan!.status).toBe('error'); + expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); + expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); + + const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); + expect(toolSuccessSpan!.status).toBe('ok'); + expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(2); - const invalidFormatSpan = spans.find(span => span.name === 'chat invalid-format'); - expect(invalidFormatSpan!.status).toBe('error'); - expect(invalidFormatSpan!.attributes[GEN_AI_REQUEST_MODEL].value).toBe('invalid-format'); - expect(invalidFormatSpan!.attributes[SENTRY_OP].value).toBe('gen_ai.chat'); - - const toolSuccessSpan = spans.find(span => span.name === 'chat claude-3-haiku-20240307'); - expect(toolSuccessSpan!.status).toBe('ok'); - expect(toolSuccessSpan!.attributes[GEN_AI_RESPONSE_TOOL_CALLS].value).toContain('tool_ok_1'); }); }); @@ -519,25 +501,23 @@ describe('Anthropic integration', () => { 'instrument-with-pii.mjs', (createRunner, test) => { test('extracts system instructions from messages', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - const expectedInstructions = JSON.stringify([{ type: 'text', content: 'You are a helpful assistant' }]); await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const spans = container.items.filter( + span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic', + ); + expect(spans).toHaveLength(1); + const [firstSpan] = spans; + + // [0] messages.create — system instructions extracted into dedicated attribute + expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); }, }) .start() .completed(); - - const spans = receivedSpans.filter(span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.anthropic'); - expect(spans).toHaveLength(1); - const [firstSpan] = spans; - - // [0] messages.create — system instructions extracted into dedicated attribute - expect(firstSpan!.attributes[GEN_AI_SYSTEM_INSTRUCTIONS].value).toBe(expectedInstructions); }); }, ); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts index db49a71dd097..05eac304d0e1 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/v0.129/test.ts @@ -1,4 +1,3 @@ -import type { SerializedStreamedSpanContainer } from '@sentry/core'; import { GEN_AI_RESPONSE_ID, GEN_AI_RESPONSE_STREAMING, SENTRY_OP } from '@sentry/conventions/attributes'; import { afterAll, describe, expect } from 'vitest'; import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; @@ -18,27 +17,23 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span for messages.stream(), not a second one for its internal create', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + const streamingSpans = container.items.filter( + span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', + ); + expect(streamingSpans).toHaveLength(1); + expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ + type: 'boolean', + value: true, + }); }, }) .start() .completed(); - - const streamingSpans = receivedSpans.filter( - span => span.attributes[GEN_AI_RESPONSE_ID]?.value === 'msg_stream_1', - ); - expect(streamingSpans).toHaveLength(1); - expect(streamingSpans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - expect(streamingSpans[0]!.attributes[GEN_AI_RESPONSE_STREAMING]).toEqual({ - type: 'boolean', - value: true, - }); }); }, { @@ -56,23 +51,19 @@ describe('Anthropic integration (0.129)', () => { 'instrument.mjs', (createRunner, test) => { test('emits one span each for beta.messages.stream() and the eager streaming tool runner', async () => { - let receivedSpans: SerializedStreamedSpanContainer['items'] = []; - await createRunner() .expect({ span: container => { expect(container.items.find(span => span.is_segment && span.name === 'main')).toBeDefined(); - receivedSpans = container.items; + for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { + const spans = container.items.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); + expect(spans, id).toHaveLength(1); + expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); + } }, }) .start() .completed(); - - for (const id of ['msg_beta_stream', 'msg_tool_runner_eager']) { - const spans = receivedSpans.filter(span => span.attributes[GEN_AI_RESPONSE_ID]?.value === id); - expect(spans, id).toHaveLength(1); - expect(spans[0]!.attributes[SENTRY_OP]).toEqual({ type: 'string', value: 'gen_ai.chat' }); - } }); }, {