From 43100efd0f2ee4fffe44b4e9a711aa6f66c71b0f Mon Sep 17 00:00:00 2001 From: mstevens843 Date: Fri, 25 Sep 2026 00:09:27 -0700 Subject: [PATCH] fix: preserve reasoning content in streamed chat traces --- .changeset/big-suits-camp.md | 10 + .github/workflows/checks.yaml | 5 + e2e/config/pr-comment-scenarios.json | 31 ++ e2e/helpers/bump-scripts.test.ts | 181 ++++++++ e2e/helpers/scenario-harness.ts | 3 + e2e/helpers/scenario-installer.ts | 1 + .../README.md | 23 + .../openai-v4-latest.cassette.json | 110 +++++ .../__cassettes__/openai-v4.cassette.json | 108 +++++ .../openai-v5-latest.cassette.json | 107 +++++ .../__cassettes__/openai-v5.cassette.json | 112 +++++ .../openai-v6-latest.cassette.json | 117 ++++++ .../__cassettes__/openai-v6.cassette.json | 107 +++++ .../openai-v4-auto-hook.span-tree.json | 66 +++ .../openai-v4-auto-hook.span-tree.txt | 54 +++ .../openai-v4-latest-auto-hook.span-tree.json | 66 +++ .../openai-v4-latest-auto-hook.span-tree.txt | 54 +++ .../openai-v4-latest-wrapped.span-tree.json | 66 +++ .../openai-v4-latest-wrapped.span-tree.txt | 54 +++ .../openai-v4-wrapped.span-tree.json | 66 +++ .../openai-v4-wrapped.span-tree.txt | 54 +++ .../openai-v5-auto-hook.span-tree.json | 66 +++ .../openai-v5-auto-hook.span-tree.txt | 54 +++ .../openai-v5-latest-auto-hook.span-tree.json | 66 +++ .../openai-v5-latest-auto-hook.span-tree.txt | 54 +++ .../openai-v5-latest-wrapped.span-tree.json | 66 +++ .../openai-v5-latest-wrapped.span-tree.txt | 54 +++ .../openai-v5-wrapped.span-tree.json | 66 +++ .../openai-v5-wrapped.span-tree.txt | 54 +++ .../openai-v6-auto-hook.span-tree.json | 66 +++ .../openai-v6-auto-hook.span-tree.txt | 54 +++ .../openai-v6-latest-auto-hook.span-tree.json | 66 +++ .../openai-v6-latest-auto-hook.span-tree.txt | 54 +++ .../openai-v6-latest-wrapped.span-tree.json | 66 +++ .../openai-v6-latest-wrapped.span-tree.txt | 54 +++ .../openai-v6-wrapped.span-tree.json | 66 +++ .../openai-v6-wrapped.span-tree.txt | 54 +++ .../assertions.ts | 145 +++++++ .../package.json | 30 ++ .../pnpm-lock.yaml | 392 ++++++++++++++++++ .../scenario.impl.mjs | 66 +++ .../scenario.mjs | 14 + .../scenario.test.ts | 62 +++ .../scenario.ts | 16 + e2e/scripts/bump-e2e-versions.mjs | 1 + e2e/scripts/run-e2e-bump-docker.mjs | 1 + .../plugins/openai-plugin.test.ts | 204 +++++++++ .../instrumentation/plugins/openai-plugin.ts | 15 +- .../plugins/openai-reasoning.test.ts | 184 ++++++++ js/src/vendor-sdk-types/openai-common.ts | 2 + 50 files changed, 3486 insertions(+), 1 deletion(-) create mode 100644 .changeset/big-suits-camp.md create mode 100644 e2e/helpers/bump-scripts.test.ts create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/README.md create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4-latest.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5-latest.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6-latest.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6.cassette.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/assertions.ts create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/package.json create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/pnpm-lock.yaml create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.impl.mjs create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.mjs create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.test.ts create mode 100644 e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.ts create mode 100644 js/src/instrumentation/plugins/openai-reasoning.test.ts diff --git a/.changeset/big-suits-camp.md b/.changeset/big-suits-camp.md new file mode 100644 index 000000000..4398a972e --- /dev/null +++ b/.changeset/big-suits-camp.md @@ -0,0 +1,10 @@ +--- +"braintrust": patch +--- + +fix: preserve reasoning content in streamed chat traces + +Preserve `reasoning_content` returned by OpenAI-compatible chat completion +streams in the recorded message, accumulating fragments separately for each +choice. Preserve empty strings and null-only values without overwriting text +with later null deltas. Other reasoning formats are unchanged. diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index acb20ac6a..547d9d47b 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -179,6 +179,11 @@ jobs: env: BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }} run: pnpm run test:e2e -- --shard=${{ matrix.shard }}/4 + - name: Test e2e bump credential handling + if: matrix.shard == 1 + env: + BRAINTRUST_API_KEY: "" + run: pnpm --filter=@braintrust/js-e2e-tests exec vitest run helpers/bump-scripts.test.ts --retry=0 - name: Upload e2e run context if: ${{ always() && steps.run_context.outputs.dir != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index 26b4ada1d..54f1054a0 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -60,6 +60,37 @@ } ] }, + { + "scenarioDirName": "openai-compatible-reasoning-instrumentation", + "label": "OpenAI-compatible Reasoning Instrumentation", + "metadataScenario": "openai-compatible-reasoning-instrumentation", + "variants": [ + { + "variantKey": "openai-v4", + "label": "v4 pinned" + }, + { + "variantKey": "openai-v4-latest", + "label": "v4 latest" + }, + { + "variantKey": "openai-v5", + "label": "v5 pinned" + }, + { + "variantKey": "openai-v5-latest", + "label": "v5 latest" + }, + { + "variantKey": "openai-v6", + "label": "v6 pinned" + }, + { + "variantKey": "openai-v6-latest", + "label": "v6 latest" + } + ] + }, { "scenarioDirName": "openai-codex-instrumentation", "label": "OpenAI Codex Instrumentation", diff --git a/e2e/helpers/bump-scripts.test.ts b/e2e/helpers/bump-scripts.test.ts new file mode 100644 index 000000000..6f5f744c7 --- /dev/null +++ b/e2e/helpers/bump-scripts.test.ts @@ -0,0 +1,181 @@ +import { EventEmitter } from "node:events"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; + +const DEEPSEEK_KEY = "dummy-deepseek-key-for-bump-test"; +const OPENAI_KEY = "dummy-openai-key-for-bump-test"; +const SCENARIO = "openai-compatible-reasoning-instrumentation"; +const E2E_ROOT = fileURLToPath(new URL("..", import.meta.url)); +const SCENARIO_DIR = path.join(E2E_ROOT, "scenarios", SCENARIO); +const MANIFEST_PATH = path.join(SCENARIO_DIR, "package.json"); +const originalEnv = process.env; +const originalArgv = process.argv; + +interface Command { + command: string; + args: string[]; + env: NodeJS.ProcessEnv; +} + +let commands: Command[]; +let logs: ReturnType[]; + +beforeEach(() => { + vi.resetModules(); + commands = []; + process.env = { + DEEPSEEK_API_KEY: DEEPSEEK_KEY, + OPENAI_API_KEY: OPENAI_KEY, + OPENAI_BASE_URL: "https://openai.example.invalid/v1", + CI: "true", + HARMLESS_TEST_SETTING: "keep-me", + }; + logs = (["error", "log", "warn", "info", "debug"] as const).map((method) => + vi.spyOn(console, method).mockImplementation(() => {}), + ); + vi.doMock("node:child_process", () => ({ + spawn: ( + command: string, + args: string[], + options: { env?: NodeJS.ProcessEnv }, + ) => { + commands.push({ + command, + args, + env: { ...(options.env ?? process.env) }, + }); + const child = Object.assign(new EventEmitter(), { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + kill: vi.fn(), + }); + queueMicrotask(() => { + if (args[0] === "config") { + child.stdout.emit( + "data", + args[2] === "minimumReleaseAge" ? "0" : "[]", + ); + } + child.emit("close", 0, null); + }); + return child; + }, + })); +}); + +afterEach(() => { + process.env = originalEnv; + process.argv = originalArgv; + vi.doUnmock("node:child_process"); + vi.doUnmock("node:fs"); + vi.doUnmock("node:fs/promises"); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +function expectNoCredentialsInOutput() { + const output = JSON.stringify({ + commands: commands.map(({ command, args }) => ({ command, args })), + logs: logs.flatMap((log) => log.mock.calls), + }); + expect(output).not.toContain(DEEPSEEK_KEY); + expect(output).not.toContain(OPENAI_KEY); +} + +it("forwards recording credentials to Docker by variable name without exposing their values", async () => { + // Never load the developer's .env files or invoke a real Docker process. + vi.doMock("node:fs", () => ({ existsSync: () => false })); + process.argv = [process.execPath, "run-e2e-bump-docker.mjs", SCENARIO]; + + await import("../scripts/run-e2e-bump-docker.mjs"); + + expect(commands.map(({ args }) => args[0])).toEqual(["build", "run"]); + const run = commands[1]; + const forwarded = run.args.flatMap((arg, index) => + arg === "--env" ? [run.args[index + 1]] : [], + ); + expect(forwarded).toEqual( + expect.arrayContaining([ + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "CI", + "HOME=/tmp", + ]), + ); + expect(forwarded).not.toContain("HARMLESS_TEST_SETTING"); + expect(run.env.OPENAI_API_KEY).toBe(OPENAI_KEY); + expect(run.env.DEEPSEEK_API_KEY).toBe(DEEPSEEK_KEY); + expect(run.args.slice(-3)).toEqual([ + "node", + "e2e/scripts/bump-e2e-versions.mjs", + SCENARIO, + ]); + expectNoCredentialsInOutput(); + expect(forwarded).toContain("DEEPSEEK_API_KEY"); +}); + +it("removes provider credentials from the actual dependency-install subprocess environment", async () => { + const manifest = { + dependencies: { "openai-latest": "npm:openai@6.0.0" }, + braintrustScenario: { + bump: { + dependencies: { "openai-latest": { package: "openai", range: "6" } }, + }, + }, + }; + const writeFile = vi.fn().mockResolvedValue(undefined); + vi.doMock("node:fs", () => ({ + existsSync: (file: string) => file === MANIFEST_PATH, + })); + vi.doMock("node:fs/promises", () => ({ + readdir: async () => [{ name: SCENARIO, isDirectory: () => true }], + readFile: async (file: string) => { + expect(file).toBe(MANIFEST_PATH); + return JSON.stringify(manifest); + }, + writeFile, + })); + const fetchMetadata = vi.fn(async (url: string) => { + expect(url).toBe("https://registry.npmjs.org/openai"); + return { ok: true, json: async () => ({ versions: { "6.0.1": {} } }) }; + }); + vi.stubGlobal("fetch", fetchMetadata); + process.argv = [ + process.execPath, + "bump-e2e-versions.mjs", + "--skip-record", + "--skip-replay", + SCENARIO, + ]; + + await import("../scripts/bump-e2e-versions.mjs"); + + expect(fetchMetadata).toHaveBeenCalledOnce(); + expect(writeFile).toHaveBeenCalledOnce(); + expect(JSON.parse(writeFile.mock.calls[0][1]).dependencies).toEqual({ + "openai-latest": "npm:openai@6.0.1", + }); + expect(commands.map(({ args }) => args[0])).toEqual([ + "config", + "config", + "install", + ]); + const install = commands[2]; + expect(install.args).toEqual([ + "install", + "--dir", + SCENARIO_DIR, + "--ignore-workspace", + "--lockfile-only", + "--strict-peer-dependencies=false", + ]); + expect(install.env.OPENAI_API_KEY).toBeUndefined(); + expect(install.env.OPENAI_BASE_URL).toBe(process.env.OPENAI_BASE_URL); + expect(install.env.CI).toBe("true"); + expect(install.env.HARMLESS_TEST_SETTING).toBe("keep-me"); + expect(process.env.DEEPSEEK_API_KEY).toBe(DEEPSEEK_KEY); + expect(process.env.OPENAI_API_KEY).toBe(OPENAI_KEY); + expectNoCredentialsInOutput(); + expect(install.env.DEEPSEEK_API_KEY).toBeUndefined(); +}); diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index b86753c4a..c6a100ece 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -302,6 +302,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] { prefix: "/aws-bedrock-runtime", upstreamOrigin: `https://bedrock-runtime.${getBedrockRegion()}.amazonaws.com`, }, + { prefix: "/deepseek", upstreamOrigin: "https://api.deepseek.com" }, { prefix: "/elevenlabs", upstreamOrigin: "https://api.elevenlabs.io" }, { prefix: "/cohere", upstreamOrigin: "https://api.cohere.com" }, { prefix: "/cursor/v1", upstreamOrigin: "https://api.cursor.com/v1" }, @@ -339,6 +340,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record { ANTHROPIC_BASE_URL: `${serverUrl}/anthropic`, ANTHROPIC_BEDROCK_BASE_URL: `${serverUrl}/anthropic-bedrock`, AWS_BEDROCK_RUNTIME_BASE_URL: `${serverUrl}/aws-bedrock-runtime`, + DEEPSEEK_BASE_URL: `${serverUrl}/deepseek/v1`, ELEVENLABS_BASE_URL: `${serverUrl}/elevenlabs`, COHERE_BASE_URL: `${serverUrl}/cohere`, COHERE_API_URL: `${serverUrl}/cohere`, @@ -397,6 +399,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{ envVars: ["COHERE_API_KEY", "CO_API_KEY"], placeholder: "cassette-placeholder", }, + { envVars: ["DEEPSEEK_API_KEY"], placeholder: "cassette-placeholder" }, { envVars: ["ELEVENLABS_API_KEY"], placeholder: "cassette-placeholder" }, { envVars: ["CURSOR_API_KEY"], placeholder: "key_cassette-placeholder" }, { diff --git a/e2e/helpers/scenario-installer.ts b/e2e/helpers/scenario-installer.ts index 5f43d09a4..ea9d97897 100644 --- a/e2e/helpers/scenario-installer.ts +++ b/e2e/helpers/scenario-installer.ts @@ -22,6 +22,7 @@ const INSTALL_SECRET_ENV_VARS = [ "CO_API_KEY", "COHERE_API_KEY", "CURSOR_API_KEY", + "DEEPSEEK_API_KEY", "GEMINI_API_KEY", "GITHUB_TOKEN", "GOOGLE_API_KEY", diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/README.md b/e2e/scenarios/openai-compatible-reasoning-instrumentation/README.md new file mode 100644 index 000000000..694fffcbc --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/README.md @@ -0,0 +1,23 @@ +# OpenAI-compatible streamed reasoning + +These cassettes contain genuine `deepseek-flash` responses recorded through the real OpenAI SDK and the repository cassette harness on 2026-09-25. Requests went to `https://api.deepseek.com/v1/chat/completions`. The prompt, thinking settings and token limit are in `scenario.impl.mjs`; the ordered raw `reasoning_content` and ordinary content fragments remain in each cassette. + +| Alias | Installed SDK | Retained entry recorded at (UTC) | Reasoning fragments | +| ---------------- | ------------- | -------------------------------- | ------------------: | +| openai-v4 | 4.104.0 | 2026-09-25T06:11:56.615Z | 40 | +| openai-v4-latest | 4.104.0 | 2026-09-25T06:11:58.950Z | 42 | +| openai-v5 | 5.11.0 | 2026-09-25T06:12:01.499Z | 44 | +| openai-v5-latest | 5.23.2 | 2026-09-25T06:12:03.958Z | 39 | +| openai-v6 | 6.25.0 | 2026-09-25T06:12:07.000Z | 39 | +| openai-v6-latest | 6.49.0 | 2026-09-25T06:12:09.757Z | 49 | + +The serial recording matrix makes twelve requests: wrapped, then auto-hook, for each of six aliases. The second run overwrites that alias's cassette, so six final auto-hook recordings are retained. Each is replayed through both entrypoints. `entry.recordedAt` identifies the retained response; `meta.createdAt` can refer to the earlier overwritten recording. + +After recording, run the keyless snapshot update to align both entrypoints with the retained responses, then normal replay. Run these from the repository root: + +```sh +pnpm --filter=@braintrust/js-e2e-tests run test:e2e:update -- openai-compatible-reasoning-instrumentation +pnpm --filter=@braintrust/js-e2e-tests run test:e2e -- openai-compatible-reasoning-instrumentation +``` + +Assertions independently concatenate the raw fragments per choice and check ordinary content before reasoning and snapshots. Retries are disabled, and the matrix stops at the first failure. The genuine responses cover one choice; constructed unit tests cover interleaved choices and absent, empty, null and malformed values. diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4-latest.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4-latest.cassette.json new file mode 100644 index 000000000..a6a47ea46 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4-latest.cassette.json @@ -0,0 +1,110 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:11:58.950Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\",\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" than\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Must\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"60b824d8-0782-43c5-a798-178172d649fa\",\"object\":\"chat.completion.chunk\",\"created\":1790316718,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":46,\"total_tokens\":97,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":42},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:11:58 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "ejJ3aV7m8QnSCau9fmnnZ6vkFKIo6zKo4XhiR1_BkxmTddMMPJSfsw==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "1cded484df907a92fe77bf6356af3257" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:11:57.881Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4.cassette.json new file mode 100644 index 000000000..d5c2c6da1 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v4.cassette.json @@ -0,0 +1,108 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:11:56.615Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" >\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Must\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" with\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"28e7d75f-6b20-49e8-bf78-400fdedf7c5e\",\"object\":\"chat.completion.chunk\",\"created\":1790316715,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":44,\"total_tokens\":95,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":40},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:11:55 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "_cxropRVmhG7Ba05C1BIFZZMpZ6s_a1hnV13pzRlD5T7A2v1aqzl1Q==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "564f13eece9f38e3d7624c67df41962c" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:11:55.186Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5-latest.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5-latest.cassette.json new file mode 100644 index 000000000..e560d5f90 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5-latest.cassette.json @@ -0,0 +1,107 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:12:03.958Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" >\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"ed180268-cfcc-4645-9a70-0fc49d8e2a8c\",\"object\":\"chat.completion.chunk\",\"created\":1790316723,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":43,\"total_tokens\":94,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":39},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:12:03 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "GaCf5DKqaIy_0i-RVTaKX8LIcv8120u6GJtRpJx-6pjP8i7XoDO3nw==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "131154829fc0165548a944ee869baa87" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:12:02.734Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5.cassette.json new file mode 100644 index 000000000..1acd856c8 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v5.cassette.json @@ -0,0 +1,112 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:12:01.499Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" >\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"a51841f3-ea00-42a9-ac89-d640dc10f623\",\"object\":\"chat.completion.chunk\",\"created\":1790316720,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":48,\"total_tokens\":99,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":44},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:12:00 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "rblnNFRyPwAveR1ngFgkEj9yQidwqaRrYkesFQant1sn9dRA_aePfQ==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "0e72f6edc3467947f6a7cf9d01018f7c" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:12:00.340Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6-latest.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6-latest.cassette.json new file mode 100644 index 000000000..efac0ef2a --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6-latest.cassette.json @@ -0,0 +1,117 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:12:09.757Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" As\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" decimals\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\":\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" >\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" So\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Must\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"994c977e-e442-4a5f-bde6-13ef09f6c375\",\"object\":\"chat.completion.chunk\",\"created\":1790316729,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":53,\"total_tokens\":104,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":49},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:12:09 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "aOPAgAFm2B4IFyadrPUDK9N1SCEHgtjtpxZbRZKErj_cCq8wuI5fFw==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "7c534367f4ebddf435f1a91380e87b7b" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:12:08.703Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6.cassette.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6.cassette.json new file mode 100644 index 000000000..6eea7aa37 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__cassettes__/openai-v6.cassette.json @@ -0,0 +1,107 @@ +{ + "entries": [ + { + "callIndex": 0, + "id": "631c45d386f59296", + "matchKey": "POST api.deepseek.com/v1/chat/completions", + "recordedAt": "2026-09-25T06:12:07.000Z", + "request": { + "body": { + "kind": "json", + "value": { + "max_tokens": 2048, + "messages": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "model": "deepseek-flash", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + }, + "headers": {}, + "method": "POST", + "url": "https://api.deepseek.com/v1/chat/completions" + }, + "response": { + "body": { + "chunks": [ + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":null,\"reasoning_content\":\"\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"We\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Compare\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" vs\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"8\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" =\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"80\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" >\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" \"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"9\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\"11\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" Need\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" answer\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" only\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" greater\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\" number\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":null,\"reasoning_content\":\".\"},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"9\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\".\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"8\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":null}],\"usage\":null}", + "data: {\"id\":\"1dd773d2-e371-4512-b864-65836766a74a\",\"object\":\"chat.completion.chunk\",\"created\":1790316725,\"model\":\"deepseek-flash\",\"system_fingerprint\":\"aeb56401ca74e127821c4f9126dcb669\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"reasoning_content\":null},\"logprobs\":null,\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":51,\"completion_tokens\":43,\"total_tokens\":94,\"prompt_tokens_details\":{\"cached_tokens\":0},\"completion_tokens_details\":{\"reasoning_tokens\":39},\"prompt_cache_hit_tokens\":0,\"prompt_cache_miss_tokens\":51}}", + "data: [DONE]" + ], + "kind": "sse" + }, + "headers": { + "access-control-allow-credentials": "true", + "cache-control": "no-cache", + "content-type": "text/event-stream; charset=utf-8", + "date": "Fri, 25 Sep 2026 06:12:06 GMT", + "server": "elb", + "strict-transport-security": "max-age=31536000; includeSubDomains; preload", + "vary": "origin, access-control-request-method, access-control-request-headers", + "via": "1.1 230a4cabf962361d7a66119623a370ee.cloudfront.net (CloudFront)", + "x-amz-cf-id": "piauLlLe7ZVysvk_9ldwbFRCBxgg54LgtR0lD-lgWi29D4b3kbVq7Q==", + "x-amz-cf-pop": "SFO53-P10", + "x-cache": "Miss from cloudfront", + "x-content-type-options": "nosniff", + "x-ds-trace-id": "3a583873b02ba7c9e16384c1371f41f1" + }, + "status": 200 + } + } + ], + "meta": { + "createdAt": "2026-09-25T06:12:05.564Z" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json new file mode 100644 index 000000000..0b89c6cf7 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Must answer with greater number only.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 40, + "completion_tokens": 44, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 95 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt new file mode 100644 index 000000000..e8a098413 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Must answer with greater number only.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 40, + "completion_tokens": 44, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 95 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json new file mode 100644 index 000000000..f480f54e0 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. 9.8 = 9.80, greater than 9.11. Must answer greater number only.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 42, + "completion_tokens": 46, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 97 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt new file mode 100644 index 000000000..52f48628d --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. 9.8 = 9.80, greater than 9.11. Must answer greater number only.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 42, + "completion_tokens": 46, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 97 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json new file mode 100644 index 000000000..f480f54e0 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. 9.8 = 9.80, greater than 9.11. Must answer greater number only.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 42, + "completion_tokens": 46, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 97 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..52f48628d --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-latest-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. 9.8 = 9.80, greater than 9.11. Must answer greater number only.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 42, + "completion_tokens": 46, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 97 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json new file mode 100644 index 000000000..0b89c6cf7 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Must answer with greater number only.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 40, + "completion_tokens": 44, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 95 + } + } + ], + "metadata": { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt new file mode 100644 index 000000000..e8a098413 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v4-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "4.104.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Must answer with greater number only.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 40, + "completion_tokens": 44, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 95 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json new file mode 100644 index 000000000..5a5539be0 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8. Need only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 44, + "completion_tokens": 48, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 99 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt new file mode 100644 index 000000000..41ca7c287 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8. Need only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 44, + "completion_tokens": 48, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 99 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json new file mode 100644 index 000000000..7787db7bb --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt new file mode 100644 index 000000000..0871c89a1 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json new file mode 100644 index 000000000..7787db7bb --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..0871c89a1 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-latest-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "5.23.2", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json new file mode 100644 index 000000000..5a5539be0 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8. Need only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 44, + "completion_tokens": 48, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 99 + } + } + ], + "metadata": { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt new file mode 100644 index 000000000..41ca7c287 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v5-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "5.11.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Answer 9.8. Need only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 44, + "completion_tokens": 48, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 99 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json new file mode 100644 index 000000000..ea7c6f0ac --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Need answer only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt new file mode 100644 index 000000000..b1ae5f437 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Need answer only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json new file mode 100644 index 000000000..60ac3b4b1 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. As decimals: 9.8 = 9.80 > 9.11. So answer 9.8. Must only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 49, + "completion_tokens": 53, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 104 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt new file mode 100644 index 000000000..e3fb21a5c --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-auto-hook.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. As decimals: 9.8 = 9.80 > 9.11. So answer 9.8. Must only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 49, + "completion_tokens": 53, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 104 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json new file mode 100644 index 000000000..60ac3b4b1 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. As decimals: 9.8 = 9.80 > 9.11. So answer 9.8. Must only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 49, + "completion_tokens": 53, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 104 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt new file mode 100644 index 000000000..e3fb21a5c --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-latest-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "6.49.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Need compare 9.11 vs 9.8. As decimals: 9.8 = 9.80 > 9.11. So answer 9.8. Must only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 49, + "completion_tokens": 53, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 104 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json new file mode 100644 index 000000000..ea7c6f0ac --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.json @@ -0,0 +1,66 @@ +{ + "span_tree": [ + { + "name": "openai-compatible-reasoning-root", + "type": "task", + "children": [ + { + "name": "Chat Completion", + "type": "llm", + "children": [], + "input": [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ], + "output": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Need answer only greater number.", + "role": "assistant" + } + } + ], + "metadata": { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + }, + "metrics": { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } + } + ], + "metadata": { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + }, + "context": { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + } + ] +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt new file mode 100644 index 000000000..b1ae5f437 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/__snapshots__/openai-v6-wrapped.span-tree.txt @@ -0,0 +1,54 @@ +span_tree: +└── openai-compatible-reasoning-root [task] + metadata: { + "openaiSdkVersion": "6.25.0", + "scenario": "openai-compatible-reasoning-instrumentation", + "testRunId": "" + } + context: { + "caller_filename": "/e2e/helpers/provider-runtime.mjs", + "caller_functionname": "runTracedScenario", + "caller_lineno": 0 + } + └── Chat Completion [llm] + input: [ + { + "content": "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + "role": "user" + } + ] + output: [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "9.8", + "reasoning_content": "We need answer only greater number. Compare 9.11 vs 9.8. 9.8 = 9.80 > 9.11. Need answer only greater number.", + "role": "assistant" + } + } + ] + metadata: { + "max_tokens": 2048, + "model": "deepseek-flash", + "provider": "openai", + "reasoning_effort": "low", + "stream": true, + "stream_options": { + "include_usage": true + }, + "thinking": { + "type": "enabled" + } + } + metrics: { + "completion_reasoning_tokens": 39, + "completion_tokens": 43, + "prompt_cache_hit_tokens": 0, + "prompt_cache_miss_tokens": 51, + "prompt_cached_tokens": 0, + "prompt_tokens": 51, + "time_to_first_token": 0, + "tokens": 94 + } diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/assertions.ts b/e2e/scenarios/openai-compatible-reasoning-instrumentation/assertions.ts new file mode 100644 index 000000000..3b2b85e07 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/assertions.ts @@ -0,0 +1,145 @@ +import { createJsonFileStore } from "@braintrust/seinfeld"; +import path from "node:path"; +import { expect } from "vitest"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { resolveFileSnapshotPath } from "../../helpers/file-snapshot"; +import { matchSpanTreeSnapshot, spanTreeFields } from "../../helpers/span-tree"; +import { findAllSpans } from "../../helpers/trace-selectors"; +import { REQUEST, ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; + +interface RawChunk { + choices: Array<{ + index: number; + delta: { content?: string | null; reasoning_content?: string | null }; + finish_reason?: string | null; + }>; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + } | null; +} + +export async function assertRecordedReasoning(options: { + events: CapturedLogEvent[]; + originalScenarioDir: string; + alias: string; + mode: string; + version: string; + testFileUrl: string; +}) { + const store = createJsonFileStore({ + rootDir: path.join(options.originalScenarioDir, "__cassettes__"), + }); + const cassette = await store.load(options.alias); + expect(cassette?.entries).toHaveLength(1); + const entry = cassette!.entries[0]; + expect(entry.request.method).toBe("POST"); + expect(entry.request.url).toBe( + "https://api.deepseek.com/v1/chat/completions", + ); + expect(entry.request.body).toEqual({ kind: "json", value: REQUEST }); + expect(entry.response.status).toBe(200); + expect(entry.response.headers["content-type"]).toContain("text/event-stream"); + expect(Number.isFinite(Date.parse(entry.recordedAt))).toBe(true); + const body = entry.response.body; + // The recorder stores larger SSE responses in a binary sidecar. + const events = + body.kind === "sse" + ? body.chunks + : body.kind === "binary" && store.loadBlob + ? new TextDecoder("utf-8", { fatal: true }) + .decode(await store.loadBlob(options.alias, body.path)) + .replace(/\r\n/g, "\n") + .split("\n\n") + .filter((event) => event.trim()) + : undefined; + if (!events) throw new Error("Expected genuine recorded SSE"); + + // Build expectations from the retained wire events, never the SDK aggregator. + const data = events.map((event) => + event + .split("\n") + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice(5).trimStart()) + .join("\n"), + ); + expect(data.at(-1)).toBe("[DONE]"); + const chunks: RawChunk[] = data + .filter((value) => value && value !== "[DONE]") + .map((value) => JSON.parse(value)); + const indexes = [ + ...new Set(chunks.flatMap((c) => c.choices.map((v) => v.index))), + ].sort((a, b) => a - b); + expect(indexes.length).toBeGreaterThan(0); + const expected = indexes.map((index) => { + const choices = chunks.flatMap((chunk) => + chunk.choices.filter((choice) => choice.index === index), + ); + const reasoning = choices + .map((choice) => choice.delta.reasoning_content) + .filter((value): value is string => typeof value === "string"); + expect( + reasoning.filter((value) => value.length > 0).length, + ).toBeGreaterThan(1); + const content = choices + .map((choice) => choice.delta.content) + .filter((value): value is string => typeof value === "string") + .join(""); + expect(content.trim().length).toBeGreaterThan(0); + expect(choices.at(-1)?.finish_reason).toBe("stop"); + return { index, content, reasoning: reasoning.join("") }; + }); + const usage = chunks.filter((chunk) => chunk.usage).at(-1)?.usage; + expect(usage?.prompt_tokens).toBeGreaterThan(0); + expect(usage?.prompt_tokens).toBeLessThanOrEqual(512); + expect(usage?.completion_tokens).toBeGreaterThan(0); + expect(usage?.completion_tokens).toBeLessThanOrEqual(2048); + + const roots = findAllSpans(options.events, ROOT_NAME); + const spans = findAllSpans(options.events, "Chat Completion"); + expect(roots).toHaveLength(1); + expect(spans).toHaveLength(1); + const span = spans[0]; + expect(span.span.parentIds).toEqual([roots[0].span.id]); + expect(roots[0].metadata).toMatchObject({ + scenario: SCENARIO_NAME, + openaiSdkVersion: options.version, + }); + const output = span.output as Array<{ + index: number; + message: { content: string; reasoning_content?: string }; + finish_reason: string; + }>; + expect( + output.map((choice) => ({ + index: choice.index, + content: choice.message.content, + })), + ).toEqual(expected.map(({ index, content }) => ({ index, content }))); + // On the untouched base this must fail on missing reasoning, before snapshots. + expect( + output.map((choice) => ({ + index: choice.index, + reasoning: choice.message.reasoning_content, + })), + ).toEqual(expected.map(({ index, reasoning }) => ({ index, reasoning }))); + expect(output.map((choice) => choice.finish_reason)).toEqual( + indexes.map(() => "stop"), + ); + expect(span.metrics).toMatchObject({ + prompt_tokens: usage!.prompt_tokens, + completion_tokens: usage!.completion_tokens, + tokens: usage!.total_tokens, + }); + await matchSpanTreeSnapshot( + [...roots, ...spans].map((event) => ({ + event, + fields: { ...spanTreeFields(event), context: event.context }, + })), + resolveFileSnapshotPath( + options.testFileUrl, + `${options.alias}-${options.mode}.span-tree.json`, + ), + ); +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/package.json b/e2e/scenarios/openai-compatible-reasoning-instrumentation/package.json new file mode 100644 index 000000000..85ee5e838 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/package.json @@ -0,0 +1,30 @@ +{ + "name": "@braintrust/e2e-openai-compatible-reasoning-instrumentation", + "private": true, + "braintrustScenario": { + "bump": { + "dependencies": { + "openai-v4-latest": { + "package": "openai", + "range": "4" + }, + "openai-v5-latest": { + "package": "openai", + "range": "5" + }, + "openai-v6-latest": { + "package": "openai", + "range": "6" + } + } + } + }, + "dependencies": { + "openai-v4": "npm:openai@4.104.0", + "openai-v4-latest": "npm:openai@4.104.0", + "openai-v5": "npm:openai@5.11.0", + "openai-v5-latest": "npm:openai@5.23.2", + "openai-v6": "npm:openai@6.25.0", + "openai-v6-latest": "npm:openai@6.49.0" + } +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/pnpm-lock.yaml b/e2e/scenarios/openai-compatible-reasoning-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..55c5c7691 --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/pnpm-lock.yaml @@ -0,0 +1,392 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + openai-v4: + specifier: npm:openai@4.104.0 + version: openai@4.104.0 + openai-v4-latest: + specifier: npm:openai@4.104.0 + version: openai@4.104.0 + openai-v5: + specifier: npm:openai@5.11.0 + version: openai@5.11.0 + openai-v5-latest: + specifier: npm:openai@5.23.2 + version: openai@5.23.2 + openai-v6: + specifier: npm:openai@6.25.0 + version: openai@6.25.0 + openai-v6-latest: + specifier: npm:openai@6.49.0 + version: openai@6.49.0 + +packages: + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + openai@5.11.0: + resolution: {integrity: sha512-+AuTc5pVjlnTuA9zvn8rA/k+1RluPIx9AD4eDcnutv6JNwHHZxIhkFy+tmMKCvmMFDQzfA/r1ujvPWB19DQkYg==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + openai@5.23.2: + resolution: {integrity: sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + openai@6.25.0: + resolution: {integrity: sha512-mEh6VZ2ds2AGGokWARo18aPISI1OhlgdEIC1ewhkZr8pSIT31dec0ecr9Nhxx0JlybyOgoAT1sWeKtwPZzJyww==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + openai@6.49.0: + resolution: {integrity: sha512-aYCc0C6L864eR6WSYIwQGyXriw/nIyZx0ObvhzOEVuk0zoBDpynjSbrionWI7q65B5H8jJX0DXR9snEzM6bfPg==} + peerDependencies: + '@aws-sdk/credential-provider-node': '>=3.972.0 <4' + '@smithy/hash-node': '>=4.3.0 <5' + '@smithy/signature-v4': '>=5.4.0 <6' + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@aws-sdk/credential-provider-node': + optional: true + '@smithy/hash-node': + optional: true + '@smithy/signature-v4': + optional: true + ws: + optional: true + zod: + optional: true + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + +snapshots: + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 18.19.130 + form-data: 4.0.5 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + asynckit@0.4.0: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + delayed-stream@1.0.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + event-target-shim@5.0.1: {} + + form-data-encoder@1.7.2: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + gopd@1.2.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + ms@2.1.3: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + openai@4.104.0: + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + openai@5.11.0: {} + + openai@5.23.2: {} + + openai@6.25.0: {} + + openai@6.49.0: {} + + tr46@0.0.3: {} + + undici-types@5.26.5: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.impl.mjs b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.impl.mjs new file mode 100644 index 000000000..263e6e85f --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.impl.mjs @@ -0,0 +1,66 @@ +import { runTracedScenario } from "../../helpers/provider-runtime.mjs"; + +export const SCENARIO_NAME = "openai-compatible-reasoning-instrumentation"; +export const ROOT_NAME = "openai-compatible-reasoning-root"; +export const REQUEST = { + model: "deepseek-flash", + messages: [ + { + role: "user", + content: + "Which is greater, 9.11 or 9.8? Answer with the greater number only.", + }, + ], + thinking: { type: "enabled" }, + reasoning_effort: "low", + max_tokens: 2048, + stream: true, + stream_options: { include_usage: true }, +}; + +export async function runReasoningScenario({ + OpenAI, + decorateClient = (client) => client, + openaiSdkVersion, +}) { + // Require cassette routing even during recording. Never default to live APIs. + const baseURL = new URL(process.env.DEEPSEEK_BASE_URL ?? ""); + if ( + baseURL.protocol !== "http:" || + !["127.0.0.1", "localhost", "[::1]"].includes(baseURL.hostname) || + baseURL.pathname !== "/deepseek/v1" || + baseURL.search || + baseURL.username || + baseURL.password + ) { + throw new Error("Expected the harness's loopback DeepSeek base URL"); + } + const client = decorateClient( + new OpenAI({ + apiKey: process.env.DEEPSEEK_API_KEY, + baseURL: baseURL.href, + maxRetries: 0, + timeout: 45_000, + }), + ); + + await runTracedScenario({ + rootName: ROOT_NAME, + projectNameBase: SCENARIO_NAME, + metadata: { scenario: SCENARIO_NAME, openaiSdkVersion }, + callback: async () => { + const controller = new AbortController(); + const deadline = setTimeout(() => controller.abort(), 45_000); + try { + const stream = await client.chat.completions.create(REQUEST, { + signal: controller.signal, + }); + for await (const _chunk of stream) { + // Consume the real SDK stream to completion; assertions use raw cassettes. + } + } finally { + clearTimeout(deadline); + } + }, + }); +} diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.mjs b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.mjs new file mode 100644 index 000000000..0a9bfb63c --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.mjs @@ -0,0 +1,14 @@ +import { + getInstalledPackageVersion, + runMain, +} from "../../helpers/provider-runtime.mjs"; +import { runReasoningScenario } from "./scenario.impl.mjs"; + +runMain(async () => { + const alias = process.env.OPENAI_PACKAGE_NAME; + const { default: OpenAI } = await import(alias); + await runReasoningScenario({ + OpenAI, + openaiSdkVersion: await getInstalledPackageVersion(import.meta.url, alias), + }); +}); diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.test.ts b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.test.ts new file mode 100644 index 000000000..3ac1966ab --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.test.ts @@ -0,0 +1,62 @@ +import { it } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { assertRecordedReasoning } from "./assertions"; + +const originalScenarioDir = resolveScenarioDir(import.meta.url); +const scenarioDir = await prepareScenarioDir({ + scenarioDir: originalScenarioDir, +}); +const aliases = [ + "openai-v4", + "openai-v4-latest", + "openai-v5", + "openai-v5-latest", + "openai-v6", + "openai-v6-latest", +]; + +// A serial, non-retrying test also stops all later live requests on first failure. +it( + "retains genuine streamed reasoning through every OpenAI alias and entrypoint", + { + retry: 0, + timeout: 900_000, + }, + async () => { + for (const alias of aliases) { + const version = await readInstalledPackageVersion(scenarioDir, alias); + for (const mode of ["wrapped", "auto-hook"]) { + await withScenarioHarness(async (harness) => { + const options = { + scenarioDir, + env: { OPENAI_PACKAGE_NAME: alias }, + runContext: { originalScenarioDir, variantKey: alias }, + timeoutMs: 60_000, + }; + if (mode === "wrapped") { + await harness.runScenarioDir({ ...options, entry: "scenario.ts" }); + } else { + await harness.runNodeScenarioDir({ + ...options, + entry: "scenario.mjs", + nodeArgs: ["--import", "braintrust/hook.mjs"], + }); + } + await assertRecordedReasoning({ + events: harness.events(), + originalScenarioDir, + alias, + mode, + version, + testFileUrl: import.meta.url, + }); + }); + } + } + }, +); diff --git a/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.ts b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.ts new file mode 100644 index 000000000..b25d64e0f --- /dev/null +++ b/e2e/scenarios/openai-compatible-reasoning-instrumentation/scenario.ts @@ -0,0 +1,16 @@ +import { wrapOpenAI } from "braintrust"; +import { + getInstalledPackageVersion, + runMain, +} from "../../helpers/scenario-runtime"; +import { runReasoningScenario } from "./scenario.impl.mjs"; + +runMain(async () => { + const alias = process.env.OPENAI_PACKAGE_NAME!; + const { default: OpenAI } = await import(alias); + await runReasoningScenario({ + OpenAI, + decorateClient: wrapOpenAI, + openaiSdkVersion: await getInstalledPackageVersion(import.meta.url, alias), + }); +}); diff --git a/e2e/scripts/bump-e2e-versions.mjs b/e2e/scripts/bump-e2e-versions.mjs index d7ed675a1..069c1981f 100644 --- a/e2e/scripts/bump-e2e-versions.mjs +++ b/e2e/scripts/bump-e2e-versions.mjs @@ -22,6 +22,7 @@ const INSTALL_SECRET_ENV_VARS = [ "COHERE_API_KEY", "ELEVENLABS_API_KEY", "CURSOR_API_KEY", + "DEEPSEEK_API_KEY", "GEMINI_API_KEY", "GH_TOKEN", "GITHUB_TOKEN", diff --git a/e2e/scripts/run-e2e-bump-docker.mjs b/e2e/scripts/run-e2e-bump-docker.mjs index 93b547b3d..84bcbabd5 100644 --- a/e2e/scripts/run-e2e-bump-docker.mjs +++ b/e2e/scripts/run-e2e-bump-docker.mjs @@ -38,6 +38,7 @@ const ALLOWED_ENV_KEYS = [ "COHERE_BASE_URL", "CURSOR_API_KEY", "CURSOR_BACKEND_URL", + "DEEPSEEK_API_KEY", "GEMINI_API_KEY", "GEMINI_BASE_URL", "GEMINI_NEXT_GEN_API_BASE_URL", diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/plugins/openai-plugin.test.ts index ddcb66e7d..4b513e133 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/plugins/openai-plugin.test.ts @@ -5,6 +5,7 @@ import { } from "./openai-plugin"; import { processImagesInOutput as processImagesInOutputWithFlag } from "./openai-span-data"; import { Attachment } from "../../logger"; +import type { OpenAIChatCompletionChunk } from "../../vendor-sdk-types/openai"; const processImagesInOutput = (output: unknown, captureAttachments = true) => processImagesInOutputWithFlag(output, captureAttachments); @@ -502,6 +503,209 @@ describe("aggregateChatCompletionChunks", () => { }); }); + describe("reasoning_content aggregation", () => { + // Constructed unit inputs, not recorded provider responses. + it("preserves reasoning_content across chunks and interleaved choices", () => { + const chunks = [ + { + choices: [ + { + index: 2, + delta: { role: "assistant", reasoning_content: "Two " }, + }, + { + index: 0, + delta: { role: "assistant", reasoning_content: "Zero " }, + }, + ], + }, + { + choices: [ + { + index: 0, + delta: { reasoning_content: "thinks. ", content: "A" }, + }, + { + index: 2, + delta: { reasoning_content: "thinks. ", content: "B" }, + }, + ], + }, + { + choices: [ + { + index: 2, + delta: { reasoning_content: "Done.", content: "2" }, + finish_reason: "stop", + }, + ], + }, + { + choices: [ + { + index: 0, + delta: { reasoning_content: "Done.", content: "0" }, + finish_reason: "length", + }, + ], + }, + { + choices: [], + usage: { prompt_tokens: 4, completion_tokens: 6, total_tokens: 10 }, + }, + ]; + + const result = aggregateChatCompletionChunks(chunks); + + expect( + result.output.map((choice) => [choice.index, choice.message.content]), + ).toEqual([ + [0, "A0"], + [2, "B2"], + ]); + expect( + result.output.map((choice) => choice.message.reasoning_content), + ).toEqual(["Zero thinks. Done.", "Two thinks. Done."]); + expect(result.output.map((choice) => choice.finish_reason)).toEqual([ + "length", + "stop", + ]); + expect(result.metrics).toEqual({ + prompt_tokens: 4, + completion_tokens: 6, + tokens: 10, + }); + }); + + it.each([ + { name: "absent", values: [undefined, undefined], expected: undefined }, + { name: "empty", values: ["", undefined, ""], expected: "" }, + { name: "null", values: [null, undefined, null], expected: null }, + { name: "null then empty", values: [null, "", null], expected: "" }, + { + name: "strings interspersed with null and absent", + values: [null, "First", undefined, null, "", " last", null], + expected: "First last", + }, + ])("preserves $name reasoning_content", ({ values, expected }) => { + const chunks = values.map((value) => ({ + choices: [ + { + index: 0, + delta: value === undefined ? {} : { reasoning_content: value }, + }, + ], + })); + const message = aggregateChatCompletionChunks(chunks).output[0].message; + if (expected === undefined) { + expect(message).not.toHaveProperty("reasoning_content"); + } else { + expect(message).toHaveProperty("reasoning_content", expected); + } + expect(message.content).toBeUndefined(); + }); + + it("ignores non-string reasoning_content values", () => { + const chunks = [false, 42, {}, []].map((value) => ({ + choices: [{ index: 0, delta: { reasoning_content: value } }], + })) as unknown as OpenAIChatCompletionChunk[]; + expect( + aggregateChatCompletionChunks(chunks).output[0].message, + ).not.toHaveProperty("reasoning_content"); + }); + + it("does not copy unknown fields or other reasoning formats into output", () => { + const result = aggregateChatCompletionChunks([ + { + choices: [ + { + index: 0, + delta: { + content: "Answer", + reasoning: "Other format", + reasoning_details: [{ text: "Structured" }], + unknown_field: "Ignore", + }, + }, + ], + }, + ]); + expect(result.output[0].message).toEqual({ + role: undefined, + content: "Answer", + tool_calls: undefined, + }); + }); + + it("preserves tool calls, refusal, audio, finish reasons, and usage alongside reasoning_content", () => { + const result = aggregateChatCompletionChunks([ + { + choices: [ + { + index: 0, + delta: { + reasoning_content: "Find ", + refusal: "Cannot ", + audio: { id: "audio-1", transcript: "Hello " }, + tool_calls: [ + { + index: 0, + id: "call-1", + type: "function", + function: { name: "weather", arguments: '{"city":' }, + }, + ], + }, + }, + ], + }, + { + choices: [ + { + index: 0, + delta: { + reasoning_content: "weather.", + refusal: "answer.", + audio: { transcript: "world", expires_at: 123 }, + tool_calls: [{ index: 0, function: { arguments: '"Paris"}' } }], + }, + finish_reason: "tool_calls", + }, + ], + }, + { choices: [{ index: 0 }] }, + { + choices: [], + usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 }, + }, + ]); + expect(result.output[0]).toEqual({ + index: 0, + finish_reason: "tool_calls", + logprobs: null, + message: { + role: undefined, + content: undefined, + reasoning_content: "Find weather.", + refusal: "Cannot answer.", + audio: { id: "audio-1", transcript: "Hello world", expires_at: 123 }, + tool_calls: [ + { + id: "call-1", + type: "function", + function: { name: "weather", arguments: '{"city":"Paris"}' }, + }, + ], + }, + }); + expect(result.metrics).toEqual({ + prompt_tokens: 5, + completion_tokens: 3, + tokens: 8, + }); + }); + }); + describe("audio aggregation", () => { it("should aggregate audio transcripts without logging base64 data", () => { const chunks = [ diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts index dc0886c50..9d410c301 100644 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ b/js/src/instrumentation/plugins/openai-plugin.ts @@ -409,6 +409,7 @@ type AggregatedChatChoice = { index: number; role: string | undefined; content: string | undefined; + reasoning_content: string | null | undefined; refusal: string | undefined; audio: NonNullable | undefined; toolCallsByIndex: Map< @@ -424,6 +425,7 @@ function createAggregatedChatChoice(index: number): AggregatedChatChoice { index, role: undefined, content: undefined, + reasoning_content: undefined, refusal: undefined, audio: undefined, toolCallsByIndex: new Map(), @@ -442,6 +444,9 @@ function toChatChoice(choice: AggregatedChatChoice): OpenAIChatChoice { message: { role: choice.role, content: choice.content, + ...(choice.reasoning_content !== undefined + ? { reasoning_content: choice.reasoning_content } + : {}), ...(choice.refusal !== undefined ? { refusal: choice.refusal } : {}), ...(choice.audio !== undefined ? { audio: choice.audio } : {}), tool_calls: toolCalls.length > 0 ? toolCalls : undefined, @@ -453,7 +458,7 @@ function toChatChoice(choice: AggregatedChatChoice): OpenAIChatChoice { /** * Aggregate chat completion chunks into a single response. - * Combines role (first), content and audio transcripts (concatenated), + * Combines role (first), content, reasoning_content and audio transcripts (concatenated), * tool_calls (by index), finish_reason (last), and usage (last chunk). */ export function aggregateChatCompletionChunks( @@ -515,6 +520,14 @@ export function aggregateChatCompletionChunks( (aggregatedChoice.content || "") + delta.content; } + if (typeof delta.reasoning_content === "string") { + aggregatedChoice.reasoning_content = + (aggregatedChoice.reasoning_content ?? "") + delta.reasoning_content; + } else if (delta.reasoning_content === null) { + // Preserve null-only responses without erasing accumulated text. + aggregatedChoice.reasoning_content ??= null; + } + if (delta.refusal) { aggregatedChoice.refusal = (aggregatedChoice.refusal || "") + delta.refusal; diff --git a/js/src/instrumentation/plugins/openai-reasoning.test.ts b/js/src/instrumentation/plugins/openai-reasoning.test.ts new file mode 100644 index 000000000..bbffec43b --- /dev/null +++ b/js/src/instrumentation/plugins/openai-reasoning.test.ts @@ -0,0 +1,184 @@ +import OpenAI from "openai"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureNode } from "../../node/config"; +import type { OpenAIChatChoice } from "../../vendor-sdk-types/openai"; +import { wrapOpenAI } from "../../wrappers/oai"; + +describe("wrapped OpenAI reasoning_content", () => { + let background: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + configureNode(); + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + background = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "openai-reasoning.test.ts", + projectId: "test-project-id", + }); + }); + + afterEach(() => { + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + // Constructed unit responses exercise SDK parsing and tracing, not a provider. + function clientFor(body: string, streaming: boolean) { + return wrapOpenAI( + new OpenAI({ + apiKey: "unit-test-key", + baseURL: "https://openai-compatible.invalid/v1", + maxRetries: 0, + fetch: async () => + new Response(body, { + headers: { + "content-type": streaming + ? "text/event-stream" + : "application/json", + }, + }), + }), + ); + } + + async function recordedOutput() { + const rows = (await background.drain()) as Array<{ + output?: OpenAIChatChoice[]; + metrics?: Record; + }>; + expect(rows).toHaveLength(1); + return rows[0]; + } + + it("records reasoning_content from interleaved streamed choices", async () => { + const chunks = [ + { + choices: [ + { index: 1, delta: { role: "assistant", reasoning_content: "One " } }, + ], + }, + { + choices: [ + { + index: 0, + delta: { + role: "assistant", + reasoning_content: "Zero ", + content: "A", + }, + }, + ], + }, + { + choices: [ + { + index: 1, + delta: { reasoning_content: "thought.", content: "B" }, + finish_reason: "stop", + }, + ], + }, + { + choices: [ + { + index: 0, + delta: { reasoning_content: "thought.", content: "0" }, + finish_reason: "stop", + }, + ], + }, + { + choices: [], + usage: { prompt_tokens: 2, completion_tokens: 3, total_tokens: 5 }, + }, + ]; + const body = + chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + + "data: [DONE]\n\n"; + const client = clientFor(body, true); + const stream = await client.chat.completions.create({ + model: "unit-test-model", + messages: [{ role: "user", content: "Constructed unit input" }], + stream: true, + n: 2, + }); + const received = []; + for await (const chunk of stream) received.push(chunk); + expect(received).toEqual(chunks); + + const row = await recordedOutput(); + expect(row.output?.map((choice) => choice.message.content)).toEqual([ + "A0", + "B", + ]); + expect(row.output).toEqual([ + { + index: 0, + message: { + role: "assistant", + content: "A0", + reasoning_content: "Zero thought.", + }, + finish_reason: "stop", + logprobs: null, + }, + { + index: 1, + message: { + role: "assistant", + content: "B", + reasoning_content: "One thought.", + }, + finish_reason: "stop", + logprobs: null, + }, + ]); + expect(row.metrics).toMatchObject({ + prompt_tokens: 2, + completion_tokens: 3, + tokens: 5, + }); + }); + + it.each([undefined, null, "", "A thought."])( + "preserves reasoning_content %s in streamed and nonstreaming span output", + async (reasoningContent) => { + const message = { + role: "assistant", + content: "Answer", + ...(reasoningContent !== undefined + ? { reasoning_content: reasoningContent } + : {}), + }; + const choices = [ + { index: 0, message, finish_reason: "stop", logprobs: null }, + ]; + const nonstreamingClient = clientFor(JSON.stringify({ choices }), false); + const response = await nonstreamingClient.chat.completions.create({ + model: "unit-test-model", + messages: [], + }); + expect(response.choices).toEqual(choices); + expect((await recordedOutput()).output).toEqual(choices); + + const streamingClient = clientFor( + `data: ${JSON.stringify({ choices: [{ index: 0, delta: message, finish_reason: "stop" }] })}\n\ndata: [DONE]\n\n`, + true, + ); + const stream = await streamingClient.chat.completions.create({ + model: "unit-test-model", + messages: [], + stream: true, + }); + for await (const _chunk of stream) { + // Consume the stream so its span is finalized. + } + expect((await recordedOutput()).output).toEqual(choices); + }, + ); +}); diff --git a/js/src/vendor-sdk-types/openai-common.ts b/js/src/vendor-sdk-types/openai-common.ts index 6b7f1ef54..15cb745fb 100644 --- a/js/src/vendor-sdk-types/openai-common.ts +++ b/js/src/vendor-sdk-types/openai-common.ts @@ -98,6 +98,7 @@ interface OpenAIChatAudio { export interface OpenAIChatMessage { role?: string; content?: unknown; + reasoning_content?: string | null; refusal?: string; audio?: OpenAIChatAudio | null; tool_calls?: OpenAIChatToolCall[]; @@ -135,6 +136,7 @@ export interface OpenAIChatCompletion { interface OpenAIChatDelta { role?: string; content?: string; + reasoning_content?: string | null; refusal?: string; audio?: OpenAIChatAudio | null; tool_calls?: OpenAIChatToolCallDelta[];