fix: preserve reasoning content in streamed chat traces

.changeset/big-suits-camp.md

@@ -0,0 +1,10 @@
+---
+"braintrust": patch
+---
+
+fix: preserve reasoning content in streamed chat traces
+
+Preserve `reasoning_content` returned by OpenAI-compatible chat completion
+streams in the recorded message, accumulating fragments separately for each
+choice. Preserve empty strings and null-only values without overwriting text
+with later null deltas. Other reasoning formats are unchanged.

.github/workflows/checks.yaml

@@ -179,6 +179,11 @@ jobs:
         env:
           BRAINTRUST_E2E_RUN_CONTEXT_DIR: ${{ steps.run_context.outputs.dir }}
         run: pnpm run test:e2e -- --shard=${{ matrix.shard }}/4
+      - name: Test e2e bump credential handling
+        if: matrix.shard == 1
+        env:
+          BRAINTRUST_API_KEY: ""
+        run: pnpm --filter=@braintrust/js-e2e-tests exec vitest run helpers/bump-scripts.test.ts --retry=0
       - name: Upload e2e run context
         if: ${{ always() && steps.run_context.outputs.dir != '' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

e2e/config/pr-comment-scenarios.json

@@ -60,6 +60,37 @@
       }
     ]
   },
+  {
+    "scenarioDirName": "openai-compatible-reasoning-instrumentation",
+    "label": "OpenAI-compatible Reasoning Instrumentation",
+    "metadataScenario": "openai-compatible-reasoning-instrumentation",
+    "variants": [
+      {
+        "variantKey": "openai-v4",
+        "label": "v4 pinned"
+      },
+      {
+        "variantKey": "openai-v4-latest",
+        "label": "v4 latest"
+      },
+      {
+        "variantKey": "openai-v5",
+        "label": "v5 pinned"
+      },
+      {
+        "variantKey": "openai-v5-latest",
+        "label": "v5 latest"
+      },
+      {
+        "variantKey": "openai-v6",
+        "label": "v6 pinned"
+      },
+      {
+        "variantKey": "openai-v6-latest",
+        "label": "v6 latest"
+      }
+    ]
+  },
   {
     "scenarioDirName": "openai-codex-instrumentation",
     "label": "OpenAI Codex Instrumentation",

e2e/helpers/bump-scripts.test.ts

@@ -0,0 +1,181 @@
+import { EventEmitter } from "node:events";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterEach, beforeEach, expect, it, vi } from "vitest";
+
+const DEEPSEEK_KEY = "dummy-deepseek-key-for-bump-test";
+const OPENAI_KEY = "dummy-openai-key-for-bump-test";
+const SCENARIO = "openai-compatible-reasoning-instrumentation";
+const E2E_ROOT = fileURLToPath(new URL("..", import.meta.url));
+const SCENARIO_DIR = path.join(E2E_ROOT, "scenarios", SCENARIO);
+const MANIFEST_PATH = path.join(SCENARIO_DIR, "package.json");
+const originalEnv = process.env;
+const originalArgv = process.argv;
+
+interface Command {
+  command: string;
+  args: string[];
+  env: NodeJS.ProcessEnv;
+}
+
+let commands: Command[];
+let logs: ReturnType<typeof vi.spyOn>[];
+
+beforeEach(() => {
+  vi.resetModules();
+  commands = [];
+  process.env = {
+    DEEPSEEK_API_KEY: DEEPSEEK_KEY,
+    OPENAI_API_KEY: OPENAI_KEY,
+    OPENAI_BASE_URL: "https://openai.example.invalid/v1",
+    CI: "true",
+    HARMLESS_TEST_SETTING: "keep-me",
+  };
+  logs = (["error", "log", "warn", "info", "debug"] as const).map((method) =>
+    vi.spyOn(console, method).mockImplementation(() => {}),
+  );
+  vi.doMock("node:child_process", () => ({
+    spawn: (
+      command: string,
+      args: string[],
+      options: { env?: NodeJS.ProcessEnv },
+    ) => {
+      commands.push({
+        command,
+        args,
+        env: { ...(options.env ?? process.env) },
+      });
+      const child = Object.assign(new EventEmitter(), {
+        stdout: new EventEmitter(),
+        stderr: new EventEmitter(),
+        kill: vi.fn(),
+      });
+      queueMicrotask(() => {
+        if (args[0] === "config") {
+          child.stdout.emit(
+            "data",
+            args[2] === "minimumReleaseAge" ? "0" : "[]",
+          );
+        }
+        child.emit("close", 0, null);
+      });
+      return child;
+    },
+  }));
+});
+
+afterEach(() => {
+  process.env = originalEnv;
+  process.argv = originalArgv;
+  vi.doUnmock("node:child_process");
+  vi.doUnmock("node:fs");
+  vi.doUnmock("node:fs/promises");
+  vi.unstubAllGlobals();
+  vi.restoreAllMocks();
+});
+
+function expectNoCredentialsInOutput() {
+  const output = JSON.stringify({
+    commands: commands.map(({ command, args }) => ({ command, args })),
+    logs: logs.flatMap((log) => log.mock.calls),
+  });
+  expect(output).not.toContain(DEEPSEEK_KEY);
+  expect(output).not.toContain(OPENAI_KEY);
+}
+
+it("forwards recording credentials to Docker by variable name without exposing their values", async () => {
+  // Never load the developer's .env files or invoke a real Docker process.
+  vi.doMock("node:fs", () => ({ existsSync: () => false }));
+  process.argv = [process.execPath, "run-e2e-bump-docker.mjs", SCENARIO];
+
+  await import("../scripts/run-e2e-bump-docker.mjs");
+
+  expect(commands.map(({ args }) => args[0])).toEqual(["build", "run"]);
+  const run = commands[1];
+  const forwarded = run.args.flatMap((arg, index) =>
+    arg === "--env" ? [run.args[index + 1]] : [],
+  );
+  expect(forwarded).toEqual(
+    expect.arrayContaining([
+      "OPENAI_API_KEY",
+      "OPENAI_BASE_URL",
+      "CI",
+      "HOME=/tmp",
+    ]),
+  );
+  expect(forwarded).not.toContain("HARMLESS_TEST_SETTING");
+  expect(run.env.OPENAI_API_KEY).toBe(OPENAI_KEY);
+  expect(run.env.DEEPSEEK_API_KEY).toBe(DEEPSEEK_KEY);
+  expect(run.args.slice(-3)).toEqual([
+    "node",
+    "e2e/scripts/bump-e2e-versions.mjs",
+    SCENARIO,
+  ]);
+  expectNoCredentialsInOutput();
+  expect(forwarded).toContain("DEEPSEEK_API_KEY");
+});
+
+it("removes provider credentials from the actual dependency-install subprocess environment", async () => {
+  const manifest = {
+    dependencies: { "openai-latest": "npm:openai@6.0.0" },
+    braintrustScenario: {
+      bump: {
+        dependencies: { "openai-latest": { package: "openai", range: "6" } },
+      },
+    },
+  };
+  const writeFile = vi.fn().mockResolvedValue(undefined);
+  vi.doMock("node:fs", () => ({
+    existsSync: (file: string) => file === MANIFEST_PATH,
+  }));
+  vi.doMock("node:fs/promises", () => ({
+    readdir: async () => [{ name: SCENARIO, isDirectory: () => true }],
+    readFile: async (file: string) => {
+      expect(file).toBe(MANIFEST_PATH);
+      return JSON.stringify(manifest);
+    },
+    writeFile,
+  }));
+  const fetchMetadata = vi.fn(async (url: string) => {
+    expect(url).toBe("https://registry.npmjs.org/openai");
+    return { ok: true, json: async () => ({ versions: { "6.0.1": {} } }) };
+  });
+  vi.stubGlobal("fetch", fetchMetadata);
+  process.argv = [
+    process.execPath,
+    "bump-e2e-versions.mjs",
+    "--skip-record",
+    "--skip-replay",
+    SCENARIO,
+  ];
+
+  await import("../scripts/bump-e2e-versions.mjs");
+
+  expect(fetchMetadata).toHaveBeenCalledOnce();
+  expect(writeFile).toHaveBeenCalledOnce();
+  expect(JSON.parse(writeFile.mock.calls[0][1]).dependencies).toEqual({
+    "openai-latest": "npm:openai@6.0.1",
+  });
+  expect(commands.map(({ args }) => args[0])).toEqual([
+    "config",
+    "config",
+    "install",
+  ]);
+  const install = commands[2];
+  expect(install.args).toEqual([
+    "install",
+    "--dir",
+    SCENARIO_DIR,
+    "--ignore-workspace",
+    "--lockfile-only",
+    "--strict-peer-dependencies=false",
+  ]);
+  expect(install.env.OPENAI_API_KEY).toBeUndefined();
+  expect(install.env.OPENAI_BASE_URL).toBe(process.env.OPENAI_BASE_URL);
+  expect(install.env.CI).toBe("true");
+  expect(install.env.HARMLESS_TEST_SETTING).toBe("keep-me");
+  expect(process.env.DEEPSEEK_API_KEY).toBe(DEEPSEEK_KEY);
+  expect(process.env.OPENAI_API_KEY).toBe(OPENAI_KEY);
+  expectNoCredentialsInOutput();
+  expect(install.env.DEEPSEEK_API_KEY).toBeUndefined();
+});

e2e/helpers/scenario-harness.ts

@@ -302,6 +302,7 @@ function getCassetteServerRoutes(): CassetteServerRoute[] {
       prefix: "/aws-bedrock-runtime",
       upstreamOrigin: `https://bedrock-runtime.${getBedrockRegion()}.amazonaws.com`,
     },
+    { prefix: "/deepseek", upstreamOrigin: "https://api.deepseek.com" },
     { prefix: "/elevenlabs", upstreamOrigin: "https://api.elevenlabs.io" },
     { prefix: "/cohere", upstreamOrigin: "https://api.cohere.com" },
     { prefix: "/cursor/v1", upstreamOrigin: "https://api.cursor.com/v1" },
@@ -339,6 +340,7 @@ function getCassetteEnv(wiring: ActiveCassetteWiring): Record<string, string> {
     ANTHROPIC_BASE_URL: `${serverUrl}/anthropic`,
     ANTHROPIC_BEDROCK_BASE_URL: `${serverUrl}/anthropic-bedrock`,
     AWS_BEDROCK_RUNTIME_BASE_URL: `${serverUrl}/aws-bedrock-runtime`,
+    DEEPSEEK_BASE_URL: `${serverUrl}/deepseek/v1`,
     ELEVENLABS_BASE_URL: `${serverUrl}/elevenlabs`,
     COHERE_BASE_URL: `${serverUrl}/cohere`,
     COHERE_API_URL: `${serverUrl}/cohere`,
@@ -397,6 +399,7 @@ const CASSETTE_PROVIDER_KEYS: Array<{
     envVars: ["COHERE_API_KEY", "CO_API_KEY"],
     placeholder: "cassette-placeholder",
   },
+  { envVars: ["DEEPSEEK_API_KEY"], placeholder: "cassette-placeholder" },
   { envVars: ["ELEVENLABS_API_KEY"], placeholder: "cassette-placeholder" },
   { envVars: ["CURSOR_API_KEY"], placeholder: "key_cassette-placeholder" },
   {

e2e/helpers/scenario-installer.ts

@@ -22,6 +22,7 @@ const INSTALL_SECRET_ENV_VARS = [
   "CO_API_KEY",
   "COHERE_API_KEY",
   "CURSOR_API_KEY",
+  "DEEPSEEK_API_KEY",
   "GEMINI_API_KEY",
   "GITHUB_TOKEN",
   "GOOGLE_API_KEY",

e2e/scenarios/openai-compatible-reasoning-instrumentation/README.md

@@ -0,0 +1,23 @@
+# OpenAI-compatible streamed reasoning
+
+These cassettes contain genuine `deepseek-flash` responses recorded through the real OpenAI SDK and the repository cassette harness on 2026-09-25. Requests went to `https://api.deepseek.com/v1/chat/completions`. The prompt, thinking settings and token limit are in `scenario.impl.mjs`; the ordered raw `reasoning_content` and ordinary content fragments remain in each cassette.
+
+| Alias            | Installed SDK | Retained entry recorded at (UTC) | Reasoning fragments |
+| ---------------- | ------------- | -------------------------------- | ------------------: |
+| openai-v4        | 4.104.0       | 2026-09-25T06:11:56.615Z         |                  40 |
+| openai-v4-latest | 4.104.0       | 2026-09-25T06:11:58.950Z         |                  42 |
+| openai-v5        | 5.11.0        | 2026-09-25T06:12:01.499Z         |                  44 |
+| openai-v5-latest | 5.23.2        | 2026-09-25T06:12:03.958Z         |                  39 |
+| openai-v6        | 6.25.0        | 2026-09-25T06:12:07.000Z         |                  39 |
+| openai-v6-latest | 6.49.0        | 2026-09-25T06:12:09.757Z         |                  49 |
+
+The serial recording matrix makes twelve requests: wrapped, then auto-hook, for each of six aliases. The second run overwrites that alias's cassette, so six final auto-hook recordings are retained. Each is replayed through both entrypoints. `entry.recordedAt` identifies the retained response; `meta.createdAt` can refer to the earlier overwritten recording.
+
+After recording, run the keyless snapshot update to align both entrypoints with the retained responses, then normal replay. Run these from the repository root:
+
+```sh
+pnpm --filter=@braintrust/js-e2e-tests run test:e2e:update -- openai-compatible-reasoning-instrumentation
+pnpm --filter=@braintrust/js-e2e-tests run test:e2e -- openai-compatible-reasoning-instrumentation
+```
+
+Assertions independently concatenate the raw fragments per choice and check ordinary content before reasoning and snapshots. Retries are disabled, and the matrix stops at the first failure. The genuine responses cover one choice; constructed unit tests cover interleaved choices and absent, empty, null and malformed values.