From 4566b5aa3ec32cff91d3959e8e4dae0f555463ba Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:36:01 -0700 Subject: [PATCH] Splice observed shapes into the MCP result envelope --- e2e/scenarios/shape-memory.test.ts | 105 +++++++++++++++++++- packages/core/sdk/src/executor.ts | 40 +++++--- packages/core/sdk/src/index.ts | 1 + packages/core/sdk/src/shape-memory.test.ts | 85 ++++++++++++++++ packages/core/sdk/src/shape-memory.ts | 91 ++++++++++++++++- packages/plugins/mcp/src/sdk/plugin.test.ts | 54 +++++++++- packages/plugins/mcp/src/sdk/plugin.ts | 51 ++++++++++ packages/plugins/mcp/src/testing/index.ts | 1 + packages/plugins/mcp/src/testing/server.ts | 24 +++++ 9 files changed, 436 insertions(+), 16 deletions(-) create mode 100644 packages/core/sdk/src/shape-memory.test.ts diff --git a/e2e/scenarios/shape-memory.test.ts b/e2e/scenarios/shape-memory.test.ts index a1d2f5e123..01cf8029eb 100644 --- a/e2e/scenarios/shape-memory.test.ts +++ b/e2e/scenarios/shape-memory.test.ts @@ -11,13 +11,15 @@ import { createServer } from "node:http"; import { expect } from "@effect/vitest"; import { Effect } from "effect"; import { composePluginApi } from "@executor-js/api/server"; +import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api"; +import { makeUndeclaredStructuredMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing"; import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; import { scenario } from "../src/scenario"; import { Api, Target } from "../src/services"; -const api = composePluginApi([openApiHttpPlugin()] as const); +const api = composePluginApi([openApiHttpPlugin(), mcpHttpPlugin()] as const); /** One GET operation whose 200 declares no response schema — the shape the * model would otherwise have to guess. */ @@ -176,3 +178,104 @@ return { ok: result.ok }; }), ), ); + +// --------------------------------------------------------------------------- +// The MCP variant — the Blacksmith post's exact case. MCP tools always carry +// a declared CallToolResult envelope, but when the server declares no output +// schema its `structuredContent` slot is an untyped placeholder. One live +// call splices the observed payload shape into exactly that slot. +// --------------------------------------------------------------------------- + +const mcpDescribeCode = (slug: string) => ` +const details = await tools.describe.tool({ path: "${slug}.org.main.undeclared_structured_echo" }); +return { + outputTypeScript: details.outputTypeScript ?? null, + note: details.outputTypeScriptNote ?? null, + error: details.error ?? null, +}; +`; + +scenario( + "Muscle memory · an MCP server's undeclared structuredContent shape reaches describe", + {}, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const { client: makeApiClient } = yield* Api; + const identity = yield* target.newIdentity(); + const client = yield* makeApiClient(api, identity); + const slug = IntegrationSlug.make(`shape_mcp_${randomBytes(4).toString("hex")}`); + + const server = yield* serveMcpServer(makeUndeclaredStructuredMcpServer); + + yield* client.mcp.addServer({ + payload: { + transport: "remote", + name: "Undeclared structured MCP", + endpoint: server.url, + slug, + }, + }); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("main"), + integration: slug, + template: AuthTemplateSlug.make("none"), + value: "", + }, + }); + + const describe = Effect.gen(function* () { + const executed = yield* client.executions.execute({ + payload: { code: mcpDescribeCode(String(slug)), autoApprove: true }, + }); + expect(executed.status, executed.text).toBe("completed"); + return JSON.parse(executed.text) as DescribeOutcome; + }); + + // Cold: the declared envelope is served, but its structuredContent + // slot carries no payload type and nothing claims to be observed. + const cold = yield* describe; + expect(cold.error, "the tool resolves").toBeNull(); + expect(cold.outputTypeScript, "the envelope is declared").toContain("structuredContent"); + expect(cold.outputTypeScript, "the payload is untyped cold").not.toContain( + "length: number", + ); + expect(cold.note, "cold describe carries no provenance note").toBeNull(); + + // One real call teaches the payload shape. + const invoked = yield* client.executions.execute({ + payload: { + code: ` +const result = await tools.${slug}.org.main.undeclared_structured_echo({ value: "hi" }); +return { ok: result.ok }; +`, + autoApprove: true, + }, + }); + expect(invoked.status, invoked.text).toBe("completed"); + expect(JSON.parse(invoked.text), "the teaching call succeeded").toEqual({ ok: true }); + + // Warm: the observed shape fills exactly the structuredContent slot. + const warm = yield* describe; + expect(warm.outputTypeScript, "the payload type is served").toContain("length: number"); + expect(warm.outputTypeScript, "the payload type is served").toContain("value: string"); + expect(warm.outputTypeScript, "the declared envelope survives").toContain("content"); + expect(warm.note, "provenance is explicit").toContain("observed from 1 live response"); + }), + Effect.gen(function* () { + yield* client.connections + .remove({ + params: { owner: "org", integration: slug, name: ConnectionName.make("main") }, + }) + .pipe(Effect.ignore); + yield* client.mcp.removeServer({ params: { slug } }).pipe(Effect.ignore); + }), + ); + }), + ), +); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 2ce9a39227..4f853ee253 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -177,7 +177,13 @@ import { } from "./oauth-ema"; import { connectionIdentifier } from "./connection-name-identifier"; import { annotateToolResultOutcome, isToolResult } from "./tool-result"; -import { makeShapeMemory, observedShapeToJsonSchema, SHAPE_MEMORY_PLUGIN_ID } from "./shape-memory"; +import { + hasShapeSlots, + makeShapeMemory, + observedShapeToJsonSchema, + SHAPE_MEMORY_PLUGIN_ID, + spliceObservedSlots, +} from "./shape-memory"; import { isUnauthorizedToolFailure } from "./auth-tool-failure"; const PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE = 90; @@ -4066,20 +4072,28 @@ export const createExecutor = 0; + } const definitionRows = yield* core.findMany("definition", { where: (b: AnyCb) => @@ -4111,7 +4125,7 @@ export const createExecutor = { + it("finds a marked slot nested in properties", () => { + expect(hasShapeSlots(envelope)).toBe(true); + }); + + it("reports false for unmarked schemas", () => { + expect(hasShapeSlots({ type: "object", properties: { a: { type: "string" } } })).toBe(false); + expect(hasShapeSlots(undefined)).toBe(false); + }); +}); + +describe("spliceObservedSlots", () => { + it("fills the marked slot from the observed counterpart and strips the marker", () => { + const observed = inferShape({ + content: [{ type: "text", text: "x" }], + structuredContent: { value: "x", length: 1, ok: true }, + }); + const { schema, filled } = spliceObservedSlots(envelope, observed); + expect(filled).toBe(1); + const out = schema as { properties: Record> }; + expect(out.properties["structuredContent"]?.[SHAPE_SLOT_KEY]).toBeUndefined(); + expect(out.properties["structuredContent"]).toMatchObject({ + type: "object", + properties: { + value: { type: "string" }, + length: { type: "number" }, + ok: { type: "boolean" }, + }, + }); + // The declared structure around the slot is untouched. + expect(out.properties["content"]).toEqual(envelope.properties.content); + expect(out.properties["isError"]).toEqual(envelope.properties.isError); + }); + + it("keeps the placeholder and reports zero when the observation lacks the slot", () => { + const observed = inferShape({ content: [{ type: "text", text: "x" }] }); + const { schema, filled } = spliceObservedSlots(envelope, observed); + expect(filled).toBe(0); + const out = schema as { properties: Record> }; + expect(out.properties["structuredContent"]?.[SHAPE_SLOT_KEY]).toBeUndefined(); + expect(out.properties["structuredContent"]?.["type"]).toBe("object"); + }); + + it("strips markers even with no observation at all", () => { + const { schema, filled } = spliceObservedSlots(envelope, null); + expect(filled).toBe(0); + expect(JSON.stringify(schema)).not.toContain(SHAPE_SLOT_KEY); + }); + + it("descends through items to reach nested slots", () => { + const declared = { + type: "object", + properties: { + rows: { type: "array", items: { type: "object", [SHAPE_SLOT_KEY]: true } }, + }, + }; + const observed = inferShape({ rows: [{ id: 7 }] }); + const { schema, filled } = spliceObservedSlots(declared, observed); + expect(filled).toBe(1); + const out = schema as { + properties: { rows: { items: Record } }; + }; + expect(out.properties.rows.items).toMatchObject({ + type: "object", + properties: { id: { type: "number" } }, + }); + }); +}); diff --git a/packages/core/sdk/src/shape-memory.ts b/packages/core/sdk/src/shape-memory.ts index d5ab9f1d22..2c2ef33934 100644 --- a/packages/core/sdk/src/shape-memory.ts +++ b/packages/core/sdk/src/shape-memory.ts @@ -19,7 +19,7 @@ import { Clock, Effect } from "effect"; import type { Owner } from "./ids"; import type { PluginStorageFacade } from "./plugin-storage"; -import { observeShape, type ObservedShape } from "./shape-inference"; +import { observeShape, type InferredShape, type ObservedShape } from "./shape-inference"; /** Reserved system namespace inside `plugin_storage`; not a real plugin. */ export const SHAPE_MEMORY_PLUGIN_ID = "executor.shape-memory"; @@ -88,3 +88,92 @@ export const observedShapeToJsonSchema = (record: ObservedShape): unknown => ({ ...record.schema, description: `Observed from ${record.observations} live response${record.observations === 1 ? "" : "s"}; fields may be incomplete.`, }); + +// --------------------------------------------------------------------------- +// Placeholder slots — partial serving inside a DECLARED schema. +// +// Some plugins must declare an output schema even when the upstream said +// nothing about the payload: the MCP plugin's CallToolResult envelope is +// genuinely declared (content blocks, isError), but its `structuredContent` +// slot is a synthesized "some object" placeholder whenever the server +// declared no output schema. A plugin marks such a slot with +// `SHAPE_SLOT_KEY: true` (typically at projection time), and serving splices +// the observed shape's counterpart into exactly that slot, keeping the +// declared structure around it. +// --------------------------------------------------------------------------- + +/** Vendor-extension marker a plugin puts on a placeholder subschema. */ +export const SHAPE_SLOT_KEY = "x-executor-shape-slot"; + +const OBSERVED_SLOT_DESCRIPTION = "Observed from live responses; fields may be incomplete."; + +/** Slot scanning/splicing depth bound — marked slots live near the root. */ +const MAX_SLOT_DEPTH = 8; + +type SchemaNode = Record; + +const isSchemaNode = (value: unknown): value is SchemaNode => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const hasShapeSlots = (schema: unknown, depth = 0): boolean => { + if (!isSchemaNode(schema) || depth >= MAX_SLOT_DEPTH) return false; + if (schema[SHAPE_SLOT_KEY] === true) return true; + const properties = schema["properties"]; + if ( + isSchemaNode(properties) && + Object.values(properties).some((child) => hasShapeSlots(child, depth + 1)) + ) { + return true; + } + return hasShapeSlots(schema["items"], depth + 1); +}; + +const isInformativeShape = (shape: InferredShape): boolean => + shape.type !== undefined || shape.anyOf !== undefined; + +/** + * Replace marked placeholder slots in a declared schema with the observed + * shape's counterpart at the same path (descending `properties` by name and + * `items`). Slots with no informative observed counterpart keep their + * declared placeholder; markers are stripped either way so they never reach + * schema consumers. `filled` reports how many slots actually got a shape. + */ +export const spliceObservedSlots = ( + declared: unknown, + observed: InferredShape | null, +): { readonly schema: unknown; readonly filled: number } => { + let filled = 0; + const walk = (node: unknown, shape: InferredShape | null, depth: number): unknown => { + if (!isSchemaNode(node) || depth >= MAX_SLOT_DEPTH) return node; + if (node[SHAPE_SLOT_KEY] === true) { + const { [SHAPE_SLOT_KEY]: _slot, ...placeholder } = node; + if (shape !== null && isInformativeShape(shape)) { + filled += 1; + return { ...shape, description: OBSERVED_SLOT_DESCRIPTION }; + } + return placeholder; + } + let next: SchemaNode = node; + const properties = node["properties"]; + if (isSchemaNode(properties)) { + const walkedProperties: Record = {}; + let changed = false; + for (const [key, child] of Object.entries(properties)) { + const counterpart = shape?.type === "object" ? (shape.properties?.[key] ?? null) : null; + const walked = walk(child, counterpart, depth + 1); + walkedProperties[key] = walked; + if (walked !== child) changed = true; + } + if (changed) next = { ...next, properties: walkedProperties }; + } + const items = node["items"]; + if (items !== undefined) { + const counterpart = shape?.type === "array" ? (shape.items ?? null) : null; + const walked = walk(items, counterpart, depth + 1); + if (walked !== items) next = { ...next, items: walked }; + } + return next; + }; + const schema = walk(declared, observed, 0); + return { schema, filled }; +}; diff --git a/packages/plugins/mcp/src/sdk/plugin.test.ts b/packages/plugins/mcp/src/sdk/plugin.test.ts index 0b8338684b..a56fcb6113 100644 --- a/packages/plugins/mcp/src/sdk/plugin.test.ts +++ b/packages/plugins/mcp/src/sdk/plugin.test.ts @@ -27,7 +27,11 @@ import { createMcpConnector } from "./connection"; import { mcpPlugin, userFacingProbeMessage } from "./plugin"; import { McpInvocationError } from "./errors"; import { extractManifestFromListToolsResult, deriveMcpNamespace, joinToolPath } from "./manifest"; -import { makeAnnotationsMcpServer, serveMcpServer } from "../testing"; +import { + makeAnnotationsMcpServer, + makeUndeclaredStructuredMcpServer, + serveMcpServer, +} from "../testing"; // removed: the v1 addSource / scopes / secrets / credential-binding / usages / // sources.configure / multi-scope shadowing suites. v2 has no scope stack, no @@ -1034,3 +1038,51 @@ describe("mcpPlugin detect URL-token fallback", () => { }), ); }); + +describe("muscle memory — structuredContent slot splice", () => { + it.effect("serves the observed structuredContent shape for a server that declared none", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* serveMcpServer(makeUndeclaredStructuredMcpServer); + const executor = yield* createExecutor( + makeTestConfig({ plugins: [memoryCredentialsPlugin(), mcpPlugin()] as const }), + ); + yield* executor.mcp.addServer({ + transport: "remote", + name: "Undeclared structured MCP", + endpoint: server.url, + slug: "shape_mcp", + }); + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration: IntegrationSlug.make("shape_mcp"), + template: AuthTemplateSlug.make("none"), + value: "", + }); + const address = ToolAddress.make("tools.shape_mcp.org.main.undeclared_structured_echo"); + + // Cold: the declared envelope survives, its placeholder slot is + // untyped, the marker never leaks, and nothing claims to be observed. + const cold = yield* executor.tools.schema(address); + expect(cold?.outputSchemaSource).toBeUndefined(); + expect(JSON.stringify(cold?.outputSchema)).not.toContain("x-executor-shape-slot"); + expect(cold?.outputTypeScript).toContain("structuredContent"); + expect(cold?.outputTypeScript).not.toContain("length: number"); + + const result = yield* executor.execute(address, { value: "hi" }); + expect(result).toMatchObject({ ok: true }); + + // Warm: the observed payload shape is spliced into exactly the + // structuredContent slot; the declared envelope stays around it. + const warm = yield* executor.tools.schema(address); + expect(warm?.outputSchemaSource).toBe("observed"); + expect(warm?.outputSchemaObservations).toBe(1); + expect(warm?.outputTypeScript).toContain("value: string"); + expect(warm?.outputTypeScript).toContain("length: number"); + expect(warm?.outputTypeScript).toContain("ok: boolean"); + expect(warm?.outputTypeScript).toContain("content"); + }), + ), + ); +}); diff --git a/packages/plugins/mcp/src/sdk/plugin.ts b/packages/plugins/mcp/src/sdk/plugin.ts index d401f896cc..eb6d168e52 100644 --- a/packages/plugins/mcp/src/sdk/plugin.ts +++ b/packages/plugins/mcp/src/sdk/plugin.ts @@ -25,6 +25,7 @@ import { type OAuthClientSummary, type Owner, type PluginCtx, + SHAPE_SLOT_KEY, type StaticToolSchema, type StorageFailure, type ToolAnnotations, @@ -425,6 +426,45 @@ const mcpCallToolResultOutputSchema = (structuredContentSchema?: unknown): JsonS }; }; +/** The persisted envelope for a server that declared no output schema: the + * builder above only lists `structuredContent` in `required` when the server + * declared one, so its absence identifies the synthesized placeholder. */ +const isGenericStructuredContentEnvelope = ( + outputSchema: unknown, +): outputSchema is JsonSchemaObject => { + if (typeof outputSchema !== "object" || outputSchema === null || Array.isArray(outputSchema)) { + return false; + } + const schema = outputSchema as JsonSchemaObject; + const required = schema.required; + return ( + schema.properties?.content !== undefined && + schema.properties?.structuredContent !== undefined && + Array.isArray(required) && + !required.includes("structuredContent") + ); +}; + +/** Mark the synthesized `structuredContent` placeholder as a shape slot so + * serving can splice the runtime-observed payload shape into it. Read-time + * only — the persisted row is untouched, so every existing catalog row gets + * the behavior without a refresh. */ +const markStructuredContentSlot = (schema: JsonSchemaObject): JsonSchemaObject => { + const slot = schema.properties?.structuredContent; + return { + ...schema, + properties: { + ...schema.properties, + structuredContent: { + ...(typeof slot === "object" && slot !== null && !Array.isArray(slot) + ? (slot as Record) + : {}), + [SHAPE_SLOT_KEY]: true, + }, + }, + }; +}; + /** Build the executor-facing ToolDef for one discovered MCP tool, stamping the * real MCP tool name + upstream annotations into the persisted annotations so * they survive to invokeTool with no plugin-side store. */ @@ -1279,6 +1319,17 @@ export const mcpPlugin = definePlugin((options?: McpPluginOptions) => { StorageFailure >, + // Read-time projection: when the persisted envelope's `structuredContent` + // is the synthesized placeholder (server declared no output schema), mark + // it as a shape slot so core serving can splice the runtime-observed + // payload shape into exactly that slot. + projectToolSchema: ({ outputSchema }) => + Effect.succeed( + isGenericStructuredContentEnvelope(outputSchema) + ? { outputSchema: markStructuredContentSlot(outputSchema) } + : {}, + ), + invokeTool: ({ ctx, toolRow, credential, args, elicit }) => Effect.gen(function* () { const parsed = parseMcpIntegrationConfig(credential.config); diff --git a/packages/plugins/mcp/src/testing/index.ts b/packages/plugins/mcp/src/testing/index.ts index 172f5ee85e..53fe0a7d39 100644 --- a/packages/plugins/mcp/src/testing/index.ts +++ b/packages/plugins/mcp/src/testing/index.ts @@ -9,6 +9,7 @@ export { makeGreetingMcpServer, makeImageMcpServer, makeMutableCatalogMcpServer, + makeUndeclaredStructuredMcpServer, serveMcpServer, serveMcpServerWithOAuth, type McpTestRequest, diff --git a/packages/plugins/mcp/src/testing/server.ts b/packages/plugins/mcp/src/testing/server.ts index dceac0db2a..df606b5cf2 100644 --- a/packages/plugins/mcp/src/testing/server.ts +++ b/packages/plugins/mcp/src/testing/server.ts @@ -514,6 +514,30 @@ export const makeElicitationMcpServer = () => { return server; }; +/** The muscle-memory case: a server returning structured data at runtime with + * NO declared output schema, like most real MCP servers. The advertised + * catalog entry gives the client nothing to type `structuredContent` with. */ +export const makeUndeclaredStructuredMcpServer = () => { + const server = new McpServer( + { name: "undeclared-structured-test-server", version: "1.0.0" }, + { capabilities: {} }, + ); + + server.registerTool( + "undeclared_structured_echo", + { + description: "Returns structured data it never declared a schema for", + inputSchema: { value: z.string() }, + }, + async ({ value }: { value: string }) => ({ + content: [{ type: "text" as const, text: value }], + structuredContent: { value, length: value.length, ok: true }, + }), + ); + + return server; +}; + /** * A server whose tool catalog mutates at runtime. `renameTool` renames the * advertised tool from `initialToolName` to `renamedToolName` via the SDK's