Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@
"@silvia-odwyer/photon-node@0.3.4": "patches/@silvia-odwyer%2Fphoton-node@0.3.4.patch",
"@standard-community/standard-openapi@0.2.9": "patches/@standard-community%2Fstandard-openapi@0.2.9.patch",
"solid-js@1.9.10": "patches/solid-js@1.9.10.patch",
"@ai-sdk/xai@3.0.102": "patches/@ai-sdk%2Fxai@3.0.102.patch",
"@ai-sdk/xai@3.0.139": "patches/@ai-sdk%2Fxai@3.0.139.patch",
"@ai-sdk/mistral@3.0.51": "patches/@ai-sdk%2Fmistral@3.0.51.patch",
"gcp-metadata@8.1.2": "patches/gcp-metadata@8.1.2.patch",
"pacote@21.5.0": "patches/pacote@21.5.0.patch",
Expand Down
2 changes: 1 addition & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
"@ai-sdk/provider-utils": "4.0.51",
"@ai-sdk/togetherai": "2.0.68",
"@ai-sdk/vercel": "2.0.39",
"@ai-sdk/xai": "3.0.102",
"@ai-sdk/xai": "3.0.139",
"@aws-sdk/credential-providers": "3.1057.0",
"@effect/opentelemetry": "catalog:",
"@effect/platform-node": "catalog:",
Expand Down
2 changes: 1 addition & 1 deletion packages/opencode/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
"@ai-sdk/provider": "3.0.16",
"@ai-sdk/togetherai": "2.0.68",
"@ai-sdk/vercel": "2.0.39",
"@ai-sdk/xai": "3.0.102",
"@ai-sdk/xai": "3.0.139",
"@aws-sdk/credential-providers": "3.1057.0",
"@clack/prompts": "1.0.0-alpha.1",
"@effect/opentelemetry": "catalog:",
Expand Down
11 changes: 10 additions & 1 deletion packages/opencode/src/session/message-v2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,12 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
return false
}

// xAI rejects any other image format (e.g. GIF) with invalid_image, failing the whole request.
const rejectedByProvider = (attachment: { mime: string }) =>
model.api.npm === "@ai-sdk/xai" &&
attachment.mime.startsWith("image/") &&
!["image/png", "image/jpeg", "image/webp"].includes(attachment.mime)

const toModelOutput = (options: { toolCallId: string; input: unknown; output: unknown }) => {
const output = options.output
if (typeof output === "string") {
Expand Down Expand Up @@ -297,7 +303,10 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
const outputText = part.state.time.compacted
? "[Old tool result content cleared]"
: truncateToolOutput(part.state.output, options?.toolOutputMaxChars)
const attachments = part.state.time.compacted || options?.stripMedia ? [] : (part.state.attachments ?? [])
const attachments =
part.state.time.compacted || options?.stripMedia
? []
: (part.state.attachments ?? []).filter((a) => !rejectedByProvider(a))

// For providers that don't support media in tool results, extract media files
// (images, PDFs) to be sent as a separate user message
Expand Down
192 changes: 191 additions & 1 deletion packages/opencode/test/session/message-v2.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, test } from "bun:test"
import { SessionV1 } from "@opencode-ai/core/v1/session"
import { APICallError } from "ai"
import { createXai } from "@ai-sdk/xai"
import { APICallError, generateText } from "ai"
import { MessageV2 } from "../../src/session/message-v2"
import { ProviderTransform } from "@/provider/transform"
import type { Provider } from "@/provider/provider"
Expand Down Expand Up @@ -494,6 +495,195 @@ describe("session.message-v2.toModelMessage", () => {
})
})

test("sends tool-result images to xai responses as input_image", async () => {
const xaiModel: Provider.Model = {
...model,
id: ModelV2.ID.make("grok-4.7"),
providerID: ProviderV2.ID.make("xai"),
api: {
id: "grok-4.7",
url: "https://api.x.ai/v1",
npm: "@ai-sdk/xai",
},
capabilities: {
...model.capabilities,
attachment: true,
input: {
...model.capabilities.input,
image: true,
},
},
}
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).toString("base64")
const userID = "m-user-xai"
const assistantID = "m-assistant-xai"
const input: SessionV1.WithParts[] = [
{
info: userInfo(userID),
parts: [
{
...basePart(userID, "u1-xai"),
type: "text",
text: "describe the diagram",
},
] as SessionV1.Part[],
},
{
info: assistantInfo(assistantID, userID),
parts: [
{
...basePart(assistantID, "a1-xai"),
type: "tool",
callID: "call-xai-1",
tool: "read",
state: {
status: "completed",
input: { filePath: "/tmp/diagram.png" },
output: "Image read successfully",
title: "Read",
metadata: {},
time: { start: 0, end: 1 },
attachments: [
{
...basePart(assistantID, "file-xai-1"),
type: "file",
mime: "image/png",
filename: "diagram.png",
url: `data:image/png;base64,${png}`,
},
],
},
},
] as SessionV1.Part[],
},
]

const bodies: Record<string, unknown>[] = []
const xai = createXai({
apiKey: "test",
fetch: Object.assign(
async (url: RequestInfo | URL, init?: RequestInit) => {
bodies.push(await new Request(url, init).json())
return new Response("{}", { status: 400 })
},
{ preconnect: globalThis.fetch.preconnect },
),
})
await generateText({
model: xai.responses(xaiModel.api.id),
messages: ProviderTransform.message(await MessageV2.toModelMessages(input, xaiModel), xaiModel, {}),
providerOptions: { xai: { promptCacheKey: "session", reasoningEffort: "xhigh" } },
maxRetries: 0,
}).catch(() => undefined)

expect(bodies).toHaveLength(1)
expect(bodies[0].prompt_cache_key).toBe("session")
expect(bodies[0].reasoning).toEqual({ effort: "xhigh" })
expect(bodies[0].input).toContainEqual({
type: "function_call_output",
call_id: "call-xai-1",
output: [
{ type: "input_text", text: "Image read successfully" },
{ type: "input_image", image_url: `data:image/png;base64,${png}` },
],
})
})

test("drops tool-result images xai rejects but keeps png and webp", async () => {
const xaiModel: Provider.Model = {
...model,
id: ModelV2.ID.make("grok-4.7"),
providerID: ProviderV2.ID.make("xai"),
api: {
id: "grok-4.7",
url: "https://api.x.ai/v1",
npm: "@ai-sdk/xai",
},
capabilities: {
...model.capabilities,
attachment: true,
input: {
...model.capabilities.input,
image: true,
},
},
}
const images = [
{ mime: "image/png", data: Buffer.from([0x89, 0x50, 0x4e, 0x47]).toString("base64") },
{ mime: "image/gif", data: Buffer.from("GIF89a").toString("base64") },
{ mime: "image/webp", data: Buffer.from("RIFFWEBP").toString("base64") },
]
const userID = "m-user-xai-gif"
const assistantID = "m-assistant-xai-gif"
const input: SessionV1.WithParts[] = [
{
info: userInfo(userID),
parts: [
{
...basePart(userID, "u1-xai-gif"),
type: "text",
text: "describe the images",
},
] as SessionV1.Part[],
},
{
info: assistantInfo(assistantID, userID),
parts: [
{
...basePart(assistantID, "a1-xai-gif"),
type: "tool",
callID: "call-xai-gif",
tool: "read",
state: {
status: "completed",
input: { filePath: "/tmp/images" },
output: "Images read successfully",
title: "Read",
metadata: {},
time: { start: 0, end: 1 },
attachments: images.map((image, index) => ({
...basePart(assistantID, `file-xai-gif-${index}`),
type: "file" as const,
mime: image.mime,
filename: `image-${index}`,
url: `data:${image.mime};base64,${image.data}`,
})),
},
},
] as SessionV1.Part[],
},
]

const bodies: Record<string, unknown>[] = []
const xai = createXai({
apiKey: "test",
fetch: Object.assign(
async (url: RequestInfo | URL, init?: RequestInit) => {
bodies.push(await new Request(url, init).json())
return new Response("{}", { status: 400 })
},
{ preconnect: globalThis.fetch.preconnect },
),
})
await generateText({
model: xai.responses(xaiModel.api.id),
messages: ProviderTransform.message(await MessageV2.toModelMessages(input, xaiModel), xaiModel, {}),
maxRetries: 0,
}).catch(() => undefined)

expect(bodies).toHaveLength(1)
expect(bodies[0].input).toContainEqual({
type: "function_call_output",
call_id: "call-xai-gif",
output: [
{ type: "input_text", text: "Images read successfully" },
{ type: "input_image", image_url: `data:image/png;base64,${images[0].data}` },
{ type: "input_image", image_url: `data:image/webp;base64,${images[2].data}` },
],
})
expect(JSON.stringify(bodies[0])).not.toContain("image/gif")
})

test("moves bedrock pdf tool-result media into a separate user message", async () => {
const bedrockModel: Provider.Model = {
...model,
Expand Down
Loading
Loading