Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/ai/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -770,7 +770,7 @@ const xai = Image.generate({
aspect: "16:9",
resolution: "1K",
providerOptions: {
responseFormat: "b64_json",
responseFormat: "url", // defaults to "b64_json" (inline bytes)
future_option: true,
},
http,
Expand Down
3 changes: 3 additions & 0 deletions packages/ai/src/protocols/fal-images.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ export type FalImageOptions = {
/** Inpainting mask, sent as `mask_url`. */
readonly mask?: Media.Asset
readonly enable_safety_checker?: boolean
/** Defaults to `true`, which returns images inline as data URIs; `false` returns hosted URLs. */
readonly sync_mode?: boolean
} & Record<string, unknown>

export type Request = ImageRequestFor<FalImageOptions>
Expand Down Expand Up @@ -96,6 +98,7 @@ const fromRequest = Effect.fn("FalImages.fromRequest")(function* (request: Reque
image_urls: list && images.length > 0 ? images : undefined,
image_url: list ? undefined : images[0],
mask_url: mask === undefined ? undefined : yield* FalQueue.mediaUrl(mask, route.name),
sync_mode: true,
},
native,
request.http?.body,
Expand Down
10 changes: 8 additions & 2 deletions packages/ai/src/protocols/openai-images.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ const Settings = {
}

const OpenAIImageResponse = Schema.Struct({
// `response_format` picks `b64_json` or `url`; DALL·E and many compatible hosts default to `url`.
// GPT Image models always return `b64_json`; compatible hosts that default to `url` can return either.
data: Schema.Array(Schema.Struct({ b64_json: optionalNull(Schema.String), url: optionalNull(Schema.String) })),
...Settings,
usage: Schema.optional(Usage),
Expand Down Expand Up @@ -141,7 +141,13 @@ const RESERVED_FORM_FIELDS = new Set(["model", "prompt", "image", "image[]", "im
const nativeOptions = (options: OpenAIImageOptions | undefined) => {
if (!options) return undefined
const { outputCompression, outputFormat, partialImages: _, mask: __, ...native } = options
return { output_compression: outputCompression, output_format: outputFormat, ...native }
// `response_format: null` drops a provider's default, for hosts that reject the field.
return {
output_compression: outputCompression,
output_format: outputFormat,
...native,
response_format: native.response_format ?? undefined,
}
}

/**
Expand Down
6 changes: 3 additions & 3 deletions packages/ai/src/protocols/xai-images.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export const EDIT_PATH = "/images/edits"

/** Provider-native options. Common fields (`n`, `aspect`, `resolution`, `images`) live on the request. */
export type XAIImageOptions = {
/** Defaults to `b64_json`; `url` returns a short-lived hosted URL instead. */
readonly responseFormat?: OpenString<"url" | "b64_json">
readonly response_format?: OpenString<"url" | "b64_json">
} & Record<string, unknown>
Expand Down Expand Up @@ -47,9 +48,8 @@ const XAIImageResponse = Schema.Struct({
// ---------------------------------------------------------------------------

const nativeOptions = (options: XAIImageOptions | undefined) => {
if (!options) return undefined
const { responseFormat, ...native } = options
return { response_format: responseFormat, ...native }
const { responseFormat, ...native } = options ?? {}
return { response_format: responseFormat ?? "b64_json", ...native }
}

/** xAI renders `1k` or `2k`; other tiers use the nearest one. */
Expand Down
14 changes: 12 additions & 2 deletions packages/ai/src/providers/openai-compatible.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
import { mergeProviderOptions, ProviderID, type ModelID, type ProviderOptions } from "../schema/index.js"
import { OpenAICompatibleChat } from "../protocols/openai-compatible-chat.js"
import { OpenAIImages } from "../protocols/openai-images.js"
import { MediaRoute } from "../route/media.js"
Expand Down Expand Up @@ -39,8 +39,18 @@ export const configure = (input: Config) => {
return {
id: ProviderID.make(provider),
model: (modelID: string | ModelID) => route.model<OpenAIProviderOptionsInput>({ id: modelID }),
// Many compatible hosts default to hosted URLs. GPT Image models always return bytes and reject the field; a
// proxy serving one under another name can drop the default with `response_format: null`.
image: (modelID: string | ModelID, providerOptions?: ProviderOptions) =>
OpenAIImages.model({ ...media, id: modelID, provider, providerOptions }),
OpenAIImages.model({
...media,
id: modelID,
provider,
providerOptions: mergeProviderOptions(
/gpt-image|chatgpt-image/.test(modelID) ? undefined : { response_format: "b64_json" },
Comment thread
rekram1-node marked this conversation as resolved.
Resolved
providerOptions,
),
}),
configure,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"headers": {
"content-type": "application/json"
},
"body": "{\"prompt\":\"A simple flat black circle centered on a plain white background.\",\"image_size\":{\"width\":512,\"height\":512},\"output_format\":\"jpeg\"}"
"body": "{\"prompt\":\"A simple flat black circle centered on a plain white background.\",\"image_size\":{\"width\":512,\"height\":512},\"sync_mode\":false,\"output_format\":\"jpeg\"}"
},
"response": {
"status": 200,
Expand Down
51 changes: 48 additions & 3 deletions packages/ai/test/image.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
Fal,
Google,
OpenAI,
OpenAICompatible,
Replicate,
Stability,
VercelAIGateway,
Expand Down Expand Up @@ -169,6 +170,39 @@ describe("Image", () => {
}),
)

it.effect("requests inline bytes by default where hosts would return URLs", () =>
Effect.gen(function* () {
const bodies: Array<unknown> = []
const compatible = OpenAICompatible.configure({ apiKey: "test", baseURL: "https://compatible.test/v1" })
const xai = XAI.configure({ apiKey: "test", baseURL: "https://api.xai.test/v1" })
const generate = (model: Parameters<typeof Image.generate>[0]["model"], providerOptions?: Record<string, unknown>) =>
Image.generate({ model, prompt: "A lighthouse", providerOptions }).pipe(
Effect.provide(
layer((input) => {
bodies.push(JSON.parse(input.text))
return Effect.succeed(json(input, { data: [{ b64_json: "AQID" }] }))
}),
),
)

yield* generate(compatible.image("flux-schnell"))
yield* generate(compatible.image("flux-schnell"), { response_format: "url" })
yield* generate(compatible.image("openai/gpt-image-2"))
yield* generate(compatible.image("image-alias"), { response_format: null })
yield* generate(xai.image("grok-imagine-image"))
yield* generate(xai.image("grok-imagine-image"), { responseFormat: "url" })

expect(bodies).toEqual([
{ model: "flux-schnell", prompt: "A lighthouse", response_format: "b64_json" },
{ model: "flux-schnell", prompt: "A lighthouse", response_format: "url" },
{ model: "openai/gpt-image-2", prompt: "A lighthouse" },
{ model: "image-alias", prompt: "A lighthouse" },
{ model: "grok-imagine-image", prompt: "A lighthouse", response_format: "b64_json" },
{ model: "grok-imagine-image", prompt: "A lighthouse", response_format: "url" },
])
}),
)

describe("OpenAI-style url results", () => {
const generate = (data: unknown) =>
Image.generate({
Expand Down Expand Up @@ -457,6 +491,7 @@ describe("Image", () => {
{ url: "https://example.test/source.jpg", type: "image_url" },
{ file_id: "file_123" },
],
response_format: "b64_json",
})
return Effect.succeed(
input.respond(JSON.stringify({ data: [{ b64_json: "AQID", mime_type: "image/png" }] }), {
Expand Down Expand Up @@ -484,6 +519,7 @@ describe("Image", () => {
model: "future-model",
prompt: "Edit this image",
image: { file_id: "file_123" },
response_format: "b64_json",
})
return Effect.succeed(
input.respond(JSON.stringify({ data: [{ b64_json: "AQID", mime_type: "image/png" }] }), {
Expand Down Expand Up @@ -945,12 +981,21 @@ describe("Image", () => {
images: images.slice(0, 1),
aspect: "16:9",
})
yield* Image.start({ model: fal.image("fal-ai/flux-pro/kontext/max/multi"), prompt: "A lighthouse", images })
yield* Image.start({
model: fal.image("fal-ai/flux-pro/kontext/max/multi"),
prompt: "A lighthouse",
images,
providerOptions: { sync_mode: false },
})

expect(rejected.reason).toMatchObject({ _tag: "UnsupportedOperation", operation: "media.resolution" })
expect(bodies).toEqual([
{ prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png" },
{ prompt: "A lighthouse", image_urls: ["https://example.test/a.png", "https://example.test/b.png"] },
{ prompt: "A lighthouse", aspect_ratio: "16:9", image_url: "https://example.test/a.png", sync_mode: true },
{
prompt: "A lighthouse",
image_urls: ["https://example.test/a.png", "https://example.test/b.png"],
sync_mode: false,
},
])
}).pipe(
Effect.provide(
Expand Down
2 changes: 1 addition & 1 deletion packages/ai/test/provider/fal-images.recorded.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ describe("fal Images recorded", () => {
prompt: "A simple flat black circle centered on a plain white background.",
aspect: "1:1",
resolution: "0.5K",
providerOptions: { output_format: "jpeg" },
providerOptions: { output_format: "jpeg", sync_mode: false },
},
{ poll: queuedPoll },
)
Expand Down
2 changes: 0 additions & 2 deletions packages/ai/test/provider/xai-images.recorded.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ describe("xAI Images recorded", () => {
prompt: "A simple flat black diamond centered on a plain white background.",
aspect: "1:1",
resolution: "1K",
providerOptions: { responseFormat: "b64_json" },
})

expect(response.images).toHaveLength(1)
Expand All @@ -46,7 +45,6 @@ describe("xAI Images recorded", () => {
],
aspect: "1:1",
resolution: "1K",
providerOptions: { responseFormat: "b64_json" },
})

expect(response.image.mediaType).toMatch(/^image\/(jpeg|png)$/)
Expand Down
Loading