feat(ai): OpenAI-compatible provider (covers Ollama)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
3c2220145b
commit
76ea0d5101
@@ -3,3 +3,4 @@ export { createSecretBox, type SecretBox } from "./crypto";
|
||||
/** Migration folders for this package. Empty: the AI package uses the ai_* tables from 0001_init.sql. */
|
||||
export const migrationsDirs: string[] = [];
|
||||
export * from "./settings";
|
||||
export * from "./providers";
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { AiProvider } from "@stc/contracts";
|
||||
import type { ProviderConfig } from "../settings";
|
||||
import { createAnthropicProvider } from "./anthropic";
|
||||
import { createOpenAiProvider } from "./openai";
|
||||
|
||||
export { createAnthropicProvider, DEFAULT_ANTHROPIC_MODEL } from "./anthropic";
|
||||
export { createOpenAiProvider } from "./openai";
|
||||
|
||||
export type ProviderFactory = (cfg: ProviderConfig, apiKey: string | null) => AiProvider;
|
||||
|
||||
export function createProvider(cfg: ProviderConfig, apiKey: string | null, fetchImpl?: typeof fetch): AiProvider {
|
||||
if (cfg.kind === "anthropic") {
|
||||
if (!apiKey) throw new Error("The Anthropic provider needs an API key");
|
||||
return createAnthropicProvider({ apiKey, model: cfg.model, baseUrl: cfg.baseUrl, effort: cfg.effort, fetch: fetchImpl });
|
||||
}
|
||||
if (!cfg.baseUrl || !cfg.model) throw new Error("An OpenAI-compatible provider needs a base URL and a model");
|
||||
return createOpenAiProvider({ baseUrl: cfg.baseUrl, model: cfg.model, apiKey, fetch: fetchImpl });
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import type { AiProvider, ChatMessage, ToolSpec } from "@stc/contracts";
|
||||
import { errorMessage, readSse } from "./sse";
|
||||
|
||||
export interface OpenAiOptions { baseUrl: string; model: string; apiKey?: string | null; fetch?: typeof fetch }
|
||||
|
||||
type OaMessage =
|
||||
| { role: "system" | "user"; content: string }
|
||||
| { role: "assistant"; content: string | null; tool_calls?: { id: string; type: "function"; function: { name: string; arguments: string } }[] }
|
||||
| { role: "tool"; tool_call_id: string; content: string };
|
||||
|
||||
function toOpenAi(messages: ChatMessage[]): OaMessage[] {
|
||||
const out: OaMessage[] = [];
|
||||
for (const m of messages) {
|
||||
if (m.role === "assistant" && m.toolCallId) {
|
||||
const call = { id: m.toolCallId, type: "function" as const, function: { name: m.name ?? "", arguments: m.content || "{}" } };
|
||||
const last = out[out.length - 1];
|
||||
if (last && last.role === "assistant") (last.tool_calls ??= []).push(call);
|
||||
else out.push({ role: "assistant", content: null, tool_calls: [call] });
|
||||
} else if (m.role === "tool") out.push({ role: "tool", tool_call_id: m.toolCallId ?? "", content: m.content });
|
||||
else out.push({ role: m.role, content: m.content } as OaMessage);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** OpenAI-compatible chat completions (OpenAI, Ollama, vLLM, LM Studio, OpenRouter …) with streaming tool calls. */
|
||||
export function createOpenAiProvider(o: OpenAiOptions): AiProvider {
|
||||
const doFetch = o.fetch ?? fetch;
|
||||
const base = o.baseUrl.replace(/\/+$/, "");
|
||||
return {
|
||||
id: "openai",
|
||||
async *chat(messages, opts: { tools?: ToolSpec[]; signal?: AbortSignal }) {
|
||||
const body: Record<string, unknown> = { model: o.model, stream: true, messages: toOpenAi(messages) };
|
||||
if (opts.tools?.length) body.tools = opts.tools.map((t) => ({ type: "function", function: { name: t.name, description: t.description, parameters: t.inputSchema } }));
|
||||
const headers: Record<string, string> = { "content-type": "application/json" };
|
||||
if (o.apiKey) headers.authorization = `Bearer ${o.apiKey}`;
|
||||
const res = await doFetch(`${base}/chat/completions`, { method: "POST", headers, body: JSON.stringify(body), signal: opts.signal });
|
||||
if (!res.ok || !res.body) throw new Error(`Chat endpoint error ${await errorMessage(res)}`);
|
||||
const pending = new Map<number, { id: string; name: string; args: string }>();
|
||||
const flush = function* () {
|
||||
for (const [, c] of [...pending].sort((a, b) => a[0] - b[0])) {
|
||||
let input: unknown = {};
|
||||
try { input = c.args ? JSON.parse(c.args) : {}; } catch { input = { _invalidJson: c.args }; }
|
||||
yield { toolCall: { id: c.id, name: c.name, input } };
|
||||
}
|
||||
pending.clear();
|
||||
};
|
||||
for await (const { data } of readSse(res.body)) {
|
||||
if (data.trim() === "[DONE]") break;
|
||||
let ev: any;
|
||||
try { ev = JSON.parse(data); } catch { continue; }
|
||||
if (ev.error) throw new Error(`Chat endpoint error: ${ev.error.message ?? JSON.stringify(ev.error)}`);
|
||||
const choice = ev.choices?.[0];
|
||||
if (!choice) continue;
|
||||
const d = choice.delta ?? {};
|
||||
if (typeof d.content === "string" && d.content) yield { text: d.content };
|
||||
for (const tc of d.tool_calls ?? []) {
|
||||
const idx = tc.index ?? 0;
|
||||
const cur = pending.get(idx) ?? { id: "", name: "", args: "" };
|
||||
if (tc.id) cur.id = tc.id;
|
||||
if (tc.function?.name) cur.name += tc.function.name;
|
||||
if (tc.function?.arguments) cur.args += tc.function.arguments;
|
||||
pending.set(idx, cur);
|
||||
}
|
||||
if (choice.finish_reason) yield* flush();
|
||||
}
|
||||
yield* flush();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { createOpenAiProvider } from "../src/providers/openai";
|
||||
import { createProvider } from "../src/providers";
|
||||
import { collect, fakeFetch, sseBody } from "./helpers";
|
||||
|
||||
const ok = (events: [string | null, unknown][]) => () => new Response(sseBody(events));
|
||||
const chunk = (delta: unknown, finish: string | null = null) => [null, { choices: [{ index: 0, delta, finish_reason: finish }] }] as [null, unknown];
|
||||
|
||||
test("streams text deltas from an Ollama-style endpoint without a key", async () => {
|
||||
const ff = fakeFetch([ok([chunk({ role: "assistant", content: "Hel" }), chunk({ content: "lo" }), chunk({}, "stop"), [null, "[DONE]"]])]);
|
||||
const p = createOpenAiProvider({ baseUrl: "http://ollama:11434/v1/", model: "llama3.1", fetch: ff.fetch });
|
||||
const r = await collect(p.chat([{ role: "system", content: "sys" }, { role: "user", content: "hi" }], {}));
|
||||
expect(r.text).toBe("Hello");
|
||||
const c = ff.calls[0]!;
|
||||
expect(c.url).toBe("http://ollama:11434/v1/chat/completions");
|
||||
expect(new Headers(c.init.headers).get("authorization")).toBeNull();
|
||||
expect(c.body).toMatchObject({ model: "llama3.1", stream: true, messages: [{ role: "system", content: "sys" }, { role: "user", content: "hi" }] });
|
||||
});
|
||||
|
||||
test("assembles fragmented tool calls and maps a tool round back", async () => {
|
||||
const ff = fakeFetch([
|
||||
ok([
|
||||
chunk({ tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "read_lesson", arguments: "" } }] }),
|
||||
chunk({ tool_calls: [{ index: 0, function: { arguments: "{\"lessonId\":" } }] }),
|
||||
chunk({ tool_calls: [{ index: 0, function: { arguments: "\"m1-intro\"}" } }] }),
|
||||
chunk({}, "tool_calls"),
|
||||
[null, "[DONE]"],
|
||||
]),
|
||||
ok([chunk({ content: "Done" }), [null, "[DONE]"]]),
|
||||
]);
|
||||
const p = createOpenAiProvider({ baseUrl: "https://api.example.com/v1", apiKey: "sk-x", model: "gpt-x", fetch: ff.fetch });
|
||||
const tools = [{ name: "read_lesson", description: "Read", inputSchema: { type: "object" } }];
|
||||
const r = await collect(p.chat([{ role: "user", content: "q" }], { tools }));
|
||||
expect(r.toolCalls).toEqual([{ id: "call_1", name: "read_lesson", input: { lessonId: "m1-intro" } }]);
|
||||
expect(ff.calls[0]!.body.tools).toEqual([{ type: "function", function: { name: "read_lesson", description: "Read", parameters: { type: "object" } } }]);
|
||||
expect(new Headers(ff.calls[0]!.init.headers).get("authorization")).toBe("Bearer sk-x");
|
||||
|
||||
await collect(p.chat([
|
||||
{ role: "user", content: "q" },
|
||||
{ role: "assistant", content: "Reading." },
|
||||
{ role: "assistant", content: "{\"lessonId\":\"m1-intro\"}", toolCallId: "call_1", name: "read_lesson" },
|
||||
{ role: "tool", content: "# Introduction", toolCallId: "call_1", name: "read_lesson" },
|
||||
], { tools }));
|
||||
expect(ff.calls[1]!.body.messages).toEqual([
|
||||
{ role: "user", content: "q" },
|
||||
{ role: "assistant", content: "Reading.", tool_calls: [{ id: "call_1", type: "function", function: { name: "read_lesson", arguments: "{\"lessonId\":\"m1-intro\"}" } }] },
|
||||
{ role: "tool", tool_call_id: "call_1", content: "# Introduction" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("errors carry the endpoint message", async () => {
|
||||
const ff = fakeFetch([() => Response.json({ error: { message: "model not found" } }, { status: 404 })]);
|
||||
await expect(collect(createOpenAiProvider({ baseUrl: "http://x/v1", model: "m", fetch: ff.fetch }).chat([{ role: "user", content: "x" }], {}))).rejects.toThrow(/model not found/);
|
||||
});
|
||||
|
||||
test("createProvider picks the implementation and defaults", () => {
|
||||
expect(createProvider({ kind: "anthropic" }, "k").id).toBe("anthropic");
|
||||
expect(createProvider({ kind: "openai", baseUrl: "http://x/v1", model: "m" }, null).id).toBe("openai");
|
||||
expect(() => createProvider({ kind: "anthropic" }, null)).toThrow(/key/);
|
||||
expect(() => createProvider({ kind: "openai" }, null)).toThrow(/base URL|model/);
|
||||
});
|
||||
Reference in New Issue
Block a user