From a14c4e77477ee9f0b87c18172228c3ea7a672c1d Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 15:35:10 +0100 Subject: [PATCH] feat(sdk): plugin contract, http client, fixture harness Co-Authored-By: Claude Opus 5.5 --- packages/plugin-sdk/src/http.ts | 101 ++++++++++++ packages/plugin-sdk/src/partial-date.ts | 24 +++ packages/plugin-sdk/src/testing.ts | 60 +++++++ packages/plugin-sdk/src/types.ts | 51 ++++++ packages/plugin-sdk/test/http.test.ts | 149 ++++++++++++++++++ packages/plugin-sdk/test/partial-date.test.ts | 35 ++++ 6 files changed, 420 insertions(+) create mode 100644 packages/plugin-sdk/src/http.ts create mode 100644 packages/plugin-sdk/src/partial-date.ts create mode 100644 packages/plugin-sdk/src/testing.ts create mode 100644 packages/plugin-sdk/src/types.ts create mode 100644 packages/plugin-sdk/test/http.test.ts create mode 100644 packages/plugin-sdk/test/partial-date.test.ts diff --git a/packages/plugin-sdk/src/http.ts b/packages/plugin-sdk/src/http.ts new file mode 100644 index 0000000..e596d25 --- /dev/null +++ b/packages/plugin-sdk/src/http.ts @@ -0,0 +1,101 @@ +import type { HttpClient } from "./types.ts"; + +export class HttpError extends Error { + readonly status: number; + readonly body: string; + constructor(status: number, body: string, url = "") { + super(`HTTP ${status}${url ? ` for ${url}` : ""}`); + this.name = "HttpError"; + this.status = status; + this.body = body; + } +} + +export interface HttpClientOptions { + ratePerMinute?: number; + burst?: number; + maxRetries?: number; + userAgent: string; + fetchImpl?: typeof fetch; + sleep?: (ms: number) => Promise; + now?: () => number; +} + +const retryable = (status: number) => status === 429 || status >= 500; + +function retryDelayMs(res: Response, attempt: number, now: number): number { + const ra = res.headers.get("retry-after"); + if (ra !== null && ra.trim() !== "") { + const secs = Number(ra); + if (Number.isFinite(secs)) return Math.max(0, secs * 1000); + const at = Date.parse(ra); + if (!Number.isNaN(at)) return Math.max(0, at - now); + } + const reset = res.headers.get("x-ratelimit-reset-in"); + if (reset !== null) { + const secs = Number(reset); + if (Number.isFinite(secs)) return Math.max(0, secs * 1000); + } + return 500 * 2 ** attempt; +} + +export function createHttpClient(opts: HttpClientOptions): HttpClient { + const doFetch = opts.fetchImpl ?? fetch; + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const now = opts.now ?? Date.now; + const maxRetries = opts.maxRetries ?? 3; + + // Token bucket: capacity `burst`, refilling at ratePerMinute / 60000 tokens per ms. + const perMs = opts.ratePerMinute ? opts.ratePerMinute / 60_000 : 0; + const capacity = Math.max(1, opts.burst ?? 1); + let tokens = capacity; + let last = now(); + let queue: Promise = Promise.resolve(); + + async function take(): Promise { + for (;;) { + const t = now(); + tokens = Math.min(capacity, tokens + (t - last) * perMs); + last = t; + if (tokens >= 1 - 1e-9) { + tokens -= 1; + return; + } + await sleep(Math.ceil((1 - tokens) / perMs)); + } + } + function acquire(): Promise { + if (!perMs) return Promise.resolve(); + const p = queue.then(take); + queue = p.catch(() => {}); + return p; + } + + async function send(url: string, init: RequestInit | undefined): Promise { + const headers = new Headers(init?.headers); + if (!headers.has("user-agent")) headers.set("user-agent", opts.userAgent); + for (let attempt = 0; ; attempt++) { + await acquire(); + const res = await doFetch(url, { ...init, headers }); + if (!retryable(res.status)) return res; + if (attempt >= maxRetries) throw new HttpError(res.status, await res.text(), url); + const delay = retryDelayMs(res, attempt, now()); + await res.body?.cancel(); + await sleep(delay); + } + } + + return { + raw: (url, init) => send(url, init), + async json(url: string, init?: RequestInit & { query?: Record }): Promise { + const { query, ...rest } = init ?? {}; + const u = new URL(url); + for (const [k, v] of Object.entries(query ?? {})) if (v !== undefined) u.searchParams.set(k, String(v)); + const headers = new Headers(rest.headers); + if (!headers.has("accept")) headers.set("accept", "application/json"); + const res = await send(u.toString(), { ...rest, headers }); + if (!res.ok) throw new HttpError(res.status, await res.text(), u.toString()); + return (await res.json()) as T; + }, + }; +} diff --git a/packages/plugin-sdk/src/partial-date.ts b/packages/plugin-sdk/src/partial-date.ts new file mode 100644 index 0000000..a71bce4 --- /dev/null +++ b/packages/plugin-sdk/src/partial-date.ts @@ -0,0 +1,24 @@ +import type { PartialDate } from "./types.ts"; + +const RE = /^(\d{4})(?:-(\d{2})(?:-(\d{2})(T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?)?)?$/; + +/** Parses by string only; never goes through Date, so no timezone shift. */ +export function parsePartialDate(input: string | null | undefined): PartialDate | null { + if (input == null) return null; + const s = input.trim(); + const m = RE.exec(s); + if (!m) return null; + const year = Number(m[1]); + if (m[2] === undefined) return { value: s, precision: "year" }; + const month = Number(m[2]); + if (month < 1 || month > 12) return null; + if (m[3] === undefined) return { value: s, precision: "month" }; + const day = Number(m[3]); + const dim = new Date(Date.UTC(year, month, 0)).getUTCDate(); + if (day < 1 || day > dim) return null; + return { value: s, precision: "day" }; +} + +export function comparePartialDates(a: PartialDate, b: PartialDate): number { + return a.value < b.value ? -1 : a.value > b.value ? 1 : 0; +} diff --git a/packages/plugin-sdk/src/testing.ts b/packages/plugin-sdk/src/testing.ts new file mode 100644 index 0000000..8b8170e --- /dev/null +++ b/packages/plugin-sdk/src/testing.ts @@ -0,0 +1,60 @@ +import { createHttpClient } from "./http.ts"; +import type { PluginContext } from "./types.ts"; + +type Route = unknown | ((req: Request) => unknown); + +/** Key is "METHOD full-url" or "METHOD url-without-query". Unknown routes throw. */ +export function createFixtureHttp(routes: Record): { fetch: typeof fetch; calls: Request[] } { + const calls: Request[] = []; + const fn = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const req = new Request(input, init); + calls.push(req); + const u = new URL(req.url); + const full = `${req.method} ${u.toString()}`; + const bare = `${req.method} ${u.origin}${u.pathname}`; + const key = full in routes ? full : bare in routes ? bare : null; + if (key === null) throw new Error(`no fixture route for ${full}`); + const r = routes[key]; + const out = typeof r === "function" ? (r as (req: Request) => unknown)(req) : r; + return out instanceof Response ? out : Response.json(out); + }; + return { fetch: fn as typeof fetch, calls }; +} + +export function createTestContext( + partial: Partial & { fetch?: typeof fetch } = {}, +): PluginContext & { saved: { credentials: Record[]; cursor: string[] } } { + const { fetch: fetchImpl, ...overrides } = partial; + const saved = { credentials: [] as Record[], cursor: [] as string[] }; + let creds: Record | null = null; + let cursor: string | null = null; + const ctx: PluginContext = { + http: createHttpClient({ userAgent: "sttracker-test/0", fetchImpl, sleep: async () => {} }), + log: () => {}, + config: {}, + credentials: { + get: async () => creds, + save: async (c) => { + creds = c; + saved.credentials.push(c); + }, + }, + cursor: { + get: async () => cursor, + set: async (c) => { + cursor = c; + saved.cursor.push(c); + }, + }, + now: () => new Date(), + signal: new AbortController().signal, + ...overrides, + }; + return Object.assign(ctx, { saved }); +} + +export async function collect(it: AsyncIterable): Promise { + const out: T[] = []; + for await (const x of it) out.push(x); + return out; +} diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts new file mode 100644 index 0000000..d57df1d --- /dev/null +++ b/packages/plugin-sdk/src/types.ts @@ -0,0 +1,51 @@ +export type MediaType = + | "book" | "comic" | "manga" | "game" | "movie" | "show" | "season" | "episode" + | "podcast" | "podcast_episode" | "audio_drama" | "artist" | "album" | "track" + | "video" | "other"; +export type Status = "planned" | "in_progress" | "paused" | "completed" | "dropped" | "none"; +export type Precision = "day" | "month" | "year" | "unknown"; +export interface PartialDate { value: string; precision: Precision } +export type Capability = "pull" | "pullChanges" | "push" | "ownedLibrary" | "ingest" | "lookup" | "importFile" | "exportFile"; +export interface ConfigField { key: string; label: string; type: "string" | "secret" | "number" | "boolean" | "select"; required?: boolean; options?: string[]; help?: string } +export interface PluginManifest { id: string; name: string; version: string; sdkVersion: 1; mediaTypes: MediaType[]; capabilities: Capability[]; config: ConfigField[]; attribution?: string } +export interface WorkHint { title: string; mediaType: MediaType; year?: number; creators?: { name: string; role?: string }[]; externalIds: Record; coverUrl?: string; description?: string; parent?: WorkHint; position?: { season?: number; episode?: number; index?: number } } +export interface EntryFields { status?: Status; rating?: number | null; review?: string | null; notes?: string | null; favorite?: boolean; tags?: string[] } +export interface EngagementFields { startedAt?: PartialDate | null; finishedAt?: PartialDate | null; progress?: Record; platform?: string | null; rating?: number | null; notes?: string | null } +export interface ListenFields { listenedAt: string; artist: string; track: string; album?: string; durationMs?: number; mbids?: { recording?: string; release?: string; artist?: string } } +interface Base { key: string; raw: unknown; deleted?: boolean; mergedIntoKey?: string } +export type SourceRecord = + | (Base & { kind: "entry"; work: WorkHint; entry: EntryFields }) + | (Base & { kind: "engagement"; work: WorkHint; engagement: EngagementFields }) + | (Base & { kind: "listen"; listen: ListenFields }) + | (Base & { kind: "list"; list: { name: string; description?: string; items: { work: WorkHint; note?: string }[] } }); +export interface HttpClient { + json(url: string, init?: RequestInit & { query?: Record }): Promise; + raw(url: string, init?: RequestInit): Promise; +} +export interface PluginContext { + http: HttpClient; + log: (msg: string) => void; + config: Record; + credentials: { get(): Promise | null>; save(c: Record): Promise }; + cursor: { get(): Promise; set(c: string): Promise }; + now(): Date; + signal: AbortSignal; +} +export interface DeviceAuth { + start(ctx: PluginContext): Promise<{ userCode: string; verificationUrl: string; deviceCode: string; intervalSec: number; expiresInSec: number }>; + poll(ctx: PluginContext, deviceCode: string): Promise<"pending" | Record>; +} +export interface SourcePlugin { + manifest: PluginManifest; + deviceAuth?: DeviceAuth; + pull?(ctx: PluginContext): AsyncIterable; + pullChanges?(ctx: PluginContext): AsyncIterable; + lookup?(ctx: PluginContext, q: { title: string; mediaType?: MediaType; year?: number }): Promise; + importFile?(ctx: PluginContext, file: { name: string; bytes: Uint8Array }): AsyncIterable; +} + +export function definePlugin(p: SourcePlugin): SourcePlugin { + if (p.manifest.sdkVersion !== 1) throw new Error(`plugin "${p.manifest.id}": unsupported sdkVersion ${String(p.manifest.sdkVersion)}`); + if (typeof p.manifest.id !== "string" || p.manifest.id.trim() === "") throw new Error("plugin manifest.id must be a non-empty string"); + return p; +} diff --git a/packages/plugin-sdk/test/http.test.ts b/packages/plugin-sdk/test/http.test.ts new file mode 100644 index 0000000..ae47907 --- /dev/null +++ b/packages/plugin-sdk/test/http.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, test } from "bun:test"; +import { createHttpClient, HttpError } from "../src/http.ts"; +import { collect, createFixtureHttp, createTestContext } from "../src/testing.ts"; +import { definePlugin } from "../src/types.ts"; + +const res = (status: number, body: unknown = {}, headers: Record = {}) => + new Response(JSON.stringify(body), { status, headers }); + +function seq(...responses: Response[]) { + const reqs: Request[] = []; + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + reqs.push(new Request(input, init)); + return responses.shift()!; + }) as typeof fetch; + return { fetchImpl, reqs }; +} + +describe("http client", () => { + test("retries 429 honouring Retry-After", async () => { + const { fetchImpl } = seq(res(429, {}, { "Retry-After": "2" }), res(200, { ok: 1 })); + const sleeps: number[] = []; + const http = createHttpClient({ userAgent: "t/1", fetchImpl, sleep: async (ms) => void sleeps.push(ms) }); + expect(await http.json<{ ok: number }>("https://x.test/a")).toEqual({ ok: 1 }); + expect(sleeps).toEqual([2000]); + }); + + test("honours Retry-After as HTTP date and X-RateLimit-Reset-In", async () => { + const t0 = Date.UTC(2026, 0, 1, 0, 0, 0); + const date = new Date(t0 + 5000).toUTCString(); + const { fetchImpl } = seq( + res(503, {}, { "Retry-After": date }), + res(429, {}, { "X-RateLimit-Reset-In": "3" }), + res(200, {}), + ); + const sleeps: number[] = []; + const http = createHttpClient({ userAgent: "t/1", fetchImpl, now: () => t0, sleep: async (ms) => void sleeps.push(ms) }); + await http.json("https://x.test/a"); + expect(sleeps).toEqual([5000, 3000]); + }); + + test("exponential backoff without headers", async () => { + const { fetchImpl } = seq(res(500), res(500), res(200)); + const sleeps: number[] = []; + const http = createHttpClient({ userAgent: "t/1", fetchImpl, sleep: async (ms) => void sleeps.push(ms) }); + await http.json("https://x.test/a"); + expect(sleeps.length).toBe(2); + expect(sleeps[1]!).toBeGreaterThan(sleeps[0]!); + }); + + test("gives up after maxRetries with HttpError", async () => { + const { fetchImpl, reqs } = seq(res(500, { e: 1 }), res(500, { e: 2 }), res(500, { e: 3 })); + const http = createHttpClient({ userAgent: "t/1", maxRetries: 2, fetchImpl, sleep: async () => {} }); + const err = (await http.json("https://x.test/a").catch((e) => e)) as HttpError; + expect(err).toBeInstanceOf(HttpError); + expect(err).toBeInstanceOf(Error); + expect(err.status).toBe(500); + expect(err.body).toBe('{"e":3}'); + expect(reqs.length).toBe(3); + }); + + test("non-retryable errors throw immediately from json", async () => { + const { fetchImpl, reqs } = seq(res(404, { nope: true })); + const http = createHttpClient({ userAgent: "t/1", fetchImpl, sleep: async () => {} }); + const err = (await http.json("https://x.test/a").catch((e) => e)) as HttpError; + expect(err).toBeInstanceOf(HttpError); + expect(err.status).toBe(404); + expect(reqs.length).toBe(1); + }); + + test("json sends Accept, User-Agent and appends query", async () => { + const { fetchImpl, reqs } = seq(res(200, {})); + const http = createHttpClient({ userAgent: "sttracker/1", fetchImpl, sleep: async () => {} }); + await http.json("https://x.test/a?z=1", { query: { q: "a b", n: 2, skip: undefined } }); + const r = reqs[0]!; + expect(r.headers.get("accept")).toBe("application/json"); + expect(r.headers.get("user-agent")).toBe("sttracker/1"); + const u = new URL(r.url); + expect(u.searchParams.get("z")).toBe("1"); + expect(u.searchParams.get("q")).toBe("a b"); + expect(u.searchParams.get("n")).toBe("2"); + expect(u.searchParams.has("skip")).toBe(false); + }); + + test("rate limits to ratePerMinute", async () => { + let clock = 1_000_000; + const sleeps: number[] = []; + const { fetchImpl } = seq(res(200), res(200), res(200)); + const http = createHttpClient({ + userAgent: "t/1", + ratePerMinute: 60, + burst: 1, + fetchImpl, + now: () => clock, + sleep: async (ms) => { + sleeps.push(ms); + clock += ms; + }, + }); + await http.json("https://x.test/1"); + await http.json("https://x.test/2"); + await http.json("https://x.test/3"); + expect(sleeps.length).toBe(2); + for (const s of sleeps) expect(Math.abs(s - 1000)).toBeLessThanOrEqual(5); + }); +}); + +describe("testing harness", () => { + test("fixture routes match by full url or url without query; unknown throws", async () => { + const fx = createFixtureHttp({ + "GET https://x.test/a": { a: 1 }, + "GET https://x.test/b?id=2": { b: 2 }, + "POST https://x.test/c": (req: Request) => ({ method: req.method }), + }); + expect(await (await fx.fetch("https://x.test/a?q=1")).json()).toEqual({ a: 1 }); + expect(await (await fx.fetch("https://x.test/b?id=2")).json()).toEqual({ b: 2 }); + expect(await (await fx.fetch("https://x.test/c", { method: "POST" })).json()).toEqual({ method: "POST" }); + expect(fx.calls.length).toBe(3); + await expect(fx.fetch("https://x.test/zzz")).rejects.toThrow(); + }); + + test("createTestContext wires http, credentials and cursor", async () => { + const fx = createFixtureHttp({ "GET https://x.test/a": { a: 1 } }); + const ctx = createTestContext({ fetch: fx.fetch }); + expect(await ctx.http.json("https://x.test/a")).toEqual({ a: 1 }); + expect(await ctx.credentials.get()).toBeNull(); + await ctx.credentials.save({ token: "t" }); + expect(await ctx.credentials.get()).toEqual({ token: "t" }); + await ctx.cursor.set("c1"); + expect(await ctx.cursor.get()).toBe("c1"); + expect(ctx.saved).toEqual({ credentials: [{ token: "t" }], cursor: ["c1"] }); + async function* gen() { + yield 1; + yield 2; + } + expect(await collect(gen())).toEqual([1, 2]); + }); +}); + +describe("definePlugin", () => { + const manifest = { id: "p", name: "P", version: "1", sdkVersion: 1 as const, mediaTypes: [], capabilities: [], config: [] }; + test("identity for valid manifest", () => { + const p = { manifest }; + expect(definePlugin(p)).toBe(p); + }); + test("rejects bad sdkVersion or empty id", () => { + expect(() => definePlugin({ manifest: { ...manifest, sdkVersion: 2 as never } })).toThrow(); + expect(() => definePlugin({ manifest: { ...manifest, id: "" } })).toThrow(); + }); +}); diff --git a/packages/plugin-sdk/test/partial-date.test.ts b/packages/plugin-sdk/test/partial-date.test.ts new file mode 100644 index 0000000..9f25b09 --- /dev/null +++ b/packages/plugin-sdk/test/partial-date.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "bun:test"; +import { comparePartialDates, parsePartialDate } from "../src/partial-date.ts"; + +describe("parsePartialDate", () => { + const table: [string | null | undefined, { value: string; precision: string } | null][] = [ + ["2019", { value: "2019", precision: "year" }], + ["2019-03", { value: "2019-03", precision: "month" }], + ["2019-03-04", { value: "2019-03-04", precision: "day" }], + ["2019-03-04T10:20:30Z", { value: "2019-03-04T10:20:30Z", precision: "day" }], + ["2019-03-04T10:20:30.123+02:00", { value: "2019-03-04T10:20:30.123+02:00", precision: "day" }], + ["03/04/2019", null], + ["", null], + [null, null], + [undefined, null], + ["2019-13", null], + ["2019-02-30", null], + ["garbage", null], + ]; + for (const [input, expected] of table) { + test(`${JSON.stringify(input)}`, () => { + expect(parsePartialDate(input)).toEqual(expected as never); + }); + } +}); + +describe("comparePartialDates", () => { + test("orders by value", () => { + const y = { value: "2019", precision: "year" as const }; + const m = { value: "2019-03", precision: "month" as const }; + const d = { value: "2019-03-04", precision: "day" as const }; + expect(comparePartialDates(y, m)).toBeLessThan(0); + expect(comparePartialDates(d, m)).toBeGreaterThan(0); + expect(comparePartialDates(d, d)).toBe(0); + }); +});