feat(core): jobs, scheduler, plugin registry, connections

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

packages/core/test/connections.test.ts

@@ -0,0 +1,98 @@
+import { expect, spyOn, test } from "bun:test";
+import { join } from "node:path";
+import { forEachDialect } from "@sttracker/db/testing";
+import { decryptJson } from "../src/crypto.ts";
+import { createConnection, deleteConnection, listConnections, makePluginContext, syncHandler, updateConnection } from "../src/connections.ts";
+import { loadPlugins } from "../src/registry.ts";
+import { makeUser } from "./sync/helpers.ts";
+import { newId, nowIso } from "@sttracker/db/ids";
+import type { Connection } from "../src/sync/run.ts";
+
+const SECRET = "test-secret";
+const warn = spyOn(console, "warn").mockImplementation(() => {});
+const reg = await loadPlugins([join(import.meta.dir, "fixtures/plugins")]);
+warn.mockRestore();
+const echo = reg.get("echo")!;
+
+forEachDialect("connections", (getDb) => {
+  const raw = (id: string) => getDb().k.selectFrom("source_connections").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
+  const asConn = async (id: string): Promise<Connection> => {
+    const r = await raw(id);
+    return { id, userId: r.user_id, pluginId: r.plugin_id, name: r.name, config: JSON.parse(r.config), credentialsEnc: r.credentials_enc, scheduleMinutes: null, cursor: r.cursor, status: r.status, lastRunAt: null, createdAt: r.created_at };
+  };
+
+  test("secrets live only in the encrypted blob and never in list", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config: { label: "hi", token: "s3cret" } }, echo.manifest);
+    const r = await raw(id);
+    expect(JSON.parse(r.config)).toEqual({ label: "hi" });
+    expect(r.credentials_enc).not.toContain("s3cret");
+    expect(await decryptJson(SECRET, r.credentials_enc!)).toEqual({ token: "s3cret" });
+    const [v] = await listConnections(db, u);
+    expect(JSON.stringify(v)).not.toContain("s3cret");
+    expect(v).toMatchObject({ id, config: { label: "hi" }, hasCredentials: true });
+    const ctx = await makePluginContext(db, SECRET, await asConn(id), "r", echo.manifest);
+    expect(ctx.config).toEqual({ label: "hi", token: "s3cret" });
+    expect(await ctx.credentials.get()).toEqual({ token: "s3cret" });
+  });
+
+  test("update keeps omitted secrets; delete pauses and clears credentials", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config: { token: "t1" } }, echo.manifest);
+    await updateConnection(db, SECRET, u, id, { config: { label: "new" }, scheduleMinutes: 30 }, echo.manifest);
+    let r = await raw(id);
+    expect(JSON.parse(r.config)).toEqual({ label: "new" });
+    expect(r.schedule_minutes).toBe(30);
+    expect(await decryptJson(SECRET, r.credentials_enc!)).toEqual({ token: "t1" });
+    await updateConnection(db, SECRET, u, id, { config: { token: "t2" } }, echo.manifest);
+    expect(await decryptJson(SECRET, (await raw(id)).credentials_enc!)).toEqual({ token: "t2" });
+    const other = await makeUser(db, "bob");
+    await expect(deleteConnection(db, other, id)).rejects.toThrow();
+    await deleteConnection(db, u, id);
+    r = await raw(id);
+    expect(r.status).toBe("paused");
+    expect(r.credentials_enc).toBeNull();
+    expect((await listConnections(db, u))[0]!.hasCredentials).toBe(false);
+  });
+
+  test("cursor persists on set; http uses manifest rate limit", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config: {} }, echo.manifest);
+    const ctx = await makePluginContext(db, SECRET, await asConn(id), "r", echo.manifest);
+    await ctx.cursor.set("c1");
+    expect((await raw(id)).cursor).toBe("c1");
+    expect(await ctx.cursor.get()).toBe("c1");
+  });
+
+  test("credentials.save persists encrypted before pull continues", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config: { label: "explode", token: "t" } }, echo.manifest);
+    const handler = syncHandler({ db, secret: SECRET, registry: reg });
+    await expect(handler({ connectionId: id })).rejects.toThrow();
+    const r = await raw(id);
+    expect(await decryptJson(SECRET, r.credentials_enc!)).toEqual({ token: "t", refresh: "new-refresh" });
+    expect(r.status).toBe("error");
+  });
+
+  test("sync handler: full first, changes after a recent full, full when payload says so or full is stale", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config: { label: "ok" } }, echo.manifest);
+    const handler = syncHandler({ db, secret: SECRET, registry: reg });
+    const modes = async () => (await db.k.selectFrom("source_runs").select(["mode", "status"]).where("connection_id", "=", id).orderBy("started_at").orderBy("id").execute()).map((r) => `${r.mode}:${r.status}`);
+    await handler({ connectionId: id });
+    await handler({ connectionId: id });
+    await handler({ connectionId: id, mode: "full" });
+    expect(await modes()).toEqual(["full:ok", "changes:ok", "full:ok"]);
+    await db.k.updateTable("source_runs").set({ started_at: new Date(Date.now() - 8 * 86400_000).toISOString() }).where("connection_id", "=", id).execute();
+    await handler({ connectionId: id });
+    expect((await modes()).at(-1)).toBe("full:ok");
+    await db.k.insertInto("source_runs").values({ id: newId(), connection_id: id, mode: "full", started_at: nowIso(), finished_at: null, status: "failed", counts: "{}", log: "", error: "x" }).execute();
+    await handler({ connectionId: id });
+    expect((await modes()).at(-1)).toBe("changes:ok");
+  });
+});

packages/core/test/fixtures/plugins/broken/index.ts

@@ -0,0 +1 @@
+export default { nope: true };

packages/core/test/fixtures/plugins/echo/index.ts

@@ -0,0 +1,35 @@
+import { definePlugin, type SourceRecord } from "@sttracker/plugin-sdk/types";
+
+export default definePlugin({
+  manifest: {
+    id: "echo",
+    name: "Echo",
+    version: "0.0.1",
+    sdkVersion: 1,
+    mediaTypes: ["book"],
+    capabilities: ["pull", "pullChanges"],
+    config: [
+      { key: "label", label: "Label", type: "string" },
+      { key: "token", label: "Token", type: "secret" },
+    ],
+    rateLimit: { perMinute: 30, burst: 2 },
+  },
+  async *pull(ctx) {
+    const cfg = ctx.config;
+    if (cfg.label === "explode") {
+      await ctx.credentials.save({ refresh: "new-refresh" });
+      throw new Error("boom after save");
+    }
+    const rec: SourceRecord = {
+      kind: "entry",
+      key: "echo-1",
+      raw: { label: cfg.label ?? null },
+      work: { title: String(cfg.label ?? "Echo"), mediaType: "book", externalIds: { isbn13: "9780000000002" } },
+      entry: { status: "completed" },
+    };
+    yield rec;
+  },
+  async *pullChanges(ctx) {
+    yield* this.pull!(ctx);
+  },
+});

packages/core/test/jobs.test.ts

@@ -0,0 +1,94 @@
+import { expect, test } from "bun:test";
+import { forEachDialect } from "@sttracker/db/testing";
+import { enqueue, startScheduler, startWorker } from "../src/jobs.ts";
+import { makeConnection, makeUser } from "./sync/helpers.ts";
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+forEachDialect("jobs", (getDb) => {
+  test("enqueue with same lockKey returns null", async () => {
+    const db = getDb();
+    expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeString();
+    expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeNull();
+    expect(await enqueue(db, "sync", {}, { lockKey: "sync:b" })).toBeString();
+    expect(await enqueue(db, "x", {})).toBeString();
+    expect(await enqueue(db, "x", {})).toBeString();
+  });
+
+  test("concurrent enqueue with one lockKey admits exactly one", async () => {
+    const db = getDb();
+    const ids = await Promise.all(Array.from({ length: 6 }, () => enqueue(db, "sync", {}, { lockKey: "k" })));
+    expect(ids.filter((i) => i !== null)).toHaveLength(1);
+  });
+
+  test("worker runs a job and releases the lock", async () => {
+    const db = getDb();
+    const seen: unknown[] = [];
+    const id = await enqueue(db, "t", { n: 1 }, { lockKey: "l" });
+    const w = startWorker(db, { t: async (p) => void seen.push(p) }, { pollMs: 10 });
+    await sleep(150);
+    await w.stop();
+    expect(seen).toEqual([{ n: 1 }]);
+    expect((await db.k.selectFrom("jobs").selectAll().where("id", "=", id!).executeTakeFirstOrThrow()).status).toBe("done");
+    expect(await enqueue(db, "t", {}, { lockKey: "l" })).toBeString();
+  });
+
+  test("a job is claimed by only one of two workers", async () => {
+    const db = getDb();
+    let runs = 0;
+    const h = { t: async () => { runs++; await sleep(30); } };
+    await enqueue(db, "t", {});
+    const a = startWorker(db, h, { pollMs: 5 });
+    const b = startWorker(db, h, { pollMs: 5 });
+    await sleep(200);
+    await Promise.all([a.stop(), b.stop()]);
+    expect(runs).toBe(1);
+  });
+
+  test("worker retries then fails", async () => {
+    const db = getDb();
+    let calls = 0;
+    const id = (await enqueue(db, "bad", {}))!;
+    const w = startWorker(db, { bad: async () => { calls++; throw new Error("nope"); } }, { pollMs: 10 });
+    const row = () => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
+    for (let attempt = 1; attempt <= 5; attempt++) {
+      for (let i = 0; i < 100 && (await row()).attempts < attempt; i++) await sleep(10);
+      const r = await row();
+      expect(r.attempts).toBe(attempt);
+      expect(r.last_error).toBe("nope");
+      if (attempt < 5) {
+        expect(r.status).toBe("queued");
+        const delay = Date.parse(r.run_at) - Date.now();
+        expect(delay).toBeGreaterThan(2 ** attempt * 60_000 - 5_000);
+        expect(delay).toBeLessThanOrEqual(2 ** attempt * 60_000);
+        await db.k.updateTable("jobs").set({ run_at: new Date(Date.now() - 1000).toISOString() }).where("id", "=", id).execute();
+      }
+    }
+    await w.stop();
+    expect((await row()).status).toBe("failed");
+    expect(calls).toBe(5);
+  });
+
+  test("scheduler enqueues due connections only", async () => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const due = await makeConnection(db, u, "p");
+    const fresh = await makeConnection(db, u, "p");
+    const manual = await makeConnection(db, u, "p");
+    const paused = await makeConnection(db, u, "p");
+    const old = new Date(Date.now() - 120 * 60_000).toISOString();
+    const recent = new Date(Date.now() - 5 * 60_000).toISOString();
+    const set = (id: string, v: object) => db.k.updateTable("source_connections").set(v).where("id", "=", id).execute();
+    await set(due, { schedule_minutes: 60, last_run_at: old });
+    await set(fresh, { schedule_minutes: 60, last_run_at: recent });
+    await set(paused, { schedule_minutes: 60, last_run_at: old, status: "paused" });
+    void manual;
+    const s = startScheduler(db, { tickMs: 3_600_000 });
+    await s.tickNow();
+    await s.tickNow();
+    s.stop();
+    const jobs = await db.k.selectFrom("jobs").selectAll().execute();
+    expect(jobs.map((j) => j.lock_key)).toEqual([`sync:${due}`]);
+    expect(JSON.parse(jobs[0]!.payload)).toEqual({ connectionId: due });
+  });
+});

packages/core/test/registry.test.ts

@@ -0,0 +1,24 @@
+import { expect, spyOn, test } from "bun:test";
+import { join } from "node:path";
+import { loadPlugins } from "../src/registry.ts";
+
+const dir = join(import.meta.dir, "fixtures/plugins");
+
+test("registry loads echo plugin and skips broken one", async () => {
+  const warn = spyOn(console, "warn").mockImplementation(() => {});
+  const reg = await loadPlugins([dir, join(dir, "does-not-exist")]);
+  expect(reg.list().map((m) => m.id)).toEqual(["echo"]);
+  expect(reg.get("echo")?.manifest.name).toBe("Echo");
+  expect(reg.get("broken")).toBeUndefined();
+  expect(warn.mock.calls.some((c) => String(c[0]).includes("broken"))).toBe(true);
+  await reg.reload();
+  expect(reg.list()).toHaveLength(1);
+  warn.mockRestore();
+});
+
+test("duplicate ids across dirs keep the first", async () => {
+  const warn = spyOn(console, "warn").mockImplementation(() => {});
+  const reg = await loadPlugins([dir, dir]);
+  expect(reg.list()).toHaveLength(1);
+  warn.mockRestore();
+});