fix(core): job failure semantics, stale-lock reaper, run-scoped context, reload

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

packages/core/test/connections.test.ts

@@ -2,6 +2,7 @@ import { expect, spyOn, test } from "bun:test";
 import { join } from "node:path";
 import { forEachDialect } from "@sttracker/db/testing";
 import { decryptJson } from "../src/crypto.ts";
+import { DropJobError, NonRetryableError } from "../src/jobs.ts";
 import { createConnection, deleteConnection, listConnections, makePluginContext, syncHandler, updateConnection } from "../src/connections.ts";
 import { loadPlugins } from "../src/registry.ts";
 import { makeUser } from "./sync/helpers.ts";
@@ -95,4 +96,69 @@ forEachDialect("connections", (getDb) => {
     await handler({ connectionId: id });
     expect((await modes()).at(-1)).toBe("changes:ok");
   });
+
+  const setup = async (config: Record<string, unknown>) => {
+    const db = getDb();
+    const u = await makeUser(db);
+    const id = await createConnection(db, SECRET, u, { pluginId: "echo", name: "E", config }, echo.manifest);
+    return { db, id };
+  };
+  const failedRuns = (id: string) => getDb().k.selectFrom("source_runs").selectAll().where("connection_id", "=", id).execute();
+
+  test("missing plugin: failed run, connection error, no retry", async () => {
+    const { db, id } = await setup({});
+    await db.k.updateTable("source_connections").set({ plugin_id: "gone" }).where("id", "=", id).execute();
+    await expect(syncHandler({ db, secret: SECRET, registry: reg })({ connectionId: id })).rejects.toBeInstanceOf(NonRetryableError);
+    const runs = await failedRuns(id);
+    expect(runs).toHaveLength(1);
+    expect(runs[0]).toMatchObject({ status: "failed", mode: "full", error: "Plugin 'gone' is not installed or failed to load." });
+    const c = await raw(id);
+    expect(c.status).toBe("error");
+    expect(c.last_run_at).not.toBeNull();
+  });
+
+  test("undecryptable credentials: failed run, connection error, no retry", async () => {
+    const { db, id } = await setup({ token: "t" });
+    await expect(syncHandler({ db, secret: "other-secret", registry: reg })({ connectionId: id })).rejects.toBeInstanceOf(NonRetryableError);
+    const runs = await failedRuns(id);
+    expect(runs[0]!.error).toBe("Cannot decrypt this source's credentials. Was STTRACKER_SECRET changed? Reconnect the source to fix it.");
+    expect((await raw(id)).status).toBe("error");
+  });
+
+  test("missing or paused connection drops the job", async () => {
+    const { db, id } = await setup({});
+    const h = syncHandler({ db, secret: SECRET, registry: reg });
+    await expect(h({ connectionId: newId() })).rejects.toBeInstanceOf(DropJobError);
+    await db.k.updateTable("source_connections").set({ status: "paused" }).where("id", "=", id).execute();
+    await expect(h({ connectionId: id })).rejects.toBeInstanceOf(DropJobError);
+    expect(await failedRuns(id)).toHaveLength(0);
+  });
+
+  test("ctx.log lands in the run's log", async () => {
+    const { db, id } = await setup({ label: "ok" });
+    const res = await syncHandler({ db, secret: SECRET, registry: reg })({ connectionId: id });
+    const run = await db.k.selectFrom("source_runs").selectAll().where("id", "=", res!.runId).executeTakeFirstOrThrow();
+    expect(run.log).toContain("echo pulled");
+  });
+
+  test("concurrent credentials.save calls keep every key", async () => {
+    const { db, id } = await setup({ token: "t" });
+    const conn = await asConn(id);
+    const ctx = await makePluginContext(db, SECRET, conn, "r", echo.manifest);
+    await Promise.all([ctx.credentials.save({ a: "1" }), ctx.credentials.save({ b: "2" })]);
+    expect(await decryptJson(SECRET, (await raw(id)).credentials_enc!)).toEqual({ token: "t", a: "1", b: "2" });
+  });
+
+  test("ctx.http is limited by manifest.rateLimit", async () => {
+    const { db, id } = await setup({});
+    let t = 0;
+    const sleeps: number[] = [];
+    const manifest = { ...echo.manifest, rateLimit: { perMinute: 60, burst: 1 } };
+    const ctx = await makePluginContext(db, SECRET, await asConn(id), "r", manifest, {
+      httpOptions: { fetchImpl: (async () => Response.json({})) as unknown as typeof fetch, now: () => t, sleep: async (ms) => { sleeps.push(ms); t += ms; } },
+    });
+    await ctx.http.json("https://example.test/a");
+    await ctx.http.json("https://example.test/b");
+    expect(sleeps).toEqual([1000]);
+  });
 });

packages/core/test/fixtures/plugins/echo/index.ts

@@ -20,6 +20,7 @@ export default definePlugin({
       await ctx.credentials.save({ refresh: "new-refresh" });
       throw new Error("boom after save");
     }
+    ctx.log("echo pulled");
     const rec: SourceRecord = {
       kind: "entry",
       key: "echo-1",

packages/core/test/jobs.test.ts

@@ -1,6 +1,6 @@
-import { expect, test } from "bun:test";
+import { expect, spyOn, test } from "bun:test";
 import { forEachDialect } from "@sttracker/db/testing";
-import { enqueue, startScheduler, startWorker } from "../src/jobs.ts";
+import { DropJobError, enqueue, NonRetryableError, startScheduler, startWorker } from "../src/jobs.ts";
 import { makeConnection, makeUser } from "./sync/helpers.ts";
 
 const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
@@ -75,12 +75,14 @@ forEachDialect("jobs", (getDb) => {
     const due = await makeConnection(db, u, "p");
     const fresh = await makeConnection(db, u, "p");
     const manual = await makeConnection(db, u, "p");
+    const errored = await makeConnection(db, u, "p");
     const paused = await makeConnection(db, u, "p");
     const old = new Date(Date.now() - 120 * 60_000).toISOString();
     const recent = new Date(Date.now() - 5 * 60_000).toISOString();
     const set = (id: string, v: object) => db.k.updateTable("source_connections").set(v).where("id", "=", id).execute();
     await set(due, { schedule_minutes: 60, last_run_at: old });
     await set(fresh, { schedule_minutes: 60, last_run_at: recent });
+    await set(errored, { schedule_minutes: 60, last_run_at: old, status: "error" });
     await set(paused, { schedule_minutes: 60, last_run_at: old, status: "paused" });
     void manual;
     const s = startScheduler(db, { tickMs: 3_600_000 });
@@ -88,7 +90,71 @@ forEachDialect("jobs", (getDb) => {
     await s.tickNow();
     s.stop();
     const jobs = await db.k.selectFrom("jobs").selectAll().execute();
-    expect(jobs.map((j) => j.lock_key)).toEqual([`sync:${due}`]);
-    expect(JSON.parse(jobs[0]!.payload)).toEqual({ connectionId: due });
+    expect(jobs.map((j) => j.lock_key).sort()).toEqual([`sync:${due}`, `sync:${errored}`].sort());
+    expect(JSON.parse(jobs.find((j) => j.lock_key === `sync:${due}`)!.payload)).toEqual({ connectionId: due });
+  });
+
+  const row = (db: ReturnType<typeof getDb>, id: string) => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
+  async function until(fn: () => Promise<boolean>) {
+    for (let i = 0; i < 200 && !(await fn()); i++) await sleep(10);
+  }
+
+  test("NonRetryableError fails at once; DropJobError completes with last_error", async () => {
+    const db = getDb();
+    const a = (await enqueue(db, "a", {}))!;
+    const b = (await enqueue(db, "b", {}))!;
+    const w = startWorker(db, { a: async () => { throw new NonRetryableError("fatal"); }, b: async () => { throw new DropJobError("gone"); } }, { pollMs: 10 });
+    await until(async () => (await row(db, a)).status !== "queued" && (await row(db, b)).status !== "queued");
+    await w.stop();
+    expect(await row(db, a)).toMatchObject({ status: "failed", attempts: 1, last_error: "fatal" });
+    expect(await row(db, b)).toMatchObject({ status: "done", last_error: "gone" });
+  });
+
+  test("a stale running job is re-queued and then runs; a fresh one is left", async () => {
+    const db = getDb();
+    const stale = (await enqueue(db, "t", {}, { lockKey: "s" }))!;
+    const fresh = (await enqueue(db, "never", {}))!;
+    await db.k.updateTable("jobs").set({ status: "running", locked_at: new Date(Date.now() - 2 * 3600_000).toISOString(), attempts: 2 }).where("id", "=", stale).execute();
+    await db.k.updateTable("jobs").set({ status: "running", locked_at: nowStr() }).where("id", "=", fresh).execute();
+    let ran = 0;
+    const w = startWorker(db, { t: async () => void ran++ }, { pollMs: 10 });
+    await until(async () => (await row(db, stale)).status === "done");
+    await w.stop();
+    expect(ran).toBe(1);
+    expect((await row(db, stale)).attempts).toBe(2);
+    expect((await row(db, fresh)).status).toBe("running");
+  });
+
+  test("failing bookkeeping is logged and stop() does not reject", async () => {
+    const db = getDb();
+    const err = spyOn(console, "error").mockImplementation(() => {});
+    await enqueue(db, "t", {});
+    let updates = 0;
+    const k = new Proxy(db.k, {
+      get(t, p) {
+        const v = Reflect.get(t, p, t);
+        if (p === "updateTable") return (...a: unknown[]) => { if (++updates > 1) throw new Error("db down"); return (v as Function).apply(t, a); };
+        return typeof v === "function" ? v.bind(t) : v;
+      },
+    });
+    let called = false;
+    const w = startWorker({ ...db, k } as typeof db, { t: async () => void (called = true) }, { pollMs: 10 });
+    await until(async () => called && err.mock.calls.length > 0);
+    await expect(w.stop()).resolves.toBeUndefined();
+    expect(err.mock.calls.some((c) => c[0] === "[jobs]")).toBe(true);
+    err.mockRestore();
+  });
+
+  test("stop() aborts the signal handed to handlers", async () => {
+    const db = getDb();
+    await enqueue(db, "t", {});
+    let aborted = false;
+    let started = false;
+    const w = startWorker(db, { t: (_p, { signal }) => new Promise<void>((res) => { started = true; signal.addEventListener("abort", () => { aborted = true; res(); }); }) }, { pollMs: 10 });
+    await until(async () => started);
+    await w.stop();
+    expect(aborted).toBe(true);
   });
 });
+
+const nowStr = () => new Date().toISOString();

packages/core/test/registry.test.ts

@@ -1,4 +1,6 @@
 import { expect, spyOn, test } from "bun:test";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { loadPlugins } from "../src/registry.ts";
 
@@ -22,3 +24,21 @@ test("duplicate ids across dirs keep the first", async () => {
   expect(reg.list()).toHaveLength(1);
   warn.mockRestore();
 });
+
+test("reload picks up an edited entry file", async () => {
+  const root = await mkdtemp(join(tmpdir(), "sttracker-plugins-"));
+  try {
+    await mkdir(join(root, "hot"));
+    const write = (v: string) =>
+      writeFile(join(root, "hot", "index.ts"), `export default { manifest: { id: "hot", name: "Hot", version: "${v}", sdkVersion: 1, mediaTypes: [], capabilities: [], config: [] } };\n`);
+    await write("1.0.0");
+    const reg = await loadPlugins([root]);
+    expect(reg.get("hot")?.manifest.version).toBe("1.0.0");
+    await new Promise((r) => setTimeout(r, 5));
+    await write("2.0.0");
+    await reg.reload();
+    expect(reg.get("hot")?.manifest.version).toBe("2.0.0");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});