import { expect, spyOn, test } from "bun:test"; import { forEachDialect } from "@sttracker/db/testing"; import { DropJobError, enqueue, NonRetryableError, startScheduler, startWorker } from "../src/jobs.ts"; import { makeConnection, makeUser } from "./sync/helpers.ts"; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); forEachDialect("jobs", (getDb) => { test("enqueue with same lockKey returns null", async () => { const db = getDb(); expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeString(); expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeNull(); expect(await enqueue(db, "sync", {}, { lockKey: "sync:b" })).toBeString(); expect(await enqueue(db, "x", {})).toBeString(); expect(await enqueue(db, "x", {})).toBeString(); }); test("concurrent enqueue with one lockKey admits exactly one", async () => { const db = getDb(); const ids = await Promise.all(Array.from({ length: 6 }, () => enqueue(db, "sync", {}, { lockKey: "k" }))); expect(ids.filter((i) => i !== null)).toHaveLength(1); }); test("worker runs a job and releases the lock", async () => { const db = getDb(); const seen: unknown[] = []; const id = await enqueue(db, "t", { n: 1 }, { lockKey: "l" }); const w = startWorker(db, { t: async (p) => void seen.push(p) }, { pollMs: 10 }); await sleep(150); await w.stop(); expect(seen).toEqual([{ n: 1 }]); expect((await db.k.selectFrom("jobs").selectAll().where("id", "=", id!).executeTakeFirstOrThrow()).status).toBe("done"); expect(await enqueue(db, "t", {}, { lockKey: "l" })).toBeString(); }); test("a job is claimed by only one of two workers", async () => { const db = getDb(); let runs = 0; const h = { t: async () => { runs++; await sleep(30); } }; await enqueue(db, "t", {}); const a = startWorker(db, h, { pollMs: 5 }); const b = startWorker(db, h, { pollMs: 5 }); await sleep(200); await Promise.all([a.stop(), b.stop()]); expect(runs).toBe(1); }); test("worker retries then fails", async () => { const db = getDb(); let calls = 0; const id = (await enqueue(db, "bad", {}))!; const w = startWorker(db, { bad: async () => { calls++; throw new Error("nope"); } }, { pollMs: 10 }); const row = () => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow(); for (let attempt = 1; attempt <= 5; attempt++) { for (let i = 0; i < 100 && (await row()).attempts < attempt; i++) await sleep(10); const r = await row(); expect(r.attempts).toBe(attempt); expect(r.last_error).toBe("nope"); if (attempt < 5) { expect(r.status).toBe("queued"); const delay = Date.parse(r.run_at) - Date.now(); expect(delay).toBeGreaterThan(2 ** attempt * 60_000 - 5_000); expect(delay).toBeLessThanOrEqual(2 ** attempt * 60_000); await db.k.updateTable("jobs").set({ run_at: new Date(Date.now() - 1000).toISOString() }).where("id", "=", id).execute(); } } await w.stop(); expect((await row()).status).toBe("failed"); expect(calls).toBe(5); }); test("scheduler enqueues due connections only", async () => { const db = getDb(); const u = await makeUser(db); const due = await makeConnection(db, u, "p"); const fresh = await makeConnection(db, u, "p"); const manual = await makeConnection(db, u, "p"); const errored = await makeConnection(db, u, "p"); const paused = await makeConnection(db, u, "p"); const old = new Date(Date.now() - 120 * 60_000).toISOString(); const recent = new Date(Date.now() - 5 * 60_000).toISOString(); const set = (id: string, v: object) => db.k.updateTable("source_connections").set(v).where("id", "=", id).execute(); await set(due, { schedule_minutes: 60, last_run_at: old }); await set(fresh, { schedule_minutes: 60, last_run_at: recent }); await set(errored, { schedule_minutes: 60, last_run_at: old, status: "error" }); await set(paused, { schedule_minutes: 60, last_run_at: old, status: "paused" }); void manual; const s = startScheduler(db, { tickMs: 3_600_000 }); await s.tickNow(); await s.tickNow(); s.stop(); const jobs = await db.k.selectFrom("jobs").selectAll().execute(); expect(jobs.map((j) => j.lock_key).sort()).toEqual([`sync:${due}`, `sync:${errored}`].sort()); expect(JSON.parse(jobs.find((j) => j.lock_key === `sync:${due}`)!.payload)).toEqual({ connectionId: due }); }); const row = (db: ReturnType, id: string) => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow(); async function until(fn: () => Promise) { for (let i = 0; i < 200 && !(await fn()); i++) await sleep(10); } test("NonRetryableError fails at once; DropJobError completes with last_error", async () => { const db = getDb(); const a = (await enqueue(db, "a", {}))!; const b = (await enqueue(db, "b", {}))!; const w = startWorker(db, { a: async () => { throw new NonRetryableError("fatal"); }, b: async () => { throw new DropJobError("gone"); } }, { pollMs: 10 }); await until(async () => (await row(db, a)).status !== "queued" && (await row(db, b)).status !== "queued"); await w.stop(); expect(await row(db, a)).toMatchObject({ status: "failed", attempts: 1, last_error: "fatal" }); expect(await row(db, b)).toMatchObject({ status: "done", last_error: "gone" }); }); test("a stale running job is re-queued and then runs; a fresh one is left", async () => { const db = getDb(); const stale = (await enqueue(db, "t", {}, { lockKey: "s" }))!; const fresh = (await enqueue(db, "never", {}))!; await db.k.updateTable("jobs").set({ status: "running", locked_at: new Date(Date.now() - 2 * 3600_000).toISOString(), attempts: 2 }).where("id", "=", stale).execute(); await db.k.updateTable("jobs").set({ status: "running", locked_at: nowStr() }).where("id", "=", fresh).execute(); let ran = 0; const w = startWorker(db, { t: async () => void ran++ }, { pollMs: 10 }); await until(async () => (await row(db, stale)).status === "done"); await w.stop(); expect(ran).toBe(1); expect((await row(db, stale)).attempts).toBe(2); expect((await row(db, fresh)).status).toBe("running"); }); test("failing bookkeeping is logged and stop() does not reject", async () => { const db = getDb(); const err = spyOn(console, "error").mockImplementation(() => {}); await enqueue(db, "t", {}); let updates = 0; const k = new Proxy(db.k, { get(t, p) { const v = Reflect.get(t, p, t); if (p === "updateTable") return (...a: unknown[]) => { if (++updates > 1) throw new Error("db down"); return (v as Function).apply(t, a); }; return typeof v === "function" ? v.bind(t) : v; }, }); let called = false; const w = startWorker({ ...db, k } as typeof db, { t: async () => void (called = true) }, { pollMs: 10 }); await until(async () => called && err.mock.calls.length > 0); await expect(w.stop()).resolves.toBeUndefined(); expect(err.mock.calls.some((c) => c[0] === "[jobs]")).toBe(true); err.mockRestore(); }); test("a job running past staleMs is not reaped by its own worker and runs once", async () => { const db = getDb(); const id = (await enqueue(db, "t", {}, { lockKey: "slow" }))!; let runs = 0; const w = startWorker(db, { t: async () => { runs++; await sleep(300); } }, { pollMs: 10, staleMs: 50, heartbeatMs: 20 }); await until(async () => (await row(db, id)).status === "done"); await w.stop(); expect(runs).toBe(1); }); test("heartbeat refreshes locked_at", async () => { const db = getDb(); const id = (await enqueue(db, "t", {}))!; let release!: () => void; const gate = new Promise((r) => (release = r)); const w = startWorker(db, { t: () => gate }, { pollMs: 10, heartbeatMs: 20 }); await until(async () => (await row(db, id)).status === "running"); const first = (await row(db, id)).locked_at!; await until(async () => (await row(db, id)).locked_at !== first); expect((await row(db, id)).locked_at).not.toBe(first); release(); await w.stop(); }); test("stop() re-queues an aborted job without burning an attempt or backoff", async () => { const db = getDb(); const id = (await enqueue(db, "t", {}))!; const before = (await row(db, id)).run_at; let started = false; const w = startWorker(db, { t: (_p, { signal }) => new Promise((_, rej) => { started = true; signal.addEventListener("abort", () => rej(new Error("aborted"))); }) }, { pollMs: 10 }); await until(async () => started); await w.stop(); expect(await row(db, id)).toMatchObject({ status: "queued", attempts: 0, run_at: before, locked_at: null }); }); test("stop() aborts the signal handed to handlers", async () => { const db = getDb(); await enqueue(db, "t", {}); let aborted = false; let started = false; const w = startWorker(db, { t: (_p, { signal }) => new Promise((res) => { started = true; signal.addEventListener("abort", () => { aborted = true; res(); }); }) }, { pollMs: 10 }); await until(async () => started); await w.stop(); expect(aborted).toBe(true); }); }); const nowStr = () => new Date().toISOString();