196 lines
9.2 KiB
TypeScript
196 lines
9.2 KiB
TypeScript
import { expect, spyOn, test } from "bun:test";
|
|
import { forEachDialect } from "@sttracker/db/testing";
|
|
import { DropJobError, enqueue, NonRetryableError, startScheduler, startWorker } from "../src/jobs.ts";
|
|
import { makeConnection, makeUser } from "./sync/helpers.ts";
|
|
|
|
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
forEachDialect("jobs", (getDb) => {
|
|
test("enqueue with same lockKey returns null", async () => {
|
|
const db = getDb();
|
|
expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeString();
|
|
expect(await enqueue(db, "sync", {}, { lockKey: "sync:a" })).toBeNull();
|
|
expect(await enqueue(db, "sync", {}, { lockKey: "sync:b" })).toBeString();
|
|
expect(await enqueue(db, "x", {})).toBeString();
|
|
expect(await enqueue(db, "x", {})).toBeString();
|
|
});
|
|
|
|
test("concurrent enqueue with one lockKey admits exactly one", async () => {
|
|
const db = getDb();
|
|
const ids = await Promise.all(Array.from({ length: 6 }, () => enqueue(db, "sync", {}, { lockKey: "k" })));
|
|
expect(ids.filter((i) => i !== null)).toHaveLength(1);
|
|
});
|
|
|
|
test("worker runs a job and releases the lock", async () => {
|
|
const db = getDb();
|
|
const seen: unknown[] = [];
|
|
const id = await enqueue(db, "t", { n: 1 }, { lockKey: "l" });
|
|
const w = startWorker(db, { t: async (p) => void seen.push(p) }, { pollMs: 10 });
|
|
await sleep(150);
|
|
await w.stop();
|
|
expect(seen).toEqual([{ n: 1 }]);
|
|
expect((await db.k.selectFrom("jobs").selectAll().where("id", "=", id!).executeTakeFirstOrThrow()).status).toBe("done");
|
|
expect(await enqueue(db, "t", {}, { lockKey: "l" })).toBeString();
|
|
});
|
|
|
|
test("a job is claimed by only one of two workers", async () => {
|
|
const db = getDb();
|
|
let runs = 0;
|
|
const h = { t: async () => { runs++; await sleep(30); } };
|
|
await enqueue(db, "t", {});
|
|
const a = startWorker(db, h, { pollMs: 5 });
|
|
const b = startWorker(db, h, { pollMs: 5 });
|
|
await sleep(200);
|
|
await Promise.all([a.stop(), b.stop()]);
|
|
expect(runs).toBe(1);
|
|
});
|
|
|
|
test("worker retries then fails", async () => {
|
|
const db = getDb();
|
|
let calls = 0;
|
|
const id = (await enqueue(db, "bad", {}))!;
|
|
const w = startWorker(db, { bad: async () => { calls++; throw new Error("nope"); } }, { pollMs: 10 });
|
|
const row = () => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
|
|
for (let attempt = 1; attempt <= 5; attempt++) {
|
|
for (let i = 0; i < 100 && (await row()).attempts < attempt; i++) await sleep(10);
|
|
const r = await row();
|
|
expect(r.attempts).toBe(attempt);
|
|
expect(r.last_error).toBe("nope");
|
|
if (attempt < 5) {
|
|
expect(r.status).toBe("queued");
|
|
const delay = Date.parse(r.run_at) - Date.now();
|
|
expect(delay).toBeGreaterThan(2 ** attempt * 60_000 - 5_000);
|
|
expect(delay).toBeLessThanOrEqual(2 ** attempt * 60_000);
|
|
await db.k.updateTable("jobs").set({ run_at: new Date(Date.now() - 1000).toISOString() }).where("id", "=", id).execute();
|
|
}
|
|
}
|
|
await w.stop();
|
|
expect((await row()).status).toBe("failed");
|
|
expect(calls).toBe(5);
|
|
});
|
|
|
|
test("scheduler enqueues due connections only", async () => {
|
|
const db = getDb();
|
|
const u = await makeUser(db);
|
|
const due = await makeConnection(db, u, "p");
|
|
const fresh = await makeConnection(db, u, "p");
|
|
const manual = await makeConnection(db, u, "p");
|
|
const errored = await makeConnection(db, u, "p");
|
|
const paused = await makeConnection(db, u, "p");
|
|
const old = new Date(Date.now() - 120 * 60_000).toISOString();
|
|
const recent = new Date(Date.now() - 5 * 60_000).toISOString();
|
|
const set = (id: string, v: object) => db.k.updateTable("source_connections").set(v).where("id", "=", id).execute();
|
|
await set(due, { schedule_minutes: 60, last_run_at: old });
|
|
await set(fresh, { schedule_minutes: 60, last_run_at: recent });
|
|
await set(errored, { schedule_minutes: 60, last_run_at: old, status: "error" });
|
|
await set(paused, { schedule_minutes: 60, last_run_at: old, status: "paused" });
|
|
void manual;
|
|
const s = startScheduler(db, { tickMs: 3_600_000 });
|
|
await s.tickNow();
|
|
await s.tickNow();
|
|
s.stop();
|
|
const jobs = await db.k.selectFrom("jobs").selectAll().execute();
|
|
expect(jobs.map((j) => j.lock_key).sort()).toEqual([`sync:${due}`, `sync:${errored}`].sort());
|
|
expect(JSON.parse(jobs.find((j) => j.lock_key === `sync:${due}`)!.payload)).toEqual({ connectionId: due });
|
|
});
|
|
|
|
const row = (db: ReturnType<typeof getDb>, id: string) => db.k.selectFrom("jobs").selectAll().where("id", "=", id).executeTakeFirstOrThrow();
|
|
async function until(fn: () => Promise<boolean>) {
|
|
for (let i = 0; i < 200 && !(await fn()); i++) await sleep(10);
|
|
}
|
|
|
|
test("NonRetryableError fails at once; DropJobError completes with last_error", async () => {
|
|
const db = getDb();
|
|
const a = (await enqueue(db, "a", {}))!;
|
|
const b = (await enqueue(db, "b", {}))!;
|
|
const w = startWorker(db, { a: async () => { throw new NonRetryableError("fatal"); }, b: async () => { throw new DropJobError("gone"); } }, { pollMs: 10 });
|
|
await until(async () => (await row(db, a)).status !== "queued" && (await row(db, b)).status !== "queued");
|
|
await w.stop();
|
|
expect(await row(db, a)).toMatchObject({ status: "failed", attempts: 1, last_error: "fatal" });
|
|
expect(await row(db, b)).toMatchObject({ status: "done", last_error: "gone" });
|
|
});
|
|
|
|
test("a stale running job is re-queued and then runs; a fresh one is left", async () => {
|
|
const db = getDb();
|
|
const stale = (await enqueue(db, "t", {}, { lockKey: "s" }))!;
|
|
const fresh = (await enqueue(db, "never", {}))!;
|
|
await db.k.updateTable("jobs").set({ status: "running", locked_at: new Date(Date.now() - 2 * 3600_000).toISOString(), attempts: 2 }).where("id", "=", stale).execute();
|
|
await db.k.updateTable("jobs").set({ status: "running", locked_at: nowStr() }).where("id", "=", fresh).execute();
|
|
let ran = 0;
|
|
const w = startWorker(db, { t: async () => void ran++ }, { pollMs: 10 });
|
|
await until(async () => (await row(db, stale)).status === "done");
|
|
await w.stop();
|
|
expect(ran).toBe(1);
|
|
expect((await row(db, stale)).attempts).toBe(2);
|
|
expect((await row(db, fresh)).status).toBe("running");
|
|
});
|
|
|
|
test("failing bookkeeping is logged and stop() does not reject", async () => {
|
|
const db = getDb();
|
|
const err = spyOn(console, "error").mockImplementation(() => {});
|
|
await enqueue(db, "t", {});
|
|
let updates = 0;
|
|
const k = new Proxy(db.k, {
|
|
get(t, p) {
|
|
const v = Reflect.get(t, p, t);
|
|
if (p === "updateTable") return (...a: unknown[]) => { if (++updates > 1) throw new Error("db down"); return (v as Function).apply(t, a); };
|
|
return typeof v === "function" ? v.bind(t) : v;
|
|
},
|
|
});
|
|
let called = false;
|
|
const w = startWorker({ ...db, k } as typeof db, { t: async () => void (called = true) }, { pollMs: 10 });
|
|
await until(async () => called && err.mock.calls.length > 0);
|
|
await expect(w.stop()).resolves.toBeUndefined();
|
|
expect(err.mock.calls.some((c) => c[0] === "[jobs]")).toBe(true);
|
|
err.mockRestore();
|
|
});
|
|
|
|
test("a job running past staleMs is not reaped by its own worker and runs once", async () => {
|
|
const db = getDb();
|
|
const id = (await enqueue(db, "t", {}, { lockKey: "slow" }))!;
|
|
let runs = 0;
|
|
const w = startWorker(db, { t: async () => { runs++; await sleep(300); } }, { pollMs: 10, staleMs: 50, heartbeatMs: 20 });
|
|
await until(async () => (await row(db, id)).status === "done");
|
|
await w.stop();
|
|
expect(runs).toBe(1);
|
|
});
|
|
|
|
test("heartbeat refreshes locked_at", async () => {
|
|
const db = getDb();
|
|
const id = (await enqueue(db, "t", {}))!;
|
|
let release!: () => void;
|
|
const gate = new Promise<void>((r) => (release = r));
|
|
const w = startWorker(db, { t: () => gate }, { pollMs: 10, heartbeatMs: 20 });
|
|
await until(async () => (await row(db, id)).status === "running");
|
|
const first = (await row(db, id)).locked_at!;
|
|
await until(async () => (await row(db, id)).locked_at !== first);
|
|
expect((await row(db, id)).locked_at).not.toBe(first);
|
|
release();
|
|
await w.stop();
|
|
});
|
|
|
|
test("stop() re-queues an aborted job without burning an attempt or backoff", async () => {
|
|
const db = getDb();
|
|
const id = (await enqueue(db, "t", {}))!;
|
|
const before = (await row(db, id)).run_at;
|
|
let started = false;
|
|
const w = startWorker(db, { t: (_p, { signal }) => new Promise<void>((_, rej) => { started = true; signal.addEventListener("abort", () => rej(new Error("aborted"))); }) }, { pollMs: 10 });
|
|
await until(async () => started);
|
|
await w.stop();
|
|
expect(await row(db, id)).toMatchObject({ status: "queued", attempts: 0, run_at: before, locked_at: null });
|
|
});
|
|
|
|
test("stop() aborts the signal handed to handlers", async () => {
|
|
const db = getDb();
|
|
await enqueue(db, "t", {});
|
|
let aborted = false;
|
|
let started = false;
|
|
const w = startWorker(db, { t: (_p, { signal }) => new Promise<void>((res) => { started = true; signal.addEventListener("abort", () => { aborted = true; res(); }); }) }, { pollMs: 10 });
|
|
await until(async () => started);
|
|
await w.stop();
|
|
expect(aborted).toBe(true);
|
|
});
|
|
});
|
|
|
|
const nowStr = () => new Date().toISOString();
|