Files
STTracker/packages/core/test/sync/run.test.ts
T
Lucas WintherandClaude Opus 5.5 1975e971f9 feat(core): sync engine and changes inbox
runSync snapshots and diffs source records, applies new records and
never-edited fields, files pending changed/conflict/removed/merged
events, de-duplicates plays across sources and ingests listens.
listChanges/resolveChange drive the Changes inbox.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 20:26:03 +01:00

410 lines
22 KiB
TypeScript

import { expect, test } from "bun:test";
import { forEachDialect } from "@sttracker/db/testing";
import { softDeleteEntry, upsertEntry } from "../../src/entries.ts";
import { listChanges, resolveChange } from "../../src/sync/changes.ts";
import { runSync } from "../../src/sync/run.ts";
import { NotFoundError, ValidationError } from "../../src/errors.ts";
import {
book,
count,
deps,
engagementRec,
entryFor,
entryRec,
events,
fakePlugin,
type FakeSource,
listenRec,
makeConnection,
makeUser,
} from "./helpers.ts";
forEachDialect("sync engine", (getDb) => {
async function setup(pluginId = "fake") {
const db = getDb();
const userId = await makeUser(db);
const connectionId = await makeConnection(db, userId, pluginId);
const src: FakeSource = { records: [] };
const plugin = fakePlugin(pluginId, src);
const actor = `plugin:${pluginId}:${connectionId}`;
return { db, userId, connectionId, src, plugin, actor };
}
test("first full pull creates entries and auto_applied events", async () => {
const { db, userId, connectionId, src, plugin, actor } = await setup();
src.records = [
entryRec("e1", book("111"), { status: "completed", rating: 80, notes: "loved it" }),
engagementRec("g1", book("111"), { finishedAt: { value: "2024-01-05", precision: "day" } }),
];
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.status).toBe("ok");
expect(s.counts).toEqual({ seen: 2, added: 2, changed: 0, removed: 0, merged: 0, conflicts: 0, autoApplied: 2, pending: 0, listens: 0 });
const e = await entryFor(db, userId, "111");
expect(e?.status).toBe("completed");
expect(e?.rating).toBe(80);
expect(e?.private_notes).toBe("loved it");
expect(e?.fieldSources).toEqual({ status: actor, rating: actor, privateNotes: actor });
const eg = await db.k.selectFrom("engagements").selectAll().execute();
expect(eg).toHaveLength(1);
expect(eg[0]!.entry_id).toBe(e!.id);
expect(eg[0]!.finished_at).toBe("2024-01-05");
expect(await count(db, "engagement_sources")).toBe(1);
const ev = await events(db);
expect(ev.map((x) => [x.type, x.status])).toEqual([["added", "auto_applied"], ["added", "auto_applied"]]);
expect(ev[0]!.run_id).toBe(s.runId);
expect(ev[0]!.target_entity).toBe("entry");
expect(ev[0]!.target_id).toBe(e!.id);
expect(ev[1]!.target_entity).toBe("engagement");
expect(await count(db, "source_records")).toBe(2);
expect(await count(db, "source_record_versions")).toBe(2);
const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "e1").executeTakeFirstOrThrow();
expect(rec.target_entity).toBe("entry");
expect(rec.target_id).toBe(e!.id);
const run = await db.k.selectFrom("source_runs").selectAll().where("id", "=", s.runId).executeTakeFirstOrThrow();
expect(run.status).toBe("ok");
expect(run.mode).toBe("full");
expect(run.finished_at).not.toBeNull();
expect(JSON.parse(run.counts)).toEqual(s.counts);
const conn = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
expect(conn.last_run_at).not.toBeNull();
expect(conn.status).toBe("active");
});
test("an entry on a child work also gets a status-none entry on the root ancestor", async () => {
const { db, userId, connectionId, src, plugin, actor } = await setup();
const show = { title: "Show", mediaType: "show" as const, externalIds: { tmdb_show: "9" } };
const season = { title: "Season 1", mediaType: "season" as const, externalIds: {}, parent: show, position: { season: 1 } };
const episode = { title: "Pilot", mediaType: "episode" as const, externalIds: { tmdb_episode: "91" }, parent: season, position: { episode: 1 } };
src.records = [engagementRec("ep1", episode, { finishedAt: { value: "2024-02-01T21:00:00Z", precision: "day" } })];
await runSync(deps(db), plugin, connectionId, "full");
const rows = await db.k
.selectFrom("entries as e")
.innerJoin("works as w", "w.id", "e.work_id")
.select(["w.media_type", "e.status", "e.field_sources"])
.where("e.user_id", "=", userId)
.orderBy("w.media_type")
.execute();
expect(rows.map((r) => [r.media_type, r.status])).toEqual([["episode", "none"], ["show", "none"]]);
const hist = await db.k.selectFrom("user_history").select("actor").execute();
expect(new Set(hist.map((h) => h.actor))).toEqual(new Set([actor]));
// An existing (even deleted) root entry is never touched.
const showEntry = await db.k
.selectFrom("entries as e")
.innerJoin("works as w", "w.id", "e.work_id")
.select("e.id")
.where("w.media_type", "=", "show")
.executeTakeFirstOrThrow();
await softDeleteEntry(db, userId, showEntry.id, "user");
src.records.push(engagementRec("ep2", { ...episode, title: "Two", externalIds: { tmdb_episode: "92" }, position: { episode: 2 } }, {}));
await runSync(deps(db), plugin, connectionId, "full");
const show2 = await db.k.selectFrom("entries").selectAll().where("id", "=", showEntry.id).executeTakeFirstOrThrow();
expect(show2.deleted_at).not.toBeNull();
});
test("second identical pull creates nothing new", async () => {
const { db, connectionId, src, plugin } = await setup();
src.records = [
entryRec("e1", book("111"), { status: "completed", rating: 80 }),
engagementRec("g1", book("111"), { finishedAt: { value: "2024-01-05", precision: "day" } }),
listenRec("l1", { listenedAt: "2024-01-01T10:00:00Z", artist: "A", track: "T" }),
];
const first = await runSync(deps(db), plugin, connectionId, "full");
expect(first.counts.listens).toBe(1);
const tables = ["source_records", "source_record_versions", "change_events", "entries", "engagements", "engagement_sources", "listens", "user_history", "works"] as const;
const before = await Promise.all(tables.map((t) => count(db, t)));
const second = await runSync(deps(db), plugin, connectionId, "full");
const after = await Promise.all(tables.map((t) => count(db, t)));
expect(after).toEqual(before);
expect(second.status).toBe("ok");
expect(second.counts).toEqual({ seen: 3, added: 0, changed: 0, removed: 0, merged: 0, conflicts: 0, autoApplied: 0, pending: 0, listens: 0 });
expect(await count(db, "source_runs")).toBe(2);
});
test("failed pull marks nothing removed", async () => {
const { db, connectionId, src, plugin } = await setup();
src.records = [entryRec("a", book("1"), { status: "completed" }), entryRec("b", book("2"), { status: "planned" })];
await runSync(deps(db), plugin, connectionId, "full");
src.failAfter = 1;
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.status).toBe("failed");
expect(s.error).toContain("source exploded");
expect(s.counts.removed).toBe(0);
const deleted = await db.k.selectFrom("source_records").select("key").where("deleted_at", "is not", null).execute();
expect(deleted).toEqual([]);
expect(await events(db, { type: "removed" })).toEqual([]);
const run = await db.k.selectFrom("source_runs").selectAll().where("id", "=", s.runId).executeTakeFirstOrThrow();
expect(run.status).toBe("failed");
expect(run.error).toContain("source exploded");
const conn = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
expect(conn.status).toBe("error");
// A later successful run puts the connection back to active.
src.failAfter = undefined;
expect((await runSync(deps(db), plugin, connectionId, "full")).status).toBe("ok");
const conn2 = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
expect(conn2.status).toBe("active");
});
test("record missing from completed full pull → pending removed; accept soft-deletes entry", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [entryRec("a", book("1"), { status: "completed" }), entryRec("b", book("2"), { status: "planned" })];
await runSync(deps(db), plugin, connectionId, "full");
src.records = [src.records[0]!];
// A "changes" pull never infers removal from absence.
expect((await runSync(deps(db), plugin, connectionId, "changes")).counts.removed).toBe(0);
expect(await events(db, { type: "removed" })).toEqual([]);
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.counts.removed).toBe(1);
expect(s.counts.pending).toBe(1);
const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "b").executeTakeFirstOrThrow();
expect(rec.deleted_at).not.toBeNull();
const removed = await events(db, { type: "removed" });
expect(removed).toHaveLength(1);
expect(removed[0]!.status).toBe("pending");
const entryB = await entryFor(db, userId, "2");
expect(removed[0]!.target_id).toBe(entryB!.id);
expect(entryB!.deleted_at).toBeNull(); // nothing deleted without the user's say
// Rerun: not re-filed.
await runSync(deps(db), plugin, connectionId, "full");
expect(await events(db, { type: "removed" })).toHaveLength(1);
await resolveChange(db, userId, removed[0]!.id, "accept");
expect((await entryFor(db, userId, "2"))!.deleted_at).not.toBeNull();
const ev = (await events(db, { type: "removed" }))[0]!;
expect(ev.status).toBe("accepted");
expect(ev.resolved_at).not.toBeNull();
});
test("deleted: true in a changes pull files a pending removed event", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [entryRec("a", book("1"), { status: "completed" })];
await runSync(deps(db), plugin, connectionId, "full");
src.records = [entryRec("a", book("1"), { status: "completed" }, { deleted: true })];
const s = await runSync(deps(db), plugin, connectionId, "changes");
expect(s.counts.removed).toBe(1);
const removed = await events(db, { type: "removed", status: "pending" });
expect(removed).toHaveLength(1);
await resolveChange(db, userId, removed[0]!.id, "ignore");
expect((await entryFor(db, userId, "1"))!.deleted_at).toBeNull();
expect((await events(db, { type: "removed" }))[0]!.status).toBe("ignored");
});
test("user-edited rating not overwritten; pending changed event", async () => {
const { db, userId, connectionId, src, plugin, actor } = await setup();
src.records = [entryRec("a", book("1"), { status: "in_progress", rating: 80 })];
await runSync(deps(db), plugin, connectionId, "full");
const e0 = (await entryFor(db, userId, "1"))!;
await upsertEntry(db, userId, e0.work_id, { rating: 90 }, "user");
src.records = [entryRec("a", book("1"), { status: "completed", rating: 70 })];
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.counts.changed).toBe(1);
expect(s.counts.autoApplied).toBe(1);
expect(s.counts.pending).toBe(1);
const e1 = (await entryFor(db, userId, "1"))!;
expect(e1.rating).toBe(90);
expect(e1.status).toBe("completed"); // never edited by the user: applied
expect(e1.fieldSources).toEqual({ status: actor, rating: "user" });
const pending = await events(db, { type: "changed", status: "pending" });
expect(pending).toHaveLength(1);
expect(pending[0]!.payload).toMatchObject({ field: "rating", from: 90, to: 70 });
const applied = await events(db, { type: "changed", status: "auto_applied" });
expect(applied.map((x) => x.payload)).toEqual([expect.objectContaining({ field: "status", from: "in_progress", to: "completed" })]);
// Rerun: nothing new.
await runSync(deps(db), plugin, connectionId, "full");
expect(await events(db, { type: "changed", status: "pending" })).toHaveLength(1);
const listed = await listChanges(db, userId, {});
expect(listed.items.map((c) => c.id)).toEqual([pending[0]!.id]);
await resolveChange(db, userId, pending[0]!.id, "accept");
const e2 = (await entryFor(db, userId, "1"))!;
expect(e2.rating).toBe(70);
expect(e2.fieldSources.rating).toBe(actor);
expect((await listChanges(db, userId, {})).items).toEqual([]);
await expect(resolveChange(db, userId, pending[0]!.id, "accept")).rejects.toBeInstanceOf(ValidationError);
});
test("a user-deleted entry is never revived by a sync", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [entryRec("a", book("1"), { rating: 80 })];
await runSync(deps(db), plugin, connectionId, "full");
const e = (await entryFor(db, userId, "1"))!;
await softDeleteEntry(db, userId, e.id, "user");
const before = await count(db, "change_events");
// A changed record, a brand-new entry record and a new engagement record, all on the deleted entry's work.
src.records = [
entryRec("a", book("1"), { rating: 50 }),
entryRec("a2", book("1"), { status: "completed" }),
engagementRec("g", book("1"), { finishedAt: { value: "2024-01-01", precision: "day" } }),
];
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.status).toBe("ok");
const after = (await entryFor(db, userId, "1"))!;
expect(after.deleted_at).not.toBeNull();
expect(after.rating).toBe(80);
expect(after.status).toBe("none");
expect(await count(db, "engagements")).toBe(0);
expect(await count(db, "change_events")).toBe(before);
});
test("an id-less record is updated through its mapping, not re-matched", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
const hint = { title: "Obscure Zine", mediaType: "book" as const, externalIds: {} };
src.records = [entryRec("csv-1", hint, { rating: 40 })];
await runSync(deps(db), plugin, connectionId, "full");
const works = await count(db, "works");
src.records = [entryRec("csv-1", hint, { rating: 60 })];
await runSync(deps(db), plugin, connectionId, "full");
expect(await count(db, "works")).toBe(works);
const rows = await db.k.selectFrom("entries").selectAll().where("user_id", "=", userId).execute();
expect(rows).toHaveLength(1);
expect(rows[0]!.rating).toBe(60);
});
test("two connections disagree → one pending conflict, not duplicated on rerun; keep_mine stops further conflicts for that value", async () => {
const { db, userId, connectionId: connA, src: srcA, plugin: pluginA, actor: actorA } = await setup("alpha");
const connB = await makeConnection(db, userId, "beta");
const srcB: FakeSource = { records: [] };
const pluginB = fakePlugin("beta", srcB);
const actorB = `plugin:beta:${connB}`;
srcA.records = [entryRec("a", book("1"), { rating: 80 })];
await runSync(deps(db), pluginA, connA, "full");
srcB.records = [entryRec("b", book("1"), { rating: 60, status: "completed" })];
const s = await runSync(deps(db), pluginB, connB, "full");
expect(s.counts.conflicts).toBe(1);
const e = (await entryFor(db, userId, "1"))!;
expect(e.rating).toBe(80);
expect(e.status).toBe("completed"); // field nobody had set: applied
expect(e.fieldSources).toEqual({ rating: actorA, status: actorB });
let conflicts = await events(db, { type: "conflict", status: "pending" });
expect(conflicts).toHaveLength(1);
expect(conflicts[0]!.payload).toMatchObject({ field: "rating", current: 80, currentSource: actorA, incoming: 60, incomingSource: actorB });
// Rerun both: nothing new. Then B flips 60 → 70 → 60: the conflict for 60 is not filed twice.
await runSync(deps(db), pluginA, connA, "full");
await runSync(deps(db), pluginB, connB, "full");
expect(await events(db, { type: "conflict" })).toHaveLength(1);
srcB.records = [entryRec("b", book("1"), { rating: 70, status: "completed" })];
await runSync(deps(db), pluginB, connB, "full");
srcB.records = [entryRec("b", book("1"), { rating: 60, status: "completed" })];
await runSync(deps(db), pluginB, connB, "full");
conflicts = await events(db, { type: "conflict", status: "pending" });
expect(conflicts.map((c) => c.payload.incoming)).toEqual([60, 70]);
await resolveChange(db, userId, conflicts[0]!.id, "keep_mine");
const kept = (await entryFor(db, userId, "1"))!;
expect(kept.rating).toBe(80);
expect(kept.fieldSources.rating).toBe("user");
expect((await events(db, { type: "conflict" }))[0]!.status).toBe("kept_mine");
await runSync(deps(db), pluginA, connA, "full");
await runSync(deps(db), pluginB, connB, "full");
expect(await events(db, { type: "conflict" })).toHaveLength(2);
expect((await entryFor(db, userId, "1"))!.rating).toBe(80);
});
test("same play from two sources dedupes into one engagement with two engagement_sources", async () => {
const { db, userId, connectionId: connA, src: srcA, plugin: pluginA } = await setup("alpha");
const connB = await makeConnection(db, userId, "beta");
const srcB: FakeSource = { records: [] };
const pluginB = fakePlugin("beta", srcB);
srcA.records = [engagementRec("play-1", book("1"), { finishedAt: { value: "2024-03-01T20:00:00Z", precision: "day" } })];
await runSync(deps(db), pluginA, connA, "full");
srcB.records = [
engagementRec("row-7", book("1"), { finishedAt: { value: "2024-03-02", precision: "day" }, rating: 90 }),
engagementRec("row-8", book("1"), { finishedAt: { value: "2024-03-10", precision: "day" } }),
];
await runSync(deps(db), pluginB, connB, "full");
const egs = await db.k.selectFrom("engagements").selectAll().orderBy("finished_at").execute();
expect(egs).toHaveLength(2);
expect(egs[0]!.finished_at).toBe("2024-03-01T20:00:00Z"); // first source's value kept
expect(egs[0]!.rating).toBe(90); // empty field filled from the second source
const srcs = await db.k.selectFrom("engagement_sources").selectAll().where("engagement_id", "=", egs[0]!.id).execute();
expect(srcs.map((x) => x.record_key).sort()).toEqual(["play-1", "row-7"]);
expect(await count(db, "engagement_sources")).toBe(3);
// Equal start datetime also dedupes.
srcA.records.push(engagementRec("play-2", book("1"), { startedAt: { value: "2024-05-01T08:00:00Z", precision: "day" } }));
srcB.records.push(engagementRec("row-9", book("1"), { startedAt: { value: "2024-05-01T08:00:00Z", precision: "day" } }));
await runSync(deps(db), pluginA, connA, "full");
await runSync(deps(db), pluginB, connB, "full");
expect(await count(db, "engagements")).toBe(3);
expect(await count(db, "engagement_sources")).toBe(5);
});
test("merged record → pending merged; accept moves entry", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [entryRec("x", book("1"), { status: "completed" })];
await runSync(deps(db), plugin, connectionId, "full");
const ex = (await entryFor(db, userId, "1"))!;
// The source merged x into y. Record y is new and creates its own (empty) entry on book 2.
src.records = [entryRec("x", book("1"), { status: "completed" }, { mergedIntoKey: "y" }), entryRec("y", book("2"), {})];
const s = await runSync(deps(db), plugin, connectionId, "full");
expect(s.counts.merged).toBe(1);
const merged = await events(db, { type: "merged", status: "pending" });
expect(merged).toHaveLength(1);
expect(merged[0]!.target_id).toBe(ex.id);
expect(merged[0]!.payload).toMatchObject({ intoKey: "y" });
expect((await entryFor(db, userId, "1"))!.deleted_at).toBeNull();
await resolveChange(db, userId, merged[0]!.id, "accept");
const ey = (await entryFor(db, userId, "2"))!;
expect(ey.deleted_at).toBeNull();
expect(ey.status).toBe("completed"); // filled from the moved entry
expect((await entryFor(db, userId, "1"))!.deleted_at).not.toBeNull();
expect((await events(db, { type: "merged" }))[0]!.status).toBe("accepted");
});
test("resolveChange is scoped to the user", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [entryRec("a", book("1"), { status: "completed" })];
await runSync(deps(db), plugin, connectionId, "full");
src.records = [];
await runSync(deps(db), plugin, connectionId, "full");
const [ev] = await events(db, { type: "removed" });
const other = await makeUser(db, "bob");
await expect(resolveChange(db, other, ev!.id, "accept")).rejects.toBeInstanceOf(NotFoundError);
expect((await listChanges(db, other, { status: "all" })).items).toEqual([]);
expect((await listChanges(db, userId, { status: "all" })).items).toHaveLength(2);
});
test("listChanges pages newest first and filters by connection", async () => {
const { db, userId, connectionId, src, plugin } = await setup();
src.records = [1, 2, 3].map((i) => entryRec(`k${i}`, book(String(i)), { status: "planned" }));
await runSync(deps(db), plugin, connectionId, "full");
const p1 = await listChanges(db, userId, { status: "all", limit: 2 });
expect(p1.items).toHaveLength(2);
expect(p1.nextCursor).not.toBeNull();
const p2 = await listChanges(db, userId, { status: "all", limit: 2, cursor: p1.nextCursor! });
expect(p2.items).toHaveLength(1);
expect(p2.nextCursor).toBeNull();
const ids = [...p1.items, ...p2.items].map((c) => c.id);
expect(ids).toEqual([...ids].sort().reverse());
expect(p1.items[0]!.payload).toMatchObject({ key: "k3" });
expect((await listChanges(db, userId, { status: "all", connectionId: "nope" })).items).toEqual([]);
});
test("a plugin that does not match the connection is refused", async () => {
const { db, connectionId } = await setup("alpha");
await expect(runSync(deps(db), fakePlugin("beta", { records: [] }), connectionId, "full")).rejects.toBeInstanceOf(ValidationError);
});
});