merge: task 8 sync engine

packages/core/test/sync/helpers.ts

@@ -0,0 +1,110 @@
+import type { Db } from "@sttracker/db/db";
+import { newId, nowIso } from "@sttracker/db/ids";
+import type { Database } from "@sttracker/db/schema";
+import { createTestContext } from "@sttracker/plugin-sdk/testing";
+import { definePlugin, type SourcePlugin, type SourceRecord, type WorkHint } from "@sttracker/plugin-sdk/types";
+import type { SyncDeps } from "../../src/sync/run.ts";
+
+export { makeUser, makeWork } from "../user-layer-fixtures.ts";
+
+/** Insert a `source_connections` row directly and return its id. */
+export async function makeConnection(db: Db, userId: string, pluginId: string): Promise<string> {
+  const id = newId();
+  await db.k
+    .insertInto("source_connections")
+    .values({
+      id,
+      user_id: userId,
+      plugin_id: pluginId,
+      name: pluginId,
+      config: "{}",
+      credentials_enc: null,
+      schedule_minutes: null,
+      cursor: null,
+      status: "active",
+      last_run_at: null,
+      created_at: nowIso(),
+    })
+    .execute();
+  return id;
+}
+
+/** A source whose records the test mutates between runs. `failAfter` makes `pull` throw after that many records. */
+export interface FakeSource {
+  records: SourceRecord[];
+  failAfter?: number;
+}
+
+export function fakePlugin(id: string, src: FakeSource): SourcePlugin {
+  async function* gen(): AsyncIterable<SourceRecord> {
+    let i = 0;
+    for (const r of src.records) {
+      if (src.failAfter !== undefined && i >= src.failAfter) throw new Error("source exploded");
+      yield r;
+      i++;
+    }
+    if (src.failAfter !== undefined && i >= src.failAfter) throw new Error("source exploded");
+  }
+  return definePlugin({
+    manifest: { id, name: id, version: "0.0.0", sdkVersion: 1, mediaTypes: ["book", "movie", "show"], capabilities: ["pull", "pullChanges"], config: [] },
+    pull: () => gen(),
+    pullChanges: () => gen(),
+  });
+}
+
+export function deps(db: Db): SyncDeps {
+  return { db, makeContext: () => createTestContext() };
+}
+
+export function book(isbn: string, title = `Book ${isbn}`): WorkHint {
+  return { title, mediaType: "book", externalIds: { isbn13: isbn } };
+}
+
+export function entryRec(key: string, work: WorkHint, entry: Extract<SourceRecord, { kind: "entry" }>["entry"], extra: Partial<SourceRecord> = {}): SourceRecord {
+  return { key, kind: "entry", work, entry, raw: { key, entry }, ...extra } as SourceRecord;
+}
+
+export function engagementRec(
+  key: string,
+  work: WorkHint,
+  engagement: Extract<SourceRecord, { kind: "engagement" }>["engagement"],
+): SourceRecord {
+  return { key, kind: "engagement", work, engagement, raw: { key } };
+}
+
+export function listenRec(key: string, listen: Extract<SourceRecord, { kind: "listen" }>["listen"]): SourceRecord {
+  return { key, kind: "listen", listen, raw: { key } };
+}
+
+export async function count(db: Db, table: keyof Database): Promise<number> {
+  const r = await db.k
+    .selectFrom(table)
+    .select((eb) => eb.fn.countAll<number>().as("n"))
+    .executeTakeFirstOrThrow();
+  return Number(r.n);
+}
+
+export async function events(db: Db, where: { type?: string; status?: string } = {}) {
+  let q = db.k.selectFrom("change_events").selectAll().orderBy("id");
+  if (where.type) q = q.where("type", "=", where.type as never);
+  if (where.status) q = q.where("status", "=", where.status as never);
+  const rows = await q.execute();
+  return rows.map((r) => ({ ...r, payload: JSON.parse(r.payload) as Record<string, unknown> }));
+}
+
+export async function entryFor(db: Db, userId: string, isbn: string) {
+  const ext = await db.k
+    .selectFrom("external_ids")
+    .select("work_id")
+    .where("namespace", "=", "isbn13")
+    .where("value", "=", isbn)
+    .executeTakeFirst();
+  if (!ext) return undefined;
+  const row = await db.k
+    .selectFrom("entries")
+    .selectAll()
+    .where("user_id", "=", userId)
+    .where("work_id", "=", ext.work_id)
+    .executeTakeFirst();
+  return row ? { ...row, fieldSources: JSON.parse(row.field_sources) as Record<string, string> } : undefined;
+}

packages/core/test/sync/listens.test.ts

@@ -0,0 +1,71 @@
+import { expect, test } from "bun:test";
+import { forEachDialect } from "@sttracker/db/testing";
+import { ingestListens } from "../../src/sync/listens.ts";
+import { runSync } from "../../src/sync/run.ts";
+import { count, deps, entryRec, book, fakePlugin, listenRec, makeConnection, makeUser, makeWork } from "./helpers.ts";
+
+forEachDialect("listens", (getDb) => {
+  test("listens insert-or-ignore and mbid matching", async () => {
+    const db = getDb();
+    const userId = await makeUser(db);
+    const known = await makeWork(db, { title: "Known Song", mediaType: "track", externalIds: { musicbrainz_recording: "rec-1" } });
+    const records = [
+      listenRec("lb:1", { listenedAt: "2024-01-01T10:00:00Z", artist: "A", track: "Known song", mbids: { recording: "rec-1" } }),
+      listenRec("lb:2", { listenedAt: "2024-01-01T10:05:00Z", artist: "B", track: "New Song", album: "LP", durationMs: 1000, mbids: { recording: "rec-2" } }),
+      listenRec("lb:3", { listenedAt: "2024-01-01T10:10:00Z", artist: "C", track: "No Ids" }),
+    ];
+    const works0 = await count(db, "works");
+    expect(await ingestListens(db, userId, "listenbrainz", records)).toEqual({ inserted: 3 });
+
+    const rows = await db.k.selectFrom("listens").selectAll().orderBy("source_key").execute();
+    expect(rows.map((r) => r.source_key)).toEqual(["lb:1", "lb:2", "lb:3"]);
+    expect(rows[0]!.work_id).toBe(known);
+    expect(rows[2]!.work_id).toBeNull();
+    expect(rows[1]!.album).toBe("LP");
+    expect(rows[1]!.duration_ms).toBe(1000);
+    expect(JSON.parse(rows[1]!.mbids)).toEqual({ recording: "rec-2" });
+    const created = await db.k.selectFrom("works").selectAll().where("id", "=", rows[1]!.work_id!).executeTakeFirstOrThrow();
+    expect(created.media_type).toBe("track");
+    expect(created.title).toBe("New Song");
+    expect(created.owner_user_id).toBeNull();
+    const ext = await db.k.selectFrom("external_ids").selectAll().where("work_id", "=", created.id).execute();
+    expect(ext.map((e) => [e.namespace, e.value])).toEqual([["musicbrainz_recording", "rec-2"]]);
+    expect(await count(db, "works")).toBe(works0 + 1);
+    // A listen never renames the shared work.
+    expect((await db.k.selectFrom("works").select("title").where("id", "=", known).executeTakeFirstOrThrow()).title).toBe("Known Song");
+
+    expect(await ingestListens(db, userId, "listenbrainz", records)).toEqual({ inserted: 0 });
+    expect(await count(db, "listens")).toBe(3);
+    expect(await count(db, "works")).toBe(works0 + 1);
+    expect(await count(db, "change_events")).toBe(0);
+
+    // Same key under another source, or for another user, is a separate listen.
+    expect(await ingestListens(db, userId, "lastfm", records.slice(0, 1))).toEqual({ inserted: 1 });
+    const bob = await makeUser(db, "bob");
+    expect(await ingestListens(db, bob, "listenbrainz", records.slice(0, 1))).toEqual({ inserted: 1 });
+  });
+
+  test("runSync routes listens to ingestListens and counts them", async () => {
+    const db = getDb();
+    const userId = await makeUser(db);
+    const connectionId = await makeConnection(db, userId, "lb");
+    const src = {
+      records: [
+        listenRec("1", { listenedAt: "2024-01-01T10:00:00Z", artist: "A", track: "T" }),
+        listenRec("2", { listenedAt: "2024-01-01T10:01:00Z", artist: "A", track: "U" }),
+        entryRec("e", book("1"), { rating: 50 }),
+      ],
+    };
+    const s = await runSync(deps(db), fakePlugin("lb", src), connectionId, "full");
+    expect(s.counts.listens).toBe(2);
+    expect(s.counts.seen).toBe(3);
+    const rows = await db.k.selectFrom("listens").select(["source", "source_key"]).orderBy("source_key").execute();
+    expect(rows).toEqual([{ source: "lb", source_key: "1" }, { source: "lb", source_key: "2" }]);
+    // Listens are not snapshotted as source records and never removed.
+    expect(await count(db, "source_records")).toBe(1);
+    src.records = [];
+    const s2 = await runSync(deps(db), fakePlugin("lb", src), connectionId, "full");
+    expect(s2.counts.removed).toBe(1);
+    expect(await count(db, "listens")).toBe(2);
+  });
+});

packages/core/test/sync/run.test.ts

@@ -0,0 +1,409 @@
+import { expect, test } from "bun:test";
+import { forEachDialect } from "@sttracker/db/testing";
+import { softDeleteEntry, upsertEntry } from "../../src/entries.ts";
+import { listChanges, resolveChange } from "../../src/sync/changes.ts";
+import { runSync } from "../../src/sync/run.ts";
+import { NotFoundError, ValidationError } from "../../src/errors.ts";
+import {
+  book,
+  count,
+  deps,
+  engagementRec,
+  entryFor,
+  entryRec,
+  events,
+  fakePlugin,
+  type FakeSource,
+  listenRec,
+  makeConnection,
+  makeUser,
+} from "./helpers.ts";
+
+forEachDialect("sync engine", (getDb) => {
+  async function setup(pluginId = "fake") {
+    const db = getDb();
+    const userId = await makeUser(db);
+    const connectionId = await makeConnection(db, userId, pluginId);
+    const src: FakeSource = { records: [] };
+    const plugin = fakePlugin(pluginId, src);
+    const actor = `plugin:${pluginId}:${connectionId}`;
+    return { db, userId, connectionId, src, plugin, actor };
+  }
+
+  test("first full pull creates entries and auto_applied events", async () => {
+    const { db, userId, connectionId, src, plugin, actor } = await setup();
+    src.records = [
+      entryRec("e1", book("111"), { status: "completed", rating: 80, notes: "loved it" }),
+      engagementRec("g1", book("111"), { finishedAt: { value: "2024-01-05", precision: "day" } }),
+    ];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.status).toBe("ok");
+    expect(s.counts).toEqual({ seen: 2, added: 2, changed: 0, removed: 0, merged: 0, conflicts: 0, autoApplied: 2, pending: 0, listens: 0 });
+
+    const e = await entryFor(db, userId, "111");
+    expect(e?.status).toBe("completed");
+    expect(e?.rating).toBe(80);
+    expect(e?.private_notes).toBe("loved it");
+    expect(e?.fieldSources).toEqual({ status: actor, rating: actor, privateNotes: actor });
+
+    const eg = await db.k.selectFrom("engagements").selectAll().execute();
+    expect(eg).toHaveLength(1);
+    expect(eg[0]!.entry_id).toBe(e!.id);
+    expect(eg[0]!.finished_at).toBe("2024-01-05");
+    expect(await count(db, "engagement_sources")).toBe(1);
+
+    const ev = await events(db);
+    expect(ev.map((x) => [x.type, x.status])).toEqual([["added", "auto_applied"], ["added", "auto_applied"]]);
+    expect(ev[0]!.run_id).toBe(s.runId);
+    expect(ev[0]!.target_entity).toBe("entry");
+    expect(ev[0]!.target_id).toBe(e!.id);
+    expect(ev[1]!.target_entity).toBe("engagement");
+
+    expect(await count(db, "source_records")).toBe(2);
+    expect(await count(db, "source_record_versions")).toBe(2);
+    const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "e1").executeTakeFirstOrThrow();
+    expect(rec.target_entity).toBe("entry");
+    expect(rec.target_id).toBe(e!.id);
+
+    const run = await db.k.selectFrom("source_runs").selectAll().where("id", "=", s.runId).executeTakeFirstOrThrow();
+    expect(run.status).toBe("ok");
+    expect(run.mode).toBe("full");
+    expect(run.finished_at).not.toBeNull();
+    expect(JSON.parse(run.counts)).toEqual(s.counts);
+    const conn = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
+    expect(conn.last_run_at).not.toBeNull();
+    expect(conn.status).toBe("active");
+  });
+
+  test("an entry on a child work also gets a status-none entry on the root ancestor", async () => {
+    const { db, userId, connectionId, src, plugin, actor } = await setup();
+    const show = { title: "Show", mediaType: "show" as const, externalIds: { tmdb_show: "9" } };
+    const season = { title: "Season 1", mediaType: "season" as const, externalIds: {}, parent: show, position: { season: 1 } };
+    const episode = { title: "Pilot", mediaType: "episode" as const, externalIds: { tmdb_episode: "91" }, parent: season, position: { episode: 1 } };
+    src.records = [engagementRec("ep1", episode, { finishedAt: { value: "2024-02-01T21:00:00Z", precision: "day" } })];
+    await runSync(deps(db), plugin, connectionId, "full");
+
+    const rows = await db.k
+      .selectFrom("entries as e")
+      .innerJoin("works as w", "w.id", "e.work_id")
+      .select(["w.media_type", "e.status", "e.field_sources"])
+      .where("e.user_id", "=", userId)
+      .orderBy("w.media_type")
+      .execute();
+    expect(rows.map((r) => [r.media_type, r.status])).toEqual([["episode", "none"], ["show", "none"]]);
+    const hist = await db.k.selectFrom("user_history").select("actor").execute();
+    expect(new Set(hist.map((h) => h.actor))).toEqual(new Set([actor]));
+
+    // An existing (even deleted) root entry is never touched.
+    const showEntry = await db.k
+      .selectFrom("entries as e")
+      .innerJoin("works as w", "w.id", "e.work_id")
+      .select("e.id")
+      .where("w.media_type", "=", "show")
+      .executeTakeFirstOrThrow();
+    await softDeleteEntry(db, userId, showEntry.id, "user");
+    src.records.push(engagementRec("ep2", { ...episode, title: "Two", externalIds: { tmdb_episode: "92" }, position: { episode: 2 } }, {}));
+    await runSync(deps(db), plugin, connectionId, "full");
+    const show2 = await db.k.selectFrom("entries").selectAll().where("id", "=", showEntry.id).executeTakeFirstOrThrow();
+    expect(show2.deleted_at).not.toBeNull();
+  });
+
+  test("second identical pull creates nothing new", async () => {
+    const { db, connectionId, src, plugin } = await setup();
+    src.records = [
+      entryRec("e1", book("111"), { status: "completed", rating: 80 }),
+      engagementRec("g1", book("111"), { finishedAt: { value: "2024-01-05", precision: "day" } }),
+      listenRec("l1", { listenedAt: "2024-01-01T10:00:00Z", artist: "A", track: "T" }),
+    ];
+    const first = await runSync(deps(db), plugin, connectionId, "full");
+    expect(first.counts.listens).toBe(1);
+    const tables = ["source_records", "source_record_versions", "change_events", "entries", "engagements", "engagement_sources", "listens", "user_history", "works"] as const;
+    const before = await Promise.all(tables.map((t) => count(db, t)));
+    const second = await runSync(deps(db), plugin, connectionId, "full");
+    const after = await Promise.all(tables.map((t) => count(db, t)));
+    expect(after).toEqual(before);
+    expect(second.status).toBe("ok");
+    expect(second.counts).toEqual({ seen: 3, added: 0, changed: 0, removed: 0, merged: 0, conflicts: 0, autoApplied: 0, pending: 0, listens: 0 });
+    expect(await count(db, "source_runs")).toBe(2);
+  });
+
+  test("failed pull marks nothing removed", async () => {
+    const { db, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "completed" }), entryRec("b", book("2"), { status: "planned" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+
+    src.failAfter = 1;
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.status).toBe("failed");
+    expect(s.error).toContain("source exploded");
+    expect(s.counts.removed).toBe(0);
+    const deleted = await db.k.selectFrom("source_records").select("key").where("deleted_at", "is not", null).execute();
+    expect(deleted).toEqual([]);
+    expect(await events(db, { type: "removed" })).toEqual([]);
+    const run = await db.k.selectFrom("source_runs").selectAll().where("id", "=", s.runId).executeTakeFirstOrThrow();
+    expect(run.status).toBe("failed");
+    expect(run.error).toContain("source exploded");
+    const conn = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
+    expect(conn.status).toBe("error");
+
+    // A later successful run puts the connection back to active.
+    src.failAfter = undefined;
+    expect((await runSync(deps(db), plugin, connectionId, "full")).status).toBe("ok");
+    const conn2 = await db.k.selectFrom("source_connections").selectAll().where("id", "=", connectionId).executeTakeFirstOrThrow();
+    expect(conn2.status).toBe("active");
+  });
+
+  test("record missing from completed full pull → pending removed; accept soft-deletes entry", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "completed" }), entryRec("b", book("2"), { status: "planned" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+
+    src.records = [src.records[0]!];
+    // A "changes" pull never infers removal from absence.
+    expect((await runSync(deps(db), plugin, connectionId, "changes")).counts.removed).toBe(0);
+    expect(await events(db, { type: "removed" })).toEqual([]);
+
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.counts.removed).toBe(1);
+    expect(s.counts.pending).toBe(1);
+    const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "b").executeTakeFirstOrThrow();
+    expect(rec.deleted_at).not.toBeNull();
+    const removed = await events(db, { type: "removed" });
+    expect(removed).toHaveLength(1);
+    expect(removed[0]!.status).toBe("pending");
+    const entryB = await entryFor(db, userId, "2");
+    expect(removed[0]!.target_id).toBe(entryB!.id);
+    expect(entryB!.deleted_at).toBeNull(); // nothing deleted without the user's say
+
+    // Rerun: not re-filed.
+    await runSync(deps(db), plugin, connectionId, "full");
+    expect(await events(db, { type: "removed" })).toHaveLength(1);
+
+    await resolveChange(db, userId, removed[0]!.id, "accept");
+    expect((await entryFor(db, userId, "2"))!.deleted_at).not.toBeNull();
+    const ev = (await events(db, { type: "removed" }))[0]!;
+    expect(ev.status).toBe("accepted");
+    expect(ev.resolved_at).not.toBeNull();
+  });
+
+  test("deleted: true in a changes pull files a pending removed event", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "completed" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    src.records = [entryRec("a", book("1"), { status: "completed" }, { deleted: true })];
+    const s = await runSync(deps(db), plugin, connectionId, "changes");
+    expect(s.counts.removed).toBe(1);
+    const removed = await events(db, { type: "removed", status: "pending" });
+    expect(removed).toHaveLength(1);
+    await resolveChange(db, userId, removed[0]!.id, "ignore");
+    expect((await entryFor(db, userId, "1"))!.deleted_at).toBeNull();
+    expect((await events(db, { type: "removed" }))[0]!.status).toBe("ignored");
+  });
+
+  test("user-edited rating not overwritten; pending changed event", async () => {
+    const { db, userId, connectionId, src, plugin, actor } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "in_progress", rating: 80 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const e0 = (await entryFor(db, userId, "1"))!;
+    await upsertEntry(db, userId, e0.work_id, { rating: 90 }, "user");
+
+    src.records = [entryRec("a", book("1"), { status: "completed", rating: 70 })];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.counts.changed).toBe(1);
+    expect(s.counts.autoApplied).toBe(1);
+    expect(s.counts.pending).toBe(1);
+    const e1 = (await entryFor(db, userId, "1"))!;
+    expect(e1.rating).toBe(90);
+    expect(e1.status).toBe("completed"); // never edited by the user: applied
+    expect(e1.fieldSources).toEqual({ status: actor, rating: "user" });
+    const pending = await events(db, { type: "changed", status: "pending" });
+    expect(pending).toHaveLength(1);
+    expect(pending[0]!.payload).toMatchObject({ field: "rating", from: 90, to: 70 });
+    const applied = await events(db, { type: "changed", status: "auto_applied" });
+    expect(applied.map((x) => x.payload)).toEqual([expect.objectContaining({ field: "status", from: "in_progress", to: "completed" })]);
+
+    // Rerun: nothing new.
+    await runSync(deps(db), plugin, connectionId, "full");
+    expect(await events(db, { type: "changed", status: "pending" })).toHaveLength(1);
+
+    const listed = await listChanges(db, userId, {});
+    expect(listed.items.map((c) => c.id)).toEqual([pending[0]!.id]);
+
+    await resolveChange(db, userId, pending[0]!.id, "accept");
+    const e2 = (await entryFor(db, userId, "1"))!;
+    expect(e2.rating).toBe(70);
+    expect(e2.fieldSources.rating).toBe(actor);
+    expect((await listChanges(db, userId, {})).items).toEqual([]);
+    await expect(resolveChange(db, userId, pending[0]!.id, "accept")).rejects.toBeInstanceOf(ValidationError);
+  });
+
+  test("a user-deleted entry is never revived by a sync", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { rating: 80 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const e = (await entryFor(db, userId, "1"))!;
+    await softDeleteEntry(db, userId, e.id, "user");
+    const before = await count(db, "change_events");
+    // A changed record, a brand-new entry record and a new engagement record, all on the deleted entry's work.
+    src.records = [
+      entryRec("a", book("1"), { rating: 50 }),
+      entryRec("a2", book("1"), { status: "completed" }),
+      engagementRec("g", book("1"), { finishedAt: { value: "2024-01-01", precision: "day" } }),
+    ];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.status).toBe("ok");
+    const after = (await entryFor(db, userId, "1"))!;
+    expect(after.deleted_at).not.toBeNull();
+    expect(after.rating).toBe(80);
+    expect(after.status).toBe("none");
+    expect(await count(db, "engagements")).toBe(0);
+    expect(await count(db, "change_events")).toBe(before);
+  });
+
+  test("an id-less record is updated through its mapping, not re-matched", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    const hint = { title: "Obscure Zine", mediaType: "book" as const, externalIds: {} };
+    src.records = [entryRec("csv-1", hint, { rating: 40 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const works = await count(db, "works");
+    src.records = [entryRec("csv-1", hint, { rating: 60 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    expect(await count(db, "works")).toBe(works);
+    const rows = await db.k.selectFrom("entries").selectAll().where("user_id", "=", userId).execute();
+    expect(rows).toHaveLength(1);
+    expect(rows[0]!.rating).toBe(60);
+  });
+
+  test("two connections disagree → one pending conflict, not duplicated on rerun; keep_mine stops further conflicts for that value", async () => {
+    const { db, userId, connectionId: connA, src: srcA, plugin: pluginA, actor: actorA } = await setup("alpha");
+    const connB = await makeConnection(db, userId, "beta");
+    const srcB: FakeSource = { records: [] };
+    const pluginB = fakePlugin("beta", srcB);
+    const actorB = `plugin:beta:${connB}`;
+
+    srcA.records = [entryRec("a", book("1"), { rating: 80 })];
+    await runSync(deps(db), pluginA, connA, "full");
+    srcB.records = [entryRec("b", book("1"), { rating: 60, status: "completed" })];
+    const s = await runSync(deps(db), pluginB, connB, "full");
+    expect(s.counts.conflicts).toBe(1);
+
+    const e = (await entryFor(db, userId, "1"))!;
+    expect(e.rating).toBe(80);
+    expect(e.status).toBe("completed"); // field nobody had set: applied
+    expect(e.fieldSources).toEqual({ rating: actorA, status: actorB });
+    let conflicts = await events(db, { type: "conflict", status: "pending" });
+    expect(conflicts).toHaveLength(1);
+    expect(conflicts[0]!.payload).toMatchObject({ field: "rating", current: 80, currentSource: actorA, incoming: 60, incomingSource: actorB });
+
+    // Rerun both: nothing new. Then B flips 60 → 70 → 60: the conflict for 60 is not filed twice.
+    await runSync(deps(db), pluginA, connA, "full");
+    await runSync(deps(db), pluginB, connB, "full");
+    expect(await events(db, { type: "conflict" })).toHaveLength(1);
+    srcB.records = [entryRec("b", book("1"), { rating: 70, status: "completed" })];
+    await runSync(deps(db), pluginB, connB, "full");
+    srcB.records = [entryRec("b", book("1"), { rating: 60, status: "completed" })];
+    await runSync(deps(db), pluginB, connB, "full");
+    conflicts = await events(db, { type: "conflict", status: "pending" });
+    expect(conflicts.map((c) => c.payload.incoming)).toEqual([60, 70]);
+
+    await resolveChange(db, userId, conflicts[0]!.id, "keep_mine");
+    const kept = (await entryFor(db, userId, "1"))!;
+    expect(kept.rating).toBe(80);
+    expect(kept.fieldSources.rating).toBe("user");
+    expect((await events(db, { type: "conflict" }))[0]!.status).toBe("kept_mine");
+
+    await runSync(deps(db), pluginA, connA, "full");
+    await runSync(deps(db), pluginB, connB, "full");
+    expect(await events(db, { type: "conflict" })).toHaveLength(2);
+    expect((await entryFor(db, userId, "1"))!.rating).toBe(80);
+  });
+
+  test("same play from two sources dedupes into one engagement with two engagement_sources", async () => {
+    const { db, userId, connectionId: connA, src: srcA, plugin: pluginA } = await setup("alpha");
+    const connB = await makeConnection(db, userId, "beta");
+    const srcB: FakeSource = { records: [] };
+    const pluginB = fakePlugin("beta", srcB);
+
+    srcA.records = [engagementRec("play-1", book("1"), { finishedAt: { value: "2024-03-01T20:00:00Z", precision: "day" } })];
+    await runSync(deps(db), pluginA, connA, "full");
+    srcB.records = [
+      engagementRec("row-7", book("1"), { finishedAt: { value: "2024-03-02", precision: "day" }, rating: 90 }),
+      engagementRec("row-8", book("1"), { finishedAt: { value: "2024-03-10", precision: "day" } }),
+    ];
+    await runSync(deps(db), pluginB, connB, "full");
+
+    const egs = await db.k.selectFrom("engagements").selectAll().orderBy("finished_at").execute();
+    expect(egs).toHaveLength(2);
+    expect(egs[0]!.finished_at).toBe("2024-03-01T20:00:00Z"); // first source's value kept
+    expect(egs[0]!.rating).toBe(90); // empty field filled from the second source
+    const srcs = await db.k.selectFrom("engagement_sources").selectAll().where("engagement_id", "=", egs[0]!.id).execute();
+    expect(srcs.map((x) => x.record_key).sort()).toEqual(["play-1", "row-7"]);
+    expect(await count(db, "engagement_sources")).toBe(3);
+
+    // Equal start datetime also dedupes.
+    srcA.records.push(engagementRec("play-2", book("1"), { startedAt: { value: "2024-05-01T08:00:00Z", precision: "day" } }));
+    srcB.records.push(engagementRec("row-9", book("1"), { startedAt: { value: "2024-05-01T08:00:00Z", precision: "day" } }));
+    await runSync(deps(db), pluginA, connA, "full");
+    await runSync(deps(db), pluginB, connB, "full");
+    expect(await count(db, "engagements")).toBe(3);
+    expect(await count(db, "engagement_sources")).toBe(5);
+  });
+
+  test("merged record → pending merged; accept moves entry", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("x", book("1"), { status: "completed" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const ex = (await entryFor(db, userId, "1"))!;
+
+    // The source merged x into y. Record y is new and creates its own (empty) entry on book 2.
+    src.records = [entryRec("x", book("1"), { status: "completed" }, { mergedIntoKey: "y" }), entryRec("y", book("2"), {})];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.counts.merged).toBe(1);
+    const merged = await events(db, { type: "merged", status: "pending" });
+    expect(merged).toHaveLength(1);
+    expect(merged[0]!.target_id).toBe(ex.id);
+    expect(merged[0]!.payload).toMatchObject({ intoKey: "y" });
+    expect((await entryFor(db, userId, "1"))!.deleted_at).toBeNull();
+
+    await resolveChange(db, userId, merged[0]!.id, "accept");
+    const ey = (await entryFor(db, userId, "2"))!;
+    expect(ey.deleted_at).toBeNull();
+    expect(ey.status).toBe("completed"); // filled from the moved entry
+    expect((await entryFor(db, userId, "1"))!.deleted_at).not.toBeNull();
+    expect((await events(db, { type: "merged" }))[0]!.status).toBe("accepted");
+  });
+
+  test("resolveChange is scoped to the user", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "completed" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    src.records = [];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const [ev] = await events(db, { type: "removed" });
+    const other = await makeUser(db, "bob");
+    await expect(resolveChange(db, other, ev!.id, "accept")).rejects.toBeInstanceOf(NotFoundError);
+    expect((await listChanges(db, other, { status: "all" })).items).toEqual([]);
+    expect((await listChanges(db, userId, { status: "all" })).items).toHaveLength(2);
+  });
+
+  test("listChanges pages newest first and filters by connection", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [1, 2, 3].map((i) => entryRec(`k${i}`, book(String(i)), { status: "planned" }));
+    await runSync(deps(db), plugin, connectionId, "full");
+    const p1 = await listChanges(db, userId, { status: "all", limit: 2 });
+    expect(p1.items).toHaveLength(2);
+    expect(p1.nextCursor).not.toBeNull();
+    const p2 = await listChanges(db, userId, { status: "all", limit: 2, cursor: p1.nextCursor! });
+    expect(p2.items).toHaveLength(1);
+    expect(p2.nextCursor).toBeNull();
+    const ids = [...p1.items, ...p2.items].map((c) => c.id);
+    expect(ids).toEqual([...ids].sort().reverse());
+    expect(p1.items[0]!.payload).toMatchObject({ key: "k3" });
+    expect((await listChanges(db, userId, { status: "all", connectionId: "nope" })).items).toEqual([]);
+  });
+
+  test("a plugin that does not match the connection is refused", async () => {
+    const { db, connectionId } = await setup("alpha");
+    await expect(runSync(deps(db), fakePlugin("beta", { records: [] }), connectionId, "full")).rejects.toBeInstanceOf(ValidationError);
+  });
+});

packages/core/test/sync/snapshot.test.ts

@@ -0,0 +1,31 @@
+import { expect, test } from "bun:test";
+import type { SourceRecord } from "@sttracker/plugin-sdk/types";
+import { decodeVersion, encodeVersion, hashRecord, normalizeRecord } from "../../src/sync/snapshot.ts";
+
+const rec = (raw: unknown, entry: Record<string, unknown>): SourceRecord =>
+  ({ key: "k", kind: "entry", work: { title: "T", mediaType: "book", externalIds: { a: "1", b: "2" } }, entry, raw }) as SourceRecord;
+
+test("hash ignores raw payload and key order", async () => {
+  const a = rec({ x: 1 }, { rating: 1, status: "planned" });
+  const b = rec({ y: 2 }, { status: "planned", rating: 1 });
+  expect(await hashRecord(a)).toBe(await hashRecord(b));
+  expect(await hashRecord(a)).toMatch(/^[0-9a-f]{64}$/);
+});
+
+test("hash covers normalised fields, deleted and mergedIntoKey", async () => {
+  const base = rec({}, { rating: 1 });
+  const h = await hashRecord(base);
+  expect(await hashRecord(rec({}, { rating: 2 }))).not.toBe(h);
+  expect(await hashRecord({ ...base, deleted: true })).not.toBe(h);
+  expect(await hashRecord({ ...base, mergedIntoKey: "z" })).not.toBe(h);
+  expect(await hashRecord({ ...base, deleted: false })).toBe(h);
+});
+
+test("versions round-trip the normalised record and the raw payload", () => {
+  const r = rec({ deep: { list: [1, 2] } }, { rating: 5 });
+  const encoded = encodeVersion(r);
+  expect(typeof encoded).toBe("string");
+  const v = decodeVersion(encoded);
+  expect(v.raw).toEqual({ deep: { list: [1, 2] } });
+  expect(v.normalized).toEqual(normalizeRecord(r));
+});