merge: feat/phase1 into task/9

packages/core/test/sync/helpers.ts

@@ -108,3 +108,7 @@ export async function entryFor(db: Db, userId: string, isbn: string) {
     .executeTakeFirst();
   return row ? { ...row, fieldSources: JSON.parse(row.field_sources) as Record<string, string> } : undefined;
 }
+
+export function listRec(key: string, name: string, items: { work: WorkHint; note?: string }[], description?: string): SourceRecord {
+  return { key, kind: "list", list: { name, items, ...(description !== undefined ? { description } : {}) }, raw: { key } };
+}

packages/core/test/sync/listens.test.ts

@@ -45,6 +45,29 @@ forEachDialect("listens", (getDb) => {
     expect(await ingestListens(db, bob, "listenbrainz", records.slice(0, 1))).toEqual({ inserted: 1 });
   });
 
+  test("1200 listens ingest correctly and idempotently", async () => {
+    const db = getDb();
+    const userId = await makeUser(db);
+    const records = Array.from({ length: 1200 }, (_, i) =>
+      listenRec(`k${i}`, {
+        listenedAt: `2024-01-01T00:00:${String(i % 60).padStart(2, "0")}Z`,
+        artist: "A",
+        track: `T${i % 7}`,
+        ...(i % 3 === 0 ? { mbids: { recording: `rec-${i % 7}` } } : {}),
+      }),
+    );
+    // A duplicate key inside the same call is inserted once.
+    records.push(records[5]!);
+    expect(await ingestListens(db, userId, "lb", records)).toEqual({ inserted: 1200 });
+    expect(await count(db, "listens")).toBe(1200);
+    expect(await count(db, "works")).toBe(7);
+    const linked = await db.k.selectFrom("listens").select("work_id").where("work_id", "is not", null).execute();
+    expect(linked).toHaveLength(400);
+    expect(await ingestListens(db, userId, "lb", records)).toEqual({ inserted: 0 });
+    expect(await count(db, "listens")).toBe(1200);
+    expect(await count(db, "works")).toBe(7);
+  });
+
   test("runSync routes listens to ingestListens and counts them", async () => {
     const db = getDb();
     const userId = await makeUser(db);

packages/core/test/sync/run.test.ts

@@ -1,6 +1,7 @@
 import { expect, test } from "bun:test";
 import { forEachDialect } from "@sttracker/db/testing";
-import { softDeleteEntry, upsertEntry } from "../../src/entries.ts";
+import { moveEntryToWork, softDeleteEntry, upsertEntry } from "../../src/entries.ts";
+import { updateList } from "../../src/lists.ts";
 import { listChanges, resolveChange } from "../../src/sync/changes.ts";
 import { runSync } from "../../src/sync/run.ts";
 import { NotFoundError, ValidationError } from "../../src/errors.ts";
@@ -15,8 +16,10 @@ import {
   fakePlugin,
   type FakeSource,
   listenRec,
+  listRec,
   makeConnection,
   makeUser,
+  makeWork,
 } from "./helpers.ts";
 
 forEachDialect("sync engine", (getDb) => {
@@ -295,26 +298,42 @@ forEachDialect("sync engine", (getDb) => {
     expect(conflicts).toHaveLength(1);
     expect(conflicts[0]!.payload).toMatchObject({ field: "rating", current: 80, currentSource: actorA, incoming: 60, incomingSource: actorB });
 
-    // Rerun both: nothing new. Then B flips 60 → 70 → 60: the conflict for 60 is not filed twice.
+    // Rerun both: nothing new.
     await runSync(deps(db), pluginA, connA, "full");
     await runSync(deps(db), pluginB, connB, "full");
     expect(await events(db, { type: "conflict" })).toHaveLength(1);
+
+    // B moves 60 → 70: the stale pending conflict is superseded, not left beside the new one.
     srcB.records = [entryRec("b", book("1"), { rating: 70, status: "completed" })];
     await runSync(deps(db), pluginB, connB, "full");
+    let all = await events(db, { type: "conflict" });
+    expect(all.map((c) => [c.payload.incoming, c.status])).toEqual([[60, "ignored"], [70, "pending"]]);
+    expect(all[0]!.payload.supersededBy).toBe(all[1]!.id);
+    expect(all[0]!.resolved_at).not.toBeNull();
+
+    // And back 70 → 60: still exactly one pending conflict for the field.
     srcB.records = [entryRec("b", book("1"), { rating: 60, status: "completed" })];
     await runSync(deps(db), pluginB, connB, "full");
     conflicts = await events(db, { type: "conflict", status: "pending" });
-    expect(conflicts.map((c) => c.payload.incoming)).toEqual([60, 70]);
+    expect(conflicts.map((c) => c.payload.incoming)).toEqual([60]);
 
     await resolveChange(db, userId, conflicts[0]!.id, "keep_mine");
     const kept = (await entryFor(db, userId, "1"))!;
     expect(kept.rating).toBe(80);
     expect(kept.fieldSources.rating).toBe("user");
-    expect((await events(db, { type: "conflict" }))[0]!.status).toBe("kept_mine");
+    all = await events(db, { type: "conflict" });
+    expect(all.find((c) => c.id === conflicts[0]!.id)!.status).toBe("kept_mine");
 
+    // Reruns file nothing; a NEW value from B is now a user-decided pending change, not a conflict.
     await runSync(deps(db), pluginA, connA, "full");
     await runSync(deps(db), pluginB, connB, "full");
-    expect(await events(db, { type: "conflict" })).toHaveLength(2);
+    expect(await events(db, { type: "conflict" })).toHaveLength(3);
+    srcB.records = [entryRec("b", book("1"), { rating: 65, status: "completed" })];
+    await runSync(deps(db), pluginB, connB, "full");
+    expect(await events(db, { type: "conflict" })).toHaveLength(3);
+    const changed = await events(db, { type: "changed", status: "pending" });
+    expect(changed).toHaveLength(1);
+    expect(changed[0]!.payload).toMatchObject({ field: "rating", from: 80, to: 65 });
     expect((await entryFor(db, userId, "1"))!.rating).toBe(80);
   });
 
@@ -373,6 +392,189 @@ forEachDialect("sync engine", (getDb) => {
     expect((await events(db, { type: "merged" }))[0]!.status).toBe("accepted");
   });
 
+  test("a deduped play with a disagreeing value files a conflict instead of dropping it", async () => {
+    const { db, userId, connectionId: connA, src: srcA, plugin: pluginA, actor: actorA } = await setup("alpha");
+    const connB = await makeConnection(db, userId, "beta");
+    const pluginB = fakePlugin("beta", { records: [engagementRec("r", book("1"), { finishedAt: { value: "2024-03-01", precision: "day" }, rating: 90 })] });
+    srcA.records = [engagementRec("p", book("1"), { finishedAt: { value: "2024-03-01T20:00:00Z", precision: "day" }, rating: 80 })];
+    await runSync(deps(db), pluginA, connA, "full");
+    const s = await runSync(deps(db), pluginB, connB, "full");
+    expect(s.counts.conflicts).toBe(1);
+    const egs = await db.k.selectFrom("engagements").selectAll().execute();
+    expect(egs).toHaveLength(1);
+    expect(egs[0]!.rating).toBe(80);
+    expect(egs[0]!.finished_at).toBe("2024-03-01T20:00:00Z"); // the matching date is not a conflict
+    const c = await events(db, { type: "conflict", status: "pending" });
+    expect(c).toHaveLength(1);
+    expect(c[0]!.target_entity).toBe("engagement");
+    expect(c[0]!.payload).toMatchObject({ field: "rating", current: 80, currentSource: actorA, incoming: 90 });
+  });
+
+  test("a merged-away record that vanishes files no removal; the survivor is untouched", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("x", book("1"), { status: "completed" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    src.records = [entryRec("x", book("1"), { status: "completed" }, { mergedIntoKey: "y" }), entryRec("y", book("2"), {})];
+    await runSync(deps(db), plugin, connectionId, "full");
+    // Before the user decides, x vanishing files nothing either.
+    src.records = [entryRec("y", book("2"), {})];
+    expect((await runSync(deps(db), plugin, connectionId, "full")).counts.removed).toBe(0);
+    expect(await events(db, { type: "removed" })).toEqual([]);
+
+    const [merged] = await events(db, { type: "merged", status: "pending" });
+    await resolveChange(db, userId, merged!.id, "accept");
+    const survivor = (await entryFor(db, userId, "2"))!;
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.counts.removed).toBe(0);
+    expect(await events(db, { type: "removed" })).toEqual([]);
+    const after = (await entryFor(db, userId, "2"))!;
+    expect(after.deleted_at).toBeNull();
+    expect(after.status).toBe(survivor.status);
+  });
+
+  test("an id-less record whose apply fails is retried against the same draft", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    const hint = { title: "Obscure Zine", mediaType: "book" as const, externalIds: {} };
+    src.records = [entryRec("csv-1", hint, { rating: 150 })];
+    const failed = await runSync(deps(db), plugin, connectionId, "full");
+    expect(failed.status).toBe("failed");
+    expect(failed.counts.added).toBe(0);
+    src.records = [entryRec("csv-1", hint, { rating: 60 })];
+    const ok = await runSync(deps(db), plugin, connectionId, "full");
+    expect(ok.status).toBe("ok");
+    expect(ok.counts.added).toBe(1);
+    const drafts = await db.k.selectFrom("works").select("id").where("owner_user_id", "=", userId).execute();
+    expect(drafts).toHaveLength(1);
+    const e = await db.k.selectFrom("entries").selectAll().where("user_id", "=", userId).execute();
+    expect(e.map((x) => [x.work_id, x.rating])).toEqual([[drafts[0]!.id, 60]]);
+  });
+
+  test("a record-level error fails the run and skips removal", async () => {
+    const { db, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { status: "completed" }), entryRec("b", book("2"), { status: "planned" })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    src.records = [entryRec("a", book("1"), { status: "completed", rating: 999 })];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.status).toBe("failed");
+    expect(s.error).toContain("record a");
+    expect(s.counts.removed).toBe(0);
+    expect(s.counts.changed).toBe(0); // the failed record's work rolled back, and is not counted
+    expect(await events(db, { type: "removed" })).toEqual([]);
+  });
+
+  test("after the user move-merges an entry, the source keeps syncing into the survivor", async () => {
+    const { db, userId, connectionId, src, plugin, actor } = await setup();
+    src.records = [entryRec("a", book("1"), { rating: 50 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const e1 = (await entryFor(db, userId, "1"))!;
+    const w2 = await makeWork(db, { title: "Other" });
+    const e2 = await upsertEntry(db, userId, w2, { status: "planned" }, "user");
+    await moveEntryToWork(db, userId, e1.id, w2, "user");
+    const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "a").executeTakeFirstOrThrow();
+    expect(rec.target_id).toBe(e2.id);
+
+    src.records = [entryRec("a", book("1"), { rating: 60 })];
+    const s = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s.counts.autoApplied).toBe(1);
+    const survivor = await db.k.selectFrom("entries").selectAll().where("id", "=", e2.id).executeTakeFirstOrThrow();
+    expect(survivor.rating).toBe(60);
+    expect(JSON.parse(survivor.field_sources).rating).toBe(actor);
+  });
+
+  test("accepting removal of a play another source still reports only detaches this source", async () => {
+    const { db, userId, connectionId: connA, src: srcA, plugin: pluginA } = await setup("alpha");
+    const connB = await makeConnection(db, userId, "beta");
+    const srcB: FakeSource = { records: [engagementRec("r", book("1"), { finishedAt: { value: "2024-03-01", precision: "day" } })] };
+    const pluginB = fakePlugin("beta", srcB);
+    srcA.records = [engagementRec("p", book("1"), { finishedAt: { value: "2024-03-01", precision: "day" } })];
+    await runSync(deps(db), pluginA, connA, "full");
+    await runSync(deps(db), pluginB, connB, "full");
+    const [eg] = await db.k.selectFrom("engagements").selectAll().execute();
+
+    srcA.records = [];
+    await runSync(deps(db), pluginA, connA, "full");
+    const [removedA] = await events(db, { type: "removed", status: "pending" });
+    expect(removedA!.target_id).toBe(eg!.id);
+    await resolveChange(db, userId, removedA!.id, "accept");
+    let row = await db.k.selectFrom("engagements").selectAll().where("id", "=", eg!.id).executeTakeFirstOrThrow();
+    expect(row.deleted_at).toBeNull();
+    const left = await db.k.selectFrom("engagement_sources").select("connection_id").where("engagement_id", "=", eg!.id).execute();
+    expect(left.map((x) => x.connection_id)).toEqual([connB]);
+
+    srcB.records = [];
+    await runSync(deps(db), pluginB, connB, "full");
+    const [removedB] = await events(db, { type: "removed", status: "pending" });
+    await resolveChange(db, userId, removedB!.id, "accept");
+    row = await db.k.selectFrom("engagements").selectAll().where("id", "=", eg!.id).executeTakeFirstOrThrow();
+    expect(row.deleted_at).not.toBeNull();
+  });
+
+  test("accepting a change whose target is gone resolves it as ignored (target_gone)", async () => {
+    const { db, userId, connectionId, src, plugin } = await setup();
+    src.records = [entryRec("a", book("1"), { rating: 80 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const e = (await entryFor(db, userId, "1"))!;
+    await upsertEntry(db, userId, e.work_id, { rating: 90 }, "user");
+    src.records = [entryRec("a", book("1"), { rating: 70 })];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const [pending] = await events(db, { type: "changed", status: "pending" });
+    await softDeleteEntry(db, userId, e.id, "user");
+    await resolveChange(db, userId, pending!.id, "accept");
+    const [ev] = await events(db, { type: "changed" });
+    expect(ev!.status).toBe("ignored");
+    expect(ev!.payload.reason).toBe("target_gone");
+    expect(ev!.resolved_at).not.toBeNull();
+  });
+
+  test("list records: create, update, user-edited → pending, removed", async () => {
+    const { db, userId, connectionId, src, plugin, actor } = await setup();
+    const draftHint = { title: "Zine", mediaType: "book" as const, externalIds: {} };
+    src.records = [listRec("l1", "Favourites", [{ work: book("1") }, { work: draftHint, note: "rare" }])];
+    const s1 = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s1.counts.added).toBe(1);
+    const list = await db.k.selectFrom("lists").selectAll().where("user_id", "=", userId).executeTakeFirstOrThrow();
+    expect([list.name, list.description, list.kind]).toEqual(["Favourites", "Imported from fake", "list"]);
+    const itemsOf = async () =>
+      (await db.k.selectFrom("list_items as i").innerJoin("works as w", "w.id", "i.work_id").select(["w.title", "i.note"]).where("i.list_id", "=", list.id).orderBy("i.position").execute()).map((r) => [r.title, r.note]);
+    expect(await itemsOf()).toEqual([["Book 1", null], ["Zine", "rare"]]);
+    const rec = await db.k.selectFrom("source_records").selectAll().where("key", "=", "l1").executeTakeFirstOrThrow();
+    expect([rec.target_entity, rec.target_id]).toEqual(["list", list.id]);
+
+    // Source update, list never touched by the user: name and items applied. The ID-less item reuses its draft.
+    src.records = [listRec("l1", "Top", [{ work: draftHint, note: "rare" }, { work: book("2") }])];
+    await runSync(deps(db), plugin, connectionId, "full");
+    expect((await db.k.selectFrom("lists").select("name").where("id", "=", list.id).executeTakeFirstOrThrow()).name).toBe("Top");
+    expect(await itemsOf()).toEqual([["Zine", "rare"], ["Book 2", null]]);
+    expect(await db.k.selectFrom("works").select("id").where("owner_user_id", "=", userId).execute()).toHaveLength(1);
+    const applied = await events(db, { type: "changed", status: "auto_applied" });
+    expect(applied.map((e) => e.payload.field).sort()).toEqual(["items", "name"]);
+
+    // The user edits the list: further source changes become pending.
+    await updateList(db, userId, list.id, { name: "Mine" }, "user");
+    src.records = [listRec("l1", "Top 2", [{ work: book("3") }])];
+    const s3 = await runSync(deps(db), plugin, connectionId, "full");
+    expect(s3.counts.pending).toBe(2);
+    expect((await db.k.selectFrom("lists").select("name").where("id", "=", list.id).executeTakeFirstOrThrow()).name).toBe("Mine");
+    expect(await itemsOf()).toEqual([["Zine", "rare"], ["Book 2", null]]);
+    const pending = await events(db, { type: "changed", status: "pending" });
+    expect(pending.map((e) => e.payload.field).sort()).toEqual(["items", "name"]);
+    const itemsEv = pending.find((e) => e.payload.field === "items")!;
+    expect((itemsEv.payload.to as unknown[]).length).toBe(1);
+    await resolveChange(db, userId, itemsEv.id, "accept");
+    expect(await itemsOf()).toEqual([["Book 3", null]]);
+    const hist = await db.k.selectFrom("user_history").select("actor").where("entity_id", "=", list.id).orderBy("created_at", "desc").orderBy("id", "desc").executeTakeFirstOrThrow();
+    expect(hist.actor).toBe(actor);
+
+    // Removed → pending; accept soft-deletes the list.
+    src.records = [];
+    await runSync(deps(db), plugin, connectionId, "full");
+    const [removed] = await events(db, { type: "removed", status: "pending" });
+    expect(removed!.target_entity).toBe("list");
+    expect((await db.k.selectFrom("lists").select("deleted_at").where("id", "=", list.id).executeTakeFirstOrThrow()).deleted_at).toBeNull();
+    await resolveChange(db, userId, removed!.id, "accept");
+    expect((await db.k.selectFrom("lists").select("deleted_at").where("id", "=", list.id).executeTakeFirstOrThrow()).deleted_at).not.toBeNull();
+  });
+
   test("resolveChange is scoped to the user", async () => {
     const { db, userId, connectionId, src, plugin } = await setup();
     src.records = [entryRec("a", book("1"), { status: "completed" })];