fix(plugins): unique csv row keys, per-file default mediaType, scale-aware zero rating

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

plugins/csv/index.ts

@@ -1,9 +1,9 @@
 import { createHash } from "node:crypto";
 import { parseCsv } from "@sttracker/plugin-sdk/csv";
 import { definePlugin } from "@sttracker/plugin-sdk/types";
-import type { EngagementFields, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
+import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
 import {
-  cleanId, defaultScale, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
+  cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
   type FieldRow,
 } from "./src/mapping.ts";
 
@@ -29,17 +29,19 @@ function jsonRows(text: string): FieldRow[] {
   });
 }
 
-function* toRecords(rows: FieldRow[], rawRows: unknown[]): Generator<SourceRecord> {
+function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator<SourceRecord> {
   const scale = defaultScale(rows);
-  const goodreadsLike = (r: FieldRow) => r.isbn13 !== undefined;
+  const seen = new Map<string, number>();
   for (const [i, r] of rows.entries()) {
     const title = str(r.title);
     if (!title) continue;
-    const mediaType = parseMediaType(str(r.mediaType), goodreadsLike(r) || r.author !== undefined ? "book" : "other");
+    const mediaType = parseMediaType(str(r.mediaType), defaultMedia);
     const finished = parseDate(str(r.finishedAt));
     const started = parseDate(str(r.startedAt));
     const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex");
-    const key = `row:${hash}`;
+    const n = (seen.get(hash) ?? 0) + 1;
+    seen.set(hash, n);
+    const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`;
 
     const externalIds: Record<string, string> = {};
     for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) {
@@ -93,13 +95,18 @@ export default definePlugin({
     const text = new TextDecoder("utf-8").decode(file.bytes);
     const lower = file.name.toLowerCase();
     if (lower.endsWith(".json")) {
-      const rows = jsonRows(text.replace(/^/, ""));
-      yield* toRecords(rows, JSON.parse(text.replace(/^/, "")) as unknown[]);
+      const clean = text.replace(/^\uFEFF/, "");
+      const rows = jsonRows(clean);
+      const raw = JSON.parse(clean) as unknown[];
+      const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o));
+      yield* toRecords(rows, raw, bookish ? "book" : "other");
     } else if (lower.endsWith(".csv")) {
       const table = parseCsv(text);
-      const rows = rowsToFieldRows(table, parseMapping(ctx.config.mapping));
       const header = table[0] ?? [];
-      yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))));
+      const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header);
+      const rows = rowsToFieldRows(table, mapping);
+      const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13");
+      yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other");
     } else {
       throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`);
     }

plugins/csv/src/mapping.ts

@@ -115,13 +115,14 @@ export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 {
   return max <= 5 ? 5 : max <= 10 ? 10 : 100;
 }
 
-/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated (Goodreads convention). */
+/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */
 export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null {
   if (r.rating === undefined) return null;
   const n = Number(r.rating);
-  if (!Number.isFinite(n) || n <= 0) return null;
+  if (!Number.isFinite(n) || n < 0) return null;
   const rs = Number(r.ratingScale);
   const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale;
+  if (n === 0 && scale === 5) return null;
   return Math.max(0, Math.min(100, Math.round((n / scale) * 100)));
 }
 

plugins/csv/test/csv.test.ts

@@ -66,4 +66,34 @@ describe("csv plugin", () => {
   test("unsupported extension throws", async () => {
     await expect(run("x.txt", new Uint8Array())).rejects.toThrow();
   });
+
+  const enc = (s: string) => new TextEncoder().encode(s);
+  test("duplicate rows get distinct deterministic keys", async () => {
+    const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n";
+    const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key);
+    const k = await keys();
+    expect(new Set(k).size).toBe(k.length);
+    expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/);
+    expect(k.some((x) => x.endsWith("#2"))).toBe(true);
+    expect(k.some((x) => x.endsWith("#3"))).toBe(true);
+    expect(await keys()).toEqual(k);
+  });
+
+  test("default mediaType is per file, not per row", async () => {
+    const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n";
+    const e = entries(await run("g.csv", enc(csv)));
+    expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]);
+    const plain = entries(await run("p.csv", enc("Title\nA\n")));
+    expect(plain[0]!.work.mediaType).toBe("other");
+    const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]')));
+    expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]);
+  });
+
+  test("rating 0 is unrated only on scale 5", async () => {
+    const m = JSON.stringify({ T: "title", R: "rating" });
+    const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null);
+    expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]);
+    expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]);
+    expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]);
+  });
 });