From 4c08c16e488e622e8ae7cc252c8add0f20fe2d3b Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 15:51:22 +0100 Subject: [PATCH] fix(plugins): unique csv row keys, per-file default mediaType, scale-aware zero rating Co-Authored-By: Claude Sonnet 5.5 --- plugins/csv/index.ts | 27 +++++++++++++++++---------- plugins/csv/src/mapping.ts | 5 +++-- plugins/csv/test/csv.test.ts | 30 ++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/plugins/csv/index.ts b/plugins/csv/index.ts index a471e59..2452cdc 100644 --- a/plugins/csv/index.ts +++ b/plugins/csv/index.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { parseCsv } from "@sttracker/plugin-sdk/csv"; import { definePlugin } from "@sttracker/plugin-sdk/types"; -import type { EngagementFields, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types"; +import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types"; import { - cleanId, defaultScale, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows, + cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows, type FieldRow, } from "./src/mapping.ts"; @@ -29,17 +29,19 @@ function jsonRows(text: string): FieldRow[] { }); } -function* toRecords(rows: FieldRow[], rawRows: unknown[]): Generator { +function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator { const scale = defaultScale(rows); - const goodreadsLike = (r: FieldRow) => r.isbn13 !== undefined; + const seen = new Map(); for (const [i, r] of rows.entries()) { const title = str(r.title); if (!title) continue; - const mediaType = parseMediaType(str(r.mediaType), goodreadsLike(r) || r.author !== undefined ? "book" : "other"); + const mediaType = parseMediaType(str(r.mediaType), defaultMedia); const finished = parseDate(str(r.finishedAt)); const started = parseDate(str(r.startedAt)); const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex"); - const key = `row:${hash}`; + const n = (seen.get(hash) ?? 0) + 1; + seen.set(hash, n); + const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`; const externalIds: Record = {}; for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) { @@ -93,13 +95,18 @@ export default definePlugin({ const text = new TextDecoder("utf-8").decode(file.bytes); const lower = file.name.toLowerCase(); if (lower.endsWith(".json")) { - const rows = jsonRows(text.replace(/^/, "")); - yield* toRecords(rows, JSON.parse(text.replace(/^/, "")) as unknown[]); + const clean = text.replace(/^\uFEFF/, ""); + const rows = jsonRows(clean); + const raw = JSON.parse(clean) as unknown[]; + const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o)); + yield* toRecords(rows, raw, bookish ? "book" : "other"); } else if (lower.endsWith(".csv")) { const table = parseCsv(text); - const rows = rowsToFieldRows(table, parseMapping(ctx.config.mapping)); const header = table[0] ?? []; - yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""])))); + const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header); + const rows = rowsToFieldRows(table, mapping); + const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13"); + yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other"); } else { throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`); } diff --git a/plugins/csv/src/mapping.ts b/plugins/csv/src/mapping.ts index 3f95d4c..3f9d82d 100644 --- a/plugins/csv/src/mapping.ts +++ b/plugins/csv/src/mapping.ts @@ -115,13 +115,14 @@ export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 { return max <= 5 ? 5 : max <= 10 ? 10 : 100; } -/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated (Goodreads convention). */ +/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */ export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null { if (r.rating === undefined) return null; const n = Number(r.rating); - if (!Number.isFinite(n) || n <= 0) return null; + if (!Number.isFinite(n) || n < 0) return null; const rs = Number(r.ratingScale); const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale; + if (n === 0 && scale === 5) return null; return Math.max(0, Math.min(100, Math.round((n / scale) * 100))); } diff --git a/plugins/csv/test/csv.test.ts b/plugins/csv/test/csv.test.ts index 151c961..c074888 100644 --- a/plugins/csv/test/csv.test.ts +++ b/plugins/csv/test/csv.test.ts @@ -66,4 +66,34 @@ describe("csv plugin", () => { test("unsupported extension throws", async () => { await expect(run("x.txt", new Uint8Array())).rejects.toThrow(); }); + + const enc = (s: string) => new TextEncoder().encode(s); + test("duplicate rows get distinct deterministic keys", async () => { + const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n"; + const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key); + const k = await keys(); + expect(new Set(k).size).toBe(k.length); + expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/); + expect(k.some((x) => x.endsWith("#2"))).toBe(true); + expect(k.some((x) => x.endsWith("#3"))).toBe(true); + expect(await keys()).toEqual(k); + }); + + test("default mediaType is per file, not per row", async () => { + const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n"; + const e = entries(await run("g.csv", enc(csv))); + expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]); + const plain = entries(await run("p.csv", enc("Title\nA\n"))); + expect(plain[0]!.work.mediaType).toBe("other"); + const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]'))); + expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]); + }); + + test("rating 0 is unrated only on scale 5", async () => { + const m = JSON.stringify({ T: "title", R: "rating" }); + const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null); + expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]); + expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]); + expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]); + }); });