merge: task 13

Co-Authored-By: Claude Opus 5.5 <[email protected]>

packages/plugin-sdk/src/csv.ts

@@ -0,0 +1,82 @@
+export type CsvDelimiter = "," | ";" | "\t";
+
+function detectDelimiter(text: string): CsvDelimiter {
+  const counts: Record<CsvDelimiter, number> = { ",": 0, ";": 0, "\t": 0 };
+  let inQuotes = false;
+  for (const ch of text) {
+    if (ch === '"') inQuotes = !inQuotes;
+    else if (!inQuotes) {
+      if (ch === "\n" || ch === "\r") break;
+      if (ch === "," || ch === ";" || ch === "\t") counts[ch]++;
+    }
+  }
+  let best: CsvDelimiter = ",";
+  for (const d of [";", "\t"] as const) if (counts[d] > counts[best]) best = d;
+  return best;
+}
+
+/** RFC 4180 parser. Strips a BOM, auto-detects the delimiter from the header line, skips blank lines. */
+export function parseCsv(text: string, opts: { delimiter?: CsvDelimiter } = {}): string[][] {
+  if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
+  const delim = opts.delimiter ?? detectDelimiter(text);
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = "";
+  let quoted = false; // current field was quoted
+  let inQuotes = false;
+  let pending = false; // something has been read for the current row
+
+  const endField = () => {
+    row.push(field);
+    field = "";
+  };
+  const endRow = () => {
+    const blank = row.length === 0 && field === "" && !quoted;
+    if (!blank) {
+      endField();
+      rows.push(row);
+    }
+    row = [];
+    field = "";
+    quoted = false;
+    pending = false;
+  };
+
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]!;
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i++;
+        } else inQuotes = false;
+      } else field += ch;
+    } else if (ch === '"' && field === "" && !quoted) {
+      inQuotes = true;
+      quoted = true;
+      pending = true;
+    } else if (ch === delim) {
+      endField();
+      quoted = false;
+      pending = true;
+    } else if (ch === "\r" || ch === "\n") {
+      if (ch === "\r" && text[i + 1] === "\n") i++;
+      endRow();
+    } else {
+      field += ch;
+      pending = true;
+    }
+  }
+  if (pending || field !== "" || row.length > 0) endRow();
+  return rows;
+}
+
+function quoteCell(s: string, single: boolean): string {
+  if (s === "" ? single : /[",;\t\r\n]/.test(s) || s !== s.trim()) return `"${s.replaceAll('"', '""')}"`;
+  return s;
+}
+
+/** Serialises with `,` and `\r\n`, quoting only where needed. */
+export function toCsv(rows: string[][]): string {
+  return rows.map((r) => r.map((c) => quoteCell(c, r.length === 1)).join(",") + "\r\n").join("");
+}

packages/plugin-sdk/test/csv.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, test } from "bun:test";
+import { parseCsv, toCsv } from "../src/csv.ts";
+
+describe("csv codec", () => {
+  test("round-trips awkward cells", () => {
+    const x = [["a", "b,c", 'q"uote', "line\nbreak", "cr\r\nlf", "", "🐉"], ["", "", "", "", "", "", ""], ["only"]];
+    expect(parseCsv(toCsv(x))).toEqual(x);
+  });
+  test("toCsv uses CRLF and quotes as needed", () => {
+    expect(toCsv([["a", "b c"], ["x,y", 'z"']])).toBe('a,b c\r\n"x,y","z"""\r\n');
+  });
+  test("strips BOM and detects semicolon / tab", () => {
+    expect(parseCsv("a;b\n1;2\n")).toEqual([["a", "b"], ["1", "2"]]);
+    expect(parseCsv("a\tb\n1\t2")).toEqual([["a", "b"], ["1", "2"]]);
+  });
+  test("explicit delimiter wins; delimiters inside quoted header are ignored", () => {
+    expect(parseCsv('"a;b;c",d\n1,2', {})).toEqual([["a;b;c", "d"], ["1", "2"]]);
+    expect(parseCsv("a,b;c\n1,2;3", { delimiter: ";" })).toEqual([["a,b", "c"], ["1,2", "3"]]);
+  });
+  test("handles CRLF, blank trailing lines, and empty input", () => {
+    expect(parseCsv("a,b\r\n1,2\r\n\r\n")).toEqual([["a", "b"], ["1", "2"]]);
+    expect(parseCsv("")).toEqual([]);
+  });
+});

plugins/csv/index.ts

@@ -0,0 +1,114 @@
+import { createHash } from "node:crypto";
+import { parseCsv } from "@sttracker/plugin-sdk/csv";
+import { definePlugin } from "@sttracker/plugin-sdk/types";
+import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
+import {
+  cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
+  type FieldRow,
+} from "./src/mapping.ts";
+
+const str = (v: string | string[] | undefined): string | undefined => (typeof v === "string" && v.trim() !== "" ? v.trim() : undefined);
+
+function jsonRows(text: string): FieldRow[] {
+  let v: unknown;
+  try {
+    v = JSON.parse(text);
+  } catch {
+    throw new Error("csv plugin: file is not valid JSON");
+  }
+  if (!Array.isArray(v)) throw new Error("csv plugin: JSON must be an array of objects");
+  return v.map((o) => {
+    const r: Record<string, string | string[]> = {};
+    if (typeof o === "object" && o !== null) {
+      for (const [k, x] of Object.entries(o)) {
+        if (x == null || x === "") continue;
+        r[k] = Array.isArray(x) ? x.map(String) : String(x);
+      }
+    }
+    return r as FieldRow;
+  });
+}
+
+function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator<SourceRecord> {
+  const scale = defaultScale(rows);
+  const seen = new Map<string, number>();
+  for (const [i, r] of rows.entries()) {
+    const title = str(r.title);
+    if (!title) continue;
+    const mediaType = parseMediaType(str(r.mediaType), defaultMedia);
+    const finished = parseDate(str(r.finishedAt));
+    const started = parseDate(str(r.startedAt));
+    const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex");
+    const n = (seen.get(hash) ?? 0) + 1;
+    seen.set(hash, n);
+    const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`;
+
+    const externalIds: Record<string, string> = {};
+    for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) {
+      const id = cleanId(r[f]);
+      if (id) externalIds[f] = id;
+    }
+    const author = str(r.author);
+    const work: WorkHint = { title, mediaType, externalIds, ...(author ? { creators: [{ name: author }] } : {}) };
+
+    const entry: EntryFields = {};
+    const status = parseStatus(str(r.status));
+    if (status) entry.status = status;
+    const rating = normalizeRating(r, scale);
+    if (rating !== null) entry.rating = rating;
+    const review = str(r.review);
+    if (review) entry.review = review;
+    const notes = str(r.notes);
+    if (notes) entry.notes = notes;
+    const tags = parseTags(r.tags);
+    if (tags) entry.tags = tags;
+    const raw = rawRows[i];
+    yield { kind: "entry", key, raw, work, entry };
+
+    if (finished || started) {
+      const engagement: EngagementFields = {};
+      if (started) engagement.startedAt = started;
+      if (finished) engagement.finishedAt = finished;
+      yield { kind: "engagement", key: `${key}:engagement`, raw, work, engagement };
+    }
+  }
+}
+
+export default definePlugin({
+  manifest: {
+    id: "csv",
+    name: "CSV / JSON import",
+    version: "0.1.0",
+    sdkVersion: 1,
+    mediaTypes: ["book", "comic", "manga", "game", "movie", "show", "podcast", "audio_drama", "album", "video", "other"],
+    capabilities: ["importFile"],
+    config: [
+      {
+        key: "mapping",
+        label: "Column mapping",
+        type: "string",
+        help: 'JSON { "column": "field" }. Fields: title mediaType status rating ratingScale(5|10|100) review finishedAt startedAt tags isbn13 imdb tmdb_movie igdb notes. Leave empty to guess from header names (Goodreads exports work out of the box). Ignored for .json.',
+      },
+    ],
+  },
+  async *importFile(ctx, file) {
+    const text = new TextDecoder("utf-8").decode(file.bytes);
+    const lower = file.name.toLowerCase();
+    if (lower.endsWith(".json")) {
+      const clean = text.replace(/^\uFEFF/, "");
+      const rows = jsonRows(clean);
+      const raw = JSON.parse(clean) as unknown[];
+      const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o));
+      yield* toRecords(rows, raw, bookish ? "book" : "other");
+    } else if (lower.endsWith(".csv")) {
+      const table = parseCsv(text);
+      const header = table[0] ?? [];
+      const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header);
+      const rows = rowsToFieldRows(table, mapping);
+      const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13");
+      yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other");
+    } else {
+      throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`);
+    }
+  },
+});

plugins/csv/src/mapping.ts

@@ -0,0 +1,140 @@
+import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
+import type { MediaType, PartialDate, Status } from "@sttracker/plugin-sdk/types";
+
+export const FIELDS = [
+  "title", "mediaType", "status", "rating", "ratingScale", "review", "finishedAt", "startedAt",
+  "tags", "isbn13", "imdb", "tmdb_movie", "igdb", "notes", "author",
+] as const;
+export type Field = (typeof FIELDS)[number];
+export type Mapping = Record<string, Field>;
+/** A row keyed by target field name. */
+export type FieldRow = Partial<Record<Field, string | string[]>>;
+
+const ALIASES: Record<Field, string[]> = {
+  title: ["title", "name"],
+  mediaType: ["mediatype", "media type", "type", "media"],
+  status: ["status", "exclusive shelf", "shelf", "state"],
+  rating: ["rating", "my rating", "score", "stars"],
+  ratingScale: ["ratingscale", "rating scale"],
+  review: ["review", "my review"],
+  finishedAt: ["finishedat", "finished", "date read", "date finished", "finished at", "date watched", "watched"],
+  startedAt: ["startedat", "started", "date started", "started at"],
+  tags: ["tags", "bookshelves", "labels", "genres"],
+  isbn13: ["isbn13", "isbn 13"],
+  imdb: ["imdb", "imdb id", "imdb_id"],
+  tmdb_movie: ["tmdb_movie", "tmdb", "tmdb id"],
+  igdb: ["igdb", "igdb id"],
+  notes: ["notes", "private notes", "comments"],
+  author: ["author", "authors", "creator"],
+};
+
+export function parseMapping(json: unknown): Mapping | null {
+  if (typeof json !== "string" || json.trim() === "") return null;
+  let v: unknown;
+  try {
+    v = JSON.parse(json);
+  } catch {
+    throw new Error("csv plugin: mapping is not valid JSON");
+  }
+  if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error("csv plugin: mapping must be an object of { column: field }");
+  const out: Mapping = {};
+  for (const [col, f] of Object.entries(v)) {
+    if (!(FIELDS as readonly unknown[]).includes(f)) throw new Error(`csv plugin: unknown mapping field "${String(f)}" for column "${col}"`);
+    out[col] = f as Field;
+  }
+  return out;
+}
+
+/** Guess a mapping from header names, case-insensitively (covers Goodreads exports). */
+export function guessMapping(headers: string[]): Mapping {
+  const out: Mapping = {};
+  const used = new Set<Field>();
+  for (const h of headers) {
+    const n = h.trim().toLowerCase();
+    const f = FIELDS.find((x) => !used.has(x) && ALIASES[x].includes(n));
+    if (f) {
+      out[h] = f;
+      used.add(f);
+    }
+  }
+  return out;
+}
+
+export function rowsToFieldRows(rows: string[][], mapping: Mapping | null): FieldRow[] {
+  const [header, ...body] = rows;
+  if (!header) return [];
+  const m = mapping ?? guessMapping(header);
+  const idx = header.map((h) => m[h] ?? null);
+  return body.map((cells) => {
+    const r: FieldRow = {};
+    idx.forEach((f, i) => {
+      const v = cells[i];
+      if (f && v !== undefined && v.trim() !== "") r[f] = v.trim();
+    });
+    return r;
+  });
+}
+
+const MEDIA: MediaType[] = [
+  "book", "comic", "manga", "game", "movie", "show", "season", "episode", "podcast", "podcast_episode",
+  "audio_drama", "artist", "album", "track", "video", "other",
+];
+export function parseMediaType(v: string | undefined, fallback: MediaType): MediaType {
+  if (!v) return fallback;
+  const n = v.trim().toLowerCase().replaceAll(/[\s-]+/g, "_");
+  return (MEDIA as string[]).includes(n) ? (n as MediaType) : fallback;
+}
+
+const STATUS: Record<string, Status> = {
+  planned: "planned", "to-read": "planned", "to read": "planned", "want to read": "planned", "want to watch": "planned", "plan to watch": "planned", wishlist: "planned", backlog: "planned",
+  in_progress: "in_progress", "in progress": "in_progress", "currently-reading": "in_progress", "currently reading": "in_progress", reading: "in_progress", watching: "in_progress", playing: "in_progress",
+  paused: "paused", "on hold": "paused", "on-hold": "paused",
+  completed: "completed", read: "completed", finished: "completed", watched: "completed", played: "completed", done: "completed", complete: "completed",
+  dropped: "dropped", dnf: "dropped", abandoned: "dropped",
+  none: "none",
+};
+export function parseStatus(v: string | undefined): Status | undefined {
+  if (!v) return undefined;
+  return STATUS[v.trim().toLowerCase()] ?? "none";
+}
+
+/** Goodreads writes `YYYY/MM/DD`; normalise to dashes. */
+export function parseDate(v: string | undefined): PartialDate | null {
+  if (!v) return null;
+  const s = v.trim();
+  return parsePartialDate(/^\d{4}\/\d{2}\/\d{2}$/.test(s) ? s.replaceAll("/", "-") : s);
+}
+
+/** Per-file default scale: 5 if max<=5, 10 if max<=10, else 100. */
+export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 {
+  let max = 0;
+  for (const r of rows) {
+    const n = Number(r.rating);
+    if (r.rating !== undefined && Number.isFinite(n) && n > max) max = n;
+  }
+  return max <= 5 ? 5 : max <= 10 ? 10 : 100;
+}
+
+/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */
+export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null {
+  if (r.rating === undefined) return null;
+  const n = Number(r.rating);
+  if (!Number.isFinite(n) || n < 0) return null;
+  const rs = Number(r.ratingScale);
+  const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale;
+  if (n === 0 && scale === 5) return null;
+  return Math.max(0, Math.min(100, Math.round((n / scale) * 100)));
+}
+
+/** Goodreads wraps ids as `="978..."`. */
+export function cleanId(v: string | string[] | undefined): string | undefined {
+  if (typeof v !== "string") return undefined;
+  const s = v.trim().replace(/^=?"?/, "").replace(/"?$/, "").trim();
+  return s === "" ? undefined : s;
+}
+
+export function parseTags(v: string | string[] | undefined): string[] | undefined {
+  const list = Array.isArray(v) ? v : typeof v === "string" ? v.split(/[,;]/) : [];
+  const t = list.map((x) => String(x).trim()).filter(Boolean);
+  return t.length ? t : undefined;
+}

plugins/csv/test/csv.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, test } from "bun:test";
+import { collect, createTestContext } from "@sttracker/plugin-sdk/testing";
+import type { SourceRecord } from "@sttracker/plugin-sdk/types";
+import plugin from "../index.ts";
+
+const fixture = async (n: string) => new Uint8Array(await Bun.file(new URL(`./fixtures/${n}`, import.meta.url)).arrayBuffer());
+const run = async (name: string, bytes: Uint8Array, config: Record<string, unknown> = {}) =>
+  collect(plugin.importFile!(createTestContext({ config }), { name, bytes }));
+const entries = (r: SourceRecord[]) => r.filter((x): x is Extract<SourceRecord, { kind: "entry" }> => x.kind === "entry");
+
+describe("csv plugin", () => {
+  test("manifest", () => {
+    expect(plugin.manifest.id).toBe("csv");
+    expect(plugin.manifest.capabilities).toEqual(["importFile"]);
+    expect(plugin.manifest.config.map((c) => c.key)).toEqual(["mapping"]);
+  });
+
+  test("messy.csv: BOM, ';', quoted newline and quotes, empty rating, year date, emoji", async () => {
+    const recs = await run("messy.csv", await fixture("messy.csv"));
+    const e = entries(recs);
+    expect(e.map((r) => r.work.title)).toEqual(['Line "one"', "🐉 Dragons"]);
+    expect(e[0]!.entry.review).toBe("multi\nline");
+    expect(e[0]!.entry.rating).toBe(80);
+    expect(e[1]!.entry.rating ?? null).toBeNull();
+    const eng = recs.filter((r) => r.kind === "engagement");
+    expect(eng).toHaveLength(2);
+    expect((eng[0] as Extract<SourceRecord, { kind: "engagement" }>).engagement.finishedAt).toEqual({ value: "2019", precision: "year" });
+    expect(e[0]!.key).toMatch(/^row:[0-9a-f]{40}$/);
+  });
+
+  test("goodreads headers auto-mapped", async () => {
+    const e = entries(await run("goodreads.csv", await fixture("goodreads.csv")));
+    expect(e.map((r) => r.entry.status)).toEqual(["completed", "planned", "in_progress"]);
+    expect(e[0]!.work.title).toBe("Dune");
+    expect(e[0]!.work.mediaType).toBe("book");
+    expect(e[0]!.work.creators).toEqual([{ name: "Frank Herbert" }]);
+    expect(e[0]!.work.externalIds).toEqual({ isbn13: "9780441172719" });
+    expect(e[0]!.entry.rating).toBe(80);
+    expect(e[0]!.entry.tags).toEqual(["sci-fi", "classics"]);
+    expect(e[1]!.entry.rating ?? null).toBeNull();
+    expect(e[1]!.work.externalIds).toEqual({});
+    const eng = (await run("goodreads.csv", await fixture("goodreads.csv"))).filter((r) => r.kind === "engagement") as Extract<SourceRecord, { kind: "engagement" }>[];
+    expect(eng).toHaveLength(1);
+    expect(eng[0]!.engagement.finishedAt).toEqual({ value: "2021-03-04", precision: "day" });
+  });
+
+  test("explicit mapping and rating scale default", async () => {
+    const csv = "Name,Score,Kind\nA,7,movie\nB,10,game\n";
+    const mapping = JSON.stringify({ Name: "title", Score: "rating", Kind: "mediaType" });
+    const e = entries(await run("x.csv", new TextEncoder().encode(csv), { mapping }));
+    expect(e.map((r) => r.entry.rating)).toEqual([70, 100]);
+    expect(e.map((r) => r.work.mediaType)).toEqual(["movie", "game"]);
+  });
+
+  test("json ignores mapping", async () => {
+    const json = JSON.stringify([{ title: "T", mediaType: "movie", rating: 3, status: "completed", finishedAt: "2020-01-02", tags: ["a"], imdb: "tt1" }]);
+    const recs = await run("x.json", new TextEncoder().encode(json), { mapping: '{"title":"notes"}' });
+    const e = entries(recs);
+    expect(e[0]!.work.title).toBe("T");
+    expect(e[0]!.entry.rating).toBe(60);
+    expect(e[0]!.entry.tags).toEqual(["a"]);
+    expect(e[0]!.work.externalIds).toEqual({ imdb: "tt1" });
+    expect(recs).toHaveLength(2);
+  });
+
+  test("unsupported extension throws", async () => {
+    await expect(run("x.txt", new Uint8Array())).rejects.toThrow();
+  });
+
+  const enc = (s: string) => new TextEncoder().encode(s);
+  test("duplicate rows get distinct deterministic keys", async () => {
+    const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n";
+    const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key);
+    const k = await keys();
+    expect(new Set(k).size).toBe(k.length);
+    expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/);
+    expect(k.some((x) => x.endsWith("#2"))).toBe(true);
+    expect(k.some((x) => x.endsWith("#3"))).toBe(true);
+    expect(await keys()).toEqual(k);
+  });
+
+  test("default mediaType is per file, not per row", async () => {
+    const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n";
+    const e = entries(await run("g.csv", enc(csv)));
+    expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]);
+    const plain = entries(await run("p.csv", enc("Title\nA\n")));
+    expect(plain[0]!.work.mediaType).toBe("other");
+    const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]')));
+    expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]);
+  });
+
+  test("rating 0 is unrated only on scale 5", async () => {
+    const m = JSON.stringify({ T: "title", R: "rating" });
+    const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null);
+    expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]);
+    expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]);
+    expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]);
+  });
+});

plugins/csv/test/fixtures/goodreads.csv

@@ -0,0 +1,4 @@
+Book Id,Title,Author,ISBN,ISBN13,My Rating,Date Read,Bookshelves,Exclusive Shelf,My Review
+1,Dune,Frank Herbert,"=""0441172717""","=""9780441172719""",4,2021/03/04,"sci-fi, classics",read,Great
+2,Neuromancer,William Gibson,"=""""","=""""",0,,,to-read,
+3,Hyperion,Dan Simmons,"=""""","=""9780553283686""",0,,,currently-reading,

plugins/csv/test/fixtures/messy.csv

@@ -0,0 +1,4 @@
+Title;Rating;Date Read;Review
+"Line ""one""";4;2019;"multi
+line"
+🐉 Dragons;;2020-05-06;