feat(plugins): csv and json import

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

plugins/csv/index.ts

@@ -0,0 +1,107 @@
+import { createHash } from "node:crypto";
+import { parseCsv } from "@sttracker/plugin-sdk/csv";
+import { definePlugin } from "@sttracker/plugin-sdk/types";
+import type { EngagementFields, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
+import {
+  cleanId, defaultScale, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
+  type FieldRow,
+} from "./src/mapping.ts";
+
+const str = (v: string | string[] | undefined): string | undefined => (typeof v === "string" && v.trim() !== "" ? v.trim() : undefined);
+
+function jsonRows(text: string): FieldRow[] {
+  let v: unknown;
+  try {
+    v = JSON.parse(text);
+  } catch {
+    throw new Error("csv plugin: file is not valid JSON");
+  }
+  if (!Array.isArray(v)) throw new Error("csv plugin: JSON must be an array of objects");
+  return v.map((o) => {
+    const r: Record<string, string | string[]> = {};
+    if (typeof o === "object" && o !== null) {
+      for (const [k, x] of Object.entries(o)) {
+        if (x == null || x === "") continue;
+        r[k] = Array.isArray(x) ? x.map(String) : String(x);
+      }
+    }
+    return r as FieldRow;
+  });
+}
+
+function* toRecords(rows: FieldRow[], rawRows: unknown[]): Generator<SourceRecord> {
+  const scale = defaultScale(rows);
+  const goodreadsLike = (r: FieldRow) => r.isbn13 !== undefined;
+  for (const [i, r] of rows.entries()) {
+    const title = str(r.title);
+    if (!title) continue;
+    const mediaType = parseMediaType(str(r.mediaType), goodreadsLike(r) || r.author !== undefined ? "book" : "other");
+    const finished = parseDate(str(r.finishedAt));
+    const started = parseDate(str(r.startedAt));
+    const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex");
+    const key = `row:${hash}`;
+
+    const externalIds: Record<string, string> = {};
+    for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) {
+      const id = cleanId(r[f]);
+      if (id) externalIds[f] = id;
+    }
+    const author = str(r.author);
+    const work: WorkHint = { title, mediaType, externalIds, ...(author ? { creators: [{ name: author }] } : {}) };
+
+    const entry: EntryFields = {};
+    const status = parseStatus(str(r.status));
+    if (status) entry.status = status;
+    const rating = normalizeRating(r, scale);
+    if (rating !== null) entry.rating = rating;
+    const review = str(r.review);
+    if (review) entry.review = review;
+    const notes = str(r.notes);
+    if (notes) entry.notes = notes;
+    const tags = parseTags(r.tags);
+    if (tags) entry.tags = tags;
+    const raw = rawRows[i];
+    yield { kind: "entry", key, raw, work, entry };
+
+    if (finished || started) {
+      const engagement: EngagementFields = {};
+      if (started) engagement.startedAt = started;
+      if (finished) engagement.finishedAt = finished;
+      yield { kind: "engagement", key: `${key}:engagement`, raw, work, engagement };
+    }
+  }
+}
+
+export default definePlugin({
+  manifest: {
+    id: "csv",
+    name: "CSV / JSON import",
+    version: "0.1.0",
+    sdkVersion: 1,
+    mediaTypes: ["book", "comic", "manga", "game", "movie", "show", "podcast", "audio_drama", "album", "video", "other"],
+    capabilities: ["importFile"],
+    config: [
+      {
+        key: "mapping",
+        label: "Column mapping",
+        type: "string",
+        help: 'JSON { "column": "field" }. Fields: title mediaType status rating ratingScale(5|10|100) review finishedAt startedAt tags isbn13 imdb tmdb_movie igdb notes. Leave empty to guess from header names (Goodreads exports work out of the box). Ignored for .json.',
+      },
+    ],
+  },
+  async *importFile(ctx, file) {
+    const text = new TextDecoder("utf-8").decode(file.bytes);
+    const lower = file.name.toLowerCase();
+    if (lower.endsWith(".json")) {
+      const rows = jsonRows(text.replace(/^/, ""));
+      yield* toRecords(rows, JSON.parse(text.replace(/^/, "")) as unknown[]);
+    } else if (lower.endsWith(".csv")) {
+      const table = parseCsv(text);
+      const rows = rowsToFieldRows(table, parseMapping(ctx.config.mapping));
+      const header = table[0] ?? [];
+      yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))));
+    } else {
+      throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`);
+    }
+  },
+});

plugins/csv/src/mapping.ts

@@ -0,0 +1,139 @@
+import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
+import type { MediaType, PartialDate, Status } from "@sttracker/plugin-sdk/types";
+
+export const FIELDS = [
+  "title", "mediaType", "status", "rating", "ratingScale", "review", "finishedAt", "startedAt",
+  "tags", "isbn13", "imdb", "tmdb_movie", "igdb", "notes", "author",
+] as const;
+export type Field = (typeof FIELDS)[number];
+export type Mapping = Record<string, Field>;
+/** A row keyed by target field name. */
+export type FieldRow = Partial<Record<Field, string | string[]>>;
+
+const ALIASES: Record<Field, string[]> = {
+  title: ["title", "name"],
+  mediaType: ["mediatype", "media type", "type", "media"],
+  status: ["status", "exclusive shelf", "shelf", "state"],
+  rating: ["rating", "my rating", "score", "stars"],
+  ratingScale: ["ratingscale", "rating scale"],
+  review: ["review", "my review"],
+  finishedAt: ["finishedat", "finished", "date read", "date finished", "finished at", "date watched", "watched"],
+  startedAt: ["startedat", "started", "date started", "started at"],
+  tags: ["tags", "bookshelves", "labels", "genres"],
+  isbn13: ["isbn13", "isbn 13"],
+  imdb: ["imdb", "imdb id", "imdb_id"],
+  tmdb_movie: ["tmdb_movie", "tmdb", "tmdb id"],
+  igdb: ["igdb", "igdb id"],
+  notes: ["notes", "private notes", "comments"],
+  author: ["author", "authors", "creator"],
+};
+
+export function parseMapping(json: unknown): Mapping | null {
+  if (typeof json !== "string" || json.trim() === "") return null;
+  let v: unknown;
+  try {
+    v = JSON.parse(json);
+  } catch {
+    throw new Error("csv plugin: mapping is not valid JSON");
+  }
+  if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error("csv plugin: mapping must be an object of { column: field }");
+  const out: Mapping = {};
+  for (const [col, f] of Object.entries(v)) {
+    if (!(FIELDS as readonly unknown[]).includes(f)) throw new Error(`csv plugin: unknown mapping field "${String(f)}" for column "${col}"`);
+    out[col] = f as Field;
+  }
+  return out;
+}
+
+/** Guess a mapping from header names, case-insensitively (covers Goodreads exports). */
+export function guessMapping(headers: string[]): Mapping {
+  const out: Mapping = {};
+  const used = new Set<Field>();
+  for (const h of headers) {
+    const n = h.trim().toLowerCase();
+    const f = FIELDS.find((x) => !used.has(x) && ALIASES[x].includes(n));
+    if (f) {
+      out[h] = f;
+      used.add(f);
+    }
+  }
+  return out;
+}
+
+export function rowsToFieldRows(rows: string[][], mapping: Mapping | null): FieldRow[] {
+  const [header, ...body] = rows;
+  if (!header) return [];
+  const m = mapping ?? guessMapping(header);
+  const idx = header.map((h) => m[h] ?? null);
+  return body.map((cells) => {
+    const r: FieldRow = {};
+    idx.forEach((f, i) => {
+      const v = cells[i];
+      if (f && v !== undefined && v.trim() !== "") r[f] = v.trim();
+    });
+    return r;
+  });
+}
+
+const MEDIA: MediaType[] = [
+  "book", "comic", "manga", "game", "movie", "show", "season", "episode", "podcast", "podcast_episode",
+  "audio_drama", "artist", "album", "track", "video", "other",
+];
+export function parseMediaType(v: string | undefined, fallback: MediaType): MediaType {
+  if (!v) return fallback;
+  const n = v.trim().toLowerCase().replaceAll(/[\s-]+/g, "_");
+  return (MEDIA as string[]).includes(n) ? (n as MediaType) : fallback;
+}
+
+const STATUS: Record<string, Status> = {
+  planned: "planned", "to-read": "planned", "to read": "planned", "want to read": "planned", "want to watch": "planned", "plan to watch": "planned", wishlist: "planned", backlog: "planned",
+  in_progress: "in_progress", "in progress": "in_progress", "currently-reading": "in_progress", "currently reading": "in_progress", reading: "in_progress", watching: "in_progress", playing: "in_progress",
+  paused: "paused", "on hold": "paused", "on-hold": "paused",
+  completed: "completed", read: "completed", finished: "completed", watched: "completed", played: "completed", done: "completed", complete: "completed",
+  dropped: "dropped", dnf: "dropped", abandoned: "dropped",
+  none: "none",
+};
+export function parseStatus(v: string | undefined): Status | undefined {
+  if (!v) return undefined;
+  return STATUS[v.trim().toLowerCase()] ?? "none";
+}
+
+/** Goodreads writes `YYYY/MM/DD`; normalise to dashes. */
+export function parseDate(v: string | undefined): PartialDate | null {
+  if (!v) return null;
+  const s = v.trim();
+  return parsePartialDate(/^\d{4}\/\d{2}\/\d{2}$/.test(s) ? s.replaceAll("/", "-") : s);
+}
+
+/** Per-file default scale: 5 if max<=5, 10 if max<=10, else 100. */
+export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 {
+  let max = 0;
+  for (const r of rows) {
+    const n = Number(r.rating);
+    if (r.rating !== undefined && Number.isFinite(n) && n > max) max = n;
+  }
+  return max <= 5 ? 5 : max <= 10 ? 10 : 100;
+}
+
+/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated (Goodreads convention). */
+export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null {
+  if (r.rating === undefined) return null;
+  const n = Number(r.rating);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  const rs = Number(r.ratingScale);
+  const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale;
+  return Math.max(0, Math.min(100, Math.round((n / scale) * 100)));
+}
+
+/** Goodreads wraps ids as `="978..."`. */
+export function cleanId(v: string | string[] | undefined): string | undefined {
+  if (typeof v !== "string") return undefined;
+  const s = v.trim().replace(/^=?"?/, "").replace(/"?$/, "").trim();
+  return s === "" ? undefined : s;
+}
+
+export function parseTags(v: string | string[] | undefined): string[] | undefined {
+  const list = Array.isArray(v) ? v : typeof v === "string" ? v.split(/[,;]/) : [];
+  const t = list.map((x) => String(x).trim()).filter(Boolean);
+  return t.length ? t : undefined;
+}

plugins/csv/test/csv.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from "bun:test";
+import { collect, createTestContext } from "@sttracker/plugin-sdk/testing";
+import type { SourceRecord } from "@sttracker/plugin-sdk/types";
+import plugin from "../index.ts";
+
+const fixture = async (n: string) => new Uint8Array(await Bun.file(new URL(`./fixtures/${n}`, import.meta.url)).arrayBuffer());
+const run = async (name: string, bytes: Uint8Array, config: Record<string, unknown> = {}) =>
+  collect(plugin.importFile!(createTestContext({ config }), { name, bytes }));
+const entries = (r: SourceRecord[]) => r.filter((x): x is Extract<SourceRecord, { kind: "entry" }> => x.kind === "entry");
+
+describe("csv plugin", () => {
+  test("manifest", () => {
+    expect(plugin.manifest.id).toBe("csv");
+    expect(plugin.manifest.capabilities).toEqual(["importFile"]);
+    expect(plugin.manifest.config.map((c) => c.key)).toEqual(["mapping"]);
+  });
+
+  test("messy.csv: BOM, ';', quoted newline and quotes, empty rating, year date, emoji", async () => {
+    const recs = await run("messy.csv", await fixture("messy.csv"));
+    const e = entries(recs);
+    expect(e.map((r) => r.work.title)).toEqual(['Line "one"', "🐉 Dragons"]);
+    expect(e[0]!.entry.review).toBe("multi\nline");
+    expect(e[0]!.entry.rating).toBe(80);
+    expect(e[1]!.entry.rating ?? null).toBeNull();
+    const eng = recs.filter((r) => r.kind === "engagement");
+    expect(eng).toHaveLength(2);
+    expect((eng[0] as Extract<SourceRecord, { kind: "engagement" }>).engagement.finishedAt).toEqual({ value: "2019", precision: "year" });
+    expect(e[0]!.key).toMatch(/^row:[0-9a-f]{40}$/);
+  });
+
+  test("goodreads headers auto-mapped", async () => {
+    const e = entries(await run("goodreads.csv", await fixture("goodreads.csv")));
+    expect(e.map((r) => r.entry.status)).toEqual(["completed", "planned", "in_progress"]);
+    expect(e[0]!.work.title).toBe("Dune");
+    expect(e[0]!.work.mediaType).toBe("book");
+    expect(e[0]!.work.creators).toEqual([{ name: "Frank Herbert" }]);
+    expect(e[0]!.work.externalIds).toEqual({ isbn13: "9780441172719" });
+    expect(e[0]!.entry.rating).toBe(80);
+    expect(e[0]!.entry.tags).toEqual(["sci-fi", "classics"]);
+    expect(e[1]!.entry.rating ?? null).toBeNull();
+    expect(e[1]!.work.externalIds).toEqual({});
+    const eng = (await run("goodreads.csv", await fixture("goodreads.csv"))).filter((r) => r.kind === "engagement") as Extract<SourceRecord, { kind: "engagement" }>[];
+    expect(eng).toHaveLength(1);
+    expect(eng[0]!.engagement.finishedAt).toEqual({ value: "2021-03-04", precision: "day" });
+  });
+
+  test("explicit mapping and rating scale default", async () => {
+    const csv = "Name,Score,Kind\nA,7,movie\nB,10,game\n";
+    const mapping = JSON.stringify({ Name: "title", Score: "rating", Kind: "mediaType" });
+    const e = entries(await run("x.csv", new TextEncoder().encode(csv), { mapping }));
+    expect(e.map((r) => r.entry.rating)).toEqual([70, 100]);
+    expect(e.map((r) => r.work.mediaType)).toEqual(["movie", "game"]);
+  });
+
+  test("json ignores mapping", async () => {
+    const json = JSON.stringify([{ title: "T", mediaType: "movie", rating: 3, status: "completed", finishedAt: "2020-01-02", tags: ["a"], imdb: "tt1" }]);
+    const recs = await run("x.json", new TextEncoder().encode(json), { mapping: '{"title":"notes"}' });
+    const e = entries(recs);
+    expect(e[0]!.work.title).toBe("T");
+    expect(e[0]!.entry.rating).toBe(60);
+    expect(e[0]!.entry.tags).toEqual(["a"]);
+    expect(e[0]!.work.externalIds).toEqual({ imdb: "tt1" });
+    expect(recs).toHaveLength(2);
+  });
+
+  test("unsupported extension throws", async () => {
+    await expect(run("x.txt", new Uint8Array())).rejects.toThrow();
+  });
+});

plugins/csv/test/fixtures/goodreads.csv

@@ -0,0 +1,4 @@
+Book Id,Title,Author,ISBN,ISBN13,My Rating,Date Read,Bookshelves,Exclusive Shelf,My Review
+1,Dune,Frank Herbert,"=""0441172717""","=""9780441172719""",4,2021/03/04,"sci-fi, classics",read,Great
+2,Neuromancer,William Gibson,"=""""","=""""",0,,,to-read,
+3,Hyperion,Dan Simmons,"=""""","=""9780553283686""",0,,,currently-reading,

plugins/csv/test/fixtures/messy.csv

@@ -0,0 +1,4 @@
+Title;Rating;Date Read;Review
+"Line ""one""";4;2019;"multi
+line"
+🐉 Dragons;;2020-05-06;