diff --git a/packages/plugin-sdk/src/csv.ts b/packages/plugin-sdk/src/csv.ts new file mode 100644 index 0000000..7e3ef1a --- /dev/null +++ b/packages/plugin-sdk/src/csv.ts @@ -0,0 +1,82 @@ +export type CsvDelimiter = "," | ";" | "\t"; + +function detectDelimiter(text: string): CsvDelimiter { + const counts: Record = { ",": 0, ";": 0, "\t": 0 }; + let inQuotes = false; + for (const ch of text) { + if (ch === '"') inQuotes = !inQuotes; + else if (!inQuotes) { + if (ch === "\n" || ch === "\r") break; + if (ch === "," || ch === ";" || ch === "\t") counts[ch]++; + } + } + let best: CsvDelimiter = ","; + for (const d of [";", "\t"] as const) if (counts[d] > counts[best]) best = d; + return best; +} + +/** RFC 4180 parser. Strips a BOM, auto-detects the delimiter from the header line, skips blank lines. */ +export function parseCsv(text: string, opts: { delimiter?: CsvDelimiter } = {}): string[][] { + if (text.charCodeAt(0) === 0xfeff) text = text.slice(1); + const delim = opts.delimiter ?? detectDelimiter(text); + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; // current field was quoted + let inQuotes = false; + let pending = false; // something has been read for the current row + + const endField = () => { + row.push(field); + field = ""; + }; + const endRow = () => { + const blank = row.length === 0 && field === "" && !quoted; + if (!blank) { + endField(); + rows.push(row); + } + row = []; + field = ""; + quoted = false; + pending = false; + }; + + for (let i = 0; i < text.length; i++) { + const ch = text[i]!; + if (inQuotes) { + if (ch === '"') { + if (text[i + 1] === '"') { + field += '"'; + i++; + } else inQuotes = false; + } else field += ch; + } else if (ch === '"' && field === "" && !quoted) { + inQuotes = true; + quoted = true; + pending = true; + } else if (ch === delim) { + endField(); + quoted = false; + pending = true; + } else if (ch === "\r" || ch === "\n") { + if (ch === "\r" && text[i + 1] === "\n") i++; + endRow(); + } else { + field += ch; + pending = true; + } + } + if (pending || field !== "" || row.length > 0) endRow(); + return rows; +} + +function quoteCell(s: string, single: boolean): string { + if (s === "" ? single : /[",;\t\r\n]/.test(s) || s !== s.trim()) return `"${s.replaceAll('"', '""')}"`; + return s; +} + +/** Serialises with `,` and `\r\n`, quoting only where needed. */ +export function toCsv(rows: string[][]): string { + return rows.map((r) => r.map((c) => quoteCell(c, r.length === 1)).join(",") + "\r\n").join(""); +} diff --git a/packages/plugin-sdk/test/csv.test.ts b/packages/plugin-sdk/test/csv.test.ts new file mode 100644 index 0000000..2f7d78f --- /dev/null +++ b/packages/plugin-sdk/test/csv.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from "bun:test"; +import { parseCsv, toCsv } from "../src/csv.ts"; + +describe("csv codec", () => { + test("round-trips awkward cells", () => { + const x = [["a", "b,c", 'q"uote', "line\nbreak", "cr\r\nlf", "", "🐉"], ["", "", "", "", "", "", ""], ["only"]]; + expect(parseCsv(toCsv(x))).toEqual(x); + }); + test("toCsv uses CRLF and quotes as needed", () => { + expect(toCsv([["a", "b c"], ["x,y", 'z"']])).toBe('a,b c\r\n"x,y","z"""\r\n'); + }); + test("strips BOM and detects semicolon / tab", () => { + expect(parseCsv("a;b\n1;2\n")).toEqual([["a", "b"], ["1", "2"]]); + expect(parseCsv("a\tb\n1\t2")).toEqual([["a", "b"], ["1", "2"]]); + }); + test("explicit delimiter wins; delimiters inside quoted header are ignored", () => { + expect(parseCsv('"a;b;c",d\n1,2', {})).toEqual([["a;b;c", "d"], ["1", "2"]]); + expect(parseCsv("a,b;c\n1,2;3", { delimiter: ";" })).toEqual([["a,b", "c"], ["1,2", "3"]]); + }); + test("handles CRLF, blank trailing lines, and empty input", () => { + expect(parseCsv("a,b\r\n1,2\r\n\r\n")).toEqual([["a", "b"], ["1", "2"]]); + expect(parseCsv("")).toEqual([]); + }); +}); diff --git a/plugins/csv/index.ts b/plugins/csv/index.ts new file mode 100644 index 0000000..2452cdc --- /dev/null +++ b/plugins/csv/index.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; +import { parseCsv } from "@sttracker/plugin-sdk/csv"; +import { definePlugin } from "@sttracker/plugin-sdk/types"; +import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types"; +import { + cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows, + type FieldRow, +} from "./src/mapping.ts"; + +const str = (v: string | string[] | undefined): string | undefined => (typeof v === "string" && v.trim() !== "" ? v.trim() : undefined); + +function jsonRows(text: string): FieldRow[] { + let v: unknown; + try { + v = JSON.parse(text); + } catch { + throw new Error("csv plugin: file is not valid JSON"); + } + if (!Array.isArray(v)) throw new Error("csv plugin: JSON must be an array of objects"); + return v.map((o) => { + const r: Record = {}; + if (typeof o === "object" && o !== null) { + for (const [k, x] of Object.entries(o)) { + if (x == null || x === "") continue; + r[k] = Array.isArray(x) ? x.map(String) : String(x); + } + } + return r as FieldRow; + }); +} + +function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator { + const scale = defaultScale(rows); + const seen = new Map(); + for (const [i, r] of rows.entries()) { + const title = str(r.title); + if (!title) continue; + const mediaType = parseMediaType(str(r.mediaType), defaultMedia); + const finished = parseDate(str(r.finishedAt)); + const started = parseDate(str(r.startedAt)); + const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex"); + const n = (seen.get(hash) ?? 0) + 1; + seen.set(hash, n); + const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`; + + const externalIds: Record = {}; + for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) { + const id = cleanId(r[f]); + if (id) externalIds[f] = id; + } + const author = str(r.author); + const work: WorkHint = { title, mediaType, externalIds, ...(author ? { creators: [{ name: author }] } : {}) }; + + const entry: EntryFields = {}; + const status = parseStatus(str(r.status)); + if (status) entry.status = status; + const rating = normalizeRating(r, scale); + if (rating !== null) entry.rating = rating; + const review = str(r.review); + if (review) entry.review = review; + const notes = str(r.notes); + if (notes) entry.notes = notes; + const tags = parseTags(r.tags); + if (tags) entry.tags = tags; + const raw = rawRows[i]; + yield { kind: "entry", key, raw, work, entry }; + + if (finished || started) { + const engagement: EngagementFields = {}; + if (started) engagement.startedAt = started; + if (finished) engagement.finishedAt = finished; + yield { kind: "engagement", key: `${key}:engagement`, raw, work, engagement }; + } + } +} + +export default definePlugin({ + manifest: { + id: "csv", + name: "CSV / JSON import", + version: "0.1.0", + sdkVersion: 1, + mediaTypes: ["book", "comic", "manga", "game", "movie", "show", "podcast", "audio_drama", "album", "video", "other"], + capabilities: ["importFile"], + config: [ + { + key: "mapping", + label: "Column mapping", + type: "string", + help: 'JSON { "column": "field" }. Fields: title mediaType status rating ratingScale(5|10|100) review finishedAt startedAt tags isbn13 imdb tmdb_movie igdb notes. Leave empty to guess from header names (Goodreads exports work out of the box). Ignored for .json.', + }, + ], + }, + async *importFile(ctx, file) { + const text = new TextDecoder("utf-8").decode(file.bytes); + const lower = file.name.toLowerCase(); + if (lower.endsWith(".json")) { + const clean = text.replace(/^\uFEFF/, ""); + const rows = jsonRows(clean); + const raw = JSON.parse(clean) as unknown[]; + const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o)); + yield* toRecords(rows, raw, bookish ? "book" : "other"); + } else if (lower.endsWith(".csv")) { + const table = parseCsv(text); + const header = table[0] ?? []; + const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header); + const rows = rowsToFieldRows(table, mapping); + const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13"); + yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other"); + } else { + throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`); + } + }, +}); diff --git a/plugins/csv/src/mapping.ts b/plugins/csv/src/mapping.ts new file mode 100644 index 0000000..3f9d82d --- /dev/null +++ b/plugins/csv/src/mapping.ts @@ -0,0 +1,140 @@ +import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date"; +import type { MediaType, PartialDate, Status } from "@sttracker/plugin-sdk/types"; + +export const FIELDS = [ + "title", "mediaType", "status", "rating", "ratingScale", "review", "finishedAt", "startedAt", + "tags", "isbn13", "imdb", "tmdb_movie", "igdb", "notes", "author", +] as const; +export type Field = (typeof FIELDS)[number]; +export type Mapping = Record; +/** A row keyed by target field name. */ +export type FieldRow = Partial>; + +const ALIASES: Record = { + title: ["title", "name"], + mediaType: ["mediatype", "media type", "type", "media"], + status: ["status", "exclusive shelf", "shelf", "state"], + rating: ["rating", "my rating", "score", "stars"], + ratingScale: ["ratingscale", "rating scale"], + review: ["review", "my review"], + finishedAt: ["finishedat", "finished", "date read", "date finished", "finished at", "date watched", "watched"], + startedAt: ["startedat", "started", "date started", "started at"], + tags: ["tags", "bookshelves", "labels", "genres"], + isbn13: ["isbn13", "isbn 13"], + imdb: ["imdb", "imdb id", "imdb_id"], + tmdb_movie: ["tmdb_movie", "tmdb", "tmdb id"], + igdb: ["igdb", "igdb id"], + notes: ["notes", "private notes", "comments"], + author: ["author", "authors", "creator"], +}; + +export function parseMapping(json: unknown): Mapping | null { + if (typeof json !== "string" || json.trim() === "") return null; + let v: unknown; + try { + v = JSON.parse(json); + } catch { + throw new Error("csv plugin: mapping is not valid JSON"); + } + if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error("csv plugin: mapping must be an object of { column: field }"); + const out: Mapping = {}; + for (const [col, f] of Object.entries(v)) { + if (!(FIELDS as readonly unknown[]).includes(f)) throw new Error(`csv plugin: unknown mapping field "${String(f)}" for column "${col}"`); + out[col] = f as Field; + } + return out; +} + +/** Guess a mapping from header names, case-insensitively (covers Goodreads exports). */ +export function guessMapping(headers: string[]): Mapping { + const out: Mapping = {}; + const used = new Set(); + for (const h of headers) { + const n = h.trim().toLowerCase(); + const f = FIELDS.find((x) => !used.has(x) && ALIASES[x].includes(n)); + if (f) { + out[h] = f; + used.add(f); + } + } + return out; +} + +export function rowsToFieldRows(rows: string[][], mapping: Mapping | null): FieldRow[] { + const [header, ...body] = rows; + if (!header) return []; + const m = mapping ?? guessMapping(header); + const idx = header.map((h) => m[h] ?? null); + return body.map((cells) => { + const r: FieldRow = {}; + idx.forEach((f, i) => { + const v = cells[i]; + if (f && v !== undefined && v.trim() !== "") r[f] = v.trim(); + }); + return r; + }); +} + +const MEDIA: MediaType[] = [ + "book", "comic", "manga", "game", "movie", "show", "season", "episode", "podcast", "podcast_episode", + "audio_drama", "artist", "album", "track", "video", "other", +]; +export function parseMediaType(v: string | undefined, fallback: MediaType): MediaType { + if (!v) return fallback; + const n = v.trim().toLowerCase().replaceAll(/[\s-]+/g, "_"); + return (MEDIA as string[]).includes(n) ? (n as MediaType) : fallback; +} + +const STATUS: Record = { + planned: "planned", "to-read": "planned", "to read": "planned", "want to read": "planned", "want to watch": "planned", "plan to watch": "planned", wishlist: "planned", backlog: "planned", + in_progress: "in_progress", "in progress": "in_progress", "currently-reading": "in_progress", "currently reading": "in_progress", reading: "in_progress", watching: "in_progress", playing: "in_progress", + paused: "paused", "on hold": "paused", "on-hold": "paused", + completed: "completed", read: "completed", finished: "completed", watched: "completed", played: "completed", done: "completed", complete: "completed", + dropped: "dropped", dnf: "dropped", abandoned: "dropped", + none: "none", +}; +export function parseStatus(v: string | undefined): Status | undefined { + if (!v) return undefined; + return STATUS[v.trim().toLowerCase()] ?? "none"; +} + +/** Goodreads writes `YYYY/MM/DD`; normalise to dashes. */ +export function parseDate(v: string | undefined): PartialDate | null { + if (!v) return null; + const s = v.trim(); + return parsePartialDate(/^\d{4}\/\d{2}\/\d{2}$/.test(s) ? s.replaceAll("/", "-") : s); +} + +/** Per-file default scale: 5 if max<=5, 10 if max<=10, else 100. */ +export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 { + let max = 0; + for (const r of rows) { + const n = Number(r.rating); + if (r.rating !== undefined && Number.isFinite(n) && n > max) max = n; + } + return max <= 5 ? 5 : max <= 10 ? 10 : 100; +} + +/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */ +export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null { + if (r.rating === undefined) return null; + const n = Number(r.rating); + if (!Number.isFinite(n) || n < 0) return null; + const rs = Number(r.ratingScale); + const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale; + if (n === 0 && scale === 5) return null; + return Math.max(0, Math.min(100, Math.round((n / scale) * 100))); +} + +/** Goodreads wraps ids as `="978..."`. */ +export function cleanId(v: string | string[] | undefined): string | undefined { + if (typeof v !== "string") return undefined; + const s = v.trim().replace(/^=?"?/, "").replace(/"?$/, "").trim(); + return s === "" ? undefined : s; +} + +export function parseTags(v: string | string[] | undefined): string[] | undefined { + const list = Array.isArray(v) ? v : typeof v === "string" ? v.split(/[,;]/) : []; + const t = list.map((x) => String(x).trim()).filter(Boolean); + return t.length ? t : undefined; +} diff --git a/plugins/csv/test/csv.test.ts b/plugins/csv/test/csv.test.ts new file mode 100644 index 0000000..c074888 --- /dev/null +++ b/plugins/csv/test/csv.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, test } from "bun:test"; +import { collect, createTestContext } from "@sttracker/plugin-sdk/testing"; +import type { SourceRecord } from "@sttracker/plugin-sdk/types"; +import plugin from "../index.ts"; + +const fixture = async (n: string) => new Uint8Array(await Bun.file(new URL(`./fixtures/${n}`, import.meta.url)).arrayBuffer()); +const run = async (name: string, bytes: Uint8Array, config: Record = {}) => + collect(plugin.importFile!(createTestContext({ config }), { name, bytes })); +const entries = (r: SourceRecord[]) => r.filter((x): x is Extract => x.kind === "entry"); + +describe("csv plugin", () => { + test("manifest", () => { + expect(plugin.manifest.id).toBe("csv"); + expect(plugin.manifest.capabilities).toEqual(["importFile"]); + expect(plugin.manifest.config.map((c) => c.key)).toEqual(["mapping"]); + }); + + test("messy.csv: BOM, ';', quoted newline and quotes, empty rating, year date, emoji", async () => { + const recs = await run("messy.csv", await fixture("messy.csv")); + const e = entries(recs); + expect(e.map((r) => r.work.title)).toEqual(['Line "one"', "🐉 Dragons"]); + expect(e[0]!.entry.review).toBe("multi\nline"); + expect(e[0]!.entry.rating).toBe(80); + expect(e[1]!.entry.rating ?? null).toBeNull(); + const eng = recs.filter((r) => r.kind === "engagement"); + expect(eng).toHaveLength(2); + expect((eng[0] as Extract).engagement.finishedAt).toEqual({ value: "2019", precision: "year" }); + expect(e[0]!.key).toMatch(/^row:[0-9a-f]{40}$/); + }); + + test("goodreads headers auto-mapped", async () => { + const e = entries(await run("goodreads.csv", await fixture("goodreads.csv"))); + expect(e.map((r) => r.entry.status)).toEqual(["completed", "planned", "in_progress"]); + expect(e[0]!.work.title).toBe("Dune"); + expect(e[0]!.work.mediaType).toBe("book"); + expect(e[0]!.work.creators).toEqual([{ name: "Frank Herbert" }]); + expect(e[0]!.work.externalIds).toEqual({ isbn13: "9780441172719" }); + expect(e[0]!.entry.rating).toBe(80); + expect(e[0]!.entry.tags).toEqual(["sci-fi", "classics"]); + expect(e[1]!.entry.rating ?? null).toBeNull(); + expect(e[1]!.work.externalIds).toEqual({}); + const eng = (await run("goodreads.csv", await fixture("goodreads.csv"))).filter((r) => r.kind === "engagement") as Extract[]; + expect(eng).toHaveLength(1); + expect(eng[0]!.engagement.finishedAt).toEqual({ value: "2021-03-04", precision: "day" }); + }); + + test("explicit mapping and rating scale default", async () => { + const csv = "Name,Score,Kind\nA,7,movie\nB,10,game\n"; + const mapping = JSON.stringify({ Name: "title", Score: "rating", Kind: "mediaType" }); + const e = entries(await run("x.csv", new TextEncoder().encode(csv), { mapping })); + expect(e.map((r) => r.entry.rating)).toEqual([70, 100]); + expect(e.map((r) => r.work.mediaType)).toEqual(["movie", "game"]); + }); + + test("json ignores mapping", async () => { + const json = JSON.stringify([{ title: "T", mediaType: "movie", rating: 3, status: "completed", finishedAt: "2020-01-02", tags: ["a"], imdb: "tt1" }]); + const recs = await run("x.json", new TextEncoder().encode(json), { mapping: '{"title":"notes"}' }); + const e = entries(recs); + expect(e[0]!.work.title).toBe("T"); + expect(e[0]!.entry.rating).toBe(60); + expect(e[0]!.entry.tags).toEqual(["a"]); + expect(e[0]!.work.externalIds).toEqual({ imdb: "tt1" }); + expect(recs).toHaveLength(2); + }); + + test("unsupported extension throws", async () => { + await expect(run("x.txt", new Uint8Array())).rejects.toThrow(); + }); + + const enc = (s: string) => new TextEncoder().encode(s); + test("duplicate rows get distinct deterministic keys", async () => { + const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n"; + const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key); + const k = await keys(); + expect(new Set(k).size).toBe(k.length); + expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/); + expect(k.some((x) => x.endsWith("#2"))).toBe(true); + expect(k.some((x) => x.endsWith("#3"))).toBe(true); + expect(await keys()).toEqual(k); + }); + + test("default mediaType is per file, not per row", async () => { + const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n"; + const e = entries(await run("g.csv", enc(csv))); + expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]); + const plain = entries(await run("p.csv", enc("Title\nA\n"))); + expect(plain[0]!.work.mediaType).toBe("other"); + const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]'))); + expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]); + }); + + test("rating 0 is unrated only on scale 5", async () => { + const m = JSON.stringify({ T: "title", R: "rating" }); + const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null); + expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]); + expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]); + expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]); + }); +}); diff --git a/plugins/csv/test/fixtures/goodreads.csv b/plugins/csv/test/fixtures/goodreads.csv new file mode 100644 index 0000000..c4c9279 --- /dev/null +++ b/plugins/csv/test/fixtures/goodreads.csv @@ -0,0 +1,4 @@ +Book Id,Title,Author,ISBN,ISBN13,My Rating,Date Read,Bookshelves,Exclusive Shelf,My Review +1,Dune,Frank Herbert,"=""0441172717""","=""9780441172719""",4,2021/03/04,"sci-fi, classics",read,Great +2,Neuromancer,William Gibson,"=""""","=""""",0,,,to-read, +3,Hyperion,Dan Simmons,"=""""","=""9780553283686""",0,,,currently-reading, diff --git a/plugins/csv/test/fixtures/messy.csv b/plugins/csv/test/fixtures/messy.csv new file mode 100644 index 0000000..8f870a3 --- /dev/null +++ b/plugins/csv/test/fixtures/messy.csv @@ -0,0 +1,4 @@ +Title;Rating;Date Read;Review +"Line ""one""";4;2019;"multi +line" +🐉 Dragons;;2020-05-06;