diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 4c69ff3..74d903d 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -4,6 +4,26 @@ Append-only. Never edit or delete an entry. Supersede it with a new one that lin One entry per decision a future maintainer would otherwise have to reverse-engineer. No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry (date, title, line number). +## 2026-10-30: Export format `sttracker-export/v1`: per-user zip, camelCase JSONL, deleted rows included, no secrets +**Status:** accepted +**Context:** Task 15 (`packages/core/src/{export,backup}.ts`) fixed the on-disk bundle format that backups and restore share. +**Decision:** +- **Files.** A zip (`fflate`) holding `manifest.json` (`{ format, exportedAt, user: { username, displayName } }`), + `works.jsonl`, `entries.jsonl`, `engagements.jsonl`, `listens.jsonl`, `lists.jsonl` (items nested), + `copies.jsonl`, `custom_fields.jsonl` (field definitions), `history.jsonl`, `attachments.jsonl` (links nested) + and `attachments/` bytes. One JSON object per line, `\n` separated. +- **Keys.** camelCase; JSON columns are parsed into real JSON; `favorite` is a boolean. `user_id` columns are omitted + (the importing user owns everything). `works.jsonl` adds `private` and `externalIds`. +- **Deleted rows are included** with their `deletedAt`, and `fieldSources`/timestamps are kept, so a restore is lossless. +- **No secrets.** No users row beyond username/display name, no sessions, tokens, connections, credentials or source snapshots. +- **Restore** (`importUserBundle`) needs an account with no entries (`account_not_empty`) and `format === "sttracker-export/v1"` + (`unsupported_format`). Works with external IDs are re-matched (else created shared); works without become private. + Ids are remapped (UUIDs inside history `before`/`after` and custom-field value keys too). Rows are inserted directly, so no + new history is written. Works merged away at export time are exported as the tombstone row without `mergedIntoId`. +- **Backups** write `/backups/'-'>/.zip` per user plus `sttracker.db` (`VACUUM INTO`, SQLite only), + and prune to the newest `keep` (default 14). +**Consequences:** Any change to these files after release needs `v2` and a migration path for old bundles. + ## 2026-10-30: User layer: per-user moves merge entries, keep aliases, and roll the library up by hierarchy **Status:** accepted **Context:** Task 6 built the user layer (`packages/core/src/{entries,engagements,history,lists,copies}.ts`). diff --git a/packages/core/src/backup.ts b/packages/core/src/backup.ts new file mode 100644 index 0000000..34546e3 --- /dev/null +++ b/packages/core/src/backup.ts @@ -0,0 +1,45 @@ +import { mkdir, readdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { sql } from "kysely"; +import type { Db } from "@sttracker/db/db"; +import { exportUserBundle } from "./export.ts"; + +const STAMP_RE = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{3}Z$/; + +/** + * Write `${dataDir}/backups//` with one bundle per user (plus `sttracker.db` on SQLite), + * then prune to the newest `keep` (default 14). Returns the new directory. + */ +export async function runBackup(db: Db, dataDir: string, opts: { keep?: number } = {}): Promise { + const keep = Math.max(1, opts.keep ?? 14); + const root = join(dataDir, "backups"); + await mkdir(root, { recursive: true }); + + // Two backups in the same millisecond must not share a directory. + const stamp = (ms: number) => new Date(ms).toISOString().replaceAll(":", "-"); + const taken = new Set(await readdir(root)); + let t = Date.now(); + while (taken.has(stamp(t))) t++; + const dir = join(root, stamp(t)); + await mkdir(dir); + + const users = await db.k.selectFrom("users").select(["id", "username"]).orderBy("id").execute(); + for (const u of users) { + const name = u.username.replace(/[^A-Za-z0-9._-]/g, "_"); + await Bun.write(join(dir, `${name}.zip`), await exportUserBundle(db, dataDir, u.id)); + } + if (db.dialect === "sqlite") { + await sql.raw(`VACUUM INTO '${join(dir, "sttracker.db").replaceAll("'", "''")}'`).execute(db.k); + } + + const all = (await readdir(root)).filter((n) => STAMP_RE.test(n)).sort(); + for (const old of all.slice(0, Math.max(0, all.length - keep))) await rm(join(root, old), { recursive: true, force: true }); + return dir; +} + +/** Job handler for the `"backup"` job type. */ +export function backupHandler(deps: { db: Db; dataDir: string; keep?: number }): () => Promise { + return async () => { + await runBackup(deps.db, deps.dataDir, { keep: deps.keep }); + }; +} diff --git a/packages/core/src/export.ts b/packages/core/src/export.ts new file mode 100644 index 0000000..f6cab20 --- /dev/null +++ b/packages/core/src/export.ts @@ -0,0 +1,382 @@ +import { strFromU8, strToU8, unzipSync, zipSync } from "fflate"; +import type { MediaType } from "@sttracker/plugin-sdk/types"; +import { toCsv } from "@sttracker/plugin-sdk/csv"; +import type { Db } from "@sttracker/db/db"; +import { newId } from "@sttracker/db/ids"; +import { fromJson, toJson } from "@sttracker/db/json"; +import type { AttachmentEntity } from "@sttracker/db/schema"; +import { storeAttachment } from "./attachments.ts"; +import { ImportError } from "./errors.ts"; +import { createWorkIn, findWorkByExternalIdsIn, type Position } from "./works.ts"; + +export const EXPORT_FORMAT = "sttracker-export/v1"; + +// Export rows are loosely typed records: the zip is untrusted input on import. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type Rec = Record; + +const camel = (s: string) => s.replace(/_([a-z0-9])/g, (_, c: string) => c.toUpperCase()); +const snake = (s: string) => s.replace(/[A-Z]/g, (c) => `_${c.toLowerCase()}`); + +const JSON_COLS = { + works: ["creators", "metadata", "position"], + entries: ["tags", "custom_fields", "overrides", "field_sources"], + engagements: ["progress", "custom_fields", "field_sources"], + listens: ["mbids"], + copies: ["custom_fields"], + defs: ["options", "applies_to"], + history: ["before", "after"], +} as const; + +/** DB row -> camelCase export record, JSON columns parsed, `drop` columns removed. */ +function out(row: Rec, jsonCols: readonly string[], drop: readonly string[] = []): Rec { + const o: Rec = {}; + for (const [k, v] of Object.entries(row)) { + if (drop.includes(k)) continue; + o[camel(k)] = jsonCols.includes(k) ? fromJson(v as string, null) : v; + } + return o; +} + +/** camelCase export record -> snake_case row, JSON columns serialised. */ +function inn(rec: Rec, jsonCols: readonly string[]): Rec { + const o: Rec = {}; + for (const [k, v] of Object.entries(rec)) { + const col = snake(k); + o[col] = jsonCols.includes(col) ? toJson(v) : v; + } + return o; +} + +const jsonl = (rows: Rec[]) => strToU8(rows.map((r) => JSON.stringify(r) + "\n").join("")); + +function readJsonl(files: Record, name: string): Rec[] { + const f = files[name]; + if (!f) return []; + return strFromU8(f) + .split("\n") + .filter((l) => l.trim() !== "") + .map((l) => JSON.parse(l) as Rec); +} + +/** Per-user export bundle (zip). Includes soft-deleted rows so a restore is lossless; never credentials. */ +export async function exportUserBundle(db: Db, dataDir: string, userId: string): Promise { + const k = db.k; + const user = await k.selectFrom("users").select(["username", "display_name"]).where("id", "=", userId).executeTakeFirstOrThrow(); + + const entries = await k.selectFrom("entries").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const engagements = ( + await k.selectFrom("engagements").innerJoin("entries", "entries.id", "engagements.entry_id").selectAll("engagements").where("entries.user_id", "=", userId).orderBy("engagements.id").execute() + ); + const listens = await k.selectFrom("listens").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const lists = await k.selectFrom("lists").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const items = lists.length + ? await k.selectFrom("list_items").selectAll().where("list_id", "in", lists.map((l) => l.id)).orderBy("position").orderBy("id").execute() + : []; + const copies = await k.selectFrom("copies").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const defs = await k.selectFrom("custom_field_definitions").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const history = await k.selectFrom("user_history").selectAll().where("user_id", "=", userId).orderBy("created_at").orderBy("id").execute(); + const atts = await k.selectFrom("attachments").selectAll().where("user_id", "=", userId).orderBy("id").execute(); + const links = atts.length + ? await k.selectFrom("attachment_links").selectAll().where("attachment_id", "in", atts.map((a) => a.id)).execute() + : []; + + // Works: everything referenced, plus ancestors. + const workIds = new Set(); + for (const e of entries) workIds.add(e.work_id); + for (const i of items) workIds.add(i.work_id); + for (const c of copies) if (c.work_id) workIds.add(c.work_id); + for (const l of listens) if (l.work_id) workIds.add(l.work_id); + const works = new Map(); + let frontier = [...workIds]; + while (frontier.length) { + const rows = await k.selectFrom("works").selectAll().where("id", "in", frontier).execute(); + frontier = []; + for (const w of rows) { + works.set(w.id, w as Rec); + if (w.parent_id && !works.has(w.parent_id)) frontier.push(w.parent_id); + } + } + const extIds = works.size + ? await k.selectFrom("external_ids").selectAll().where("work_id", "in", [...works.keys()]).orderBy("namespace").execute() + : []; + const extByWork = new Map>(); + for (const x of extIds) { + const m = extByWork.get(x.work_id) ?? {}; + m[x.namespace] = x.value; + extByWork.set(x.work_id, m); + } + const workRecs = [...works.values()] + .sort((a, b) => (a.id < b.id ? -1 : 1)) + .map((w) => ({ + ...out(w, JSON_COLS.works, ["owner_user_id", "merged_into_id", "cover_attachment_id"]), + private: w.owner_user_id != null, + externalIds: extByWork.get(w.id) ?? {}, + })); + + const files: Record = {}; + const attRecs: Rec[] = []; + for (const a of atts) { + const file = Bun.file(`${dataDir}/attachments/${a.sha256.slice(0, 2)}/${a.sha256.slice(2, 4)}/${a.sha256}`); + if (!(await file.exists())) continue; + files[`attachments/${a.sha256}`] = new Uint8Array(await file.arrayBuffer()); + attRecs.push({ + ...out(a, [], ["user_id"]), + links: links.filter((l) => l.attachment_id === a.id).map((l) => ({ entity: l.entity, entityId: l.entity_id })), + }); + } + + const listRecs = lists.map((l) => ({ + ...out(l, [], ["user_id"]), + items: items.filter((i) => i.list_id === l.id).map((i) => out(i, [], ["list_id"])), + })); + + const manifest = { + format: EXPORT_FORMAT, + exportedAt: new Date().toISOString(), + user: { username: user.username, displayName: user.display_name }, + }; + return zipSync({ + "manifest.json": strToU8(JSON.stringify(manifest, null, 2)), + "works.jsonl": jsonl(workRecs), + "entries.jsonl": jsonl(entries.map((e) => ({ ...out(e, JSON_COLS.entries, ["user_id"]), favorite: e.favorite !== 0 }))), + "engagements.jsonl": jsonl(engagements.map((e) => out(e, JSON_COLS.engagements))), + "listens.jsonl": jsonl(listens.map((l) => out(l, JSON_COLS.listens, ["user_id"]))), + "lists.jsonl": jsonl(listRecs), + "copies.jsonl": jsonl(copies.map((c) => out(c, JSON_COLS.copies, ["user_id"]))), + "custom_fields.jsonl": jsonl(defs.map((d) => out(d, JSON_COLS.defs, ["user_id"]))), + "history.jsonl": jsonl(history.map((h) => out(h, JSON_COLS.history, ["user_id"]))), + "attachments.jsonl": jsonl(attRecs), + ...files, + }); +} + +/** CSV per entity type (live rows only): entries, engagements, listens. */ +export async function exportUserCsv(db: Db, userId: string): Promise> { + const k = db.k; + const s = (v: unknown) => (v == null ? "" : String(v)); + const entries = await k + .selectFrom("entries") + .innerJoin("works", "works.id", "entries.work_id") + .select(["entries.id", "entries.work_id", "works.title", "works.media_type", "works.year", "entries.status", "entries.rating", "entries.favorite", "entries.tags", "entries.review", "entries.created_at", "entries.updated_at"]) + .where("entries.user_id", "=", userId) + .where("entries.deleted_at", "is", null) + .orderBy("entries.id") + .execute(); + const engagements = await k + .selectFrom("engagements") + .innerJoin("entries", "entries.id", "engagements.entry_id") + .innerJoin("works", "works.id", "entries.work_id") + .select(["engagements.id", "engagements.entry_id", "works.title", "engagements.started_at", "engagements.started_precision", "engagements.finished_at", "engagements.finished_precision", "engagements.platform", "engagements.rating", "engagements.notes"]) + .where("entries.user_id", "=", userId) + .where("entries.deleted_at", "is", null) + .where("engagements.deleted_at", "is", null) + .orderBy("engagements.id") + .execute(); + const listens = await k.selectFrom("listens").selectAll().where("user_id", "=", userId).orderBy("listened_at").orderBy("id").execute(); + + return { + "entries.csv": toCsv([ + ["id", "workId", "title", "mediaType", "year", "status", "rating", "favorite", "tags", "review", "createdAt", "updatedAt"], + ...entries.map((e) => [e.id, e.work_id, e.title, e.media_type, s(e.year), e.status, s(e.rating), e.favorite ? "true" : "false", fromJson(e.tags, []).join("; "), s(e.review), e.created_at, e.updated_at]), + ]), + "engagements.csv": toCsv([ + ["id", "entryId", "title", "startedAt", "startedPrecision", "finishedAt", "finishedPrecision", "platform", "rating", "notes"], + ...engagements.map((e) => [e.id, e.entry_id, e.title, s(e.started_at), e.started_precision, s(e.finished_at), e.finished_precision, s(e.platform), s(e.rating), s(e.notes)]), + ]), + "listens.csv": toCsv([ + ["id", "listenedAt", "artist", "track", "album", "durationMs", "source"], + ...listens.map((l) => [l.id, l.listened_at, l.artist, l.track, s(l.album), s(l.duration_ms), l.source]), + ]), + }; +} + +const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; + +/** Restore a bundle into an account with no entries. Works are re-matched by external id. */ +export async function importUserBundle( + db: Db, + dataDir: string, + userId: string, + zip: Uint8Array, +): Promise<{ works: number; entries: number }> { + let files: Record; + try { + files = unzipSync(zip); + } catch { + throw new ImportError("unsupported_format"); + } + const mf = files["manifest.json"]; + let manifest: Rec = {}; + try { + manifest = mf ? (JSON.parse(strFromU8(mf)) as Rec) : {}; + } catch { + /* handled below */ + } + if (manifest.format !== EXPORT_FORMAT) throw new ImportError("unsupported_format"); + + const existing = await db.k.selectFrom("entries").select("id").where("user_id", "=", userId).limit(1).executeTakeFirst(); + if (existing) throw new ImportError("account_not_empty"); + + const works = readJsonl(files, "works.jsonl"); + const entries = readJsonl(files, "entries.jsonl"); + const engagements = readJsonl(files, "engagements.jsonl"); + const listens = readJsonl(files, "listens.jsonl"); + const lists = readJsonl(files, "lists.jsonl"); + const copies = readJsonl(files, "copies.jsonl"); + const defs = readJsonl(files, "custom_fields.jsonl"); + const history = readJsonl(files, "history.jsonl"); + const atts = readJsonl(files, "attachments.jsonl"); + + const ids = new Map(); + const remap = (old: unknown): string | null => (typeof old === "string" ? (ids.get(old) ?? null) : null); + const remapCustom = (v: unknown): Rec => { + const o: Rec = {}; + for (const [key, val] of Object.entries((v ?? {}) as Rec)) o[ids.get(key) ?? key] = val; + return o; + }; + + // Parents before children. + const byId = new Map(works.map((w) => [w.id as string, w])); + const depth = (w: Rec): number => { + let d = 0; + const seen = new Set(); + for (let p = w.parentId ? byId.get(w.parentId) : undefined; p && !seen.has(p.id); p = p.parentId ? byId.get(p.parentId) : undefined) { + seen.add(p.id); + d++; + } + return d; + }; + const ordered = works.map((w) => ({ w, d: depth(w) })).sort((a, b) => a.d - b.d).map((x) => x.w); + + await db.k.transaction().execute(async (tx) => { + for (const w of ordered) { + const externalIds = (w.externalIds ?? {}) as Record; + const hasExt = Object.keys(externalIds).length > 0; + const matched = hasExt ? await findWorkByExternalIdsIn(tx, externalIds, userId) : null; + if (matched) { + ids.set(w.id, matched); + continue; + } + const newWorkId = await createWorkIn( + tx, + { + hint: { + title: w.title, + mediaType: w.mediaType as MediaType, + year: w.year ?? undefined, + creators: w.creators ?? [], + externalIds, + coverUrl: w.coverUrl ?? "", + description: w.description ?? "", + position: (w.position ?? undefined) as Position | undefined, + }, + ownerUserId: hasExt ? null : userId, + parentId: remap(w.parentId), + }, + "import", + ); + await tx + .updateTable("works") + .set({ + original_title: w.originalTitle ?? "", + release_date: w.releaseDate ?? null, + metadata: toJson(w.metadata ?? {}), + created_at: w.createdAt, + updated_at: w.updatedAt, + }) + .where("id", "=", newWorkId) + .execute(); + ids.set(w.id, newWorkId); + } + + for (const d of defs) { + const id = newId(); + ids.set(d.id, id); + await tx.insertInto("custom_field_definitions").values({ ...inn(d, JSON_COLS.defs), id, user_id: userId } as never).execute(); + } + + for (const e of entries) { + const id = newId(); + ids.set(e.id, id); + const workId = remap(e.workId); + if (!workId) throw new ImportError("corrupt_bundle"); + await tx + .insertInto("entries") + .values({ ...inn(e, JSON_COLS.entries), id, user_id: userId, work_id: workId, favorite: e.favorite ? 1 : 0, custom_fields: toJson(remapCustom(e.customFields)) } as never) + .execute(); + } + + for (const g of engagements) { + const id = newId(); + ids.set(g.id, id); + const entryId = remap(g.entryId); + if (!entryId) throw new ImportError("corrupt_bundle"); + await tx + .insertInto("engagements") + .values({ ...inn(g, JSON_COLS.engagements), id, entry_id: entryId, custom_fields: toJson(remapCustom(g.customFields)) } as never) + .execute(); + } + + for (const l of listens) { + await tx + .insertInto("listens") + .values({ ...inn(l, JSON_COLS.listens), id: newId(), user_id: userId, work_id: remap(l.workId) } as never) + .onConflict((oc) => oc.doNothing()) + .execute(); + } + + for (const l of lists) { + const id = newId(); + ids.set(l.id, id); + const { items, ...rest } = l; + await tx.insertInto("lists").values({ ...inn(rest, []), id, user_id: userId } as never).execute(); + for (const it of (items ?? []) as Rec[]) { + const workId = remap(it.workId); + if (!workId) continue; + await tx.insertInto("list_items").values({ ...inn(it, []), id: newId(), list_id: id, work_id: workId } as never).execute(); + } + } + + for (const c of copies) { + const id = newId(); + ids.set(c.id, id); + await tx + .insertInto("copies") + .values({ ...inn(c, JSON_COLS.copies), id, user_id: userId, work_id: remap(c.workId), custom_fields: toJson(remapCustom(c.customFields)) } as never) + .execute(); + } + + for (const h of history) { + const sub = (v: unknown) => toJson(JSON.parse(JSON.stringify(v).replace(UUID_RE, (m) => ids.get(m) ?? m))); + await tx + .insertInto("user_history") + .values({ ...inn(h, []), id: newId(), user_id: userId, entity_id: remap(h.entityId) ?? h.entityId, before: sub(h.before), after: sub(h.after) } as never) + .execute(); + } + }); + + // Attachments go through storeAttachment (own transaction), so after the main one. + for (const a of atts) { + const bytes = files[`attachments/${a.sha256}`]; + if (!bytes) continue; + const links = ((a.links ?? []) as Rec[]) + .map((l) => ({ entity: l.entity as AttachmentEntity, entityId: remap(l.entityId) })) + .filter((l): l is { entity: AttachmentEntity; entityId: string } => l.entityId != null); + const [first, ...rest] = links; + if (!first) continue; + const row = await storeAttachment( + db, + dataDir, + userId, + { bytes, mime: a.mime, filename: a.filename, caption: a.caption ?? undefined, takenAt: a.takenAt ?? undefined }, + first, + ); + await db.k.updateTable("attachments").set({ visibility: a.visibility ?? null, created_at: a.createdAt }).where("id", "=", row.id).execute(); + for (const l of rest) { + await db.k.insertInto("attachment_links").values({ id: newId(), attachment_id: row.id, entity: l.entity, entity_id: l.entityId }).execute(); + } + } + + return { works: works.length, entries: entries.length }; +} diff --git a/packages/core/test/backup.test.ts b/packages/core/test/backup.test.ts new file mode 100644 index 0000000..0e899a9 --- /dev/null +++ b/packages/core/test/backup.test.ts @@ -0,0 +1,44 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { forEachDialect } from "@sttracker/db/testing"; +import { backupHandler, runBackup } from "../src/backup.ts"; +import { makeUser } from "./user-layer-fixtures.ts"; + +forEachDialect("backup", (getDb) => { + let dir: string; + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "sttracker-bak-")); + }); + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + test("backup writes a bundle per user (and the db on sqlite)", async () => { + const db = getDb(); + await makeUser(db, "ada"); + await makeUser(db, "bob"); + const out = await runBackup(db, dir, { keep: 14 }); + expect(out.startsWith(join(dir, "backups"))).toBe(true); + const files = (await readdir(out)).sort(); + expect(files).toEqual(db.dialect === "sqlite" ? ["ada.zip", "bob.zip", "sttracker.db"] : ["ada.zip", "bob.zip"]); + }); + + test("backup prunes to keep", async () => { + const db = getDb(); + await makeUser(db, "ada"); + const made: string[] = []; + for (let i = 0; i < 4; i++) made.push(await runBackup(db, dir, { keep: 2 })); + const left = (await readdir(join(dir, "backups"))).sort(); + expect(left).toHaveLength(2); + expect(left.map((n) => join(dir, "backups", n))).toEqual(made.slice(2)); + }); + + test("backupHandler runs a backup", async () => { + const db = getDb(); + await makeUser(db, "ada"); + await backupHandler({ db, dataDir: dir, keep: 3 })(); + expect(await readdir(join(dir, "backups"))).toHaveLength(1); + }); +}); diff --git a/packages/core/test/export.test.ts b/packages/core/test/export.test.ts new file mode 100644 index 0000000..7226d07 --- /dev/null +++ b/packages/core/test/export.test.ts @@ -0,0 +1,135 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { strFromU8, unzipSync, zipSync, strToU8 } from "fflate"; +import { forEachDialect } from "@sttracker/db/testing"; +import { storeAttachment } from "../src/attachments.ts"; +import { ImportError } from "../src/errors.ts"; +import { exportUserBundle, exportUserCsv, importUserBundle } from "../src/export.ts"; +import { makeUser, makeWork } from "./user-layer-fixtures.ts"; + +forEachDialect("export bundle", (getDb) => { + let dir: string; + let dir2: string; + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "sttracker-exp-")); + dir2 = await mkdtemp(join(tmpdir(), "sttracker-exp-")); + }); + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + await rm(dir2, { recursive: true, force: true }); + }); + + async function seed(db: ReturnType, u: string) { + const shared = await makeWork(db, { title: "Dune", mediaType: "book", year: 1965, externalIds: { hardcover: "dune" } }); + const parent = await makeWork(db, { title: "Show", mediaType: "tv", externalIds: { trakt: "show" } }); + const child = await makeWork(db, { title: "Pilot", mediaType: "tv", parentId: parent, externalIds: { trakt: "pilot" } }); + const priv = await makeWork(db, { title: "My Zine", mediaType: "book", ownerUserId: u }); + const now = "2024-01-02T03:04:05.000Z"; + const ent = async (id: string, workId: string, extra: object = {}) => + db.k.insertInto("entries").values({ + id, user_id: u, work_id: workId, status: "completed", rating: 85, review: "great", private_notes: null, favorite: 1, + tags: '["a","b"]', custom_fields: "{}", visibility: "private", overrides: "{}", media_type_override: null, + field_sources: '{"rating":"user"}', created_at: now, updated_at: now, deleted_at: null, ...extra, + }).execute(); + const e1 = crypto.randomUUID(), e2 = crypto.randomUUID(), e3 = crypto.randomUUID(), e4 = crypto.randomUUID(); + await ent(e1, shared); + await ent(e2, child); + await ent(e3, priv, { rating: null }); + await ent(e4, parent, { deleted_at: now }); + const eng = crypto.randomUUID(); + await db.k.insertInto("engagements").values({ + id: eng, entry_id: e1, started_at: "1999-05", started_precision: "month", finished_at: "2000", finished_precision: "year", + progress: "{}", platform: "kindle", rating: 70, notes: "n", custom_fields: "{}", field_sources: "{}", created_at: now, updated_at: now, deleted_at: null, + }).execute(); + const list = crypto.randomUUID(); + await db.k.insertInto("lists").values({ id: list, user_id: u, name: "Faves", description: "d", kind: "list", visibility: "private", rating: null, review: null, created_at: now, updated_at: now, deleted_at: null }).execute(); + await db.k.insertInto("list_items").values({ id: crypto.randomUUID(), list_id: list, work_id: shared, position: 0, note: "first" }).execute(); + await db.k.insertInto("user_history").values({ id: crypto.randomUUID(), user_id: u, entity: "entry", entity_id: e1, before: "{}", after: '{"rating":85}', actor: "user", created_at: now }).execute(); + await storeAttachment(db, dir, u, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "n.txt" }, { entity: "entry", entityId: e1 }); + return { e1, shared }; + } + + test("export then import into fresh user reproduces entries, ratings, engagements with precision, lists and attachments", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const zip = await exportUserBundle(db, dir, u); + + const v = await makeUser(db, "bob"); + expect(await importUserBundle(db, dir2, v, zip)).toEqual({ works: 4, entries: 4 }); + + const rows = await db.k.selectFrom("entries").innerJoin("works", "works.id", "entries.work_id").select(["works.title", "entries.rating", "entries.favorite", "entries.field_sources", "entries.deleted_at", "entries.created_at", "works.owner_user_id", "works.parent_id", "works.id as wid"]).where("entries.user_id", "=", v).execute(); + const byTitle = Object.fromEntries(rows.map((r) => [r.title, r])); + expect(rows).toHaveLength(4); + expect(byTitle["Dune"]!.rating).toBe(85); + expect(byTitle["Dune"]!.favorite).toBe(1); + expect(byTitle["Dune"]!.field_sources).toBe('{"rating":"user"}'); + expect(byTitle["Dune"]!.created_at).toBe("2024-01-02T03:04:05.000Z"); + expect(byTitle["Show"]!.deleted_at).not.toBeNull(); + expect(byTitle["My Zine"]!.owner_user_id).toBe(v); + expect(byTitle["Pilot"]!.parent_id).toBe(byTitle["Show"]!.wid); + // shared works were re-matched, not duplicated + expect(await db.k.selectFrom("works").select("id").where("title", "=", "Dune").execute()).toHaveLength(1); + + const eng = await db.k.selectFrom("engagements").innerJoin("entries", "entries.id", "engagements.entry_id").selectAll("engagements").where("entries.user_id", "=", v).execute(); + expect(eng).toHaveLength(1); + expect(eng[0]).toMatchObject({ started_at: "1999-05", started_precision: "month", finished_precision: "year", platform: "kindle" }); + + const lists = await db.k.selectFrom("lists").selectAll().where("user_id", "=", v).execute(); + expect(lists).toHaveLength(1); + const items = await db.k.selectFrom("list_items").selectAll().where("list_id", "=", lists[0]!.id).execute(); + expect(items).toMatchObject([{ note: "first", work_id: byTitle["Dune"]!.wid }]); + + const hist = await db.k.selectFrom("user_history").selectAll().where("user_id", "=", v).execute(); + expect(hist).toHaveLength(1); + const dune = await db.k.selectFrom("entries").select("id").where("user_id", "=", v).where("work_id", "=", byTitle["Dune"]!.wid).executeTakeFirstOrThrow(); + expect(hist[0]!.entity_id).toBe(dune.id); + + const atts = await db.k.selectFrom("attachments").innerJoin("attachment_links", "attachment_links.attachment_id", "attachments.id").selectAll("attachments").select("attachment_links.entity_id").where("attachments.user_id", "=", v).execute(); + expect(atts).toHaveLength(1); + expect(atts[0]!.entity_id).toBe(dune.id); + expect(await Bun.file(join(dir2, "attachments", atts[0]!.sha256.slice(0, 2), atts[0]!.sha256.slice(2, 4), atts[0]!.sha256)).text()).toBe("hello"); + }); + + test("bundle has no credential fields", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const files = unzipSync(await exportUserBundle(db, dir, u)); + expect(Object.keys(files)).toContain("manifest.json"); + const text = Object.entries(files).filter(([n]) => !n.startsWith("attachments/")).map(([, b]) => strFromU8(b)).join("\n").toLowerCase(); + for (const w of ["password", "token", "credentials"]) expect(text).not.toContain(w); + expect(JSON.parse(strFromU8(files["manifest.json"]!)).format).toBe("sttracker-export/v1"); + }); + + test("import into non-empty account refused", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const zip = await exportUserBundle(db, dir, u); + const err = await importUserBundle(db, dir2, u, zip).catch((e) => e); + expect(err).toBeInstanceOf(ImportError); + expect(err.code).toBe("account_not_empty"); + }); + + test("import of an unknown format is refused", async () => { + const db = getDb(); + const v = await makeUser(db, "bob"); + const zip = zipSync({ "manifest.json": strToU8('{"format":"other"}') }); + const err = await importUserBundle(db, dir2, v, zip).catch((e) => e); + expect(err.code).toBe("unsupported_format"); + }); + + test("csv export has entries, engagements and listens", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const csv = await exportUserCsv(db, u); + expect(Object.keys(csv).sort()).toEqual(["engagements.csv", "entries.csv", "listens.csv"]); + expect(csv["entries.csv"]).toContain("Dune"); + expect(csv["entries.csv"]).not.toContain("Show,"); + expect(csv["engagements.csv"]).toContain("kindle"); + }); +});