From 66b155876f7949a6a013f17be33b60fa1f8c3073 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 20:24:10 +0100 Subject: [PATCH] fix(core): atomic restore with aliases, snapshot export, hardened backups Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/DECISIONS.md | 10 +- docs/TODO.md | 5 + packages/core/src/attachments.ts | 35 ++- packages/core/src/backup.ts | 34 ++- packages/core/src/export.ts | 396 ++++++++++++++++++------------ packages/core/test/backup.test.ts | 11 +- packages/core/test/export.test.ts | 113 ++++++++- 8 files changed, 424 insertions(+), 182 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 16699ad..3ff0f22 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,7 +16,7 @@ user layer or files a Changes-inbox event. |---|---|---| | `apps/server` | `Bun.serve`: `/api/v1`, `/ingest/`, job worker, static web | planned | | `apps/web` | React app (HTML imports) | planned | -| `packages/core` | domain services, one module per area (import by path, no barrel): `auth`, `crypto`, `works`, `matching`, `entries` (entries, library listing, quick add, per-user moves, match suggestions), `engagements`, `history` (`user_history`, restore), `lists`, `copies`, `custom-fields`, `attachments`. Change engine and export planned | exists (partial) | +| `packages/core` | domain services, one module per area (import by path, no barrel): `auth`, `crypto`, `works`, `matching`, `entries` (entries, library listing, quick add, per-user moves, match suggestions), `engagements`, `history` (`user_history`, restore), `lists`, `copies`, `custom-fields`, `attachments`, `export` (per-user bundle, CSV, restore), `backup` (scheduled bundles + SQLite `VACUUM INTO`). Change engine planned | exists (partial) | | `packages/db` | Kysely over one Bun.SQL dialect (SQLite + Postgres), ids, JSON helpers, migrations, `forEachDialect` test harness | exists (schema pending) | | `packages/plugin-sdk` | plugin contract, `SourceRecord`, http/rate-limit helpers, fixture test harness | planned | | `plugins/` | one source integration each (hardcover, trakt, mangabaka, listenbrainz, lastfm, csv) | planned | diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 74d903d..6afe5f0 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -16,12 +16,18 @@ No index: `grep -n '^## ' docs/DECISIONS.md` lists every entry (date, title, lin (the importing user owns everything). `works.jsonl` adds `private` and `externalIds`. - **Deleted rows are included** with their `deletedAt`, and `fieldSources`/timestamps are kept, so a restore is lossless. - **No secrets.** No users row beyond username/display name, no sessions, tokens, connections, credentials or source snapshots. +- **Merges and aliases.** `works.jsonl` rows carry `mergedIntoId` (the survivor and ancestors are exported too) and + `work_aliases.jsonl` holds the user's `{ fromWorkId, toWorkId }` rows. Both are additive to v1. Restore re-creates + tombstones (for works it creates) and aliases, so a later write to the old work still resolves to the moved entry. +- **Export is one snapshot:** all reads run in a single transaction (repeatable read on Postgres). - **Restore** (`importUserBundle`) needs an account with no entries (`account_not_empty`) and `format === "sttracker-export/v1"` (`unsupported_format`). Works with external IDs are re-matched (else created shared); works without become private. Ids are remapped (UUIDs inside history `before`/`after` and custom-field value keys too). Rows are inserted directly, so no - new history is written. Works merged away at export time are exported as the tombstone row without `mergedIntoId`. + new history is written. The import validates first (attachment mime/size/sha256, rating 0-100, status, visibility, 2 GB + uncompressed cap) and writes all rows, attachments included, in one transaction; errors are + `invalid_attachment`, `corrupt_bundle`, `bundle_too_large`, `conflict`. Works merged away at export time are exported as the tombstone row without `mergedIntoId`. - **Backups** write `/backups/'-'>/.zip` per user plus `sttracker.db` (`VACUUM INTO`, SQLite only), - and prune to the newest `keep` (default 14). + are built in `.tmp` then renamed (modes 0700/0600), and prune to the newest `keep` (default 14). **Consequences:** Any change to these files after release needs `v2` and a migration path for old bundles. ## 2026-10-30: User layer: per-user moves merge entries, keep aliases, and roll the library up by hierarchy diff --git a/docs/TODO.md b/docs/TODO.md index 75fa908..6ee3b41 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -24,6 +24,11 @@ Small, specific, independently actionable. Remove only when actually done. file, and compare steps and token use. ## Known issues +- [ ] Restore: a new child work whose parent+position collides with an existing child of a re-matched parent fails + with `ImportError("conflict")` instead of being merged or repositioned. +- [ ] Restore: an unlink of identical attachment bytes between the pre-write and the commit could remove the file; + mitigated by re-writing after commit, not locked. +- [ ] Export bundles hold attachment bytes in memory; very large libraries need a streaming zip. - [ ] Admin merge does not merge colliding children recursively; colliding children stay under the tombstoned parent. - [ ] Matching: a shared parent resolved without external IDs or position (title/year reuse in `packages/core/src/matching.ts`) has no unique key, so concurrent matches can create it twice. diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 02d3019..f86b93b 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -7,14 +7,14 @@ import { ValidationError } from "./errors.ts"; export type Attachment = AttachmentsTable; -const MAX_BYTES = 25 * 1024 * 1024; +export const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024; /** Lowercase and strip parameters (`; charset=...`). */ -function normalizeMime(mime: string): string { +export function normalizeMime(mime: string): string { return (mime.split(";")[0] ?? "").trim().toLowerCase(); } -function mimeAllowed(mime: string): boolean { +export function mimeAllowed(mime: string): boolean { if (mime === "image/svg+xml") return false; // scriptable; never serve user SVG return mime.startsWith("image/") || mime === "application/pdf" || mime === "text/plain"; } @@ -41,6 +41,33 @@ function pathFor(dataDir: string, sha: string): string { return join(dataDir, "attachments", sha.slice(0, 2), sha.slice(2, 4), sha); } +export function sha256Hex(bytes: Uint8Array): string { + return new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); +} + +/** + * Write bytes to the content-addressed store (idempotent, deduplicated) and return their sha256. + * Callers that insert rows afterwards should call it again after committing, since an unlink of + * identical bytes in between may have removed the file. + */ +export async function writeAttachmentFile(dataDir: string, bytes: Uint8Array): Promise { + const sha256 = sha256Hex(bytes); + const path = pathFor(dataDir, sha256); + await withLock(sha256, async () => { + if (await Bun.file(path).exists()) return; + await mkdir(dirname(path), { recursive: true }); + const tmp = `${path}.tmp-${newId()}`; + try { + await Bun.write(tmp, bytes); + await rename(tmp, path); + } catch (e) { + await rm(tmp, { force: true }); + throw e; + } + }); + return sha256; +} + /** * Store a file content-addressed by sha256 (deduplicated on disk) and link it to an entity. * Each call creates a new attachment row and link; identical bytes share one file. @@ -54,7 +81,7 @@ export async function storeAttachment( ): Promise { const mime = normalizeMime(file.mime); if (!mimeAllowed(mime)) throw new ValidationError("mime", "unsupported file type"); - if (file.bytes.byteLength > MAX_BYTES) throw new ValidationError("file", "larger than 25 MB"); + if (file.bytes.byteLength > MAX_ATTACHMENT_BYTES) throw new ValidationError("file", "larger than 25 MB"); const sha256 = new Bun.CryptoHasher("sha256").update(file.bytes).digest("hex"); const path = pathFor(dataDir, sha256); diff --git a/packages/core/src/backup.ts b/packages/core/src/backup.ts index 34546e3..c9c75df 100644 --- a/packages/core/src/backup.ts +++ b/packages/core/src/backup.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir, rm } from "node:fs/promises"; +import { chmod, mkdir, readdir, rename, rm } from "node:fs/promises"; import { join } from "node:path"; import { sql } from "kysely"; import type { Db } from "@sttracker/db/db"; @@ -14,6 +14,7 @@ export async function runBackup(db: Db, dataDir: string, opts: { keep?: number } const keep = Math.max(1, opts.keep ?? 14); const root = join(dataDir, "backups"); await mkdir(root, { recursive: true }); + await chmod(root, 0o700); // Two backups in the same millisecond must not share a directory. const stamp = (ms: number) => new Date(ms).toISOString().replaceAll(":", "-"); @@ -21,15 +22,28 @@ export async function runBackup(db: Db, dataDir: string, opts: { keep?: number } let t = Date.now(); while (taken.has(stamp(t))) t++; const dir = join(root, stamp(t)); - await mkdir(dir); - - const users = await db.k.selectFrom("users").select(["id", "username"]).orderBy("id").execute(); - for (const u of users) { - const name = u.username.replace(/[^A-Za-z0-9._-]/g, "_"); - await Bun.write(join(dir, `${name}.zip`), await exportUserBundle(db, dataDir, u.id)); - } - if (db.dialect === "sqlite") { - await sql.raw(`VACUUM INTO '${join(dir, "sttracker.db").replaceAll("'", "''")}'`).execute(db.k); + // Build in `.tmp` and rename at the end, so a failed run never leaves a counted partial backup. + const tmp = `${dir}.tmp`; + await rm(tmp, { recursive: true, force: true }); + await mkdir(tmp); + await chmod(tmp, 0o700); + try { + const users = await db.k.selectFrom("users").select(["id", "username"]).orderBy("id").execute(); + for (const u of users) { + const name = u.username.replace(/[^A-Za-z0-9._-]/g, "_"); + const file = join(tmp, `${name}.zip`); + await Bun.write(file, await exportUserBundle(db, dataDir, u.id)); + await chmod(file, 0o600); + } + if (db.dialect === "sqlite") { + const file = join(tmp, "sttracker.db"); + await sql.raw(`VACUUM INTO '${file.replaceAll("'", "''")}'`).execute(db.k); + await chmod(file, 0o600); + } + await rename(tmp, dir); + } catch (e) { + await rm(tmp, { recursive: true, force: true }); + throw e; } const all = (await readdir(root)).filter((n) => STAMP_RE.test(n)).sort(); diff --git a/packages/core/src/export.ts b/packages/core/src/export.ts index f6cab20..09a463a 100644 --- a/packages/core/src/export.ts +++ b/packages/core/src/export.ts @@ -4,10 +4,11 @@ import { toCsv } from "@sttracker/plugin-sdk/csv"; import type { Db } from "@sttracker/db/db"; import { newId } from "@sttracker/db/ids"; import { fromJson, toJson } from "@sttracker/db/json"; -import type { AttachmentEntity } from "@sttracker/db/schema"; -import { storeAttachment } from "./attachments.ts"; +import type { AttachmentEntity, EntryStatus, Visibility } from "@sttracker/db/schema"; +import { MAX_ATTACHMENT_BYTES, mimeAllowed, normalizeMime, sha256Hex, writeAttachmentFile } from "./attachments.ts"; +import { isUniqueViolation } from "./matching.ts"; import { ImportError } from "./errors.ts"; -import { createWorkIn, findWorkByExternalIdsIn, type Position } from "./works.ts"; +import { createWorkIn, findWorkByExternalIdsIn, type K, type Position } from "./works.ts"; export const EXPORT_FORMAT = "sttracker-export/v1"; @@ -59,9 +60,26 @@ function readJsonl(files: Record, name: string): Rec[] { .map((l) => JSON.parse(l) as Rec); } -/** Per-user export bundle (zip). Includes soft-deleted rows so a restore is lossless; never credentials. */ -export async function exportUserBundle(db: Db, dataDir: string, userId: string): Promise { - const k = db.k; +const CHUNK = 500; +const MAX_UNCOMPRESSED = 2 * 1024 * 1024 * 1024; + +/** Run `fn` over `ids` in batches (keeps `IN (...)` lists under driver parameter limits). */ +async function inChunks(ids: readonly string[], fn: (chunk: string[]) => Promise): Promise { + const all: T[] = []; + for (let i = 0; i < ids.length; i += CHUNK) all.push(...(await fn(ids.slice(i, i + CHUNK)))); + return all; +} + +/** + * Per-user export bundle (zip). Includes soft-deleted rows so a restore is lossless; never credentials. + * All reads happen in one transaction (repeatable read on Postgres) so the bundle is a consistent snapshot. + */ +export function exportUserBundle(db: Db, dataDir: string, userId: string): Promise { + const tb = db.k.transaction(); + return (db.dialect === "postgres" ? tb.setIsolationLevel("repeatable read") : tb).execute((tx) => buildBundle(tx, dataDir, userId)); +} + +async function buildBundle(k: K, dataDir: string, userId: string): Promise { const user = await k.selectFrom("users").select(["username", "display_name"]).where("id", "=", userId).executeTakeFirstOrThrow(); const entries = await k.selectFrom("entries").selectAll().where("user_id", "=", userId).orderBy("id").execute(); @@ -70,16 +88,15 @@ export async function exportUserBundle(db: Db, dataDir: string, userId: string): ); const listens = await k.selectFrom("listens").selectAll().where("user_id", "=", userId).orderBy("id").execute(); const lists = await k.selectFrom("lists").selectAll().where("user_id", "=", userId).orderBy("id").execute(); - const items = lists.length - ? await k.selectFrom("list_items").selectAll().where("list_id", "in", lists.map((l) => l.id)).orderBy("position").orderBy("id").execute() - : []; + const items = await inChunks(lists.map((l) => l.id), (c) => + k.selectFrom("list_items").selectAll().where("list_id", "in", c).orderBy("position").orderBy("id").execute(), + ); const copies = await k.selectFrom("copies").selectAll().where("user_id", "=", userId).orderBy("id").execute(); const defs = await k.selectFrom("custom_field_definitions").selectAll().where("user_id", "=", userId).orderBy("id").execute(); const history = await k.selectFrom("user_history").selectAll().where("user_id", "=", userId).orderBy("created_at").orderBy("id").execute(); const atts = await k.selectFrom("attachments").selectAll().where("user_id", "=", userId).orderBy("id").execute(); - const links = atts.length - ? await k.selectFrom("attachment_links").selectAll().where("attachment_id", "in", atts.map((a) => a.id)).execute() - : []; + const links = await inChunks(atts.map((a) => a.id), (c) => k.selectFrom("attachment_links").selectAll().where("attachment_id", "in", c).execute()); + const aliases = await k.selectFrom("work_aliases").selectAll().where("user_id", "=", userId).orderBy("id").execute(); // Works: everything referenced, plus ancestors. const workIds = new Set(); @@ -87,19 +104,22 @@ export async function exportUserBundle(db: Db, dataDir: string, userId: string): for (const i of items) workIds.add(i.work_id); for (const c of copies) if (c.work_id) workIds.add(c.work_id); for (const l of listens) if (l.work_id) workIds.add(l.work_id); + for (const a of aliases) { + workIds.add(a.from_work_id); + workIds.add(a.to_work_id); + } const works = new Map(); let frontier = [...workIds]; while (frontier.length) { - const rows = await k.selectFrom("works").selectAll().where("id", "in", frontier).execute(); + const rows = await inChunks(frontier, (c) => k.selectFrom("works").selectAll().where("id", "in", c).execute()); frontier = []; for (const w of rows) { works.set(w.id, w as Rec); - if (w.parent_id && !works.has(w.parent_id)) frontier.push(w.parent_id); + // Ancestors and merge survivors are part of the bundle too. + for (const ref of [w.parent_id, w.merged_into_id]) if (ref && !works.has(ref)) frontier.push(ref); } } - const extIds = works.size - ? await k.selectFrom("external_ids").selectAll().where("work_id", "in", [...works.keys()]).orderBy("namespace").execute() - : []; + const extIds = await inChunks([...works.keys()], (c) => k.selectFrom("external_ids").selectAll().where("work_id", "in", c).orderBy("namespace").execute()); const extByWork = new Map>(); for (const x of extIds) { const m = extByWork.get(x.work_id) ?? {}; @@ -109,7 +129,7 @@ export async function exportUserBundle(db: Db, dataDir: string, userId: string): const workRecs = [...works.values()] .sort((a, b) => (a.id < b.id ? -1 : 1)) .map((w) => ({ - ...out(w, JSON_COLS.works, ["owner_user_id", "merged_into_id", "cover_attachment_id"]), + ...out(w, JSON_COLS.works, ["owner_user_id", "cover_attachment_id"]), private: w.owner_user_id != null, externalIds: extByWork.get(w.id) ?? {}, })); @@ -145,6 +165,7 @@ export async function exportUserBundle(db: Db, dataDir: string, userId: string): "lists.jsonl": jsonl(listRecs), "copies.jsonl": jsonl(copies.map((c) => out(c, JSON_COLS.copies, ["user_id"]))), "custom_fields.jsonl": jsonl(defs.map((d) => out(d, JSON_COLS.defs, ["user_id"]))), + "work_aliases.jsonl": jsonl(aliases.map((a) => ({ fromWorkId: a.from_work_id, toWorkId: a.to_work_id }))), "history.jsonl": jsonl(history.map((h) => out(h, JSON_COLS.history, ["user_id"]))), "attachments.jsonl": jsonl(attRecs), ...files, @@ -192,8 +213,29 @@ export async function exportUserCsv(db: Db, userId: string): Promise= 0 && v <= 100)) corrupt(); +} + +function checkRows(rows: Rec[], opts: { status?: boolean; visibility?: boolean; rating?: boolean }): void { + for (const r of rows) { + if (opts.rating) checkRatingValue(r.rating); + if (opts.status && !STATUSES.includes(r.status)) corrupt(); + if (opts.visibility && !VISIBILITIES.includes(r.visibility)) corrupt(); + } +} + +/** + * Restore a bundle into an account with no entries. Works are re-matched by external id. Everything is + * validated before anything is written; rows (attachments included) are inserted in one transaction. + */ export async function importUserBundle( db: Db, dataDir: string, @@ -201,9 +243,17 @@ export async function importUserBundle( zip: Uint8Array, ): Promise<{ works: number; entries: number }> { let files: Record; + let total = 0; try { - files = unzipSync(zip); - } catch { + files = unzipSync(zip, { + filter: (f) => { + total += f.originalSize; + if (total > MAX_UNCOMPRESSED) throw new ImportError("bundle_too_large"); + return true; + }, + }); + } catch (e) { + if (e instanceof ImportError) throw e; throw new ImportError("unsupported_format"); } const mf = files["manifest.json"]; @@ -218,15 +268,39 @@ export async function importUserBundle( const existing = await db.k.selectFrom("entries").select("id").where("user_id", "=", userId).limit(1).executeTakeFirst(); if (existing) throw new ImportError("account_not_empty"); - const works = readJsonl(files, "works.jsonl"); - const entries = readJsonl(files, "entries.jsonl"); - const engagements = readJsonl(files, "engagements.jsonl"); - const listens = readJsonl(files, "listens.jsonl"); - const lists = readJsonl(files, "lists.jsonl"); - const copies = readJsonl(files, "copies.jsonl"); - const defs = readJsonl(files, "custom_fields.jsonl"); - const history = readJsonl(files, "history.jsonl"); - const atts = readJsonl(files, "attachments.jsonl"); + let works: Rec[], entries: Rec[], engagements: Rec[], listens: Rec[], lists: Rec[], copies: Rec[]; + let defs: Rec[], history: Rec[], atts: Rec[], aliases: Rec[]; + try { + works = readJsonl(files, "works.jsonl"); + entries = readJsonl(files, "entries.jsonl"); + engagements = readJsonl(files, "engagements.jsonl"); + listens = readJsonl(files, "listens.jsonl"); + lists = readJsonl(files, "lists.jsonl"); + copies = readJsonl(files, "copies.jsonl"); + defs = readJsonl(files, "custom_fields.jsonl"); + history = readJsonl(files, "history.jsonl"); + atts = readJsonl(files, "attachments.jsonl"); + aliases = readJsonl(files, "work_aliases.jsonl"); + } catch { + return corrupt(); + } + checkRows(entries, { rating: true, status: true, visibility: true }); + checkRows(engagements, { rating: true }); + checkRows(lists, { rating: true, visibility: true }); + + // Attachments: validate everything before writing anything. + for (const a of atts) { + const bytes = files[`attachments/${a.sha256}`]; + if ( + !bytes || + !mimeAllowed(normalizeMime(String(a.mime ?? ""))) || + bytes.byteLength > MAX_ATTACHMENT_BYTES || + sha256Hex(bytes) !== a.sha256 + ) { + throw new ImportError("invalid_attachment"); + } + } + for (const a of atts) await writeAttachmentFile(dataDir, files[`attachments/${a.sha256}`]!); const ids = new Map(); const remap = (old: unknown): string | null => (typeof old === "string" ? (ids.get(old) ?? null) : null); @@ -249,134 +323,152 @@ export async function importUserBundle( }; const ordered = works.map((w) => ({ w, d: depth(w) })).sort((a, b) => a.d - b.d).map((x) => x.w); - await db.k.transaction().execute(async (tx) => { - for (const w of ordered) { - const externalIds = (w.externalIds ?? {}) as Record; - const hasExt = Object.keys(externalIds).length > 0; - const matched = hasExt ? await findWorkByExternalIdsIn(tx, externalIds, userId) : null; - if (matched) { - ids.set(w.id, matched); - continue; - } - const newWorkId = await createWorkIn( - tx, - { - hint: { - title: w.title, - mediaType: w.mediaType as MediaType, - year: w.year ?? undefined, - creators: w.creators ?? [], - externalIds, - coverUrl: w.coverUrl ?? "", - description: w.description ?? "", - position: (w.position ?? undefined) as Position | undefined, + try { + await db.k.transaction().execute(async (tx) => { + const created = new Set(); + for (const w of ordered) { + const externalIds = (w.externalIds ?? {}) as Record; + const hasExt = Object.keys(externalIds).length > 0; + const matched = hasExt ? await findWorkByExternalIdsIn(tx, externalIds, userId) : null; + if (matched) { + ids.set(w.id, matched); + continue; + } + const newWorkId = await createWorkIn( + tx, + { + hint: { + title: w.title, + mediaType: w.mediaType as MediaType, + year: w.year ?? undefined, + creators: w.creators ?? [], + externalIds, + coverUrl: w.coverUrl ?? "", + description: w.description ?? "", + position: (w.position ?? undefined) as Position | undefined, + }, + ownerUserId: hasExt ? null : userId, + parentId: remap(w.parentId), }, - ownerUserId: hasExt ? null : userId, - parentId: remap(w.parentId), - }, - "import", - ); - await tx - .updateTable("works") - .set({ - original_title: w.originalTitle ?? "", - release_date: w.releaseDate ?? null, - metadata: toJson(w.metadata ?? {}), - created_at: w.createdAt, - updated_at: w.updatedAt, - }) - .where("id", "=", newWorkId) - .execute(); - ids.set(w.id, newWorkId); - } - - for (const d of defs) { - const id = newId(); - ids.set(d.id, id); - await tx.insertInto("custom_field_definitions").values({ ...inn(d, JSON_COLS.defs), id, user_id: userId } as never).execute(); - } - - for (const e of entries) { - const id = newId(); - ids.set(e.id, id); - const workId = remap(e.workId); - if (!workId) throw new ImportError("corrupt_bundle"); - await tx - .insertInto("entries") - .values({ ...inn(e, JSON_COLS.entries), id, user_id: userId, work_id: workId, favorite: e.favorite ? 1 : 0, custom_fields: toJson(remapCustom(e.customFields)) } as never) - .execute(); - } - - for (const g of engagements) { - const id = newId(); - ids.set(g.id, id); - const entryId = remap(g.entryId); - if (!entryId) throw new ImportError("corrupt_bundle"); - await tx - .insertInto("engagements") - .values({ ...inn(g, JSON_COLS.engagements), id, entry_id: entryId, custom_fields: toJson(remapCustom(g.customFields)) } as never) - .execute(); - } - - for (const l of listens) { - await tx - .insertInto("listens") - .values({ ...inn(l, JSON_COLS.listens), id: newId(), user_id: userId, work_id: remap(l.workId) } as never) - .onConflict((oc) => oc.doNothing()) - .execute(); - } - - for (const l of lists) { - const id = newId(); - ids.set(l.id, id); - const { items, ...rest } = l; - await tx.insertInto("lists").values({ ...inn(rest, []), id, user_id: userId } as never).execute(); - for (const it of (items ?? []) as Rec[]) { - const workId = remap(it.workId); - if (!workId) continue; - await tx.insertInto("list_items").values({ ...inn(it, []), id: newId(), list_id: id, work_id: workId } as never).execute(); + "import", + ); + await tx + .updateTable("works") + .set({ + original_title: w.originalTitle ?? "", + release_date: w.releaseDate ?? null, + metadata: toJson(w.metadata ?? {}), + created_at: w.createdAt, + updated_at: w.updatedAt, + }) + .where("id", "=", newWorkId) + .execute(); + ids.set(w.id, newWorkId); + created.add(w.id); } - } - for (const c of copies) { - const id = newId(); - ids.set(c.id, id); - await tx - .insertInto("copies") - .values({ ...inn(c, JSON_COLS.copies), id, user_id: userId, work_id: remap(c.workId), custom_fields: toJson(remapCustom(c.customFields)) } as never) - .execute(); - } + // Tombstones: only for works created here (a matched existing work keeps its own state). + for (const w of works) { + const target = remap(w.mergedIntoId); + if (!created.has(w.id) || !target) continue; + await tx.updateTable("works").set({ merged_into_id: target }).where("id", "=", ids.get(w.id)!).execute(); + } + for (const a of aliases) { + const from = remap(a.fromWorkId); + const to = remap(a.toWorkId); + if (!from || !to || from === to) continue; + await tx.insertInto("work_aliases").values({ id: newId(), user_id: userId, from_work_id: from, to_work_id: to }).execute(); + } - for (const h of history) { - const sub = (v: unknown) => toJson(JSON.parse(JSON.stringify(v).replace(UUID_RE, (m) => ids.get(m) ?? m))); - await tx - .insertInto("user_history") - .values({ ...inn(h, []), id: newId(), user_id: userId, entity_id: remap(h.entityId) ?? h.entityId, before: sub(h.before), after: sub(h.after) } as never) - .execute(); - } - }); + for (const d of defs) { + const id = newId(); + ids.set(d.id, id); + await tx.insertInto("custom_field_definitions").values({ ...inn(d, JSON_COLS.defs), id, user_id: userId } as never).execute(); + } - // Attachments go through storeAttachment (own transaction), so after the main one. - for (const a of atts) { - const bytes = files[`attachments/${a.sha256}`]; - if (!bytes) continue; - const links = ((a.links ?? []) as Rec[]) - .map((l) => ({ entity: l.entity as AttachmentEntity, entityId: remap(l.entityId) })) - .filter((l): l is { entity: AttachmentEntity; entityId: string } => l.entityId != null); - const [first, ...rest] = links; - if (!first) continue; - const row = await storeAttachment( - db, - dataDir, - userId, - { bytes, mime: a.mime, filename: a.filename, caption: a.caption ?? undefined, takenAt: a.takenAt ?? undefined }, - first, - ); - await db.k.updateTable("attachments").set({ visibility: a.visibility ?? null, created_at: a.createdAt }).where("id", "=", row.id).execute(); - for (const l of rest) { - await db.k.insertInto("attachment_links").values({ id: newId(), attachment_id: row.id, entity: l.entity, entity_id: l.entityId }).execute(); - } + for (const e of entries) { + const id = newId(); + ids.set(e.id, id); + const workId = remap(e.workId); + if (!workId) corrupt(); + await tx + .insertInto("entries") + .values({ ...inn(e, JSON_COLS.entries), id, user_id: userId, work_id: workId, favorite: e.favorite ? 1 : 0, custom_fields: toJson(remapCustom(e.customFields)) } as never) + .execute(); + } + + for (const g of engagements) { + const id = newId(); + ids.set(g.id, id); + const entryId = remap(g.entryId); + if (!entryId) corrupt(); + await tx + .insertInto("engagements") + .values({ ...inn(g, JSON_COLS.engagements), id, entry_id: entryId, custom_fields: toJson(remapCustom(g.customFields)) } as never) + .execute(); + } + + for (const l of listens) { + await tx + .insertInto("listens") + .values({ ...inn(l, JSON_COLS.listens), id: newId(), user_id: userId, work_id: remap(l.workId) } as never) + .onConflict((oc) => oc.doNothing()) + .execute(); + } + + for (const l of lists) { + const id = newId(); + ids.set(l.id, id); + const { items, ...rest } = l; + await tx.insertInto("lists").values({ ...inn(rest, []), id, user_id: userId } as never).execute(); + for (const it of (items ?? []) as Rec[]) { + const workId = remap(it.workId); + if (!workId) continue; + await tx.insertInto("list_items").values({ ...inn(it, []), id: newId(), list_id: id, work_id: workId } as never).execute(); + } + } + + for (const c of copies) { + const id = newId(); + ids.set(c.id, id); + await tx + .insertInto("copies") + .values({ ...inn(c, JSON_COLS.copies), id, user_id: userId, work_id: remap(c.workId), custom_fields: toJson(remapCustom(c.customFields)) } as never) + .execute(); + } + + for (const a of atts) { + const links = ((a.links ?? []) as Rec[]) + .map((l) => ({ entity: l.entity as AttachmentEntity, entityId: remap(l.entityId) })) + .filter((l): l is { entity: AttachmentEntity; entityId: string } => l.entityId != null); + if (links.length === 0) continue; + const id = newId(); + const { links: _links, ...attRow } = a; + await tx + .insertInto("attachments") + .values({ ...inn(attRow, []), id, user_id: userId, mime: normalizeMime(a.mime), size: files[`attachments/${a.sha256}`]!.byteLength } as never) + .execute(); + for (const l of links) { + await tx.insertInto("attachment_links").values({ id: newId(), attachment_id: id, entity: l.entity, entity_id: l.entityId }).execute(); + } + } + + // History last, so every referenced id has been remapped. + for (const h of history) { + const sub = (v: unknown) => toJson(JSON.parse(JSON.stringify(v).replace(UUID_RE, (m) => ids.get(m) ?? m))); + await tx + .insertInto("user_history") + .values({ ...inn(h, []), id: newId(), user_id: userId, entity_id: remap(h.entityId) ?? h.entityId, before: sub(h.before), after: sub(h.after) } as never) + .execute(); + } + }); + } catch (e) { + if (e instanceof ImportError) throw e; + if (isUniqueViolation(e)) throw new ImportError("conflict"); + throw e; } + // An unlink of identical bytes between the pre-write and the commit may have removed a file. + for (const a of atts) await writeAttachmentFile(dataDir, files[`attachments/${a.sha256}`]!); return { works: works.length, entries: entries.length }; } diff --git a/packages/core/test/backup.test.ts b/packages/core/test/backup.test.ts index 0e899a9..ba26550 100644 --- a/packages/core/test/backup.test.ts +++ b/packages/core/test/backup.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; -import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { forEachDialect } from "@sttracker/db/testing"; @@ -41,4 +41,13 @@ forEachDialect("backup", (getDb) => { await backupHandler({ db, dataDir: dir, keep: 3 })(); expect(await readdir(join(dir, "backups"))).toHaveLength(1); }); + + test("backup files are private and no partial directory remains", async () => { + const db = getDb(); + await makeUser(db, "ada"); + const out = await runBackup(db, dir, { keep: 5 }); + expect((await stat(out)).mode & 0o777).toBe(0o700); + expect((await stat(join(out, "ada.zip"))).mode & 0o777).toBe(0o600); + expect((await readdir(join(dir, "backups"))).some((n) => n.endsWith(".tmp"))).toBe(false); + }); }); diff --git a/packages/core/test/export.test.ts b/packages/core/test/export.test.ts index 7226d07..ce42e83 100644 --- a/packages/core/test/export.test.ts +++ b/packages/core/test/export.test.ts @@ -6,6 +6,7 @@ import { strFromU8, unzipSync, zipSync, strToU8 } from "fflate"; import { forEachDialect } from "@sttracker/db/testing"; import { storeAttachment } from "../src/attachments.ts"; import { ImportError } from "../src/errors.ts"; +import { moveEntryToWork, quickAdd, upsertEntry } from "../src/entries.ts"; import { exportUserBundle, exportUserCsv, importUserBundle } from "../src/export.ts"; import { makeUser, makeWork } from "./user-layer-fixtures.ts"; @@ -33,8 +34,10 @@ forEachDialect("export bundle", (getDb) => { tags: '["a","b"]', custom_fields: "{}", visibility: "private", overrides: "{}", media_type_override: null, field_sources: '{"rating":"user"}', created_at: now, updated_at: now, deleted_at: null, ...extra, }).execute(); + const defId = crypto.randomUUID(); + await db.k.insertInto("custom_field_definitions").values({ id: defId, user_id: u, name: "Shelf", type: "text", options: "[]", applies_to: '"all"', level: "entry", created_at: now }).execute(); const e1 = crypto.randomUUID(), e2 = crypto.randomUUID(), e3 = crypto.randomUUID(), e4 = crypto.randomUUID(); - await ent(e1, shared); + await ent(e1, shared, { overrides: '{"title":"Dune X"}', visibility: "public", custom_fields: JSON.stringify({ [defId]: "top shelf" }) }); await ent(e2, child); await ent(e3, priv, { rating: null }); await ent(e4, parent, { deleted_at: now }); @@ -43,22 +46,40 @@ forEachDialect("export bundle", (getDb) => { id: eng, entry_id: e1, started_at: "1999-05", started_precision: "month", finished_at: "2000", finished_precision: "year", progress: "{}", platform: "kindle", rating: 70, notes: "n", custom_fields: "{}", field_sources: "{}", created_at: now, updated_at: now, deleted_at: null, }).execute(); + await db.k.insertInto("copies").values({ id: crypto.randomUUID(), user_id: u, work_id: shared, title_hint: null, format: "hardcover", acquired_from: "shop", acquired_at: "2020-03", acquired_precision: "month", price_cents: 1299, currency: "EUR", notes: null, custom_fields: "{}", source_ref: null, created_at: now, updated_at: now, deleted_at: null }).execute(); + await db.k.insertInto("listens").values({ id: crypto.randomUUID(), user_id: u, listened_at: now, artist: "Artist", track: "Track", album: null, duration_ms: 1000, mbids: "{}", work_id: null, source: "lastfm", source_key: "k1", created_at: now }).execute(); + const grp = crypto.randomUUID(); + await db.k.insertInto("lists").values({ id: grp, user_id: u, name: "Grouped", description: "", kind: "grouping", visibility: "public", rating: 90, review: "nice set", created_at: now, updated_at: now, deleted_at: null }).execute(); const list = crypto.randomUUID(); await db.k.insertInto("lists").values({ id: list, user_id: u, name: "Faves", description: "d", kind: "list", visibility: "private", rating: null, review: null, created_at: now, updated_at: now, deleted_at: null }).execute(); await db.k.insertInto("list_items").values({ id: crypto.randomUUID(), list_id: list, work_id: shared, position: 0, note: "first" }).execute(); await db.k.insertInto("user_history").values({ id: crypto.randomUUID(), user_id: u, entity: "entry", entity_id: e1, before: "{}", after: '{"rating":85}', actor: "user", created_at: now }).execute(); - await storeAttachment(db, dir, u, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "n.txt" }, { entity: "entry", entityId: e1 }); - return { e1, shared }; + await db.k.insertInto("user_history").values({ id: crypto.randomUUID(), user_id: u, entity: "entry", entity_id: e1, before: "{}", after: JSON.stringify({ workId: shared, note: "moved" }), actor: "user", created_at: now }).execute(); + const att = await storeAttachment(db, dir, u, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "n.txt", caption: "a caption" }, { entity: "entry", entityId: e1 }); + await db.k.insertInto("attachment_links").values({ id: crypto.randomUUID(), attachment_id: att.id, entity: "engagement", entity_id: eng }).execute(); + return { e1, shared, defId }; } test("export then import into fresh user reproduces entries, ratings, engagements with precision, lists and attachments", async () => { const db = getDb(); const u = await makeUser(db, "ada"); - await seed(db, u); + const { defId } = await seed(db, u); + // A second user's data must never leak into ada's bundle. + const other = await makeUser(db, "eve"); + const otherWork = await makeWork(db, { title: "Other Zine", mediaType: "book", ownerUserId: other }); + const oe = crypto.randomUUID(); + await db.k.insertInto("entries").values({ id: oe, user_id: other, work_id: otherWork, status: "planned", rating: null, review: null, private_notes: "secret", favorite: 0, tags: "[]", custom_fields: "{}", visibility: "private", overrides: "{}", media_type_override: null, field_sources: "{}", created_at: "2024-01-01T00:00:00.000Z", updated_at: "2024-01-01T00:00:00.000Z", deleted_at: null }).execute(); + await storeAttachment(db, dir, other, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "eve.txt" }, { entity: "entry", entityId: oe }); const zip = await exportUserBundle(db, dir, u); + const files = unzipSync(zip); + const allText = Object.entries(files).filter(([n]) => n.endsWith(".jsonl")).map(([, b]) => strFromU8(b)).join("\n"); + expect(allText).not.toContain("Other Zine"); + expect(allText).not.toContain("secret"); + expect(strFromU8(files["attachments.jsonl"]!).trim().split("\n")).toHaveLength(1); const v = await makeUser(db, "bob"); expect(await importUserBundle(db, dir2, v, zip)).toEqual({ works: 4, entries: 4 }); + expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", other).execute()).toHaveLength(1); const rows = await db.k.selectFrom("entries").innerJoin("works", "works.id", "entries.work_id").select(["works.title", "entries.rating", "entries.favorite", "entries.field_sources", "entries.deleted_at", "entries.created_at", "works.owner_user_id", "works.parent_id", "works.id as wid"]).where("entries.user_id", "=", v).execute(); const byTitle = Object.fromEntries(rows.map((r) => [r.title, r])); @@ -78,18 +99,34 @@ forEachDialect("export bundle", (getDb) => { expect(eng[0]).toMatchObject({ started_at: "1999-05", started_precision: "month", finished_precision: "year", platform: "kindle" }); const lists = await db.k.selectFrom("lists").selectAll().where("user_id", "=", v).execute(); - expect(lists).toHaveLength(1); - const items = await db.k.selectFrom("list_items").selectAll().where("list_id", "=", lists[0]!.id).execute(); + expect(lists).toHaveLength(2); + const items = await db.k.selectFrom("list_items").selectAll().where("list_id", "=", lists.find((l) => l.name === "Faves")!.id).execute(); expect(items).toMatchObject([{ note: "first", work_id: byTitle["Dune"]!.wid }]); const hist = await db.k.selectFrom("user_history").selectAll().where("user_id", "=", v).execute(); - expect(hist).toHaveLength(1); + expect(hist).toHaveLength(2); const dune = await db.k.selectFrom("entries").select("id").where("user_id", "=", v).where("work_id", "=", byTitle["Dune"]!.wid).executeTakeFirstOrThrow(); - expect(hist[0]!.entity_id).toBe(dune.id); + expect(hist.every((h) => h.entity_id === dune.id)).toBe(true); const atts = await db.k.selectFrom("attachments").innerJoin("attachment_links", "attachment_links.attachment_id", "attachments.id").selectAll("attachments").select("attachment_links.entity_id").where("attachments.user_id", "=", v).execute(); - expect(atts).toHaveLength(1); - expect(atts[0]!.entity_id).toBe(dune.id); + expect(atts).toHaveLength(2); + expect(atts.every((a) => a.caption === "a caption")).toBe(true); + const engRow = eng[0]!; + expect(atts.map((a) => a.entity_id).sort()).toEqual([dune.id, engRow.id].sort()); + + const full = await db.k.selectFrom("entries").selectAll().where("id", "=", dune.id).executeTakeFirstOrThrow(); + expect(full).toMatchObject({ visibility: "public", overrides: '{"title":"Dune X"}' }); + const newDef = await db.k.selectFrom("custom_field_definitions").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow(); + expect(newDef.id).not.toBe(defId); + expect(JSON.parse(full.custom_fields)).toEqual({ [newDef.id]: "top shelf" }); + + const copy = await db.k.selectFrom("copies").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow(); + expect(copy).toMatchObject({ price_cents: 1299, currency: "EUR", acquired_precision: "month", work_id: byTitle["Dune"]!.wid }); + expect(await db.k.selectFrom("listens").select("id").where("user_id", "=", v).execute()).toHaveLength(1); + const grouping = await db.k.selectFrom("lists").selectAll().where("user_id", "=", v).where("kind", "=", "grouping").executeTakeFirstOrThrow(); + expect(grouping).toMatchObject({ rating: 90, review: "nice set", visibility: "public" }); + const remapped = (await db.k.selectFrom("user_history").selectAll().where("user_id", "=", v).execute()).find((h) => h.after.includes("moved"))!; + expect(JSON.parse(remapped.after).workId).toBe(byTitle["Dune"]!.wid); expect(await Bun.file(join(dir2, "attachments", atts[0]!.sha256.slice(0, 2), atts[0]!.sha256.slice(2, 4), atts[0]!.sha256)).text()).toBe("hello"); }); @@ -99,8 +136,16 @@ forEachDialect("export bundle", (getDb) => { await seed(db, u); const files = unzipSync(await exportUserBundle(db, dir, u)); expect(Object.keys(files)).toContain("manifest.json"); - const text = Object.entries(files).filter(([n]) => !n.startsWith("attachments/")).map(([, b]) => strFromU8(b)).join("\n").toLowerCase(); - for (const w of ["password", "token", "credentials"]) expect(text).not.toContain(w); + const keys = new Set(); + const walk = (v: unknown) => { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === "object") for (const [k, x] of Object.entries(v)) (keys.add(k.toLowerCase().replaceAll("_", "")), walk(x)); + }; + for (const [n, b] of Object.entries(files)) { + if (n.endsWith(".jsonl")) strFromU8(b).split("\n").filter(Boolean).forEach((l) => walk(JSON.parse(l))); + } + expect(keys.size).toBeGreaterThan(10); + for (const bad of ["passwordhash", "tokenhash", "credentialsenc", "codehash"]) expect(keys.has(bad)).toBe(false); expect(JSON.parse(strFromU8(files["manifest.json"]!)).format).toBe("sttracker-export/v1"); }); @@ -132,4 +177,48 @@ forEachDialect("export bundle", (getDb) => { expect(csv["entries.csv"]).not.toContain("Show,"); expect(csv["engagements.csv"]).toContain("kindle"); }); + + test("a per-user move restores as alias plus tombstone and later upserts resolve to the moved entry", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + const draft = await quickAdd(db, u, { title: "Dune??", mediaType: "book" }); + const shared = await makeWork(db, { title: "Dune", mediaType: "book", externalIds: { isbn13: "9780441013593" } }); + await moveEntryToWork(db, u, draft.id, shared, "user"); + const zip = await exportUserBundle(db, dir, u); + + const v = await makeUser(db, "bob"); + await importUserBundle(db, dir2, v, zip); + const alias = await db.k.selectFrom("work_aliases").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow(); + expect(alias.to_work_id).toBe(shared); + const tomb = await db.k.selectFrom("works").selectAll().where("id", "=", alias.from_work_id).executeTakeFirstOrThrow(); + expect(tomb).toMatchObject({ merged_into_id: shared, owner_user_id: v }); + const moved = await upsertEntry(db, v, tomb.id, { rating: 10 }, "user"); + expect(moved.workId).toBe(shared); + expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", v).execute()).toHaveLength(1); + }); + + test("an invalid attachment aborts the import with nothing written", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const files = unzipSync(await exportUserBundle(db, dir, u)); + const key = Object.keys(files).find((n) => n.startsWith("attachments/"))!; + files[key] = new TextEncoder().encode("tampered"); + const v = await makeUser(db, "bob"); + const err = await importUserBundle(db, dir2, v, zipSync(files)).catch((e) => e); + expect(err.code).toBe("invalid_attachment"); + expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", v).execute()).toHaveLength(0); + expect(await db.k.selectFrom("attachments").select("id").where("user_id", "=", v).execute()).toHaveLength(0); + }); + + test("an out-of-range rating is a corrupt bundle", async () => { + const db = getDb(); + const u = await makeUser(db, "ada"); + await seed(db, u); + const files = unzipSync(await exportUserBundle(db, dir, u)); + files["entries.jsonl"] = strToU8(strFromU8(files["entries.jsonl"]!).replaceAll('"rating":85', '"rating":185')); + const v = await makeUser(db, "bob"); + const err = await importUserBundle(db, dir2, v, zipSync(files)).catch((e) => e); + expect(err.code).toBe("corrupt_bundle"); + }); });