Merge branch 'feat/phase1' into task/8
# Conflicts: # docs/ARCHITECTURE.md # docs/DECISIONS.md # docs/TODO.md
This commit is contained in:
Lucas Winther committed
commit
b2e6e1948b
8 files changed
+873
-12
No files matched your search
@@ -0,0 +1,53 @@
|
||||
import { afterEach, beforeEach, expect, test } from "bun:test";
|
||||
import { mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import { backupHandler, runBackup } from "../src/backup.ts";
|
||||
import { makeUser } from "./user-layer-fixtures.ts";
|
||||
|
||||
forEachDialect("backup", (getDb) => {
|
||||
let dir: string;
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), "sttracker-bak-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("backup writes a bundle per user (and the db on sqlite)", async () => {
|
||||
const db = getDb();
|
||||
await makeUser(db, "ada");
|
||||
await makeUser(db, "bob");
|
||||
const out = await runBackup(db, dir, { keep: 14 });
|
||||
expect(out.startsWith(join(dir, "backups"))).toBe(true);
|
||||
const files = (await readdir(out)).sort();
|
||||
expect(files).toEqual(db.dialect === "sqlite" ? ["ada.zip", "bob.zip", "sttracker.db"] : ["ada.zip", "bob.zip"]);
|
||||
});
|
||||
|
||||
test("backup prunes to keep", async () => {
|
||||
const db = getDb();
|
||||
await makeUser(db, "ada");
|
||||
const made: string[] = [];
|
||||
for (let i = 0; i < 4; i++) made.push(await runBackup(db, dir, { keep: 2 }));
|
||||
const left = (await readdir(join(dir, "backups"))).sort();
|
||||
expect(left).toHaveLength(2);
|
||||
expect(left.map((n) => join(dir, "backups", n))).toEqual(made.slice(2));
|
||||
});
|
||||
|
||||
test("backupHandler runs a backup", async () => {
|
||||
const db = getDb();
|
||||
await makeUser(db, "ada");
|
||||
await backupHandler({ db, dataDir: dir, keep: 3 })();
|
||||
expect(await readdir(join(dir, "backups"))).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("backup files are private and no partial directory remains", async () => {
|
||||
const db = getDb();
|
||||
await makeUser(db, "ada");
|
||||
const out = await runBackup(db, dir, { keep: 5 });
|
||||
expect((await stat(out)).mode & 0o777).toBe(0o700);
|
||||
expect((await stat(join(out, "ada.zip"))).mode & 0o777).toBe(0o600);
|
||||
expect((await readdir(join(dir, "backups"))).some((n) => n.endsWith(".tmp"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
import { afterEach, beforeEach, expect, test } from "bun:test";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { strFromU8, unzipSync, zipSync, strToU8 } from "fflate";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import { storeAttachment } from "../src/attachments.ts";
|
||||
import { ImportError } from "../src/errors.ts";
|
||||
import { moveEntryToWork, quickAdd, upsertEntry } from "../src/entries.ts";
|
||||
import { exportUserBundle, exportUserCsv, importUserBundle } from "../src/export.ts";
|
||||
import { makeUser, makeWork } from "./user-layer-fixtures.ts";
|
||||
|
||||
forEachDialect("export bundle", (getDb) => {
|
||||
let dir: string;
|
||||
let dir2: string;
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), "sttracker-exp-"));
|
||||
dir2 = await mkdtemp(join(tmpdir(), "sttracker-exp-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
await rm(dir2, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function seed(db: ReturnType<typeof getDb>, u: string) {
|
||||
const shared = await makeWork(db, { title: "Dune", mediaType: "book", year: 1965, externalIds: { hardcover: "dune" } });
|
||||
const parent = await makeWork(db, { title: "Show", mediaType: "tv", externalIds: { trakt: "show" } });
|
||||
const child = await makeWork(db, { title: "Pilot", mediaType: "tv", parentId: parent, externalIds: { trakt: "pilot" } });
|
||||
const priv = await makeWork(db, { title: "My Zine", mediaType: "book", ownerUserId: u });
|
||||
const now = "2024-01-02T03:04:05.000Z";
|
||||
const ent = async (id: string, workId: string, extra: object = {}) =>
|
||||
db.k.insertInto("entries").values({
|
||||
id, user_id: u, work_id: workId, status: "completed", rating: 85, review: "great", private_notes: null, favorite: 1,
|
||||
tags: '["a","b"]', custom_fields: "{}", visibility: "private", overrides: "{}", media_type_override: null,
|
||||
field_sources: '{"rating":"user"}', created_at: now, updated_at: now, deleted_at: null, ...extra,
|
||||
}).execute();
|
||||
const defId = crypto.randomUUID();
|
||||
await db.k.insertInto("custom_field_definitions").values({ id: defId, user_id: u, name: "Shelf", type: "text", options: "[]", applies_to: '"all"', level: "entry", created_at: now }).execute();
|
||||
const e1 = crypto.randomUUID(), e2 = crypto.randomUUID(), e3 = crypto.randomUUID(), e4 = crypto.randomUUID();
|
||||
await ent(e1, shared, { overrides: '{"title":"Dune X"}', visibility: "public", custom_fields: JSON.stringify({ [defId]: "top shelf" }) });
|
||||
await ent(e2, child);
|
||||
await ent(e3, priv, { rating: null });
|
||||
await ent(e4, parent, { deleted_at: now });
|
||||
const eng = crypto.randomUUID();
|
||||
await db.k.insertInto("engagements").values({
|
||||
id: eng, entry_id: e1, started_at: "1999-05", started_precision: "month", finished_at: "2000", finished_precision: "year",
|
||||
progress: "{}", platform: "kindle", rating: 70, notes: "n", custom_fields: "{}", field_sources: "{}", created_at: now, updated_at: now, deleted_at: null,
|
||||
}).execute();
|
||||
await db.k.insertInto("copies").values({ id: crypto.randomUUID(), user_id: u, work_id: shared, title_hint: null, format: "hardcover", acquired_from: "shop", acquired_at: "2020-03", acquired_precision: "month", price_cents: 1299, currency: "EUR", notes: null, custom_fields: "{}", source_ref: null, created_at: now, updated_at: now, deleted_at: null }).execute();
|
||||
await db.k.insertInto("listens").values({ id: crypto.randomUUID(), user_id: u, listened_at: now, artist: "Artist", track: "Track", album: null, duration_ms: 1000, mbids: "{}", work_id: null, source: "lastfm", source_key: "k1", created_at: now }).execute();
|
||||
const grp = crypto.randomUUID();
|
||||
await db.k.insertInto("lists").values({ id: grp, user_id: u, name: "Grouped", description: "", kind: "grouping", visibility: "public", rating: 90, review: "nice set", created_at: now, updated_at: now, deleted_at: null }).execute();
|
||||
const list = crypto.randomUUID();
|
||||
await db.k.insertInto("lists").values({ id: list, user_id: u, name: "Faves", description: "d", kind: "list", visibility: "private", rating: null, review: null, created_at: now, updated_at: now, deleted_at: null }).execute();
|
||||
await db.k.insertInto("list_items").values({ id: crypto.randomUUID(), list_id: list, work_id: shared, position: 0, note: "first" }).execute();
|
||||
await db.k.insertInto("user_history").values({ id: crypto.randomUUID(), user_id: u, entity: "entry", entity_id: e1, before: "{}", after: '{"rating":85}', actor: "user", created_at: now }).execute();
|
||||
await db.k.insertInto("user_history").values({ id: crypto.randomUUID(), user_id: u, entity: "entry", entity_id: e1, before: "{}", after: JSON.stringify({ workId: shared, note: "moved" }), actor: "user", created_at: now }).execute();
|
||||
const att = await storeAttachment(db, dir, u, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "n.txt", caption: "a caption" }, { entity: "entry", entityId: e1 });
|
||||
await db.k.insertInto("attachment_links").values({ id: crypto.randomUUID(), attachment_id: att.id, entity: "engagement", entity_id: eng }).execute();
|
||||
return { e1, shared, defId };
|
||||
}
|
||||
|
||||
test("export then import into fresh user reproduces entries, ratings, engagements with precision, lists and attachments", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
const { defId } = await seed(db, u);
|
||||
// A second user's data must never leak into ada's bundle.
|
||||
const other = await makeUser(db, "eve");
|
||||
const otherWork = await makeWork(db, { title: "Other Zine", mediaType: "book", ownerUserId: other });
|
||||
const oe = crypto.randomUUID();
|
||||
await db.k.insertInto("entries").values({ id: oe, user_id: other, work_id: otherWork, status: "planned", rating: null, review: null, private_notes: "secret", favorite: 0, tags: "[]", custom_fields: "{}", visibility: "private", overrides: "{}", media_type_override: null, field_sources: "{}", created_at: "2024-01-01T00:00:00.000Z", updated_at: "2024-01-01T00:00:00.000Z", deleted_at: null }).execute();
|
||||
await storeAttachment(db, dir, other, { bytes: new TextEncoder().encode("hello"), mime: "text/plain", filename: "eve.txt" }, { entity: "entry", entityId: oe });
|
||||
const zip = await exportUserBundle(db, dir, u);
|
||||
const files = unzipSync(zip);
|
||||
const allText = Object.entries(files).filter(([n]) => n.endsWith(".jsonl")).map(([, b]) => strFromU8(b)).join("\n");
|
||||
expect(allText).not.toContain("Other Zine");
|
||||
expect(allText).not.toContain("secret");
|
||||
expect(strFromU8(files["attachments.jsonl"]!).trim().split("\n")).toHaveLength(1);
|
||||
|
||||
const v = await makeUser(db, "bob");
|
||||
expect(await importUserBundle(db, dir2, v, zip)).toEqual({ works: 4, entries: 4 });
|
||||
expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", other).execute()).toHaveLength(1);
|
||||
|
||||
const rows = await db.k.selectFrom("entries").innerJoin("works", "works.id", "entries.work_id").select(["works.title", "entries.rating", "entries.favorite", "entries.field_sources", "entries.deleted_at", "entries.created_at", "works.owner_user_id", "works.parent_id", "works.id as wid"]).where("entries.user_id", "=", v).execute();
|
||||
const byTitle = Object.fromEntries(rows.map((r) => [r.title, r]));
|
||||
expect(rows).toHaveLength(4);
|
||||
expect(byTitle["Dune"]!.rating).toBe(85);
|
||||
expect(byTitle["Dune"]!.favorite).toBe(1);
|
||||
expect(byTitle["Dune"]!.field_sources).toBe('{"rating":"user"}');
|
||||
expect(byTitle["Dune"]!.created_at).toBe("2024-01-02T03:04:05.000Z");
|
||||
expect(byTitle["Show"]!.deleted_at).not.toBeNull();
|
||||
expect(byTitle["My Zine"]!.owner_user_id).toBe(v);
|
||||
expect(byTitle["Pilot"]!.parent_id).toBe(byTitle["Show"]!.wid);
|
||||
// shared works were re-matched, not duplicated
|
||||
expect(await db.k.selectFrom("works").select("id").where("title", "=", "Dune").execute()).toHaveLength(1);
|
||||
|
||||
const eng = await db.k.selectFrom("engagements").innerJoin("entries", "entries.id", "engagements.entry_id").selectAll("engagements").where("entries.user_id", "=", v).execute();
|
||||
expect(eng).toHaveLength(1);
|
||||
expect(eng[0]).toMatchObject({ started_at: "1999-05", started_precision: "month", finished_precision: "year", platform: "kindle" });
|
||||
|
||||
const lists = await db.k.selectFrom("lists").selectAll().where("user_id", "=", v).execute();
|
||||
expect(lists).toHaveLength(2);
|
||||
const items = await db.k.selectFrom("list_items").selectAll().where("list_id", "=", lists.find((l) => l.name === "Faves")!.id).execute();
|
||||
expect(items).toMatchObject([{ note: "first", work_id: byTitle["Dune"]!.wid }]);
|
||||
|
||||
const hist = await db.k.selectFrom("user_history").selectAll().where("user_id", "=", v).execute();
|
||||
expect(hist).toHaveLength(2);
|
||||
const dune = await db.k.selectFrom("entries").select("id").where("user_id", "=", v).where("work_id", "=", byTitle["Dune"]!.wid).executeTakeFirstOrThrow();
|
||||
expect(hist.every((h) => h.entity_id === dune.id)).toBe(true);
|
||||
|
||||
const atts = await db.k.selectFrom("attachments").innerJoin("attachment_links", "attachment_links.attachment_id", "attachments.id").selectAll("attachments").select("attachment_links.entity_id").where("attachments.user_id", "=", v).execute();
|
||||
expect(atts).toHaveLength(2);
|
||||
expect(atts.every((a) => a.caption === "a caption")).toBe(true);
|
||||
const engRow = eng[0]!;
|
||||
expect(atts.map((a) => a.entity_id).sort()).toEqual([dune.id, engRow.id].sort());
|
||||
|
||||
const full = await db.k.selectFrom("entries").selectAll().where("id", "=", dune.id).executeTakeFirstOrThrow();
|
||||
expect(full).toMatchObject({ visibility: "public", overrides: '{"title":"Dune X"}' });
|
||||
const newDef = await db.k.selectFrom("custom_field_definitions").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow();
|
||||
expect(newDef.id).not.toBe(defId);
|
||||
expect(JSON.parse(full.custom_fields)).toEqual({ [newDef.id]: "top shelf" });
|
||||
|
||||
const copy = await db.k.selectFrom("copies").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow();
|
||||
expect(copy).toMatchObject({ price_cents: 1299, currency: "EUR", acquired_precision: "month", work_id: byTitle["Dune"]!.wid });
|
||||
expect(await db.k.selectFrom("listens").select("id").where("user_id", "=", v).execute()).toHaveLength(1);
|
||||
const grouping = await db.k.selectFrom("lists").selectAll().where("user_id", "=", v).where("kind", "=", "grouping").executeTakeFirstOrThrow();
|
||||
expect(grouping).toMatchObject({ rating: 90, review: "nice set", visibility: "public" });
|
||||
const remapped = (await db.k.selectFrom("user_history").selectAll().where("user_id", "=", v).execute()).find((h) => h.after.includes("moved"))!;
|
||||
expect(JSON.parse(remapped.after).workId).toBe(byTitle["Dune"]!.wid);
|
||||
expect(await Bun.file(join(dir2, "attachments", atts[0]!.sha256.slice(0, 2), atts[0]!.sha256.slice(2, 4), atts[0]!.sha256)).text()).toBe("hello");
|
||||
});
|
||||
|
||||
test("bundle has no credential fields", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
await seed(db, u);
|
||||
const files = unzipSync(await exportUserBundle(db, dir, u));
|
||||
expect(Object.keys(files)).toContain("manifest.json");
|
||||
const keys = new Set<string>();
|
||||
const walk = (v: unknown) => {
|
||||
if (Array.isArray(v)) v.forEach(walk);
|
||||
else if (v && typeof v === "object") for (const [k, x] of Object.entries(v)) (keys.add(k.toLowerCase().replaceAll("_", "")), walk(x));
|
||||
};
|
||||
for (const [n, b] of Object.entries(files)) {
|
||||
if (n.endsWith(".jsonl")) strFromU8(b).split("\n").filter(Boolean).forEach((l) => walk(JSON.parse(l)));
|
||||
}
|
||||
expect(keys.size).toBeGreaterThan(10);
|
||||
for (const bad of ["passwordhash", "tokenhash", "credentialsenc", "codehash"]) expect(keys.has(bad)).toBe(false);
|
||||
expect(JSON.parse(strFromU8(files["manifest.json"]!)).format).toBe("sttracker-export/v1");
|
||||
});
|
||||
|
||||
test("import into non-empty account refused", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
await seed(db, u);
|
||||
const zip = await exportUserBundle(db, dir, u);
|
||||
const err = await importUserBundle(db, dir2, u, zip).catch((e) => e);
|
||||
expect(err).toBeInstanceOf(ImportError);
|
||||
expect(err.code).toBe("account_not_empty");
|
||||
});
|
||||
|
||||
test("import of an unknown format is refused", async () => {
|
||||
const db = getDb();
|
||||
const v = await makeUser(db, "bob");
|
||||
const zip = zipSync({ "manifest.json": strToU8('{"format":"other"}') });
|
||||
const err = await importUserBundle(db, dir2, v, zip).catch((e) => e);
|
||||
expect(err.code).toBe("unsupported_format");
|
||||
});
|
||||
|
||||
test("csv export has entries, engagements and listens", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
await seed(db, u);
|
||||
const csv = await exportUserCsv(db, u);
|
||||
expect(Object.keys(csv).sort()).toEqual(["engagements.csv", "entries.csv", "listens.csv"]);
|
||||
expect(csv["entries.csv"]).toContain("Dune");
|
||||
expect(csv["entries.csv"]).not.toContain("Show,");
|
||||
expect(csv["engagements.csv"]).toContain("kindle");
|
||||
});
|
||||
|
||||
test("a per-user move restores as alias plus tombstone and later upserts resolve to the moved entry", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
const draft = await quickAdd(db, u, { title: "Dune??", mediaType: "book" });
|
||||
const shared = await makeWork(db, { title: "Dune", mediaType: "book", externalIds: { isbn13: "9780441013593" } });
|
||||
await moveEntryToWork(db, u, draft.id, shared, "user");
|
||||
const zip = await exportUserBundle(db, dir, u);
|
||||
|
||||
const v = await makeUser(db, "bob");
|
||||
await importUserBundle(db, dir2, v, zip);
|
||||
const alias = await db.k.selectFrom("work_aliases").selectAll().where("user_id", "=", v).executeTakeFirstOrThrow();
|
||||
expect(alias.to_work_id).toBe(shared);
|
||||
const tomb = await db.k.selectFrom("works").selectAll().where("id", "=", alias.from_work_id).executeTakeFirstOrThrow();
|
||||
expect(tomb).toMatchObject({ merged_into_id: shared, owner_user_id: v });
|
||||
const moved = await upsertEntry(db, v, tomb.id, { rating: 10 }, "user");
|
||||
expect(moved.workId).toBe(shared);
|
||||
expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", v).execute()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("an invalid attachment aborts the import with nothing written", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
await seed(db, u);
|
||||
const files = unzipSync(await exportUserBundle(db, dir, u));
|
||||
const key = Object.keys(files).find((n) => n.startsWith("attachments/"))!;
|
||||
files[key] = new TextEncoder().encode("tampered");
|
||||
const v = await makeUser(db, "bob");
|
||||
const err = await importUserBundle(db, dir2, v, zipSync(files)).catch((e) => e);
|
||||
expect(err.code).toBe("invalid_attachment");
|
||||
expect(await db.k.selectFrom("entries").select("id").where("user_id", "=", v).execute()).toHaveLength(0);
|
||||
expect(await db.k.selectFrom("attachments").select("id").where("user_id", "=", v).execute()).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("an out-of-range rating is a corrupt bundle", async () => {
|
||||
const db = getDb();
|
||||
const u = await makeUser(db, "ada");
|
||||
await seed(db, u);
|
||||
const files = unzipSync(await exportUserBundle(db, dir, u));
|
||||
files["entries.jsonl"] = strToU8(strFromU8(files["entries.jsonl"]!).replaceAll('"rating":85', '"rating":185'));
|
||||
const v = await makeUser(db, "bob");
|
||||
const err = await importUserBundle(db, dir2, v, zipSync(files)).catch((e) => e);
|
||||
expect(err.code).toBe("corrupt_bundle");
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user