merge: feat/phase1 into task/6

This commit is contained in:
Lucas Winther committed 2026-10-30 19:54:33 +01:00
commit 4c65939d61
63 files changed
+4092 -2

No files matched your search

+1
View File
@@ -23,3 +23,4 @@ CLAUDE.local.md
.DS_Store
.idea/
.vscode/
.claude/worktrees/
+23
View File
@@ -81,3 +81,26 @@ instead of 26. Where `AGENTS.md` or the spec say "ULID", read "UUIDv7".
**Consequences:** Rule changes happen in one file, and the pointer files must stay thin. Plans are working
artefacts: kept after merge, they go stale and contradict the code, so the spec is the record.
## 2026-10-30: Catalogue matching is race-safe through unique keys plus whole-transaction retry
**Status:** accepted
**Context:** Two syncs matching the same new work at once both miss the lookup and both insert. On
Postgres the loser hit the `external_ids (namespace, value)` unique key and crashed the sync; seasons and
episodes had no key at all and were duplicated.
**Decision:**
- Migration `0002_works_position_unique` adds a unique index on `works (parent_id, position)`. NULLs are
distinct on both dialects, so top-level works and position-less children are unaffected.
- `matchWork` re-runs the whole resolve in a fresh transaction on a unique violation (SQLite
"UNIQUE constraint failed", Postgres SQLSTATE 23505), up to 3 attempts. Postgres aborts the transaction
on the error, so a statement-level retry is not possible.
- Adding missing external IDs to an existing work uses `ON CONFLICT DO NOTHING`.
- An ID match only counts if the work is shared or owned by the matching user, and private works can
never carry external IDs (`createWork` refuses; global merges accept only shared works).
- A global merge repoints children of `from` to `to` unless `to` already has a child at that position.
**Alternatives:**
- Advisory or table locks: dialect-specific, and SQLite already serialises writes.
- Upserts throughout `createWork`: would hide genuine conflicts and need per-dialect `RETURNING` handling.
**Consequences:** Parents resolved without IDs or position (title/year reuse) still have no unique key
and can be duplicated under concurrency; tracked in `docs/TODO.md`.
+4
View File
@@ -9,6 +9,7 @@ Small, specific, independently actionable. Remove only when actually done.
## Next
- [ ] Phase 2 and Phase 3 plans (spec §12).
- [ ] Generate image thumbnails for attachments (Phase 1 sizes images with CSS).
## Someday
@@ -23,4 +24,7 @@ Small, specific, independently actionable. Remove only when actually done.
file, and compare steps and token use.
## Known issues
- [ ] Admin merge does not merge colliding children recursively; colliding children stay under the tombstoned parent.
- [ ] Matching: a shared parent resolved without external IDs or position (title/year reuse in
`packages/core/src/matching.ts`) has no unique key, so concurrent matches can create it twice.
- [ ] `bun run typecheck` fails with TS18003 until source files exist under `apps/`, `packages/` or `plugins/`.
+176
View File
@@ -0,0 +1,176 @@
import { mkdir, rename, rm } from "node:fs/promises";
import { dirname, join } from "node:path";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import type { AttachmentEntity, AttachmentsTable } from "@sttracker/db/schema";
import { ValidationError } from "./errors.ts";
export type Attachment = AttachmentsTable;
const MAX_BYTES = 25 * 1024 * 1024;
/** Lowercase and strip parameters (`; charset=...`). */
function normalizeMime(mime: string): string {
return (mime.split(";")[0] ?? "").trim().toLowerCase();
}
function mimeAllowed(mime: string): boolean {
if (mime === "image/svg+xml") return false; // scriptable; never serve user SVG
return mime.startsWith("image/") || mime === "application/pdf" || mime === "text/plain";
}
const locks = new Map<string, Promise<void>>();
/** Serialise file mutations per sha within this process. */
async function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
const prev = locks.get(key) ?? Promise.resolve();
const run = prev.then(fn, fn);
const tail = run.then(
() => undefined,
() => undefined,
);
locks.set(key, tail);
try {
return await run;
} finally {
if (locks.get(key) === tail) locks.delete(key);
}
}
function pathFor(dataDir: string, sha: string): string {
return join(dataDir, "attachments", sha.slice(0, 2), sha.slice(2, 4), sha);
}
/**
* Store a file content-addressed by sha256 (deduplicated on disk) and link it to an entity.
* Each call creates a new attachment row and link; identical bytes share one file.
*/
export async function storeAttachment(
db: Db,
dataDir: string,
userId: string,
file: { bytes: Uint8Array; mime: string; filename: string; caption?: string; takenAt?: string },
link: { entity: AttachmentEntity; entityId: string },
): Promise<Attachment> {
const mime = normalizeMime(file.mime);
if (!mimeAllowed(mime)) throw new ValidationError("mime", "unsupported file type");
if (file.bytes.byteLength > MAX_BYTES) throw new ValidationError("file", "larger than 25 MB");
const sha256 = new Bun.CryptoHasher("sha256").update(file.bytes).digest("hex");
const path = pathFor(dataDir, sha256);
const row: Attachment = {
id: newId(),
user_id: userId,
sha256,
mime,
size: file.bytes.byteLength,
filename: file.filename,
caption: file.caption ?? null,
taken_at: file.takenAt ?? null,
visibility: null,
created_at: nowIso(),
};
await withLock(sha256, async () => {
if (!(await Bun.file(path).exists())) {
await mkdir(dirname(path), { recursive: true });
const tmp = `${path}.tmp-${newId()}`;
try {
await Bun.write(tmp, file.bytes);
await rename(tmp, path);
} catch (e) {
await rm(tmp, { force: true });
throw e;
}
}
await db.k.transaction().execute(async (tx) => {
await tx.insertInto("attachments").values(row).execute();
await tx
.insertInto("attachment_links")
.values({ id: newId(), attachment_id: row.id, entity: link.entity, entity_id: link.entityId })
.execute();
});
});
return row;
}
/** Open an attachment's file. Returns null when it does not exist or belongs to another user. */
export async function readAttachment(
db: Db,
dataDir: string,
userId: string,
id: string,
): Promise<{ file: ReturnType<typeof Bun.file>; mime: string } | null> {
const row = await db.k
.selectFrom("attachments")
.select(["sha256", "mime"])
.where("id", "=", id)
.where("user_id", "=", userId)
.executeTakeFirst();
if (!row) return null;
return { file: Bun.file(pathFor(dataDir, row.sha256)), mime: row.mime };
}
/** Attachments of `userId` linked to the given entity, oldest first. */
export async function listAttachments(
db: Db,
userId: string,
entity: AttachmentEntity,
entityId: string,
): Promise<Attachment[]> {
return db.k
.selectFrom("attachments")
.innerJoin("attachment_links", "attachment_links.attachment_id", "attachments.id")
.selectAll("attachments")
.where("attachments.user_id", "=", userId)
.where("attachment_links.entity", "=", entity)
.where("attachment_links.entity_id", "=", entityId)
.orderBy("attachments.created_at")
.orderBy("attachments.id")
.execute();
}
/**
* Remove an attachment's link to an entity. When no links remain the attachment row is
* deleted, and the file too if no other row references its content. No-op for other users.
*/
export async function unlinkAttachment(
db: Db,
dataDir: string,
userId: string,
attachmentId: string,
entity: AttachmentEntity,
entityId: string,
): Promise<void> {
const att = await db.k
.selectFrom("attachments")
.select("sha256")
.where("id", "=", attachmentId)
.where("user_id", "=", userId)
.executeTakeFirst();
if (!att) return;
await withLock(att.sha256, async () => {
const orphan = await db.k.transaction().execute(async (tx) => {
await tx
.deleteFrom("attachment_links")
.where("attachment_id", "=", attachmentId)
.where("entity", "=", entity)
.where("entity_id", "=", entityId)
.execute();
const left = await tx
.selectFrom("attachment_links")
.select("id")
.where("attachment_id", "=", attachmentId)
.executeTakeFirst();
if (left) return false;
await tx.deleteFrom("attachments").where("id", "=", attachmentId).execute();
const shared = await tx
.selectFrom("attachments")
.select("id")
.where("sha256", "=", att.sha256)
.executeTakeFirst();
return !shared;
});
if (orphan) await rm(pathFor(dataDir, att.sha256), { force: true });
});
}
+200
View File
@@ -0,0 +1,200 @@
import type { TokenScope } from "@sttracker/db/schema";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import { fromJson, toJson } from "@sttracker/db/json";
import { sql } from "kysely";
import { AuthError } from "./errors.ts";
export type User = { id: string; username: string; displayName: string; role: "admin" | "member" };
const USERNAME_RE = /^[a-z0-9_]{2,32}$/;
const DAY_MS = 86_400_000;
const SESSION_DAYS = 30;
function sha256Hex(s: string): string {
return new Bun.CryptoHasher("sha256").update(s).digest("hex");
}
let dummy: Promise<string> | undefined;
/** Fixed argon2id hash, computed once, so unknown-user logins cost the same as real ones. */
function dummyHash(): Promise<string> {
return (dummy ??= Bun.password.hash("sttracker-dummy-password", { algorithm: "argon2id" }));
}
function randomToken(bytes = 32): string {
return crypto.getRandomValues(new Uint8Array(bytes)).toBase64({ alphabet: "base64url", omitPadding: true });
}
function toUser(r: { id: string; username: string; display_name: string; role: "admin" | "member" }): User {
return { id: r.id, username: r.username, displayName: r.display_name, role: r.role };
}
export async function registerUser(
db: Db,
input: { username: string; password: string; displayName?: string; inviteCode?: string },
): Promise<User> {
const username = input.username.toLowerCase();
if (!USERNAME_RE.test(username)) throw new AuthError("invalid_username");
const passwordHash = await Bun.password.hash(input.password, { algorithm: "argon2id" });
return db.k.transaction().execute(async (tx) => {
// Serialise registrations on Postgres so two first-time sign-ups cannot both become admin.
// SQLite is already serialised by its single connection.
if (db.dialect === "postgres") await sql`select pg_advisory_xact_lock(727401)`.execute(tx);
const anyUser = await tx.selectFrom("users").select("id").limit(1).executeTakeFirst();
if (anyUser && !input.inviteCode) throw new AuthError("invite_required");
const existing = await tx.selectFrom("users").select("id").where("username", "=", username).executeTakeFirst();
if (existing) throw new AuthError("username_taken");
const id = newId();
const now = nowIso();
const role = anyUser ? "member" : "admin";
const display_name = input.displayName ?? username;
await tx
.insertInto("users")
.values({
id,
username,
display_name,
password_hash: passwordHash,
role,
preferences: toJson({}),
created_at: now,
updated_at: now,
})
.execute();
// used_by references users.id, so claim after the insert; the tx rolls back on failure.
if (anyUser) {
const claimed = await tx
.updateTable("invites")
.set({ used_by: id })
.where("code_hash", "=", sha256Hex(input.inviteCode!))
.where("used_by", "is", null)
.where("expires_at", ">", now)
.executeTakeFirst();
if (Number(claimed.numUpdatedRows) !== 1) throw new AuthError("invite_required");
}
return { id, username, displayName: display_name, role };
});
}
export async function login(
db: Db,
username: string,
password: string,
): Promise<{ user: User; sessionToken: string }> {
const row = await db.k
.selectFrom("users")
.selectAll()
.where("username", "=", username.toLowerCase())
.executeTakeFirst();
const hash = row?.password_hash ?? (await dummyHash());
const ok = await Bun.password.verify(password, hash);
if (!row || !ok) {
throw new AuthError("invalid_credentials");
}
const sessionToken = randomToken();
await db.k
.insertInto("sessions")
.values({
id: newId(),
user_id: row.id,
token_hash: sha256Hex(sessionToken),
expires_at: new Date(Date.now() + SESSION_DAYS * DAY_MS).toISOString(),
created_at: nowIso(),
})
.execute();
return { user: toUser(row), sessionToken };
}
export async function getSessionUser(db: Db, token: string): Promise<User | null> {
const row = await db.k
.selectFrom("sessions")
.innerJoin("users", "users.id", "sessions.user_id")
.select(["users.id", "users.username", "users.display_name", "users.role"])
.where("sessions.token_hash", "=", sha256Hex(token))
.where("sessions.expires_at", ">", nowIso())
.executeTakeFirst();
return row ? toUser(row) : null;
}
export async function logout(db: Db, token: string): Promise<void> {
await db.k.deleteFrom("sessions").where("token_hash", "=", sha256Hex(token)).execute();
}
/** Admin-only. Returns the invite code (shown once; only its hash is stored). */
export async function createInvite(db: Db, adminId: string, ttlDays = 7): Promise<string> {
const admin = await db.k.selectFrom("users").select("role").where("id", "=", adminId).executeTakeFirst();
if (admin?.role !== "admin") throw new AuthError("forbidden");
const code = randomToken(16);
await db.k
.insertInto("invites")
.values({
id: newId(),
code_hash: sha256Hex(code),
created_by: adminId,
used_by: null,
expires_at: new Date(Date.now() + ttlDays * DAY_MS).toISOString(),
created_at: nowIso(),
})
.execute();
return code;
}
export async function createAppToken(
db: Db,
userId: string,
name: string,
scopes: TokenScope[],
): Promise<{ id: string; token: string }> {
const id = newId();
const token = `stt_${randomToken()}`;
await db.k
.insertInto("app_tokens")
.values({
id,
user_id: userId,
name,
token_hash: sha256Hex(token),
scopes: toJson(scopes),
last_used_at: null,
revoked_at: null,
created_at: nowIso(),
})
.execute();
return { id, token };
}
export async function verifyAppToken(db: Db, token: string, scope: TokenScope): Promise<User | null> {
const row = await db.k
.selectFrom("app_tokens")
.innerJoin("users", "users.id", "app_tokens.user_id")
.select([
"app_tokens.id as token_id",
"app_tokens.scopes",
"users.id",
"users.username",
"users.display_name",
"users.role",
])
.where("app_tokens.token_hash", "=", sha256Hex(token))
.where("app_tokens.revoked_at", "is", null)
.executeTakeFirst();
if (!row || !fromJson<string[]>(row.scopes, []).includes(scope)) return null;
await db.k.updateTable("app_tokens").set({ last_used_at: nowIso() }).where("id", "=", row.token_id).execute();
return toUser(row);
}
export async function revokeAppToken(db: Db, userId: string, id: string): Promise<void> {
await db.k
.updateTable("app_tokens")
.set({ revoked_at: nowIso() })
.where("id", "=", id)
.where("user_id", "=", userId)
.where("revoked_at", "is", null)
.execute();
}
+45
View File
@@ -0,0 +1,45 @@
import { CryptoError } from "./errors.ts";
const SALT = new TextEncoder().encode("sttracker-v1");
const INFO = new TextEncoder().encode("sttracker-credentials");
function b64(bytes: Uint8Array): string {
return Buffer.from(bytes).toString("base64");
}
async function deriveKey(secret: string): Promise<CryptoKey> {
const material = await crypto.subtle.importKey("raw", new TextEncoder().encode(secret), "HKDF", false, [
"deriveKey",
]);
return crypto.subtle.deriveKey(
{ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO },
material,
{ name: "AES-GCM", length: 256 },
false,
["encrypt", "decrypt"],
);
}
/** Encrypt a JSON value with AES-256-GCM. Returns `v1.<b64 iv>.<b64 ciphertext>`. */
export async function encryptJson(secret: string, value: unknown): Promise<string> {
const key = await deriveKey(secret);
const iv = crypto.getRandomValues(new Uint8Array(12));
const plain = new TextEncoder().encode(JSON.stringify(value));
const ct = new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv }, key, plain));
return `v1.${b64(iv)}.${b64(ct)}`;
}
/** Decrypt a blob from {@link encryptJson}. Throws `CryptoError` on a wrong secret or bad blob. */
export async function decryptJson<T>(secret: string, blob: string): Promise<T> {
const parts = blob.split(".");
if (parts.length !== 3 || parts[0] !== "v1") throw new CryptoError("malformed credential blob");
try {
const iv = new Uint8Array(Buffer.from(parts[1]!, "base64"));
const ct = new Uint8Array(Buffer.from(parts[2]!, "base64"));
const key = await deriveKey(secret);
const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ct);
return JSON.parse(new TextDecoder().decode(plain)) as T;
} catch (cause) {
throw new CryptoError("decryption failed", { cause });
}
}
+192
View File
@@ -0,0 +1,192 @@
import { sql } from "kysely";
import type { WorkHint } from "@sttracker/plugin-sdk/types";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import {
createWorkIn,
finalWorkId,
findWorkByExternalIdsIn,
normalizePosition,
resolveAliasIn,
toWork,
updateWorkFromHintIn,
type K,
} from "./works.ts";
export interface FuzzyInput {
title: string;
year?: number | null;
creators?: { name: string }[];
}
function normalize(s: string): string {
return s
.normalize("NFD")
.replace(/\p{M}+/gu, "")
.toLowerCase()
.replace(/[^\p{L}\p{N}]+/gu, " ")
.trim()
.replace(/^(the|a|an)\s+/, "");
}
function tokens(s: string): string[] {
return normalize(s).split(" ").filter(Boolean);
}
/**
* 0..1 similarity. Title token-set Jaccard weighs 0.7; year 0.2 (±1 year counts half); any shared
* creator 0.1. A component missing on either side is left out and the weights renormalised, so a
* title-only comparison can still reach 1.
*/
export function fuzzyScore(a: FuzzyInput, b: FuzzyInput): number {
const ta = new Set(tokens(a.title));
const tb = new Set(tokens(b.title));
let inter = 0;
for (const t of ta) if (tb.has(t)) inter++;
const union = ta.size + tb.size - inter;
const jaccard = union === 0 ? 0 : inter / union;
let score = jaccard * 0.7;
let weight = 0.7;
if (a.year != null && b.year != null) {
const d = Math.abs(a.year - b.year);
score += d === 0 ? 0.2 : d === 1 ? 0.1 : 0;
weight += 0.2;
}
const ca = new Set((a.creators ?? []).map((c) => normalize(c.name)).filter(Boolean));
const cb = (b.creators ?? []).map((c) => normalize(c.name)).filter(Boolean);
if (ca.size > 0 && cb.length > 0) {
if (cb.some((n) => ca.has(n))) score += 0.1;
weight += 0.1;
}
return score / weight;
}
interface Resolved {
workId: string;
created: boolean;
draft: boolean;
}
async function resolve(k: K, userId: string | null, hint: WorkHint, actor: string): Promise<Resolved> {
// 1. Any external ID on a work this user may see (parents resolve with userId null: shared only).
const byId = await findWorkByExternalIdsIn(k, hint.externalIds, userId);
if (byId) {
await updateWorkFromHintIn(k, byId, hint, actor);
const workId = userId ? await resolveAliasIn(k, userId, byId) : byId;
const owner = await k.selectFrom("works").select("owner_user_id").where("id", "=", workId).executeTakeFirstOrThrow();
return { workId, created: false, draft: owner.owner_user_id !== null };
}
// 2. Parent + position.
const parentId = hint.parent ? (await resolve(k, null, hint.parent, actor)).workId : null;
const position = normalizePosition(hint.position);
if (parentId && position) {
const child = await k
.selectFrom("works")
.select("id")
.where("parent_id", "=", parentId)
.where("position", "=", position)
.where("owner_user_id", "is", null)
.executeTakeFirst();
if (child) {
// A merged tombstone still holds its (parent_id, position) slot: follow it to the surviving work.
const workId = await finalWorkId(k, child.id);
await updateWorkFromHintIn(k, workId, hint, actor);
return { workId, created: false, draft: false };
}
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
}
// 3. External IDs but no match: new shared work.
if (Object.keys(hint.externalIds).length > 0) {
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
}
if (userId === null) {
// Parent of a shared child: reuse an identical shared work, else create one.
let q = k
.selectFrom("works")
.select("id")
.where("owner_user_id", "is", null)
.where("merged_into_id", "is", null)
.where("media_type", "=", hint.mediaType)
.where(sql<boolean>`lower(title) = ${hint.title.toLowerCase()}`);
q = hint.year === undefined ? q : q.where("year", "=", hint.year);
q = parentId ? q.where("parent_id", "=", parentId) : q.where("parent_id", "is", null);
const same = await q.executeTakeFirst();
if (same) return { workId: same.id, created: false, draft: false };
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
}
// 4. Private draft plus fuzzy suggestions.
const workId = await createWorkIn(k, { hint, ownerUserId: userId }, actor);
await suggest(k, userId, workId, hint);
return { workId, created: true, draft: true };
}
async function suggest(k: K, userId: string, draftId: string, hint: WorkHint): Promise<void> {
// Bound the candidate set with LIKE on the significant title tokens (OR), so "The Expanse: Leviathan
// Wakes" still finds "Leviathan Wakes". Tokens of 3+ chars, first five; falls back to the first token.
const all = tokens(hint.title);
const sig = all.filter((t) => t.length >= 3).slice(0, 5);
const likeTokens = sig.length > 0 ? sig : all.slice(0, 1);
if (likeTokens.length === 0) return;
const rows = await k
.selectFrom("works")
.selectAll()
.where("merged_into_id", "is", null)
.where("id", "!=", draftId)
.where((eb) =>
eb.or([
eb.and([eb("owner_user_id", "is", null), eb("media_type", "=", hint.mediaType)]),
eb("owner_user_id", "=", userId),
]),
)
.where((eb) =>
eb.or(likeTokens.map((t) => eb(sql<string>`lower(title)`, "like", `%${t}%`))),
)
.limit(200)
.execute();
const scored = rows
.map(toWork)
.map((w) => ({ id: w.id, score: fuzzyScore(hint, w) }))
.filter((c) => c.score >= 0.6)
.sort((a, b) => b.score - a.score)
.slice(0, 5);
const now = nowIso();
for (const c of scored) {
await k
.insertInto("match_suggestions")
.values({ id: newId(), user_id: userId, work_id: draftId, candidate_work_id: c.id, score: c.score, status: "pending", created_at: now })
.execute();
}
}
const MAX_ATTEMPTS = 3;
/** SQLite "UNIQUE constraint failed", or Postgres SQLSTATE 23505, anywhere in the cause chain. */
export function isUniqueViolation(err: unknown): boolean {
for (let e: unknown = err, depth = 0; e && typeof e === "object" && depth < 5; e = (e as { cause?: unknown }).cause, depth++) {
const { code, errno, message } = e as { code?: unknown; errno?: unknown; message?: unknown };
if (code === "23505" || errno === "23505") return true;
if (typeof message === "string" && (message.includes("UNIQUE constraint failed") || message.includes("duplicate key"))) {
return true;
}
}
return false;
}
/**
* Resolve a hint to a work per spec §8.4. Never auto-links on fuzzy matches; those become suggestions.
* A concurrent match may create the same work first (unique external id or parent+position); the losing
* transaction is discarded and the whole resolve re-run, which then finds the winner's rows.
*/
export async function matchWork(db: Db, userId: string, hint: WorkHint, actor: string): Promise<Resolved> {
for (let attempt = 1; ; attempt++) {
try {
return await db.k.transaction().execute((tx) => resolve(tx, userId, hint, actor));
} catch (err) {
if (attempt >= MAX_ATTEMPTS || !isUniqueViolation(err)) throw err;
}
}
}
+330
View File
@@ -0,0 +1,330 @@
import { sql, type Kysely } from "kysely";
import type { MediaType, WorkHint } from "@sttracker/plugin-sdk/types";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import { fromJson, toJson } from "@sttracker/db/json";
import type { Database, WorksTable } from "@sttracker/db/schema";
import { AuthError, NotFoundError, ValidationError } from "./errors.ts";
export type Creator = { name: string; role?: string };
export type Position = { season?: number; episode?: number; index?: number };
/** A `works` row with JSON columns parsed. Other fields keep their column names. */
export interface Work extends Omit<WorksTable, "creators" | "metadata" | "position"> {
creators: Creator[];
metadata: Record<string, unknown>;
position: Position | null;
}
/** Any Kysely handle: the root one or a transaction. */
export type K = Kysely<Database>;
/** Normalised JSON for `works.position`: keys in a fixed order (season, episode, index), or null when empty. */
export function normalizePosition(p: Position | undefined | null): string | null {
if (!p) return null;
const o: Position = {};
if (p.season !== undefined) o.season = p.season;
if (p.episode !== undefined) o.episode = p.episode;
if (p.index !== undefined) o.index = p.index;
return Object.keys(o).length === 0 ? null : toJson(o);
}
export function toWork(r: WorksTable): Work {
return {
...r,
creators: fromJson<Creator[]>(r.creators, []),
metadata: fromJson<Record<string, unknown>>(r.metadata, {}),
position: fromJson<Position | null>(r.position, null),
};
}
export async function createWorkIn(
k: K,
input: { hint: WorkHint; ownerUserId?: string | null; parentId?: string | null },
actor: string,
): Promise<string> {
const { hint } = input;
if (hint.title.trim() === "") throw new ValidationError("title", "required");
if (input.ownerUserId && Object.keys(hint.externalIds).length > 0) {
throw new ValidationError("externalIds", "private works cannot carry external ids");
}
const id = newId();
const now = nowIso();
await k
.insertInto("works")
.values({
id,
media_type: hint.mediaType,
title: hint.title,
original_title: "",
year: hint.year ?? null,
release_date: null,
creators: toJson(hint.creators ?? []),
description: hint.description ?? "",
cover_url: hint.coverUrl ?? "",
cover_attachment_id: null,
metadata: "{}",
parent_id: input.parentId ?? null,
position: normalizePosition(hint.position),
owner_user_id: input.ownerUserId ?? null,
merged_into_id: null,
created_at: now,
updated_at: now,
})
.execute();
for (const [namespace, value] of Object.entries(hint.externalIds)) {
await k.insertInto("external_ids").values({ id: newId(), work_id: id, namespace, value }).execute();
}
await k
.insertInto("work_history")
.values({ id: newId(), work_id: id, before: "{}", after: toJson({ created: true, title: hint.title }), actor, created_at: now })
.execute();
return id;
}
/** Insert the work and its external IDs. */
export function createWork(
db: Db,
input: { hint: WorkHint; ownerUserId?: string | null; parentId?: string | null },
actor: string,
): Promise<string> {
return db.k.transaction().execute((tx) => createWorkIn(tx, input, actor));
}
export async function getWorkIn(k: K, id: string): Promise<Work | null> {
const seen = new Set<string>();
let cur: string | null = id;
while (cur && !seen.has(cur)) {
seen.add(cur);
const row: WorksTable | undefined = await k.selectFrom("works").selectAll().where("id", "=", cur).executeTakeFirst();
if (!row) return null;
if (!row.merged_into_id) return toWork(row);
cur = row.merged_into_id;
}
return null;
}
/** Fetch a work, following `merged_into_id` tombstones to the surviving one. */
export function getWork(db: Db, id: string): Promise<Work | null> {
return getWorkIn(db.k, id);
}
/** Follow `merged_into_id` to the surviving id (the input id if it is not merged). */
export async function finalWorkId(k: K, id: string): Promise<string> {
return (await getWorkIn(k, id))?.id ?? id;
}
/**
* First work matching any (namespace, value) pair, following merges. Only works visible to `userId`
* count (shared, or private to that user); with no user, only shared works. Others are treated as absent.
*/
export async function findWorkByExternalIdsIn(
k: K,
ids: Record<string, string>,
userId: string | null = null,
): Promise<string | null> {
for (const [namespace, value] of Object.entries(ids)) {
const row = await k
.selectFrom("external_ids")
.select("work_id")
.where("namespace", "=", namespace)
.where("value", "=", value)
.executeTakeFirst();
if (!row) continue;
const work = await getWorkIn(k, row.work_id);
if (work && (work.owner_user_id === null || work.owner_user_id === userId)) return work.id;
}
return null;
}
/** First work visible to `userId` (shared works only when omitted) matching any (namespace, value) pair. */
export function findWorkByExternalIds(db: Db, ids: Record<string, string>, userId: string | null = null): Promise<string | null> {
return findWorkByExternalIdsIn(db.k, ids, userId);
}
async function addMissingExternalIds(k: K, workId: string, ids: Record<string, string>): Promise<string[]> {
const added: string[] = [];
for (const [namespace, value] of Object.entries(ids)) {
const exists = await k
.selectFrom("external_ids")
.select("id")
.where("namespace", "=", namespace)
.where("value", "=", value)
.executeTakeFirst();
if (exists) continue;
// A concurrent sync may have added the same pair since the check: skip it rather than fail.
const res = await k
.insertInto("external_ids")
.values({ id: newId(), work_id: workId, namespace, value })
.onConflict((oc) => oc.columns(["namespace", "value"]).doNothing())
.executeTakeFirst();
if (Number(res.numInsertedOrUpdatedRows ?? 0n) > 0) added.push(`${namespace}:${value}`);
}
return added;
}
export async function updateWorkFromHintIn(k: K, workId: string, hint: WorkHint, actor: string): Promise<void> {
const row = await k.selectFrom("works").selectAll().where("id", "=", workId).executeTakeFirst();
if (!row || row.owner_user_id !== null) return; // private works are never touched by syncs
const before: Record<string, unknown> = {};
const after: Record<string, unknown> = {};
const patch: Partial<WorksTable> = {};
const set = <F extends "title" | "year" | "creators" | "description" | "cover_url">(field: F, next: WorksTable[F] | undefined, empty: boolean) => {
if (next === undefined || empty || next === row[field]) return;
before[field] = row[field];
after[field] = next;
patch[field] = next;
};
set("title", hint.title, hint.title.trim() === "");
set("year", hint.year, hint.year === undefined);
set("creators", hint.creators ? toJson(hint.creators) : undefined, !hint.creators || hint.creators.length === 0);
set("description", hint.description, !hint.description);
set("cover_url", hint.coverUrl, !hint.coverUrl);
const addedIds = await addMissingExternalIds(k, workId, hint.externalIds);
if (addedIds.length > 0) after.externalIds = addedIds;
if (Object.keys(after).length === 0) return;
const now = nowIso();
if (Object.keys(patch).length > 0) {
await k.updateTable("works").set({ ...patch, updated_at: now }).where("id", "=", workId).execute();
}
await k
.insertInto("work_history")
.values({ id: newId(), work_id: workId, before: toJson(before), after: toJson(after), actor, created_at: now })
.execute();
}
/** Fill or update a shared work's display fields from a hint, add missing external IDs, log to `work_history`. */
export function updateWorkFromHint(db: Db, workId: string, hint: WorkHint, actor: string): Promise<void> {
return db.k.transaction().execute((tx) => updateWorkFromHintIn(tx, workId, hint, actor));
}
function escapeLike(s: string): string {
return s.replace(/[!%_]/g, (c) => `!${c}`);
}
/** Shared works plus the user's own private ones, case-insensitive substring match on title. */
export async function searchWorks(
db: Db,
userId: string,
q: string,
opts: { mediaType?: MediaType; limit?: number } = {},
): Promise<Work[]> {
let qb = db.k
.selectFrom("works")
.selectAll()
.where("merged_into_id", "is", null)
.where((eb) => eb.or([eb("owner_user_id", "is", null), eb("owner_user_id", "=", userId)]))
.where(sql<boolean>`lower(title) like ${`%${escapeLike(q.toLowerCase())}%`} escape '!'`);
if (opts.mediaType) qb = qb.where("media_type", "=", opts.mediaType);
const rows = await qb.orderBy("title").limit(opts.limit ?? 50).execute();
return rows.map(toWork);
}
/** Follow the user's aliases and global merge tombstones to the work id they should see. */
export async function resolveAliasIn(k: K, userId: string, workId: string): Promise<string> {
const seen = new Set<string>();
let cur = workId;
while (!seen.has(cur)) {
seen.add(cur);
const alias = await k
.selectFrom("work_aliases")
.select("to_work_id")
.where("user_id", "=", userId)
.where("from_work_id", "=", cur)
.executeTakeFirst();
if (alias) {
cur = alias.to_work_id;
continue;
}
const w = await k.selectFrom("works").select("merged_into_id").where("id", "=", cur).executeTakeFirst();
if (!w?.merged_into_id) break;
cur = w.merged_into_id;
}
return cur;
}
export function resolveAlias(db: Db, userId: string, workId: string): Promise<string> {
return resolveAliasIn(db.k, userId, workId);
}
/**
* Admin-only global merge of shared work `fromId` into `toId`. Leaves a `merged_into_id` tombstone and
* moves external IDs and the children whose position `toId` does not already have. Both works must
* be shared. Users without an entry on `toId` get their entry repointed; users who already
* have one keep their `from` entry and get a pending `merged` change event.
*/
export async function adminMergeWorks(db: Db, adminUserId: string, fromId: string, toId: string): Promise<void> {
await db.k.transaction().execute(async (tx) => {
const admin = await tx.selectFrom("users").select("role").where("id", "=", adminUserId).executeTakeFirst();
if (admin?.role !== "admin") throw new AuthError("forbidden");
if (fromId === toId) throw new ValidationError("toId", "cannot merge a work into itself");
const from = await tx.selectFrom("works").selectAll().where("id", "=", fromId).executeTakeFirst();
const to = await tx.selectFrom("works").selectAll().where("id", "=", toId).executeTakeFirst();
if (!from) throw new NotFoundError("work");
if (!to) throw new NotFoundError("work");
if (from.merged_into_id) throw new ValidationError("fromId", "already merged");
if (to.merged_into_id) throw new ValidationError("toId", "already merged");
if (from.owner_user_id !== null) throw new ValidationError("fromId", "only shared works can be merged");
if (to.owner_user_id !== null) throw new ValidationError("toId", "only shared works can be merged");
const now = nowIso();
await tx.updateTable("works").set({ merged_into_id: toId, updated_at: now }).where("id", "=", fromId).execute();
const moved = await tx.selectFrom("external_ids").select(["namespace", "value"]).where("work_id", "=", fromId).execute();
await tx.updateTable("external_ids").set({ work_id: toId }).where("work_id", "=", fromId).execute();
// Repoint children so matching through `toId` finds them. A child whose position `toId` already
// has stays under `fromId` (unique parent_id+position); it remains reachable via the tombstone.
const toPositions = new Set(
(await tx.selectFrom("works").select("position").where("parent_id", "=", toId).where("position", "is not", null).execute()).map(
(r) => r.position,
),
);
const children = await tx.selectFrom("works").select(["id", "position"]).where("parent_id", "=", fromId).execute();
const movedChildren: string[] = [];
for (const c of children) {
if (c.position !== null && toPositions.has(c.position)) continue;
await tx.updateTable("works").set({ parent_id: toId, updated_at: now }).where("id", "=", c.id).execute();
movedChildren.push(c.id);
}
const entries = await tx.selectFrom("entries").select(["id", "user_id"]).where("work_id", "=", fromId).execute();
for (const e of entries) {
const existing = await tx
.selectFrom("entries")
.select("id")
.where("user_id", "=", e.user_id)
.where("work_id", "=", toId)
.executeTakeFirst();
if (existing) {
await tx
.insertInto("change_events")
.values({
id: newId(),
user_id: e.user_id,
connection_id: null,
run_id: null,
record_id: null,
type: "merged",
status: "pending",
target_entity: "entry",
target_id: e.id,
payload: toJson({ fromWorkId: fromId, toWorkId: toId }),
created_at: now,
resolved_at: null,
})
.execute();
} else {
await tx.updateTable("entries").set({ work_id: toId, updated_at: now }).where("id", "=", e.id).execute();
}
}
await tx
.insertInto("work_history")
.values({ id: newId(), work_id: fromId, before: toJson({ merged_into_id: null }), after: toJson({ merged_into_id: toId }), actor: adminUserId, created_at: now })
.execute();
await tx
.insertInto("work_history")
.values({ id: newId(), work_id: toId, before: "{}", after: toJson({ mergedFrom: fromId, externalIds: moved, children: movedChildren }), actor: adminUserId, created_at: now })
.execute();
});
}
+173
View File
@@ -0,0 +1,173 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { mkdtemp, readdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { newId, nowIso } from "@sttracker/db/ids";
import { forEachDialect } from "@sttracker/db/testing";
import type { Db } from "@sttracker/db/db";
import {
listAttachments,
readAttachment,
storeAttachment,
unlinkAttachment,
} from "../src/attachments.ts";
import { ValidationError } from "../src/errors.ts";
async function addUser(db: Db, username: string): Promise<string> {
const id = newId();
const now = nowIso();
await db.k
.insertInto("users")
.values({
id,
username,
display_name: username,
password_hash: "x",
role: "member",
preferences: "{}",
created_at: now,
updated_at: now,
})
.execute();
return id;
}
async function filesUnder(dir: string): Promise<string[]> {
const all = await readdir(dir, { recursive: true, withFileTypes: true });
return all.filter((e) => e.isFile()).map((e) => join(e.parentPath, e.name));
}
forEachDialect("attachments", (getDb) => {
let dataDir: string;
beforeEach(async () => {
dataDir = await mkdtemp(join(tmpdir(), "sttracker-att-"));
});
afterEach(async () => {
await rm(dataDir, { recursive: true, force: true });
});
const png = (s: string) => ({
bytes: new TextEncoder().encode(s),
mime: "image/png",
filename: "a.png",
});
test("same bytes twice: one file, two links", async () => {
const db = getDb();
const u = await addUser(db, "u1");
const a = await storeAttachment(db, dataDir, u, png("hello"), { entity: "work", entityId: "w1" });
const b = await storeAttachment(db, dataDir, u, png("hello"), { entity: "entry", entityId: "e1" });
expect(a.sha256).toBe(b.sha256);
expect(a.id).not.toBe(b.id);
const files = await filesUnder(join(dataDir, "attachments"));
expect(files).toHaveLength(1);
expect(files[0]).toBe(
join(dataDir, "attachments", a.sha256.slice(0, 2), a.sha256.slice(2, 4), a.sha256),
);
const links = await db.k.selectFrom("attachment_links").selectAll().execute();
expect(links).toHaveLength(2);
expect(await listAttachments(db, u, "work", "w1")).toHaveLength(1);
});
test("rejects disallowed mime and oversize", async () => {
const db = getDb();
const u = await addUser(db, "u1");
const link = { entity: "work", entityId: "w1" } as const;
await expect(
storeAttachment(db, dataDir, u, { ...png("x"), mime: "application/x-msdownload", filename: "a.exe" }, link),
).rejects.toBeInstanceOf(ValidationError);
await expect(
storeAttachment(db, dataDir, u, { ...png("x"), bytes: new Uint8Array(25 * 1024 * 1024 + 1) }, link),
).rejects.toBeInstanceOf(ValidationError);
});
test("reads own file; another user gets null", async () => {
const db = getDb();
const u1 = await addUser(db, "u1");
const u2 = await addUser(db, "u2");
const a = await storeAttachment(db, dataDir, u1, png("secret"), { entity: "work", entityId: "w1" });
const r = await readAttachment(db, dataDir, u1, a.id);
expect(r?.mime).toBe("image/png");
expect(await r?.file.text()).toBe("secret");
expect(await readAttachment(db, dataDir, u2, a.id)).toBeNull();
expect(await readAttachment(db, dataDir, u1, "nope")).toBeNull();
});
test("unlink removes link, orphan row and unshared file", async () => {
const db = getDb();
const u = await addUser(db, "u1");
const a = await storeAttachment(db, dataDir, u, png("bye"), { entity: "work", entityId: "w1" });
const b = await storeAttachment(db, dataDir, u, png("bye"), { entity: "work", entityId: "w2" });
await unlinkAttachment(db, dataDir, u, a.id, "work", "w1");
expect(await listAttachments(db, u, "work", "w1")).toHaveLength(0);
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
await unlinkAttachment(db, dataDir, u, b.id, "work", "w2");
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(0);
expect(await db.k.selectFrom("attachments").selectAll().execute()).toHaveLength(0);
});
test("rejects svg and normalizes mime", async () => {
const db = getDb();
const u = await addUser(db, "u1");
const link = { entity: "work", entityId: "w1" } as const;
for (const mime of ["image/svg+xml", "IMAGE/SVG+XML; charset=utf-8"]) {
await expect(storeAttachment(db, dataDir, u, { ...png("x"), mime }, link)).rejects.toBeInstanceOf(
ValidationError,
);
}
const a = await storeAttachment(db, dataDir, u, { ...png("x"), mime: "Image/PNG; foo=bar" }, link);
expect(a.mime).toBe("image/png");
expect((await listAttachments(db, u, "work", "w1"))[0]?.mime).toBe("image/png");
});
test("shared bytes across users survive the other's unlink", async () => {
const db = getDb();
const ua = await addUser(db, "a");
const ub = await addUser(db, "b");
const a = await storeAttachment(db, dataDir, ua, png("same"), { entity: "work", entityId: "w1" });
const b = await storeAttachment(db, dataDir, ub, png("same"), { entity: "work", entityId: "w1" });
await unlinkAttachment(db, dataDir, ua, a.id, "work", "w1");
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
const r = await readAttachment(db, dataDir, ub, b.id);
expect(await r?.file.text()).toBe("same");
});
test("another user's unlink is a no-op", async () => {
const db = getDb();
const ua = await addUser(db, "a");
const ub = await addUser(db, "b");
const a = await storeAttachment(db, dataDir, ua, png("mine"), { entity: "work", entityId: "w1" });
await unlinkAttachment(db, dataDir, ub, a.id, "work", "w1");
expect(await listAttachments(db, ua, "work", "w1")).toHaveLength(1);
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
});
test("listAttachments is scoped to the user", async () => {
const db = getDb();
const ua = await addUser(db, "a");
const ub = await addUser(db, "b");
await storeAttachment(db, dataDir, ua, png("one"), { entity: "work", entityId: "w1" });
expect(await listAttachments(db, ub, "work", "w1")).toHaveLength(0);
expect(await listAttachments(db, ua, "work", "w1")).toHaveLength(1);
});
test("concurrent store and unlink never leave a row without its file", async () => {
const db = getDb();
const u = await addUser(db, "u1");
const first = await storeAttachment(db, dataDir, u, png("race"), { entity: "work", entityId: "w1" });
const [, second] = await Promise.all([
unlinkAttachment(db, dataDir, u, first.id, "work", "w1"),
storeAttachment(db, dataDir, u, png("race"), { entity: "work", entityId: "w2" }),
]);
const r = await readAttachment(db, dataDir, u, second.id);
expect(await r?.file.text()).toBe("race");
});
test("leaves no temp files behind", async () => {
const db = getDb();
const u = await addUser(db, "u1");
await storeAttachment(db, dataDir, u, png("clean"), { entity: "work", entityId: "w1" });
const files = await filesUnder(join(dataDir, "attachments"));
expect(files.some((f) => f.includes(".tmp-"))).toBe(false);
});
});
+155
View File
@@ -0,0 +1,155 @@
import { expect, test } from "bun:test";
import { forEachDialect } from "@sttracker/db/testing";
import {
createAppToken,
createInvite,
getSessionUser,
login,
logout,
registerUser,
revokeAppToken,
verifyAppToken,
} from "../src/auth.ts";
import { AuthError } from "../src/errors.ts";
async function code(p: Promise<unknown>): Promise<string | undefined> {
try {
await p;
} catch (e) {
return e instanceof AuthError ? e.code : "other";
}
return undefined;
}
forEachDialect("auth", (getDb) => {
test("first user is admin and username is lowercased", async () => {
const u = await registerUser(getDb(), { username: "Alice", password: "pw-one" });
expect(u.role).toBe("admin");
expect(u.username).toBe("alice");
expect(u.displayName).toBe("alice");
});
test("invalid and duplicate usernames are rejected", async () => {
const db = getDb();
expect(await code(registerUser(db, { username: "a b", password: "x" }))).toBe("invalid_username");
await registerUser(db, { username: "alice", password: "x" });
const inv = await createInvite(db, (await login(db, "alice", "x")).user.id);
expect(await code(registerUser(db, { username: "ALICE", password: "x", inviteCode: inv }))).toBe(
"username_taken",
);
});
test("second user needs a single-use invite", async () => {
const db = getDb();
const admin = await registerUser(db, { username: "alice", password: "x" });
expect(await code(registerUser(db, { username: "bob", password: "x" }))).toBe("invite_required");
expect(await code(registerUser(db, { username: "bob", password: "x", inviteCode: "nope" }))).toBe(
"invite_required",
);
const inv = await createInvite(db, admin.id);
const bob = await registerUser(db, { username: "bob", password: "x", inviteCode: inv });
expect(bob.role).toBe("member");
expect(await code(registerUser(db, { username: "carol", password: "x", inviteCode: inv }))).toBe(
"invite_required",
);
});
test("expired invite is rejected; members cannot create invites", async () => {
const db = getDb();
const admin = await registerUser(db, { username: "alice", password: "x" });
const inv = await createInvite(db, admin.id, -1);
expect(await code(registerUser(db, { username: "bob", password: "x", inviteCode: inv }))).toBe(
"invite_required",
);
const ok = await createInvite(db, admin.id);
const bob = await registerUser(db, { username: "bob", password: "x", inviteCode: ok });
expect(await code(createInvite(db, bob.id))).toBe("forbidden");
});
test("wrong password or unknown user -> invalid_credentials", async () => {
const db = getDb();
await registerUser(db, { username: "alice", password: "right" });
expect(await code(login(db, "alice", "wrong"))).toBe("invalid_credentials");
expect(await code(login(db, "nobody", "x"))).toBe("invalid_credentials");
});
test("session round-trip and logout", async () => {
const db = getDb();
await registerUser(db, { username: "alice", password: "pw" });
const { user, sessionToken } = await login(db, "Alice", "pw");
expect(await getSessionUser(db, sessionToken)).toEqual(user);
const stored = await db.k.selectFrom("sessions").selectAll().execute();
expect(stored[0]!.token_hash).not.toBe(sessionToken);
await logout(db, sessionToken);
expect(await getSessionUser(db, sessionToken)).toBeNull();
expect(await getSessionUser(db, "garbage")).toBeNull();
});
test("expired session returns null", async () => {
const db = getDb();
await registerUser(db, { username: "alice", password: "pw" });
const { sessionToken } = await login(db, "alice", "pw");
await db.k.updateTable("sessions").set({ expires_at: new Date(Date.now() - 1000).toISOString() }).execute();
expect(await getSessionUser(db, sessionToken)).toBeNull();
});
test("app tokens: scope check, last_used_at, revoke", async () => {
const db = getDb();
const u = await registerUser(db, { username: "alice", password: "pw" });
const { id, token } = await createAppToken(db, u.id, "cli", ["read"]);
expect(token.startsWith("stt_")).toBe(true);
expect(await verifyAppToken(db, token, "write")).toBeNull();
expect(await verifyAppToken(db, token, "read")).toEqual(u);
const row = await db.k.selectFrom("app_tokens").selectAll().executeTakeFirstOrThrow();
expect(row.last_used_at).not.toBeNull();
expect(await verifyAppToken(db, "stt_bogus", "read")).toBeNull();
await revokeAppToken(db, u.id, id);
expect(await verifyAppToken(db, token, "read")).toBeNull();
});
test("concurrent first registrations yield exactly one admin", async () => {
const db = getDb();
const results = await Promise.allSettled([
registerUser(db, { username: "alice", password: "x" }),
registerUser(db, { username: "bob", password: "x" }),
]);
const ok = results.filter((r) => r.status === "fulfilled");
const bad = results.filter((r) => r.status === "rejected");
expect(ok.length).toBe(1);
expect((ok[0] as PromiseFulfilledResult<{ role: string }>).value.role).toBe("admin");
expect(((bad[0] as PromiseRejectedResult).reason as AuthError).code).toBe("invite_required");
const admins = await db.k.selectFrom("users").select("id").where("role", "=", "admin").execute();
expect(admins.length).toBe(1);
});
test("no invite means invite_required even for a taken username", async () => {
const db = getDb();
await registerUser(db, { username: "alice", password: "x" });
expect(await code(registerUser(db, { username: "alice", password: "x" }))).toBe("invite_required");
});
test("only hashes are stored for invites and app tokens", async () => {
const db = getDb();
const u = await registerUser(db, { username: "alice", password: "x" });
const inv = await createInvite(db, u.id);
const { token } = await createAppToken(db, u.id, "cli", ["read"]);
const invRow = await db.k.selectFrom("invites").selectAll().executeTakeFirstOrThrow();
const tokRow = await db.k.selectFrom("app_tokens").selectAll().executeTakeFirstOrThrow();
expect(invRow.code_hash).not.toBe(inv);
expect(tokRow.token_hash).not.toBe(token);
expect(tokRow.token_hash).not.toContain(token);
});
test("revoking with another user's id leaves the token valid", async () => {
const db = getDb();
const a = await registerUser(db, { username: "alice", password: "x" });
const b = await registerUser(db, {
username: "bob",
password: "x",
inviteCode: await createInvite(db, a.id),
});
const { id, token } = await createAppToken(db, a.id, "cli", ["read"]);
await revokeAppToken(db, b.id, id);
expect(await verifyAppToken(db, token, "read")).toEqual(a);
});
});
+39
View File
@@ -0,0 +1,39 @@
import { describe, expect, test } from "bun:test";
import { decryptJson, encryptJson } from "../src/crypto.ts";
import { CryptoError } from "../src/errors.ts";
describe("crypto", () => {
test("round-trips JSON", async () => {
const value = { token: "abc", n: [1, 2, { x: null }] };
const blob = await encryptJson("s3cret", value);
expect(blob).toMatch(/^v1\.[A-Za-z0-9+/=]+\.[A-Za-z0-9+/=]+$/);
expect(await decryptJson("s3cret", blob)).toEqual(value);
});
test("uses a fresh IV each time", async () => {
expect(await encryptJson("k", 1)).not.toBe(await encryptJson("k", 1));
});
test("wrong secret throws CryptoError", async () => {
const blob = await encryptJson("right", { a: 1 });
await expect(decryptJson("wrong", blob)).rejects.toBeInstanceOf(CryptoError);
});
test("malformed blob throws CryptoError", async () => {
await expect(decryptJson("k", "garbage")).rejects.toBeInstanceOf(CryptoError);
await expect(decryptJson("k", "v2.AA==.AA==")).rejects.toBeInstanceOf(CryptoError);
});
test("IV decodes to 12 bytes", async () => {
const blob = await encryptJson("k", { a: 1 });
expect(Buffer.from(blob.split(".")[1]!, "base64")).toHaveLength(12);
});
test("flipping a ciphertext byte throws CryptoError", async () => {
const [v, iv, ct] = (await encryptJson("k", { a: 1 })).split(".");
const bytes = Buffer.from(ct!, "base64");
bytes[0] = bytes[0]! ^ 0xff;
const tampered = `${v}.${iv}.${bytes.toString("base64")}`;
await expect(decryptJson("k", tampered)).rejects.toBeInstanceOf(CryptoError);
});
});
+163
View File
@@ -0,0 +1,163 @@
import { expect, test } from "bun:test";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import { forEachDialect } from "@sttracker/db/testing";
import { fuzzyScore, matchWork } from "../src/matching.ts";
import { adminMergeWorks, createWork, getWork } from "../src/works.ts";
async function addUser(db: Db): Promise<string> {
const id = newId();
const now = nowIso();
await db.k
.insertInto("users")
.values({ id, username: `u${id}`, display_name: "U", password_hash: "x", role: "member", preferences: "{}", created_at: now, updated_at: now })
.execute();
return id;
}
test("fuzzyScore", () => {
expect(fuzzyScore({ title: "The Expanse: Leviathan Wakes" }, { title: "Leviathan Wakes" })).toBeGreaterThan(0.6);
expect(fuzzyScore({ title: "Dune" }, { title: "Dune Messiah" })).toBeLessThan(0.6);
expect(fuzzyScore({ title: "Pokémon" }, { title: "Pokemon" })).toBe(1);
expect(
fuzzyScore({ title: "Pokémon", year: 1996, creators: [{ name: "Satoshi" }] }, { title: "pokemon!", year: 1996, creators: [{ name: "satoshi" }] }),
).toBe(1);
// year off by one counts half, creator mismatch counts zero
expect(fuzzyScore({ title: "A B", year: 2000, creators: [{ name: "x" }] }, { title: "A B", year: 2001, creators: [{ name: "y" }] })).toBeCloseTo(0.8, 5);
});
forEachDialect("matching", (getDb) => {
test("match by isbn13 links existing", async () => {
const db = getDb();
const u = await addUser(db);
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "9780441013593" } } }, "t");
const r = await matchWork(db, u, { title: "Dune (Ace)", mediaType: "book", year: 1990, externalIds: { isbn13: "9780441013593", asin: "B1" } }, "plug");
expect(r).toEqual({ workId: id, created: false, draft: false });
expect((await getWork(db, id))?.year).toBe(1990);
});
test("unseen external id creates shared work", async () => {
const db = getDb();
const u = await addUser(db);
const r = await matchWork(db, u, { title: "Heat", mediaType: "movie", externalIds: { tmdb_movie: "949" } }, "p");
expect(r.created).toBe(true);
expect(r.draft).toBe(false);
expect((await getWork(db, r.workId))?.owner_user_id).toBeNull();
});
const chain = () => ({
title: "Pilot",
mediaType: "episode" as const,
externalIds: {},
position: { episode: 1, season: 1 },
parent: {
title: "Season 1",
mediaType: "season" as const,
externalIds: {},
position: { season: 1 },
parent: { title: "Lost", mediaType: "show" as const, externalIds: { tmdb_show: "4607" } },
},
});
test("episode hint creates show→season→episode chain once", async () => {
const db = getDb();
const u = await addUser(db);
const r1 = await matchWork(db, u, chain(), "p");
const r2 = await matchWork(db, u, chain(), "p");
expect(r1.created).toBe(true);
expect(r2).toEqual({ workId: r1.workId, created: false, draft: false });
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
const ep = await getWork(db, r1.workId);
expect(ep?.position).toEqual({ season: 1, episode: 1 });
expect(ep?.parent_id).not.toBeNull();
const season = await getWork(db, ep!.parent_id!);
expect(season?.media_type).toBe("season");
expect((await getWork(db, season!.parent_id!))?.media_type).toBe("show");
});
test("position key order does not matter", async () => {
const db = getDb();
const u = await addUser(db);
const a = await matchWork(db, u, chain(), "p");
const h = chain();
h.position = { season: 1, episode: 1 };
expect((await matchWork(db, u, h, "p")).workId).toBe(a.workId);
});
test("hint without ids creates private draft with suggestions", async () => {
const db = getDb();
const u = await addUser(db);
const shared = await createWork(db, { hint: { title: "Leviathan Wakes", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
await createWork(db, { hint: { title: "Leviathan Wakes", mediaType: "game", externalIds: { igdb: "1" } } }, "t");
await createWork(db, { hint: { title: "Unrelated", mediaType: "book", externalIds: { isbn13: "2" } } }, "t");
const r = await matchWork(db, u, { title: "The Expanse: Leviathan Wakes", mediaType: "book", externalIds: {} }, "p");
expect(r.created).toBe(true);
expect(r.draft).toBe(true);
expect((await getWork(db, r.workId))?.owner_user_id).toBe(u);
const sug = await db.k.selectFrom("match_suggestions").selectAll().execute();
expect(sug).toHaveLength(1);
expect(sug[0]).toMatchObject({ user_id: u, work_id: r.workId, candidate_work_id: shared, status: "pending" });
expect(sug[0]!.score).toBeGreaterThan(0.6);
});
test("concurrent matches of the same new external id create one work", async () => {
const db = getDb();
const u = await addUser(db);
const hint = () => ({ title: "Heat", mediaType: "movie" as const, externalIds: { tmdb_movie: "949" } });
const rs = await Promise.all([matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p")]);
expect(new Set(rs.map((r) => r.workId)).size).toBe(1);
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(1);
expect(await db.k.selectFrom("external_ids").select("id").execute()).toHaveLength(1);
});
test("concurrent matches of the same new episode chain create it once", async () => {
const db = getDb();
const u = await addUser(db);
const rs = await Promise.all([matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p")]);
expect(new Set(rs.map((r) => r.workId)).size).toBe(1);
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
});
test("concurrent matches of new episodes under an existing show create one season", async () => {
const db = getDb();
const u = await addUser(db);
await createWork(db, { hint: { title: "Lost", mediaType: "show", externalIds: { tmdb_show: "4607" } } }, "t");
const rs = await Promise.all([matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p")]);
expect(rs[0]!.workId).toBe(rs[1]!.workId);
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
});
test("concurrent updates adding the same external id do not fail", async () => {
const db = getDb();
const u = await addUser(db);
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
const hint = () => ({ title: "Dune", mediaType: "book" as const, externalIds: { isbn13: "1", asin: "B1" } });
const rs = await Promise.all([matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p")]);
expect(rs.map((r) => r.workId)).toEqual([id, id, id]);
expect(await db.k.selectFrom("external_ids").select("id").where("namespace", "=", "asin").execute()).toHaveLength(1);
});
test("draft reflects the aliased work, not the matched one", async () => {
const db = getDb();
const u = await addUser(db);
const shared = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
const priv = await createWork(db, { hint: { title: "My Dune", mediaType: "book", externalIds: {} }, ownerUserId: u }, "t");
await db.k.insertInto("work_aliases").values({ id: newId(), user_id: u, from_work_id: shared, to_work_id: priv }).execute();
expect(await matchWork(db, u, { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } }, "p")).toEqual({ workId: priv, created: false, draft: true });
});
test("position lookup follows a merged tombstone instead of colliding with it", async () => {
const db = getDb();
const u = await addUser(db);
await db.k.updateTable("users").set({ role: "admin" }).where("id", "=", u).execute();
const e1 = await matchWork(db, u, chain(), "p");
const h2 = chain();
h2.title = "Episode 2";
h2.position = { season: 1, episode: 2 };
const e2 = await matchWork(db, u, h2, "p");
await adminMergeWorks(db, u, e1.workId, e2.workId);
const before = (await db.k.selectFrom("works").select("id").execute()).length;
expect(await matchWork(db, u, chain(), "p")).toEqual({ workId: e2.workId, created: false, draft: false });
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(before);
});
});
+196
View File
@@ -0,0 +1,196 @@
import { expect, test } from "bun:test";
import type { Db } from "@sttracker/db/db";
import { newId, nowIso } from "@sttracker/db/ids";
import { fromJson } from "@sttracker/db/json";
import { forEachDialect } from "@sttracker/db/testing";
import { AuthError, ValidationError } from "../src/errors.ts";
import { matchWork } from "../src/matching.ts";
import {
adminMergeWorks,
createWork,
findWorkByExternalIds,
getWork,
resolveAlias,
searchWorks,
updateWorkFromHint,
} from "../src/works.ts";
async function addUser(db: Db, role: "admin" | "member" = "member"): Promise<string> {
const id = newId();
const now = nowIso();
await db.k
.insertInto("users")
.values({ id, username: `u${id}`, display_name: "U", password_hash: "x", role, preferences: "{}", created_at: now, updated_at: now })
.execute();
return id;
}
async function addEntry(db: Db, userId: string, workId: string): Promise<string> {
const id = newId();
const now = nowIso();
await db.k
.insertInto("entries")
.values({
id, user_id: userId, work_id: workId, status: "none", rating: null, review: null, private_notes: null,
favorite: 0, tags: "[]", custom_fields: "{}", visibility: "private", overrides: "{}",
media_type_override: null, field_sources: "{}", created_at: now, updated_at: now, deleted_at: null,
})
.execute();
return id;
}
forEachDialect("works", (getDb) => {
test("createWork stores external ids; findWorkByExternalIds finds any namespace", async () => {
const db = getDb();
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", year: 1965, externalIds: { isbn13: "9780441013593", hardcover_book: "7" } } }, "test");
expect(await findWorkByExternalIds(db, { nope: "1", hardcover_book: "7" })).toBe(id);
expect(await findWorkByExternalIds(db, { isbn13: "0" })).toBeNull();
expect((await getWork(db, id))?.year).toBe(1965);
});
test("updateWorkFromHint updates shared work, logs history, never touches private", async () => {
const db = getDb();
const u = await addUser(db);
const shared = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { a: "1" } } }, "t");
await updateWorkFromHint(db, shared, { title: "Dune", mediaType: "book", year: 1965, description: "Sand", externalIds: { a: "1", b: "2" } }, "plug");
const w = await getWork(db, shared);
expect(w?.year).toBe(1965);
expect(w?.description).toBe("Sand");
expect(await findWorkByExternalIds(db, { b: "2" })).toBe(shared);
const hist = await db.k.selectFrom("work_history").selectAll().where("work_id", "=", shared).where("actor", "=", "plug").execute();
expect(hist).toHaveLength(1);
expect(fromJson<Record<string, unknown>>(hist[0]!.before, {}).year).toBeNull();
const priv = await createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: {} }, ownerUserId: u }, "t");
await updateWorkFromHint(db, priv, { title: "Changed", mediaType: "book", externalIds: {} }, "plug");
expect((await getWork(db, priv))?.title).toBe("Mine");
});
test("private works invisible to other users in search", async () => {
const db = getDb();
const u1 = await addUser(db);
const u2 = await addUser(db);
await createWork(db, { hint: { title: "Secret 100%", mediaType: "book", externalIds: {} }, ownerUserId: u1 }, "t");
await createWork(db, { hint: { title: "Public Thing", mediaType: "book", externalIds: { x: "1" } } }, "t");
expect((await searchWorks(db, u1, "secret 100%")).map((w) => w.title)).toEqual(["Secret 100%"]);
expect(await searchWorks(db, u2, "secret")).toEqual([]);
expect((await searchWorks(db, u2, "PUBLIC")).map((w) => w.title)).toEqual(["Public Thing"]);
expect(await searchWorks(db, u2, "public", { mediaType: "game" })).toEqual([]);
});
test("adminMergeWorks requires admin", async () => {
const db = getDb();
const member = await addUser(db);
const a = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
const b = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: { x: "2" } } }, "t");
await expect(adminMergeWorks(db, member, a, b)).rejects.toBeInstanceOf(AuthError);
});
test("adminMergeWorks tombstones, moves ids, repoints entries, emits merged event on conflict", async () => {
const db = getDb();
const admin = await addUser(db, "admin");
const u1 = await addUser(db);
const u2 = await addUser(db);
const from = await createWork(db, { hint: { title: "Old", mediaType: "book", externalIds: { x: "1" } } }, "t");
const to = await createWork(db, { hint: { title: "New", mediaType: "book", externalIds: { x: "2" } } }, "t");
const e1 = await addEntry(db, u1, from);
const e2from = await addEntry(db, u2, from);
await addEntry(db, u2, to);
await adminMergeWorks(db, admin, from, to);
expect((await getWork(db, from))?.id).toBe(to);
expect(await findWorkByExternalIds(db, { x: "1" })).toBe(to);
expect(await resolveAlias(db, u1, from)).toBe(to);
expect((await db.k.selectFrom("entries").select("work_id").where("id", "=", e1).executeTakeFirstOrThrow()).work_id).toBe(to);
expect((await db.k.selectFrom("entries").select("work_id").where("id", "=", e2from).executeTakeFirstOrThrow()).work_id).toBe(from);
const ev = await db.k.selectFrom("change_events").selectAll().execute();
expect(ev).toHaveLength(1);
expect(ev[0]).toMatchObject({ user_id: u2, type: "merged", status: "pending", target_entity: "entry", target_id: e2from });
expect(fromJson(ev[0]!.payload, {})).toEqual({ fromWorkId: from, toWorkId: to });
expect(await db.k.selectFrom("work_history").select("id").where("work_id", "=", from).where("actor", "=", admin).execute()).toHaveLength(1);
});
test("createWork refuses external ids on a private work", async () => {
const db = getDb();
const u = await addUser(db);
const p = createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: { isbn13: "1" } }, ownerUserId: u }, "t");
await expect(p).rejects.toBeInstanceOf(ValidationError);
await expect(p).rejects.toMatchObject({ field: "externalIds", reason: "private works cannot carry external ids" });
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(0);
});
test("findWorkByExternalIds only returns works visible to the user", async () => {
const db = getDb();
const a = await addUser(db);
const b = await addUser(db);
const priv = await createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: {} }, ownerUserId: a }, "t");
// Legacy data: a private work carrying an external id (no longer creatable through createWork).
await db.k.insertInto("external_ids").values({ id: newId(), work_id: priv, namespace: "isbn13", value: "42" }).execute();
expect(await findWorkByExternalIds(db, { isbn13: "42" }, a)).toBe(priv);
expect(await findWorkByExternalIds(db, { isbn13: "42" }, b)).toBeNull();
expect(await findWorkByExternalIds(db, { isbn13: "42" })).toBeNull();
const shared = await createWork(db, { hint: { title: "Theirs", mediaType: "book", externalIds: { asin: "X" } } }, "t");
const hint = { title: "Mine", mediaType: "book" as const, externalIds: { isbn13: "42", asin: "X" } };
expect(await matchWork(db, b, hint, "p")).toEqual({ workId: shared, created: false, draft: false });
expect(await matchWork(db, a, hint, "p")).toEqual({ workId: priv, created: false, draft: true });
});
test("getWork returns null on a merged_into cycle", async () => {
const db = getDb();
const a = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
const b = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: { x: "2" } } }, "t");
await db.k.updateTable("works").set({ merged_into_id: b }).where("id", "=", a).execute();
await db.k.updateTable("works").set({ merged_into_id: a }).where("id", "=", b).execute();
expect(await getWork(db, a)).toBeNull();
});
test("search treats a literal _ as a character, not a wildcard", async () => {
const db = getDb();
const u = await addUser(db);
await createWork(db, { hint: { title: "snake_case", mediaType: "book", externalIds: { x: "1" } } }, "t");
await createWork(db, { hint: { title: "snakeXcase", mediaType: "book", externalIds: { x: "2" } } }, "t");
expect((await searchWorks(db, u, "e_c")).map((w) => w.title)).toEqual(["snake_case"]);
});
test("adminMergeWorks refuses private works", async () => {
const db = getDb();
const admin = await addUser(db, "admin");
const shared = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
const priv = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: {} }, ownerUserId: admin }, "t");
await expect(adminMergeWorks(db, admin, priv, shared)).rejects.toBeInstanceOf(ValidationError);
await expect(adminMergeWorks(db, admin, shared, priv)).rejects.toBeInstanceOf(ValidationError);
expect((await getWork(db, priv))?.id).toBe(priv);
});
test("adminMergeWorks moves children that do not collide, so matching does not recreate them", async () => {
const db = getDb();
const admin = await addUser(db, "admin");
const u = await addUser(db);
const show = (id: string) => ({ title: "Lost", mediaType: "show" as const, externalIds: { tmdb_show: id } });
const ep = (showId: string, season: number) => ({
title: "Pilot",
mediaType: "episode" as const,
externalIds: {},
position: { season, episode: 1 },
parent: { title: `Season ${season}`, mediaType: "season" as const, externalIds: {}, position: { season }, parent: show(showId) },
});
const a1 = await matchWork(db, u, ep("A", 1), "p");
const a2 = await matchWork(db, u, ep("A", 2), "p");
const b2 = await matchWork(db, u, ep("B", 2), "p");
const showA = (await findWorkByExternalIds(db, { tmdb_show: "A" }))!;
const showB = (await findWorkByExternalIds(db, { tmdb_show: "B" }))!;
const seasonA1 = (await getWork(db, a1.workId))!.parent_id!;
const seasonA2 = (await getWork(db, a2.workId))!.parent_id!;
const before = (await db.k.selectFrom("works").select("id").execute()).length;
await adminMergeWorks(db, admin, showA, showB);
expect((await getWork(db, seasonA1))?.parent_id).toBe(showB); // moved: B had no season 1
expect((await getWork(db, seasonA2))?.parent_id).toBe(showA); // left: B already had season 2
const again = await matchWork(db, u, ep("A", 1), "p");
expect(again).toEqual({ workId: a1.workId, created: false, draft: false });
expect((await matchWork(db, u, ep("A", 2), "p")).workId).toBe(b2.workId);
expect((await db.k.selectFrom("works").select("id").execute()).length).toBe(before);
});
});
@@ -0,0 +1,16 @@
import type { Kysely } from "kysely";
import type { Migration } from "kysely/migration";
/**
* One child per (parent, position), so concurrent matches cannot duplicate seasons or episodes.
* NULLs are distinct on both dialects: top-level works and children without a position are unaffected.
*/
export const worksPositionUnique: Migration = {
async up(db: Kysely<any>) {
await db.schema.createIndex("works_parent_id_position_uq").unique().on("works").columns(["parent_id", "position"]).execute();
},
async down(db: Kysely<any>) {
await db.schema.dropIndex("works_parent_id_position_uq").execute();
},
};
+2
View File
@@ -1,7 +1,9 @@
import type { Migration } from "kysely/migration";
import { initial } from "./0001_initial.ts";
import { worksPositionUnique } from "./0002_works_position_unique.ts";
/** All migrations, keyed by sortable name. Imported statically so bundles need no filesystem. */
export const migrations: Record<string, Migration> = {
"0001_initial": initial,
"0002_works_position_unique": worksPositionUnique,
};
+21 -2
View File
@@ -1,6 +1,6 @@
import { expect, test } from "bun:test";
import { sql, type Kysely } from "kysely";
import { Migrator } from "kysely/migration";
import { Migrator, NO_MIGRATIONS } from "kysely/migration";
import { newId, nowIso } from "../src/ids.ts";
import { toJson } from "../src/json.ts";
import { migrations } from "../src/migrations/index.ts";
@@ -79,10 +79,29 @@ forEachDialect("schema", (getDb) => {
await expect(k.insertInto("entries").values({ id: newId(), ...entry }).execute()).rejects.toThrow();
});
test("works (parent_id, position) is unique; null positions and null parents are not", async () => {
const k = getDb().k;
const now = nowIso();
const work = (id: string, parent: string | null, position: string | null) => ({
id, media_type: "show", title: "T", original_title: "", year: null, release_date: null, creators: "[]",
description: "", cover_url: "", cover_attachment_id: null, metadata: "{}", parent_id: parent, position,
owner_user_id: null, merged_into_id: null, created_at: now, updated_at: now,
});
const show = newId();
await k.insertInto("works").values(work(show, null, null)).execute();
await k.insertInto("works").values(work(newId(), null, null)).execute();
await k.insertInto("works").values(work(newId(), null, '{"season":1}')).execute();
await k.insertInto("works").values(work(newId(), null, '{"season":1}')).execute();
await k.insertInto("works").values(work(newId(), show, null)).execute();
await k.insertInto("works").values(work(newId(), show, null)).execute();
await k.insertInto("works").values(work(newId(), show, '{"season":1}')).execute();
await expect(k.insertInto("works").values(work(newId(), show, '{"season":1}')).execute()).rejects.toThrow();
});
test("migrating down removes the tables", async () => {
const k = getDb().k as Kysely<any>;
const migrator = new Migrator({ db: k, provider: { getMigrations: async () => migrations } });
const { error } = await migrator.migrateDown();
const { error } = await migrator.migrateTo(NO_MIGRATIONS);
expect(error).toBeUndefined();
const probe = sql`select 1 from users`.execute(k);
await expect(probe).rejects.toThrow();
+82
View File
@@ -0,0 +1,82 @@
export type CsvDelimiter = "," | ";" | "\t";
function detectDelimiter(text: string): CsvDelimiter {
const counts: Record<CsvDelimiter, number> = { ",": 0, ";": 0, "\t": 0 };
let inQuotes = false;
for (const ch of text) {
if (ch === '"') inQuotes = !inQuotes;
else if (!inQuotes) {
if (ch === "\n" || ch === "\r") break;
if (ch === "," || ch === ";" || ch === "\t") counts[ch]++;
}
}
let best: CsvDelimiter = ",";
for (const d of [";", "\t"] as const) if (counts[d] > counts[best]) best = d;
return best;
}
/** RFC 4180 parser. Strips a BOM, auto-detects the delimiter from the header line, skips blank lines. */
export function parseCsv(text: string, opts: { delimiter?: CsvDelimiter } = {}): string[][] {
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
const delim = opts.delimiter ?? detectDelimiter(text);
const rows: string[][] = [];
let row: string[] = [];
let field = "";
let quoted = false; // current field was quoted
let inQuotes = false;
let pending = false; // something has been read for the current row
const endField = () => {
row.push(field);
field = "";
};
const endRow = () => {
const blank = row.length === 0 && field === "" && !quoted;
if (!blank) {
endField();
rows.push(row);
}
row = [];
field = "";
quoted = false;
pending = false;
};
for (let i = 0; i < text.length; i++) {
const ch = text[i]!;
if (inQuotes) {
if (ch === '"') {
if (text[i + 1] === '"') {
field += '"';
i++;
} else inQuotes = false;
} else field += ch;
} else if (ch === '"' && field === "" && !quoted) {
inQuotes = true;
quoted = true;
pending = true;
} else if (ch === delim) {
endField();
quoted = false;
pending = true;
} else if (ch === "\r" || ch === "\n") {
if (ch === "\r" && text[i + 1] === "\n") i++;
endRow();
} else {
field += ch;
pending = true;
}
}
if (pending || field !== "" || row.length > 0) endRow();
return rows;
}
function quoteCell(s: string, single: boolean): string {
if (s === "" ? single : /[",;\t\r\n]/.test(s) || s !== s.trim()) return `"${s.replaceAll('"', '""')}"`;
return s;
}
/** Serialises with `,` and `\r\n`, quoting only where needed. */
export function toCsv(rows: string[][]): string {
return rows.map((r) => r.map((c) => quoteCell(c, r.length === 1)).join(",") + "\r\n").join("");
}
+24
View File
@@ -0,0 +1,24 @@
import { describe, expect, test } from "bun:test";
import { parseCsv, toCsv } from "../src/csv.ts";
describe("csv codec", () => {
test("round-trips awkward cells", () => {
const x = [["a", "b,c", 'q"uote', "line\nbreak", "cr\r\nlf", "", "🐉"], ["", "", "", "", "", "", ""], ["only"]];
expect(parseCsv(toCsv(x))).toEqual(x);
});
test("toCsv uses CRLF and quotes as needed", () => {
expect(toCsv([["a", "b c"], ["x,y", 'z"']])).toBe('a,b c\r\n"x,y","z"""\r\n');
});
test("strips BOM and detects semicolon / tab", () => {
expect(parseCsv("a;b\n1;2\n")).toEqual([["a", "b"], ["1", "2"]]);
expect(parseCsv("a\tb\n1\t2")).toEqual([["a", "b"], ["1", "2"]]);
});
test("explicit delimiter wins; delimiters inside quoted header are ignored", () => {
expect(parseCsv('"a;b;c",d\n1,2', {})).toEqual([["a;b;c", "d"], ["1", "2"]]);
expect(parseCsv("a,b;c\n1,2;3", { delimiter: ";" })).toEqual([["a,b", "c"], ["1,2", "3"]]);
});
test("handles CRLF, blank trailing lines, and empty input", () => {
expect(parseCsv("a,b\r\n1,2\r\n\r\n")).toEqual([["a", "b"], ["1", "2"]]);
expect(parseCsv("")).toEqual([]);
});
});
+114
View File
@@ -0,0 +1,114 @@
import { createHash } from "node:crypto";
import { parseCsv } from "@sttracker/plugin-sdk/csv";
import { definePlugin } from "@sttracker/plugin-sdk/types";
import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
import {
cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
type FieldRow,
} from "./src/mapping.ts";
const str = (v: string | string[] | undefined): string | undefined => (typeof v === "string" && v.trim() !== "" ? v.trim() : undefined);
function jsonRows(text: string): FieldRow[] {
let v: unknown;
try {
v = JSON.parse(text);
} catch {
throw new Error("csv plugin: file is not valid JSON");
}
if (!Array.isArray(v)) throw new Error("csv plugin: JSON must be an array of objects");
return v.map((o) => {
const r: Record<string, string | string[]> = {};
if (typeof o === "object" && o !== null) {
for (const [k, x] of Object.entries(o)) {
if (x == null || x === "") continue;
r[k] = Array.isArray(x) ? x.map(String) : String(x);
}
}
return r as FieldRow;
});
}
function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator<SourceRecord> {
const scale = defaultScale(rows);
const seen = new Map<string, number>();
for (const [i, r] of rows.entries()) {
const title = str(r.title);
if (!title) continue;
const mediaType = parseMediaType(str(r.mediaType), defaultMedia);
const finished = parseDate(str(r.finishedAt));
const started = parseDate(str(r.startedAt));
const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex");
const n = (seen.get(hash) ?? 0) + 1;
seen.set(hash, n);
const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`;
const externalIds: Record<string, string> = {};
for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) {
const id = cleanId(r[f]);
if (id) externalIds[f] = id;
}
const author = str(r.author);
const work: WorkHint = { title, mediaType, externalIds, ...(author ? { creators: [{ name: author }] } : {}) };
const entry: EntryFields = {};
const status = parseStatus(str(r.status));
if (status) entry.status = status;
const rating = normalizeRating(r, scale);
if (rating !== null) entry.rating = rating;
const review = str(r.review);
if (review) entry.review = review;
const notes = str(r.notes);
if (notes) entry.notes = notes;
const tags = parseTags(r.tags);
if (tags) entry.tags = tags;
const raw = rawRows[i];
yield { kind: "entry", key, raw, work, entry };
if (finished || started) {
const engagement: EngagementFields = {};
if (started) engagement.startedAt = started;
if (finished) engagement.finishedAt = finished;
yield { kind: "engagement", key: `${key}:engagement`, raw, work, engagement };
}
}
}
export default definePlugin({
manifest: {
id: "csv",
name: "CSV / JSON import",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["book", "comic", "manga", "game", "movie", "show", "podcast", "audio_drama", "album", "video", "other"],
capabilities: ["importFile"],
config: [
{
key: "mapping",
label: "Column mapping",
type: "string",
help: 'JSON { "column": "field" }. Fields: title mediaType status rating ratingScale(5|10|100) review finishedAt startedAt tags isbn13 imdb tmdb_movie igdb notes. Leave empty to guess from header names (Goodreads exports work out of the box). Ignored for .json.',
},
],
},
async *importFile(ctx, file) {
const text = new TextDecoder("utf-8").decode(file.bytes);
const lower = file.name.toLowerCase();
if (lower.endsWith(".json")) {
const clean = text.replace(/^\uFEFF/, "");
const rows = jsonRows(clean);
const raw = JSON.parse(clean) as unknown[];
const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o));
yield* toRecords(rows, raw, bookish ? "book" : "other");
} else if (lower.endsWith(".csv")) {
const table = parseCsv(text);
const header = table[0] ?? [];
const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header);
const rows = rowsToFieldRows(table, mapping);
const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13");
yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other");
} else {
throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`);
}
},
});
+140
View File
@@ -0,0 +1,140 @@
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
import type { MediaType, PartialDate, Status } from "@sttracker/plugin-sdk/types";
export const FIELDS = [
"title", "mediaType", "status", "rating", "ratingScale", "review", "finishedAt", "startedAt",
"tags", "isbn13", "imdb", "tmdb_movie", "igdb", "notes", "author",
] as const;
export type Field = (typeof FIELDS)[number];
export type Mapping = Record<string, Field>;
/** A row keyed by target field name. */
export type FieldRow = Partial<Record<Field, string | string[]>>;
const ALIASES: Record<Field, string[]> = {
title: ["title", "name"],
mediaType: ["mediatype", "media type", "type", "media"],
status: ["status", "exclusive shelf", "shelf", "state"],
rating: ["rating", "my rating", "score", "stars"],
ratingScale: ["ratingscale", "rating scale"],
review: ["review", "my review"],
finishedAt: ["finishedat", "finished", "date read", "date finished", "finished at", "date watched", "watched"],
startedAt: ["startedat", "started", "date started", "started at"],
tags: ["tags", "bookshelves", "labels", "genres"],
isbn13: ["isbn13", "isbn 13"],
imdb: ["imdb", "imdb id", "imdb_id"],
tmdb_movie: ["tmdb_movie", "tmdb", "tmdb id"],
igdb: ["igdb", "igdb id"],
notes: ["notes", "private notes", "comments"],
author: ["author", "authors", "creator"],
};
export function parseMapping(json: unknown): Mapping | null {
if (typeof json !== "string" || json.trim() === "") return null;
let v: unknown;
try {
v = JSON.parse(json);
} catch {
throw new Error("csv plugin: mapping is not valid JSON");
}
if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error("csv plugin: mapping must be an object of { column: field }");
const out: Mapping = {};
for (const [col, f] of Object.entries(v)) {
if (!(FIELDS as readonly unknown[]).includes(f)) throw new Error(`csv plugin: unknown mapping field "${String(f)}" for column "${col}"`);
out[col] = f as Field;
}
return out;
}
/** Guess a mapping from header names, case-insensitively (covers Goodreads exports). */
export function guessMapping(headers: string[]): Mapping {
const out: Mapping = {};
const used = new Set<Field>();
for (const h of headers) {
const n = h.trim().toLowerCase();
const f = FIELDS.find((x) => !used.has(x) && ALIASES[x].includes(n));
if (f) {
out[h] = f;
used.add(f);
}
}
return out;
}
export function rowsToFieldRows(rows: string[][], mapping: Mapping | null): FieldRow[] {
const [header, ...body] = rows;
if (!header) return [];
const m = mapping ?? guessMapping(header);
const idx = header.map((h) => m[h] ?? null);
return body.map((cells) => {
const r: FieldRow = {};
idx.forEach((f, i) => {
const v = cells[i];
if (f && v !== undefined && v.trim() !== "") r[f] = v.trim();
});
return r;
});
}
const MEDIA: MediaType[] = [
"book", "comic", "manga", "game", "movie", "show", "season", "episode", "podcast", "podcast_episode",
"audio_drama", "artist", "album", "track", "video", "other",
];
export function parseMediaType(v: string | undefined, fallback: MediaType): MediaType {
if (!v) return fallback;
const n = v.trim().toLowerCase().replaceAll(/[\s-]+/g, "_");
return (MEDIA as string[]).includes(n) ? (n as MediaType) : fallback;
}
const STATUS: Record<string, Status> = {
planned: "planned", "to-read": "planned", "to read": "planned", "want to read": "planned", "want to watch": "planned", "plan to watch": "planned", wishlist: "planned", backlog: "planned",
in_progress: "in_progress", "in progress": "in_progress", "currently-reading": "in_progress", "currently reading": "in_progress", reading: "in_progress", watching: "in_progress", playing: "in_progress",
paused: "paused", "on hold": "paused", "on-hold": "paused",
completed: "completed", read: "completed", finished: "completed", watched: "completed", played: "completed", done: "completed", complete: "completed",
dropped: "dropped", dnf: "dropped", abandoned: "dropped",
none: "none",
};
export function parseStatus(v: string | undefined): Status | undefined {
if (!v) return undefined;
return STATUS[v.trim().toLowerCase()] ?? "none";
}
/** Goodreads writes `YYYY/MM/DD`; normalise to dashes. */
export function parseDate(v: string | undefined): PartialDate | null {
if (!v) return null;
const s = v.trim();
return parsePartialDate(/^\d{4}\/\d{2}\/\d{2}$/.test(s) ? s.replaceAll("/", "-") : s);
}
/** Per-file default scale: 5 if max<=5, 10 if max<=10, else 100. */
export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 {
let max = 0;
for (const r of rows) {
const n = Number(r.rating);
if (r.rating !== undefined && Number.isFinite(n) && n > max) max = n;
}
return max <= 5 ? 5 : max <= 10 ? 10 : 100;
}
/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */
export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null {
if (r.rating === undefined) return null;
const n = Number(r.rating);
if (!Number.isFinite(n) || n < 0) return null;
const rs = Number(r.ratingScale);
const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale;
if (n === 0 && scale === 5) return null;
return Math.max(0, Math.min(100, Math.round((n / scale) * 100)));
}
/** Goodreads wraps ids as `="978..."`. */
export function cleanId(v: string | string[] | undefined): string | undefined {
if (typeof v !== "string") return undefined;
const s = v.trim().replace(/^=?"?/, "").replace(/"?$/, "").trim();
return s === "" ? undefined : s;
}
export function parseTags(v: string | string[] | undefined): string[] | undefined {
const list = Array.isArray(v) ? v : typeof v === "string" ? v.split(/[,;]/) : [];
const t = list.map((x) => String(x).trim()).filter(Boolean);
return t.length ? t : undefined;
}
+99
View File
@@ -0,0 +1,99 @@
import { describe, expect, test } from "bun:test";
import { collect, createTestContext } from "@sttracker/plugin-sdk/testing";
import type { SourceRecord } from "@sttracker/plugin-sdk/types";
import plugin from "../index.ts";
const fixture = async (n: string) => new Uint8Array(await Bun.file(new URL(`./fixtures/${n}`, import.meta.url)).arrayBuffer());
const run = async (name: string, bytes: Uint8Array, config: Record<string, unknown> = {}) =>
collect(plugin.importFile!(createTestContext({ config }), { name, bytes }));
const entries = (r: SourceRecord[]) => r.filter((x): x is Extract<SourceRecord, { kind: "entry" }> => x.kind === "entry");
describe("csv plugin", () => {
test("manifest", () => {
expect(plugin.manifest.id).toBe("csv");
expect(plugin.manifest.capabilities).toEqual(["importFile"]);
expect(plugin.manifest.config.map((c) => c.key)).toEqual(["mapping"]);
});
test("messy.csv: BOM, ';', quoted newline and quotes, empty rating, year date, emoji", async () => {
const recs = await run("messy.csv", await fixture("messy.csv"));
const e = entries(recs);
expect(e.map((r) => r.work.title)).toEqual(['Line "one"', "🐉 Dragons"]);
expect(e[0]!.entry.review).toBe("multi\nline");
expect(e[0]!.entry.rating).toBe(80);
expect(e[1]!.entry.rating ?? null).toBeNull();
const eng = recs.filter((r) => r.kind === "engagement");
expect(eng).toHaveLength(2);
expect((eng[0] as Extract<SourceRecord, { kind: "engagement" }>).engagement.finishedAt).toEqual({ value: "2019", precision: "year" });
expect(e[0]!.key).toMatch(/^row:[0-9a-f]{40}$/);
});
test("goodreads headers auto-mapped", async () => {
const e = entries(await run("goodreads.csv", await fixture("goodreads.csv")));
expect(e.map((r) => r.entry.status)).toEqual(["completed", "planned", "in_progress"]);
expect(e[0]!.work.title).toBe("Dune");
expect(e[0]!.work.mediaType).toBe("book");
expect(e[0]!.work.creators).toEqual([{ name: "Frank Herbert" }]);
expect(e[0]!.work.externalIds).toEqual({ isbn13: "9780441172719" });
expect(e[0]!.entry.rating).toBe(80);
expect(e[0]!.entry.tags).toEqual(["sci-fi", "classics"]);
expect(e[1]!.entry.rating ?? null).toBeNull();
expect(e[1]!.work.externalIds).toEqual({});
const eng = (await run("goodreads.csv", await fixture("goodreads.csv"))).filter((r) => r.kind === "engagement") as Extract<SourceRecord, { kind: "engagement" }>[];
expect(eng).toHaveLength(1);
expect(eng[0]!.engagement.finishedAt).toEqual({ value: "2021-03-04", precision: "day" });
});
test("explicit mapping and rating scale default", async () => {
const csv = "Name,Score,Kind\nA,7,movie\nB,10,game\n";
const mapping = JSON.stringify({ Name: "title", Score: "rating", Kind: "mediaType" });
const e = entries(await run("x.csv", new TextEncoder().encode(csv), { mapping }));
expect(e.map((r) => r.entry.rating)).toEqual([70, 100]);
expect(e.map((r) => r.work.mediaType)).toEqual(["movie", "game"]);
});
test("json ignores mapping", async () => {
const json = JSON.stringify([{ title: "T", mediaType: "movie", rating: 3, status: "completed", finishedAt: "2020-01-02", tags: ["a"], imdb: "tt1" }]);
const recs = await run("x.json", new TextEncoder().encode(json), { mapping: '{"title":"notes"}' });
const e = entries(recs);
expect(e[0]!.work.title).toBe("T");
expect(e[0]!.entry.rating).toBe(60);
expect(e[0]!.entry.tags).toEqual(["a"]);
expect(e[0]!.work.externalIds).toEqual({ imdb: "tt1" });
expect(recs).toHaveLength(2);
});
test("unsupported extension throws", async () => {
await expect(run("x.txt", new Uint8Array())).rejects.toThrow();
});
const enc = (s: string) => new TextEncoder().encode(s);
test("duplicate rows get distinct deterministic keys", async () => {
const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n";
const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key);
const k = await keys();
expect(new Set(k).size).toBe(k.length);
expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/);
expect(k.some((x) => x.endsWith("#2"))).toBe(true);
expect(k.some((x) => x.endsWith("#3"))).toBe(true);
expect(await keys()).toEqual(k);
});
test("default mediaType is per file, not per row", async () => {
const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n";
const e = entries(await run("g.csv", enc(csv)));
expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]);
const plain = entries(await run("p.csv", enc("Title\nA\n")));
expect(plain[0]!.work.mediaType).toBe("other");
const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]')));
expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]);
});
test("rating 0 is unrated only on scale 5", async () => {
const m = JSON.stringify({ T: "title", R: "rating" });
const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null);
expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]);
expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]);
expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]);
});
});
+4
View File
@@ -0,0 +1,4 @@
Book Id,Title,Author,ISBN,ISBN13,My Rating,Date Read,Bookshelves,Exclusive Shelf,My Review
1,Dune,Frank Herbert,"=""0441172717""","=""9780441172719""",4,2021/03/04,"sci-fi, classics",read,Great
2,Neuromancer,William Gibson,"=""""","=""""",0,,,to-read,
3,Hyperion,Dan Simmons,"=""""","=""9780553283686""",0,,,currently-reading,
1 Book Id Title Author ISBN ISBN13 My Rating Date Read Bookshelves Exclusive Shelf My Review
2 1 Dune Frank Herbert ="0441172717" ="9780441172719" 4 2021/03/04 sci-fi, classics read Great
3 2 Neuromancer William Gibson ="" ="" 0 to-read
4 3 Hyperion Dan Simmons ="" ="9780553283686" 0 currently-reading
+4
View File
@@ -0,0 +1,4 @@
Title;Rating;Date Read;Review
"Line ""one""";4;2019;"multi
line"
🐉 Dragons;;2020-05-06;
1 Title Rating Date Read Review
2 Line "one" 4 2019 multi line
3 🐉 Dragons 2020-05-06
+155
View File
@@ -0,0 +1,155 @@
import { describe, expect, test } from "bun:test";
import { HttpError } from "@sttracker/plugin-sdk/http";
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
import plugin from "./index.ts";
import { PAGE_SIZE } from "./src/graphql.ts";
const short = await Bun.file(new URL("./test/fixtures/page-short.json", import.meta.url)).json();
const lists = await Bun.file(new URL("./test/fixtures/lists.json", import.meta.url)).json();
const errors = await Bun.file(new URL("./test/fixtures/errors.json", import.meta.url)).json();
const URL_ = "POST https://api.hardcover.app/v1/graphql";
type Body = { query: string; variables: { offset?: number; where?: unknown } };
function ctxFor(userBooks: (b: Body) => unknown) {
const calls: Body[] = [];
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
const body = JSON.parse(String(init?.body)) as Body;
calls.push(body);
const out = body.query.includes("query Lists") ? lists : userBooks(body);
const inner = createFixtureHttp({ [URL_]: out });
return inner.fetch(input, init);
}) as typeof fetch;
return { ctx: createTestContext({ fetch: fetchImpl, config: { token: "tok" } }), calls };
}
describe("hardcover pull", () => {
test("maps status, rating, series parent, favorite", async () => {
const { ctx } = ctxFor(() => short);
const recs = await collect(plugin.pull!(ctx));
const e = recs.find((r) => r.key === "ub:1");
if (e?.kind !== "entry") throw new Error("no entry");
expect(e.entry).toMatchObject({ status: "completed", rating: 90, review: "Great", notes: "lent to Sam", favorite: true });
expect(e.work.externalIds).toEqual({ hardcover_book: "10", hardcover_edition: "500", isbn13: "9780441013593", isbn10: "0441013597" });
expect(e.work.parent).toMatchObject({ title: "Dune Chronicles", externalIds: { hardcover_series: "77" } });
expect(e.work.position).toEqual({ index: 1 });
const e2 = recs.find((r) => r.key === "ub:2");
if (e2?.kind !== "entry") throw new Error("no entry");
expect(e2.entry).toMatchObject({ status: "in_progress", rating: null, favorite: false });
});
test("re-reads yield two engagements with day precision", async () => {
const { ctx } = ctxFor(() => short);
const eng = (await collect(plugin.pull!(ctx))).filter((r) => r.kind === "engagement");
expect(eng.map((r) => r.key)).toEqual(["read:900", "read:901"]);
if (eng[0]?.kind !== "engagement") throw new Error();
expect(eng[0].engagement.startedAt).toEqual({ value: "2024-01-05", precision: "day" });
expect(eng[0].engagement.progress).toEqual({ pages: 412 });
});
test("merge yields synthetic record with mergedIntoKey", async () => {
const { ctx } = ctxFor(() => short);
const m = (await collect(plugin.pull!(ctx))).find((r) => r.key === "ub:2@book:20");
expect(m?.mergedIntoKey).toBe("ub:2");
if (m?.kind !== "entry") throw new Error();
expect(m.work.externalIds).toEqual({ hardcover_book: "20" });
expect((await collect(plugin.pull!(ctxFor(() => short).ctx))).some((r) => r.key === "ub:1@book:10")).toBe(false);
});
test("lists yield list records", async () => {
const { ctx } = ctxFor(() => short);
const l = (await collect(plugin.pull!(ctx))).find((r) => r.kind === "list");
expect(l?.key).toBe("list:5");
if (l?.kind !== "list") throw new Error();
expect(l.list.items[0]?.work.externalIds).toEqual({ hardcover_book: "10" });
});
test("sends bearer token", async () => {
let auth = "";
const f = (async (_i: unknown, init?: RequestInit) => {
auth = new Headers(init?.headers).get("authorization") ?? "";
return Response.json({ data: { me: [{ user_books: [], lists: [] }] } });
}) as typeof fetch;
await collect(plugin.pull!(createTestContext({ fetch: f, config: { token: "tok" } })));
expect(auth).toBe("Bearer tok");
});
test("GraphQL errors throw HttpError", async () => {
const { ctx } = ctxFor(() => errors);
await expect(collect(plugin.pull!(ctx))).rejects.toBeInstanceOf(HttpError);
});
test("offset paging continues on full page and stops on short page", async () => {
const row = short.data.me[0].user_books[0];
const full = { data: { me: [{ user_books: Array.from({ length: PAGE_SIZE }, (_, i) => ({ ...row, id: 1000 + i, user_book_reads: [] })) }] } };
const { ctx, calls } = ctxFor((b) => (b.variables.offset === 0 ? full : short));
await collect(plugin.pull!(ctx));
const ub = calls.filter((c) => c.query.includes("UserBooks"));
expect(ub.map((c) => c.variables.offset)).toEqual([0, 100]);
});
});
describe("hardcover pullChanges", () => {
test("filters by cursor and advances it", async () => {
const { ctx, calls } = ctxFor(() => short);
await ctx.cursor.set("2024-05-01T00:00:00+00:00");
const recs = await collect(plugin.pullChanges!(ctx));
expect(calls[0]?.variables.where).toEqual({ updated_at: { _gt: "2024-05-01T00:00:00+00:00" } });
expect(recs.some((r) => r.kind === "list")).toBe(false);
expect(ctx.saved.cursor.at(-1)).toBe("2024-06-01T10:00:00+00:00");
});
});
test("manifest", () => {
expect(plugin.manifest).toMatchObject({ id: "hardcover", capabilities: ["pull", "pullChanges", "lookup"], mediaTypes: ["book"], attribution: "Book data from Hardcover." });
});
describe("hardcover lookup", () => {
const search = (results: unknown) => {
const calls: unknown[] = [];
const f = (async (_i: unknown, init?: RequestInit) => {
calls.push(JSON.parse(String(init?.body)));
return Response.json({ data: { search: { results } } });
}) as typeof fetch;
return { ctx: createTestContext({ fetch: f, config: { token: "tok" } }), calls };
};
test("maps hits to WorkHints", async () => {
const { ctx, calls } = search({
hits: [
{ document: { id: 10, title: "Dune", release_year: 1965, author_names: ["Frank Herbert"], image: { url: "https://img/dune.jpg" } } },
{ document: { id: "11", title: "Bare" } },
],
});
const out = await plugin.lookup!(ctx, { title: "dune" });
expect(out).toEqual([
{ title: "Dune", mediaType: "book", year: 1965, creators: [{ name: "Frank Herbert" }], externalIds: { hardcover_book: "10" }, coverUrl: "https://img/dune.jpg" },
{ title: "Bare", mediaType: "book", externalIds: { hardcover_book: "11" } },
]);
expect((calls[0] as { variables: { q: string } }).variables.q).toBe("dune");
});
test("null results returns []", async () => {
expect(await plugin.lookup!(search(null).ctx, { title: "x" })).toEqual([]);
});
test("non-book mediaType returns [] without a request", async () => {
const { ctx, calls } = search({ hits: [] });
expect(await plugin.lookup!(ctx, { title: "x", mediaType: "movie" })).toEqual([]);
expect(calls).toHaveLength(0);
});
});
describe("status mapping", () => {
const cases: [number | null, string][] = [[1, "planned"], [2, "in_progress"], [3, "completed"], [4, "paused"], [5, "dropped"], [6, "none"], [null, "none"]];
for (const [id, want] of cases) {
test(`status_id ${id} -> ${want}`, async () => {
const row = { ...short.data.me[0].user_books[1], status_id: id, merged_at: null };
const ctx = createTestContext({
fetch: (async () => Response.json({ data: { me: [{ user_books: [row], lists: [] }] } })) as unknown as typeof fetch,
});
const e = (await collect(plugin.pull!(ctx))).find((r) => r.key === "ub:2");
if (e?.kind !== "entry") throw new Error();
expect(e.entry.status).toBe(want as never);
});
}
});
+67
View File
@@ -0,0 +1,67 @@
import { definePlugin } from "@sttracker/plugin-sdk/types";
import type { MediaType, PluginContext, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
import { gql, LISTS_QUERY, PAGE_SIZE, SEARCH_QUERY, USER_BOOKS_QUERY } from "./src/graphql.ts";
import { mapList, mapUserBook } from "./src/map.ts";
import type { HcList, HcUserBook } from "./src/map.ts";
export { RATE_PER_MINUTE } from "./src/graphql.ts";
async function* pageUserBooks(ctx: PluginContext, where: Record<string, unknown> | null): AsyncGenerator<HcUserBook> {
for (let offset = 0; ; offset += PAGE_SIZE) {
const d = await gql<{ me: { user_books: HcUserBook[] }[] }>(ctx, USER_BOOKS_QUERY, { limit: PAGE_SIZE, offset, where });
const rows = d.me[0]?.user_books ?? [];
yield* rows;
if (rows.length < PAGE_SIZE) return;
}
}
async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const cursor = await ctx.cursor.get();
const where = cursor ? { updated_at: { _gt: cursor } } : null;
let max = cursor;
for await (const ub of pageUserBooks(ctx, where)) {
if (max === null || ub.updated_at > max) max = ub.updated_at;
yield* mapUserBook(ub);
}
if (max !== null && max !== cursor) await ctx.cursor.set(max);
}
async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
let max: string | null = null;
for await (const ub of pageUserBooks(ctx, null)) {
if (max === null || ub.updated_at > max) max = ub.updated_at;
yield* mapUserBook(ub);
}
const d = await gql<{ me: { lists: HcList[] }[] }>(ctx, LISTS_QUERY, {});
for (const l of d.me[0]?.lists ?? []) yield mapList(l);
if (max !== null) await ctx.cursor.set(max);
}
interface SearchDoc { id: number | string; title: string; release_year?: number; author_names?: string[]; image?: { url?: string } }
export default definePlugin({
manifest: {
id: "hardcover",
name: "Hardcover",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["book"],
capabilities: ["pull", "pullChanges", "lookup"],
config: [{ key: "token", label: "Personal access token", type: "secret", required: true }],
attribution: "Book data from Hardcover.",
},
pull,
pullChanges,
async lookup(ctx, q: { title: string; mediaType?: MediaType; year?: number }): Promise<WorkHint[]> {
if (q.mediaType && q.mediaType !== "book") return [];
const d = await gql<{ search: { results: { hits?: { document: SearchDoc }[] } | null } }>(ctx, SEARCH_QUERY, { q: q.title });
return (d.search.results?.hits ?? []).map(({ document: x }) => ({
title: x.title,
mediaType: "book" as const,
externalIds: { hardcover_book: String(x.id) },
...(x.release_year ? { year: x.release_year } : {}),
...(x.author_names?.length ? { creators: x.author_names.map((name) => ({ name })) } : {}),
...(x.image?.url ? { coverUrl: x.image.url } : {}),
}));
},
});
+36
View File
@@ -0,0 +1,36 @@
import { HttpError } from "@sttracker/plugin-sdk/http";
import type { PluginContext } from "@sttracker/plugin-sdk/types";
export const ENDPOINT = "https://api.hardcover.app/v1/graphql";
export const RATE_PER_MINUTE = 50;
export const PAGE_SIZE = 100;
const BOOK = `id title release_year contributions { author { name } }`;
const BOOK_FULL = `id title release_year contributions { author { name } } book_series { position series { id name } }`;
export const USER_BOOKS_QUERY = `query UserBooks($limit: Int!, $offset: Int!, $where: user_books_bool_exp) {
me { user_books(limit: $limit, offset: $offset, where: $where, order_by: {id: asc}) {
id status_id rating review_raw private_notes starred updated_at merged_at original_book_id
book { ${BOOK_FULL} }
edition { id isbn_13 isbn_10 asin }
user_book_reads { id started_at finished_at progress_pages }
} }
}`;
export const LISTS_QUERY = `query Lists { me { lists { id name description list_books { book { ${BOOK} } } } } }`;
export const SEARCH_QUERY = `query Search($q: String!) { search(query: $q, query_type: "Book", per_page: 10) { results } }`;
export async function gql<T>(ctx: PluginContext, query: string, variables: Record<string, unknown>): Promise<T> {
const creds = await ctx.credentials.get();
const token = creds?.token ?? (typeof ctx.config.token === "string" ? ctx.config.token : "");
const res = await ctx.http.json<{ data?: T; errors?: { message: string }[] }>(ENDPOINT, {
method: "POST",
headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
body: JSON.stringify({ query, variables }),
});
if (res.errors?.length || res.data === undefined) {
throw new HttpError(200, JSON.stringify(res.errors ?? res), ENDPOINT);
}
return res.data;
}
+102
View File
@@ -0,0 +1,102 @@
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
import type { EntryFields, SourceRecord, Status, WorkHint } from "@sttracker/plugin-sdk/types";
export interface HcBook {
id: number;
title: string;
release_year?: number | null;
contributions?: { author: { name: string } | null }[];
book_series?: { position: number | null; series: { id: number; name: string } }[];
}
export interface HcRead { id: number; started_at: string | null; finished_at: string | null; progress_pages: number | null }
export interface HcUserBook {
id: number;
status_id: number | null;
rating: number | null;
review_raw: string | null;
private_notes: string | null;
starred: boolean | null;
updated_at: string;
merged_at: string | null;
original_book_id: number | null;
book: HcBook;
edition: { id: number; isbn_13: string | null; isbn_10: string | null; asin: string | null } | null;
user_book_reads: HcRead[];
}
export interface HcList { id: number; name: string; description: string | null; list_books: { book: HcBook }[] }
export const STATUS: Record<number, Status> = { 1: "planned", 2: "in_progress", 3: "completed", 4: "paused", 5: "dropped", 6: "none" };
export function bookHint(b: HcBook): WorkHint {
const creators = (b.contributions ?? []).flatMap((c) => (c.author ? [{ name: c.author.name }] : []));
const h: WorkHint = { title: b.title, mediaType: "book", externalIds: { hardcover_book: String(b.id) } };
if (b.release_year != null) h.year = b.release_year;
if (creators.length) h.creators = creators;
return h;
}
function workHint(ub: HcUserBook): WorkHint {
const h = bookHint(ub.book);
const e = ub.edition;
if (e) {
h.externalIds.hardcover_edition = String(e.id);
if (e.isbn_13) h.externalIds.isbn13 = e.isbn_13;
if (e.isbn_10) h.externalIds.isbn10 = e.isbn_10;
if (e.asin) h.externalIds.asin = e.asin;
}
const s = ub.book.book_series?.[0];
if (s) {
h.parent = { title: s.series.name, mediaType: "book", externalIds: { hardcover_series: String(s.series.id) } };
if (s.position != null) h.position = { index: s.position };
}
return h;
}
export function* mapUserBook(ub: HcUserBook): Generator<SourceRecord> {
const work = workHint(ub);
const entry: EntryFields = {
status: STATUS[ub.status_id ?? 6] ?? "none",
rating: ub.rating == null ? null : Math.round(ub.rating * 20),
review: ub.review_raw,
notes: ub.private_notes,
favorite: ub.starred === true,
};
const key = `ub:${ub.id}`;
yield { kind: "entry", key, raw: ub, work, entry };
for (const r of ub.user_book_reads) {
yield {
kind: "engagement",
key: `read:${r.id}`,
raw: r,
work,
engagement: {
startedAt: parsePartialDate(r.started_at),
finishedAt: parsePartialDate(r.finished_at),
...(r.progress_pages != null ? { progress: { pages: r.progress_pages } } : {}),
},
};
}
if (ub.merged_at && ub.original_book_id != null && ub.original_book_id !== ub.book.id) {
yield {
kind: "entry",
key: `${key}@book:${ub.original_book_id}`,
raw: ub,
mergedIntoKey: key,
work: { title: ub.book.title, mediaType: "book", externalIds: { hardcover_book: String(ub.original_book_id) } },
entry,
};
}
}
export function mapList(l: HcList): SourceRecord {
return {
kind: "list",
key: `list:${l.id}`,
raw: l,
list: {
name: l.name,
...(l.description ? { description: l.description } : {}),
items: l.list_books.map((lb) => ({ work: bookHint(lb.book) })),
},
};
}
+1
View File
@@ -0,0 +1 @@
{"errors":[{"message":"Invalid token"}]}
+1
View File
@@ -0,0 +1 @@
{"data":{"me":[{"lists":[{"id":5,"name":"Favorites","description":"best","list_books":[{"book":{"id":10,"title":"Dune","release_year":1965,"contributions":[{"author":{"name":"Frank Herbert"}}]}}]}]}]}}
+9
View File
@@ -0,0 +1,9 @@
{"data":{"me":[{"user_books":[
{"id":1,"status_id":3,"rating":4.5,"review_raw":"Great","private_notes":"lent to Sam","starred":true,"updated_at":"2024-05-02T10:00:00+00:00","merged_at":null,"original_book_id":10,
"book":{"id":10,"title":"Dune","release_year":1965,"contributions":[{"author":{"name":"Frank Herbert"}}],"book_series":[{"position":1,"series":{"id":77,"name":"Dune Chronicles"}}]},
"edition":{"id":500,"isbn_13":"9780441013593","isbn_10":"0441013597","asin":null},
"user_book_reads":[{"id":900,"started_at":"2024-01-05","finished_at":"2024-01-20","progress_pages":412},{"id":901,"started_at":"2024-04-01","finished_at":"2024-04-15","progress_pages":null}]},
{"id":2,"status_id":2,"rating":null,"review_raw":null,"private_notes":null,"starred":false,"updated_at":"2024-06-01T10:00:00+00:00","merged_at":"2024-05-30T00:00:00+00:00","original_book_id":20,
"book":{"id":21,"title":"Standalone","release_year":null,"contributions":[],"book_series":[]},
"edition":null,"user_book_reads":[]}
]}]}}
+19
View File
@@ -0,0 +1,19 @@
import { definePlugin } from "@sttracker/plugin-sdk/types";
import { pull, pullChanges } from "./src/lastfm.ts";
export default definePlugin({
manifest: {
id: "lastfm",
name: "Last.fm",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["track", "album", "artist"],
capabilities: ["pull", "pullChanges"],
config: [
{ key: "username", label: "Username", type: "string", required: true },
{ key: "apiKey", label: "API key", type: "secret", required: true },
],
},
pull,
pullChanges,
});
+137
View File
@@ -0,0 +1,137 @@
import { HttpError } from "@sttracker/plugin-sdk/http";
import type { ListenFields, PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
import { sleep } from "./options.ts";
export const BASE = "https://ws.audioscrobbler.com/2.0/";
export const MAX_ATTEMPTS = 3;
export class LastfmError extends Error {
readonly code: number;
constructor(code: number, message: string) {
super(`Last.fm error ${code}: ${message}`);
this.name = "LastfmError";
this.code = code;
}
}
type Q = Record<string, string | number | undefined>;
interface Text { "#text"?: string; mbid?: string; name?: string }
interface LfmTrack {
name?: string;
mbid?: string;
artist?: Text;
album?: Text;
date?: { uts?: string };
"@attr"?: { nowplaying?: string };
}
function errorOf(body: unknown): { code: number; message: string } | null {
if (body && typeof body === "object" && "error" in body && typeof (body as { error: unknown }).error === "number") {
const b = body as { error: number; message?: string };
return { code: b.error, message: b.message ?? "unknown error" };
}
return null;
}
/** Last.fm reports errors in the JSON body (HTTP 200 or 4xx); error 29 (rate limit) is retried here. */
export async function call<T>(ctx: PluginContext, query: Q): Promise<T> {
const apiKey = (await ctx.credentials.get())?.apiKey ?? String(ctx.config.apiKey ?? "");
for (let attempt = 1; ; attempt++) {
ctx.signal.throwIfAborted();
let body: unknown;
try {
body = await ctx.http.json<unknown>(BASE, { query: { ...query, api_key: apiKey, format: "json" } });
} catch (e) {
if (!(e instanceof HttpError)) throw e;
let parsed: unknown;
try { parsed = JSON.parse(e.body); } catch { throw e; }
const err = errorOf(parsed);
if (!err) throw e;
body = parsed;
}
const err = errorOf(body);
if (!err) return body as T;
if (err.code === 29 && attempt < MAX_ATTEMPTS) {
await sleep(2 ** attempt * 1000);
continue;
}
throw new LastfmError(err.code, err.message);
}
}
const arr = <T>(v: T | T[] | undefined): T[] => (v === undefined ? [] : Array.isArray(v) ? v : [v]);
const nonEmpty = (s: string | undefined): string | undefined => (s && s.trim() !== "" ? s : undefined);
function toListen(t: LfmTrack): SourceRecord | null {
const uts = t.date?.uts;
const artist = nonEmpty(t.artist?.["#text"] ?? t.artist?.name);
const track = nonEmpty(t.name);
if (!uts || !artist || !track) return null;
const listen: ListenFields = { listenedAt: new Date(Number(uts) * 1000).toISOString(), artist, track };
const album = nonEmpty(t.album?.["#text"]);
if (album) listen.album = album;
const mbids: NonNullable<ListenFields["mbids"]> = {};
const rec = nonEmpty(t.mbid), rel = nonEmpty(t.album?.mbid), art = nonEmpty(t.artist?.mbid);
if (rec) mbids.recording = rec;
if (rel) mbids.release = rel;
if (art) mbids.artist = art;
if (Object.keys(mbids).length > 0) listen.mbids = mbids;
return { kind: "listen", key: `${uts}:${artist}:${track}`, raw: t, listen };
}
async function* recent(ctx: PluginContext, from: number | undefined): AsyncGenerator<SourceRecord> {
const user = String(ctx.config.username ?? "");
let max = from ?? 0;
for (let page = 1, total = 1; page <= total; page++) {
const res = await call<{ recenttracks?: { track?: LfmTrack | LfmTrack[]; "@attr"?: { totalPages?: string } } }>(ctx, {
method: "user.getRecentTracks", user, limit: 200, page, from,
});
total = Number(res.recenttracks?.["@attr"]?.totalPages ?? 1) || 1;
for (const t of arr(res.recenttracks?.track)) {
if (t["@attr"]?.nowplaying === "true") continue;
const rec = toListen(t);
if (!rec) continue;
max = Math.max(max, Number(t.date?.uts));
yield rec;
}
}
if (max > 0) await ctx.cursor.set(String(max));
}
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
yield* recent(ctx, undefined);
yield* loved(ctx);
}
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const c = await ctx.cursor.get();
yield* recent(ctx, c ? Number(c) : undefined);
}
async function* loved(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const user = String(ctx.config.username ?? "");
for (let page = 1, total = 1; page <= total; page++) {
const res = await call<{ lovedtracks?: { track?: LfmTrack | LfmTrack[]; "@attr"?: { totalPages?: string } } }>(ctx, {
method: "user.getLovedTracks", user, limit: 200, page,
});
total = Number(res.lovedtracks?.["@attr"]?.totalPages ?? 1) || 1;
for (const t of arr(res.lovedtracks?.track)) {
const artist = nonEmpty(t.artist?.name ?? t.artist?.["#text"]);
const title = nonEmpty(t.name);
if (!artist || !title) continue;
const mbid = nonEmpty(t.mbid);
yield {
kind: "entry",
key: `loved:${artist}:${title}`,
raw: t,
work: {
title,
mediaType: "track",
creators: [{ name: artist, role: "artist" }],
externalIds: mbid ? { musicbrainz_recording: mbid } : {},
},
entry: { favorite: true },
};
}
}
}
+8
View File
@@ -0,0 +1,8 @@
/** Injectable sleep. Tests replace it with an immediate one via `setSleep`. */
let sleepImpl: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms);
export function setSleep(fn: ((ms: number) => Promise<void>) | null): void {
sleepImpl = fn ?? ((ms) => Bun.sleep(ms));
}
export const sleep = (ms: number): Promise<void> => sleepImpl(ms);
+92
View File
@@ -0,0 +1,92 @@
import { afterEach, beforeEach, expect, test } from "bun:test";
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
import plugin from "../index.ts";
import { setSleep } from "../src/options.ts";
const sleeps: number[] = [];
beforeEach(() => { sleeps.length = 0; setSleep(async (ms) => { sleeps.push(ms); }); });
afterEach(() => setSleep(null));
const URL0 = "GET https://ws.audioscrobbler.com/2.0/";
const rt = (name: string, uts: string, mbid = "", extra = {}) => ({
name, mbid, artist: { "#text": "Art", mbid: "" }, album: { "#text": "Alb", mbid: "" }, date: { uts }, ...extra,
});
function routes(handler: (q: URLSearchParams) => unknown) {
return { [URL0]: (req: Request) => handler(new URL(req.url).searchParams) };
}
test("skips nowplaying, omits empty mbids, includes loved entries", async () => {
const f = createFixtureHttp(routes((q) => {
expect(q.get("api_key")).toBe("key");
expect(q.get("format")).toBe("json");
if (q.get("method") === "user.getLovedTracks")
return { lovedtracks: { track: [{ name: "Fav", mbid: "m9", artist: { name: "Art" } }], "@attr": { totalPages: "1" } } };
return { recenttracks: { track: [
{ name: "Now", artist: { "#text": "Art" }, date: { uts: "1700000005" }, "@attr": { nowplaying: "true" } },
rt("One", "1700000000", "rec1", { artist: { "#text": "Art", mbid: "a1" } }),
rt("Two", "1699999999"),
], "@attr": { totalPages: "1" } } };
}));
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "key" } });
const recs = await collect(plugin.pull!(ctx));
expect(recs.map((r) => r.key)).toEqual(["1700000000:Art:One", "1699999999:Art:Two", "loved:Art:Fav"]);
const one = recs[0]!, two = recs[1]!, fav = recs[2]!;
expect(one.kind === "listen" && one.listen.mbids).toEqual({ recording: "rec1", artist: "a1" });
expect(two.kind === "listen" && two.listen.mbids).toBeUndefined();
expect(fav.kind === "entry" && fav.entry.favorite).toBe(true);
expect(fav.kind === "entry" && fav.work.externalIds).toEqual({ musicbrainz_recording: "m9" });
expect(ctx.saved.cursor).toEqual(["1700000000"]);
});
test("paginates recent tracks", async () => {
const pagesSeen: (string | null)[] = [];
const f = createFixtureHttp(routes((q) => {
if (q.get("method") === "user.getLovedTracks") return { lovedtracks: { track: [], "@attr": { totalPages: "0" } } };
pagesSeen.push(q.get("page"));
return { recenttracks: { track: [rt(`T${q.get("page")}`, `10${q.get("page")}`)], "@attr": { totalPages: "2" } } };
}));
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } })));
expect(pagesSeen).toEqual(["1", "2"]);
expect(recs).toHaveLength(2);
});
test("error 29 is retried with exponential sleeps (body on HTTP 200 and 403)", async () => {
let n = 0;
const f = createFixtureHttp(routes((q) => {
if (q.get("method") === "user.getLovedTracks") return { lovedtracks: { track: [] } };
n++;
if (n === 1) return { error: 29, message: "Rate Limit Exceeded" };
if (n === 2) return new Response(JSON.stringify({ error: 29, message: "Rate Limit Exceeded" }), { status: 403 });
return { recenttracks: { track: rt("X", "5") } };
}));
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } })));
expect(n).toBe(3);
expect(sleeps).toEqual([2000, 4000]);
expect(recs).toHaveLength(1);
});
test("error 29 gives up after 3 attempts; other codes throw immediately", async () => {
const f29 = createFixtureHttp(routes(() => ({ error: 29, message: "Rate Limit Exceeded" })));
const p = collect(plugin.pull!(createTestContext({ fetch: f29.fetch, config: { username: "u", apiKey: "k" } })));
await expect(p).rejects.toMatchObject({ code: 29 });
expect(f29.calls).toHaveLength(3);
const f6 = createFixtureHttp(routes(() => new Response(JSON.stringify({ error: 6, message: "User not found" }), { status: 404 })));
const p6 = collect(plugin.pull!(createTestContext({ fetch: f6.fetch, config: { username: "u", apiKey: "k" } })));
await expect(p6).rejects.toThrow(/Last\.fm error 6: User not found/);
expect(f6.calls).toHaveLength(1);
});
test("pullChanges sends from=<cursor> and advances it", async () => {
let from: string | null = null;
const f = createFixtureHttp(routes((q) => {
from = q.get("from");
return { recenttracks: { track: [rt("N", "2000")], "@attr": { totalPages: "1" } } };
}));
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } });
await ctx.cursor.set("1500");
await collect(plugin.pullChanges!(ctx));
expect(from as string | null).toBe("1500");
expect(await ctx.cursor.get()).toBe("2000");
});
+19
View File
@@ -0,0 +1,19 @@
import { definePlugin } from "@sttracker/plugin-sdk/types";
import { pull, pullChanges } from "./src/listenbrainz.ts";
export default definePlugin({
manifest: {
id: "listenbrainz",
name: "ListenBrainz",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["track", "album", "artist"],
capabilities: ["pull", "pullChanges"],
config: [
{ key: "username", label: "Username", type: "string", required: true },
{ key: "token", label: "User token", type: "secret", required: true },
],
},
pull,
pullChanges,
});
+183
View File
@@ -0,0 +1,183 @@
import { strFromU8, unzipSync } from "fflate";
import { HttpError } from "@sttracker/plugin-sdk/http";
import type { ListenFields, PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
import { sleep } from "./options.ts";
export const BASE = "https://api.listenbrainz.org";
export const POLL_INTERVAL_MS = 30_000;
export const MAX_POLLS = 60;
interface TrackMetadata {
artist_name?: string;
track_name?: string;
release_name?: string;
additional_info?: { duration_ms?: number };
mbid_mapping?: { recording_mbid?: string; release_mbid?: string; artist_mbids?: string[] };
}
export interface LbListen {
listened_at: number | string;
recording_msid?: string;
track_metadata?: TrackMetadata;
}
interface LbFeedback {
recording_mbid?: string;
recording_msid?: string;
score?: number;
track_metadata?: TrackMetadata;
}
interface ExportInfo { export_id: number | string; status: string; created?: string }
async function authHeaders(ctx: PluginContext): Promise<Record<string, string>> {
const creds = await ctx.credentials.get();
const token = creds?.token ?? (typeof ctx.config.token === "string" ? ctx.config.token : undefined);
return token ? { authorization: `Token ${token}` } : {};
}
/** Unix seconds from either a number or an ISO string; null if unparseable. */
export function toSeconds(v: number | string | undefined): number | null {
if (typeof v === "number") return Number.isFinite(v) ? Math.floor(v) : null;
if (typeof v === "string" && v.trim() !== "") {
if (/^\d+$/.test(v.trim())) return Number(v);
const ms = Date.parse(v);
return Number.isNaN(ms) ? null : Math.floor(ms / 1000);
}
return null;
}
export function listenToRecord(l: LbListen): SourceRecord | null {
const secs = toSeconds(l.listened_at);
const tm = l.track_metadata;
if (secs === null || !tm?.artist_name || !tm.track_name) return null;
const listen: ListenFields = {
listenedAt: new Date(secs * 1000).toISOString(),
artist: tm.artist_name,
track: tm.track_name,
};
if (tm.release_name) listen.album = tm.release_name;
const dur = tm.additional_info?.duration_ms;
if (typeof dur === "number" && dur > 0) listen.durationMs = dur;
const m = tm.mbid_mapping;
const mbids: NonNullable<ListenFields["mbids"]> = {};
if (m?.recording_mbid) mbids.recording = m.recording_mbid;
if (m?.release_mbid) mbids.release = m.release_mbid;
if (m?.artist_mbids?.[0]) mbids.artist = m.artist_mbids[0];
if (Object.keys(mbids).length > 0) listen.mbids = mbids;
return { kind: "listen", key: `${secs}:${l.recording_msid ?? ""}`, raw: l, listen };
}
function feedbackToRecord(f: LbFeedback): SourceRecord | null {
if (f.score !== 1 || !f.recording_mbid) return null;
return {
kind: "entry",
key: `feedback:${f.recording_mbid}`,
raw: f,
work: {
title: f.track_metadata?.track_name ?? f.recording_mbid,
mediaType: "track",
externalIds: { musicbrainz_recording: f.recording_mbid },
...(f.track_metadata?.artist_name ? { creators: [{ name: f.track_metadata.artist_name, role: "artist" }] } : {}),
},
entry: { favorite: true },
};
}
function* jsonlLines<T>(bytes: Uint8Array): Generator<T> {
for (const line of strFromU8(bytes).split("\n")) {
const s = line.trim();
if (s === "") continue;
yield JSON.parse(s) as T;
}
}
async function requestExport(ctx: PluginContext, headers: Record<string, string>): Promise<void> {
try {
await ctx.http.json(`${BASE}/1/export/`, { method: "POST", headers });
} catch (e) {
if (e instanceof HttpError && e.status === 400 && /pending/i.test(e.body)) return;
throw e;
}
}
async function waitForExport(ctx: PluginContext, headers: Record<string, string>): Promise<string | number> {
for (let poll = 0; poll < MAX_POLLS; poll++) {
if (poll > 0) await sleep(POLL_INTERVAL_MS);
ctx.signal.throwIfAborted();
const list = await ctx.http.json<ExportInfo[]>(`${BASE}/1/export/list`, { headers });
const newest = [...list].sort((a, b) => (b.created ?? "").localeCompare(a.created ?? ""))[0];
if (newest?.status === "complete") return newest.export_id;
if (newest?.status === "failed") throw new Error(`ListenBrainz export ${newest.export_id} failed`);
}
throw new Error(`ListenBrainz export did not complete after ${MAX_POLLS} polls`);
}
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const headers = await authHeaders(ctx);
await requestExport(ctx, headers);
const id = await waitForExport(ctx, headers);
const res = await ctx.http.raw(`${BASE}/1/export/${id}/download`, { headers });
if (!res.ok) throw new HttpError(res.status, await res.text(), res.url);
const files = unzipSync(new Uint8Array(await res.arrayBuffer()), {
filter: (f) => f.name.endsWith(".jsonl"),
});
let max = 0;
for (const name of Object.keys(files).sort()) {
const bytes = files[name]!;
if (name === "feedback.jsonl" || name.endsWith("/feedback.jsonl")) {
for (const f of jsonlLines<LbFeedback>(bytes)) {
const rec = feedbackToRecord(f);
if (rec) yield rec;
}
} else if (/(^|\/)listens\/.+\.jsonl$/.test(name)) {
for (const l of jsonlLines<LbListen>(bytes)) {
const rec = listenToRecord(l);
if (!rec) continue;
max = Math.max(max, toSeconds(l.listened_at) ?? 0);
yield rec;
}
}
}
if (max > 0) await ctx.cursor.set(String(max));
}
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const headers = await authHeaders(ctx);
const username = String(ctx.config.username ?? "");
const cursor = Number((await ctx.cursor.get()) ?? 0) || 0;
// We do not rely on the ordering of `min_ts` queries (and LB forbids min_ts together with max_ts).
// Instead walk backwards from now with `max_ts` (listens older than max_ts, newest first) until we
// reach a listen at or before the cursor, or run out of pages.
// max_ts = smallest + 1 re-includes the page-boundary second so listens sharing that second are not
// lost; the duplicates are dropped by the seen-set here and by idempotent keys downstream.
// If that would not make progress (a whole page in one second), fall back to max_ts = smallest.
const seen = new Set<string>();
let max = cursor;
let maxTs: number | undefined;
for (;;) {
ctx.signal.throwIfAborted();
const res = await ctx.http.json<{ payload?: { listens?: LbListen[] } }>(
`${BASE}/1/user/${encodeURIComponent(username)}/listens`,
{ headers, query: { max_ts: maxTs, count: 1000 } },
);
const listens = res.payload?.listens ?? [];
if (listens.length === 0) break;
let smallest = Infinity;
let reachedCursor = false;
for (const l of listens) {
const secs = toSeconds(l.listened_at);
if (secs === null) continue;
smallest = Math.min(smallest, secs);
if (secs <= cursor) { reachedCursor = true; continue; }
const rec = listenToRecord(l);
if (!rec || seen.has(rec.key)) continue;
seen.add(rec.key);
max = Math.max(max, secs);
yield rec;
}
if (reachedCursor || !Number.isFinite(smallest)) break;
let next = smallest + 1;
if (maxTs !== undefined && next >= maxTs) next = smallest;
if (maxTs !== undefined && next >= maxTs) break;
maxTs = next;
}
if (max > cursor) await ctx.cursor.set(String(max));
}
+8
View File
@@ -0,0 +1,8 @@
/** Injectable sleep. Tests replace it with an immediate one via `setSleep`. */
let sleepImpl: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms);
export function setSleep(fn: ((ms: number) => Promise<void>) | null): void {
sleepImpl = fn ?? ((ms) => Bun.sleep(ms));
}
export const sleep = (ms: number): Promise<void> => sleepImpl(ms);
@@ -0,0 +1,113 @@
import { afterEach, beforeEach, expect, test } from "bun:test";
import { zipSync, strToU8 } from "fflate";
import { HttpError } from "@sttracker/plugin-sdk/http";
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
import plugin from "../index.ts";
import { setSleep } from "../src/options.ts";
const sleeps: number[] = [];
beforeEach(() => { sleeps.length = 0; setSleep(async (ms) => { sleeps.push(ms); }); });
afterEach(() => setSleep(null));
const jl = (rows: unknown[]) => strToU8(rows.map((r) => JSON.stringify(r)).join("\n") + "\n");
const listen = (at: number | string, msid: string, extra = {}) => ({
listened_at: at, recording_msid: msid,
track_metadata: { artist_name: "A", track_name: `T${msid}`, release_name: "R", additional_info: { duration_ms: 1000 }, ...extra },
});
const mapped = { mbid_mapping: { recording_mbid: "rec", release_mbid: "rel", artist_mbids: ["art"] } };
function exportRoutes(listRuns: unknown[][], post: () => Response) {
let i = 0;
const zip = zipSync({
"listens/2024/1.jsonl": jl([listen(1700000000, "m1", mapped), listen("2023-11-14T22:13:20Z", "m2")]),
"feedback.jsonl": jl([
{ recording_mbid: "love1", score: 1, track_metadata: { track_name: "Loved", artist_name: "A" } },
{ recording_mbid: "hate1", score: -1 },
]),
});
return {
"POST https://api.listenbrainz.org/1/export/": post,
"GET https://api.listenbrainz.org/1/export/list": () => Response.json(listRuns[Math.min(i++, listRuns.length - 1)]),
"GET https://api.listenbrainz.org/1/export/7/download": () => new Response(zip),
};
}
const pending = [{ export_id: 7, status: "in_progress", created: "2024-01-01" }];
const done = [{ export_id: 6, status: "complete", created: "2023-01-01" }, { export_id: 7, status: "complete", created: "2024-01-01" }];
test("pull: polls pending then complete, parses listens and feedback", async () => {
const f = createFixtureHttp(exportRoutes([pending, done], () => Response.json({ export_id: 7 })));
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", token: "tok" } });
const recs = await collect(plugin.pull!(ctx));
expect(sleeps).toEqual([30_000]);
expect(f.calls[0]!.headers.get("authorization")).toBe("Token tok");
expect(recs.map((r) => r.key)).toEqual(["feedback:love1", "1700000000:m1", "1700000000:m2"]);
const l = recs.find((r) => r.key === "1700000000:m1")!;
expect(l.kind === "listen" && l.listen).toEqual({
listenedAt: "2023-11-14T22:13:20.000Z", artist: "A", track: "Tm1", album: "R", durationMs: 1000,
mbids: { recording: "rec", release: "rel", artist: "art" },
});
const m2 = recs.find((r) => r.key === "1700000000:m2")!;
expect(m2.kind === "listen" && m2.listen.mbids).toBeUndefined();
const fav = recs[0]!;
expect(fav.kind === "entry" && fav.entry.favorite).toBe(true);
expect(fav.kind === "entry" && fav.work.externalIds).toEqual({ musicbrainz_recording: "love1" });
expect(ctx.saved.cursor).toEqual(["1700000000"]);
});
test("pull: 400 already pending falls through to listing", async () => {
const f = createFixtureHttp(exportRoutes([done], () => new Response("An export is already pending", { status: 400 })));
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
expect(recs).toHaveLength(3);
});
test("pull: other POST errors propagate", async () => {
const f = createFixtureHttp(exportRoutes([done], () => new Response("nope", { status: 401 })));
const p = collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
await expect(p).rejects.toBeInstanceOf(HttpError);
});
test("pull: gives up after 60 polls", async () => {
const f = createFixtureHttp(exportRoutes([pending], () => Response.json({ export_id: 7 })));
const p = collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
await expect(p).rejects.toThrow(/60 polls/);
expect(sleeps).toHaveLength(59);
});
function changesFixture(pages: unknown[][], seen: (string | null)[]) {
let i = 0;
return createFixtureHttp({
"GET https://api.listenbrainz.org/1/user/u/listens": (req: Request) => {
const u = new URL(req.url);
expect(u.searchParams.get("min_ts")).toBeNull();
seen.push(u.searchParams.get("max_ts"));
return { payload: { listens: pages[i++] ?? [] } };
},
});
}
test("pullChanges: walks back over 3 pages with max_ts, cursor ends at max", async () => {
const seen: (string | null)[] = [];
const f = changesFixture(
[
[listen(600, "f"), listen(500, "e")],
[listen(500, "e2"), listen(400, "d")],
[listen(300, "c"), listen(200, "b")],
[listen(100, "a"), listen(50, "z")],
],
seen,
);
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u" } });
await ctx.cursor.set("150");
const recs = await collect(plugin.pullChanges!(ctx));
expect(recs.map((r) => r.key)).toEqual(["600:f", "500:e", "500:e2", "400:d", "300:c", "200:b"]);
expect(seen).toEqual([null, "501", "401", "201"]);
expect(ctx.saved.cursor).toEqual(["150", "600"]);
});
test("pullChanges: unchanged cursor yields nothing", async () => {
const f = changesFixture([[listen(600, "f"), listen(500, "e")]], []);
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u" } });
await ctx.cursor.set("600");
expect(await collect(plugin.pullChanges!(ctx))).toEqual([]);
expect(ctx.saved.cursor).toEqual(["600"]);
});
+19
View File
@@ -0,0 +1,19 @@
import { definePlugin } from "@sttracker/plugin-sdk/types";
import { lookup } from "./src/lookup.ts";
import { pull, pullChanges } from "./src/pull.ts";
export default definePlugin({
manifest: {
id: "mangabaka",
name: "MangaBaka",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["manga", "comic"],
capabilities: ["pull", "pullChanges", "lookup"],
config: [{ key: "token", label: "API token", type: "secret", required: true, help: "Sent as the x-api-key header." }],
attribution: "Manga data from MangaBaka (CC BY-NC-SA 4.0).",
},
pull,
pullChanges,
lookup,
});
+46
View File
@@ -0,0 +1,46 @@
import type { PluginContext } from "@sttracker/plugin-sdk/types";
export const BASE_URL = "https://api.mangabaka.org";
/** The host builds the HttpClient; it should use this rate for this plugin. */
export const RATE_PER_MINUTE = 150;
export interface ApiSource { id?: number | string | null }
export interface ApiSeries {
id: number;
title: string;
type?: string | null;
state?: string | null;
merged_with?: number | null;
year?: number | null;
source?: {
anilist?: ApiSource | null;
my_anime_list?: ApiSource | null;
manga_updates?: ApiSource | null;
kitsu?: ApiSource | null;
} | null;
}
export interface ApiEntry {
series_id: number;
state?: string | null;
rating?: number | null;
progress_chapter?: number | null;
progress_volume?: number | null;
start_date?: string | null;
finish_date?: string | null;
note?: string | null;
series?: ApiSeries | null;
}
export interface LibraryPage {
data: ApiEntry[];
pagination?: { page?: number; total_pages?: number } | null;
}
export interface ChangesPage {
data: { type: "upsert" | "delete"; entry?: ApiEntry; series_id: number }[];
next_cursor?: string | null;
}
export function authHeaders(ctx: PluginContext): Record<string, string> {
const token = ctx.config.token;
if (typeof token !== "string" || token === "") throw new Error("mangabaka: token is not configured");
return { "x-api-key": token };
}
+14
View File
@@ -0,0 +1,14 @@
import type { MediaType, PluginContext, WorkHint } from "@sttracker/plugin-sdk/types";
import { BASE_URL, authHeaders, type ApiSeries } from "./api.ts";
import { mapEntry } from "./map.ts";
export async function lookup(ctx: PluginContext, q: { title: string; mediaType?: MediaType; year?: number }): Promise<WorkHint[]> {
const res = await ctx.http.json<{ data: ApiSeries[] }>(`${BASE_URL}/v1/series/search`, {
headers: authHeaders(ctx),
query: { q: q.title },
signal: ctx.signal,
});
return res.data
.map((s) => (mapEntry({ series_id: s.id, series: s })[0] as { work: WorkHint }).work)
.filter((w) => (q.mediaType ? w.mediaType === q.mediaType : true) && (q.year == null || w.year == null || w.year === q.year));
}
+75
View File
@@ -0,0 +1,75 @@
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
import type { SourceRecord, Status, WorkHint } from "@sttracker/plugin-sdk/types";
import type { ApiEntry } from "./api.ts";
export const STATE_MAP: Record<string, Status> = {
considering: "planned",
plan_to_read: "planned",
reading: "in_progress",
rereading: "in_progress",
paused: "paused",
dropped: "dropped",
completed: "completed",
};
export function mapState(s: string | null | undefined): Status {
return s != null && Object.hasOwn(STATE_MAP, s) ? STATE_MAP[s]! : "none";
}
function hint(e: ApiEntry): WorkHint {
const s = e.series;
const externalIds: Record<string, string> = { mangabaka: String(e.series_id) };
const add = (ns: string, v: { id?: number | string | null } | null | undefined) => {
if (v?.id != null && v.id !== "") externalIds[ns] = String(v.id);
};
add("anilist_manga", s?.source?.anilist);
add("mal_manga", s?.source?.my_anime_list);
add("mangaupdates", s?.source?.manga_updates);
add("kitsu_manga", s?.source?.kitsu);
return {
title: s?.title ?? `MangaBaka series ${e.series_id}`,
mediaType: s?.type === "comic" ? "comic" : "manga",
...(s?.year != null ? { year: s.year } : {}),
externalIds,
};
}
export function mapEntry(e: ApiEntry): SourceRecord[] {
const work = hint(e);
const key = `series:${e.series_id}`;
const merged = e.series?.state === "merged" && e.series.merged_with != null;
const entry: SourceRecord = {
kind: "entry",
key,
raw: e,
work,
entry: {
status: mapState(e.state),
...(e.rating !== undefined ? { rating: e.rating } : {}),
...(e.note !== undefined ? { notes: e.note } : {}),
},
...(merged ? { mergedIntoKey: `series:${e.series!.merged_with}` } : {}),
};
const progress: Record<string, number> = {};
if (e.progress_chapter != null) progress.chapter = e.progress_chapter;
if (e.progress_volume != null) progress.volume = e.progress_volume;
const started = parsePartialDate(e.start_date);
const finished = parsePartialDate(e.finish_date);
if (Object.keys(progress).length === 0 && !started && !finished) return [entry];
const engagement: SourceRecord = {
kind: "engagement",
key: `progress:${e.series_id}`,
raw: e,
work,
engagement: { progress, startedAt: started, finishedAt: finished },
};
return [entry, engagement];
}
export function mapDeletion(seriesId: number): SourceRecord[] {
const work: WorkHint = { title: `MangaBaka series ${seriesId}`, mediaType: "manga", externalIds: { mangabaka: String(seriesId) } };
return [
{ kind: "entry", key: `series:${seriesId}`, raw: { series_id: seriesId }, work, entry: {}, deleted: true },
{ kind: "engagement", key: `progress:${seriesId}`, raw: { series_id: seriesId }, work, engagement: {}, deleted: true },
];
}
+33
View File
@@ -0,0 +1,33 @@
import type { PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
import { BASE_URL, authHeaders, type ChangesPage, type LibraryPage } from "./api.ts";
import { mapDeletion, mapEntry } from "./map.ts";
export async function* pull(ctx: PluginContext): AsyncIterable<SourceRecord> {
const headers = authHeaders(ctx);
for (let page = 1; ; page++) {
const res = await ctx.http.json<LibraryPage>(`${BASE_URL}/v1/my/library`, { headers, query: { page }, signal: ctx.signal });
for (const e of res.data) yield* mapEntry(e);
const total = res.pagination?.total_pages;
if (res.data.length === 0 || (total != null && page >= total)) return;
}
}
export async function* pullChanges(ctx: PluginContext): AsyncIterable<SourceRecord> {
const headers = authHeaders(ctx);
let cursor = (await ctx.cursor.get()) ?? undefined;
for (;;) {
const res = await ctx.http.json<ChangesPage>(`${BASE_URL}/v2/my/library/changes`, { headers, query: { cursor }, signal: ctx.signal });
for (const c of res.data) {
if (c.type === "delete") yield* mapDeletion(c.series_id);
else if (c.entry) yield* mapEntry(c.entry);
else ctx.log(`mangabaka: upsert without entry for series ${c.series_id}, skipped`);
}
const next = res.next_cursor;
if (next) {
await ctx.cursor.set(next);
if (next === cursor) return;
cursor = next;
}
if (!next || res.data.length === 0) return;
}
}
+4
View File
@@ -0,0 +1,4 @@
{"data":[
{"type":"upsert","series_id":1,"entry":{"series_id":1,"state":"paused","rating":70,"progress_chapter":13,"progress_volume":2,"start_date":"2024-03-05","finish_date":null,"note":null,"series":{"id":1,"title":"Alpha","type":"manga","state":"active","merged_with":null,"year":2020,"source":{}}}},
{"type":"delete","series_id":7}
],"next_cursor":"cur-2"}
+4
View File
@@ -0,0 +1,4 @@
{"data":[
{"series_id":1,"state":"reading","rating":80,"progress_chapter":12,"progress_volume":2,"start_date":"2024-03-05","finish_date":null,"note":"good","series":{"id":1,"title":"Alpha","type":"manga","state":"active","merged_with":null,"year":2020,"source":{"anilist":{"id":11},"my_anime_list":{"id":22},"manga_updates":{"id":33},"kitsu":{"id":44}}}},
{"series_id":2,"state":"plan_to_read","rating":null,"progress_chapter":null,"progress_volume":null,"start_date":null,"finish_date":null,"note":null,"series":{"id":2,"title":"Beta","type":"manga","state":"merged","merged_with":9,"year":null,"source":{}}}
],"pagination":{"page":1,"total_pages":2}}
+3
View File
@@ -0,0 +1,3 @@
{"data":[
{"series_id":3,"state":"completed","rating":95,"progress_chapter":50,"progress_volume":null,"start_date":"2023-01","finish_date":"2023-06-30","note":null,"series":{"id":3,"title":"Gamma","type":"manga","state":"active","merged_with":null,"year":2019,"source":{}}}
],"pagination":{"page":2,"total_pages":2}}
+1
View File
@@ -0,0 +1 @@
{"data":[{"id":5,"title":"Delta","type":"manga","state":"active","merged_with":null,"year":2018,"source":{"anilist":{"id":55}}}]}
+97
View File
@@ -0,0 +1,97 @@
import { describe, expect, test } from "bun:test";
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
import type { Status } from "@sttracker/plugin-sdk/types";
import plugin from "../index.ts";
import { mapDeletion, mapState } from "../src/map.ts";
import page1 from "./fixtures/library-page1.json";
import page2 from "./fixtures/library-page2.json";
import changes from "./fixtures/changes.json";
import search from "./fixtures/search.json";
const B = "https://api.mangabaka.org";
const ctxFor = (routes: Record<string, unknown>) => {
const h = createFixtureHttp(routes);
return { h, ctx: createTestContext({ fetch: h.fetch, config: { token: "tok" } }) };
};
describe("mangabaka", () => {
test("manifest", () => {
expect(plugin.manifest.id).toBe("mangabaka");
expect(plugin.manifest.capabilities).toEqual(["pull", "pullChanges", "lookup"]);
});
test.each<[string, Status]>([
["considering", "planned"], ["plan_to_read", "planned"], ["reading", "in_progress"],
["rereading", "in_progress"], ["paused", "paused"], ["dropped", "dropped"], ["completed", "completed"],
])("state %s -> %s", (a, b) => expect(mapState(a)).toBe(b));
test("pull pages, maps entries and engagements, sends api key", async () => {
const { h, ctx } = ctxFor({
[`GET ${B}/v1/my/library?page=1`]: page1,
[`GET ${B}/v1/my/library?page=2`]: page2,
});
const recs = await collect(plugin.pull!(ctx));
expect(recs.map((r) => r.key)).toEqual(["series:1", "progress:1", "series:2", "series:3", "progress:3"]);
expect(h.calls.every((c) => c.headers.get("x-api-key") === "tok")).toBe(true);
const a = recs[0]!;
if (a.kind !== "entry") throw new Error();
expect(a.entry).toEqual({ status: "in_progress", rating: 80, notes: "good" });
expect(a.work.externalIds).toEqual({ mangabaka: "1", anilist_manga: "11", mal_manga: "22", mangaupdates: "33", kitsu_manga: "44" });
const p = recs[1]!;
if (p.kind !== "engagement") throw new Error();
expect(p.engagement.progress).toEqual({ chapter: 12, volume: 2 });
expect(p.engagement.startedAt).toEqual({ value: "2024-03-05", precision: "day" });
expect(p.engagement.finishedAt).toBeNull();
});
test("merged series sets mergedIntoKey", async () => {
const { ctx } = ctxFor({ [`GET ${B}/v1/my/library?page=1`]: page1, [`GET ${B}/v1/my/library?page=2`]: page2 });
const recs = await collect(plugin.pull!(ctx));
expect(recs.find((r) => r.key === "series:2")!.mergedIntoKey).toBe("series:9");
expect(recs.find((r) => r.key === "series:1")!.mergedIntoKey).toBeUndefined();
});
test("pullChanges yields upsert and deletion, saves cursor", async () => {
const { h, ctx } = ctxFor({
[`GET ${B}/v2/my/library/changes`]: (req: Request) =>
new URL(req.url).searchParams.get("cursor") === "cur-2" ? { data: [], next_cursor: "cur-2" } : changes,
});
await ctx.cursor.set("cur-1");
ctx.saved.cursor.length = 0;
const recs = await collect(plugin.pullChanges!(ctx));
expect(recs.filter((r) => r.deleted).map((r) => r.key)).toEqual(["series:7", "progress:7"]);
expect(recs[0]!.key).toBe("series:1");
expect(ctx.saved.cursor).toEqual(["cur-2", "cur-2"]);
expect(await ctx.cursor.get()).toBe("cur-2");
expect(new URL(h.calls[0]!.url).searchParams.get("cursor")).toBe("cur-1");
});
test("upsert without entry is skipped and logged, never deleted", async () => {
const logs: string[] = [];
const h = createFixtureHttp({
[`GET ${B}/v2/my/library/changes`]: (req: Request) =>
new URL(req.url).searchParams.has("cursor") ? { data: [], next_cursor: "c" } : { data: [{ type: "upsert", series_id: 4 }], next_cursor: "c" },
});
const ctx = createTestContext({ fetch: h.fetch, config: { token: "tok" }, log: (m) => logs.push(m) });
const recs = await collect(plugin.pullChanges!(ctx));
expect(recs).toEqual([]);
expect(logs).toEqual(["mangabaka: upsert without entry for series 4, skipped"]);
});
test("inherited keys are not states", () => expect(mapState("constructor")).toBe("none"));
test("delete keys are a subset of keys a full pull produced", async () => {
const { ctx } = ctxFor({ [`GET ${B}/v1/my/library?page=1`]: page1, [`GET ${B}/v1/my/library?page=2`]: page2 });
const pulled = (await collect(plugin.pull!(ctx))).map((r) => r.key);
const del = mapDeletion(1).map((r) => r.key);
expect(del).toEqual(["series:1", "progress:1"]);
for (const k of del) expect(pulled).toContain(k);
});
test("lookup searches series", async () => {
const { h, ctx } = ctxFor({ [`GET ${B}/v1/series/search`]: search });
const r = await plugin.lookup!(ctx, { title: "Delta", mediaType: "manga" });
expect(r).toEqual([{ title: "Delta", mediaType: "manga", year: 2018, externalIds: { mangabaka: "5", anilist_manga: "55" } }]);
expect(new URL(h.calls[0]!.url).searchParams.get("q")).toBe("Delta");
});
});
+55
View File
@@ -0,0 +1,55 @@
import { HttpError } from "@sttracker/plugin-sdk/http";
import { definePlugin } from "@sttracker/plugin-sdk/types";
import { BASE, clientCreds, post, toCredentials, type TokenResponse } from "./src/api.ts";
import { searchHint } from "./src/map.ts";
import { pull, pullChanges } from "./src/sync.ts";
interface DeviceCode { device_code: string; user_code: string; verification_url: string; expires_in: number; interval: number }
export default definePlugin({
manifest: {
id: "trakt",
name: "Trakt",
version: "0.1.0",
sdkVersion: 1,
mediaTypes: ["movie", "show", "season", "episode"],
capabilities: ["pull", "pullChanges", "lookup"],
config: [
{ key: "clientId", label: "Client ID", type: "string", required: true },
{ key: "clientSecret", label: "Client secret", type: "secret", required: true },
],
attribution: "STTracker uses Trakt but is not endorsed or certified by Trakt.",
},
deviceAuth: {
async start(ctx) {
const { clientId } = clientCreds(ctx);
const d = await post<DeviceCode>(ctx, "/oauth/device/code", { client_id: clientId });
return { userCode: d.user_code, verificationUrl: d.verification_url, deviceCode: d.device_code, intervalSec: d.interval, expiresInSec: d.expires_in };
},
async poll(ctx, deviceCode) {
const { clientId, clientSecret } = clientCreds(ctx);
try {
const t = await post<TokenResponse>(ctx, "/oauth/device/token", { code: deviceCode, client_id: clientId, client_secret: clientSecret });
return toCredentials(t, ctx.now());
} catch (e) {
// 400 = pending, 429 = polling too fast; anything else (404/409/410/418) is terminal.
if (e instanceof HttpError && (e.status === 400 || e.status === 429)) return "pending";
throw e;
}
},
},
pull,
pullChanges,
async lookup(ctx, q) {
const { clientId } = clientCreds(ctx);
const type = q.mediaType === "movie" || q.mediaType === "show" ? q.mediaType : "movie,show";
const res = await ctx.http.json<Parameters<typeof searchHint>[0][]>(`${BASE}/search/${type}`, {
headers: { "trakt-api-version": "2", "trakt-api-key": clientId },
query: { query: q.title, years: q.year },
});
return res.flatMap((r) => {
const h = searchHint(r);
return h ? [h] : [];
});
},
});
+82
View File
@@ -0,0 +1,82 @@
import { HttpError } from "@sttracker/plugin-sdk/http";
import type { PluginContext } from "@sttracker/plugin-sdk/types";
export const BASE = "https://api.trakt.tv";
/** Rate the host should configure for this plugin's HttpClient. */
export const RATE_PER_MINUTE = 100;
const REFRESH_WINDOW_MS = 24 * 60 * 60 * 1000;
const REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob";
export interface TokenResponse {
access_token: string;
refresh_token: string;
expires_in: number;
created_at?: number;
}
export function clientCreds(ctx: PluginContext): { clientId: string; clientSecret: string } {
const clientId = ctx.config.clientId;
const clientSecret = ctx.config.clientSecret;
if (typeof clientId !== "string" || clientId === "") throw new Error("trakt: clientId is not configured");
if (typeof clientSecret !== "string" || clientSecret === "") throw new Error("trakt: clientSecret is not configured");
return { clientId, clientSecret };
}
function headers(ctx: PluginContext, accessToken?: string, body = false): Headers {
const h = new Headers({ "trakt-api-version": "2", "trakt-api-key": clientCreds(ctx).clientId });
if (body) h.set("content-type", "application/json");
if (accessToken) h.set("authorization", `Bearer ${accessToken}`);
return h;
}
export function toCredentials(t: TokenResponse, now: Date): Record<string, string> {
const createdMs = typeof t.created_at === "number" ? t.created_at * 1000 : now.getTime();
return {
accessToken: t.access_token,
refreshToken: t.refresh_token,
expiresAt: new Date(createdMs + t.expires_in * 1000).toISOString(),
};
}
export async function post<T>(ctx: PluginContext, path: string, body: unknown): Promise<T> {
return ctx.http.json<T>(BASE + path, { method: "POST", headers: headers(ctx, undefined, true), body: JSON.stringify(body) });
}
/**
* Returns a usable access token. Refreshes when it expires within 24 h; Trakt refresh tokens are
* single-use, so the rotated credentials are saved before this returns (before any further request).
*/
export async function ensureAccessToken(ctx: PluginContext): Promise<string> {
const creds = await ctx.credentials.get();
if (!creds?.accessToken) throw new Error("trakt: not connected (no credentials)");
const expiresMs = Date.parse(creds.expiresAt ?? "");
if (!Number.isNaN(expiresMs) && expiresMs - ctx.now().getTime() > REFRESH_WINDOW_MS) return creds.accessToken;
if (!creds.refreshToken) throw new Error("trakt: access token expiring and no refresh token");
const { clientId, clientSecret } = clientCreds(ctx);
const t = await post<TokenResponse>(ctx, "/oauth/token", {
refresh_token: creds.refreshToken,
client_id: clientId,
client_secret: clientSecret,
redirect_uri: REDIRECT_URI,
grant_type: "refresh_token",
});
await ctx.credentials.save(toCredentials(t, ctx.now()));
return t.access_token;
}
/** Authenticated GET. Returns the parsed body plus the X-Pagination-Page-Count header (1 if absent). */
export async function get<T>(
ctx: PluginContext,
token: string,
path: string,
query: Record<string, string | number | undefined> = {},
): Promise<{ data: T; pageCount: number }> {
const u = new URL(BASE + path);
for (const [k, v] of Object.entries(query)) if (v !== undefined) u.searchParams.set(k, String(v));
const h = headers(ctx, token);
h.set("accept", "application/json");
const res = await ctx.http.raw(u.toString(), { headers: h });
if (!res.ok) throw new HttpError(res.status, await res.text(), u.toString());
const pc = Number(res.headers.get("x-pagination-page-count"));
return { data: (await res.json()) as T, pageCount: Number.isFinite(pc) && pc >= 1 ? pc : 1 };
}
+128
View File
@@ -0,0 +1,128 @@
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
import type { MediaType, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
interface Ids { trakt?: number; slug?: string; imdb?: string | null; tmdb?: number | null; tvdb?: number | null }
interface TMovie { title: string; year?: number | null; ids: Ids }
interface TShow { title: string; year?: number | null; ids: Ids }
interface TSeason { number: number; ids?: Ids }
interface TEpisode { season: number; number: number; title?: string | null; ids?: Ids }
export interface TItem {
type: string;
id?: number;
watched_at?: string;
rating?: number;
notes?: string | null;
movie?: TMovie;
show?: TShow;
season?: TSeason;
episode?: TEpisode;
}
function ids(map: Record<string, unknown>): Record<string, string> {
const out: Record<string, string> = {};
for (const [k, v] of Object.entries(map)) if (v !== undefined && v !== null && v !== "") out[k] = String(v);
return out;
}
function movieHint(m: TMovie): WorkHint {
return {
title: m.title,
mediaType: "movie",
...(m.year ? { year: m.year } : {}),
externalIds: ids({ trakt_movie: m.ids.trakt, tmdb_movie: m.ids.tmdb, imdb: m.ids.imdb }),
};
}
function showHint(s: TShow): WorkHint {
return {
title: s.title,
mediaType: "show",
...(s.year ? { year: s.year } : {}),
externalIds: ids({ trakt_show: s.ids.trakt, tmdb_show: s.ids.tmdb, tvdb_show: s.ids.tvdb, imdb: s.ids.imdb }),
};
}
function seasonHint(show: TShow, n: number, sid?: Ids): WorkHint {
return {
title: `Season ${n}`,
mediaType: "season",
externalIds: sid ? ids({ trakt_season: sid.trakt, tmdb_season: sid.tmdb }) : {},
parent: showHint(show),
position: { season: n },
};
}
function episodeHint(show: TShow, e: TEpisode): WorkHint {
const eid = e.ids;
return {
title: e.title || `Episode ${e.number}`,
mediaType: "episode",
externalIds: eid ? ids({ trakt_episode: eid.trakt, tvdb_episode: eid.tvdb, tmdb_episode: eid.tmdb, imdb: eid.imdb }) : {},
parent: seasonHint(show, e.season),
position: { season: e.season, episode: e.number },
};
}
/** Maps a Trakt item (history, rating, watchlist or list item) to a WorkHint, or null if unsupported. */
export function hintFor(item: TItem): WorkHint | null {
switch (item.type) {
case "movie": return item.movie ? movieHint(item.movie) : null;
case "show": return item.show ? showHint(item.show) : null;
case "season": return item.show && item.season ? seasonHint(item.show, item.season.number, item.season.ids) : null;
case "episode": return item.show && item.episode ? episodeHint(item.show, item.episode) : null;
default: return null;
}
}
export function traktIdOf(item: TItem): number | undefined {
switch (item.type) {
case "movie": return item.movie?.ids.trakt;
case "show": return item.show?.ids.trakt;
case "season": return item.season?.ids?.trakt;
case "episode": return item.episode?.ids?.trakt;
default: return undefined;
}
}
export function historyRecord(item: TItem): SourceRecord | null {
const work = hintFor(item);
if (!work || item.id === undefined) return null;
const finishedAt = parsePartialDate(item.watched_at);
return { kind: "engagement", key: `history:${item.id}`, raw: item, work, engagement: { finishedAt } };
}
export function ratingRecord(item: TItem): SourceRecord | null {
const work = hintFor(item);
const tid = traktIdOf(item);
if (!work || tid === undefined || typeof item.rating !== "number") return null;
return { kind: "entry", key: `rating:${item.type}:${tid}`, raw: item, work, entry: { rating: item.rating * 10 } };
}
export function watchlistRecord(item: TItem): SourceRecord | null {
const work = hintFor(item);
const tid = traktIdOf(item);
if (!work || tid === undefined) return null;
return { kind: "entry", key: `watchlist:${item.type}:${tid}`, raw: item, work, entry: { status: "planned" } };
}
export interface TList { name: string; description?: string | null; ids: Ids }
export function listRecord(list: TList, items: TItem[]): SourceRecord {
const mapped: { work: WorkHint; note?: string }[] = [];
for (const it of items) {
const work = hintFor(it);
if (work) mapped.push({ work, ...(it.notes ? { note: it.notes } : {}) });
}
return {
kind: "list",
key: `list:${list.ids.trakt}`,
raw: { list, items },
list: { name: list.name, ...(list.description ? { description: list.description } : {}), items: mapped },
};
}
export function searchHint(r: { type: string; movie?: TMovie; show?: TShow }): WorkHint | null {
return hintFor({ type: r.type, ...(r.movie ? { movie: r.movie } : {}), ...(r.show ? { show: r.show } : {}) });
}
export type { MediaType };
+92
View File
@@ -0,0 +1,92 @@
import type { PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
import { ensureAccessToken, get } from "./api.ts";
import { historyRecord, listRecord, ratingRecord, watchlistRecord, type TItem, type TList } from "./map.ts";
const PAGE_LIMIT = 250;
async function* history(ctx: PluginContext, token: string, startAt?: string): AsyncGenerator<SourceRecord> {
for (let page = 1, pages = 1; page <= pages; page++) {
const r = await get<TItem[]>(ctx, token, "/sync/history", { page, limit: PAGE_LIMIT, start_at: startAt });
pages = r.pageCount;
for (const item of r.data) {
const rec = historyRecord(item);
if (rec) yield rec;
}
}
}
async function* ratings(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
for (const item of (await get<TItem[]>(ctx, token, "/sync/ratings")).data) {
const rec = ratingRecord(item);
if (rec) yield rec;
}
}
async function* watchlist(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
for (const item of (await get<TItem[]>(ctx, token, "/sync/watchlist")).data) {
const rec = watchlistRecord(item);
if (rec) yield rec;
}
}
async function* lists(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
for (const list of (await get<TList[]>(ctx, token, "/users/me/lists")).data) {
const items = (await get<TItem[]>(ctx, token, `/users/me/lists/${list.ids.trakt}/items`)).data;
yield listRecord(list, items);
}
}
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const token = await ensureAccessToken(ctx);
yield* history(ctx, token);
yield* ratings(ctx, token);
yield* watchlist(ctx, token);
yield* lists(ctx, token);
}
interface Activity { watched_at?: string | null; rated_at?: string | null; watchlisted_at?: string | null; updated_at?: string | null }
interface LastActivities { movies?: Activity; episodes?: Activity; shows?: Activity; seasons?: Activity; lists?: Activity }
interface Cursor { history: string; ratings: string; watchlist: string; lists: string }
const maxIso = (xs: (string | null | undefined)[]): string =>
xs.reduce<string>((a, x) => (x && x > a ? x : a), "");
function cursorFrom(a: LastActivities): Cursor {
return {
history: maxIso([a.movies?.watched_at, a.episodes?.watched_at]),
ratings: maxIso([a.movies?.rated_at, a.episodes?.rated_at, a.shows?.rated_at, a.seasons?.rated_at]),
watchlist: maxIso([a.movies?.watchlisted_at, a.episodes?.watchlisted_at, a.shows?.watchlisted_at, a.seasons?.watchlisted_at]),
lists: maxIso([a.lists?.updated_at]),
};
}
function parseCursor(s: string | null): Cursor | null {
if (!s) return null;
try {
const c = JSON.parse(s) as Partial<Cursor>;
return { history: c.history ?? "", ratings: c.ratings ?? "", watchlist: c.watchlist ?? "", lists: c.lists ?? "" };
} catch {
return null;
}
}
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
const token = await ensureAccessToken(ctx);
const raw = await ctx.cursor.get();
const prev = parseCursor(raw);
const next = cursorFrom((await get<LastActivities>(ctx, token, "/sync/last_activities")).data);
if (!prev) {
// No usable cursor: full pull, then remember where we are.
yield* history(ctx, token);
yield* ratings(ctx, token);
yield* watchlist(ctx, token);
yield* lists(ctx, token);
} else {
if (next.history !== prev.history) yield* history(ctx, token, prev.history || undefined);
if (next.ratings !== prev.ratings) yield* ratings(ctx, token);
if (next.watchlist !== prev.watchlist) yield* watchlist(ctx, token);
if (next.lists !== prev.lists) yield* lists(ctx, token);
}
const encoded = JSON.stringify(next);
if (encoded !== raw) await ctx.cursor.set(encoded);
}
+1
View File
@@ -0,0 +1 @@
{ "device_code": "dev123", "user_code": "ABCD1234", "verification_url": "https://trakt.tv/activate", "expires_in": 600, "interval": 5 }
+11
View File
@@ -0,0 +1,11 @@
[
{
"id": 9001, "watched_at": "2024-03-05T20:15:00.000Z", "action": "watch", "type": "movie",
"movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } }
},
{
"id": 9002, "watched_at": "2024-03-06T21:30:00.000Z", "action": "watch", "type": "episode",
"episode": { "season": 2, "number": 3, "title": "The Heist", "ids": { "trakt": 5003, "tvdb": 7003, "imdb": "tt0000003", "tmdb": 8003 } },
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
}
]
+6
View File
@@ -0,0 +1,6 @@
[
{
"id": 9003, "watched_at": "2023-12-31T23:59:59.000Z", "action": "watch", "type": "movie",
"movie": { "title": "Heat", "year": 1995, "ids": { "trakt": 1003, "slug": "heat-1995", "imdb": "tt0113277", "tmdb": 949 } }
}
]
+8
View File
@@ -0,0 +1,8 @@
{
"all": "2024-03-07T10:00:00.000Z",
"movies": { "watched_at": "2024-03-05T20:15:00.000Z", "collected_at": null, "rated_at": "2024-03-05T22:00:00.000Z", "watchlisted_at": "2024-02-01T08:00:00.000Z", "favorited_at": null, "commented_at": null, "paused_at": null, "hidden_at": null },
"episodes": { "watched_at": "2024-03-06T21:30:00.000Z", "collected_at": null, "rated_at": null, "watchlisted_at": null, "commented_at": null, "paused_at": null },
"shows": { "rated_at": "2024-03-07T10:00:00.000Z", "watchlisted_at": null, "favorited_at": null, "commented_at": null, "hidden_at": null },
"seasons": { "rated_at": null, "watchlisted_at": null, "commented_at": null, "hidden_at": null },
"lists": { "liked_at": null, "updated_at": "2024-01-02T00:00:00.000Z" }
}
+10
View File
@@ -0,0 +1,10 @@
[
{
"rank": 1, "id": 1, "listed_at": "2024-01-01T00:00:00.000Z", "notes": "rewatch yearly", "type": "movie",
"movie": { "title": "Heat", "year": 1995, "ids": { "trakt": 1003, "slug": "heat-1995", "imdb": "tt0113277", "tmdb": 949 } }
},
{
"rank": 2, "id": 2, "listed_at": "2024-01-02T00:00:00.000Z", "notes": null, "type": "show",
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
}
]
+3
View File
@@ -0,0 +1,3 @@
[
{ "name": "Favourites", "description": "All-time best", "privacy": "private", "item_count": 2, "ids": { "trakt": 7777, "slug": "favourites" } }
]
+10
View File
@@ -0,0 +1,10 @@
[
{
"rated_at": "2024-03-05T22:00:00.000Z", "rating": 8, "type": "movie",
"movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } }
},
{
"rated_at": "2024-03-07T10:00:00.000Z", "rating": 10, "type": "show",
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
}
]
+4
View File
@@ -0,0 +1,4 @@
[
{ "type": "movie", "score": 90.1, "movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } } },
{ "type": "show", "score": 40.2, "show": { "title": "Arrival Show", "year": 2020, "ids": { "trakt": 2010, "slug": "arrival-show", "tvdb": 71010, "imdb": "tt7100010", "tmdb": 81010 } } }
]
+1
View File
@@ -0,0 +1 @@
{ "access_token": "new-access", "token_type": "bearer", "expires_in": 7776000, "refresh_token": "new-refresh", "scope": "public", "created_at": 1772000000 }
+6
View File
@@ -0,0 +1,6 @@
[
{
"rank": 1, "id": 333, "listed_at": "2024-02-01T08:00:00.000Z", "notes": null, "type": "movie",
"movie": { "title": "Dune: Part Two", "year": 2024, "ids": { "trakt": 1004, "slug": "dune-part-two-2024", "imdb": "tt15239678", "tmdb": 693134 } }
}
]
+226
View File
@@ -0,0 +1,226 @@
import { describe, expect, test } from "bun:test";
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
import type { PluginContext } from "@sttracker/plugin-sdk/types";
import plugin from "../index.ts";
import history1 from "./fixtures/history-page1.json";
import history2 from "./fixtures/history-page2.json";
import ratings from "./fixtures/ratings.json";
import watchlist from "./fixtures/watchlist.json";
import lists from "./fixtures/lists.json";
import listItems from "./fixtures/list-items.json";
import lastActivities from "./fixtures/last-activities.json";
import tokenRefresh from "./fixtures/token-refresh.json";
import deviceCode from "./fixtures/device-code.json";
import search from "./fixtures/search.json";
const B = "https://api.trakt.tv";
const NOW = new Date("2024-04-01T00:00:00.000Z");
const FRESH = { accessToken: "old-access", refreshToken: "old-refresh", expiresAt: "2024-06-01T00:00:00.000Z" };
const paged = (req: Request) => {
const page = new URL(req.url).searchParams.get("page");
return Response.json(page === "2" ? history2 : history1, { headers: { "X-Pagination-Page-Count": "2" } });
};
function setup(routes: Record<string, unknown>, creds: Record<string, string> | null = FRESH, cursor?: string) {
const fx = createFixtureHttp(routes);
const ctx = createTestContext({ fetch: fx.fetch, now: () => NOW, config: { clientId: "cid", clientSecret: "csec" } });
return (async () => {
if (creds) await ctx.credentials.save(creds);
ctx.saved.credentials.length = 0;
if (cursor) await ctx.cursor.set(cursor);
ctx.saved.cursor.length = 0;
return { fx, ctx };
})();
}
const fullRoutes = {
[`GET ${B}/sync/history`]: paged,
[`GET ${B}/sync/ratings`]: ratings,
[`GET ${B}/sync/watchlist`]: watchlist,
[`GET ${B}/users/me/lists`]: lists,
[`GET ${B}/users/me/lists/7777/items`]: listItems,
};
describe("trakt pull", () => {
test("history pages through X-Pagination-Page-Count", async () => {
const { fx, ctx } = await setup(fullRoutes);
const recs = await collect(plugin.pull!(ctx));
const hist = recs.filter((r) => r.key.startsWith("history:"));
expect(hist.map((r) => r.key)).toEqual(["history:9001", "history:9002", "history:9003"]);
const pages = fx.calls.filter((c) => c.url.includes("/sync/history")).map((c) => new URL(c.url).searchParams.get("page"));
expect(pages).toEqual(["1", "2"]);
expect(new URL(fx.calls[0]!.url).searchParams.get("limit")).toBe("250");
expect(fx.calls[0]!.headers.get("trakt-api-version")).toBe("2");
expect(fx.calls[0]!.headers.get("trakt-api-key")).toBe("cid");
expect(fx.calls[0]!.headers.get("authorization")).toBe("Bearer old-access");
});
test("movie history: day precision, full datetime kept, ids", async () => {
const { ctx } = await setup(fullRoutes);
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "history:9001")!;
if (r.kind !== "engagement") throw new Error("kind");
expect(r.work.mediaType).toBe("movie");
expect(r.work.externalIds).toEqual({ trakt_movie: "1001", tmdb_movie: "329865", imdb: "tt2543164" });
expect(r.engagement.finishedAt).toEqual({ value: "2024-03-05T20:15:00.000Z", precision: "day" });
});
test("episode history: episode -> season -> show chain with position", async () => {
const { ctx } = await setup(fullRoutes);
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "history:9002")!;
if (r.kind !== "engagement") throw new Error("kind");
expect(r.work.mediaType).toBe("episode");
expect(r.work.position).toEqual({ season: 2, episode: 3 });
expect(r.work.externalIds).toEqual({ trakt_episode: "5003", tvdb_episode: "7003", tmdb_episode: "8003", imdb: "tt0000003" });
expect(r.work.parent?.mediaType).toBe("season");
expect(r.work.parent?.position).toEqual({ season: 2 });
const show = r.work.parent?.parent;
expect(show?.mediaType).toBe("show");
expect(show?.externalIds).toEqual({ trakt_show: "2002", tmdb_show: "80002", tvdb_show: "70002", imdb: "tt7000002" });
});
test("rating 8 becomes 80", async () => {
const { ctx } = await setup(fullRoutes);
const recs = await collect(plugin.pull!(ctx));
const m = recs.find((x) => x.key === "rating:movie:1001")!;
const s = recs.find((x) => x.key === "rating:show:2002")!;
if (m.kind !== "entry" || s.kind !== "entry") throw new Error("kind");
expect(m.entry.rating).toBe(80);
expect(s.entry.rating).toBe(100);
});
test("watchlist becomes planned", async () => {
const { ctx } = await setup(fullRoutes);
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "watchlist:movie:1004")!;
if (r.kind !== "entry") throw new Error("kind");
expect(r.entry.status).toBe("planned");
expect(r.work.title).toBe("Dune: Part Two");
});
test("custom lists with items and notes", async () => {
const { ctx } = await setup(fullRoutes);
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "list:7777")!;
if (r.kind !== "list") throw new Error("kind");
expect(r.list.name).toBe("Favourites");
expect(r.list.items.map((i) => i.work.title)).toEqual(["Heat", "Great Show"]);
expect(r.list.items[0]!.note).toBe("rewatch yearly");
});
test("a token near expiry is refreshed and saved before the first /sync call", async () => {
let savedWhenSyncCalled = -1;
const routes = {
...fullRoutes,
[`POST ${B}/oauth/token`]: () => tokenRefresh,
[`GET ${B}/sync/history`]: (req: Request) => {
if (savedWhenSyncCalled < 0) savedWhenSyncCalled = ctxRef.saved.credentials.length;
return paged(req);
},
};
const { fx, ctx } = await setup(routes, { ...FRESH, expiresAt: "2024-04-01T12:00:00.000Z" });
ctxRef = ctx;
await collect(plugin.pull!(ctx));
expect(fx.calls[0]!.method).toBe("POST");
expect(await fx.calls[0]!.json()).toEqual({
refresh_token: "old-refresh", client_id: "cid", client_secret: "csec",
redirect_uri: "urn:ietf:wg:oauth:2.0:oob", grant_type: "refresh_token",
});
expect(new URL(fx.calls[0]!.url).pathname).toBe("/oauth/token");
expect(new URL(fx.calls[1]!.url).pathname).toBe("/sync/history");
expect(savedWhenSyncCalled).toBe(1);
expect(ctx.saved.credentials).toEqual([
{ accessToken: "new-access", refreshToken: "new-refresh", expiresAt: new Date((1772000000 + 7776000) * 1000).toISOString() },
]);
expect(fx.calls[1]!.headers.get("authorization")).toBe("Bearer new-access");
});
test("a failed refresh saves nothing and never calls /sync", async () => {
const { fx, ctx } = await setup(
{ ...fullRoutes, [`POST ${B}/oauth/token`]: () => new Response("{}", { status: 400 }) },
{ ...FRESH, expiresAt: "2024-04-01T12:00:00.000Z" },
);
await expect(collect(plugin.pull!(ctx))).rejects.toThrow("400");
expect(ctx.saved.credentials).toEqual([]);
expect(fx.calls.map((c) => new URL(c.url).pathname)).toEqual(["/oauth/token"]);
});
test("a token with more than 24h left is not refreshed", async () => {
const { fx, ctx } = await setup(fullRoutes);
await collect(plugin.pull!(ctx));
expect(fx.calls.some((c) => c.url.includes("/oauth/token"))).toBe(false);
expect(ctx.saved.credentials).toEqual([]);
});
});
let ctxRef: ReturnType<typeof createTestContext>;
describe("trakt pullChanges", () => {
const cursorOf = (o: object) => JSON.stringify(o);
const same = {
history: "2024-03-06T21:30:00.000Z",
ratings: "2024-03-07T10:00:00.000Z",
watchlist: "2024-02-01T08:00:00.000Z",
lists: "2024-01-02T00:00:00.000Z",
};
const routes = { ...fullRoutes, [`GET ${B}/sync/last_activities`]: lastActivities };
test("unchanged last_activities yields nothing and fetches nothing else", async () => {
const { fx, ctx } = await setup(routes, FRESH, cursorOf(same));
expect(await collect(plugin.pullChanges!(ctx))).toEqual([]);
expect(fx.calls.map((c) => new URL(c.url).pathname)).toEqual(["/sync/last_activities"]);
expect(ctx.saved.cursor).toEqual([]);
});
test("only changed categories are re-fetched; history uses start_at; cursor advances", async () => {
const prev = { ...same, history: "2024-03-01T00:00:00.000Z", watchlist: "2024-01-01T00:00:00.000Z" };
const { fx, ctx } = await setup(routes, FRESH, cursorOf(prev));
const recs = await collect(plugin.pullChanges!(ctx));
expect(recs.some((r) => r.key.startsWith("history:"))).toBe(true);
expect(recs.some((r) => r.key.startsWith("watchlist:"))).toBe(true);
expect(recs.some((r) => r.key.startsWith("rating:"))).toBe(false);
const h = fx.calls.find((c) => c.url.includes("/sync/history"))!;
expect(new URL(h.url).searchParams.get("start_at")).toBe("2024-03-01T00:00:00.000Z");
expect(fx.calls.some((c) => c.url.includes("/sync/ratings"))).toBe(false);
expect(ctx.saved.cursor).toEqual([cursorOf(same)]);
});
test("no cursor: full pull and cursor set", async () => {
const { ctx } = await setup(routes);
const recs = await collect(plugin.pullChanges!(ctx));
expect(recs.length).toBeGreaterThan(5);
expect(ctx.saved.cursor).toEqual([cursorOf(same)]);
});
});
describe("trakt auth and lookup", () => {
test("manifest", () => {
expect(plugin.manifest.id).toBe("trakt");
expect(plugin.manifest.capabilities).toEqual(["pull", "pullChanges", "lookup"]);
expect(plugin.manifest.attribution).toBe("STTracker uses Trakt but is not endorsed or certified by Trakt.");
});
test("device auth start and poll", async () => {
let n = 0;
const { ctx } = await setup({
[`POST ${B}/oauth/device/code`]: deviceCode,
[`POST ${B}/oauth/device/token`]: () => (n++ === 0 ? new Response("", { status: 400 }) : tokenRefresh),
}, null);
const start = await plugin.deviceAuth!.start(ctx);
expect(start).toEqual({ userCode: "ABCD1234", verificationUrl: "https://trakt.tv/activate", deviceCode: "dev123", intervalSec: 5, expiresInSec: 600 });
expect(await plugin.deviceAuth!.poll(ctx, "dev123")).toBe("pending");
expect(await plugin.deviceAuth!.poll(ctx, "dev123")).toEqual({
accessToken: "new-access", refreshToken: "new-refresh", expiresAt: new Date((1772000000 + 7776000) * 1000).toISOString(),
});
});
test("device auth poll throws on terminal errors", async () => {
const { ctx } = await setup({ [`POST ${B}/oauth/device/token`]: () => new Response("", { status: 418 }) }, null);
await expect(plugin.deviceAuth!.poll(ctx as PluginContext, "x")).rejects.toThrow("418");
});
test("lookup searches movie,show", async () => {
const { fx, ctx } = await setup({ [`GET ${B}/search/movie,show`]: search }, null);
const hints = await plugin.lookup!(ctx, { title: "Arrival" });
expect(new URL(fx.calls[0]!.url).searchParams.get("query")).toBe("Arrival");
expect(hints.map((h) => [h.mediaType, h.title])).toEqual([["movie", "Arrival"], ["show", "Arrival Show"]]);
});
});