merge: feat/phase1 into task/6
This commit is contained in:
commit
4c65939d61
63 files changed
+4092
-2
No files matched your search
@@ -23,3 +23,4 @@ CLAUDE.local.md
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
.claude/worktrees/
|
||||
@@ -81,3 +81,26 @@ instead of 26. Where `AGENTS.md` or the spec say "ULID", read "UUIDv7".
|
||||
|
||||
**Consequences:** Rule changes happen in one file, and the pointer files must stay thin. Plans are working
|
||||
artefacts: kept after merge, they go stale and contradict the code, so the spec is the record.
|
||||
|
||||
## 2026-10-30: Catalogue matching is race-safe through unique keys plus whole-transaction retry
|
||||
**Status:** accepted
|
||||
**Context:** Two syncs matching the same new work at once both miss the lookup and both insert. On
|
||||
Postgres the loser hit the `external_ids (namespace, value)` unique key and crashed the sync; seasons and
|
||||
episodes had no key at all and were duplicated.
|
||||
**Decision:**
|
||||
- Migration `0002_works_position_unique` adds a unique index on `works (parent_id, position)`. NULLs are
|
||||
distinct on both dialects, so top-level works and position-less children are unaffected.
|
||||
- `matchWork` re-runs the whole resolve in a fresh transaction on a unique violation (SQLite
|
||||
"UNIQUE constraint failed", Postgres SQLSTATE 23505), up to 3 attempts. Postgres aborts the transaction
|
||||
on the error, so a statement-level retry is not possible.
|
||||
- Adding missing external IDs to an existing work uses `ON CONFLICT DO NOTHING`.
|
||||
- An ID match only counts if the work is shared or owned by the matching user, and private works can
|
||||
never carry external IDs (`createWork` refuses; global merges accept only shared works).
|
||||
- A global merge repoints children of `from` to `to` unless `to` already has a child at that position.
|
||||
|
||||
**Alternatives:**
|
||||
- Advisory or table locks: dialect-specific, and SQLite already serialises writes.
|
||||
- Upserts throughout `createWork`: would hide genuine conflicts and need per-dialect `RETURNING` handling.
|
||||
|
||||
**Consequences:** Parents resolved without IDs or position (title/year reuse) still have no unique key
|
||||
and can be duplicated under concurrency; tracked in `docs/TODO.md`.
|
||||
@@ -9,6 +9,7 @@ Small, specific, independently actionable. Remove only when actually done.
|
||||
|
||||
## Next
|
||||
- [ ] Phase 2 and Phase 3 plans (spec §12).
|
||||
- [ ] Generate image thumbnails for attachments (Phase 1 sizes images with CSS).
|
||||
|
||||
## Someday
|
||||
|
||||
@@ -23,4 +24,7 @@ Small, specific, independently actionable. Remove only when actually done.
|
||||
file, and compare steps and token use.
|
||||
|
||||
## Known issues
|
||||
- [ ] Admin merge does not merge colliding children recursively; colliding children stay under the tombstoned parent.
|
||||
- [ ] Matching: a shared parent resolved without external IDs or position (title/year reuse in
|
||||
`packages/core/src/matching.ts`) has no unique key, so concurrent matches can create it twice.
|
||||
- [ ] `bun run typecheck` fails with TS18003 until source files exist under `apps/`, `packages/` or `plugins/`.
|
||||
@@ -0,0 +1,176 @@
|
||||
import { mkdir, rename, rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import type { AttachmentEntity, AttachmentsTable } from "@sttracker/db/schema";
|
||||
import { ValidationError } from "./errors.ts";
|
||||
|
||||
export type Attachment = AttachmentsTable;
|
||||
|
||||
const MAX_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
/** Lowercase and strip parameters (`; charset=...`). */
|
||||
function normalizeMime(mime: string): string {
|
||||
return (mime.split(";")[0] ?? "").trim().toLowerCase();
|
||||
}
|
||||
|
||||
function mimeAllowed(mime: string): boolean {
|
||||
if (mime === "image/svg+xml") return false; // scriptable; never serve user SVG
|
||||
return mime.startsWith("image/") || mime === "application/pdf" || mime === "text/plain";
|
||||
}
|
||||
|
||||
const locks = new Map<string, Promise<void>>();
|
||||
|
||||
/** Serialise file mutations per sha within this process. */
|
||||
async function withLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
|
||||
const prev = locks.get(key) ?? Promise.resolve();
|
||||
const run = prev.then(fn, fn);
|
||||
const tail = run.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
locks.set(key, tail);
|
||||
try {
|
||||
return await run;
|
||||
} finally {
|
||||
if (locks.get(key) === tail) locks.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
function pathFor(dataDir: string, sha: string): string {
|
||||
return join(dataDir, "attachments", sha.slice(0, 2), sha.slice(2, 4), sha);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a file content-addressed by sha256 (deduplicated on disk) and link it to an entity.
|
||||
* Each call creates a new attachment row and link; identical bytes share one file.
|
||||
*/
|
||||
export async function storeAttachment(
|
||||
db: Db,
|
||||
dataDir: string,
|
||||
userId: string,
|
||||
file: { bytes: Uint8Array; mime: string; filename: string; caption?: string; takenAt?: string },
|
||||
link: { entity: AttachmentEntity; entityId: string },
|
||||
): Promise<Attachment> {
|
||||
const mime = normalizeMime(file.mime);
|
||||
if (!mimeAllowed(mime)) throw new ValidationError("mime", "unsupported file type");
|
||||
if (file.bytes.byteLength > MAX_BYTES) throw new ValidationError("file", "larger than 25 MB");
|
||||
|
||||
const sha256 = new Bun.CryptoHasher("sha256").update(file.bytes).digest("hex");
|
||||
const path = pathFor(dataDir, sha256);
|
||||
|
||||
const row: Attachment = {
|
||||
id: newId(),
|
||||
user_id: userId,
|
||||
sha256,
|
||||
mime,
|
||||
size: file.bytes.byteLength,
|
||||
filename: file.filename,
|
||||
caption: file.caption ?? null,
|
||||
taken_at: file.takenAt ?? null,
|
||||
visibility: null,
|
||||
created_at: nowIso(),
|
||||
};
|
||||
await withLock(sha256, async () => {
|
||||
if (!(await Bun.file(path).exists())) {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const tmp = `${path}.tmp-${newId()}`;
|
||||
try {
|
||||
await Bun.write(tmp, file.bytes);
|
||||
await rename(tmp, path);
|
||||
} catch (e) {
|
||||
await rm(tmp, { force: true });
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
await db.k.transaction().execute(async (tx) => {
|
||||
await tx.insertInto("attachments").values(row).execute();
|
||||
await tx
|
||||
.insertInto("attachment_links")
|
||||
.values({ id: newId(), attachment_id: row.id, entity: link.entity, entity_id: link.entityId })
|
||||
.execute();
|
||||
});
|
||||
});
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Open an attachment's file. Returns null when it does not exist or belongs to another user. */
|
||||
export async function readAttachment(
|
||||
db: Db,
|
||||
dataDir: string,
|
||||
userId: string,
|
||||
id: string,
|
||||
): Promise<{ file: ReturnType<typeof Bun.file>; mime: string } | null> {
|
||||
const row = await db.k
|
||||
.selectFrom("attachments")
|
||||
.select(["sha256", "mime"])
|
||||
.where("id", "=", id)
|
||||
.where("user_id", "=", userId)
|
||||
.executeTakeFirst();
|
||||
if (!row) return null;
|
||||
return { file: Bun.file(pathFor(dataDir, row.sha256)), mime: row.mime };
|
||||
}
|
||||
|
||||
/** Attachments of `userId` linked to the given entity, oldest first. */
|
||||
export async function listAttachments(
|
||||
db: Db,
|
||||
userId: string,
|
||||
entity: AttachmentEntity,
|
||||
entityId: string,
|
||||
): Promise<Attachment[]> {
|
||||
return db.k
|
||||
.selectFrom("attachments")
|
||||
.innerJoin("attachment_links", "attachment_links.attachment_id", "attachments.id")
|
||||
.selectAll("attachments")
|
||||
.where("attachments.user_id", "=", userId)
|
||||
.where("attachment_links.entity", "=", entity)
|
||||
.where("attachment_links.entity_id", "=", entityId)
|
||||
.orderBy("attachments.created_at")
|
||||
.orderBy("attachments.id")
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an attachment's link to an entity. When no links remain the attachment row is
|
||||
* deleted, and the file too if no other row references its content. No-op for other users.
|
||||
*/
|
||||
export async function unlinkAttachment(
|
||||
db: Db,
|
||||
dataDir: string,
|
||||
userId: string,
|
||||
attachmentId: string,
|
||||
entity: AttachmentEntity,
|
||||
entityId: string,
|
||||
): Promise<void> {
|
||||
const att = await db.k
|
||||
.selectFrom("attachments")
|
||||
.select("sha256")
|
||||
.where("id", "=", attachmentId)
|
||||
.where("user_id", "=", userId)
|
||||
.executeTakeFirst();
|
||||
if (!att) return;
|
||||
await withLock(att.sha256, async () => {
|
||||
const orphan = await db.k.transaction().execute(async (tx) => {
|
||||
await tx
|
||||
.deleteFrom("attachment_links")
|
||||
.where("attachment_id", "=", attachmentId)
|
||||
.where("entity", "=", entity)
|
||||
.where("entity_id", "=", entityId)
|
||||
.execute();
|
||||
const left = await tx
|
||||
.selectFrom("attachment_links")
|
||||
.select("id")
|
||||
.where("attachment_id", "=", attachmentId)
|
||||
.executeTakeFirst();
|
||||
if (left) return false;
|
||||
await tx.deleteFrom("attachments").where("id", "=", attachmentId).execute();
|
||||
const shared = await tx
|
||||
.selectFrom("attachments")
|
||||
.select("id")
|
||||
.where("sha256", "=", att.sha256)
|
||||
.executeTakeFirst();
|
||||
return !shared;
|
||||
});
|
||||
if (orphan) await rm(pathFor(dataDir, att.sha256), { force: true });
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import type { TokenScope } from "@sttracker/db/schema";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import { fromJson, toJson } from "@sttracker/db/json";
|
||||
import { sql } from "kysely";
|
||||
import { AuthError } from "./errors.ts";
|
||||
|
||||
export type User = { id: string; username: string; displayName: string; role: "admin" | "member" };
|
||||
|
||||
const USERNAME_RE = /^[a-z0-9_]{2,32}$/;
|
||||
const DAY_MS = 86_400_000;
|
||||
const SESSION_DAYS = 30;
|
||||
|
||||
function sha256Hex(s: string): string {
|
||||
return new Bun.CryptoHasher("sha256").update(s).digest("hex");
|
||||
}
|
||||
|
||||
let dummy: Promise<string> | undefined;
|
||||
/** Fixed argon2id hash, computed once, so unknown-user logins cost the same as real ones. */
|
||||
function dummyHash(): Promise<string> {
|
||||
return (dummy ??= Bun.password.hash("sttracker-dummy-password", { algorithm: "argon2id" }));
|
||||
}
|
||||
|
||||
function randomToken(bytes = 32): string {
|
||||
return crypto.getRandomValues(new Uint8Array(bytes)).toBase64({ alphabet: "base64url", omitPadding: true });
|
||||
}
|
||||
|
||||
function toUser(r: { id: string; username: string; display_name: string; role: "admin" | "member" }): User {
|
||||
return { id: r.id, username: r.username, displayName: r.display_name, role: r.role };
|
||||
}
|
||||
|
||||
export async function registerUser(
|
||||
db: Db,
|
||||
input: { username: string; password: string; displayName?: string; inviteCode?: string },
|
||||
): Promise<User> {
|
||||
const username = input.username.toLowerCase();
|
||||
if (!USERNAME_RE.test(username)) throw new AuthError("invalid_username");
|
||||
const passwordHash = await Bun.password.hash(input.password, { algorithm: "argon2id" });
|
||||
|
||||
return db.k.transaction().execute(async (tx) => {
|
||||
// Serialise registrations on Postgres so two first-time sign-ups cannot both become admin.
|
||||
// SQLite is already serialised by its single connection.
|
||||
if (db.dialect === "postgres") await sql`select pg_advisory_xact_lock(727401)`.execute(tx);
|
||||
|
||||
const anyUser = await tx.selectFrom("users").select("id").limit(1).executeTakeFirst();
|
||||
if (anyUser && !input.inviteCode) throw new AuthError("invite_required");
|
||||
|
||||
const existing = await tx.selectFrom("users").select("id").where("username", "=", username).executeTakeFirst();
|
||||
if (existing) throw new AuthError("username_taken");
|
||||
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
|
||||
const role = anyUser ? "member" : "admin";
|
||||
const display_name = input.displayName ?? username;
|
||||
await tx
|
||||
.insertInto("users")
|
||||
.values({
|
||||
id,
|
||||
username,
|
||||
display_name,
|
||||
password_hash: passwordHash,
|
||||
role,
|
||||
preferences: toJson({}),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
.execute();
|
||||
|
||||
// used_by references users.id, so claim after the insert; the tx rolls back on failure.
|
||||
if (anyUser) {
|
||||
const claimed = await tx
|
||||
.updateTable("invites")
|
||||
.set({ used_by: id })
|
||||
.where("code_hash", "=", sha256Hex(input.inviteCode!))
|
||||
.where("used_by", "is", null)
|
||||
.where("expires_at", ">", now)
|
||||
.executeTakeFirst();
|
||||
if (Number(claimed.numUpdatedRows) !== 1) throw new AuthError("invite_required");
|
||||
}
|
||||
return { id, username, displayName: display_name, role };
|
||||
});
|
||||
}
|
||||
|
||||
export async function login(
|
||||
db: Db,
|
||||
username: string,
|
||||
password: string,
|
||||
): Promise<{ user: User; sessionToken: string }> {
|
||||
const row = await db.k
|
||||
.selectFrom("users")
|
||||
.selectAll()
|
||||
.where("username", "=", username.toLowerCase())
|
||||
.executeTakeFirst();
|
||||
const hash = row?.password_hash ?? (await dummyHash());
|
||||
const ok = await Bun.password.verify(password, hash);
|
||||
if (!row || !ok) {
|
||||
throw new AuthError("invalid_credentials");
|
||||
}
|
||||
const sessionToken = randomToken();
|
||||
await db.k
|
||||
.insertInto("sessions")
|
||||
.values({
|
||||
id: newId(),
|
||||
user_id: row.id,
|
||||
token_hash: sha256Hex(sessionToken),
|
||||
expires_at: new Date(Date.now() + SESSION_DAYS * DAY_MS).toISOString(),
|
||||
created_at: nowIso(),
|
||||
})
|
||||
.execute();
|
||||
return { user: toUser(row), sessionToken };
|
||||
}
|
||||
|
||||
export async function getSessionUser(db: Db, token: string): Promise<User | null> {
|
||||
const row = await db.k
|
||||
.selectFrom("sessions")
|
||||
.innerJoin("users", "users.id", "sessions.user_id")
|
||||
.select(["users.id", "users.username", "users.display_name", "users.role"])
|
||||
.where("sessions.token_hash", "=", sha256Hex(token))
|
||||
.where("sessions.expires_at", ">", nowIso())
|
||||
.executeTakeFirst();
|
||||
return row ? toUser(row) : null;
|
||||
}
|
||||
|
||||
export async function logout(db: Db, token: string): Promise<void> {
|
||||
await db.k.deleteFrom("sessions").where("token_hash", "=", sha256Hex(token)).execute();
|
||||
}
|
||||
|
||||
/** Admin-only. Returns the invite code (shown once; only its hash is stored). */
|
||||
export async function createInvite(db: Db, adminId: string, ttlDays = 7): Promise<string> {
|
||||
const admin = await db.k.selectFrom("users").select("role").where("id", "=", adminId).executeTakeFirst();
|
||||
if (admin?.role !== "admin") throw new AuthError("forbidden");
|
||||
const code = randomToken(16);
|
||||
await db.k
|
||||
.insertInto("invites")
|
||||
.values({
|
||||
id: newId(),
|
||||
code_hash: sha256Hex(code),
|
||||
created_by: adminId,
|
||||
used_by: null,
|
||||
expires_at: new Date(Date.now() + ttlDays * DAY_MS).toISOString(),
|
||||
created_at: nowIso(),
|
||||
})
|
||||
.execute();
|
||||
return code;
|
||||
}
|
||||
|
||||
export async function createAppToken(
|
||||
db: Db,
|
||||
userId: string,
|
||||
name: string,
|
||||
scopes: TokenScope[],
|
||||
): Promise<{ id: string; token: string }> {
|
||||
const id = newId();
|
||||
const token = `stt_${randomToken()}`;
|
||||
await db.k
|
||||
.insertInto("app_tokens")
|
||||
.values({
|
||||
id,
|
||||
user_id: userId,
|
||||
name,
|
||||
token_hash: sha256Hex(token),
|
||||
scopes: toJson(scopes),
|
||||
last_used_at: null,
|
||||
revoked_at: null,
|
||||
created_at: nowIso(),
|
||||
})
|
||||
.execute();
|
||||
return { id, token };
|
||||
}
|
||||
|
||||
export async function verifyAppToken(db: Db, token: string, scope: TokenScope): Promise<User | null> {
|
||||
const row = await db.k
|
||||
.selectFrom("app_tokens")
|
||||
.innerJoin("users", "users.id", "app_tokens.user_id")
|
||||
.select([
|
||||
"app_tokens.id as token_id",
|
||||
"app_tokens.scopes",
|
||||
"users.id",
|
||||
"users.username",
|
||||
"users.display_name",
|
||||
"users.role",
|
||||
])
|
||||
.where("app_tokens.token_hash", "=", sha256Hex(token))
|
||||
.where("app_tokens.revoked_at", "is", null)
|
||||
.executeTakeFirst();
|
||||
if (!row || !fromJson<string[]>(row.scopes, []).includes(scope)) return null;
|
||||
await db.k.updateTable("app_tokens").set({ last_used_at: nowIso() }).where("id", "=", row.token_id).execute();
|
||||
return toUser(row);
|
||||
}
|
||||
|
||||
export async function revokeAppToken(db: Db, userId: string, id: string): Promise<void> {
|
||||
await db.k
|
||||
.updateTable("app_tokens")
|
||||
.set({ revoked_at: nowIso() })
|
||||
.where("id", "=", id)
|
||||
.where("user_id", "=", userId)
|
||||
.where("revoked_at", "is", null)
|
||||
.execute();
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { CryptoError } from "./errors.ts";
|
||||
|
||||
const SALT = new TextEncoder().encode("sttracker-v1");
|
||||
const INFO = new TextEncoder().encode("sttracker-credentials");
|
||||
|
||||
function b64(bytes: Uint8Array): string {
|
||||
return Buffer.from(bytes).toString("base64");
|
||||
}
|
||||
|
||||
async function deriveKey(secret: string): Promise<CryptoKey> {
|
||||
const material = await crypto.subtle.importKey("raw", new TextEncoder().encode(secret), "HKDF", false, [
|
||||
"deriveKey",
|
||||
]);
|
||||
return crypto.subtle.deriveKey(
|
||||
{ name: "HKDF", hash: "SHA-256", salt: SALT, info: INFO },
|
||||
material,
|
||||
{ name: "AES-GCM", length: 256 },
|
||||
false,
|
||||
["encrypt", "decrypt"],
|
||||
);
|
||||
}
|
||||
|
||||
/** Encrypt a JSON value with AES-256-GCM. Returns `v1.<b64 iv>.<b64 ciphertext>`. */
|
||||
export async function encryptJson(secret: string, value: unknown): Promise<string> {
|
||||
const key = await deriveKey(secret);
|
||||
const iv = crypto.getRandomValues(new Uint8Array(12));
|
||||
const plain = new TextEncoder().encode(JSON.stringify(value));
|
||||
const ct = new Uint8Array(await crypto.subtle.encrypt({ name: "AES-GCM", iv }, key, plain));
|
||||
return `v1.${b64(iv)}.${b64(ct)}`;
|
||||
}
|
||||
|
||||
/** Decrypt a blob from {@link encryptJson}. Throws `CryptoError` on a wrong secret or bad blob. */
|
||||
export async function decryptJson<T>(secret: string, blob: string): Promise<T> {
|
||||
const parts = blob.split(".");
|
||||
if (parts.length !== 3 || parts[0] !== "v1") throw new CryptoError("malformed credential blob");
|
||||
try {
|
||||
const iv = new Uint8Array(Buffer.from(parts[1]!, "base64"));
|
||||
const ct = new Uint8Array(Buffer.from(parts[2]!, "base64"));
|
||||
const key = await deriveKey(secret);
|
||||
const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv }, key, ct);
|
||||
return JSON.parse(new TextDecoder().decode(plain)) as T;
|
||||
} catch (cause) {
|
||||
throw new CryptoError("decryption failed", { cause });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import { sql } from "kysely";
|
||||
import type { WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import {
|
||||
createWorkIn,
|
||||
finalWorkId,
|
||||
findWorkByExternalIdsIn,
|
||||
normalizePosition,
|
||||
resolveAliasIn,
|
||||
toWork,
|
||||
updateWorkFromHintIn,
|
||||
type K,
|
||||
} from "./works.ts";
|
||||
|
||||
export interface FuzzyInput {
|
||||
title: string;
|
||||
year?: number | null;
|
||||
creators?: { name: string }[];
|
||||
}
|
||||
|
||||
function normalize(s: string): string {
|
||||
return s
|
||||
.normalize("NFD")
|
||||
.replace(/\p{M}+/gu, "")
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, " ")
|
||||
.trim()
|
||||
.replace(/^(the|a|an)\s+/, "");
|
||||
}
|
||||
|
||||
function tokens(s: string): string[] {
|
||||
return normalize(s).split(" ").filter(Boolean);
|
||||
}
|
||||
|
||||
/**
|
||||
* 0..1 similarity. Title token-set Jaccard weighs 0.7; year 0.2 (±1 year counts half); any shared
|
||||
* creator 0.1. A component missing on either side is left out and the weights renormalised, so a
|
||||
* title-only comparison can still reach 1.
|
||||
*/
|
||||
export function fuzzyScore(a: FuzzyInput, b: FuzzyInput): number {
|
||||
const ta = new Set(tokens(a.title));
|
||||
const tb = new Set(tokens(b.title));
|
||||
let inter = 0;
|
||||
for (const t of ta) if (tb.has(t)) inter++;
|
||||
const union = ta.size + tb.size - inter;
|
||||
const jaccard = union === 0 ? 0 : inter / union;
|
||||
let score = jaccard * 0.7;
|
||||
let weight = 0.7;
|
||||
if (a.year != null && b.year != null) {
|
||||
const d = Math.abs(a.year - b.year);
|
||||
score += d === 0 ? 0.2 : d === 1 ? 0.1 : 0;
|
||||
weight += 0.2;
|
||||
}
|
||||
const ca = new Set((a.creators ?? []).map((c) => normalize(c.name)).filter(Boolean));
|
||||
const cb = (b.creators ?? []).map((c) => normalize(c.name)).filter(Boolean);
|
||||
if (ca.size > 0 && cb.length > 0) {
|
||||
if (cb.some((n) => ca.has(n))) score += 0.1;
|
||||
weight += 0.1;
|
||||
}
|
||||
return score / weight;
|
||||
}
|
||||
|
||||
interface Resolved {
|
||||
workId: string;
|
||||
created: boolean;
|
||||
draft: boolean;
|
||||
}
|
||||
|
||||
async function resolve(k: K, userId: string | null, hint: WorkHint, actor: string): Promise<Resolved> {
|
||||
// 1. Any external ID on a work this user may see (parents resolve with userId null: shared only).
|
||||
const byId = await findWorkByExternalIdsIn(k, hint.externalIds, userId);
|
||||
if (byId) {
|
||||
await updateWorkFromHintIn(k, byId, hint, actor);
|
||||
const workId = userId ? await resolveAliasIn(k, userId, byId) : byId;
|
||||
const owner = await k.selectFrom("works").select("owner_user_id").where("id", "=", workId).executeTakeFirstOrThrow();
|
||||
return { workId, created: false, draft: owner.owner_user_id !== null };
|
||||
}
|
||||
|
||||
// 2. Parent + position.
|
||||
const parentId = hint.parent ? (await resolve(k, null, hint.parent, actor)).workId : null;
|
||||
const position = normalizePosition(hint.position);
|
||||
if (parentId && position) {
|
||||
const child = await k
|
||||
.selectFrom("works")
|
||||
.select("id")
|
||||
.where("parent_id", "=", parentId)
|
||||
.where("position", "=", position)
|
||||
.where("owner_user_id", "is", null)
|
||||
.executeTakeFirst();
|
||||
if (child) {
|
||||
// A merged tombstone still holds its (parent_id, position) slot: follow it to the surviving work.
|
||||
const workId = await finalWorkId(k, child.id);
|
||||
await updateWorkFromHintIn(k, workId, hint, actor);
|
||||
return { workId, created: false, draft: false };
|
||||
}
|
||||
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
|
||||
}
|
||||
|
||||
// 3. External IDs but no match: new shared work.
|
||||
if (Object.keys(hint.externalIds).length > 0) {
|
||||
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
|
||||
}
|
||||
|
||||
if (userId === null) {
|
||||
// Parent of a shared child: reuse an identical shared work, else create one.
|
||||
let q = k
|
||||
.selectFrom("works")
|
||||
.select("id")
|
||||
.where("owner_user_id", "is", null)
|
||||
.where("merged_into_id", "is", null)
|
||||
.where("media_type", "=", hint.mediaType)
|
||||
.where(sql<boolean>`lower(title) = ${hint.title.toLowerCase()}`);
|
||||
q = hint.year === undefined ? q : q.where("year", "=", hint.year);
|
||||
q = parentId ? q.where("parent_id", "=", parentId) : q.where("parent_id", "is", null);
|
||||
const same = await q.executeTakeFirst();
|
||||
if (same) return { workId: same.id, created: false, draft: false };
|
||||
return { workId: await createWorkIn(k, { hint, parentId }, actor), created: true, draft: false };
|
||||
}
|
||||
|
||||
// 4. Private draft plus fuzzy suggestions.
|
||||
const workId = await createWorkIn(k, { hint, ownerUserId: userId }, actor);
|
||||
await suggest(k, userId, workId, hint);
|
||||
return { workId, created: true, draft: true };
|
||||
}
|
||||
|
||||
async function suggest(k: K, userId: string, draftId: string, hint: WorkHint): Promise<void> {
|
||||
// Bound the candidate set with LIKE on the significant title tokens (OR), so "The Expanse: Leviathan
|
||||
// Wakes" still finds "Leviathan Wakes". Tokens of 3+ chars, first five; falls back to the first token.
|
||||
const all = tokens(hint.title);
|
||||
const sig = all.filter((t) => t.length >= 3).slice(0, 5);
|
||||
const likeTokens = sig.length > 0 ? sig : all.slice(0, 1);
|
||||
if (likeTokens.length === 0) return;
|
||||
const rows = await k
|
||||
.selectFrom("works")
|
||||
.selectAll()
|
||||
.where("merged_into_id", "is", null)
|
||||
.where("id", "!=", draftId)
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb.and([eb("owner_user_id", "is", null), eb("media_type", "=", hint.mediaType)]),
|
||||
eb("owner_user_id", "=", userId),
|
||||
]),
|
||||
)
|
||||
.where((eb) =>
|
||||
eb.or(likeTokens.map((t) => eb(sql<string>`lower(title)`, "like", `%${t}%`))),
|
||||
)
|
||||
.limit(200)
|
||||
.execute();
|
||||
const scored = rows
|
||||
.map(toWork)
|
||||
.map((w) => ({ id: w.id, score: fuzzyScore(hint, w) }))
|
||||
.filter((c) => c.score >= 0.6)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, 5);
|
||||
const now = nowIso();
|
||||
for (const c of scored) {
|
||||
await k
|
||||
.insertInto("match_suggestions")
|
||||
.values({ id: newId(), user_id: userId, work_id: draftId, candidate_work_id: c.id, score: c.score, status: "pending", created_at: now })
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_ATTEMPTS = 3;
|
||||
|
||||
/** SQLite "UNIQUE constraint failed", or Postgres SQLSTATE 23505, anywhere in the cause chain. */
|
||||
export function isUniqueViolation(err: unknown): boolean {
|
||||
for (let e: unknown = err, depth = 0; e && typeof e === "object" && depth < 5; e = (e as { cause?: unknown }).cause, depth++) {
|
||||
const { code, errno, message } = e as { code?: unknown; errno?: unknown; message?: unknown };
|
||||
if (code === "23505" || errno === "23505") return true;
|
||||
if (typeof message === "string" && (message.includes("UNIQUE constraint failed") || message.includes("duplicate key"))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a hint to a work per spec §8.4. Never auto-links on fuzzy matches; those become suggestions.
|
||||
* A concurrent match may create the same work first (unique external id or parent+position); the losing
|
||||
* transaction is discarded and the whole resolve re-run, which then finds the winner's rows.
|
||||
*/
|
||||
export async function matchWork(db: Db, userId: string, hint: WorkHint, actor: string): Promise<Resolved> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await db.k.transaction().execute((tx) => resolve(tx, userId, hint, actor));
|
||||
} catch (err) {
|
||||
if (attempt >= MAX_ATTEMPTS || !isUniqueViolation(err)) throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import type { MediaType, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import { fromJson, toJson } from "@sttracker/db/json";
|
||||
import type { Database, WorksTable } from "@sttracker/db/schema";
|
||||
import { AuthError, NotFoundError, ValidationError } from "./errors.ts";
|
||||
|
||||
export type Creator = { name: string; role?: string };
|
||||
export type Position = { season?: number; episode?: number; index?: number };
|
||||
|
||||
/** A `works` row with JSON columns parsed. Other fields keep their column names. */
|
||||
export interface Work extends Omit<WorksTable, "creators" | "metadata" | "position"> {
|
||||
creators: Creator[];
|
||||
metadata: Record<string, unknown>;
|
||||
position: Position | null;
|
||||
}
|
||||
|
||||
/** Any Kysely handle: the root one or a transaction. */
|
||||
export type K = Kysely<Database>;
|
||||
|
||||
/** Normalised JSON for `works.position`: keys in a fixed order (season, episode, index), or null when empty. */
|
||||
export function normalizePosition(p: Position | undefined | null): string | null {
|
||||
if (!p) return null;
|
||||
const o: Position = {};
|
||||
if (p.season !== undefined) o.season = p.season;
|
||||
if (p.episode !== undefined) o.episode = p.episode;
|
||||
if (p.index !== undefined) o.index = p.index;
|
||||
return Object.keys(o).length === 0 ? null : toJson(o);
|
||||
}
|
||||
|
||||
export function toWork(r: WorksTable): Work {
|
||||
return {
|
||||
...r,
|
||||
creators: fromJson<Creator[]>(r.creators, []),
|
||||
metadata: fromJson<Record<string, unknown>>(r.metadata, {}),
|
||||
position: fromJson<Position | null>(r.position, null),
|
||||
};
|
||||
}
|
||||
|
||||
export async function createWorkIn(
|
||||
k: K,
|
||||
input: { hint: WorkHint; ownerUserId?: string | null; parentId?: string | null },
|
||||
actor: string,
|
||||
): Promise<string> {
|
||||
const { hint } = input;
|
||||
if (hint.title.trim() === "") throw new ValidationError("title", "required");
|
||||
if (input.ownerUserId && Object.keys(hint.externalIds).length > 0) {
|
||||
throw new ValidationError("externalIds", "private works cannot carry external ids");
|
||||
}
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
await k
|
||||
.insertInto("works")
|
||||
.values({
|
||||
id,
|
||||
media_type: hint.mediaType,
|
||||
title: hint.title,
|
||||
original_title: "",
|
||||
year: hint.year ?? null,
|
||||
release_date: null,
|
||||
creators: toJson(hint.creators ?? []),
|
||||
description: hint.description ?? "",
|
||||
cover_url: hint.coverUrl ?? "",
|
||||
cover_attachment_id: null,
|
||||
metadata: "{}",
|
||||
parent_id: input.parentId ?? null,
|
||||
position: normalizePosition(hint.position),
|
||||
owner_user_id: input.ownerUserId ?? null,
|
||||
merged_into_id: null,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
.execute();
|
||||
for (const [namespace, value] of Object.entries(hint.externalIds)) {
|
||||
await k.insertInto("external_ids").values({ id: newId(), work_id: id, namespace, value }).execute();
|
||||
}
|
||||
await k
|
||||
.insertInto("work_history")
|
||||
.values({ id: newId(), work_id: id, before: "{}", after: toJson({ created: true, title: hint.title }), actor, created_at: now })
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
/** Insert the work and its external IDs. */
|
||||
export function createWork(
|
||||
db: Db,
|
||||
input: { hint: WorkHint; ownerUserId?: string | null; parentId?: string | null },
|
||||
actor: string,
|
||||
): Promise<string> {
|
||||
return db.k.transaction().execute((tx) => createWorkIn(tx, input, actor));
|
||||
}
|
||||
|
||||
export async function getWorkIn(k: K, id: string): Promise<Work | null> {
|
||||
const seen = new Set<string>();
|
||||
let cur: string | null = id;
|
||||
while (cur && !seen.has(cur)) {
|
||||
seen.add(cur);
|
||||
const row: WorksTable | undefined = await k.selectFrom("works").selectAll().where("id", "=", cur).executeTakeFirst();
|
||||
if (!row) return null;
|
||||
if (!row.merged_into_id) return toWork(row);
|
||||
cur = row.merged_into_id;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Fetch a work, following `merged_into_id` tombstones to the surviving one. */
|
||||
export function getWork(db: Db, id: string): Promise<Work | null> {
|
||||
return getWorkIn(db.k, id);
|
||||
}
|
||||
|
||||
/** Follow `merged_into_id` to the surviving id (the input id if it is not merged). */
|
||||
export async function finalWorkId(k: K, id: string): Promise<string> {
|
||||
return (await getWorkIn(k, id))?.id ?? id;
|
||||
}
|
||||
|
||||
/**
|
||||
* First work matching any (namespace, value) pair, following merges. Only works visible to `userId`
|
||||
* count (shared, or private to that user); with no user, only shared works. Others are treated as absent.
|
||||
*/
|
||||
export async function findWorkByExternalIdsIn(
|
||||
k: K,
|
||||
ids: Record<string, string>,
|
||||
userId: string | null = null,
|
||||
): Promise<string | null> {
|
||||
for (const [namespace, value] of Object.entries(ids)) {
|
||||
const row = await k
|
||||
.selectFrom("external_ids")
|
||||
.select("work_id")
|
||||
.where("namespace", "=", namespace)
|
||||
.where("value", "=", value)
|
||||
.executeTakeFirst();
|
||||
if (!row) continue;
|
||||
const work = await getWorkIn(k, row.work_id);
|
||||
if (work && (work.owner_user_id === null || work.owner_user_id === userId)) return work.id;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** First work visible to `userId` (shared works only when omitted) matching any (namespace, value) pair. */
|
||||
export function findWorkByExternalIds(db: Db, ids: Record<string, string>, userId: string | null = null): Promise<string | null> {
|
||||
return findWorkByExternalIdsIn(db.k, ids, userId);
|
||||
}
|
||||
|
||||
async function addMissingExternalIds(k: K, workId: string, ids: Record<string, string>): Promise<string[]> {
|
||||
const added: string[] = [];
|
||||
for (const [namespace, value] of Object.entries(ids)) {
|
||||
const exists = await k
|
||||
.selectFrom("external_ids")
|
||||
.select("id")
|
||||
.where("namespace", "=", namespace)
|
||||
.where("value", "=", value)
|
||||
.executeTakeFirst();
|
||||
if (exists) continue;
|
||||
// A concurrent sync may have added the same pair since the check: skip it rather than fail.
|
||||
const res = await k
|
||||
.insertInto("external_ids")
|
||||
.values({ id: newId(), work_id: workId, namespace, value })
|
||||
.onConflict((oc) => oc.columns(["namespace", "value"]).doNothing())
|
||||
.executeTakeFirst();
|
||||
if (Number(res.numInsertedOrUpdatedRows ?? 0n) > 0) added.push(`${namespace}:${value}`);
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
export async function updateWorkFromHintIn(k: K, workId: string, hint: WorkHint, actor: string): Promise<void> {
|
||||
const row = await k.selectFrom("works").selectAll().where("id", "=", workId).executeTakeFirst();
|
||||
if (!row || row.owner_user_id !== null) return; // private works are never touched by syncs
|
||||
const before: Record<string, unknown> = {};
|
||||
const after: Record<string, unknown> = {};
|
||||
const patch: Partial<WorksTable> = {};
|
||||
const set = <F extends "title" | "year" | "creators" | "description" | "cover_url">(field: F, next: WorksTable[F] | undefined, empty: boolean) => {
|
||||
if (next === undefined || empty || next === row[field]) return;
|
||||
before[field] = row[field];
|
||||
after[field] = next;
|
||||
patch[field] = next;
|
||||
};
|
||||
set("title", hint.title, hint.title.trim() === "");
|
||||
set("year", hint.year, hint.year === undefined);
|
||||
set("creators", hint.creators ? toJson(hint.creators) : undefined, !hint.creators || hint.creators.length === 0);
|
||||
set("description", hint.description, !hint.description);
|
||||
set("cover_url", hint.coverUrl, !hint.coverUrl);
|
||||
const addedIds = await addMissingExternalIds(k, workId, hint.externalIds);
|
||||
if (addedIds.length > 0) after.externalIds = addedIds;
|
||||
if (Object.keys(after).length === 0) return;
|
||||
const now = nowIso();
|
||||
if (Object.keys(patch).length > 0) {
|
||||
await k.updateTable("works").set({ ...patch, updated_at: now }).where("id", "=", workId).execute();
|
||||
}
|
||||
await k
|
||||
.insertInto("work_history")
|
||||
.values({ id: newId(), work_id: workId, before: toJson(before), after: toJson(after), actor, created_at: now })
|
||||
.execute();
|
||||
}
|
||||
|
||||
/** Fill or update a shared work's display fields from a hint, add missing external IDs, log to `work_history`. */
|
||||
export function updateWorkFromHint(db: Db, workId: string, hint: WorkHint, actor: string): Promise<void> {
|
||||
return db.k.transaction().execute((tx) => updateWorkFromHintIn(tx, workId, hint, actor));
|
||||
}
|
||||
|
||||
function escapeLike(s: string): string {
|
||||
return s.replace(/[!%_]/g, (c) => `!${c}`);
|
||||
}
|
||||
|
||||
/** Shared works plus the user's own private ones, case-insensitive substring match on title. */
|
||||
export async function searchWorks(
|
||||
db: Db,
|
||||
userId: string,
|
||||
q: string,
|
||||
opts: { mediaType?: MediaType; limit?: number } = {},
|
||||
): Promise<Work[]> {
|
||||
let qb = db.k
|
||||
.selectFrom("works")
|
||||
.selectAll()
|
||||
.where("merged_into_id", "is", null)
|
||||
.where((eb) => eb.or([eb("owner_user_id", "is", null), eb("owner_user_id", "=", userId)]))
|
||||
.where(sql<boolean>`lower(title) like ${`%${escapeLike(q.toLowerCase())}%`} escape '!'`);
|
||||
if (opts.mediaType) qb = qb.where("media_type", "=", opts.mediaType);
|
||||
const rows = await qb.orderBy("title").limit(opts.limit ?? 50).execute();
|
||||
return rows.map(toWork);
|
||||
}
|
||||
|
||||
/** Follow the user's aliases and global merge tombstones to the work id they should see. */
|
||||
export async function resolveAliasIn(k: K, userId: string, workId: string): Promise<string> {
|
||||
const seen = new Set<string>();
|
||||
let cur = workId;
|
||||
while (!seen.has(cur)) {
|
||||
seen.add(cur);
|
||||
const alias = await k
|
||||
.selectFrom("work_aliases")
|
||||
.select("to_work_id")
|
||||
.where("user_id", "=", userId)
|
||||
.where("from_work_id", "=", cur)
|
||||
.executeTakeFirst();
|
||||
if (alias) {
|
||||
cur = alias.to_work_id;
|
||||
continue;
|
||||
}
|
||||
const w = await k.selectFrom("works").select("merged_into_id").where("id", "=", cur).executeTakeFirst();
|
||||
if (!w?.merged_into_id) break;
|
||||
cur = w.merged_into_id;
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
export function resolveAlias(db: Db, userId: string, workId: string): Promise<string> {
|
||||
return resolveAliasIn(db.k, userId, workId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Admin-only global merge of shared work `fromId` into `toId`. Leaves a `merged_into_id` tombstone and
|
||||
* moves external IDs and the children whose position `toId` does not already have. Both works must
|
||||
* be shared. Users without an entry on `toId` get their entry repointed; users who already
|
||||
* have one keep their `from` entry and get a pending `merged` change event.
|
||||
*/
|
||||
export async function adminMergeWorks(db: Db, adminUserId: string, fromId: string, toId: string): Promise<void> {
|
||||
await db.k.transaction().execute(async (tx) => {
|
||||
const admin = await tx.selectFrom("users").select("role").where("id", "=", adminUserId).executeTakeFirst();
|
||||
if (admin?.role !== "admin") throw new AuthError("forbidden");
|
||||
if (fromId === toId) throw new ValidationError("toId", "cannot merge a work into itself");
|
||||
const from = await tx.selectFrom("works").selectAll().where("id", "=", fromId).executeTakeFirst();
|
||||
const to = await tx.selectFrom("works").selectAll().where("id", "=", toId).executeTakeFirst();
|
||||
if (!from) throw new NotFoundError("work");
|
||||
if (!to) throw new NotFoundError("work");
|
||||
if (from.merged_into_id) throw new ValidationError("fromId", "already merged");
|
||||
if (to.merged_into_id) throw new ValidationError("toId", "already merged");
|
||||
if (from.owner_user_id !== null) throw new ValidationError("fromId", "only shared works can be merged");
|
||||
if (to.owner_user_id !== null) throw new ValidationError("toId", "only shared works can be merged");
|
||||
const now = nowIso();
|
||||
|
||||
await tx.updateTable("works").set({ merged_into_id: toId, updated_at: now }).where("id", "=", fromId).execute();
|
||||
const moved = await tx.selectFrom("external_ids").select(["namespace", "value"]).where("work_id", "=", fromId).execute();
|
||||
await tx.updateTable("external_ids").set({ work_id: toId }).where("work_id", "=", fromId).execute();
|
||||
|
||||
// Repoint children so matching through `toId` finds them. A child whose position `toId` already
|
||||
// has stays under `fromId` (unique parent_id+position); it remains reachable via the tombstone.
|
||||
const toPositions = new Set(
|
||||
(await tx.selectFrom("works").select("position").where("parent_id", "=", toId).where("position", "is not", null).execute()).map(
|
||||
(r) => r.position,
|
||||
),
|
||||
);
|
||||
const children = await tx.selectFrom("works").select(["id", "position"]).where("parent_id", "=", fromId).execute();
|
||||
const movedChildren: string[] = [];
|
||||
for (const c of children) {
|
||||
if (c.position !== null && toPositions.has(c.position)) continue;
|
||||
await tx.updateTable("works").set({ parent_id: toId, updated_at: now }).where("id", "=", c.id).execute();
|
||||
movedChildren.push(c.id);
|
||||
}
|
||||
|
||||
const entries = await tx.selectFrom("entries").select(["id", "user_id"]).where("work_id", "=", fromId).execute();
|
||||
for (const e of entries) {
|
||||
const existing = await tx
|
||||
.selectFrom("entries")
|
||||
.select("id")
|
||||
.where("user_id", "=", e.user_id)
|
||||
.where("work_id", "=", toId)
|
||||
.executeTakeFirst();
|
||||
if (existing) {
|
||||
await tx
|
||||
.insertInto("change_events")
|
||||
.values({
|
||||
id: newId(),
|
||||
user_id: e.user_id,
|
||||
connection_id: null,
|
||||
run_id: null,
|
||||
record_id: null,
|
||||
type: "merged",
|
||||
status: "pending",
|
||||
target_entity: "entry",
|
||||
target_id: e.id,
|
||||
payload: toJson({ fromWorkId: fromId, toWorkId: toId }),
|
||||
created_at: now,
|
||||
resolved_at: null,
|
||||
})
|
||||
.execute();
|
||||
} else {
|
||||
await tx.updateTable("entries").set({ work_id: toId, updated_at: now }).where("id", "=", e.id).execute();
|
||||
}
|
||||
}
|
||||
|
||||
await tx
|
||||
.insertInto("work_history")
|
||||
.values({ id: newId(), work_id: fromId, before: toJson({ merged_into_id: null }), after: toJson({ merged_into_id: toId }), actor: adminUserId, created_at: now })
|
||||
.execute();
|
||||
await tx
|
||||
.insertInto("work_history")
|
||||
.values({ id: newId(), work_id: toId, before: "{}", after: toJson({ mergedFrom: fromId, externalIds: moved, children: movedChildren }), actor: adminUserId, created_at: now })
|
||||
.execute();
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
||||
import { mkdtemp, readdir, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import {
|
||||
listAttachments,
|
||||
readAttachment,
|
||||
storeAttachment,
|
||||
unlinkAttachment,
|
||||
} from "../src/attachments.ts";
|
||||
import { ValidationError } from "../src/errors.ts";
|
||||
|
||||
async function addUser(db: Db, username: string): Promise<string> {
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
await db.k
|
||||
.insertInto("users")
|
||||
.values({
|
||||
id,
|
||||
username,
|
||||
display_name: username,
|
||||
password_hash: "x",
|
||||
role: "member",
|
||||
preferences: "{}",
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
async function filesUnder(dir: string): Promise<string[]> {
|
||||
const all = await readdir(dir, { recursive: true, withFileTypes: true });
|
||||
return all.filter((e) => e.isFile()).map((e) => join(e.parentPath, e.name));
|
||||
}
|
||||
|
||||
forEachDialect("attachments", (getDb) => {
|
||||
let dataDir: string;
|
||||
beforeEach(async () => {
|
||||
dataDir = await mkdtemp(join(tmpdir(), "sttracker-att-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(dataDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const png = (s: string) => ({
|
||||
bytes: new TextEncoder().encode(s),
|
||||
mime: "image/png",
|
||||
filename: "a.png",
|
||||
});
|
||||
|
||||
test("same bytes twice: one file, two links", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
const a = await storeAttachment(db, dataDir, u, png("hello"), { entity: "work", entityId: "w1" });
|
||||
const b = await storeAttachment(db, dataDir, u, png("hello"), { entity: "entry", entityId: "e1" });
|
||||
expect(a.sha256).toBe(b.sha256);
|
||||
expect(a.id).not.toBe(b.id);
|
||||
const files = await filesUnder(join(dataDir, "attachments"));
|
||||
expect(files).toHaveLength(1);
|
||||
expect(files[0]).toBe(
|
||||
join(dataDir, "attachments", a.sha256.slice(0, 2), a.sha256.slice(2, 4), a.sha256),
|
||||
);
|
||||
const links = await db.k.selectFrom("attachment_links").selectAll().execute();
|
||||
expect(links).toHaveLength(2);
|
||||
expect(await listAttachments(db, u, "work", "w1")).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("rejects disallowed mime and oversize", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
const link = { entity: "work", entityId: "w1" } as const;
|
||||
await expect(
|
||||
storeAttachment(db, dataDir, u, { ...png("x"), mime: "application/x-msdownload", filename: "a.exe" }, link),
|
||||
).rejects.toBeInstanceOf(ValidationError);
|
||||
await expect(
|
||||
storeAttachment(db, dataDir, u, { ...png("x"), bytes: new Uint8Array(25 * 1024 * 1024 + 1) }, link),
|
||||
).rejects.toBeInstanceOf(ValidationError);
|
||||
});
|
||||
|
||||
test("reads own file; another user gets null", async () => {
|
||||
const db = getDb();
|
||||
const u1 = await addUser(db, "u1");
|
||||
const u2 = await addUser(db, "u2");
|
||||
const a = await storeAttachment(db, dataDir, u1, png("secret"), { entity: "work", entityId: "w1" });
|
||||
const r = await readAttachment(db, dataDir, u1, a.id);
|
||||
expect(r?.mime).toBe("image/png");
|
||||
expect(await r?.file.text()).toBe("secret");
|
||||
expect(await readAttachment(db, dataDir, u2, a.id)).toBeNull();
|
||||
expect(await readAttachment(db, dataDir, u1, "nope")).toBeNull();
|
||||
});
|
||||
|
||||
test("unlink removes link, orphan row and unshared file", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
const a = await storeAttachment(db, dataDir, u, png("bye"), { entity: "work", entityId: "w1" });
|
||||
const b = await storeAttachment(db, dataDir, u, png("bye"), { entity: "work", entityId: "w2" });
|
||||
await unlinkAttachment(db, dataDir, u, a.id, "work", "w1");
|
||||
expect(await listAttachments(db, u, "work", "w1")).toHaveLength(0);
|
||||
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
|
||||
await unlinkAttachment(db, dataDir, u, b.id, "work", "w2");
|
||||
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(0);
|
||||
expect(await db.k.selectFrom("attachments").selectAll().execute()).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("rejects svg and normalizes mime", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
const link = { entity: "work", entityId: "w1" } as const;
|
||||
for (const mime of ["image/svg+xml", "IMAGE/SVG+XML; charset=utf-8"]) {
|
||||
await expect(storeAttachment(db, dataDir, u, { ...png("x"), mime }, link)).rejects.toBeInstanceOf(
|
||||
ValidationError,
|
||||
);
|
||||
}
|
||||
const a = await storeAttachment(db, dataDir, u, { ...png("x"), mime: "Image/PNG; foo=bar" }, link);
|
||||
expect(a.mime).toBe("image/png");
|
||||
expect((await listAttachments(db, u, "work", "w1"))[0]?.mime).toBe("image/png");
|
||||
});
|
||||
|
||||
test("shared bytes across users survive the other's unlink", async () => {
|
||||
const db = getDb();
|
||||
const ua = await addUser(db, "a");
|
||||
const ub = await addUser(db, "b");
|
||||
const a = await storeAttachment(db, dataDir, ua, png("same"), { entity: "work", entityId: "w1" });
|
||||
const b = await storeAttachment(db, dataDir, ub, png("same"), { entity: "work", entityId: "w1" });
|
||||
await unlinkAttachment(db, dataDir, ua, a.id, "work", "w1");
|
||||
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
|
||||
const r = await readAttachment(db, dataDir, ub, b.id);
|
||||
expect(await r?.file.text()).toBe("same");
|
||||
});
|
||||
|
||||
test("another user's unlink is a no-op", async () => {
|
||||
const db = getDb();
|
||||
const ua = await addUser(db, "a");
|
||||
const ub = await addUser(db, "b");
|
||||
const a = await storeAttachment(db, dataDir, ua, png("mine"), { entity: "work", entityId: "w1" });
|
||||
await unlinkAttachment(db, dataDir, ub, a.id, "work", "w1");
|
||||
expect(await listAttachments(db, ua, "work", "w1")).toHaveLength(1);
|
||||
expect(await filesUnder(join(dataDir, "attachments"))).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("listAttachments is scoped to the user", async () => {
|
||||
const db = getDb();
|
||||
const ua = await addUser(db, "a");
|
||||
const ub = await addUser(db, "b");
|
||||
await storeAttachment(db, dataDir, ua, png("one"), { entity: "work", entityId: "w1" });
|
||||
expect(await listAttachments(db, ub, "work", "w1")).toHaveLength(0);
|
||||
expect(await listAttachments(db, ua, "work", "w1")).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("concurrent store and unlink never leave a row without its file", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
const first = await storeAttachment(db, dataDir, u, png("race"), { entity: "work", entityId: "w1" });
|
||||
const [, second] = await Promise.all([
|
||||
unlinkAttachment(db, dataDir, u, first.id, "work", "w1"),
|
||||
storeAttachment(db, dataDir, u, png("race"), { entity: "work", entityId: "w2" }),
|
||||
]);
|
||||
const r = await readAttachment(db, dataDir, u, second.id);
|
||||
expect(await r?.file.text()).toBe("race");
|
||||
});
|
||||
|
||||
test("leaves no temp files behind", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db, "u1");
|
||||
await storeAttachment(db, dataDir, u, png("clean"), { entity: "work", entityId: "w1" });
|
||||
const files = await filesUnder(join(dataDir, "attachments"));
|
||||
expect(files.some((f) => f.includes(".tmp-"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import {
|
||||
createAppToken,
|
||||
createInvite,
|
||||
getSessionUser,
|
||||
login,
|
||||
logout,
|
||||
registerUser,
|
||||
revokeAppToken,
|
||||
verifyAppToken,
|
||||
} from "../src/auth.ts";
|
||||
import { AuthError } from "../src/errors.ts";
|
||||
|
||||
async function code(p: Promise<unknown>): Promise<string | undefined> {
|
||||
try {
|
||||
await p;
|
||||
} catch (e) {
|
||||
return e instanceof AuthError ? e.code : "other";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
forEachDialect("auth", (getDb) => {
|
||||
test("first user is admin and username is lowercased", async () => {
|
||||
const u = await registerUser(getDb(), { username: "Alice", password: "pw-one" });
|
||||
expect(u.role).toBe("admin");
|
||||
expect(u.username).toBe("alice");
|
||||
expect(u.displayName).toBe("alice");
|
||||
});
|
||||
|
||||
test("invalid and duplicate usernames are rejected", async () => {
|
||||
const db = getDb();
|
||||
expect(await code(registerUser(db, { username: "a b", password: "x" }))).toBe("invalid_username");
|
||||
await registerUser(db, { username: "alice", password: "x" });
|
||||
const inv = await createInvite(db, (await login(db, "alice", "x")).user.id);
|
||||
expect(await code(registerUser(db, { username: "ALICE", password: "x", inviteCode: inv }))).toBe(
|
||||
"username_taken",
|
||||
);
|
||||
});
|
||||
|
||||
test("second user needs a single-use invite", async () => {
|
||||
const db = getDb();
|
||||
const admin = await registerUser(db, { username: "alice", password: "x" });
|
||||
expect(await code(registerUser(db, { username: "bob", password: "x" }))).toBe("invite_required");
|
||||
expect(await code(registerUser(db, { username: "bob", password: "x", inviteCode: "nope" }))).toBe(
|
||||
"invite_required",
|
||||
);
|
||||
const inv = await createInvite(db, admin.id);
|
||||
const bob = await registerUser(db, { username: "bob", password: "x", inviteCode: inv });
|
||||
expect(bob.role).toBe("member");
|
||||
expect(await code(registerUser(db, { username: "carol", password: "x", inviteCode: inv }))).toBe(
|
||||
"invite_required",
|
||||
);
|
||||
});
|
||||
|
||||
test("expired invite is rejected; members cannot create invites", async () => {
|
||||
const db = getDb();
|
||||
const admin = await registerUser(db, { username: "alice", password: "x" });
|
||||
const inv = await createInvite(db, admin.id, -1);
|
||||
expect(await code(registerUser(db, { username: "bob", password: "x", inviteCode: inv }))).toBe(
|
||||
"invite_required",
|
||||
);
|
||||
const ok = await createInvite(db, admin.id);
|
||||
const bob = await registerUser(db, { username: "bob", password: "x", inviteCode: ok });
|
||||
expect(await code(createInvite(db, bob.id))).toBe("forbidden");
|
||||
});
|
||||
|
||||
test("wrong password or unknown user -> invalid_credentials", async () => {
|
||||
const db = getDb();
|
||||
await registerUser(db, { username: "alice", password: "right" });
|
||||
expect(await code(login(db, "alice", "wrong"))).toBe("invalid_credentials");
|
||||
expect(await code(login(db, "nobody", "x"))).toBe("invalid_credentials");
|
||||
});
|
||||
|
||||
test("session round-trip and logout", async () => {
|
||||
const db = getDb();
|
||||
await registerUser(db, { username: "alice", password: "pw" });
|
||||
const { user, sessionToken } = await login(db, "Alice", "pw");
|
||||
expect(await getSessionUser(db, sessionToken)).toEqual(user);
|
||||
const stored = await db.k.selectFrom("sessions").selectAll().execute();
|
||||
expect(stored[0]!.token_hash).not.toBe(sessionToken);
|
||||
await logout(db, sessionToken);
|
||||
expect(await getSessionUser(db, sessionToken)).toBeNull();
|
||||
expect(await getSessionUser(db, "garbage")).toBeNull();
|
||||
});
|
||||
|
||||
test("expired session returns null", async () => {
|
||||
const db = getDb();
|
||||
await registerUser(db, { username: "alice", password: "pw" });
|
||||
const { sessionToken } = await login(db, "alice", "pw");
|
||||
await db.k.updateTable("sessions").set({ expires_at: new Date(Date.now() - 1000).toISOString() }).execute();
|
||||
expect(await getSessionUser(db, sessionToken)).toBeNull();
|
||||
});
|
||||
|
||||
test("app tokens: scope check, last_used_at, revoke", async () => {
|
||||
const db = getDb();
|
||||
const u = await registerUser(db, { username: "alice", password: "pw" });
|
||||
const { id, token } = await createAppToken(db, u.id, "cli", ["read"]);
|
||||
expect(token.startsWith("stt_")).toBe(true);
|
||||
expect(await verifyAppToken(db, token, "write")).toBeNull();
|
||||
expect(await verifyAppToken(db, token, "read")).toEqual(u);
|
||||
const row = await db.k.selectFrom("app_tokens").selectAll().executeTakeFirstOrThrow();
|
||||
expect(row.last_used_at).not.toBeNull();
|
||||
expect(await verifyAppToken(db, "stt_bogus", "read")).toBeNull();
|
||||
await revokeAppToken(db, u.id, id);
|
||||
expect(await verifyAppToken(db, token, "read")).toBeNull();
|
||||
});
|
||||
|
||||
test("concurrent first registrations yield exactly one admin", async () => {
|
||||
const db = getDb();
|
||||
const results = await Promise.allSettled([
|
||||
registerUser(db, { username: "alice", password: "x" }),
|
||||
registerUser(db, { username: "bob", password: "x" }),
|
||||
]);
|
||||
const ok = results.filter((r) => r.status === "fulfilled");
|
||||
const bad = results.filter((r) => r.status === "rejected");
|
||||
expect(ok.length).toBe(1);
|
||||
expect((ok[0] as PromiseFulfilledResult<{ role: string }>).value.role).toBe("admin");
|
||||
expect(((bad[0] as PromiseRejectedResult).reason as AuthError).code).toBe("invite_required");
|
||||
const admins = await db.k.selectFrom("users").select("id").where("role", "=", "admin").execute();
|
||||
expect(admins.length).toBe(1);
|
||||
});
|
||||
|
||||
test("no invite means invite_required even for a taken username", async () => {
|
||||
const db = getDb();
|
||||
await registerUser(db, { username: "alice", password: "x" });
|
||||
expect(await code(registerUser(db, { username: "alice", password: "x" }))).toBe("invite_required");
|
||||
});
|
||||
|
||||
test("only hashes are stored for invites and app tokens", async () => {
|
||||
const db = getDb();
|
||||
const u = await registerUser(db, { username: "alice", password: "x" });
|
||||
const inv = await createInvite(db, u.id);
|
||||
const { token } = await createAppToken(db, u.id, "cli", ["read"]);
|
||||
const invRow = await db.k.selectFrom("invites").selectAll().executeTakeFirstOrThrow();
|
||||
const tokRow = await db.k.selectFrom("app_tokens").selectAll().executeTakeFirstOrThrow();
|
||||
expect(invRow.code_hash).not.toBe(inv);
|
||||
expect(tokRow.token_hash).not.toBe(token);
|
||||
expect(tokRow.token_hash).not.toContain(token);
|
||||
});
|
||||
|
||||
test("revoking with another user's id leaves the token valid", async () => {
|
||||
const db = getDb();
|
||||
const a = await registerUser(db, { username: "alice", password: "x" });
|
||||
const b = await registerUser(db, {
|
||||
username: "bob",
|
||||
password: "x",
|
||||
inviteCode: await createInvite(db, a.id),
|
||||
});
|
||||
const { id, token } = await createAppToken(db, a.id, "cli", ["read"]);
|
||||
await revokeAppToken(db, b.id, id);
|
||||
expect(await verifyAppToken(db, token, "read")).toEqual(a);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { decryptJson, encryptJson } from "../src/crypto.ts";
|
||||
import { CryptoError } from "../src/errors.ts";
|
||||
|
||||
describe("crypto", () => {
|
||||
test("round-trips JSON", async () => {
|
||||
const value = { token: "abc", n: [1, 2, { x: null }] };
|
||||
const blob = await encryptJson("s3cret", value);
|
||||
expect(blob).toMatch(/^v1\.[A-Za-z0-9+/=]+\.[A-Za-z0-9+/=]+$/);
|
||||
expect(await decryptJson("s3cret", blob)).toEqual(value);
|
||||
});
|
||||
|
||||
test("uses a fresh IV each time", async () => {
|
||||
expect(await encryptJson("k", 1)).not.toBe(await encryptJson("k", 1));
|
||||
});
|
||||
|
||||
test("wrong secret throws CryptoError", async () => {
|
||||
const blob = await encryptJson("right", { a: 1 });
|
||||
await expect(decryptJson("wrong", blob)).rejects.toBeInstanceOf(CryptoError);
|
||||
});
|
||||
|
||||
test("malformed blob throws CryptoError", async () => {
|
||||
await expect(decryptJson("k", "garbage")).rejects.toBeInstanceOf(CryptoError);
|
||||
await expect(decryptJson("k", "v2.AA==.AA==")).rejects.toBeInstanceOf(CryptoError);
|
||||
});
|
||||
|
||||
test("IV decodes to 12 bytes", async () => {
|
||||
const blob = await encryptJson("k", { a: 1 });
|
||||
expect(Buffer.from(blob.split(".")[1]!, "base64")).toHaveLength(12);
|
||||
});
|
||||
|
||||
test("flipping a ciphertext byte throws CryptoError", async () => {
|
||||
const [v, iv, ct] = (await encryptJson("k", { a: 1 })).split(".");
|
||||
const bytes = Buffer.from(ct!, "base64");
|
||||
bytes[0] = bytes[0]! ^ 0xff;
|
||||
const tampered = `${v}.${iv}.${bytes.toString("base64")}`;
|
||||
await expect(decryptJson("k", tampered)).rejects.toBeInstanceOf(CryptoError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import { fuzzyScore, matchWork } from "../src/matching.ts";
|
||||
import { adminMergeWorks, createWork, getWork } from "../src/works.ts";
|
||||
|
||||
async function addUser(db: Db): Promise<string> {
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
await db.k
|
||||
.insertInto("users")
|
||||
.values({ id, username: `u${id}`, display_name: "U", password_hash: "x", role: "member", preferences: "{}", created_at: now, updated_at: now })
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
test("fuzzyScore", () => {
|
||||
expect(fuzzyScore({ title: "The Expanse: Leviathan Wakes" }, { title: "Leviathan Wakes" })).toBeGreaterThan(0.6);
|
||||
expect(fuzzyScore({ title: "Dune" }, { title: "Dune Messiah" })).toBeLessThan(0.6);
|
||||
expect(fuzzyScore({ title: "Pokémon" }, { title: "Pokemon" })).toBe(1);
|
||||
expect(
|
||||
fuzzyScore({ title: "Pokémon", year: 1996, creators: [{ name: "Satoshi" }] }, { title: "pokemon!", year: 1996, creators: [{ name: "satoshi" }] }),
|
||||
).toBe(1);
|
||||
// year off by one counts half, creator mismatch counts zero
|
||||
expect(fuzzyScore({ title: "A B", year: 2000, creators: [{ name: "x" }] }, { title: "A B", year: 2001, creators: [{ name: "y" }] })).toBeCloseTo(0.8, 5);
|
||||
});
|
||||
|
||||
forEachDialect("matching", (getDb) => {
|
||||
test("match by isbn13 links existing", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "9780441013593" } } }, "t");
|
||||
const r = await matchWork(db, u, { title: "Dune (Ace)", mediaType: "book", year: 1990, externalIds: { isbn13: "9780441013593", asin: "B1" } }, "plug");
|
||||
expect(r).toEqual({ workId: id, created: false, draft: false });
|
||||
expect((await getWork(db, id))?.year).toBe(1990);
|
||||
});
|
||||
|
||||
test("unseen external id creates shared work", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const r = await matchWork(db, u, { title: "Heat", mediaType: "movie", externalIds: { tmdb_movie: "949" } }, "p");
|
||||
expect(r.created).toBe(true);
|
||||
expect(r.draft).toBe(false);
|
||||
expect((await getWork(db, r.workId))?.owner_user_id).toBeNull();
|
||||
});
|
||||
|
||||
const chain = () => ({
|
||||
title: "Pilot",
|
||||
mediaType: "episode" as const,
|
||||
externalIds: {},
|
||||
position: { episode: 1, season: 1 },
|
||||
parent: {
|
||||
title: "Season 1",
|
||||
mediaType: "season" as const,
|
||||
externalIds: {},
|
||||
position: { season: 1 },
|
||||
parent: { title: "Lost", mediaType: "show" as const, externalIds: { tmdb_show: "4607" } },
|
||||
},
|
||||
});
|
||||
|
||||
test("episode hint creates show→season→episode chain once", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const r1 = await matchWork(db, u, chain(), "p");
|
||||
const r2 = await matchWork(db, u, chain(), "p");
|
||||
expect(r1.created).toBe(true);
|
||||
expect(r2).toEqual({ workId: r1.workId, created: false, draft: false });
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
|
||||
const ep = await getWork(db, r1.workId);
|
||||
expect(ep?.position).toEqual({ season: 1, episode: 1 });
|
||||
expect(ep?.parent_id).not.toBeNull();
|
||||
const season = await getWork(db, ep!.parent_id!);
|
||||
expect(season?.media_type).toBe("season");
|
||||
expect((await getWork(db, season!.parent_id!))?.media_type).toBe("show");
|
||||
});
|
||||
|
||||
test("position key order does not matter", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const a = await matchWork(db, u, chain(), "p");
|
||||
const h = chain();
|
||||
h.position = { season: 1, episode: 1 };
|
||||
expect((await matchWork(db, u, h, "p")).workId).toBe(a.workId);
|
||||
});
|
||||
|
||||
test("hint without ids creates private draft with suggestions", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const shared = await createWork(db, { hint: { title: "Leviathan Wakes", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
|
||||
await createWork(db, { hint: { title: "Leviathan Wakes", mediaType: "game", externalIds: { igdb: "1" } } }, "t");
|
||||
await createWork(db, { hint: { title: "Unrelated", mediaType: "book", externalIds: { isbn13: "2" } } }, "t");
|
||||
const r = await matchWork(db, u, { title: "The Expanse: Leviathan Wakes", mediaType: "book", externalIds: {} }, "p");
|
||||
expect(r.created).toBe(true);
|
||||
expect(r.draft).toBe(true);
|
||||
expect((await getWork(db, r.workId))?.owner_user_id).toBe(u);
|
||||
const sug = await db.k.selectFrom("match_suggestions").selectAll().execute();
|
||||
expect(sug).toHaveLength(1);
|
||||
expect(sug[0]).toMatchObject({ user_id: u, work_id: r.workId, candidate_work_id: shared, status: "pending" });
|
||||
expect(sug[0]!.score).toBeGreaterThan(0.6);
|
||||
});
|
||||
|
||||
test("concurrent matches of the same new external id create one work", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const hint = () => ({ title: "Heat", mediaType: "movie" as const, externalIds: { tmdb_movie: "949" } });
|
||||
const rs = await Promise.all([matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p")]);
|
||||
expect(new Set(rs.map((r) => r.workId)).size).toBe(1);
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(1);
|
||||
expect(await db.k.selectFrom("external_ids").select("id").execute()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("concurrent matches of the same new episode chain create it once", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const rs = await Promise.all([matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p")]);
|
||||
expect(new Set(rs.map((r) => r.workId)).size).toBe(1);
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
|
||||
});
|
||||
|
||||
test("concurrent matches of new episodes under an existing show create one season", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
await createWork(db, { hint: { title: "Lost", mediaType: "show", externalIds: { tmdb_show: "4607" } } }, "t");
|
||||
const rs = await Promise.all([matchWork(db, u, chain(), "p"), matchWork(db, u, chain(), "p")]);
|
||||
expect(rs[0]!.workId).toBe(rs[1]!.workId);
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(3);
|
||||
});
|
||||
|
||||
test("concurrent updates adding the same external id do not fail", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
|
||||
const hint = () => ({ title: "Dune", mediaType: "book" as const, externalIds: { isbn13: "1", asin: "B1" } });
|
||||
const rs = await Promise.all([matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p"), matchWork(db, u, hint(), "p")]);
|
||||
expect(rs.map((r) => r.workId)).toEqual([id, id, id]);
|
||||
expect(await db.k.selectFrom("external_ids").select("id").where("namespace", "=", "asin").execute()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("draft reflects the aliased work, not the matched one", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const shared = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } } }, "t");
|
||||
const priv = await createWork(db, { hint: { title: "My Dune", mediaType: "book", externalIds: {} }, ownerUserId: u }, "t");
|
||||
await db.k.insertInto("work_aliases").values({ id: newId(), user_id: u, from_work_id: shared, to_work_id: priv }).execute();
|
||||
expect(await matchWork(db, u, { title: "Dune", mediaType: "book", externalIds: { isbn13: "1" } }, "p")).toEqual({ workId: priv, created: false, draft: true });
|
||||
});
|
||||
|
||||
test("position lookup follows a merged tombstone instead of colliding with it", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
await db.k.updateTable("users").set({ role: "admin" }).where("id", "=", u).execute();
|
||||
const e1 = await matchWork(db, u, chain(), "p");
|
||||
const h2 = chain();
|
||||
h2.title = "Episode 2";
|
||||
h2.position = { season: 1, episode: 2 };
|
||||
const e2 = await matchWork(db, u, h2, "p");
|
||||
await adminMergeWorks(db, u, e1.workId, e2.workId);
|
||||
const before = (await db.k.selectFrom("works").select("id").execute()).length;
|
||||
expect(await matchWork(db, u, chain(), "p")).toEqual({ workId: e2.workId, created: false, draft: false });
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(before);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import type { Db } from "@sttracker/db/db";
|
||||
import { newId, nowIso } from "@sttracker/db/ids";
|
||||
import { fromJson } from "@sttracker/db/json";
|
||||
import { forEachDialect } from "@sttracker/db/testing";
|
||||
import { AuthError, ValidationError } from "../src/errors.ts";
|
||||
import { matchWork } from "../src/matching.ts";
|
||||
import {
|
||||
adminMergeWorks,
|
||||
createWork,
|
||||
findWorkByExternalIds,
|
||||
getWork,
|
||||
resolveAlias,
|
||||
searchWorks,
|
||||
updateWorkFromHint,
|
||||
} from "../src/works.ts";
|
||||
|
||||
async function addUser(db: Db, role: "admin" | "member" = "member"): Promise<string> {
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
await db.k
|
||||
.insertInto("users")
|
||||
.values({ id, username: `u${id}`, display_name: "U", password_hash: "x", role, preferences: "{}", created_at: now, updated_at: now })
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
async function addEntry(db: Db, userId: string, workId: string): Promise<string> {
|
||||
const id = newId();
|
||||
const now = nowIso();
|
||||
await db.k
|
||||
.insertInto("entries")
|
||||
.values({
|
||||
id, user_id: userId, work_id: workId, status: "none", rating: null, review: null, private_notes: null,
|
||||
favorite: 0, tags: "[]", custom_fields: "{}", visibility: "private", overrides: "{}",
|
||||
media_type_override: null, field_sources: "{}", created_at: now, updated_at: now, deleted_at: null,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
forEachDialect("works", (getDb) => {
|
||||
test("createWork stores external ids; findWorkByExternalIds finds any namespace", async () => {
|
||||
const db = getDb();
|
||||
const id = await createWork(db, { hint: { title: "Dune", mediaType: "book", year: 1965, externalIds: { isbn13: "9780441013593", hardcover_book: "7" } } }, "test");
|
||||
expect(await findWorkByExternalIds(db, { nope: "1", hardcover_book: "7" })).toBe(id);
|
||||
expect(await findWorkByExternalIds(db, { isbn13: "0" })).toBeNull();
|
||||
expect((await getWork(db, id))?.year).toBe(1965);
|
||||
});
|
||||
|
||||
test("updateWorkFromHint updates shared work, logs history, never touches private", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const shared = await createWork(db, { hint: { title: "Dune", mediaType: "book", externalIds: { a: "1" } } }, "t");
|
||||
await updateWorkFromHint(db, shared, { title: "Dune", mediaType: "book", year: 1965, description: "Sand", externalIds: { a: "1", b: "2" } }, "plug");
|
||||
const w = await getWork(db, shared);
|
||||
expect(w?.year).toBe(1965);
|
||||
expect(w?.description).toBe("Sand");
|
||||
expect(await findWorkByExternalIds(db, { b: "2" })).toBe(shared);
|
||||
const hist = await db.k.selectFrom("work_history").selectAll().where("work_id", "=", shared).where("actor", "=", "plug").execute();
|
||||
expect(hist).toHaveLength(1);
|
||||
expect(fromJson<Record<string, unknown>>(hist[0]!.before, {}).year).toBeNull();
|
||||
|
||||
const priv = await createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: {} }, ownerUserId: u }, "t");
|
||||
await updateWorkFromHint(db, priv, { title: "Changed", mediaType: "book", externalIds: {} }, "plug");
|
||||
expect((await getWork(db, priv))?.title).toBe("Mine");
|
||||
});
|
||||
|
||||
test("private works invisible to other users in search", async () => {
|
||||
const db = getDb();
|
||||
const u1 = await addUser(db);
|
||||
const u2 = await addUser(db);
|
||||
await createWork(db, { hint: { title: "Secret 100%", mediaType: "book", externalIds: {} }, ownerUserId: u1 }, "t");
|
||||
await createWork(db, { hint: { title: "Public Thing", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
expect((await searchWorks(db, u1, "secret 100%")).map((w) => w.title)).toEqual(["Secret 100%"]);
|
||||
expect(await searchWorks(db, u2, "secret")).toEqual([]);
|
||||
expect((await searchWorks(db, u2, "PUBLIC")).map((w) => w.title)).toEqual(["Public Thing"]);
|
||||
expect(await searchWorks(db, u2, "public", { mediaType: "game" })).toEqual([]);
|
||||
});
|
||||
|
||||
test("adminMergeWorks requires admin", async () => {
|
||||
const db = getDb();
|
||||
const member = await addUser(db);
|
||||
const a = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
const b = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: { x: "2" } } }, "t");
|
||||
await expect(adminMergeWorks(db, member, a, b)).rejects.toBeInstanceOf(AuthError);
|
||||
});
|
||||
|
||||
test("adminMergeWorks tombstones, moves ids, repoints entries, emits merged event on conflict", async () => {
|
||||
const db = getDb();
|
||||
const admin = await addUser(db, "admin");
|
||||
const u1 = await addUser(db);
|
||||
const u2 = await addUser(db);
|
||||
const from = await createWork(db, { hint: { title: "Old", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
const to = await createWork(db, { hint: { title: "New", mediaType: "book", externalIds: { x: "2" } } }, "t");
|
||||
const e1 = await addEntry(db, u1, from);
|
||||
const e2from = await addEntry(db, u2, from);
|
||||
await addEntry(db, u2, to);
|
||||
|
||||
await adminMergeWorks(db, admin, from, to);
|
||||
|
||||
expect((await getWork(db, from))?.id).toBe(to);
|
||||
expect(await findWorkByExternalIds(db, { x: "1" })).toBe(to);
|
||||
expect(await resolveAlias(db, u1, from)).toBe(to);
|
||||
expect((await db.k.selectFrom("entries").select("work_id").where("id", "=", e1).executeTakeFirstOrThrow()).work_id).toBe(to);
|
||||
expect((await db.k.selectFrom("entries").select("work_id").where("id", "=", e2from).executeTakeFirstOrThrow()).work_id).toBe(from);
|
||||
const ev = await db.k.selectFrom("change_events").selectAll().execute();
|
||||
expect(ev).toHaveLength(1);
|
||||
expect(ev[0]).toMatchObject({ user_id: u2, type: "merged", status: "pending", target_entity: "entry", target_id: e2from });
|
||||
expect(fromJson(ev[0]!.payload, {})).toEqual({ fromWorkId: from, toWorkId: to });
|
||||
expect(await db.k.selectFrom("work_history").select("id").where("work_id", "=", from).where("actor", "=", admin).execute()).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("createWork refuses external ids on a private work", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
const p = createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: { isbn13: "1" } }, ownerUserId: u }, "t");
|
||||
await expect(p).rejects.toBeInstanceOf(ValidationError);
|
||||
await expect(p).rejects.toMatchObject({ field: "externalIds", reason: "private works cannot carry external ids" });
|
||||
expect(await db.k.selectFrom("works").select("id").execute()).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("findWorkByExternalIds only returns works visible to the user", async () => {
|
||||
const db = getDb();
|
||||
const a = await addUser(db);
|
||||
const b = await addUser(db);
|
||||
const priv = await createWork(db, { hint: { title: "Mine", mediaType: "book", externalIds: {} }, ownerUserId: a }, "t");
|
||||
// Legacy data: a private work carrying an external id (no longer creatable through createWork).
|
||||
await db.k.insertInto("external_ids").values({ id: newId(), work_id: priv, namespace: "isbn13", value: "42" }).execute();
|
||||
expect(await findWorkByExternalIds(db, { isbn13: "42" }, a)).toBe(priv);
|
||||
expect(await findWorkByExternalIds(db, { isbn13: "42" }, b)).toBeNull();
|
||||
expect(await findWorkByExternalIds(db, { isbn13: "42" })).toBeNull();
|
||||
const shared = await createWork(db, { hint: { title: "Theirs", mediaType: "book", externalIds: { asin: "X" } } }, "t");
|
||||
const hint = { title: "Mine", mediaType: "book" as const, externalIds: { isbn13: "42", asin: "X" } };
|
||||
expect(await matchWork(db, b, hint, "p")).toEqual({ workId: shared, created: false, draft: false });
|
||||
expect(await matchWork(db, a, hint, "p")).toEqual({ workId: priv, created: false, draft: true });
|
||||
});
|
||||
|
||||
test("getWork returns null on a merged_into cycle", async () => {
|
||||
const db = getDb();
|
||||
const a = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
const b = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: { x: "2" } } }, "t");
|
||||
await db.k.updateTable("works").set({ merged_into_id: b }).where("id", "=", a).execute();
|
||||
await db.k.updateTable("works").set({ merged_into_id: a }).where("id", "=", b).execute();
|
||||
expect(await getWork(db, a)).toBeNull();
|
||||
});
|
||||
|
||||
test("search treats a literal _ as a character, not a wildcard", async () => {
|
||||
const db = getDb();
|
||||
const u = await addUser(db);
|
||||
await createWork(db, { hint: { title: "snake_case", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
await createWork(db, { hint: { title: "snakeXcase", mediaType: "book", externalIds: { x: "2" } } }, "t");
|
||||
expect((await searchWorks(db, u, "e_c")).map((w) => w.title)).toEqual(["snake_case"]);
|
||||
});
|
||||
|
||||
test("adminMergeWorks refuses private works", async () => {
|
||||
const db = getDb();
|
||||
const admin = await addUser(db, "admin");
|
||||
const shared = await createWork(db, { hint: { title: "A", mediaType: "book", externalIds: { x: "1" } } }, "t");
|
||||
const priv = await createWork(db, { hint: { title: "B", mediaType: "book", externalIds: {} }, ownerUserId: admin }, "t");
|
||||
await expect(adminMergeWorks(db, admin, priv, shared)).rejects.toBeInstanceOf(ValidationError);
|
||||
await expect(adminMergeWorks(db, admin, shared, priv)).rejects.toBeInstanceOf(ValidationError);
|
||||
expect((await getWork(db, priv))?.id).toBe(priv);
|
||||
});
|
||||
|
||||
test("adminMergeWorks moves children that do not collide, so matching does not recreate them", async () => {
|
||||
const db = getDb();
|
||||
const admin = await addUser(db, "admin");
|
||||
const u = await addUser(db);
|
||||
const show = (id: string) => ({ title: "Lost", mediaType: "show" as const, externalIds: { tmdb_show: id } });
|
||||
const ep = (showId: string, season: number) => ({
|
||||
title: "Pilot",
|
||||
mediaType: "episode" as const,
|
||||
externalIds: {},
|
||||
position: { season, episode: 1 },
|
||||
parent: { title: `Season ${season}`, mediaType: "season" as const, externalIds: {}, position: { season }, parent: show(showId) },
|
||||
});
|
||||
const a1 = await matchWork(db, u, ep("A", 1), "p");
|
||||
const a2 = await matchWork(db, u, ep("A", 2), "p");
|
||||
const b2 = await matchWork(db, u, ep("B", 2), "p");
|
||||
const showA = (await findWorkByExternalIds(db, { tmdb_show: "A" }))!;
|
||||
const showB = (await findWorkByExternalIds(db, { tmdb_show: "B" }))!;
|
||||
const seasonA1 = (await getWork(db, a1.workId))!.parent_id!;
|
||||
const seasonA2 = (await getWork(db, a2.workId))!.parent_id!;
|
||||
const before = (await db.k.selectFrom("works").select("id").execute()).length;
|
||||
|
||||
await adminMergeWorks(db, admin, showA, showB);
|
||||
|
||||
expect((await getWork(db, seasonA1))?.parent_id).toBe(showB); // moved: B had no season 1
|
||||
expect((await getWork(db, seasonA2))?.parent_id).toBe(showA); // left: B already had season 2
|
||||
const again = await matchWork(db, u, ep("A", 1), "p");
|
||||
expect(again).toEqual({ workId: a1.workId, created: false, draft: false });
|
||||
expect((await matchWork(db, u, ep("A", 2), "p")).workId).toBe(b2.workId);
|
||||
expect((await db.k.selectFrom("works").select("id").execute()).length).toBe(before);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
import type { Kysely } from "kysely";
|
||||
import type { Migration } from "kysely/migration";
|
||||
|
||||
/**
|
||||
* One child per (parent, position), so concurrent matches cannot duplicate seasons or episodes.
|
||||
* NULLs are distinct on both dialects: top-level works and children without a position are unaffected.
|
||||
*/
|
||||
export const worksPositionUnique: Migration = {
|
||||
async up(db: Kysely<any>) {
|
||||
await db.schema.createIndex("works_parent_id_position_uq").unique().on("works").columns(["parent_id", "position"]).execute();
|
||||
},
|
||||
|
||||
async down(db: Kysely<any>) {
|
||||
await db.schema.dropIndex("works_parent_id_position_uq").execute();
|
||||
},
|
||||
};
|
||||
@@ -1,7 +1,9 @@
|
||||
import type { Migration } from "kysely/migration";
|
||||
import { initial } from "./0001_initial.ts";
|
||||
import { worksPositionUnique } from "./0002_works_position_unique.ts";
|
||||
|
||||
/** All migrations, keyed by sortable name. Imported statically so bundles need no filesystem. */
|
||||
export const migrations: Record<string, Migration> = {
|
||||
"0001_initial": initial,
|
||||
"0002_works_position_unique": worksPositionUnique,
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { sql, type Kysely } from "kysely";
|
||||
import { Migrator } from "kysely/migration";
|
||||
import { Migrator, NO_MIGRATIONS } from "kysely/migration";
|
||||
import { newId, nowIso } from "../src/ids.ts";
|
||||
import { toJson } from "../src/json.ts";
|
||||
import { migrations } from "../src/migrations/index.ts";
|
||||
@@ -79,10 +79,29 @@ forEachDialect("schema", (getDb) => {
|
||||
await expect(k.insertInto("entries").values({ id: newId(), ...entry }).execute()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("works (parent_id, position) is unique; null positions and null parents are not", async () => {
|
||||
const k = getDb().k;
|
||||
const now = nowIso();
|
||||
const work = (id: string, parent: string | null, position: string | null) => ({
|
||||
id, media_type: "show", title: "T", original_title: "", year: null, release_date: null, creators: "[]",
|
||||
description: "", cover_url: "", cover_attachment_id: null, metadata: "{}", parent_id: parent, position,
|
||||
owner_user_id: null, merged_into_id: null, created_at: now, updated_at: now,
|
||||
});
|
||||
const show = newId();
|
||||
await k.insertInto("works").values(work(show, null, null)).execute();
|
||||
await k.insertInto("works").values(work(newId(), null, null)).execute();
|
||||
await k.insertInto("works").values(work(newId(), null, '{"season":1}')).execute();
|
||||
await k.insertInto("works").values(work(newId(), null, '{"season":1}')).execute();
|
||||
await k.insertInto("works").values(work(newId(), show, null)).execute();
|
||||
await k.insertInto("works").values(work(newId(), show, null)).execute();
|
||||
await k.insertInto("works").values(work(newId(), show, '{"season":1}')).execute();
|
||||
await expect(k.insertInto("works").values(work(newId(), show, '{"season":1}')).execute()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("migrating down removes the tables", async () => {
|
||||
const k = getDb().k as Kysely<any>;
|
||||
const migrator = new Migrator({ db: k, provider: { getMigrations: async () => migrations } });
|
||||
const { error } = await migrator.migrateDown();
|
||||
const { error } = await migrator.migrateTo(NO_MIGRATIONS);
|
||||
expect(error).toBeUndefined();
|
||||
const probe = sql`select 1 from users`.execute(k);
|
||||
await expect(probe).rejects.toThrow();
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
export type CsvDelimiter = "," | ";" | "\t";
|
||||
|
||||
function detectDelimiter(text: string): CsvDelimiter {
|
||||
const counts: Record<CsvDelimiter, number> = { ",": 0, ";": 0, "\t": 0 };
|
||||
let inQuotes = false;
|
||||
for (const ch of text) {
|
||||
if (ch === '"') inQuotes = !inQuotes;
|
||||
else if (!inQuotes) {
|
||||
if (ch === "\n" || ch === "\r") break;
|
||||
if (ch === "," || ch === ";" || ch === "\t") counts[ch]++;
|
||||
}
|
||||
}
|
||||
let best: CsvDelimiter = ",";
|
||||
for (const d of [";", "\t"] as const) if (counts[d] > counts[best]) best = d;
|
||||
return best;
|
||||
}
|
||||
|
||||
/** RFC 4180 parser. Strips a BOM, auto-detects the delimiter from the header line, skips blank lines. */
|
||||
export function parseCsv(text: string, opts: { delimiter?: CsvDelimiter } = {}): string[][] {
|
||||
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
|
||||
const delim = opts.delimiter ?? detectDelimiter(text);
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = "";
|
||||
let quoted = false; // current field was quoted
|
||||
let inQuotes = false;
|
||||
let pending = false; // something has been read for the current row
|
||||
|
||||
const endField = () => {
|
||||
row.push(field);
|
||||
field = "";
|
||||
};
|
||||
const endRow = () => {
|
||||
const blank = row.length === 0 && field === "" && !quoted;
|
||||
if (!blank) {
|
||||
endField();
|
||||
rows.push(row);
|
||||
}
|
||||
row = [];
|
||||
field = "";
|
||||
quoted = false;
|
||||
pending = false;
|
||||
};
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i]!;
|
||||
if (inQuotes) {
|
||||
if (ch === '"') {
|
||||
if (text[i + 1] === '"') {
|
||||
field += '"';
|
||||
i++;
|
||||
} else inQuotes = false;
|
||||
} else field += ch;
|
||||
} else if (ch === '"' && field === "" && !quoted) {
|
||||
inQuotes = true;
|
||||
quoted = true;
|
||||
pending = true;
|
||||
} else if (ch === delim) {
|
||||
endField();
|
||||
quoted = false;
|
||||
pending = true;
|
||||
} else if (ch === "\r" || ch === "\n") {
|
||||
if (ch === "\r" && text[i + 1] === "\n") i++;
|
||||
endRow();
|
||||
} else {
|
||||
field += ch;
|
||||
pending = true;
|
||||
}
|
||||
}
|
||||
if (pending || field !== "" || row.length > 0) endRow();
|
||||
return rows;
|
||||
}
|
||||
|
||||
function quoteCell(s: string, single: boolean): string {
|
||||
if (s === "" ? single : /[",;\t\r\n]/.test(s) || s !== s.trim()) return `"${s.replaceAll('"', '""')}"`;
|
||||
return s;
|
||||
}
|
||||
|
||||
/** Serialises with `,` and `\r\n`, quoting only where needed. */
|
||||
export function toCsv(rows: string[][]): string {
|
||||
return rows.map((r) => r.map((c) => quoteCell(c, r.length === 1)).join(",") + "\r\n").join("");
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { parseCsv, toCsv } from "../src/csv.ts";
|
||||
|
||||
describe("csv codec", () => {
|
||||
test("round-trips awkward cells", () => {
|
||||
const x = [["a", "b,c", 'q"uote', "line\nbreak", "cr\r\nlf", "", "🐉"], ["", "", "", "", "", "", ""], ["only"]];
|
||||
expect(parseCsv(toCsv(x))).toEqual(x);
|
||||
});
|
||||
test("toCsv uses CRLF and quotes as needed", () => {
|
||||
expect(toCsv([["a", "b c"], ["x,y", 'z"']])).toBe('a,b c\r\n"x,y","z"""\r\n');
|
||||
});
|
||||
test("strips BOM and detects semicolon / tab", () => {
|
||||
expect(parseCsv("a;b\n1;2\n")).toEqual([["a", "b"], ["1", "2"]]);
|
||||
expect(parseCsv("a\tb\n1\t2")).toEqual([["a", "b"], ["1", "2"]]);
|
||||
});
|
||||
test("explicit delimiter wins; delimiters inside quoted header are ignored", () => {
|
||||
expect(parseCsv('"a;b;c",d\n1,2', {})).toEqual([["a;b;c", "d"], ["1", "2"]]);
|
||||
expect(parseCsv("a,b;c\n1,2;3", { delimiter: ";" })).toEqual([["a,b", "c"], ["1,2", "3"]]);
|
||||
});
|
||||
test("handles CRLF, blank trailing lines, and empty input", () => {
|
||||
expect(parseCsv("a,b\r\n1,2\r\n\r\n")).toEqual([["a", "b"], ["1", "2"]]);
|
||||
expect(parseCsv("")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { parseCsv } from "@sttracker/plugin-sdk/csv";
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import type { EngagementFields, MediaType, EntryFields, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import {
|
||||
cleanId, defaultScale, guessMapping, normalizeRating, parseDate, parseMapping, parseMediaType, parseStatus, parseTags, rowsToFieldRows,
|
||||
type FieldRow,
|
||||
} from "./src/mapping.ts";
|
||||
|
||||
const str = (v: string | string[] | undefined): string | undefined => (typeof v === "string" && v.trim() !== "" ? v.trim() : undefined);
|
||||
|
||||
function jsonRows(text: string): FieldRow[] {
|
||||
let v: unknown;
|
||||
try {
|
||||
v = JSON.parse(text);
|
||||
} catch {
|
||||
throw new Error("csv plugin: file is not valid JSON");
|
||||
}
|
||||
if (!Array.isArray(v)) throw new Error("csv plugin: JSON must be an array of objects");
|
||||
return v.map((o) => {
|
||||
const r: Record<string, string | string[]> = {};
|
||||
if (typeof o === "object" && o !== null) {
|
||||
for (const [k, x] of Object.entries(o)) {
|
||||
if (x == null || x === "") continue;
|
||||
r[k] = Array.isArray(x) ? x.map(String) : String(x);
|
||||
}
|
||||
}
|
||||
return r as FieldRow;
|
||||
});
|
||||
}
|
||||
|
||||
function* toRecords(rows: FieldRow[], rawRows: unknown[], defaultMedia: MediaType): Generator<SourceRecord> {
|
||||
const scale = defaultScale(rows);
|
||||
const seen = new Map<string, number>();
|
||||
for (const [i, r] of rows.entries()) {
|
||||
const title = str(r.title);
|
||||
if (!title) continue;
|
||||
const mediaType = parseMediaType(str(r.mediaType), defaultMedia);
|
||||
const finished = parseDate(str(r.finishedAt));
|
||||
const started = parseDate(str(r.startedAt));
|
||||
const hash = createHash("sha1").update([title, mediaType, finished?.value ?? ""].join("\u0000")).digest("hex");
|
||||
const n = (seen.get(hash) ?? 0) + 1;
|
||||
seen.set(hash, n);
|
||||
const key = n === 1 ? `row:${hash}` : `row:${hash}#${n}`;
|
||||
|
||||
const externalIds: Record<string, string> = {};
|
||||
for (const f of ["isbn13", "imdb", "tmdb_movie", "igdb"] as const) {
|
||||
const id = cleanId(r[f]);
|
||||
if (id) externalIds[f] = id;
|
||||
}
|
||||
const author = str(r.author);
|
||||
const work: WorkHint = { title, mediaType, externalIds, ...(author ? { creators: [{ name: author }] } : {}) };
|
||||
|
||||
const entry: EntryFields = {};
|
||||
const status = parseStatus(str(r.status));
|
||||
if (status) entry.status = status;
|
||||
const rating = normalizeRating(r, scale);
|
||||
if (rating !== null) entry.rating = rating;
|
||||
const review = str(r.review);
|
||||
if (review) entry.review = review;
|
||||
const notes = str(r.notes);
|
||||
if (notes) entry.notes = notes;
|
||||
const tags = parseTags(r.tags);
|
||||
if (tags) entry.tags = tags;
|
||||
const raw = rawRows[i];
|
||||
yield { kind: "entry", key, raw, work, entry };
|
||||
|
||||
if (finished || started) {
|
||||
const engagement: EngagementFields = {};
|
||||
if (started) engagement.startedAt = started;
|
||||
if (finished) engagement.finishedAt = finished;
|
||||
yield { kind: "engagement", key: `${key}:engagement`, raw, work, engagement };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "csv",
|
||||
name: "CSV / JSON import",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["book", "comic", "manga", "game", "movie", "show", "podcast", "audio_drama", "album", "video", "other"],
|
||||
capabilities: ["importFile"],
|
||||
config: [
|
||||
{
|
||||
key: "mapping",
|
||||
label: "Column mapping",
|
||||
type: "string",
|
||||
help: 'JSON { "column": "field" }. Fields: title mediaType status rating ratingScale(5|10|100) review finishedAt startedAt tags isbn13 imdb tmdb_movie igdb notes. Leave empty to guess from header names (Goodreads exports work out of the box). Ignored for .json.',
|
||||
},
|
||||
],
|
||||
},
|
||||
async *importFile(ctx, file) {
|
||||
const text = new TextDecoder("utf-8").decode(file.bytes);
|
||||
const lower = file.name.toLowerCase();
|
||||
if (lower.endsWith(".json")) {
|
||||
const clean = text.replace(/^\uFEFF/, "");
|
||||
const rows = jsonRows(clean);
|
||||
const raw = JSON.parse(clean) as unknown[];
|
||||
const bookish = raw.some((o) => typeof o === "object" && o !== null && ("author" in o || "isbn13" in o));
|
||||
yield* toRecords(rows, raw, bookish ? "book" : "other");
|
||||
} else if (lower.endsWith(".csv")) {
|
||||
const table = parseCsv(text);
|
||||
const header = table[0] ?? [];
|
||||
const mapping = parseMapping(ctx.config.mapping) ?? guessMapping(header);
|
||||
const rows = rowsToFieldRows(table, mapping);
|
||||
const bookish = header.some((h) => mapping[h] === "author" || mapping[h] === "isbn13");
|
||||
yield* toRecords(rows, table.slice(1).map((cells) => Object.fromEntries(header.map((h, i) => [h, cells[i] ?? ""]))), bookish ? "book" : "other");
|
||||
} else {
|
||||
throw new Error(`csv plugin: unsupported file "${file.name}" (expected .csv or .json)`);
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
|
||||
import type { MediaType, PartialDate, Status } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
export const FIELDS = [
|
||||
"title", "mediaType", "status", "rating", "ratingScale", "review", "finishedAt", "startedAt",
|
||||
"tags", "isbn13", "imdb", "tmdb_movie", "igdb", "notes", "author",
|
||||
] as const;
|
||||
export type Field = (typeof FIELDS)[number];
|
||||
export type Mapping = Record<string, Field>;
|
||||
/** A row keyed by target field name. */
|
||||
export type FieldRow = Partial<Record<Field, string | string[]>>;
|
||||
|
||||
const ALIASES: Record<Field, string[]> = {
|
||||
title: ["title", "name"],
|
||||
mediaType: ["mediatype", "media type", "type", "media"],
|
||||
status: ["status", "exclusive shelf", "shelf", "state"],
|
||||
rating: ["rating", "my rating", "score", "stars"],
|
||||
ratingScale: ["ratingscale", "rating scale"],
|
||||
review: ["review", "my review"],
|
||||
finishedAt: ["finishedat", "finished", "date read", "date finished", "finished at", "date watched", "watched"],
|
||||
startedAt: ["startedat", "started", "date started", "started at"],
|
||||
tags: ["tags", "bookshelves", "labels", "genres"],
|
||||
isbn13: ["isbn13", "isbn 13"],
|
||||
imdb: ["imdb", "imdb id", "imdb_id"],
|
||||
tmdb_movie: ["tmdb_movie", "tmdb", "tmdb id"],
|
||||
igdb: ["igdb", "igdb id"],
|
||||
notes: ["notes", "private notes", "comments"],
|
||||
author: ["author", "authors", "creator"],
|
||||
};
|
||||
|
||||
export function parseMapping(json: unknown): Mapping | null {
|
||||
if (typeof json !== "string" || json.trim() === "") return null;
|
||||
let v: unknown;
|
||||
try {
|
||||
v = JSON.parse(json);
|
||||
} catch {
|
||||
throw new Error("csv plugin: mapping is not valid JSON");
|
||||
}
|
||||
if (typeof v !== "object" || v === null || Array.isArray(v)) throw new Error("csv plugin: mapping must be an object of { column: field }");
|
||||
const out: Mapping = {};
|
||||
for (const [col, f] of Object.entries(v)) {
|
||||
if (!(FIELDS as readonly unknown[]).includes(f)) throw new Error(`csv plugin: unknown mapping field "${String(f)}" for column "${col}"`);
|
||||
out[col] = f as Field;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Guess a mapping from header names, case-insensitively (covers Goodreads exports). */
|
||||
export function guessMapping(headers: string[]): Mapping {
|
||||
const out: Mapping = {};
|
||||
const used = new Set<Field>();
|
||||
for (const h of headers) {
|
||||
const n = h.trim().toLowerCase();
|
||||
const f = FIELDS.find((x) => !used.has(x) && ALIASES[x].includes(n));
|
||||
if (f) {
|
||||
out[h] = f;
|
||||
used.add(f);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function rowsToFieldRows(rows: string[][], mapping: Mapping | null): FieldRow[] {
|
||||
const [header, ...body] = rows;
|
||||
if (!header) return [];
|
||||
const m = mapping ?? guessMapping(header);
|
||||
const idx = header.map((h) => m[h] ?? null);
|
||||
return body.map((cells) => {
|
||||
const r: FieldRow = {};
|
||||
idx.forEach((f, i) => {
|
||||
const v = cells[i];
|
||||
if (f && v !== undefined && v.trim() !== "") r[f] = v.trim();
|
||||
});
|
||||
return r;
|
||||
});
|
||||
}
|
||||
|
||||
const MEDIA: MediaType[] = [
|
||||
"book", "comic", "manga", "game", "movie", "show", "season", "episode", "podcast", "podcast_episode",
|
||||
"audio_drama", "artist", "album", "track", "video", "other",
|
||||
];
|
||||
export function parseMediaType(v: string | undefined, fallback: MediaType): MediaType {
|
||||
if (!v) return fallback;
|
||||
const n = v.trim().toLowerCase().replaceAll(/[\s-]+/g, "_");
|
||||
return (MEDIA as string[]).includes(n) ? (n as MediaType) : fallback;
|
||||
}
|
||||
|
||||
const STATUS: Record<string, Status> = {
|
||||
planned: "planned", "to-read": "planned", "to read": "planned", "want to read": "planned", "want to watch": "planned", "plan to watch": "planned", wishlist: "planned", backlog: "planned",
|
||||
in_progress: "in_progress", "in progress": "in_progress", "currently-reading": "in_progress", "currently reading": "in_progress", reading: "in_progress", watching: "in_progress", playing: "in_progress",
|
||||
paused: "paused", "on hold": "paused", "on-hold": "paused",
|
||||
completed: "completed", read: "completed", finished: "completed", watched: "completed", played: "completed", done: "completed", complete: "completed",
|
||||
dropped: "dropped", dnf: "dropped", abandoned: "dropped",
|
||||
none: "none",
|
||||
};
|
||||
export function parseStatus(v: string | undefined): Status | undefined {
|
||||
if (!v) return undefined;
|
||||
return STATUS[v.trim().toLowerCase()] ?? "none";
|
||||
}
|
||||
|
||||
/** Goodreads writes `YYYY/MM/DD`; normalise to dashes. */
|
||||
export function parseDate(v: string | undefined): PartialDate | null {
|
||||
if (!v) return null;
|
||||
const s = v.trim();
|
||||
return parsePartialDate(/^\d{4}\/\d{2}\/\d{2}$/.test(s) ? s.replaceAll("/", "-") : s);
|
||||
}
|
||||
|
||||
/** Per-file default scale: 5 if max<=5, 10 if max<=10, else 100. */
|
||||
export function defaultScale(rows: FieldRow[]): 5 | 10 | 100 {
|
||||
let max = 0;
|
||||
for (const r of rows) {
|
||||
const n = Number(r.rating);
|
||||
if (r.rating !== undefined && Number.isFinite(n) && n > max) max = n;
|
||||
}
|
||||
return max <= 5 ? 5 : max <= 10 ? 10 : 100;
|
||||
}
|
||||
|
||||
/** Returns a 0-100 rating, or null when unrated. 0 counts as unrated only on scale 5 (Goodreads convention). */
|
||||
export function normalizeRating(r: FieldRow, fileScale: 5 | 10 | 100): number | null {
|
||||
if (r.rating === undefined) return null;
|
||||
const n = Number(r.rating);
|
||||
if (!Number.isFinite(n) || n < 0) return null;
|
||||
const rs = Number(r.ratingScale);
|
||||
const scale = rs === 5 || rs === 10 || rs === 100 ? rs : fileScale;
|
||||
if (n === 0 && scale === 5) return null;
|
||||
return Math.max(0, Math.min(100, Math.round((n / scale) * 100)));
|
||||
}
|
||||
|
||||
/** Goodreads wraps ids as `="978..."`. */
|
||||
export function cleanId(v: string | string[] | undefined): string | undefined {
|
||||
if (typeof v !== "string") return undefined;
|
||||
const s = v.trim().replace(/^=?"?/, "").replace(/"?$/, "").trim();
|
||||
return s === "" ? undefined : s;
|
||||
}
|
||||
|
||||
export function parseTags(v: string | string[] | undefined): string[] | undefined {
|
||||
const list = Array.isArray(v) ? v : typeof v === "string" ? v.split(/[,;]/) : [];
|
||||
const t = list.map((x) => String(x).trim()).filter(Boolean);
|
||||
return t.length ? t : undefined;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { collect, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import type { SourceRecord } from "@sttracker/plugin-sdk/types";
|
||||
import plugin from "../index.ts";
|
||||
|
||||
const fixture = async (n: string) => new Uint8Array(await Bun.file(new URL(`./fixtures/${n}`, import.meta.url)).arrayBuffer());
|
||||
const run = async (name: string, bytes: Uint8Array, config: Record<string, unknown> = {}) =>
|
||||
collect(plugin.importFile!(createTestContext({ config }), { name, bytes }));
|
||||
const entries = (r: SourceRecord[]) => r.filter((x): x is Extract<SourceRecord, { kind: "entry" }> => x.kind === "entry");
|
||||
|
||||
describe("csv plugin", () => {
|
||||
test("manifest", () => {
|
||||
expect(plugin.manifest.id).toBe("csv");
|
||||
expect(plugin.manifest.capabilities).toEqual(["importFile"]);
|
||||
expect(plugin.manifest.config.map((c) => c.key)).toEqual(["mapping"]);
|
||||
});
|
||||
|
||||
test("messy.csv: BOM, ';', quoted newline and quotes, empty rating, year date, emoji", async () => {
|
||||
const recs = await run("messy.csv", await fixture("messy.csv"));
|
||||
const e = entries(recs);
|
||||
expect(e.map((r) => r.work.title)).toEqual(['Line "one"', "🐉 Dragons"]);
|
||||
expect(e[0]!.entry.review).toBe("multi\nline");
|
||||
expect(e[0]!.entry.rating).toBe(80);
|
||||
expect(e[1]!.entry.rating ?? null).toBeNull();
|
||||
const eng = recs.filter((r) => r.kind === "engagement");
|
||||
expect(eng).toHaveLength(2);
|
||||
expect((eng[0] as Extract<SourceRecord, { kind: "engagement" }>).engagement.finishedAt).toEqual({ value: "2019", precision: "year" });
|
||||
expect(e[0]!.key).toMatch(/^row:[0-9a-f]{40}$/);
|
||||
});
|
||||
|
||||
test("goodreads headers auto-mapped", async () => {
|
||||
const e = entries(await run("goodreads.csv", await fixture("goodreads.csv")));
|
||||
expect(e.map((r) => r.entry.status)).toEqual(["completed", "planned", "in_progress"]);
|
||||
expect(e[0]!.work.title).toBe("Dune");
|
||||
expect(e[0]!.work.mediaType).toBe("book");
|
||||
expect(e[0]!.work.creators).toEqual([{ name: "Frank Herbert" }]);
|
||||
expect(e[0]!.work.externalIds).toEqual({ isbn13: "9780441172719" });
|
||||
expect(e[0]!.entry.rating).toBe(80);
|
||||
expect(e[0]!.entry.tags).toEqual(["sci-fi", "classics"]);
|
||||
expect(e[1]!.entry.rating ?? null).toBeNull();
|
||||
expect(e[1]!.work.externalIds).toEqual({});
|
||||
const eng = (await run("goodreads.csv", await fixture("goodreads.csv"))).filter((r) => r.kind === "engagement") as Extract<SourceRecord, { kind: "engagement" }>[];
|
||||
expect(eng).toHaveLength(1);
|
||||
expect(eng[0]!.engagement.finishedAt).toEqual({ value: "2021-03-04", precision: "day" });
|
||||
});
|
||||
|
||||
test("explicit mapping and rating scale default", async () => {
|
||||
const csv = "Name,Score,Kind\nA,7,movie\nB,10,game\n";
|
||||
const mapping = JSON.stringify({ Name: "title", Score: "rating", Kind: "mediaType" });
|
||||
const e = entries(await run("x.csv", new TextEncoder().encode(csv), { mapping }));
|
||||
expect(e.map((r) => r.entry.rating)).toEqual([70, 100]);
|
||||
expect(e.map((r) => r.work.mediaType)).toEqual(["movie", "game"]);
|
||||
});
|
||||
|
||||
test("json ignores mapping", async () => {
|
||||
const json = JSON.stringify([{ title: "T", mediaType: "movie", rating: 3, status: "completed", finishedAt: "2020-01-02", tags: ["a"], imdb: "tt1" }]);
|
||||
const recs = await run("x.json", new TextEncoder().encode(json), { mapping: '{"title":"notes"}' });
|
||||
const e = entries(recs);
|
||||
expect(e[0]!.work.title).toBe("T");
|
||||
expect(e[0]!.entry.rating).toBe(60);
|
||||
expect(e[0]!.entry.tags).toEqual(["a"]);
|
||||
expect(e[0]!.work.externalIds).toEqual({ imdb: "tt1" });
|
||||
expect(recs).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("unsupported extension throws", async () => {
|
||||
await expect(run("x.txt", new Uint8Array())).rejects.toThrow();
|
||||
});
|
||||
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
test("duplicate rows get distinct deterministic keys", async () => {
|
||||
const csv = "Title,Date Read\nA,2020-01-01\nA,2020-01-01\nB,\nB,\nB,\n";
|
||||
const keys = async () => (await run("d.csv", enc(csv))).map((r) => r.key);
|
||||
const k = await keys();
|
||||
expect(new Set(k).size).toBe(k.length);
|
||||
expect(k[0]).toMatch(/^row:[0-9a-f]{40}$/);
|
||||
expect(k.some((x) => x.endsWith("#2"))).toBe(true);
|
||||
expect(k.some((x) => x.endsWith("#3"))).toBe(true);
|
||||
expect(await keys()).toEqual(k);
|
||||
});
|
||||
|
||||
test("default mediaType is per file, not per row", async () => {
|
||||
const csv = "Title,Author,ISBN13\nA,Someone,=\"9780441172719\"\nB,,\n";
|
||||
const e = entries(await run("g.csv", enc(csv)));
|
||||
expect(e.map((r) => r.work.mediaType)).toEqual(["book", "book"]);
|
||||
const plain = entries(await run("p.csv", enc("Title\nA\n")));
|
||||
expect(plain[0]!.work.mediaType).toBe("other");
|
||||
const j = entries(await run("j.json", enc('[{"title":"A","author":"x"},{"title":"B"},{"title":"C","mediaType":"game"}]')));
|
||||
expect(j.map((r) => r.work.mediaType)).toEqual(["book", "book", "game"]);
|
||||
});
|
||||
|
||||
test("rating 0 is unrated only on scale 5", async () => {
|
||||
const m = JSON.stringify({ T: "title", R: "rating" });
|
||||
const rate = async (csv: string) => entries(await run("r.csv", enc(csv), { mapping: m })).map((r) => r.entry.rating ?? null);
|
||||
expect(await rate("T,R\nA,0\nB,4\n")).toEqual([null, 80]);
|
||||
expect(await rate("T,R\nA,0\nB,8\n")).toEqual([0, 80]);
|
||||
expect(await rate("T,R\nA,0\nB,80\n")).toEqual([0, 80]);
|
||||
});
|
||||
});
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
Book Id,Title,Author,ISBN,ISBN13,My Rating,Date Read,Bookshelves,Exclusive Shelf,My Review
|
||||
1,Dune,Frank Herbert,"=""0441172717""","=""9780441172719""",4,2021/03/04,"sci-fi, classics",read,Great
|
||||
2,Neuromancer,William Gibson,"=""""","=""""",0,,,to-read,
|
||||
3,Hyperion,Dan Simmons,"=""""","=""9780553283686""",0,,,currently-reading,
|
||||
|
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
Title;Rating;Date Read;Review
|
||||
"Line ""one""";4;2019;"multi
|
||||
line"
|
||||
🐉 Dragons;;2020-05-06;
|
||||
|
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import plugin from "./index.ts";
|
||||
import { PAGE_SIZE } from "./src/graphql.ts";
|
||||
|
||||
const short = await Bun.file(new URL("./test/fixtures/page-short.json", import.meta.url)).json();
|
||||
const lists = await Bun.file(new URL("./test/fixtures/lists.json", import.meta.url)).json();
|
||||
const errors = await Bun.file(new URL("./test/fixtures/errors.json", import.meta.url)).json();
|
||||
const URL_ = "POST https://api.hardcover.app/v1/graphql";
|
||||
|
||||
type Body = { query: string; variables: { offset?: number; where?: unknown } };
|
||||
function ctxFor(userBooks: (b: Body) => unknown) {
|
||||
const calls: Body[] = [];
|
||||
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const body = JSON.parse(String(init?.body)) as Body;
|
||||
calls.push(body);
|
||||
const out = body.query.includes("query Lists") ? lists : userBooks(body);
|
||||
const inner = createFixtureHttp({ [URL_]: out });
|
||||
return inner.fetch(input, init);
|
||||
}) as typeof fetch;
|
||||
return { ctx: createTestContext({ fetch: fetchImpl, config: { token: "tok" } }), calls };
|
||||
}
|
||||
|
||||
describe("hardcover pull", () => {
|
||||
test("maps status, rating, series parent, favorite", async () => {
|
||||
const { ctx } = ctxFor(() => short);
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
const e = recs.find((r) => r.key === "ub:1");
|
||||
if (e?.kind !== "entry") throw new Error("no entry");
|
||||
expect(e.entry).toMatchObject({ status: "completed", rating: 90, review: "Great", notes: "lent to Sam", favorite: true });
|
||||
expect(e.work.externalIds).toEqual({ hardcover_book: "10", hardcover_edition: "500", isbn13: "9780441013593", isbn10: "0441013597" });
|
||||
expect(e.work.parent).toMatchObject({ title: "Dune Chronicles", externalIds: { hardcover_series: "77" } });
|
||||
expect(e.work.position).toEqual({ index: 1 });
|
||||
const e2 = recs.find((r) => r.key === "ub:2");
|
||||
if (e2?.kind !== "entry") throw new Error("no entry");
|
||||
expect(e2.entry).toMatchObject({ status: "in_progress", rating: null, favorite: false });
|
||||
});
|
||||
|
||||
test("re-reads yield two engagements with day precision", async () => {
|
||||
const { ctx } = ctxFor(() => short);
|
||||
const eng = (await collect(plugin.pull!(ctx))).filter((r) => r.kind === "engagement");
|
||||
expect(eng.map((r) => r.key)).toEqual(["read:900", "read:901"]);
|
||||
if (eng[0]?.kind !== "engagement") throw new Error();
|
||||
expect(eng[0].engagement.startedAt).toEqual({ value: "2024-01-05", precision: "day" });
|
||||
expect(eng[0].engagement.progress).toEqual({ pages: 412 });
|
||||
});
|
||||
|
||||
test("merge yields synthetic record with mergedIntoKey", async () => {
|
||||
const { ctx } = ctxFor(() => short);
|
||||
const m = (await collect(plugin.pull!(ctx))).find((r) => r.key === "ub:2@book:20");
|
||||
expect(m?.mergedIntoKey).toBe("ub:2");
|
||||
if (m?.kind !== "entry") throw new Error();
|
||||
expect(m.work.externalIds).toEqual({ hardcover_book: "20" });
|
||||
expect((await collect(plugin.pull!(ctxFor(() => short).ctx))).some((r) => r.key === "ub:1@book:10")).toBe(false);
|
||||
});
|
||||
|
||||
test("lists yield list records", async () => {
|
||||
const { ctx } = ctxFor(() => short);
|
||||
const l = (await collect(plugin.pull!(ctx))).find((r) => r.kind === "list");
|
||||
expect(l?.key).toBe("list:5");
|
||||
if (l?.kind !== "list") throw new Error();
|
||||
expect(l.list.items[0]?.work.externalIds).toEqual({ hardcover_book: "10" });
|
||||
});
|
||||
|
||||
test("sends bearer token", async () => {
|
||||
let auth = "";
|
||||
const f = (async (_i: unknown, init?: RequestInit) => {
|
||||
auth = new Headers(init?.headers).get("authorization") ?? "";
|
||||
return Response.json({ data: { me: [{ user_books: [], lists: [] }] } });
|
||||
}) as typeof fetch;
|
||||
await collect(plugin.pull!(createTestContext({ fetch: f, config: { token: "tok" } })));
|
||||
expect(auth).toBe("Bearer tok");
|
||||
});
|
||||
|
||||
test("GraphQL errors throw HttpError", async () => {
|
||||
const { ctx } = ctxFor(() => errors);
|
||||
await expect(collect(plugin.pull!(ctx))).rejects.toBeInstanceOf(HttpError);
|
||||
});
|
||||
|
||||
test("offset paging continues on full page and stops on short page", async () => {
|
||||
const row = short.data.me[0].user_books[0];
|
||||
const full = { data: { me: [{ user_books: Array.from({ length: PAGE_SIZE }, (_, i) => ({ ...row, id: 1000 + i, user_book_reads: [] })) }] } };
|
||||
const { ctx, calls } = ctxFor((b) => (b.variables.offset === 0 ? full : short));
|
||||
await collect(plugin.pull!(ctx));
|
||||
const ub = calls.filter((c) => c.query.includes("UserBooks"));
|
||||
expect(ub.map((c) => c.variables.offset)).toEqual([0, 100]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("hardcover pullChanges", () => {
|
||||
test("filters by cursor and advances it", async () => {
|
||||
const { ctx, calls } = ctxFor(() => short);
|
||||
await ctx.cursor.set("2024-05-01T00:00:00+00:00");
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(calls[0]?.variables.where).toEqual({ updated_at: { _gt: "2024-05-01T00:00:00+00:00" } });
|
||||
expect(recs.some((r) => r.kind === "list")).toBe(false);
|
||||
expect(ctx.saved.cursor.at(-1)).toBe("2024-06-01T10:00:00+00:00");
|
||||
});
|
||||
});
|
||||
|
||||
test("manifest", () => {
|
||||
expect(plugin.manifest).toMatchObject({ id: "hardcover", capabilities: ["pull", "pullChanges", "lookup"], mediaTypes: ["book"], attribution: "Book data from Hardcover." });
|
||||
});
|
||||
|
||||
describe("hardcover lookup", () => {
|
||||
const search = (results: unknown) => {
|
||||
const calls: unknown[] = [];
|
||||
const f = (async (_i: unknown, init?: RequestInit) => {
|
||||
calls.push(JSON.parse(String(init?.body)));
|
||||
return Response.json({ data: { search: { results } } });
|
||||
}) as typeof fetch;
|
||||
return { ctx: createTestContext({ fetch: f, config: { token: "tok" } }), calls };
|
||||
};
|
||||
|
||||
test("maps hits to WorkHints", async () => {
|
||||
const { ctx, calls } = search({
|
||||
hits: [
|
||||
{ document: { id: 10, title: "Dune", release_year: 1965, author_names: ["Frank Herbert"], image: { url: "https://img/dune.jpg" } } },
|
||||
{ document: { id: "11", title: "Bare" } },
|
||||
],
|
||||
});
|
||||
const out = await plugin.lookup!(ctx, { title: "dune" });
|
||||
expect(out).toEqual([
|
||||
{ title: "Dune", mediaType: "book", year: 1965, creators: [{ name: "Frank Herbert" }], externalIds: { hardcover_book: "10" }, coverUrl: "https://img/dune.jpg" },
|
||||
{ title: "Bare", mediaType: "book", externalIds: { hardcover_book: "11" } },
|
||||
]);
|
||||
expect((calls[0] as { variables: { q: string } }).variables.q).toBe("dune");
|
||||
});
|
||||
|
||||
test("null results returns []", async () => {
|
||||
expect(await plugin.lookup!(search(null).ctx, { title: "x" })).toEqual([]);
|
||||
});
|
||||
|
||||
test("non-book mediaType returns [] without a request", async () => {
|
||||
const { ctx, calls } = search({ hits: [] });
|
||||
expect(await plugin.lookup!(ctx, { title: "x", mediaType: "movie" })).toEqual([]);
|
||||
expect(calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("status mapping", () => {
|
||||
const cases: [number | null, string][] = [[1, "planned"], [2, "in_progress"], [3, "completed"], [4, "paused"], [5, "dropped"], [6, "none"], [null, "none"]];
|
||||
for (const [id, want] of cases) {
|
||||
test(`status_id ${id} -> ${want}`, async () => {
|
||||
const row = { ...short.data.me[0].user_books[1], status_id: id, merged_at: null };
|
||||
const ctx = createTestContext({
|
||||
fetch: (async () => Response.json({ data: { me: [{ user_books: [row], lists: [] }] } })) as unknown as typeof fetch,
|
||||
});
|
||||
const e = (await collect(plugin.pull!(ctx))).find((r) => r.key === "ub:2");
|
||||
if (e?.kind !== "entry") throw new Error();
|
||||
expect(e.entry.status).toBe(want as never);
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import type { MediaType, PluginContext, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import { gql, LISTS_QUERY, PAGE_SIZE, SEARCH_QUERY, USER_BOOKS_QUERY } from "./src/graphql.ts";
|
||||
import { mapList, mapUserBook } from "./src/map.ts";
|
||||
import type { HcList, HcUserBook } from "./src/map.ts";
|
||||
|
||||
export { RATE_PER_MINUTE } from "./src/graphql.ts";
|
||||
|
||||
async function* pageUserBooks(ctx: PluginContext, where: Record<string, unknown> | null): AsyncGenerator<HcUserBook> {
|
||||
for (let offset = 0; ; offset += PAGE_SIZE) {
|
||||
const d = await gql<{ me: { user_books: HcUserBook[] }[] }>(ctx, USER_BOOKS_QUERY, { limit: PAGE_SIZE, offset, where });
|
||||
const rows = d.me[0]?.user_books ?? [];
|
||||
yield* rows;
|
||||
if (rows.length < PAGE_SIZE) return;
|
||||
}
|
||||
}
|
||||
|
||||
async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const cursor = await ctx.cursor.get();
|
||||
const where = cursor ? { updated_at: { _gt: cursor } } : null;
|
||||
let max = cursor;
|
||||
for await (const ub of pageUserBooks(ctx, where)) {
|
||||
if (max === null || ub.updated_at > max) max = ub.updated_at;
|
||||
yield* mapUserBook(ub);
|
||||
}
|
||||
if (max !== null && max !== cursor) await ctx.cursor.set(max);
|
||||
}
|
||||
|
||||
async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
let max: string | null = null;
|
||||
for await (const ub of pageUserBooks(ctx, null)) {
|
||||
if (max === null || ub.updated_at > max) max = ub.updated_at;
|
||||
yield* mapUserBook(ub);
|
||||
}
|
||||
const d = await gql<{ me: { lists: HcList[] }[] }>(ctx, LISTS_QUERY, {});
|
||||
for (const l of d.me[0]?.lists ?? []) yield mapList(l);
|
||||
if (max !== null) await ctx.cursor.set(max);
|
||||
}
|
||||
|
||||
interface SearchDoc { id: number | string; title: string; release_year?: number; author_names?: string[]; image?: { url?: string } }
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "hardcover",
|
||||
name: "Hardcover",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["book"],
|
||||
capabilities: ["pull", "pullChanges", "lookup"],
|
||||
config: [{ key: "token", label: "Personal access token", type: "secret", required: true }],
|
||||
attribution: "Book data from Hardcover.",
|
||||
},
|
||||
pull,
|
||||
pullChanges,
|
||||
async lookup(ctx, q: { title: string; mediaType?: MediaType; year?: number }): Promise<WorkHint[]> {
|
||||
if (q.mediaType && q.mediaType !== "book") return [];
|
||||
const d = await gql<{ search: { results: { hits?: { document: SearchDoc }[] } | null } }>(ctx, SEARCH_QUERY, { q: q.title });
|
||||
return (d.search.results?.hits ?? []).map(({ document: x }) => ({
|
||||
title: x.title,
|
||||
mediaType: "book" as const,
|
||||
externalIds: { hardcover_book: String(x.id) },
|
||||
...(x.release_year ? { year: x.release_year } : {}),
|
||||
...(x.author_names?.length ? { creators: x.author_names.map((name) => ({ name })) } : {}),
|
||||
...(x.image?.url ? { coverUrl: x.image.url } : {}),
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import type { PluginContext } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
export const ENDPOINT = "https://api.hardcover.app/v1/graphql";
|
||||
export const RATE_PER_MINUTE = 50;
|
||||
export const PAGE_SIZE = 100;
|
||||
|
||||
const BOOK = `id title release_year contributions { author { name } }`;
|
||||
const BOOK_FULL = `id title release_year contributions { author { name } } book_series { position series { id name } }`;
|
||||
|
||||
export const USER_BOOKS_QUERY = `query UserBooks($limit: Int!, $offset: Int!, $where: user_books_bool_exp) {
|
||||
me { user_books(limit: $limit, offset: $offset, where: $where, order_by: {id: asc}) {
|
||||
id status_id rating review_raw private_notes starred updated_at merged_at original_book_id
|
||||
book { ${BOOK_FULL} }
|
||||
edition { id isbn_13 isbn_10 asin }
|
||||
user_book_reads { id started_at finished_at progress_pages }
|
||||
} }
|
||||
}`;
|
||||
|
||||
export const LISTS_QUERY = `query Lists { me { lists { id name description list_books { book { ${BOOK} } } } } }`;
|
||||
|
||||
export const SEARCH_QUERY = `query Search($q: String!) { search(query: $q, query_type: "Book", per_page: 10) { results } }`;
|
||||
|
||||
export async function gql<T>(ctx: PluginContext, query: string, variables: Record<string, unknown>): Promise<T> {
|
||||
const creds = await ctx.credentials.get();
|
||||
const token = creds?.token ?? (typeof ctx.config.token === "string" ? ctx.config.token : "");
|
||||
const res = await ctx.http.json<{ data?: T; errors?: { message: string }[] }>(ENDPOINT, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", authorization: `Bearer ${token}` },
|
||||
body: JSON.stringify({ query, variables }),
|
||||
});
|
||||
if (res.errors?.length || res.data === undefined) {
|
||||
throw new HttpError(200, JSON.stringify(res.errors ?? res), ENDPOINT);
|
||||
}
|
||||
return res.data;
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
|
||||
import type { EntryFields, SourceRecord, Status, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
export interface HcBook {
|
||||
id: number;
|
||||
title: string;
|
||||
release_year?: number | null;
|
||||
contributions?: { author: { name: string } | null }[];
|
||||
book_series?: { position: number | null; series: { id: number; name: string } }[];
|
||||
}
|
||||
export interface HcRead { id: number; started_at: string | null; finished_at: string | null; progress_pages: number | null }
|
||||
export interface HcUserBook {
|
||||
id: number;
|
||||
status_id: number | null;
|
||||
rating: number | null;
|
||||
review_raw: string | null;
|
||||
private_notes: string | null;
|
||||
starred: boolean | null;
|
||||
updated_at: string;
|
||||
merged_at: string | null;
|
||||
original_book_id: number | null;
|
||||
book: HcBook;
|
||||
edition: { id: number; isbn_13: string | null; isbn_10: string | null; asin: string | null } | null;
|
||||
user_book_reads: HcRead[];
|
||||
}
|
||||
export interface HcList { id: number; name: string; description: string | null; list_books: { book: HcBook }[] }
|
||||
|
||||
export const STATUS: Record<number, Status> = { 1: "planned", 2: "in_progress", 3: "completed", 4: "paused", 5: "dropped", 6: "none" };
|
||||
|
||||
export function bookHint(b: HcBook): WorkHint {
|
||||
const creators = (b.contributions ?? []).flatMap((c) => (c.author ? [{ name: c.author.name }] : []));
|
||||
const h: WorkHint = { title: b.title, mediaType: "book", externalIds: { hardcover_book: String(b.id) } };
|
||||
if (b.release_year != null) h.year = b.release_year;
|
||||
if (creators.length) h.creators = creators;
|
||||
return h;
|
||||
}
|
||||
|
||||
function workHint(ub: HcUserBook): WorkHint {
|
||||
const h = bookHint(ub.book);
|
||||
const e = ub.edition;
|
||||
if (e) {
|
||||
h.externalIds.hardcover_edition = String(e.id);
|
||||
if (e.isbn_13) h.externalIds.isbn13 = e.isbn_13;
|
||||
if (e.isbn_10) h.externalIds.isbn10 = e.isbn_10;
|
||||
if (e.asin) h.externalIds.asin = e.asin;
|
||||
}
|
||||
const s = ub.book.book_series?.[0];
|
||||
if (s) {
|
||||
h.parent = { title: s.series.name, mediaType: "book", externalIds: { hardcover_series: String(s.series.id) } };
|
||||
if (s.position != null) h.position = { index: s.position };
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
export function* mapUserBook(ub: HcUserBook): Generator<SourceRecord> {
|
||||
const work = workHint(ub);
|
||||
const entry: EntryFields = {
|
||||
status: STATUS[ub.status_id ?? 6] ?? "none",
|
||||
rating: ub.rating == null ? null : Math.round(ub.rating * 20),
|
||||
review: ub.review_raw,
|
||||
notes: ub.private_notes,
|
||||
favorite: ub.starred === true,
|
||||
};
|
||||
const key = `ub:${ub.id}`;
|
||||
yield { kind: "entry", key, raw: ub, work, entry };
|
||||
for (const r of ub.user_book_reads) {
|
||||
yield {
|
||||
kind: "engagement",
|
||||
key: `read:${r.id}`,
|
||||
raw: r,
|
||||
work,
|
||||
engagement: {
|
||||
startedAt: parsePartialDate(r.started_at),
|
||||
finishedAt: parsePartialDate(r.finished_at),
|
||||
...(r.progress_pages != null ? { progress: { pages: r.progress_pages } } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
if (ub.merged_at && ub.original_book_id != null && ub.original_book_id !== ub.book.id) {
|
||||
yield {
|
||||
kind: "entry",
|
||||
key: `${key}@book:${ub.original_book_id}`,
|
||||
raw: ub,
|
||||
mergedIntoKey: key,
|
||||
work: { title: ub.book.title, mediaType: "book", externalIds: { hardcover_book: String(ub.original_book_id) } },
|
||||
entry,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export function mapList(l: HcList): SourceRecord {
|
||||
return {
|
||||
kind: "list",
|
||||
key: `list:${l.id}`,
|
||||
raw: l,
|
||||
list: {
|
||||
name: l.name,
|
||||
...(l.description ? { description: l.description } : {}),
|
||||
items: l.list_books.map((lb) => ({ work: bookHint(lb.book) })),
|
||||
},
|
||||
};
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"errors":[{"message":"Invalid token"}]}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"data":{"me":[{"lists":[{"id":5,"name":"Favorites","description":"best","list_books":[{"book":{"id":10,"title":"Dune","release_year":1965,"contributions":[{"author":{"name":"Frank Herbert"}}]}}]}]}]}}
|
||||
@@ -0,0 +1,9 @@
|
||||
{"data":{"me":[{"user_books":[
|
||||
{"id":1,"status_id":3,"rating":4.5,"review_raw":"Great","private_notes":"lent to Sam","starred":true,"updated_at":"2024-05-02T10:00:00+00:00","merged_at":null,"original_book_id":10,
|
||||
"book":{"id":10,"title":"Dune","release_year":1965,"contributions":[{"author":{"name":"Frank Herbert"}}],"book_series":[{"position":1,"series":{"id":77,"name":"Dune Chronicles"}}]},
|
||||
"edition":{"id":500,"isbn_13":"9780441013593","isbn_10":"0441013597","asin":null},
|
||||
"user_book_reads":[{"id":900,"started_at":"2024-01-05","finished_at":"2024-01-20","progress_pages":412},{"id":901,"started_at":"2024-04-01","finished_at":"2024-04-15","progress_pages":null}]},
|
||||
{"id":2,"status_id":2,"rating":null,"review_raw":null,"private_notes":null,"starred":false,"updated_at":"2024-06-01T10:00:00+00:00","merged_at":"2024-05-30T00:00:00+00:00","original_book_id":20,
|
||||
"book":{"id":21,"title":"Standalone","release_year":null,"contributions":[],"book_series":[]},
|
||||
"edition":null,"user_book_reads":[]}
|
||||
]}]}}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import { pull, pullChanges } from "./src/lastfm.ts";
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "lastfm",
|
||||
name: "Last.fm",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["track", "album", "artist"],
|
||||
capabilities: ["pull", "pullChanges"],
|
||||
config: [
|
||||
{ key: "username", label: "Username", type: "string", required: true },
|
||||
{ key: "apiKey", label: "API key", type: "secret", required: true },
|
||||
],
|
||||
},
|
||||
pull,
|
||||
pullChanges,
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import type { ListenFields, PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
|
||||
import { sleep } from "./options.ts";
|
||||
|
||||
export const BASE = "https://ws.audioscrobbler.com/2.0/";
|
||||
export const MAX_ATTEMPTS = 3;
|
||||
|
||||
export class LastfmError extends Error {
|
||||
readonly code: number;
|
||||
constructor(code: number, message: string) {
|
||||
super(`Last.fm error ${code}: ${message}`);
|
||||
this.name = "LastfmError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
type Q = Record<string, string | number | undefined>;
|
||||
interface Text { "#text"?: string; mbid?: string; name?: string }
|
||||
interface LfmTrack {
|
||||
name?: string;
|
||||
mbid?: string;
|
||||
artist?: Text;
|
||||
album?: Text;
|
||||
date?: { uts?: string };
|
||||
"@attr"?: { nowplaying?: string };
|
||||
}
|
||||
|
||||
function errorOf(body: unknown): { code: number; message: string } | null {
|
||||
if (body && typeof body === "object" && "error" in body && typeof (body as { error: unknown }).error === "number") {
|
||||
const b = body as { error: number; message?: string };
|
||||
return { code: b.error, message: b.message ?? "unknown error" };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Last.fm reports errors in the JSON body (HTTP 200 or 4xx); error 29 (rate limit) is retried here. */
|
||||
export async function call<T>(ctx: PluginContext, query: Q): Promise<T> {
|
||||
const apiKey = (await ctx.credentials.get())?.apiKey ?? String(ctx.config.apiKey ?? "");
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
ctx.signal.throwIfAborted();
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await ctx.http.json<unknown>(BASE, { query: { ...query, api_key: apiKey, format: "json" } });
|
||||
} catch (e) {
|
||||
if (!(e instanceof HttpError)) throw e;
|
||||
let parsed: unknown;
|
||||
try { parsed = JSON.parse(e.body); } catch { throw e; }
|
||||
const err = errorOf(parsed);
|
||||
if (!err) throw e;
|
||||
body = parsed;
|
||||
}
|
||||
const err = errorOf(body);
|
||||
if (!err) return body as T;
|
||||
if (err.code === 29 && attempt < MAX_ATTEMPTS) {
|
||||
await sleep(2 ** attempt * 1000);
|
||||
continue;
|
||||
}
|
||||
throw new LastfmError(err.code, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
const arr = <T>(v: T | T[] | undefined): T[] => (v === undefined ? [] : Array.isArray(v) ? v : [v]);
|
||||
const nonEmpty = (s: string | undefined): string | undefined => (s && s.trim() !== "" ? s : undefined);
|
||||
|
||||
function toListen(t: LfmTrack): SourceRecord | null {
|
||||
const uts = t.date?.uts;
|
||||
const artist = nonEmpty(t.artist?.["#text"] ?? t.artist?.name);
|
||||
const track = nonEmpty(t.name);
|
||||
if (!uts || !artist || !track) return null;
|
||||
const listen: ListenFields = { listenedAt: new Date(Number(uts) * 1000).toISOString(), artist, track };
|
||||
const album = nonEmpty(t.album?.["#text"]);
|
||||
if (album) listen.album = album;
|
||||
const mbids: NonNullable<ListenFields["mbids"]> = {};
|
||||
const rec = nonEmpty(t.mbid), rel = nonEmpty(t.album?.mbid), art = nonEmpty(t.artist?.mbid);
|
||||
if (rec) mbids.recording = rec;
|
||||
if (rel) mbids.release = rel;
|
||||
if (art) mbids.artist = art;
|
||||
if (Object.keys(mbids).length > 0) listen.mbids = mbids;
|
||||
return { kind: "listen", key: `${uts}:${artist}:${track}`, raw: t, listen };
|
||||
}
|
||||
|
||||
async function* recent(ctx: PluginContext, from: number | undefined): AsyncGenerator<SourceRecord> {
|
||||
const user = String(ctx.config.username ?? "");
|
||||
let max = from ?? 0;
|
||||
for (let page = 1, total = 1; page <= total; page++) {
|
||||
const res = await call<{ recenttracks?: { track?: LfmTrack | LfmTrack[]; "@attr"?: { totalPages?: string } } }>(ctx, {
|
||||
method: "user.getRecentTracks", user, limit: 200, page, from,
|
||||
});
|
||||
total = Number(res.recenttracks?.["@attr"]?.totalPages ?? 1) || 1;
|
||||
for (const t of arr(res.recenttracks?.track)) {
|
||||
if (t["@attr"]?.nowplaying === "true") continue;
|
||||
const rec = toListen(t);
|
||||
if (!rec) continue;
|
||||
max = Math.max(max, Number(t.date?.uts));
|
||||
yield rec;
|
||||
}
|
||||
}
|
||||
if (max > 0) await ctx.cursor.set(String(max));
|
||||
}
|
||||
|
||||
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
yield* recent(ctx, undefined);
|
||||
yield* loved(ctx);
|
||||
}
|
||||
|
||||
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const c = await ctx.cursor.get();
|
||||
yield* recent(ctx, c ? Number(c) : undefined);
|
||||
}
|
||||
|
||||
async function* loved(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const user = String(ctx.config.username ?? "");
|
||||
for (let page = 1, total = 1; page <= total; page++) {
|
||||
const res = await call<{ lovedtracks?: { track?: LfmTrack | LfmTrack[]; "@attr"?: { totalPages?: string } } }>(ctx, {
|
||||
method: "user.getLovedTracks", user, limit: 200, page,
|
||||
});
|
||||
total = Number(res.lovedtracks?.["@attr"]?.totalPages ?? 1) || 1;
|
||||
for (const t of arr(res.lovedtracks?.track)) {
|
||||
const artist = nonEmpty(t.artist?.name ?? t.artist?.["#text"]);
|
||||
const title = nonEmpty(t.name);
|
||||
if (!artist || !title) continue;
|
||||
const mbid = nonEmpty(t.mbid);
|
||||
yield {
|
||||
kind: "entry",
|
||||
key: `loved:${artist}:${title}`,
|
||||
raw: t,
|
||||
work: {
|
||||
title,
|
||||
mediaType: "track",
|
||||
creators: [{ name: artist, role: "artist" }],
|
||||
externalIds: mbid ? { musicbrainz_recording: mbid } : {},
|
||||
},
|
||||
entry: { favorite: true },
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Injectable sleep. Tests replace it with an immediate one via `setSleep`. */
|
||||
let sleepImpl: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms);
|
||||
|
||||
export function setSleep(fn: ((ms: number) => Promise<void>) | null): void {
|
||||
sleepImpl = fn ?? ((ms) => Bun.sleep(ms));
|
||||
}
|
||||
|
||||
export const sleep = (ms: number): Promise<void> => sleepImpl(ms);
|
||||
@@ -0,0 +1,92 @@
|
||||
import { afterEach, beforeEach, expect, test } from "bun:test";
|
||||
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import plugin from "../index.ts";
|
||||
import { setSleep } from "../src/options.ts";
|
||||
|
||||
const sleeps: number[] = [];
|
||||
beforeEach(() => { sleeps.length = 0; setSleep(async (ms) => { sleeps.push(ms); }); });
|
||||
afterEach(() => setSleep(null));
|
||||
|
||||
const URL0 = "GET https://ws.audioscrobbler.com/2.0/";
|
||||
const rt = (name: string, uts: string, mbid = "", extra = {}) => ({
|
||||
name, mbid, artist: { "#text": "Art", mbid: "" }, album: { "#text": "Alb", mbid: "" }, date: { uts }, ...extra,
|
||||
});
|
||||
|
||||
function routes(handler: (q: URLSearchParams) => unknown) {
|
||||
return { [URL0]: (req: Request) => handler(new URL(req.url).searchParams) };
|
||||
}
|
||||
|
||||
test("skips nowplaying, omits empty mbids, includes loved entries", async () => {
|
||||
const f = createFixtureHttp(routes((q) => {
|
||||
expect(q.get("api_key")).toBe("key");
|
||||
expect(q.get("format")).toBe("json");
|
||||
if (q.get("method") === "user.getLovedTracks")
|
||||
return { lovedtracks: { track: [{ name: "Fav", mbid: "m9", artist: { name: "Art" } }], "@attr": { totalPages: "1" } } };
|
||||
return { recenttracks: { track: [
|
||||
{ name: "Now", artist: { "#text": "Art" }, date: { uts: "1700000005" }, "@attr": { nowplaying: "true" } },
|
||||
rt("One", "1700000000", "rec1", { artist: { "#text": "Art", mbid: "a1" } }),
|
||||
rt("Two", "1699999999"),
|
||||
], "@attr": { totalPages: "1" } } };
|
||||
}));
|
||||
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "key" } });
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
expect(recs.map((r) => r.key)).toEqual(["1700000000:Art:One", "1699999999:Art:Two", "loved:Art:Fav"]);
|
||||
const one = recs[0]!, two = recs[1]!, fav = recs[2]!;
|
||||
expect(one.kind === "listen" && one.listen.mbids).toEqual({ recording: "rec1", artist: "a1" });
|
||||
expect(two.kind === "listen" && two.listen.mbids).toBeUndefined();
|
||||
expect(fav.kind === "entry" && fav.entry.favorite).toBe(true);
|
||||
expect(fav.kind === "entry" && fav.work.externalIds).toEqual({ musicbrainz_recording: "m9" });
|
||||
expect(ctx.saved.cursor).toEqual(["1700000000"]);
|
||||
});
|
||||
|
||||
test("paginates recent tracks", async () => {
|
||||
const pagesSeen: (string | null)[] = [];
|
||||
const f = createFixtureHttp(routes((q) => {
|
||||
if (q.get("method") === "user.getLovedTracks") return { lovedtracks: { track: [], "@attr": { totalPages: "0" } } };
|
||||
pagesSeen.push(q.get("page"));
|
||||
return { recenttracks: { track: [rt(`T${q.get("page")}`, `10${q.get("page")}`)], "@attr": { totalPages: "2" } } };
|
||||
}));
|
||||
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } })));
|
||||
expect(pagesSeen).toEqual(["1", "2"]);
|
||||
expect(recs).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("error 29 is retried with exponential sleeps (body on HTTP 200 and 403)", async () => {
|
||||
let n = 0;
|
||||
const f = createFixtureHttp(routes((q) => {
|
||||
if (q.get("method") === "user.getLovedTracks") return { lovedtracks: { track: [] } };
|
||||
n++;
|
||||
if (n === 1) return { error: 29, message: "Rate Limit Exceeded" };
|
||||
if (n === 2) return new Response(JSON.stringify({ error: 29, message: "Rate Limit Exceeded" }), { status: 403 });
|
||||
return { recenttracks: { track: rt("X", "5") } };
|
||||
}));
|
||||
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } })));
|
||||
expect(n).toBe(3);
|
||||
expect(sleeps).toEqual([2000, 4000]);
|
||||
expect(recs).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("error 29 gives up after 3 attempts; other codes throw immediately", async () => {
|
||||
const f29 = createFixtureHttp(routes(() => ({ error: 29, message: "Rate Limit Exceeded" })));
|
||||
const p = collect(plugin.pull!(createTestContext({ fetch: f29.fetch, config: { username: "u", apiKey: "k" } })));
|
||||
await expect(p).rejects.toMatchObject({ code: 29 });
|
||||
expect(f29.calls).toHaveLength(3);
|
||||
|
||||
const f6 = createFixtureHttp(routes(() => new Response(JSON.stringify({ error: 6, message: "User not found" }), { status: 404 })));
|
||||
const p6 = collect(plugin.pull!(createTestContext({ fetch: f6.fetch, config: { username: "u", apiKey: "k" } })));
|
||||
await expect(p6).rejects.toThrow(/Last\.fm error 6: User not found/);
|
||||
expect(f6.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("pullChanges sends from=<cursor> and advances it", async () => {
|
||||
let from: string | null = null;
|
||||
const f = createFixtureHttp(routes((q) => {
|
||||
from = q.get("from");
|
||||
return { recenttracks: { track: [rt("N", "2000")], "@attr": { totalPages: "1" } } };
|
||||
}));
|
||||
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", apiKey: "k" } });
|
||||
await ctx.cursor.set("1500");
|
||||
await collect(plugin.pullChanges!(ctx));
|
||||
expect(from as string | null).toBe("1500");
|
||||
expect(await ctx.cursor.get()).toBe("2000");
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import { pull, pullChanges } from "./src/listenbrainz.ts";
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "listenbrainz",
|
||||
name: "ListenBrainz",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["track", "album", "artist"],
|
||||
capabilities: ["pull", "pullChanges"],
|
||||
config: [
|
||||
{ key: "username", label: "Username", type: "string", required: true },
|
||||
{ key: "token", label: "User token", type: "secret", required: true },
|
||||
],
|
||||
},
|
||||
pull,
|
||||
pullChanges,
|
||||
});
|
||||
@@ -0,0 +1,183 @@
|
||||
import { strFromU8, unzipSync } from "fflate";
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import type { ListenFields, PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
|
||||
import { sleep } from "./options.ts";
|
||||
|
||||
export const BASE = "https://api.listenbrainz.org";
|
||||
export const POLL_INTERVAL_MS = 30_000;
|
||||
export const MAX_POLLS = 60;
|
||||
|
||||
interface TrackMetadata {
|
||||
artist_name?: string;
|
||||
track_name?: string;
|
||||
release_name?: string;
|
||||
additional_info?: { duration_ms?: number };
|
||||
mbid_mapping?: { recording_mbid?: string; release_mbid?: string; artist_mbids?: string[] };
|
||||
}
|
||||
export interface LbListen {
|
||||
listened_at: number | string;
|
||||
recording_msid?: string;
|
||||
track_metadata?: TrackMetadata;
|
||||
}
|
||||
interface LbFeedback {
|
||||
recording_mbid?: string;
|
||||
recording_msid?: string;
|
||||
score?: number;
|
||||
track_metadata?: TrackMetadata;
|
||||
}
|
||||
interface ExportInfo { export_id: number | string; status: string; created?: string }
|
||||
|
||||
async function authHeaders(ctx: PluginContext): Promise<Record<string, string>> {
|
||||
const creds = await ctx.credentials.get();
|
||||
const token = creds?.token ?? (typeof ctx.config.token === "string" ? ctx.config.token : undefined);
|
||||
return token ? { authorization: `Token ${token}` } : {};
|
||||
}
|
||||
|
||||
/** Unix seconds from either a number or an ISO string; null if unparseable. */
|
||||
export function toSeconds(v: number | string | undefined): number | null {
|
||||
if (typeof v === "number") return Number.isFinite(v) ? Math.floor(v) : null;
|
||||
if (typeof v === "string" && v.trim() !== "") {
|
||||
if (/^\d+$/.test(v.trim())) return Number(v);
|
||||
const ms = Date.parse(v);
|
||||
return Number.isNaN(ms) ? null : Math.floor(ms / 1000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function listenToRecord(l: LbListen): SourceRecord | null {
|
||||
const secs = toSeconds(l.listened_at);
|
||||
const tm = l.track_metadata;
|
||||
if (secs === null || !tm?.artist_name || !tm.track_name) return null;
|
||||
const listen: ListenFields = {
|
||||
listenedAt: new Date(secs * 1000).toISOString(),
|
||||
artist: tm.artist_name,
|
||||
track: tm.track_name,
|
||||
};
|
||||
if (tm.release_name) listen.album = tm.release_name;
|
||||
const dur = tm.additional_info?.duration_ms;
|
||||
if (typeof dur === "number" && dur > 0) listen.durationMs = dur;
|
||||
const m = tm.mbid_mapping;
|
||||
const mbids: NonNullable<ListenFields["mbids"]> = {};
|
||||
if (m?.recording_mbid) mbids.recording = m.recording_mbid;
|
||||
if (m?.release_mbid) mbids.release = m.release_mbid;
|
||||
if (m?.artist_mbids?.[0]) mbids.artist = m.artist_mbids[0];
|
||||
if (Object.keys(mbids).length > 0) listen.mbids = mbids;
|
||||
return { kind: "listen", key: `${secs}:${l.recording_msid ?? ""}`, raw: l, listen };
|
||||
}
|
||||
|
||||
function feedbackToRecord(f: LbFeedback): SourceRecord | null {
|
||||
if (f.score !== 1 || !f.recording_mbid) return null;
|
||||
return {
|
||||
kind: "entry",
|
||||
key: `feedback:${f.recording_mbid}`,
|
||||
raw: f,
|
||||
work: {
|
||||
title: f.track_metadata?.track_name ?? f.recording_mbid,
|
||||
mediaType: "track",
|
||||
externalIds: { musicbrainz_recording: f.recording_mbid },
|
||||
...(f.track_metadata?.artist_name ? { creators: [{ name: f.track_metadata.artist_name, role: "artist" }] } : {}),
|
||||
},
|
||||
entry: { favorite: true },
|
||||
};
|
||||
}
|
||||
|
||||
function* jsonlLines<T>(bytes: Uint8Array): Generator<T> {
|
||||
for (const line of strFromU8(bytes).split("\n")) {
|
||||
const s = line.trim();
|
||||
if (s === "") continue;
|
||||
yield JSON.parse(s) as T;
|
||||
}
|
||||
}
|
||||
|
||||
async function requestExport(ctx: PluginContext, headers: Record<string, string>): Promise<void> {
|
||||
try {
|
||||
await ctx.http.json(`${BASE}/1/export/`, { method: "POST", headers });
|
||||
} catch (e) {
|
||||
if (e instanceof HttpError && e.status === 400 && /pending/i.test(e.body)) return;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForExport(ctx: PluginContext, headers: Record<string, string>): Promise<string | number> {
|
||||
for (let poll = 0; poll < MAX_POLLS; poll++) {
|
||||
if (poll > 0) await sleep(POLL_INTERVAL_MS);
|
||||
ctx.signal.throwIfAborted();
|
||||
const list = await ctx.http.json<ExportInfo[]>(`${BASE}/1/export/list`, { headers });
|
||||
const newest = [...list].sort((a, b) => (b.created ?? "").localeCompare(a.created ?? ""))[0];
|
||||
if (newest?.status === "complete") return newest.export_id;
|
||||
if (newest?.status === "failed") throw new Error(`ListenBrainz export ${newest.export_id} failed`);
|
||||
}
|
||||
throw new Error(`ListenBrainz export did not complete after ${MAX_POLLS} polls`);
|
||||
}
|
||||
|
||||
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const headers = await authHeaders(ctx);
|
||||
await requestExport(ctx, headers);
|
||||
const id = await waitForExport(ctx, headers);
|
||||
const res = await ctx.http.raw(`${BASE}/1/export/${id}/download`, { headers });
|
||||
if (!res.ok) throw new HttpError(res.status, await res.text(), res.url);
|
||||
const files = unzipSync(new Uint8Array(await res.arrayBuffer()), {
|
||||
filter: (f) => f.name.endsWith(".jsonl"),
|
||||
});
|
||||
let max = 0;
|
||||
for (const name of Object.keys(files).sort()) {
|
||||
const bytes = files[name]!;
|
||||
if (name === "feedback.jsonl" || name.endsWith("/feedback.jsonl")) {
|
||||
for (const f of jsonlLines<LbFeedback>(bytes)) {
|
||||
const rec = feedbackToRecord(f);
|
||||
if (rec) yield rec;
|
||||
}
|
||||
} else if (/(^|\/)listens\/.+\.jsonl$/.test(name)) {
|
||||
for (const l of jsonlLines<LbListen>(bytes)) {
|
||||
const rec = listenToRecord(l);
|
||||
if (!rec) continue;
|
||||
max = Math.max(max, toSeconds(l.listened_at) ?? 0);
|
||||
yield rec;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (max > 0) await ctx.cursor.set(String(max));
|
||||
}
|
||||
|
||||
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const headers = await authHeaders(ctx);
|
||||
const username = String(ctx.config.username ?? "");
|
||||
const cursor = Number((await ctx.cursor.get()) ?? 0) || 0;
|
||||
// We do not rely on the ordering of `min_ts` queries (and LB forbids min_ts together with max_ts).
|
||||
// Instead walk backwards from now with `max_ts` (listens older than max_ts, newest first) until we
|
||||
// reach a listen at or before the cursor, or run out of pages.
|
||||
// max_ts = smallest + 1 re-includes the page-boundary second so listens sharing that second are not
|
||||
// lost; the duplicates are dropped by the seen-set here and by idempotent keys downstream.
|
||||
// If that would not make progress (a whole page in one second), fall back to max_ts = smallest.
|
||||
const seen = new Set<string>();
|
||||
let max = cursor;
|
||||
let maxTs: number | undefined;
|
||||
for (;;) {
|
||||
ctx.signal.throwIfAborted();
|
||||
const res = await ctx.http.json<{ payload?: { listens?: LbListen[] } }>(
|
||||
`${BASE}/1/user/${encodeURIComponent(username)}/listens`,
|
||||
{ headers, query: { max_ts: maxTs, count: 1000 } },
|
||||
);
|
||||
const listens = res.payload?.listens ?? [];
|
||||
if (listens.length === 0) break;
|
||||
let smallest = Infinity;
|
||||
let reachedCursor = false;
|
||||
for (const l of listens) {
|
||||
const secs = toSeconds(l.listened_at);
|
||||
if (secs === null) continue;
|
||||
smallest = Math.min(smallest, secs);
|
||||
if (secs <= cursor) { reachedCursor = true; continue; }
|
||||
const rec = listenToRecord(l);
|
||||
if (!rec || seen.has(rec.key)) continue;
|
||||
seen.add(rec.key);
|
||||
max = Math.max(max, secs);
|
||||
yield rec;
|
||||
}
|
||||
if (reachedCursor || !Number.isFinite(smallest)) break;
|
||||
let next = smallest + 1;
|
||||
if (maxTs !== undefined && next >= maxTs) next = smallest;
|
||||
if (maxTs !== undefined && next >= maxTs) break;
|
||||
maxTs = next;
|
||||
}
|
||||
if (max > cursor) await ctx.cursor.set(String(max));
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Injectable sleep. Tests replace it with an immediate one via `setSleep`. */
|
||||
let sleepImpl: (ms: number) => Promise<void> = (ms) => Bun.sleep(ms);
|
||||
|
||||
export function setSleep(fn: ((ms: number) => Promise<void>) | null): void {
|
||||
sleepImpl = fn ?? ((ms) => Bun.sleep(ms));
|
||||
}
|
||||
|
||||
export const sleep = (ms: number): Promise<void> => sleepImpl(ms);
|
||||
@@ -0,0 +1,113 @@
|
||||
import { afterEach, beforeEach, expect, test } from "bun:test";
|
||||
import { zipSync, strToU8 } from "fflate";
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import plugin from "../index.ts";
|
||||
import { setSleep } from "../src/options.ts";
|
||||
|
||||
const sleeps: number[] = [];
|
||||
beforeEach(() => { sleeps.length = 0; setSleep(async (ms) => { sleeps.push(ms); }); });
|
||||
afterEach(() => setSleep(null));
|
||||
|
||||
const jl = (rows: unknown[]) => strToU8(rows.map((r) => JSON.stringify(r)).join("\n") + "\n");
|
||||
const listen = (at: number | string, msid: string, extra = {}) => ({
|
||||
listened_at: at, recording_msid: msid,
|
||||
track_metadata: { artist_name: "A", track_name: `T${msid}`, release_name: "R", additional_info: { duration_ms: 1000 }, ...extra },
|
||||
});
|
||||
const mapped = { mbid_mapping: { recording_mbid: "rec", release_mbid: "rel", artist_mbids: ["art"] } };
|
||||
|
||||
function exportRoutes(listRuns: unknown[][], post: () => Response) {
|
||||
let i = 0;
|
||||
const zip = zipSync({
|
||||
"listens/2024/1.jsonl": jl([listen(1700000000, "m1", mapped), listen("2023-11-14T22:13:20Z", "m2")]),
|
||||
"feedback.jsonl": jl([
|
||||
{ recording_mbid: "love1", score: 1, track_metadata: { track_name: "Loved", artist_name: "A" } },
|
||||
{ recording_mbid: "hate1", score: -1 },
|
||||
]),
|
||||
});
|
||||
return {
|
||||
"POST https://api.listenbrainz.org/1/export/": post,
|
||||
"GET https://api.listenbrainz.org/1/export/list": () => Response.json(listRuns[Math.min(i++, listRuns.length - 1)]),
|
||||
"GET https://api.listenbrainz.org/1/export/7/download": () => new Response(zip),
|
||||
};
|
||||
}
|
||||
const pending = [{ export_id: 7, status: "in_progress", created: "2024-01-01" }];
|
||||
const done = [{ export_id: 6, status: "complete", created: "2023-01-01" }, { export_id: 7, status: "complete", created: "2024-01-01" }];
|
||||
|
||||
test("pull: polls pending then complete, parses listens and feedback", async () => {
|
||||
const f = createFixtureHttp(exportRoutes([pending, done], () => Response.json({ export_id: 7 })));
|
||||
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u", token: "tok" } });
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
expect(sleeps).toEqual([30_000]);
|
||||
expect(f.calls[0]!.headers.get("authorization")).toBe("Token tok");
|
||||
expect(recs.map((r) => r.key)).toEqual(["feedback:love1", "1700000000:m1", "1700000000:m2"]);
|
||||
const l = recs.find((r) => r.key === "1700000000:m1")!;
|
||||
expect(l.kind === "listen" && l.listen).toEqual({
|
||||
listenedAt: "2023-11-14T22:13:20.000Z", artist: "A", track: "Tm1", album: "R", durationMs: 1000,
|
||||
mbids: { recording: "rec", release: "rel", artist: "art" },
|
||||
});
|
||||
const m2 = recs.find((r) => r.key === "1700000000:m2")!;
|
||||
expect(m2.kind === "listen" && m2.listen.mbids).toBeUndefined();
|
||||
const fav = recs[0]!;
|
||||
expect(fav.kind === "entry" && fav.entry.favorite).toBe(true);
|
||||
expect(fav.kind === "entry" && fav.work.externalIds).toEqual({ musicbrainz_recording: "love1" });
|
||||
expect(ctx.saved.cursor).toEqual(["1700000000"]);
|
||||
});
|
||||
|
||||
test("pull: 400 already pending falls through to listing", async () => {
|
||||
const f = createFixtureHttp(exportRoutes([done], () => new Response("An export is already pending", { status: 400 })));
|
||||
const recs = await collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
|
||||
expect(recs).toHaveLength(3);
|
||||
});
|
||||
|
||||
test("pull: other POST errors propagate", async () => {
|
||||
const f = createFixtureHttp(exportRoutes([done], () => new Response("nope", { status: 401 })));
|
||||
const p = collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
|
||||
await expect(p).rejects.toBeInstanceOf(HttpError);
|
||||
});
|
||||
|
||||
test("pull: gives up after 60 polls", async () => {
|
||||
const f = createFixtureHttp(exportRoutes([pending], () => Response.json({ export_id: 7 })));
|
||||
const p = collect(plugin.pull!(createTestContext({ fetch: f.fetch, config: { username: "u" } })));
|
||||
await expect(p).rejects.toThrow(/60 polls/);
|
||||
expect(sleeps).toHaveLength(59);
|
||||
});
|
||||
|
||||
function changesFixture(pages: unknown[][], seen: (string | null)[]) {
|
||||
let i = 0;
|
||||
return createFixtureHttp({
|
||||
"GET https://api.listenbrainz.org/1/user/u/listens": (req: Request) => {
|
||||
const u = new URL(req.url);
|
||||
expect(u.searchParams.get("min_ts")).toBeNull();
|
||||
seen.push(u.searchParams.get("max_ts"));
|
||||
return { payload: { listens: pages[i++] ?? [] } };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("pullChanges: walks back over 3 pages with max_ts, cursor ends at max", async () => {
|
||||
const seen: (string | null)[] = [];
|
||||
const f = changesFixture(
|
||||
[
|
||||
[listen(600, "f"), listen(500, "e")],
|
||||
[listen(500, "e2"), listen(400, "d")],
|
||||
[listen(300, "c"), listen(200, "b")],
|
||||
[listen(100, "a"), listen(50, "z")],
|
||||
],
|
||||
seen,
|
||||
);
|
||||
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u" } });
|
||||
await ctx.cursor.set("150");
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(recs.map((r) => r.key)).toEqual(["600:f", "500:e", "500:e2", "400:d", "300:c", "200:b"]);
|
||||
expect(seen).toEqual([null, "501", "401", "201"]);
|
||||
expect(ctx.saved.cursor).toEqual(["150", "600"]);
|
||||
});
|
||||
|
||||
test("pullChanges: unchanged cursor yields nothing", async () => {
|
||||
const f = changesFixture([[listen(600, "f"), listen(500, "e")]], []);
|
||||
const ctx = createTestContext({ fetch: f.fetch, config: { username: "u" } });
|
||||
await ctx.cursor.set("600");
|
||||
expect(await collect(plugin.pullChanges!(ctx))).toEqual([]);
|
||||
expect(ctx.saved.cursor).toEqual(["600"]);
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import { lookup } from "./src/lookup.ts";
|
||||
import { pull, pullChanges } from "./src/pull.ts";
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "mangabaka",
|
||||
name: "MangaBaka",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["manga", "comic"],
|
||||
capabilities: ["pull", "pullChanges", "lookup"],
|
||||
config: [{ key: "token", label: "API token", type: "secret", required: true, help: "Sent as the x-api-key header." }],
|
||||
attribution: "Manga data from MangaBaka (CC BY-NC-SA 4.0).",
|
||||
},
|
||||
pull,
|
||||
pullChanges,
|
||||
lookup,
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { PluginContext } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
export const BASE_URL = "https://api.mangabaka.org";
|
||||
/** The host builds the HttpClient; it should use this rate for this plugin. */
|
||||
export const RATE_PER_MINUTE = 150;
|
||||
|
||||
export interface ApiSource { id?: number | string | null }
|
||||
export interface ApiSeries {
|
||||
id: number;
|
||||
title: string;
|
||||
type?: string | null;
|
||||
state?: string | null;
|
||||
merged_with?: number | null;
|
||||
year?: number | null;
|
||||
source?: {
|
||||
anilist?: ApiSource | null;
|
||||
my_anime_list?: ApiSource | null;
|
||||
manga_updates?: ApiSource | null;
|
||||
kitsu?: ApiSource | null;
|
||||
} | null;
|
||||
}
|
||||
export interface ApiEntry {
|
||||
series_id: number;
|
||||
state?: string | null;
|
||||
rating?: number | null;
|
||||
progress_chapter?: number | null;
|
||||
progress_volume?: number | null;
|
||||
start_date?: string | null;
|
||||
finish_date?: string | null;
|
||||
note?: string | null;
|
||||
series?: ApiSeries | null;
|
||||
}
|
||||
export interface LibraryPage {
|
||||
data: ApiEntry[];
|
||||
pagination?: { page?: number; total_pages?: number } | null;
|
||||
}
|
||||
export interface ChangesPage {
|
||||
data: { type: "upsert" | "delete"; entry?: ApiEntry; series_id: number }[];
|
||||
next_cursor?: string | null;
|
||||
}
|
||||
|
||||
export function authHeaders(ctx: PluginContext): Record<string, string> {
|
||||
const token = ctx.config.token;
|
||||
if (typeof token !== "string" || token === "") throw new Error("mangabaka: token is not configured");
|
||||
return { "x-api-key": token };
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { MediaType, PluginContext, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import { BASE_URL, authHeaders, type ApiSeries } from "./api.ts";
|
||||
import { mapEntry } from "./map.ts";
|
||||
|
||||
export async function lookup(ctx: PluginContext, q: { title: string; mediaType?: MediaType; year?: number }): Promise<WorkHint[]> {
|
||||
const res = await ctx.http.json<{ data: ApiSeries[] }>(`${BASE_URL}/v1/series/search`, {
|
||||
headers: authHeaders(ctx),
|
||||
query: { q: q.title },
|
||||
signal: ctx.signal,
|
||||
});
|
||||
return res.data
|
||||
.map((s) => (mapEntry({ series_id: s.id, series: s })[0] as { work: WorkHint }).work)
|
||||
.filter((w) => (q.mediaType ? w.mediaType === q.mediaType : true) && (q.year == null || w.year == null || w.year === q.year));
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
|
||||
import type { SourceRecord, Status, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
import type { ApiEntry } from "./api.ts";
|
||||
|
||||
export const STATE_MAP: Record<string, Status> = {
|
||||
considering: "planned",
|
||||
plan_to_read: "planned",
|
||||
reading: "in_progress",
|
||||
rereading: "in_progress",
|
||||
paused: "paused",
|
||||
dropped: "dropped",
|
||||
completed: "completed",
|
||||
};
|
||||
|
||||
export function mapState(s: string | null | undefined): Status {
|
||||
return s != null && Object.hasOwn(STATE_MAP, s) ? STATE_MAP[s]! : "none";
|
||||
}
|
||||
|
||||
function hint(e: ApiEntry): WorkHint {
|
||||
const s = e.series;
|
||||
const externalIds: Record<string, string> = { mangabaka: String(e.series_id) };
|
||||
const add = (ns: string, v: { id?: number | string | null } | null | undefined) => {
|
||||
if (v?.id != null && v.id !== "") externalIds[ns] = String(v.id);
|
||||
};
|
||||
add("anilist_manga", s?.source?.anilist);
|
||||
add("mal_manga", s?.source?.my_anime_list);
|
||||
add("mangaupdates", s?.source?.manga_updates);
|
||||
add("kitsu_manga", s?.source?.kitsu);
|
||||
return {
|
||||
title: s?.title ?? `MangaBaka series ${e.series_id}`,
|
||||
mediaType: s?.type === "comic" ? "comic" : "manga",
|
||||
...(s?.year != null ? { year: s.year } : {}),
|
||||
externalIds,
|
||||
};
|
||||
}
|
||||
|
||||
export function mapEntry(e: ApiEntry): SourceRecord[] {
|
||||
const work = hint(e);
|
||||
const key = `series:${e.series_id}`;
|
||||
const merged = e.series?.state === "merged" && e.series.merged_with != null;
|
||||
const entry: SourceRecord = {
|
||||
kind: "entry",
|
||||
key,
|
||||
raw: e,
|
||||
work,
|
||||
entry: {
|
||||
status: mapState(e.state),
|
||||
...(e.rating !== undefined ? { rating: e.rating } : {}),
|
||||
...(e.note !== undefined ? { notes: e.note } : {}),
|
||||
},
|
||||
...(merged ? { mergedIntoKey: `series:${e.series!.merged_with}` } : {}),
|
||||
};
|
||||
const progress: Record<string, number> = {};
|
||||
if (e.progress_chapter != null) progress.chapter = e.progress_chapter;
|
||||
if (e.progress_volume != null) progress.volume = e.progress_volume;
|
||||
const started = parsePartialDate(e.start_date);
|
||||
const finished = parsePartialDate(e.finish_date);
|
||||
if (Object.keys(progress).length === 0 && !started && !finished) return [entry];
|
||||
const engagement: SourceRecord = {
|
||||
kind: "engagement",
|
||||
key: `progress:${e.series_id}`,
|
||||
raw: e,
|
||||
work,
|
||||
engagement: { progress, startedAt: started, finishedAt: finished },
|
||||
};
|
||||
return [entry, engagement];
|
||||
}
|
||||
|
||||
export function mapDeletion(seriesId: number): SourceRecord[] {
|
||||
const work: WorkHint = { title: `MangaBaka series ${seriesId}`, mediaType: "manga", externalIds: { mangabaka: String(seriesId) } };
|
||||
return [
|
||||
{ kind: "entry", key: `series:${seriesId}`, raw: { series_id: seriesId }, work, entry: {}, deleted: true },
|
||||
{ kind: "engagement", key: `progress:${seriesId}`, raw: { series_id: seriesId }, work, engagement: {}, deleted: true },
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import type { PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
|
||||
import { BASE_URL, authHeaders, type ChangesPage, type LibraryPage } from "./api.ts";
|
||||
import { mapDeletion, mapEntry } from "./map.ts";
|
||||
|
||||
export async function* pull(ctx: PluginContext): AsyncIterable<SourceRecord> {
|
||||
const headers = authHeaders(ctx);
|
||||
for (let page = 1; ; page++) {
|
||||
const res = await ctx.http.json<LibraryPage>(`${BASE_URL}/v1/my/library`, { headers, query: { page }, signal: ctx.signal });
|
||||
for (const e of res.data) yield* mapEntry(e);
|
||||
const total = res.pagination?.total_pages;
|
||||
if (res.data.length === 0 || (total != null && page >= total)) return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function* pullChanges(ctx: PluginContext): AsyncIterable<SourceRecord> {
|
||||
const headers = authHeaders(ctx);
|
||||
let cursor = (await ctx.cursor.get()) ?? undefined;
|
||||
for (;;) {
|
||||
const res = await ctx.http.json<ChangesPage>(`${BASE_URL}/v2/my/library/changes`, { headers, query: { cursor }, signal: ctx.signal });
|
||||
for (const c of res.data) {
|
||||
if (c.type === "delete") yield* mapDeletion(c.series_id);
|
||||
else if (c.entry) yield* mapEntry(c.entry);
|
||||
else ctx.log(`mangabaka: upsert without entry for series ${c.series_id}, skipped`);
|
||||
}
|
||||
const next = res.next_cursor;
|
||||
if (next) {
|
||||
await ctx.cursor.set(next);
|
||||
if (next === cursor) return;
|
||||
cursor = next;
|
||||
}
|
||||
if (!next || res.data.length === 0) return;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{"data":[
|
||||
{"type":"upsert","series_id":1,"entry":{"series_id":1,"state":"paused","rating":70,"progress_chapter":13,"progress_volume":2,"start_date":"2024-03-05","finish_date":null,"note":null,"series":{"id":1,"title":"Alpha","type":"manga","state":"active","merged_with":null,"year":2020,"source":{}}}},
|
||||
{"type":"delete","series_id":7}
|
||||
],"next_cursor":"cur-2"}
|
||||
@@ -0,0 +1,4 @@
|
||||
{"data":[
|
||||
{"series_id":1,"state":"reading","rating":80,"progress_chapter":12,"progress_volume":2,"start_date":"2024-03-05","finish_date":null,"note":"good","series":{"id":1,"title":"Alpha","type":"manga","state":"active","merged_with":null,"year":2020,"source":{"anilist":{"id":11},"my_anime_list":{"id":22},"manga_updates":{"id":33},"kitsu":{"id":44}}}},
|
||||
{"series_id":2,"state":"plan_to_read","rating":null,"progress_chapter":null,"progress_volume":null,"start_date":null,"finish_date":null,"note":null,"series":{"id":2,"title":"Beta","type":"manga","state":"merged","merged_with":9,"year":null,"source":{}}}
|
||||
],"pagination":{"page":1,"total_pages":2}}
|
||||
@@ -0,0 +1,3 @@
|
||||
{"data":[
|
||||
{"series_id":3,"state":"completed","rating":95,"progress_chapter":50,"progress_volume":null,"start_date":"2023-01","finish_date":"2023-06-30","note":null,"series":{"id":3,"title":"Gamma","type":"manga","state":"active","merged_with":null,"year":2019,"source":{}}}
|
||||
],"pagination":{"page":2,"total_pages":2}}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"data":[{"id":5,"title":"Delta","type":"manga","state":"active","merged_with":null,"year":2018,"source":{"anilist":{"id":55}}}]}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import type { Status } from "@sttracker/plugin-sdk/types";
|
||||
import plugin from "../index.ts";
|
||||
import { mapDeletion, mapState } from "../src/map.ts";
|
||||
import page1 from "./fixtures/library-page1.json";
|
||||
import page2 from "./fixtures/library-page2.json";
|
||||
import changes from "./fixtures/changes.json";
|
||||
import search from "./fixtures/search.json";
|
||||
|
||||
const B = "https://api.mangabaka.org";
|
||||
const ctxFor = (routes: Record<string, unknown>) => {
|
||||
const h = createFixtureHttp(routes);
|
||||
return { h, ctx: createTestContext({ fetch: h.fetch, config: { token: "tok" } }) };
|
||||
};
|
||||
|
||||
describe("mangabaka", () => {
|
||||
test("manifest", () => {
|
||||
expect(plugin.manifest.id).toBe("mangabaka");
|
||||
expect(plugin.manifest.capabilities).toEqual(["pull", "pullChanges", "lookup"]);
|
||||
});
|
||||
|
||||
test.each<[string, Status]>([
|
||||
["considering", "planned"], ["plan_to_read", "planned"], ["reading", "in_progress"],
|
||||
["rereading", "in_progress"], ["paused", "paused"], ["dropped", "dropped"], ["completed", "completed"],
|
||||
])("state %s -> %s", (a, b) => expect(mapState(a)).toBe(b));
|
||||
|
||||
test("pull pages, maps entries and engagements, sends api key", async () => {
|
||||
const { h, ctx } = ctxFor({
|
||||
[`GET ${B}/v1/my/library?page=1`]: page1,
|
||||
[`GET ${B}/v1/my/library?page=2`]: page2,
|
||||
});
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
expect(recs.map((r) => r.key)).toEqual(["series:1", "progress:1", "series:2", "series:3", "progress:3"]);
|
||||
expect(h.calls.every((c) => c.headers.get("x-api-key") === "tok")).toBe(true);
|
||||
const a = recs[0]!;
|
||||
if (a.kind !== "entry") throw new Error();
|
||||
expect(a.entry).toEqual({ status: "in_progress", rating: 80, notes: "good" });
|
||||
expect(a.work.externalIds).toEqual({ mangabaka: "1", anilist_manga: "11", mal_manga: "22", mangaupdates: "33", kitsu_manga: "44" });
|
||||
const p = recs[1]!;
|
||||
if (p.kind !== "engagement") throw new Error();
|
||||
expect(p.engagement.progress).toEqual({ chapter: 12, volume: 2 });
|
||||
expect(p.engagement.startedAt).toEqual({ value: "2024-03-05", precision: "day" });
|
||||
expect(p.engagement.finishedAt).toBeNull();
|
||||
});
|
||||
|
||||
test("merged series sets mergedIntoKey", async () => {
|
||||
const { ctx } = ctxFor({ [`GET ${B}/v1/my/library?page=1`]: page1, [`GET ${B}/v1/my/library?page=2`]: page2 });
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
expect(recs.find((r) => r.key === "series:2")!.mergedIntoKey).toBe("series:9");
|
||||
expect(recs.find((r) => r.key === "series:1")!.mergedIntoKey).toBeUndefined();
|
||||
});
|
||||
|
||||
test("pullChanges yields upsert and deletion, saves cursor", async () => {
|
||||
const { h, ctx } = ctxFor({
|
||||
[`GET ${B}/v2/my/library/changes`]: (req: Request) =>
|
||||
new URL(req.url).searchParams.get("cursor") === "cur-2" ? { data: [], next_cursor: "cur-2" } : changes,
|
||||
});
|
||||
await ctx.cursor.set("cur-1");
|
||||
ctx.saved.cursor.length = 0;
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(recs.filter((r) => r.deleted).map((r) => r.key)).toEqual(["series:7", "progress:7"]);
|
||||
expect(recs[0]!.key).toBe("series:1");
|
||||
expect(ctx.saved.cursor).toEqual(["cur-2", "cur-2"]);
|
||||
expect(await ctx.cursor.get()).toBe("cur-2");
|
||||
expect(new URL(h.calls[0]!.url).searchParams.get("cursor")).toBe("cur-1");
|
||||
});
|
||||
|
||||
test("upsert without entry is skipped and logged, never deleted", async () => {
|
||||
const logs: string[] = [];
|
||||
const h = createFixtureHttp({
|
||||
[`GET ${B}/v2/my/library/changes`]: (req: Request) =>
|
||||
new URL(req.url).searchParams.has("cursor") ? { data: [], next_cursor: "c" } : { data: [{ type: "upsert", series_id: 4 }], next_cursor: "c" },
|
||||
});
|
||||
const ctx = createTestContext({ fetch: h.fetch, config: { token: "tok" }, log: (m) => logs.push(m) });
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(recs).toEqual([]);
|
||||
expect(logs).toEqual(["mangabaka: upsert without entry for series 4, skipped"]);
|
||||
});
|
||||
|
||||
test("inherited keys are not states", () => expect(mapState("constructor")).toBe("none"));
|
||||
|
||||
test("delete keys are a subset of keys a full pull produced", async () => {
|
||||
const { ctx } = ctxFor({ [`GET ${B}/v1/my/library?page=1`]: page1, [`GET ${B}/v1/my/library?page=2`]: page2 });
|
||||
const pulled = (await collect(plugin.pull!(ctx))).map((r) => r.key);
|
||||
const del = mapDeletion(1).map((r) => r.key);
|
||||
expect(del).toEqual(["series:1", "progress:1"]);
|
||||
for (const k of del) expect(pulled).toContain(k);
|
||||
});
|
||||
|
||||
test("lookup searches series", async () => {
|
||||
const { h, ctx } = ctxFor({ [`GET ${B}/v1/series/search`]: search });
|
||||
const r = await plugin.lookup!(ctx, { title: "Delta", mediaType: "manga" });
|
||||
expect(r).toEqual([{ title: "Delta", mediaType: "manga", year: 2018, externalIds: { mangabaka: "5", anilist_manga: "55" } }]);
|
||||
expect(new URL(h.calls[0]!.url).searchParams.get("q")).toBe("Delta");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import { definePlugin } from "@sttracker/plugin-sdk/types";
|
||||
import { BASE, clientCreds, post, toCredentials, type TokenResponse } from "./src/api.ts";
|
||||
import { searchHint } from "./src/map.ts";
|
||||
import { pull, pullChanges } from "./src/sync.ts";
|
||||
|
||||
interface DeviceCode { device_code: string; user_code: string; verification_url: string; expires_in: number; interval: number }
|
||||
|
||||
export default definePlugin({
|
||||
manifest: {
|
||||
id: "trakt",
|
||||
name: "Trakt",
|
||||
version: "0.1.0",
|
||||
sdkVersion: 1,
|
||||
mediaTypes: ["movie", "show", "season", "episode"],
|
||||
capabilities: ["pull", "pullChanges", "lookup"],
|
||||
config: [
|
||||
{ key: "clientId", label: "Client ID", type: "string", required: true },
|
||||
{ key: "clientSecret", label: "Client secret", type: "secret", required: true },
|
||||
],
|
||||
attribution: "STTracker uses Trakt but is not endorsed or certified by Trakt.",
|
||||
},
|
||||
deviceAuth: {
|
||||
async start(ctx) {
|
||||
const { clientId } = clientCreds(ctx);
|
||||
const d = await post<DeviceCode>(ctx, "/oauth/device/code", { client_id: clientId });
|
||||
return { userCode: d.user_code, verificationUrl: d.verification_url, deviceCode: d.device_code, intervalSec: d.interval, expiresInSec: d.expires_in };
|
||||
},
|
||||
async poll(ctx, deviceCode) {
|
||||
const { clientId, clientSecret } = clientCreds(ctx);
|
||||
try {
|
||||
const t = await post<TokenResponse>(ctx, "/oauth/device/token", { code: deviceCode, client_id: clientId, client_secret: clientSecret });
|
||||
return toCredentials(t, ctx.now());
|
||||
} catch (e) {
|
||||
// 400 = pending, 429 = polling too fast; anything else (404/409/410/418) is terminal.
|
||||
if (e instanceof HttpError && (e.status === 400 || e.status === 429)) return "pending";
|
||||
throw e;
|
||||
}
|
||||
},
|
||||
},
|
||||
pull,
|
||||
pullChanges,
|
||||
async lookup(ctx, q) {
|
||||
const { clientId } = clientCreds(ctx);
|
||||
const type = q.mediaType === "movie" || q.mediaType === "show" ? q.mediaType : "movie,show";
|
||||
const res = await ctx.http.json<Parameters<typeof searchHint>[0][]>(`${BASE}/search/${type}`, {
|
||||
headers: { "trakt-api-version": "2", "trakt-api-key": clientId },
|
||||
query: { query: q.title, years: q.year },
|
||||
});
|
||||
return res.flatMap((r) => {
|
||||
const h = searchHint(r);
|
||||
return h ? [h] : [];
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { HttpError } from "@sttracker/plugin-sdk/http";
|
||||
import type { PluginContext } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
export const BASE = "https://api.trakt.tv";
|
||||
/** Rate the host should configure for this plugin's HttpClient. */
|
||||
export const RATE_PER_MINUTE = 100;
|
||||
const REFRESH_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
const REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob";
|
||||
|
||||
export interface TokenResponse {
|
||||
access_token: string;
|
||||
refresh_token: string;
|
||||
expires_in: number;
|
||||
created_at?: number;
|
||||
}
|
||||
|
||||
export function clientCreds(ctx: PluginContext): { clientId: string; clientSecret: string } {
|
||||
const clientId = ctx.config.clientId;
|
||||
const clientSecret = ctx.config.clientSecret;
|
||||
if (typeof clientId !== "string" || clientId === "") throw new Error("trakt: clientId is not configured");
|
||||
if (typeof clientSecret !== "string" || clientSecret === "") throw new Error("trakt: clientSecret is not configured");
|
||||
return { clientId, clientSecret };
|
||||
}
|
||||
|
||||
function headers(ctx: PluginContext, accessToken?: string, body = false): Headers {
|
||||
const h = new Headers({ "trakt-api-version": "2", "trakt-api-key": clientCreds(ctx).clientId });
|
||||
if (body) h.set("content-type", "application/json");
|
||||
if (accessToken) h.set("authorization", `Bearer ${accessToken}`);
|
||||
return h;
|
||||
}
|
||||
|
||||
export function toCredentials(t: TokenResponse, now: Date): Record<string, string> {
|
||||
const createdMs = typeof t.created_at === "number" ? t.created_at * 1000 : now.getTime();
|
||||
return {
|
||||
accessToken: t.access_token,
|
||||
refreshToken: t.refresh_token,
|
||||
expiresAt: new Date(createdMs + t.expires_in * 1000).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export async function post<T>(ctx: PluginContext, path: string, body: unknown): Promise<T> {
|
||||
return ctx.http.json<T>(BASE + path, { method: "POST", headers: headers(ctx, undefined, true), body: JSON.stringify(body) });
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a usable access token. Refreshes when it expires within 24 h; Trakt refresh tokens are
|
||||
* single-use, so the rotated credentials are saved before this returns (before any further request).
|
||||
*/
|
||||
export async function ensureAccessToken(ctx: PluginContext): Promise<string> {
|
||||
const creds = await ctx.credentials.get();
|
||||
if (!creds?.accessToken) throw new Error("trakt: not connected (no credentials)");
|
||||
const expiresMs = Date.parse(creds.expiresAt ?? "");
|
||||
if (!Number.isNaN(expiresMs) && expiresMs - ctx.now().getTime() > REFRESH_WINDOW_MS) return creds.accessToken;
|
||||
if (!creds.refreshToken) throw new Error("trakt: access token expiring and no refresh token");
|
||||
const { clientId, clientSecret } = clientCreds(ctx);
|
||||
const t = await post<TokenResponse>(ctx, "/oauth/token", {
|
||||
refresh_token: creds.refreshToken,
|
||||
client_id: clientId,
|
||||
client_secret: clientSecret,
|
||||
redirect_uri: REDIRECT_URI,
|
||||
grant_type: "refresh_token",
|
||||
});
|
||||
await ctx.credentials.save(toCredentials(t, ctx.now()));
|
||||
return t.access_token;
|
||||
}
|
||||
|
||||
/** Authenticated GET. Returns the parsed body plus the X-Pagination-Page-Count header (1 if absent). */
|
||||
export async function get<T>(
|
||||
ctx: PluginContext,
|
||||
token: string,
|
||||
path: string,
|
||||
query: Record<string, string | number | undefined> = {},
|
||||
): Promise<{ data: T; pageCount: number }> {
|
||||
const u = new URL(BASE + path);
|
||||
for (const [k, v] of Object.entries(query)) if (v !== undefined) u.searchParams.set(k, String(v));
|
||||
const h = headers(ctx, token);
|
||||
h.set("accept", "application/json");
|
||||
const res = await ctx.http.raw(u.toString(), { headers: h });
|
||||
if (!res.ok) throw new HttpError(res.status, await res.text(), u.toString());
|
||||
const pc = Number(res.headers.get("x-pagination-page-count"));
|
||||
return { data: (await res.json()) as T, pageCount: Number.isFinite(pc) && pc >= 1 ? pc : 1 };
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { parsePartialDate } from "@sttracker/plugin-sdk/partial-date";
|
||||
import type { MediaType, SourceRecord, WorkHint } from "@sttracker/plugin-sdk/types";
|
||||
|
||||
interface Ids { trakt?: number; slug?: string; imdb?: string | null; tmdb?: number | null; tvdb?: number | null }
|
||||
interface TMovie { title: string; year?: number | null; ids: Ids }
|
||||
interface TShow { title: string; year?: number | null; ids: Ids }
|
||||
interface TSeason { number: number; ids?: Ids }
|
||||
interface TEpisode { season: number; number: number; title?: string | null; ids?: Ids }
|
||||
export interface TItem {
|
||||
type: string;
|
||||
id?: number;
|
||||
watched_at?: string;
|
||||
rating?: number;
|
||||
notes?: string | null;
|
||||
movie?: TMovie;
|
||||
show?: TShow;
|
||||
season?: TSeason;
|
||||
episode?: TEpisode;
|
||||
}
|
||||
|
||||
function ids(map: Record<string, unknown>): Record<string, string> {
|
||||
const out: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(map)) if (v !== undefined && v !== null && v !== "") out[k] = String(v);
|
||||
return out;
|
||||
}
|
||||
|
||||
function movieHint(m: TMovie): WorkHint {
|
||||
return {
|
||||
title: m.title,
|
||||
mediaType: "movie",
|
||||
...(m.year ? { year: m.year } : {}),
|
||||
externalIds: ids({ trakt_movie: m.ids.trakt, tmdb_movie: m.ids.tmdb, imdb: m.ids.imdb }),
|
||||
};
|
||||
}
|
||||
|
||||
function showHint(s: TShow): WorkHint {
|
||||
return {
|
||||
title: s.title,
|
||||
mediaType: "show",
|
||||
...(s.year ? { year: s.year } : {}),
|
||||
externalIds: ids({ trakt_show: s.ids.trakt, tmdb_show: s.ids.tmdb, tvdb_show: s.ids.tvdb, imdb: s.ids.imdb }),
|
||||
};
|
||||
}
|
||||
|
||||
function seasonHint(show: TShow, n: number, sid?: Ids): WorkHint {
|
||||
return {
|
||||
title: `Season ${n}`,
|
||||
mediaType: "season",
|
||||
externalIds: sid ? ids({ trakt_season: sid.trakt, tmdb_season: sid.tmdb }) : {},
|
||||
parent: showHint(show),
|
||||
position: { season: n },
|
||||
};
|
||||
}
|
||||
|
||||
function episodeHint(show: TShow, e: TEpisode): WorkHint {
|
||||
const eid = e.ids;
|
||||
return {
|
||||
title: e.title || `Episode ${e.number}`,
|
||||
mediaType: "episode",
|
||||
externalIds: eid ? ids({ trakt_episode: eid.trakt, tvdb_episode: eid.tvdb, tmdb_episode: eid.tmdb, imdb: eid.imdb }) : {},
|
||||
parent: seasonHint(show, e.season),
|
||||
position: { season: e.season, episode: e.number },
|
||||
};
|
||||
}
|
||||
|
||||
/** Maps a Trakt item (history, rating, watchlist or list item) to a WorkHint, or null if unsupported. */
|
||||
export function hintFor(item: TItem): WorkHint | null {
|
||||
switch (item.type) {
|
||||
case "movie": return item.movie ? movieHint(item.movie) : null;
|
||||
case "show": return item.show ? showHint(item.show) : null;
|
||||
case "season": return item.show && item.season ? seasonHint(item.show, item.season.number, item.season.ids) : null;
|
||||
case "episode": return item.show && item.episode ? episodeHint(item.show, item.episode) : null;
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function traktIdOf(item: TItem): number | undefined {
|
||||
switch (item.type) {
|
||||
case "movie": return item.movie?.ids.trakt;
|
||||
case "show": return item.show?.ids.trakt;
|
||||
case "season": return item.season?.ids?.trakt;
|
||||
case "episode": return item.episode?.ids?.trakt;
|
||||
default: return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function historyRecord(item: TItem): SourceRecord | null {
|
||||
const work = hintFor(item);
|
||||
if (!work || item.id === undefined) return null;
|
||||
const finishedAt = parsePartialDate(item.watched_at);
|
||||
return { kind: "engagement", key: `history:${item.id}`, raw: item, work, engagement: { finishedAt } };
|
||||
}
|
||||
|
||||
export function ratingRecord(item: TItem): SourceRecord | null {
|
||||
const work = hintFor(item);
|
||||
const tid = traktIdOf(item);
|
||||
if (!work || tid === undefined || typeof item.rating !== "number") return null;
|
||||
return { kind: "entry", key: `rating:${item.type}:${tid}`, raw: item, work, entry: { rating: item.rating * 10 } };
|
||||
}
|
||||
|
||||
export function watchlistRecord(item: TItem): SourceRecord | null {
|
||||
const work = hintFor(item);
|
||||
const tid = traktIdOf(item);
|
||||
if (!work || tid === undefined) return null;
|
||||
return { kind: "entry", key: `watchlist:${item.type}:${tid}`, raw: item, work, entry: { status: "planned" } };
|
||||
}
|
||||
|
||||
export interface TList { name: string; description?: string | null; ids: Ids }
|
||||
|
||||
export function listRecord(list: TList, items: TItem[]): SourceRecord {
|
||||
const mapped: { work: WorkHint; note?: string }[] = [];
|
||||
for (const it of items) {
|
||||
const work = hintFor(it);
|
||||
if (work) mapped.push({ work, ...(it.notes ? { note: it.notes } : {}) });
|
||||
}
|
||||
return {
|
||||
kind: "list",
|
||||
key: `list:${list.ids.trakt}`,
|
||||
raw: { list, items },
|
||||
list: { name: list.name, ...(list.description ? { description: list.description } : {}), items: mapped },
|
||||
};
|
||||
}
|
||||
|
||||
export function searchHint(r: { type: string; movie?: TMovie; show?: TShow }): WorkHint | null {
|
||||
return hintFor({ type: r.type, ...(r.movie ? { movie: r.movie } : {}), ...(r.show ? { show: r.show } : {}) });
|
||||
}
|
||||
|
||||
export type { MediaType };
|
||||
@@ -0,0 +1,92 @@
|
||||
import type { PluginContext, SourceRecord } from "@sttracker/plugin-sdk/types";
|
||||
import { ensureAccessToken, get } from "./api.ts";
|
||||
import { historyRecord, listRecord, ratingRecord, watchlistRecord, type TItem, type TList } from "./map.ts";
|
||||
|
||||
const PAGE_LIMIT = 250;
|
||||
|
||||
async function* history(ctx: PluginContext, token: string, startAt?: string): AsyncGenerator<SourceRecord> {
|
||||
for (let page = 1, pages = 1; page <= pages; page++) {
|
||||
const r = await get<TItem[]>(ctx, token, "/sync/history", { page, limit: PAGE_LIMIT, start_at: startAt });
|
||||
pages = r.pageCount;
|
||||
for (const item of r.data) {
|
||||
const rec = historyRecord(item);
|
||||
if (rec) yield rec;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function* ratings(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
|
||||
for (const item of (await get<TItem[]>(ctx, token, "/sync/ratings")).data) {
|
||||
const rec = ratingRecord(item);
|
||||
if (rec) yield rec;
|
||||
}
|
||||
}
|
||||
|
||||
async function* watchlist(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
|
||||
for (const item of (await get<TItem[]>(ctx, token, "/sync/watchlist")).data) {
|
||||
const rec = watchlistRecord(item);
|
||||
if (rec) yield rec;
|
||||
}
|
||||
}
|
||||
|
||||
async function* lists(ctx: PluginContext, token: string): AsyncGenerator<SourceRecord> {
|
||||
for (const list of (await get<TList[]>(ctx, token, "/users/me/lists")).data) {
|
||||
const items = (await get<TItem[]>(ctx, token, `/users/me/lists/${list.ids.trakt}/items`)).data;
|
||||
yield listRecord(list, items);
|
||||
}
|
||||
}
|
||||
|
||||
export async function* pull(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const token = await ensureAccessToken(ctx);
|
||||
yield* history(ctx, token);
|
||||
yield* ratings(ctx, token);
|
||||
yield* watchlist(ctx, token);
|
||||
yield* lists(ctx, token);
|
||||
}
|
||||
|
||||
interface Activity { watched_at?: string | null; rated_at?: string | null; watchlisted_at?: string | null; updated_at?: string | null }
|
||||
interface LastActivities { movies?: Activity; episodes?: Activity; shows?: Activity; seasons?: Activity; lists?: Activity }
|
||||
interface Cursor { history: string; ratings: string; watchlist: string; lists: string }
|
||||
|
||||
const maxIso = (xs: (string | null | undefined)[]): string =>
|
||||
xs.reduce<string>((a, x) => (x && x > a ? x : a), "");
|
||||
|
||||
function cursorFrom(a: LastActivities): Cursor {
|
||||
return {
|
||||
history: maxIso([a.movies?.watched_at, a.episodes?.watched_at]),
|
||||
ratings: maxIso([a.movies?.rated_at, a.episodes?.rated_at, a.shows?.rated_at, a.seasons?.rated_at]),
|
||||
watchlist: maxIso([a.movies?.watchlisted_at, a.episodes?.watchlisted_at, a.shows?.watchlisted_at, a.seasons?.watchlisted_at]),
|
||||
lists: maxIso([a.lists?.updated_at]),
|
||||
};
|
||||
}
|
||||
|
||||
function parseCursor(s: string | null): Cursor | null {
|
||||
if (!s) return null;
|
||||
try {
|
||||
const c = JSON.parse(s) as Partial<Cursor>;
|
||||
return { history: c.history ?? "", ratings: c.ratings ?? "", watchlist: c.watchlist ?? "", lists: c.lists ?? "" };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function* pullChanges(ctx: PluginContext): AsyncGenerator<SourceRecord> {
|
||||
const token = await ensureAccessToken(ctx);
|
||||
const raw = await ctx.cursor.get();
|
||||
const prev = parseCursor(raw);
|
||||
const next = cursorFrom((await get<LastActivities>(ctx, token, "/sync/last_activities")).data);
|
||||
if (!prev) {
|
||||
// No usable cursor: full pull, then remember where we are.
|
||||
yield* history(ctx, token);
|
||||
yield* ratings(ctx, token);
|
||||
yield* watchlist(ctx, token);
|
||||
yield* lists(ctx, token);
|
||||
} else {
|
||||
if (next.history !== prev.history) yield* history(ctx, token, prev.history || undefined);
|
||||
if (next.ratings !== prev.ratings) yield* ratings(ctx, token);
|
||||
if (next.watchlist !== prev.watchlist) yield* watchlist(ctx, token);
|
||||
if (next.lists !== prev.lists) yield* lists(ctx, token);
|
||||
}
|
||||
const encoded = JSON.stringify(next);
|
||||
if (encoded !== raw) await ctx.cursor.set(encoded);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{ "device_code": "dev123", "user_code": "ABCD1234", "verification_url": "https://trakt.tv/activate", "expires_in": 600, "interval": 5 }
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
[
|
||||
{
|
||||
"id": 9001, "watched_at": "2024-03-05T20:15:00.000Z", "action": "watch", "type": "movie",
|
||||
"movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } }
|
||||
},
|
||||
{
|
||||
"id": 9002, "watched_at": "2024-03-06T21:30:00.000Z", "action": "watch", "type": "episode",
|
||||
"episode": { "season": 2, "number": 3, "title": "The Heist", "ids": { "trakt": 5003, "tvdb": 7003, "imdb": "tt0000003", "tmdb": 8003 } },
|
||||
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{
|
||||
"id": 9003, "watched_at": "2023-12-31T23:59:59.000Z", "action": "watch", "type": "movie",
|
||||
"movie": { "title": "Heat", "year": 1995, "ids": { "trakt": 1003, "slug": "heat-1995", "imdb": "tt0113277", "tmdb": 949 } }
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"all": "2024-03-07T10:00:00.000Z",
|
||||
"movies": { "watched_at": "2024-03-05T20:15:00.000Z", "collected_at": null, "rated_at": "2024-03-05T22:00:00.000Z", "watchlisted_at": "2024-02-01T08:00:00.000Z", "favorited_at": null, "commented_at": null, "paused_at": null, "hidden_at": null },
|
||||
"episodes": { "watched_at": "2024-03-06T21:30:00.000Z", "collected_at": null, "rated_at": null, "watchlisted_at": null, "commented_at": null, "paused_at": null },
|
||||
"shows": { "rated_at": "2024-03-07T10:00:00.000Z", "watchlisted_at": null, "favorited_at": null, "commented_at": null, "hidden_at": null },
|
||||
"seasons": { "rated_at": null, "watchlisted_at": null, "commented_at": null, "hidden_at": null },
|
||||
"lists": { "liked_at": null, "updated_at": "2024-01-02T00:00:00.000Z" }
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"rank": 1, "id": 1, "listed_at": "2024-01-01T00:00:00.000Z", "notes": "rewatch yearly", "type": "movie",
|
||||
"movie": { "title": "Heat", "year": 1995, "ids": { "trakt": 1003, "slug": "heat-1995", "imdb": "tt0113277", "tmdb": 949 } }
|
||||
},
|
||||
{
|
||||
"rank": 2, "id": 2, "listed_at": "2024-01-02T00:00:00.000Z", "notes": null, "type": "show",
|
||||
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
|
||||
}
|
||||
]
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
[
|
||||
{ "name": "Favourites", "description": "All-time best", "privacy": "private", "item_count": 2, "ids": { "trakt": 7777, "slug": "favourites" } }
|
||||
]
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
[
|
||||
{
|
||||
"rated_at": "2024-03-05T22:00:00.000Z", "rating": 8, "type": "movie",
|
||||
"movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } }
|
||||
},
|
||||
{
|
||||
"rated_at": "2024-03-07T10:00:00.000Z", "rating": 10, "type": "show",
|
||||
"show": { "title": "Great Show", "year": 2019, "ids": { "trakt": 2002, "slug": "great-show", "tvdb": 70002, "imdb": "tt7000002", "tmdb": 80002 } }
|
||||
}
|
||||
]
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{ "type": "movie", "score": 90.1, "movie": { "title": "Arrival", "year": 2016, "ids": { "trakt": 1001, "slug": "arrival-2016", "imdb": "tt2543164", "tmdb": 329865 } } },
|
||||
{ "type": "show", "score": 40.2, "show": { "title": "Arrival Show", "year": 2020, "ids": { "trakt": 2010, "slug": "arrival-show", "tvdb": 71010, "imdb": "tt7100010", "tmdb": 81010 } } }
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
{ "access_token": "new-access", "token_type": "bearer", "expires_in": 7776000, "refresh_token": "new-refresh", "scope": "public", "created_at": 1772000000 }
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{
|
||||
"rank": 1, "id": 333, "listed_at": "2024-02-01T08:00:00.000Z", "notes": null, "type": "movie",
|
||||
"movie": { "title": "Dune: Part Two", "year": 2024, "ids": { "trakt": 1004, "slug": "dune-part-two-2024", "imdb": "tt15239678", "tmdb": 693134 } }
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,226 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { collect, createFixtureHttp, createTestContext } from "@sttracker/plugin-sdk/testing";
|
||||
import type { PluginContext } from "@sttracker/plugin-sdk/types";
|
||||
import plugin from "../index.ts";
|
||||
import history1 from "./fixtures/history-page1.json";
|
||||
import history2 from "./fixtures/history-page2.json";
|
||||
import ratings from "./fixtures/ratings.json";
|
||||
import watchlist from "./fixtures/watchlist.json";
|
||||
import lists from "./fixtures/lists.json";
|
||||
import listItems from "./fixtures/list-items.json";
|
||||
import lastActivities from "./fixtures/last-activities.json";
|
||||
import tokenRefresh from "./fixtures/token-refresh.json";
|
||||
import deviceCode from "./fixtures/device-code.json";
|
||||
import search from "./fixtures/search.json";
|
||||
|
||||
const B = "https://api.trakt.tv";
|
||||
const NOW = new Date("2024-04-01T00:00:00.000Z");
|
||||
const FRESH = { accessToken: "old-access", refreshToken: "old-refresh", expiresAt: "2024-06-01T00:00:00.000Z" };
|
||||
|
||||
const paged = (req: Request) => {
|
||||
const page = new URL(req.url).searchParams.get("page");
|
||||
return Response.json(page === "2" ? history2 : history1, { headers: { "X-Pagination-Page-Count": "2" } });
|
||||
};
|
||||
|
||||
function setup(routes: Record<string, unknown>, creds: Record<string, string> | null = FRESH, cursor?: string) {
|
||||
const fx = createFixtureHttp(routes);
|
||||
const ctx = createTestContext({ fetch: fx.fetch, now: () => NOW, config: { clientId: "cid", clientSecret: "csec" } });
|
||||
return (async () => {
|
||||
if (creds) await ctx.credentials.save(creds);
|
||||
ctx.saved.credentials.length = 0;
|
||||
if (cursor) await ctx.cursor.set(cursor);
|
||||
ctx.saved.cursor.length = 0;
|
||||
return { fx, ctx };
|
||||
})();
|
||||
}
|
||||
|
||||
const fullRoutes = {
|
||||
[`GET ${B}/sync/history`]: paged,
|
||||
[`GET ${B}/sync/ratings`]: ratings,
|
||||
[`GET ${B}/sync/watchlist`]: watchlist,
|
||||
[`GET ${B}/users/me/lists`]: lists,
|
||||
[`GET ${B}/users/me/lists/7777/items`]: listItems,
|
||||
};
|
||||
|
||||
describe("trakt pull", () => {
|
||||
test("history pages through X-Pagination-Page-Count", async () => {
|
||||
const { fx, ctx } = await setup(fullRoutes);
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
const hist = recs.filter((r) => r.key.startsWith("history:"));
|
||||
expect(hist.map((r) => r.key)).toEqual(["history:9001", "history:9002", "history:9003"]);
|
||||
const pages = fx.calls.filter((c) => c.url.includes("/sync/history")).map((c) => new URL(c.url).searchParams.get("page"));
|
||||
expect(pages).toEqual(["1", "2"]);
|
||||
expect(new URL(fx.calls[0]!.url).searchParams.get("limit")).toBe("250");
|
||||
expect(fx.calls[0]!.headers.get("trakt-api-version")).toBe("2");
|
||||
expect(fx.calls[0]!.headers.get("trakt-api-key")).toBe("cid");
|
||||
expect(fx.calls[0]!.headers.get("authorization")).toBe("Bearer old-access");
|
||||
});
|
||||
|
||||
test("movie history: day precision, full datetime kept, ids", async () => {
|
||||
const { ctx } = await setup(fullRoutes);
|
||||
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "history:9001")!;
|
||||
if (r.kind !== "engagement") throw new Error("kind");
|
||||
expect(r.work.mediaType).toBe("movie");
|
||||
expect(r.work.externalIds).toEqual({ trakt_movie: "1001", tmdb_movie: "329865", imdb: "tt2543164" });
|
||||
expect(r.engagement.finishedAt).toEqual({ value: "2024-03-05T20:15:00.000Z", precision: "day" });
|
||||
});
|
||||
|
||||
test("episode history: episode -> season -> show chain with position", async () => {
|
||||
const { ctx } = await setup(fullRoutes);
|
||||
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "history:9002")!;
|
||||
if (r.kind !== "engagement") throw new Error("kind");
|
||||
expect(r.work.mediaType).toBe("episode");
|
||||
expect(r.work.position).toEqual({ season: 2, episode: 3 });
|
||||
expect(r.work.externalIds).toEqual({ trakt_episode: "5003", tvdb_episode: "7003", tmdb_episode: "8003", imdb: "tt0000003" });
|
||||
expect(r.work.parent?.mediaType).toBe("season");
|
||||
expect(r.work.parent?.position).toEqual({ season: 2 });
|
||||
const show = r.work.parent?.parent;
|
||||
expect(show?.mediaType).toBe("show");
|
||||
expect(show?.externalIds).toEqual({ trakt_show: "2002", tmdb_show: "80002", tvdb_show: "70002", imdb: "tt7000002" });
|
||||
});
|
||||
|
||||
test("rating 8 becomes 80", async () => {
|
||||
const { ctx } = await setup(fullRoutes);
|
||||
const recs = await collect(plugin.pull!(ctx));
|
||||
const m = recs.find((x) => x.key === "rating:movie:1001")!;
|
||||
const s = recs.find((x) => x.key === "rating:show:2002")!;
|
||||
if (m.kind !== "entry" || s.kind !== "entry") throw new Error("kind");
|
||||
expect(m.entry.rating).toBe(80);
|
||||
expect(s.entry.rating).toBe(100);
|
||||
});
|
||||
|
||||
test("watchlist becomes planned", async () => {
|
||||
const { ctx } = await setup(fullRoutes);
|
||||
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "watchlist:movie:1004")!;
|
||||
if (r.kind !== "entry") throw new Error("kind");
|
||||
expect(r.entry.status).toBe("planned");
|
||||
expect(r.work.title).toBe("Dune: Part Two");
|
||||
});
|
||||
|
||||
test("custom lists with items and notes", async () => {
|
||||
const { ctx } = await setup(fullRoutes);
|
||||
const r = (await collect(plugin.pull!(ctx))).find((x) => x.key === "list:7777")!;
|
||||
if (r.kind !== "list") throw new Error("kind");
|
||||
expect(r.list.name).toBe("Favourites");
|
||||
expect(r.list.items.map((i) => i.work.title)).toEqual(["Heat", "Great Show"]);
|
||||
expect(r.list.items[0]!.note).toBe("rewatch yearly");
|
||||
});
|
||||
|
||||
test("a token near expiry is refreshed and saved before the first /sync call", async () => {
|
||||
let savedWhenSyncCalled = -1;
|
||||
const routes = {
|
||||
...fullRoutes,
|
||||
[`POST ${B}/oauth/token`]: () => tokenRefresh,
|
||||
[`GET ${B}/sync/history`]: (req: Request) => {
|
||||
if (savedWhenSyncCalled < 0) savedWhenSyncCalled = ctxRef.saved.credentials.length;
|
||||
return paged(req);
|
||||
},
|
||||
};
|
||||
const { fx, ctx } = await setup(routes, { ...FRESH, expiresAt: "2024-04-01T12:00:00.000Z" });
|
||||
ctxRef = ctx;
|
||||
await collect(plugin.pull!(ctx));
|
||||
expect(fx.calls[0]!.method).toBe("POST");
|
||||
expect(await fx.calls[0]!.json()).toEqual({
|
||||
refresh_token: "old-refresh", client_id: "cid", client_secret: "csec",
|
||||
redirect_uri: "urn:ietf:wg:oauth:2.0:oob", grant_type: "refresh_token",
|
||||
});
|
||||
expect(new URL(fx.calls[0]!.url).pathname).toBe("/oauth/token");
|
||||
expect(new URL(fx.calls[1]!.url).pathname).toBe("/sync/history");
|
||||
expect(savedWhenSyncCalled).toBe(1);
|
||||
expect(ctx.saved.credentials).toEqual([
|
||||
{ accessToken: "new-access", refreshToken: "new-refresh", expiresAt: new Date((1772000000 + 7776000) * 1000).toISOString() },
|
||||
]);
|
||||
expect(fx.calls[1]!.headers.get("authorization")).toBe("Bearer new-access");
|
||||
});
|
||||
|
||||
test("a failed refresh saves nothing and never calls /sync", async () => {
|
||||
const { fx, ctx } = await setup(
|
||||
{ ...fullRoutes, [`POST ${B}/oauth/token`]: () => new Response("{}", { status: 400 }) },
|
||||
{ ...FRESH, expiresAt: "2024-04-01T12:00:00.000Z" },
|
||||
);
|
||||
await expect(collect(plugin.pull!(ctx))).rejects.toThrow("400");
|
||||
expect(ctx.saved.credentials).toEqual([]);
|
||||
expect(fx.calls.map((c) => new URL(c.url).pathname)).toEqual(["/oauth/token"]);
|
||||
});
|
||||
|
||||
test("a token with more than 24h left is not refreshed", async () => {
|
||||
const { fx, ctx } = await setup(fullRoutes);
|
||||
await collect(plugin.pull!(ctx));
|
||||
expect(fx.calls.some((c) => c.url.includes("/oauth/token"))).toBe(false);
|
||||
expect(ctx.saved.credentials).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
let ctxRef: ReturnType<typeof createTestContext>;
|
||||
|
||||
describe("trakt pullChanges", () => {
|
||||
const cursorOf = (o: object) => JSON.stringify(o);
|
||||
const same = {
|
||||
history: "2024-03-06T21:30:00.000Z",
|
||||
ratings: "2024-03-07T10:00:00.000Z",
|
||||
watchlist: "2024-02-01T08:00:00.000Z",
|
||||
lists: "2024-01-02T00:00:00.000Z",
|
||||
};
|
||||
const routes = { ...fullRoutes, [`GET ${B}/sync/last_activities`]: lastActivities };
|
||||
|
||||
test("unchanged last_activities yields nothing and fetches nothing else", async () => {
|
||||
const { fx, ctx } = await setup(routes, FRESH, cursorOf(same));
|
||||
expect(await collect(plugin.pullChanges!(ctx))).toEqual([]);
|
||||
expect(fx.calls.map((c) => new URL(c.url).pathname)).toEqual(["/sync/last_activities"]);
|
||||
expect(ctx.saved.cursor).toEqual([]);
|
||||
});
|
||||
|
||||
test("only changed categories are re-fetched; history uses start_at; cursor advances", async () => {
|
||||
const prev = { ...same, history: "2024-03-01T00:00:00.000Z", watchlist: "2024-01-01T00:00:00.000Z" };
|
||||
const { fx, ctx } = await setup(routes, FRESH, cursorOf(prev));
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(recs.some((r) => r.key.startsWith("history:"))).toBe(true);
|
||||
expect(recs.some((r) => r.key.startsWith("watchlist:"))).toBe(true);
|
||||
expect(recs.some((r) => r.key.startsWith("rating:"))).toBe(false);
|
||||
const h = fx.calls.find((c) => c.url.includes("/sync/history"))!;
|
||||
expect(new URL(h.url).searchParams.get("start_at")).toBe("2024-03-01T00:00:00.000Z");
|
||||
expect(fx.calls.some((c) => c.url.includes("/sync/ratings"))).toBe(false);
|
||||
expect(ctx.saved.cursor).toEqual([cursorOf(same)]);
|
||||
});
|
||||
|
||||
test("no cursor: full pull and cursor set", async () => {
|
||||
const { ctx } = await setup(routes);
|
||||
const recs = await collect(plugin.pullChanges!(ctx));
|
||||
expect(recs.length).toBeGreaterThan(5);
|
||||
expect(ctx.saved.cursor).toEqual([cursorOf(same)]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("trakt auth and lookup", () => {
|
||||
test("manifest", () => {
|
||||
expect(plugin.manifest.id).toBe("trakt");
|
||||
expect(plugin.manifest.capabilities).toEqual(["pull", "pullChanges", "lookup"]);
|
||||
expect(plugin.manifest.attribution).toBe("STTracker uses Trakt but is not endorsed or certified by Trakt.");
|
||||
});
|
||||
|
||||
test("device auth start and poll", async () => {
|
||||
let n = 0;
|
||||
const { ctx } = await setup({
|
||||
[`POST ${B}/oauth/device/code`]: deviceCode,
|
||||
[`POST ${B}/oauth/device/token`]: () => (n++ === 0 ? new Response("", { status: 400 }) : tokenRefresh),
|
||||
}, null);
|
||||
const start = await plugin.deviceAuth!.start(ctx);
|
||||
expect(start).toEqual({ userCode: "ABCD1234", verificationUrl: "https://trakt.tv/activate", deviceCode: "dev123", intervalSec: 5, expiresInSec: 600 });
|
||||
expect(await plugin.deviceAuth!.poll(ctx, "dev123")).toBe("pending");
|
||||
expect(await plugin.deviceAuth!.poll(ctx, "dev123")).toEqual({
|
||||
accessToken: "new-access", refreshToken: "new-refresh", expiresAt: new Date((1772000000 + 7776000) * 1000).toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
test("device auth poll throws on terminal errors", async () => {
|
||||
const { ctx } = await setup({ [`POST ${B}/oauth/device/token`]: () => new Response("", { status: 418 }) }, null);
|
||||
await expect(plugin.deviceAuth!.poll(ctx as PluginContext, "x")).rejects.toThrow("418");
|
||||
});
|
||||
|
||||
test("lookup searches movie,show", async () => {
|
||||
const { fx, ctx } = await setup({ [`GET ${B}/search/movie,show`]: search }, null);
|
||||
const hints = await plugin.lookup!(ctx, { title: "Arrival" });
|
||||
expect(new URL(fx.calls[0]!.url).searchParams.get("query")).toBe("Arrival");
|
||||
expect(hints.map((h) => [h.mediaType, h.title])).toEqual([["movie", "Arrival"], ["show", "Arrival Show"]]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user