From ad192365a11dafce825486e3f2a1cf7f6d48e56c Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 26 Sep 2026 11:15:32 +0200 Subject: [PATCH] feat: add authenticated HTTP service --- docs/ARCHITECTURE.md | 12 ++- docs/DECISIONS.md | 12 +++ docs/TODO.md | 17 ++-- docs/plans/001-search-engine-api.md | 12 +-- src/recoll_modern/api/__init__.py | 1 + src/recoll_modern/api/auth.py | 45 ++++++++++ src/recoll_modern/api/collections.py | 82 ++++++++++++++++++ src/recoll_modern/api/dependencies.py | 21 +++++ src/recoll_modern/api/errors.py | 51 ++++++++++++ src/recoll_modern/api/files.py | 43 ++++++++++ src/recoll_modern/api/health.py | 14 ++++ src/recoll_modern/api/middleware.py | 59 +++++++++++++ src/recoll_modern/api/schemas.py | 15 ++++ src/recoll_modern/app.py | 105 ++++++++++++++++++++++-- src/recoll_modern/config.py | 3 + src/recoll_modern/domain/auth.py | 9 ++ src/recoll_modern/storage/auth.py | 7 ++ src/recoll_modern/storage/membership.py | 8 ++ tests/api/__init__.py | 1 + tests/api/conftest.py | 47 +++++++++++ tests/api/test_auth.py | 46 +++++++++++ tests/api/test_collections.py | 48 +++++++++++ tests/api/test_errors.py | 37 +++++++++ tests/api/test_files.py | 23 ++++++ tests/api/test_health.py | 29 +++++++ tests/api/test_root_path.py | 37 +++++++++ tests/domain/__init__.py | 1 + tests/helper/__init__.py | 1 + tests/indexing/__init__.py | 1 + tests/storage/__init__.py | 1 + tests/unit/__init__.py | 1 + 31 files changed, 768 insertions(+), 21 deletions(-) create mode 100644 src/recoll_modern/api/__init__.py create mode 100644 src/recoll_modern/api/auth.py create mode 100644 src/recoll_modern/api/collections.py create mode 100644 src/recoll_modern/api/dependencies.py create mode 100644 src/recoll_modern/api/errors.py create mode 100644 src/recoll_modern/api/files.py create mode 100644 src/recoll_modern/api/health.py create mode 100644 src/recoll_modern/api/middleware.py create mode 100644 src/recoll_modern/api/schemas.py create mode 100644 tests/api/__init__.py create mode 100644 tests/api/conftest.py create mode 100644 tests/api/test_auth.py create mode 100644 tests/api/test_collections.py create mode 100644 tests/api/test_errors.py create mode 100644 tests/api/test_files.py create mode 100644 tests/api/test_health.py create mode 100644 tests/api/test_root_path.py create mode 100644 tests/domain/__init__.py create mode 100644 tests/helper/__init__.py create mode 100644 tests/indexing/__init__.py create mode 100644 tests/storage/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 07f3461..ce81845 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,8 +5,9 @@ How `recoll-modern` is built today. Update in the same commit as any structural ## Overview The Python service scaffold, SQLite schema, collection and membership repositories, document -discovery/identity modules, durable job worker, engine adapter, and authentication service are built. -The HTTP API remains intended structure. Recoll behaviour below was verified against a real container. +discovery/identity modules, durable job worker, engine adapter, authentication service, and initial +HTTP shell are built. Membership, search and job resource routes remain to be added. Recoll behaviour +below was verified against a real container. The goal: a self-hosted service that wraps Recoll's indexing and full-text search behind (a) a modern web UI and (b) an MCP server, running in Docker, with several independent collections such as @@ -30,6 +31,7 @@ web UI and (b) an MCP server, running in Docker, with several independent collec | `helper/` | C reader run by Recoll's `metadatacmds` for each indexed file | | `src/recoll_modern/engine/` | Shared Protocol, deterministic fake, and isolated Recoll Python/indexer adapter | | `src/recoll_modern/domain/auth.py`, `storage/auth.py` | Argon2 password checks, seeded administrator, hashed server-side sessions and CSRF verification | +| `src/recoll_modern/api/` | HTTP auth, health, file browsing, collection metadata, prefix/proxy handling and JSON errors | | `tests/` | Automated service tests | Discovery produces content hashes and root-relative paths without writing to the document volume. @@ -96,6 +98,12 @@ contract in `docs/specs/001-search-engine-api.md`. ### HTTP boundary +The app lifespan creates state, migrates SQLite, seeds the first administrator, exports the initial +membership map, recovers interrupted jobs, reconciles persistent rules, and starts/stops one worker. +Core routes use a server-side session cookie and CSRF token; `GET /auth/session` rotates the token so +the database only stores hashes. Forwarded scheme, host and prefix are applied only for configured +proxy networks. + The resource groups are auth, health, read-only file browsing, collections and membership sources, directory rules, scoped search, document text/original retrieval, and indexing jobs. Search callers select returned fields; only `document_id` is unconditional, and inline full text is bounded. Every diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index b80aa73..91d6f7e 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -289,3 +289,15 @@ is Xapian's raw weight. undocumented Recoll binding attribute for score. **Consequences:** unusual filenames do not enter a query parser. Clients can compare score order within one query, but the value is a rank proxy. Revisit this once the real Recoll container can be tested. + +--- + +## 2026-09-26 — Rotate CSRF tokens on session reads +**Status:** accepted +**Context:** sessions store only SHA-256 token digests, but `GET /auth/session` must return a usable CSRF +token after a page reload. The raw token cannot be recovered from SQLite. +**Decision:** authenticate the session, replace its stored CSRF digest, and return a newly generated +token. The previous token becomes invalid immediately. +**Alternatives:** store the raw CSRF token; derive a repeatable token from the cookie and a server key. +**Consequences:** clients should refresh their CSRF token after calling the session endpoint. Session +storage contains no raw cookie or CSRF token. diff --git a/docs/TODO.md b/docs/TODO.md index 33fe2fe..e7f61b2 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -15,12 +15,12 @@ Where the workflow stands (`AGENTS.md` -> Workflow: brainstorm -> spec -> plan - selectable search fields, subpath hosting and the packaging seam for deferred desktop spec 005. - Plan `docs/plans/001-search-engine-api.md`: **approved 2026-09-18.** Its listed Python runtime and development dependencies are approved. Execution follows the plan task by task. -- Plan Tasks 1–3 are complete and independently reviewed. Tasks 4–9 add discovery, content-hash - identity, additive membership, durable jobs, the atomic membership map, the engine boundary, and - session authentication. Their checkboxes are current in the plan. The next task is the HTTP shell. +- Plan Tasks 1–3 are complete and independently reviewed. Tasks 4–10 add discovery, content-hash + identity, additive membership, durable jobs, the atomic membership map, the engine boundary, + session authentication and the HTTP shell. Their checkboxes are current in the plan. The next task + is membership, rule and job routes. -The next action is Task 10, "Build the FastAPI shell, middleware, and core resources." Do not repeat -Tasks 1–9. +The next action is Task 11, "Expose membership, rule, and job APIs." Do not repeat Tasks 1–10. Things a resuming agent will otherwise get wrong: @@ -56,7 +56,8 @@ Things a resuming agent will otherwise get wrong: extraction cost should dwarf the helper, but that is an assumption until measured. - [ ] Promote a new canonical path and force a reindex when a canonical path disappears, or the document silently leaves every collection it is in. -- [ ] Recover interrupted indexing jobs at startup; they do not survive a process restart on their own. +- [x] Recover interrupted indexing jobs at startup; Task 6 requeues valid running jobs and fails + unreadable requests. - [ ] Write `PRODUCT.md` properly via `/impeccable init` — the current file is a stub from the initial brief, not an interview. Blocks spec 002. @@ -68,6 +69,10 @@ Things a resuming agent will otherwise get wrong: paths exist, and fail if `AGENTS.md` exceeds 150 lines. ## Known issues +- The API test suite reports deprecation warnings from the installed FastAPI/Starlette TestClient + dependency. Replace its client integration when the dependency stack supports the successor API. +- Directory sources currently require a named child of the document root; decide whether the root + itself should be accepted as a persistent rule before finishing membership endpoints. - Task 3's repository ID test checks the generated collection ID prefix and length rather than asserting the full `^c[0-9a-f]{10}$` regex. The implementation uses `secrets.token_hex(5)` and is correct; add the stronger regression assertion when work resumes. diff --git a/docs/plans/001-search-engine-api.md b/docs/plans/001-search-engine-api.md index 0a97522..1e2052f 100644 --- a/docs/plans/001-search-engine-api.md +++ b/docs/plans/001-search-engine-api.md @@ -719,7 +719,7 @@ git commit -m "feat: add session authentication" - Consumes: `Settings`, repositories, `AuthService`, `Engine`, `JobWorker` - Produces: `create_app(settings: Settings | None = None, engine: Engine | None = None) -> FastAPI` -- [ ] **Step 1: Write failing API contract tests** +- [x] **Step 1: Write failing API contract tests** ```python def test_error_envelope(client: TestClient) -> None: @@ -741,18 +741,18 @@ def test_untrusted_forwarded_prefix_is_ignored(client: TestClient) -> None: Add tests for every auth route, CSRF on unsafe methods, strict same-origin login checks when `Origin` is present, `Secure` production cookies, request IDs, health 200/503, path traversal, one-level file browsing, list pagination, CRUD, confirmation, and no job on rename/empty creation. -- [ ] **Step 2: Run API tests and observe failure** +- [x] **Step 2: Run API tests and observe failure** Run: `uv run pytest tests/api/test_auth.py tests/api/test_health.py tests/api/test_files.py tests/api/test_collections.py tests/api/test_errors.py tests/api/test_root_path.py -v` -- [ ] **Step 3: Implement lifespan and dependency wiring** +- [x] **Step 3: Implement lifespan and dependency wiring** The lifespan creates state directories, migrates the database, seeds/refuses auth, recovers jobs, enqueues one reconciliation job for all persistent rules, starts the worker, and on exit calls `await worker.stop()`. Dependency callables read services from `app.state`. Tests inject a fake engine and temporary settings; production constructs `RecollEngine` only through the engine factory. -- [ ] **Step 4: Implement middleware and error mapping** +- [x] **Step 4: Implement middleware and error mapping** Map every `AppError` code to the spec status and envelope. Add an opaque request ID to unexpected 500 responses and logs. Prefix middleware strips the configured prefix from incoming paths before routing, @@ -761,11 +761,11 @@ configured proxy network, it may instead validate and apply forwarded scheme/hos forwarded prefix only when it is present on the incoming path. Ignore all such headers from other clients. Do not enable permissive CORS. -- [ ] **Step 5: Implement auth, health, files, and collections routes** +- [x] **Step 5: Implement auth, health, files, and collections routes** Use JSON login. Set cookie `recoll_modern_session` as HttpOnly, SameSite=Lax, root-path scoped, and Secure outside explicit development mode. Return CSRF token in the login/session JSON and require `X-CSRF-Token` on POST/PATCH/DELETE. `GET /files` returns name, relative path, kind, size, and mtime for one sorted directory level. Collection create with no sources returns 201; with sources delegates to Task 11 and returns 202. Deletion validates exact confirmation, captures affected paths, deletes database state, and enqueues cleanup. -- [ ] **Step 6: Verify and commit** +- [x] **Step 6: Verify and commit** Run: `uv run pytest tests/api -v && uv run ruff check . && uv run mypy src tests` diff --git a/src/recoll_modern/api/__init__.py b/src/recoll_modern/api/__init__.py new file mode 100644 index 0000000..94e33c6 --- /dev/null +++ b/src/recoll_modern/api/__init__.py @@ -0,0 +1 @@ +"""HTTP routes and transport adapters.""" diff --git a/src/recoll_modern/api/auth.py b/src/recoll_modern/api/auth.py new file mode 100644 index 0000000..7b29f62 --- /dev/null +++ b/src/recoll_modern/api/auth.py @@ -0,0 +1,45 @@ +"""Login, session refresh, and logout routes.""" + +from fastapi import APIRouter, Request, Response +from pydantic import BaseModel + +from recoll_modern.api.dependencies import COOKIE_NAME, auth_service, require_user +from recoll_modern.domain.errors import AppError + +router = APIRouter() + + +class LoginBody(BaseModel): + username: str + password: str + + +@router.post("/auth/login") +def login(body: LoginBody, request: Request, response: Response) -> dict[str, str]: + origin = request.headers.get("origin") + expected = f"{request.url.scheme}://{request.url.netloc}" + if origin is not None and origin != expected: + raise AppError("csrf_failed", "Origin is invalid", 403) + credentials = auth_service(request).login(body.username, body.password) + settings = request.app.state.settings + response.set_cookie( + COOKIE_NAME, credentials.cookie_token, httponly=True, samesite="lax", + secure=not settings.development_mode, + path=settings.root_path, + expires=credentials.expires_at, + ) + return {"csrf_token": credentials.csrf_token} + + +@router.get("/auth/session") +def session(request: Request) -> dict[str, str]: + user = require_user(request) + token = request.cookies[COOKIE_NAME] + return {"username": user.username, "csrf_token": auth_service(request).rotate_csrf(token)} + + +@router.post("/auth/logout", status_code=204) +def logout(request: Request, response: Response) -> None: + require_user(request) + auth_service(request).logout(request.cookies.get(COOKIE_NAME)) + response.delete_cookie(COOKIE_NAME, path=request.app.state.settings.root_path) diff --git a/src/recoll_modern/api/collections.py b/src/recoll_modern/api/collections.py new file mode 100644 index 0000000..5d7f23a --- /dev/null +++ b/src/recoll_modern/api/collections.py @@ -0,0 +1,82 @@ +"""Collection metadata HTTP routes.""" + +from dataclasses import asdict +from typing import cast + +from fastapi import APIRouter, Request, Response +from fastapi.encoders import jsonable_encoder + +from recoll_modern.api.dependencies import require_user +from recoll_modern.api.schemas import CollectionCreateBody, CollectionPatchBody +from recoll_modern.domain.errors import AppError +from recoll_modern.domain.models import UNSET, CollectionCreate, CollectionPatch + +router = APIRouter() + + +@router.get("/collections") +def list_collections(request: Request, limit: int = 50, offset: int = 0) -> dict[str, object]: + require_user(request) + if not 1 <= limit <= 200 or offset < 0: + raise AppError("validation_failed", "Invalid pagination", 422) + collections, total = request.app.state.collections.list(limit, offset) + return {"items": jsonable_encoder([asdict(item) for item in collections]), "total": total} + + +@router.get("/collections/{collection_id}") +def get_collection(request: Request, collection_id: str) -> dict[str, object]: + require_user(request) + return cast( + dict[str, object], + jsonable_encoder(asdict(request.app.state.collections.get(collection_id))), + ) + + +@router.post("/collections") +def create_collection( + body: CollectionCreateBody, request: Request, response: Response +) -> dict[str, object]: + require_user(request) + collection = request.app.state.collections.create( + CollectionCreate(body.name, body.description) + ) + payload = jsonable_encoder(asdict(collection)) + if body.sources: + job = request.app.state.jobs.enqueue( + "membership_add", + {"collection_id": collection.id, "sources": body.sources}, + body.fail_on_file_error, + ) + request.app.state.worker.notify() + response.status_code = 202 + return {"collection": payload, "job_id": job.id} + response.status_code = 201 + return cast(dict[str, object], payload) + + +@router.patch("/collections/{collection_id}") +def patch_collection( + body: CollectionPatchBody, request: Request, collection_id: str +) -> dict[str, object]: + require_user(request) + description = body.description if "description" in body.model_fields_set else UNSET + changed = request.app.state.collections.patch( + collection_id, CollectionPatch(body.name, description) + ) + return cast(dict[str, object], jsonable_encoder(asdict(changed))) + + +@router.delete("/collections/{collection_id}", status_code=202) +def delete_collection( + request: Request, collection_id: str, confirmation: str = "" +) -> dict[str, str]: + require_user(request) + collection = request.app.state.collections.get(collection_id) + if confirmation != collection.name: + raise AppError("confirmation_required", "Collection name confirmation is required", 409) + paths = request.app.state.collections.delete(collection_id) + job = request.app.state.jobs.enqueue( + "collection_cleanup", {"paths": paths}, False + ) + request.app.state.worker.notify() + return {"job_id": job.id} diff --git a/src/recoll_modern/api/dependencies.py b/src/recoll_modern/api/dependencies.py new file mode 100644 index 0000000..4c1af2f --- /dev/null +++ b/src/recoll_modern/api/dependencies.py @@ -0,0 +1,21 @@ +"""Read application services and enforce cookie sessions.""" + +from typing import cast + +from fastapi import Request + +from recoll_modern.domain.auth import AuthenticatedUser, AuthService + +COOKIE_NAME = "recoll_modern_session" + + +def auth_service(request: Request) -> AuthService: + return cast(AuthService, request.app.state.auth) + + +def require_user(request: Request) -> AuthenticatedUser: + return auth_service(request).authenticate( + request.cookies.get(COOKIE_NAME), + csrf_token=request.headers.get("X-CSRF-Token"), + unsafe=request.method not in ("GET", "HEAD", "OPTIONS"), + ) diff --git a/src/recoll_modern/api/errors.py b/src/recoll_modern/api/errors.py new file mode 100644 index 0000000..791c90c --- /dev/null +++ b/src/recoll_modern/api/errors.py @@ -0,0 +1,51 @@ +"""Stable JSON error envelopes for expected and unexpected failures.""" + +import logging +import secrets + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from recoll_modern.domain.errors import AppError + +_LOG = logging.getLogger(__name__) + + +def install_error_handlers(app: FastAPI) -> None: + @app.exception_handler(StarletteHTTPException) + async def http_error(_request: Request, error: StarletteHTTPException) -> JSONResponse: + code = "path_not_found" if error.status_code == 404 else "invalid_query" + message = "Path was not found" if error.status_code == 404 else "Request is invalid" + return JSONResponse( + {"error": {"code": code, "message": message, "details": {}}}, + status_code=error.status_code, + ) + + @app.exception_handler(AppError) + async def app_error(_request: Request, error: AppError) -> JSONResponse: + return JSONResponse( + {"error": {"code": error.code, "message": error.message, + "details": dict(error.details)}}, + status_code=error.status, + ) + + @app.exception_handler(RequestValidationError) + async def validation_error(_request: Request, _error: RequestValidationError) -> JSONResponse: + return JSONResponse( + {"error": {"code": "validation_failed", "message": "Request validation failed", + "details": {}}}, + status_code=422, + ) + + @app.exception_handler(Exception) + async def unexpected_error(_request: Request, error: Exception) -> JSONResponse: + request_id = secrets.token_hex(8) + _LOG.exception("Unhandled request error %s", request_id, exc_info=error) + return JSONResponse( + {"error": {"code": "internal_error", "message": "Internal server error", + "details": {"request_id": request_id}}}, + status_code=500, + headers={"X-Request-ID": request_id}, + ) diff --git a/src/recoll_modern/api/files.py b/src/recoll_modern/api/files.py new file mode 100644 index 0000000..ab6a396 --- /dev/null +++ b/src/recoll_modern/api/files.py @@ -0,0 +1,43 @@ +"""Read-only single-directory browsing beneath the configured document root.""" + +import os +from pathlib import Path + +from fastapi import APIRouter, Request + +from recoll_modern.api.dependencies import require_user +from recoll_modern.domain.errors import AppError +from recoll_modern.indexing.paths import PathPolicyError + +router = APIRouter() + + +@router.get("/files") +def list_files(request: Request, path: str = "") -> dict[str, object]: + require_user(request) + policy = request.app.state.path_policy + try: + directory = policy.root if path == "" else policy.resolve_directory(path) + except PathPolicyError: + raise AppError("path_outside_root", "Path is outside document root", 422) from None + if not directory.is_dir(): + raise AppError("path_not_found", "Path was not found", 404) + items: list[dict[str, object]] = [] + try: + with os.scandir(directory) as entries: + for entry in entries: + candidate = Path(entry.path) + if entry.is_symlink() and not candidate.resolve().is_relative_to(policy.root): + continue + stat = entry.stat(follow_symlinks=False) + relative = candidate.relative_to(policy.root).as_posix() + items.append({ + "name": entry.name, + "path": relative, + "kind": "directory" if entry.is_dir(follow_symlinks=False) else "file", + "size": stat.st_size, + "mtime": stat.st_mtime, + }) + except OSError: + raise AppError("path_not_found", "Path was not found", 404) from None + return {"path": path, "items": sorted(items, key=lambda item: str(item["name"]))} diff --git a/src/recoll_modern/api/health.py b/src/recoll_modern/api/health.py new file mode 100644 index 0000000..44196bd --- /dev/null +++ b/src/recoll_modern/api/health.py @@ -0,0 +1,14 @@ +"""Public readiness probe.""" + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +router = APIRouter() + + +@router.get("/health") +def health(request: Request) -> JSONResponse: + status = request.app.state.engine.health() + if not status.available: + return JSONResponse({"status": "unavailable"}, status_code=503) + return JSONResponse({"status": "ready"}) diff --git a/src/recoll_modern/api/middleware.py b/src/recoll_modern/api/middleware.py new file mode 100644 index 0000000..c94f887 --- /dev/null +++ b/src/recoll_modern/api/middleware.py @@ -0,0 +1,59 @@ +"""Mount-prefix routing and conservative proxy-header handling.""" + +import ipaddress + +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from recoll_modern.config import Settings + + +class PrefixMiddleware: + def __init__(self, app: ASGIApp, settings: Settings) -> None: + self.app = app + self.settings = settings + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] not in ("http", "websocket"): + await self.app(scope, receive, send) + return + scope = dict(scope) + prefix = self.settings.root_path + client = scope.get("client") + try: + trusted = client is not None and any( + ipaddress.ip_address(client[0]) in network + for network in self.settings.trusted_proxies + ) + except ValueError: + trusted = False + if trusted: + headers = {name.lower(): value for name, value in scope["headers"]} + forwarded_prefix = headers.get(b"x-forwarded-prefix", b"").decode("ascii", "ignore") + if ( + forwarded_prefix.startswith("/") + and not forwarded_prefix.endswith("/") + and ".." not in forwarded_prefix + ): + prefix = forwarded_prefix + scheme = headers.get(b"x-forwarded-proto", b"").decode("ascii", "ignore") + if scheme in ("http", "https"): + scope["scheme"] = scheme + host = headers.get(b"x-forwarded-host", b"") + if host and b"/" not in host and b"\r" not in host and b"\n" not in host: + scope["headers"] = [ + (name, value) for name, value in scope["headers"] if name.lower() != b"host" + ] + [(b"host", host)] + path = scope["path"] + if prefix != "/": + if path != prefix and not path.startswith(prefix + "/") and not trusted: + response = JSONResponse( + {"error": {"code": "path_not_found", "message": "Path was not found", + "details": {}}}, status_code=404 + ) + await response(scope, receive, send) + return + if path == prefix or path.startswith(prefix + "/"): + scope["path"] = path[len(prefix):] or "/" + scope["root_path"] = prefix + await self.app(scope, receive, send) diff --git a/src/recoll_modern/api/schemas.py b/src/recoll_modern/api/schemas.py new file mode 100644 index 0000000..2d70a2f --- /dev/null +++ b/src/recoll_modern/api/schemas.py @@ -0,0 +1,15 @@ +"""Small HTTP request bodies for collection metadata.""" + +from pydantic import BaseModel + + +class CollectionCreateBody(BaseModel): + name: str + description: str | None = None + sources: list[dict[str, object]] | None = None + fail_on_file_error: bool = False + + +class CollectionPatchBody(BaseModel): + name: str | None = None + description: str | None = None diff --git a/src/recoll_modern/app.py b/src/recoll_modern/app.py index 7e84b70..6c6070a 100644 --- a/src/recoll_modern/app.py +++ b/src/recoll_modern/app.py @@ -1,16 +1,107 @@ -"""FastAPI application factory.""" +"""FastAPI application factory and durable-service lifecycle.""" + +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import timedelta from fastapi import FastAPI +from recoll_modern.api import auth, collections, files, health +from recoll_modern.api.errors import install_error_handlers +from recoll_modern.api.middleware import PrefixMiddleware from recoll_modern.config import Settings +from recoll_modern.domain.auth import AuthService +from recoll_modern.domain.collections import CollectionService +from recoll_modern.domain.jobs import Job +from recoll_modern.engine.protocol import Engine +from recoll_modern.engine.recoll import RecollEngine +from recoll_modern.indexing.handlers import MembershipJobHandler +from recoll_modern.indexing.map_export import MembershipMapRows, export_membership_map +from recoll_modern.indexing.paths import PathPolicy +from recoll_modern.indexing.reconcile import reconcile_rule +from recoll_modern.indexing.worker import JobWorker +from recoll_modern.storage.auth import SQLiteAuthRepository +from recoll_modern.storage.collections import SQLiteCollectionRepository +from recoll_modern.storage.database import Database +from recoll_modern.storage.documents import DocumentRepository +from recoll_modern.storage.jobs import JobRepository +from recoll_modern.storage.membership import MembershipRepository -def create_app(settings: Settings) -> FastAPI: - """Create the HTTP application with its configured mount prefix.""" - app = FastAPI(root_path=settings.root_path) +def create_app(settings: Settings | None = None, engine: Engine | None = None) -> FastAPI: + """Wire config, storage, indexing, and HTTP routes in one process boundary.""" + settings = settings or Settings.from_env() + engine = engine or RecollEngine(settings.recoll_config_dir) - @app.get("/health") - def health() -> dict[str, str]: - return {"status": "starting"} + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + settings.state_root.mkdir(parents=True, exist_ok=True) + database = Database(settings.database_path) + database.migrate() + auth_service = AuthService( + SQLiteAuthRepository(database), + lifetime=timedelta(hours=settings.session_lifetime_hours), + ) + auth_service.seed_initial_admin( + os.environ.get("RECOLL_MODERN_ADMIN_USERNAME"), + os.environ.get("RECOLL_MODERN_ADMIN_PASSWORD"), + ) + policy = PathPolicy(settings.document_root) + memberships = MembershipRepository(database) + documents = DocumentRepository(database) + jobs = JobRepository(database) + map_rows = MembershipMapRows(memberships, policy.root) + export_membership_map(map_rows, settings.membership_map_path) + def cleanup(job: Job) -> None: + export_membership_map(map_rows, settings.membership_map_path) + paths = job.request.get("paths", []) + if isinstance(paths, list): + absolute = tuple(str(policy.root / path) for path in paths if isinstance(path, str)) + if absolute: + engine.force_reindex(absolute) + + def reconcile_all(_job: Job) -> None: + changed: set[str] = set() + for rule in memberships.list_rules(): + result = reconcile_rule(rule.id, policy, documents, memberships) + changed.update(result.changed_paths) + export_membership_map(map_rows, settings.membership_map_path) + if changed: + engine.force_reindex(tuple(str(policy.root / path) for path in sorted(changed))) + + membership_handler = MembershipJobHandler( + policy, documents, memberships, jobs, settings.membership_map_path, engine + ) + worker = JobWorker(jobs, { + "membership_add": membership_handler.handle, + "collection_cleanup": cleanup, + "reconcile_rules": reconcile_all, + }) + app.state.settings = settings + app.state.database = database + app.state.auth = auth_service + app.state.engine = engine + app.state.path_policy = policy + app.state.collections = CollectionService(SQLiteCollectionRepository(database)) + app.state.memberships = memberships + app.state.documents = documents + app.state.jobs = jobs + app.state.worker = worker + if memberships.list_rules(): + jobs.enqueue("reconcile_rules", {}, False) + worker.start() + try: + yield + finally: + await worker.stop() + + app = FastAPI(root_path=settings.root_path, lifespan=lifespan) + app.add_middleware(PrefixMiddleware, settings=settings) + install_error_handlers(app) + app.include_router(auth.router) + app.include_router(health.router) + app.include_router(files.router) + app.include_router(collections.router) return app diff --git a/src/recoll_modern/config.py b/src/recoll_modern/config.py index 167075b..e94afce 100644 --- a/src/recoll_modern/config.py +++ b/src/recoll_modern/config.py @@ -21,6 +21,7 @@ class Settings: inline_text_limit: int inline_text_limit_max: int session_lifetime_hours: int + development_mode: bool @property def database_path(self) -> Path: @@ -70,4 +71,6 @@ class Settings: inline_text_limit=text_default, inline_text_limit_max=text_max, session_lifetime_hours=session_hours, + development_mode=environ.get("RECOLL_MODERN_DEVELOPMENT_MODE", "false").lower() + == "true", ) diff --git a/src/recoll_modern/domain/auth.py b/src/recoll_modern/domain/auth.py index d726e19..c113d31 100644 --- a/src/recoll_modern/domain/auth.py +++ b/src/recoll_modern/domain/auth.py @@ -65,6 +65,8 @@ class AuthRepository(Protocol): def touch_session(self, cookie_token: str, now: datetime) -> None: ... + def update_csrf(self, cookie_token: str, csrf_token: str) -> None: ... + def normalize_username(username: str) -> tuple[str, str]: display = normalize("NFC", username.strip()) @@ -142,6 +144,13 @@ class AuthService: if cookie_token: self._repository.delete_session(cookie_token) + def rotate_csrf(self, cookie_token: str) -> str: + """Issue a fresh readable CSRF token for an authenticated session.""" + self.authenticate(cookie_token) + csrf_token = secrets.token_urlsafe(32) + self._repository.update_csrf(cookie_token, csrf_token) + return csrf_token + def _invalid_credentials() -> AppError: return AppError("invalid_credentials", "Invalid username or password", 401) diff --git a/src/recoll_modern/storage/auth.py b/src/recoll_modern/storage/auth.py index 76933af..f6a3e9c 100644 --- a/src/recoll_modern/storage/auth.py +++ b/src/recoll_modern/storage/auth.py @@ -115,6 +115,13 @@ class SQLiteAuthRepository: (now.isoformat(), _hash_token(cookie_token)), ) + def update_csrf(self, cookie_token: str, csrf_token: str) -> None: + with self._database.transaction() as connection: + connection.execute( + "UPDATE sessions SET csrf_hash = ? WHERE token_hash = ?", + (_hash_token(csrf_token), _hash_token(cookie_token)), + ) + def cleanup_expired(self, now: datetime) -> int: with self._database.transaction() as connection: cursor = connection.execute( diff --git a/src/recoll_modern/storage/membership.py b/src/recoll_modern/storage/membership.py index 1716029..fa99044 100644 --- a/src/recoll_modern/storage/membership.py +++ b/src/recoll_modern/storage/membership.py @@ -64,6 +64,14 @@ class MembershipRepository: ) return DirectoryRule(str(row[0]), str(row[1]), str(row[2]), exclusions) + def list_rules(self) -> list[DirectoryRule]: + with self._database.connect() as connection: + rule_ids = [ + str(row[0]) + for row in connection.execute("SELECT id FROM directory_rules ORDER BY id") + ] + return [rule for rule_id in rule_ids if (rule := self.get_rule(rule_id)) is not None] + def patch_rule(self, rule_id: str, excluded_directories: tuple[str, ...]) -> None: with self._database.transaction() as connection: connection.execute( diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..9831674 --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1 @@ +"""HTTP contract tests.""" diff --git a/tests/api/conftest.py b/tests/api/conftest.py new file mode 100644 index 0000000..46cf19c --- /dev/null +++ b/tests/api/conftest.py @@ -0,0 +1,47 @@ +from collections.abc import Generator +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from recoll_modern.app import create_app +from recoll_modern.config import Settings +from recoll_modern.engine.fake import FakeEngine + + +@pytest.fixture +def app_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[TestClient]: + monkeypatch.setenv("RECOLL_MODERN_ADMIN_USERNAME", "admin") + monkeypatch.setenv("RECOLL_MODERN_ADMIN_PASSWORD", "secret-password") + (tmp_path / "docs").mkdir() + settings = Settings.from_env({ + "RECOLL_MODERN_DOCUMENT_ROOT": str(tmp_path / "docs"), + "RECOLL_MODERN_STATE_ROOT": str(tmp_path / "state"), + "RECOLL_MODERN_RECOLL_CONFIG_DIR": str(tmp_path / "recoll"), + }) + with TestClient(create_app(settings, FakeEngine([])), base_url="https://testserver") as client: + yield client + + +@pytest.fixture +def prefixed_client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[TestClient]: + monkeypatch.setenv("RECOLL_MODERN_ADMIN_USERNAME", "admin") + monkeypatch.setenv("RECOLL_MODERN_ADMIN_PASSWORD", "secret-password") + (tmp_path / "docs").mkdir() + settings = Settings.from_env({ + "RECOLL_MODERN_DOCUMENT_ROOT": str(tmp_path / "docs"), + "RECOLL_MODERN_STATE_ROOT": str(tmp_path / "state"), + "RECOLL_MODERN_RECOLL_CONFIG_DIR": str(tmp_path / "recoll"), + "RECOLL_MODERN_ROOT_PATH": "/recoll", + }) + with TestClient(create_app(settings, FakeEngine([])), base_url="https://testserver") as client: + yield client + + +@pytest.fixture +def authenticated(app_client: TestClient) -> tuple[TestClient, str]: + response = app_client.post( + "/auth/login", json={"username": "admin", "password": "secret-password"} + ) + assert response.status_code == 200 + return app_client, response.json()["csrf_token"] diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py new file mode 100644 index 0000000..50da41b --- /dev/null +++ b/tests/api/test_auth.py @@ -0,0 +1,46 @@ +from fastapi.testclient import TestClient + + +def test_login_sets_scoped_secure_cookie(app_client: TestClient) -> None: + response = app_client.post( + "/auth/login", json={"username": "admin", "password": "secret-password"} + ) + assert response.status_code == 200 + cookie = response.headers["set-cookie"] + assert "HttpOnly" in cookie and "SameSite=lax" in cookie + assert "Secure" in cookie and "Path=/" in cookie + assert response.json()["csrf_token"] + + +def test_unauthenticated_request_is_refused(app_client: TestClient) -> None: + response = app_client.get("/collections") + assert response.status_code == 401 + assert response.json()["error"]["code"] == "unauthenticated" + + +def test_csrf_required_for_unsafe_request(authenticated: tuple[TestClient, str]) -> None: + client, csrf = authenticated + assert client.post("/collections", json={"name": "Books"}).status_code == 403 + response = client.post( + "/collections", json={"name": "Books"}, headers={"X-CSRF-Token": csrf} + ) + assert response.status_code == 201 + + +def test_session_rotates_csrf_and_logout(authenticated: tuple[TestClient, str]) -> None: + client, old_csrf = authenticated + session = client.get("/auth/session") + assert session.status_code == 200 + new_csrf = session.json()["csrf_token"] + assert new_csrf != old_csrf + assert client.post("/auth/logout", headers={"X-CSRF-Token": old_csrf}).status_code == 403 + assert client.post("/auth/logout", headers={"X-CSRF-Token": new_csrf}).status_code == 204 + assert client.get("/auth/session").status_code == 401 + + +def test_login_rejects_cross_origin(app_client: TestClient) -> None: + response = app_client.post( + "/auth/login", json={"username": "admin", "password": "secret-password"}, + headers={"Origin": "https://evil.example"}, + ) + assert response.status_code == 403 diff --git a/tests/api/test_collections.py b/tests/api/test_collections.py new file mode 100644 index 0000000..cd0dd69 --- /dev/null +++ b/tests/api/test_collections.py @@ -0,0 +1,48 @@ +from typing import cast + +from fastapi import FastAPI +from fastapi.testclient import TestClient + + +def test_collection_crud_and_confirmation(authenticated: tuple[TestClient, str]) -> None: + client, csrf = authenticated + headers = {"X-CSRF-Token": csrf} + created = client.post("/collections", json={"name": "Books"}, headers=headers) + assert created.status_code == 201 + collection_id = created.json()["id"] + assert client.get(f"/collections/{collection_id}").json()["name"] == "Books" + assert client.get("/collections").json()["total"] == 1 + renamed = client.patch( + f"/collections/{collection_id}", json={"name": "Reading"}, headers=headers + ) + assert renamed.status_code == 200 + assert renamed.json()["name"] == "Reading" + denied = client.delete( + f"/collections/{collection_id}", params={"confirmation": "Books"}, headers=headers + ) + assert denied.status_code == 409 + deleted = client.delete( + f"/collections/{collection_id}", params={"confirmation": "Reading"}, headers=headers + ) + assert deleted.status_code == 202 + assert client.get(f"/collections/{collection_id}").status_code == 404 + + +def test_collection_list_validates_pagination(authenticated: tuple[TestClient, str]) -> None: + client, _ = authenticated + response = client.get("/collections", params={"limit": 201}) + assert response.status_code == 422 + assert response.json()["error"]["code"] == "validation_failed" + + +def test_empty_create_and_rename_do_not_enqueue_jobs( + authenticated: tuple[TestClient, str] +) -> None: + client, csrf = authenticated + headers = {"X-CSRF-Token": csrf} + created = client.post("/collections", json={"name": "Books"}, headers=headers) + collection_id = created.json()["id"] + client.patch(f"/collections/{collection_id}", json={"name": "Reading"}, headers=headers) + with cast(FastAPI, client.app).state.database.connect() as connection: + total = connection.execute("SELECT COUNT(*) FROM jobs").fetchone()[0] + assert total == 0 diff --git a/tests/api/test_errors.py b/tests/api/test_errors.py new file mode 100644 index 0000000..cd516f3 --- /dev/null +++ b/tests/api/test_errors.py @@ -0,0 +1,37 @@ +from typing import cast + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + + +def test_error_envelope(authenticated: tuple[TestClient, str]) -> None: + client, _ = authenticated + response = client.get("/collections/cffffffffff") + assert response.status_code == 404 + assert response.json() == {"error": { + "code": "collection_not_found", "message": "Collection was not found", "details": {} + }} + + +def test_unknown_route_uses_error_envelope(app_client: TestClient) -> None: + response = app_client.get("/no-such-route") + assert response.status_code == 404 + assert response.json()["error"]["code"] == "path_not_found" + + +def test_unexpected_error_has_request_id( + app_client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def fail() -> None: + raise RuntimeError("private filesystem path") + + app = cast(FastAPI, app_client.app) + monkeypatch.setattr(app.state.engine, "health", fail) + with TestClient( + app_client.app, base_url="https://testserver", raise_server_exceptions=False + ) as client: + response = client.get("/health") + assert response.status_code == 500 + assert response.headers["X-Request-ID"] == response.json()["error"]["details"]["request_id"] + assert "private" not in response.text diff --git a/tests/api/test_files.py b/tests/api/test_files.py new file mode 100644 index 0000000..e37019b --- /dev/null +++ b/tests/api/test_files.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from fastapi.testclient import TestClient + + +def test_files_browses_one_level_read_only( + authenticated: tuple[TestClient, str], tmp_path: Path +) -> None: + client, _ = authenticated + root = tmp_path / "docs" + (root / "a.txt").write_text("a") + (root / "sub").mkdir() + (root / "sub" / "deep.txt").write_text("deep") + response = client.get("/files") + assert response.status_code == 200 + assert [item["name"] for item in response.json()["items"]] == ["a.txt", "sub"] + + +def test_files_rejects_traversal(authenticated: tuple[TestClient, str]) -> None: + client, _ = authenticated + response = client.get("/files", params={"path": "../"}) + assert response.status_code == 422 + assert response.json()["error"]["code"] == "path_outside_root" diff --git a/tests/api/test_health.py b/tests/api/test_health.py new file mode 100644 index 0000000..92c47b1 --- /dev/null +++ b/tests/api/test_health.py @@ -0,0 +1,29 @@ +from typing import cast + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from recoll_modern.engine.protocol import EngineHealth + + +def test_health_is_public_and_ready(app_client: TestClient) -> None: + response = app_client.get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "ready" + + +def test_untrusted_forwarded_prefix_is_ignored(app_client: TestClient) -> None: + response = app_client.get("/health", headers={"X-Forwarded-Prefix": "/evil"}) + assert response.status_code == 200 + assert "/evil" not in response.text + + +def test_health_reports_engine_unavailable( + app_client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + cast(FastAPI, app_client.app).state.engine, "health", lambda: EngineHealth(False) + ) + response = app_client.get("/health") + assert response.status_code == 503 diff --git a/tests/api/test_root_path.py b/tests/api/test_root_path.py new file mode 100644 index 0000000..265025c --- /dev/null +++ b/tests/api/test_root_path.py @@ -0,0 +1,37 @@ +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + +from recoll_modern.app import create_app +from recoll_modern.config import Settings +from recoll_modern.engine.fake import FakeEngine + + +def test_cookie_and_openapi_honor_root_path(prefixed_client: TestClient) -> None: + response = prefixed_client.post( + "/recoll/auth/login", json={"username": "admin", "password": "secret-password"} + ) + assert response.status_code == 200 + assert "Path=/recoll" in response.headers["set-cookie"] + assert prefixed_client.get("/recoll/openapi.json").status_code == 200 + assert prefixed_client.get("/auth/login").status_code == 404 + + +def test_trusted_forwarded_prefix_routes_request( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("RECOLL_MODERN_ADMIN_USERNAME", "admin") + monkeypatch.setenv("RECOLL_MODERN_ADMIN_PASSWORD", "secret-password") + (tmp_path / "docs").mkdir() + settings = Settings.from_env({ + "RECOLL_MODERN_DOCUMENT_ROOT": str(tmp_path / "docs"), + "RECOLL_MODERN_STATE_ROOT": str(tmp_path / "state"), + "RECOLL_MODERN_TRUSTED_PROXIES": "127.0.0.1/32", + }) + with TestClient( + create_app(settings, FakeEngine([])), base_url="https://testserver", + client=("127.0.0.1", 50000), + ) as client: + response = client.get("/recoll/health", headers={"X-Forwarded-Prefix": "/recoll"}) + assert response.status_code == 200 diff --git a/tests/domain/__init__.py b/tests/domain/__init__.py new file mode 100644 index 0000000..a241b0e --- /dev/null +++ b/tests/domain/__init__.py @@ -0,0 +1 @@ +"""Domain behavior tests.""" diff --git a/tests/helper/__init__.py b/tests/helper/__init__.py new file mode 100644 index 0000000..ad8d491 --- /dev/null +++ b/tests/helper/__init__.py @@ -0,0 +1 @@ +"""Native metadata helper tests.""" diff --git a/tests/indexing/__init__.py b/tests/indexing/__init__.py new file mode 100644 index 0000000..0d31b69 --- /dev/null +++ b/tests/indexing/__init__.py @@ -0,0 +1 @@ +"""Discovery and job tests.""" diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py new file mode 100644 index 0000000..6ee80ff --- /dev/null +++ b/tests/storage/__init__.py @@ -0,0 +1 @@ +"""SQLite persistence tests.""" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..b27d053 --- /dev/null +++ b/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Configuration tests."""