From 56805a97cf37c6f17a756a4be0b7b2b4b019646a Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 26 Sep 2026 13:05:15 +0200 Subject: [PATCH] feat: package Recoll service --- .dockerignore | 14 +++++ .gitignore | 1 + Dockerfile | 39 ++++++++++++++ README.md | 40 ++++++++++++++- compose.yaml | 29 +++++++++++ docker/entrypoint.sh | 12 +++++ docker/fields | 5 ++ docker/recoll.conf | 4 ++ docs/ARCHITECTURE.md | 12 ++++- docs/DECISIONS.md | 15 ++++++ docs/TODO.md | 11 ++-- docs/plans/001-search-engine-api.md | 10 ++-- tests/integration/__init__.py | 1 + tests/integration/conftest.py | 31 +++++++++++ tests/integration/test_container_smoke.py | 13 +++++ tests/integration/test_read_only_corpus.py | 60 ++++++++++++++++++++++ 16 files changed, 284 insertions(+), 13 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 docker/entrypoint.sh create mode 100644 docker/fields create mode 100644 docker/recoll.conf create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_container_smoke.py create mode 100644 tests/integration/test_read_only_corpus.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d8e3a00 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.venv +.pytest_cache +.ruff_cache +.mypy_cache +__pycache__ +dist +*.sqlite3 +*.sqlite3-shm +*.sqlite3-wal +membership.map +helper/recoll-membership +.env +.env.* diff --git a/.gitignore b/.gitignore index d19192f..942d6c8 100644 --- a/.gitignore +++ b/.gitignore @@ -34,5 +34,6 @@ helper/recoll-membership # Recoll runtime data (configuration templates remain tracked) recoll.conf +!docker/recoll.conf recoll-config/ xapiandb/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a92b1d0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM debian:trixie-slim AS helper-build +RUN apt-get update && apt-get install -y --no-install-recommends build-essential \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /build +COPY helper/recoll_membership.c ./ +RUN cc -std=c17 -O2 -Wall -Wextra -Werror recoll_membership.c -o recoll-membership + +FROM ghcr.io/astral-sh/uv:0.12.16 AS uv-bin + +FROM debian:trixie-slim +COPY --from=uv-bin /uv /usr/local/bin/uv +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates python3 python3-recoll recoll poppler-utils python3-lxml unzip \ + && rm -rf /var/lib/apt/lists/* +COPY --from=helper-build /build/recoll-membership /usr/local/bin/recoll-membership +WORKDIR /app +COPY pyproject.toml uv.lock README.md ./ +COPY src ./src +RUN uv venv --python /usr/bin/python3 --system-site-packages /opt/venv +ENV VIRTUAL_ENV=/opt/venv \ + UV_PROJECT_ENVIRONMENT=/opt/venv \ + PATH="/opt/venv/bin:$PATH" \ + UV_LINK_MODE=copy \ + RECOLL_MODERN_DOCUMENT_ROOT=/documents \ + RECOLL_MODERN_STATE_ROOT=/var/lib/recoll-modern \ + RECOLL_MODERN_RECOLL_CONFIG_DIR=/var/lib/recoll-modern/recoll-config \ + RECOLL_MODERN_MEMBERSHIP_MAP=/var/lib/recoll-modern/membership.map +RUN uv sync --locked --no-dev && python -c 'from recoll import recoll' +COPY docker/recoll.conf docker/fields /etc/recoll-modern/ +COPY docker/entrypoint.sh /usr/local/bin/recoll-modern-entrypoint +RUN chmod 755 /usr/local/bin/recoll-modern-entrypoint \ + && groupadd --gid 10001 recoll-modern \ + && useradd --uid 10001 --gid 10001 --home-dir /var/lib/recoll-modern \ + --shell /usr/sbin/nologin recoll-modern \ + && mkdir -p /documents /var/lib/recoll-modern /var/lib/recoll \ + && chown -R 10001:10001 /var/lib/recoll-modern /var/lib/recoll +USER 10001:10001 +EXPOSE 8000 +ENTRYPOINT ["recoll-modern-entrypoint"] diff --git a/README.md b/README.md index 4184d7b..ef8cd5c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,45 @@ A modern, self-hosted web UI and MCP server on top of [Recoll](https://www.lesbonscomptes.com/recoll/) full-text search, with support for multiple independent collections. -Status: initial Python service scaffold in place; the Search engine and HTTP API are under development. +Status: the authenticated search API and Docker deployment are implemented. Final contract and +security acceptance coverage for spec 001 remains; the web UI and MCP server are later specs. + +## Run with Docker + +Set `RECOLL_MODERN_DOCUMENTS` to an absolute host directory containing documents and provide a first +administrator username and password: + +```sh +export RECOLL_MODERN_DOCUMENTS=/srv/documents +export RECOLL_MODERN_ADMIN_USERNAME=admin +export RECOLL_MODERN_ADMIN_PASSWORD='use-a-long-unique-password' +docker compose up -d --build +``` + +The service listens on host port 8000 by default; set `RECOLL_MODERN_PUBLISH_PORT` to change it. The +document directory is mounted read-only. SQLite, the membership map and generated Recoll configuration +live in the `service-state` volume; the index lives in the separate `recoll-index` volume. Back up both +volumes together and keep a separate copy of the documents. `docker compose down` preserves the volumes; +never pass `-v` unless you intentionally want to delete the index and service state. + +On first boot, the API is available but `/health` may return 503 until a membership or reindex job has +built the first Recoll index. Log in through `POST /auth/login`, create a collection, then add file or +directory sources through `POST /collections/{id}/memberships`. Mutations return a durable job ID; +inspect `GET /jobs/{id}` for completion or file errors. `POST /jobs/reindex` refreshes the index without +deleting it. The login seed applies only when no administrator exists; changing the environment value +later does not rotate an existing password. Protect and back up the service-state volume. + +Production sessions use Secure cookies, so serve the API over HTTPS. For a trusted local HTTP trial, +set `RECOLL_MODERN_DEVELOPMENT_MODE=true`; do not use that setting on a public endpoint. Set +`RECOLL_MODERN_ROOT_PATH=/recoll` when hosting under a path prefix, and make the reverse proxy preserve +that path. `RECOLL_MODERN_TRUSTED_PROXIES` accepts comma-separated IP networks; only forwarded host, +scheme and prefix headers from those networks are honored. The health URL follows the prefix, for +example `/recoll/health`. + +To run the generated-corpus container checks, start Compose with a test document containing the unique +word `zebraplatypus`, set `RECOLL_MODERN_TEST_CORPUS_FILE` to its path relative to the document root, +and run `RECOLL_MODERN_REQUIRE_DOCKER=1 uv run pytest tests/integration`. Ordinary host test runs skip +these checks when Compose is not running. Recoll commands run only inside the container. Working here (human or agent): read [`AGENTS.md`](AGENTS.md). It is the single source of truth for commands, workflow, and boundaries. Deeper context lives in [`docs/`](docs/). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..2267a87 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,29 @@ +services: + app: + build: . + image: recoll-modern:local + ports: + - "${RECOLL_MODERN_PUBLISH_PORT:-8000}:8000" + environment: + RECOLL_MODERN_ADMIN_USERNAME: ${RECOLL_MODERN_ADMIN_USERNAME:?Set administrator username} + RECOLL_MODERN_ADMIN_PASSWORD: ${RECOLL_MODERN_ADMIN_PASSWORD:?Set administrator password} + RECOLL_MODERN_HOST: 0.0.0.0 + RECOLL_MODERN_PORT: 8000 + RECOLL_MODERN_ROOT_PATH: ${RECOLL_MODERN_ROOT_PATH:-/} + RECOLL_MODERN_TRUSTED_PROXIES: ${RECOLL_MODERN_TRUSTED_PROXIES:-} + RECOLL_MODERN_DEVELOPMENT_MODE: ${RECOLL_MODERN_DEVELOPMENT_MODE:-false} + RECOLL_MODERN_MEMBERSHIP_MAP: /var/lib/recoll-modern/membership.map + volumes: + - ${RECOLL_MODERN_DOCUMENTS:?Set documents directory}:/documents:ro + - service-state:/var/lib/recoll-modern + - recoll-index:/var/lib/recoll + healthcheck: + test: ["CMD", "python", "-c", "import os,urllib.request; p=os.environ.get('RECOLL_MODERN_ROOT_PATH','/').rstrip('/'); urllib.request.urlopen('http://127.0.0.1:8000'+p+'/health',timeout=5)"] + interval: 30s + timeout: 8s + retries: 3 + start_period: 30s + restart: unless-stopped +volumes: + service-state: + recoll-index: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..cd99d59 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +mkdir -p "${RECOLL_MODERN_STATE_ROOT:-/var/lib/recoll-modern}" \ + /var/lib/recoll "${RECOLL_MODERN_RECOLL_CONFIG_DIR:-/var/lib/recoll-modern/recoll-config}" +config_dir="${RECOLL_MODERN_RECOLL_CONFIG_DIR:-/var/lib/recoll-modern/recoll-config}" +for name in recoll.conf fields; do + if [ ! -e "$config_dir/$name" ]; then + cp "/etc/recoll-modern/$name" "$config_dir/$name" + fi +done +exec recoll-modern diff --git a/docker/fields b/docker/fields new file mode 100644 index 0000000..5ada3cc --- /dev/null +++ b/docker/fields @@ -0,0 +1,5 @@ +[prefixes] +collection = XYCOL + +[stored] +collection = diff --git a/docker/recoll.conf b/docker/recoll.conf new file mode 100644 index 0000000..3bf1207 --- /dev/null +++ b/docker/recoll.conf @@ -0,0 +1,4 @@ +topdirs = /documents +dbdir = /var/lib/recoll +idxstoredocuments = 1 +metadatacmds = ; collection = /usr/local/bin/recoll-membership %f diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index aa6f761..1199001 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -6,8 +6,8 @@ How `recoll-modern` is built today. Update in the same commit as any structural The Python service scaffold, SQLite schema, collection and membership repositories, document discovery/identity modules, durable job worker, engine adapter, authentication service, and initial -HTTP shell, membership routes, job routes, scoped search, and document retrieval are built. Docker -packaging and real-container contract coverage remain. Recoll behaviour +HTTP shell, membership routes, job routes, scoped search, document retrieval, and Docker packaging are +built. Final contract, recovery and security acceptance coverage remains. Recoll behaviour below was verified against a real container. The goal: a self-hosted service that wraps Recoll's indexing and full-text search behind (a) a modern @@ -30,6 +30,7 @@ web UI and (b) an MCP server, running in Docker, with several independent collec | `src/recoll_modern/indexing/handlers.py` | Scans membership sources before strict-mode mutations and records file errors | | `src/recoll_modern/indexing/map_export.py` | Writes a durable atomic map from canonical paths to opaque collection IDs | | `helper/` | C reader run by Recoll's `metadatacmds` for each indexed file | +| `Dockerfile`, `compose.yaml`, `docker/` | Non-root Recoll runtime, read-only document mount, persistent state/index volumes and config templates | | `src/recoll_modern/engine/` | Shared Protocol, deterministic fake, and isolated Recoll Python/indexer adapter | | `src/recoll_modern/domain/auth.py`, `storage/auth.py` | Argon2 password checks, seeded administrator, hashed server-side sessions and CSRF verification | | `src/recoll_modern/api/` | HTTP auth, health, file browsing, collection/membership/rule/job resources, search, document retrieval, prefix/proxy handling and JSON errors | @@ -115,6 +116,13 @@ membership in SQLite before returning them. Full-text and original-file routes r path under the configured root. Original responses use range-capable `FileResponse`, with inline display limited to safe media types. +The image installs Recoll and its Python binding from Debian trixie, and syncs the application into a +system-site-packages venv so the binding remains importable. A separate build stage compiles the C +helper. The non-root process copies Recoll config templates into the persistent state volume only when +absent; Recoll may generate additional support files there. Compose mounts documents read-only and +keeps the SQLite/config and Xapian index volumes separate. Initial health can be 503 until an indexing +job creates the first index. + The resource groups are auth, health, read-only file browsing, collections and membership sources, directory rules, scoped search, document text/original retrieval, and indexing jobs. Search callers select returned fields; only `document_id` is unconditional, and inline full text is bounded. Every diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 91d6f7e..6e9b40a 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -301,3 +301,18 @@ token. The previous token becomes invalid immediately. **Alternatives:** store the raw CSRF token; derive a repeatable token from the cookie and a server key. **Consequences:** clients should refresh their CSRF token after calling the session endpoint. Session storage contains no raw cookie or CSRF token. + +--- + +## 2026-09-26 — Persist writable Recoll config beside service state +**Status:** accepted +**Context:** Debian Recoll 1.43 writes generated support files into its config directory. A non-root +container cannot write to image-owned `/etc/recoll-modern`; a live first-boot test surfaced this. +**Decision:** keep immutable templates under `/etc/recoll-modern`, copy each missing template into +`/var/lib/recoll-modern/recoll-config` on startup, and point both the engine and indexer there. Never +overwrite an existing runtime config file. Keep the Recoll index in its own named volume. +**Alternatives:** chown `/etc/recoll-modern` in the image (runtime changes would not survive container +replacement); run the service as root. +**Consequences:** service-state backups include generated Recoll configuration, and config changes made +by an operator persist across image upgrades. Template updates require an explicit migration if an +existing config should adopt them. diff --git a/docs/TODO.md b/docs/TODO.md index 54eb82a..7e62980 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -15,13 +15,14 @@ Where the workflow stands (`AGENTS.md` -> Workflow: brainstorm -> spec -> plan - selectable search fields, subpath hosting and the packaging seam for deferred desktop spec 005. - Plan `docs/plans/001-search-engine-api.md`: **approved 2026-09-18.** Its listed Python runtime and development dependencies are approved. Execution follows the plan task by task. -- Plan Tasks 1–3 are complete and independently reviewed. Tasks 4–12 add discovery, content-hash +- Plan Tasks 1–3 are complete and independently reviewed. Tasks 4–13 add discovery, content-hash identity, additive membership, durable jobs, the atomic membership map, the engine boundary, session authentication, the HTTP shell, membership/rule/job routes, scoped search and document - retrieval. Their checkboxes are current in the plan. The next task is Docker packaging. + retrieval and Docker packaging. Their checkboxes are current in the plan. The next task is final + contract, recovery, security and acceptance coverage. -The next action is Task 13, "Package Recoll and the service with Docker Compose." Do not repeat Tasks -1–12. +The next action is Task 14, "Complete contract, recovery, security, and acceptance coverage." Do not +repeat Tasks 1–13. Things a resuming agent will otherwise get wrong: @@ -48,7 +49,7 @@ Things a resuming agent will otherwise get wrong: exact-path text retrieval and whether the Python API exposes a native numeric relevance score. - [x] Scaffold Python + FastAPI with `uv`; verify install/dev/test/lint/typecheck/build by running them, then record the verified ones in `AGENTS.md`. -- [ ] Dockerfile + compose: Recoll in the image, document volume mounted `:ro`, writable volumes for +- [x] Dockerfile + compose: Recoll in the image, document volume mounted `:ro`, writable volumes for the index and SQLite. - [x] Write the `metadatacmds` membership helper as a fast-starting non-Python program, fed by a flat map exported from SQLite. A `python3` helper made a 400-file index 9x slower; Recoll spawns it diff --git a/docs/plans/001-search-engine-api.md b/docs/plans/001-search-engine-api.md index 632595e..3af9581 100644 --- a/docs/plans/001-search-engine-api.md +++ b/docs/plans/001-search-engine-api.md @@ -902,7 +902,7 @@ git commit -m "feat: add scoped search and retrieval" - Produces: `docker compose up --build` service on configured host port and root path - Consumes: `recoll-modern`, `recoll-membership`, persisted state and document volumes -- [ ] **Step 1: Write failing container smoke tests** +- [x] **Step 1: Write failing container smoke tests** ```python def test_document_mount_is_read_only(compose_service: ComposeService) -> None: @@ -916,7 +916,7 @@ def test_recoll_binding_and_helper_are_present(compose_service: ComposeService) The fixture skips with `Docker Compose service is not running` when unavailable for normal host test runs; a dedicated `RECOLL_MODERN_REQUIRE_DOCKER=1` mode converts absence into failure for container CI. -- [ ] **Step 2: Add the multi-stage image and Compose deployment** +- [x] **Step 2: Add the multi-stage image and Compose deployment** Base both stages on `debian:trixie-slim`; copy uv from `ghcr.io/astral-sh/uv:0.12.16`. Install `build-essential` only in the helper build stage. Runtime installs `python3`, `python3-recoll`, `recoll`, @@ -933,7 +933,7 @@ Before accepting the image, assert `python -c 'from recoll import recoll'` durin Compose mounts `${RECOLL_MODERN_DOCUMENTS:?}:/documents:ro`, separate named volumes at `/var/lib/recoll-modern` and `/var/lib/recoll`, and seeds admin credentials through environment variables without defaults. Healthcheck calls `/health` at the configured prefix. -- [ ] **Step 3: Configure Recoll** +- [x] **Step 3: Configure Recoll** `recoll.conf` sets `topdirs = /documents`, a persistent `dbdir`, text storage, and: @@ -943,7 +943,7 @@ metadatacmds = ; collection = /usr/local/bin/recoll-membership %f Define `collection` as a stored/queryable field in `fields`. The service passes `RECOLL_MODERN_MEMBERSHIP_MAP` to all `recollindex` subprocesses. Never initialize by deleting an existing index. -- [ ] **Step 4: Run image, integration, and read-only tests** +- [x] **Step 4: Run image, integration, and read-only tests** Run: @@ -957,7 +957,7 @@ docker compose down Expected: image and configuration succeed; generated corpus indexes and searches on a read-only mount; shutdown preserves volumes. Do not pass `-v` to `docker compose down`. -- [ ] **Step 5: Document and commit** +- [x] **Step 5: Document and commit** Document first boot, subpath proxying, trusted proxies, volumes, admin seed rotation, backups, health, and explicit non-destructive reindex behavior. Record image/process boundaries in architecture and any packaging choices in decisions. diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..86d51b9 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Container-level service checks.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..56462f8 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,31 @@ +"""Run integration checks only against an explicitly running Compose service.""" + +import os +import subprocess +from collections.abc import Callable + +import pytest + + +@pytest.fixture +def compose_exec() -> Callable[[list[str]], subprocess.CompletedProcess[str]]: + try: + status = subprocess.run( + ["docker", "compose", "ps", "-q", "app"], capture_output=True, text=True, + check=False, timeout=5, + ) + running = status.returncode == 0 and bool(status.stdout.strip()) + except (OSError, subprocess.TimeoutExpired): + running = False + if not running: + if os.environ.get("RECOLL_MODERN_REQUIRE_DOCKER") == "1": + pytest.fail("Docker Compose service is not running") + pytest.skip("Docker Compose service is not running") + + def run(command: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", "compose", "exec", "-T", "app", *command], + capture_output=True, text=True, check=False, timeout=30, + ) + + return run diff --git a/tests/integration/test_container_smoke.py b/tests/integration/test_container_smoke.py new file mode 100644 index 0000000..511a6e8 --- /dev/null +++ b/tests/integration/test_container_smoke.py @@ -0,0 +1,13 @@ +"""Image runtime and Recoll binding contracts.""" + +import subprocess +from collections.abc import Callable + + +def test_recoll_binding_and_helper_are_present( + compose_exec: Callable[[list[str]], subprocess.CompletedProcess[str]], +) -> None: + binding = compose_exec(["python", "-c", "from recoll import recoll"]) + assert binding.returncode == 0, binding.stderr + helper = compose_exec(["recoll-membership", "/documents/none"]) + assert helper.returncode == 0, helper.stderr diff --git a/tests/integration/test_read_only_corpus.py b/tests/integration/test_read_only_corpus.py new file mode 100644 index 0000000..df4fdc7 --- /dev/null +++ b/tests/integration/test_read_only_corpus.py @@ -0,0 +1,60 @@ +"""The service must not be able to mutate its document mount.""" + +import os +import subprocess +import time +from collections.abc import Callable + +import httpx +import pytest + + +def test_document_mount_is_read_only( + compose_exec: Callable[[list[str]], subprocess.CompletedProcess[str]], +) -> None: + result = compose_exec(["sh", "-c", "touch /documents/must-fail"]) + assert result.returncode != 0 + + +def test_generated_corpus_indexes_and_searches( + compose_exec: Callable[[list[str]], subprocess.CompletedProcess[str]], +) -> None: + corpus_file = os.environ.get("RECOLL_MODERN_TEST_CORPUS_FILE") + if corpus_file is None: + pytest.skip("Set RECOLL_MODERN_TEST_CORPUS_FILE for generated-corpus verification") + assert compose_exec(["test", "-f", f"/documents/{corpus_file}"]).returncode == 0 + port = os.environ.get("RECOLL_MODERN_PUBLISH_PORT", "8000") + username = os.environ["RECOLL_MODERN_ADMIN_USERNAME"] + password = os.environ["RECOLL_MODERN_ADMIN_PASSWORD"] + prefix = os.environ.get("RECOLL_MODERN_ROOT_PATH", "/").rstrip("/") + with httpx.Client(base_url=f"http://127.0.0.1:{port}{prefix}", timeout=10) as client: + login = client.post("/auth/login", json={"username": username, "password": password}) + assert login.status_code == 200, login.text + csrf = login.json()["csrf_token"] + cookie = login.cookies.get("recoll_modern_session") + assert cookie + headers = {"X-CSRF-Token": csrf, "Cookie": f"recoll_modern_session={cookie}"} + created = client.post( + "/collections", json={"name": f"Integration {time.time_ns()}"}, headers=headers + ) + assert created.status_code == 201, created.text + collection_id = created.json()["id"] + queued = client.post( + f"/collections/{collection_id}/memberships", headers=headers, + json={"sources": [{"type": "file", "path": corpus_file}]}, + ) + assert queued.status_code == 202, queued.text + job_id = queued.json()["job"]["id"] + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + job = client.get(f"/jobs/{job_id}", headers=headers).json()["job"] + if job["state"] not in ("queued", "running"): + break + time.sleep(0.1) + assert job["state"] == "completed", job + result = client.get( + "/search", params={"q": "zebraplatypus", "collections": collection_id}, + headers=headers, + ) + assert result.status_code == 200, result.text + assert result.json()["hits"], result.json()