feat: add membership provenance and rules

This commit is contained in:
Lucas Winther
2026-09-26 10:32:49 +02:00
parent 9e896fc410
commit e80d59e7d7
10 changed files with 524 additions and 12 deletions
+5 -3
View File
@@ -4,9 +4,9 @@ How `recoll-modern` is built today. Update in the same commit as any structural
## Overview
The Python service scaffold, SQLite schema, collection repository, and document discovery/identity
modules are built. The API, job queue, and Recoll adapter remain intended structure. Recoll behaviour
below was verified against a real container.
The Python service scaffold, SQLite schema, collection and membership repositories, and document
discovery/identity modules are built. The API, job queue, and Recoll adapter remain intended structure.
Recoll behaviour below was verified against a real container.
The goal: a self-hosted service that wraps Recoll's indexing and full-text search behind (a) a modern
web UI and (b) an MCP server, running in Docker, with several independent collections such as
@@ -22,6 +22,8 @@ web UI and (b) an MCP server, running in Docker, with several independent collec
| `src/recoll_modern/` | Python service package; configuration, application factory and CLI boundary |
| `src/recoll_modern/indexing/` | Root-relative path policy, read-only discovery, stable file hashing |
| `src/recoll_modern/storage/` | SQLite migrations, collection metadata and content-hash identity |
| `src/recoll_modern/domain/membership.py` | Validates sources and applies direct additions and rule changes |
| `src/recoll_modern/indexing/reconcile.py` | Refreshes persistent directory rules while retaining excluded or unreadable files as present |
| `tests/` | Automated service tests |
Discovery produces content hashes and root-relative paths without writing to the document volume.
+4 -5
View File
@@ -15,12 +15,11 @@ Where the workflow stands (`AGENTS.md` -> Workflow: brainstorm -> spec -> plan -
selectable search fields, subpath hosting and the packaging seam for deferred desktop spec 005.
- Plan `docs/plans/001-search-engine-api.md`: **approved 2026-09-18.** Its listed Python runtime and
development dependencies are approved. Execution follows the plan task by task.
- Plan Tasks 1–3 are complete and independently reviewed. Task 4 adds read-only discovery, stable
hashing and content-hash identity; its checkboxes are current in the plan. The next task is additive
membership and directory rules.
- Plan Tasks 1–3 are complete and independently reviewed. Tasks 4–5 add read-only discovery,
content-hash identity, additive membership and directory rules. Their checkboxes are current in the
plan. The next task is the durable job queue.
The next action is Task 5, "Implement additive membership and directory rules." Do not repeat
Tasks 1–4.
The next action is Task 6, "Add the durable single-worker job queue." Do not repeat Tasks 1–5.
Things a resuming agent will otherwise get wrong:
+4 -4
View File
@@ -412,7 +412,7 @@ git commit -m "feat: add document discovery and identity"
- Produces: `MembershipService.plan_add`, `plan_remove_document`, `plan_patch_rule`, `plan_delete_rule`
- Produces: `reconcile_rule(rule_id, policy, repositories) -> ReconcileResult`
- [ ] **Step 1: Write failing provenance tests**
- [x] **Step 1: Write failing provenance tests**
```python
def test_direct_membership_wins_over_rule_exclusions(repo: MembershipRepository) -> None:
@@ -432,11 +432,11 @@ def test_rule_reconciliation_removes_departed_file_without_touching_direct(repo:
assert repo.has_direct(COLLECTION_ID, DEPARTED_HASH)
```
- [ ] **Step 2: Run tests and observe failure**
- [x] **Step 2: Run tests and observe failure**
Run: `uv run pytest tests/domain/test_membership.py tests/indexing/test_reconcile.py tests/storage/test_membership_repository.py -v`
- [ ] **Step 3: Implement desired-state operations**
- [x] **Step 3: Implement desired-state operations**
Snapshot directory results and file sources call `add_direct`; persistent directories create `r` plus ten hex digits, validate exclusions as exact descendants, then atomically replace that rule's materialized membership. Effective membership is this SQL shape:
@@ -455,7 +455,7 @@ ORDER BY collection_id;
Removing a document deletes its direct row and upserts a collection-level rule override. Explicitly adding that file again deletes the override and adds direct membership. Rule deletion returns canonical paths whose effective collection set changed.
- [ ] **Step 4: Verify and commit**
- [x] **Step 4: Verify and commit**
Run: `uv run pytest tests/domain/test_membership.py tests/indexing/test_reconcile.py tests/storage/test_membership_repository.py -v && uv run ruff check . && uv run mypy src tests`
+95
View File
@@ -0,0 +1,95 @@
"""Validate and apply collection membership sources."""
from dataclasses import dataclass
from recoll_modern.indexing.discovery import DiscoveryReport, Source, discover_sources
from recoll_modern.indexing.paths import PathPolicy
from recoll_modern.indexing.reconcile import reconcile_rule
from recoll_modern.storage.documents import DocumentRepository
from recoll_modern.storage.membership import MembershipRepository
@dataclass(frozen=True, slots=True)
class MembershipPlanResult:
discovery: DiscoveryReport
changed_paths: tuple[str, ...]
class MembershipService:
def __init__(
self,
policy: PathPolicy,
documents: DocumentRepository,
memberships: MembershipRepository,
) -> None:
self._policy = policy
self._documents = documents
self._memberships = memberships
def plan_add(self, collection_id: str, sources: list[Source]) -> MembershipPlanResult:
for source in sources:
if source.type == "directory":
parent = self._policy.normalize(source.path)
if any(
not self._policy.normalize(path).startswith(parent + "/")
for path in source.excluded_directories
):
raise ValueError("excluded directory must be a descendant")
discovered = [(source, discover_sources(self._policy, [source])) for source in sources]
files = list({
file.relative_path: file
for _, report in discovered
for file in report.files
}.values())
observations = self._documents.observe_files(files)
changed = {
document.canonical_path
for document in observations.documents
if document.canonical_path is not None
}
for source, report in discovered:
if source.type == "directory" and source.mode == "persistent":
rule = self._memberships.create_rule(
collection_id,
self._policy.normalize(source.path),
tuple(self._policy.normalize(path) for path in source.excluded_directories),
)
changed.update(self._memberships.replace_rule_members(
rule.id, [file.content_hash for file in report.files]
))
else:
for file in report.files:
self._memberships.add_direct(collection_id, file.content_hash)
record = self._documents.get(file.content_hash)
if record is not None and record.canonical_path is not None:
changed.add(record.canonical_path)
return MembershipPlanResult(
DiscoveryReport(
tuple(files), tuple(error for _, report in discovered for error in report.errors)
),
tuple(sorted(changed)),
)
def plan_remove_document(self, collection_id: str, content_hash: str) -> tuple[str, ...]:
record = self._documents.get(content_hash)
self._memberships.remove_document(collection_id, content_hash)
return (record.canonical_path,) if record and record.canonical_path else ()
def plan_patch_rule(
self, rule_id: str, excluded_directories: tuple[str, ...]
) -> MembershipPlanResult:
rule = self._memberships.get_rule(rule_id)
if rule is None:
raise ValueError("directory rule was not found")
parent = rule.relative_path
if any(
not self._policy.normalize(path).startswith(parent + "/")
for path in excluded_directories
):
raise ValueError("excluded directory must be a descendant")
self._memberships.patch_rule(rule_id, excluded_directories)
result = reconcile_rule(rule_id, self._policy, self._documents, self._memberships)
return MembershipPlanResult(result.discovery, result.changed_paths)
def plan_delete_rule(self, rule_id: str) -> tuple[str, ...]:
return tuple(self._memberships.delete_rule(rule_id))
+45
View File
@@ -0,0 +1,45 @@
"""Refresh one persistent directory rule from the read-only document tree."""
from dataclasses import dataclass
from recoll_modern.indexing.discovery import DiscoveryReport, Source, discover_sources
from recoll_modern.indexing.paths import PathPolicy
from recoll_modern.storage.documents import DocumentRepository
from recoll_modern.storage.membership import MembershipRepository
@dataclass(frozen=True, slots=True)
class ReconcileResult:
discovery: DiscoveryReport
changed_paths: tuple[str, ...]
def reconcile_rule(
rule_id: str,
policy: PathPolicy,
documents: DocumentRepository,
memberships: MembershipRepository,
) -> ReconcileResult:
rule = memberships.get_rule(rule_id)
if rule is None:
raise ValueError("directory rule was not found")
report = discover_sources(policy, [Source.directory(
rule.relative_path,
mode="persistent",
excluded_directories=rule.excluded_directories,
)])
skipped = (*rule.excluded_directories, *(error.relative_path for error in report.errors))
known_paths = tuple(
path for path in documents.known_paths_under(rule.relative_path)
if not any(path == prefix or path.startswith(prefix + "/") for prefix in skipped)
)
observed = documents.observe_files(list(report.files), reconcile_paths=known_paths)
changed = memberships.replace_rule_members(
rule_id, [file.content_hash for file in report.files]
)
canonical_changes = [
document.canonical_path
for document in observed.documents
if document.canonical_path is not None
]
return ReconcileResult(report, tuple(sorted(set(changed + canonical_changes))))
+13
View File
@@ -32,6 +32,19 @@ class DocumentRepository:
with self._database.connect() as connection:
return _record(connection, content_hash)
def known_paths_under(self, relative_directory: str) -> tuple[str, ...]:
"""Return known live paths beneath a directory for explicit reconciliation."""
prefix = relative_directory + "/"
with self._database.connect() as connection:
return tuple(
str(row[0])
for row in connection.execute(
"SELECT relative_path FROM document_paths WHERE missing_at IS NULL "
"ORDER BY relative_path"
)
if str(row[0]).startswith(prefix)
)
def observe_files(
self,
files: list[ObservedFile],
+203
View File
@@ -0,0 +1,203 @@
"""SQLite provenance for direct and directory-rule membership."""
import secrets
import sqlite3
from dataclasses import dataclass
from datetime import UTC, datetime
from recoll_modern.storage.database import Database
@dataclass(frozen=True, slots=True)
class DirectoryRule:
id: str
collection_id: str
relative_path: str
excluded_directories: tuple[str, ...]
class MembershipRepository:
def __init__(self, database: Database) -> None:
self._database = database
def create_rule(
self, collection_id: str, relative_path: str, excluded_directories: tuple[str, ...]
) -> DirectoryRule:
now = datetime.now(UTC).isoformat()
while True:
rule_id = f"r{secrets.token_hex(5)}"
try:
with self._database.transaction() as connection:
connection.execute(
"INSERT INTO directory_rules "
"(id, collection_id, relative_path, created_at, updated_at) "
"VALUES (?, ?, ?, ?, ?)",
(rule_id, collection_id, relative_path, now, now),
)
connection.executemany(
"INSERT INTO directory_rule_exclusions(rule_id, relative_path) "
"VALUES (?, ?)",
[(rule_id, path) for path in excluded_directories],
)
except sqlite3.IntegrityError as error:
if "directory_rules.id" in str(error):
continue
raise
return DirectoryRule(rule_id, collection_id, relative_path, excluded_directories)
def get_rule(self, rule_id: str) -> DirectoryRule | None:
with self._database.connect() as connection:
row = connection.execute(
"SELECT id, collection_id, relative_path FROM directory_rules WHERE id = ?",
(rule_id,),
).fetchone()
if row is None:
return None
exclusions = tuple(
str(item[0])
for item in connection.execute(
"SELECT relative_path FROM directory_rule_exclusions "
"WHERE rule_id = ? ORDER BY relative_path",
(rule_id,),
)
)
return DirectoryRule(str(row[0]), str(row[1]), str(row[2]), exclusions)
def patch_rule(self, rule_id: str, excluded_directories: tuple[str, ...]) -> None:
with self._database.transaction() as connection:
connection.execute(
"DELETE FROM directory_rule_exclusions WHERE rule_id = ?", (rule_id,)
)
connection.executemany(
"INSERT INTO directory_rule_exclusions(rule_id, relative_path) VALUES (?, ?)",
[(rule_id, path) for path in excluded_directories],
)
connection.execute(
"UPDATE directory_rules SET updated_at = ? WHERE id = ?",
(datetime.now(UTC).isoformat(), rule_id),
)
def rule_documents(self, rule_id: str) -> set[str]:
with self._database.connect() as connection:
return {
str(row[0])
for row in connection.execute(
"SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
)
}
def replace_rule_members(self, rule_id: str, hashes: list[str]) -> list[str]:
with self._database.transaction() as connection:
old = {
str(row[0])
for row in connection.execute(
"SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
)
}
affected = old | set(hashes)
before = _effective_for_hashes(connection, affected)
connection.execute("DELETE FROM rule_memberships WHERE rule_id = ?", (rule_id,))
now = datetime.now(UTC).isoformat()
connection.executemany(
"INSERT INTO rule_memberships(rule_id, document_hash, last_seen_at) "
"VALUES (?, ?, ?)",
[(rule_id, content_hash, now) for content_hash in set(hashes)],
)
return _changed_paths(connection, affected, before)
def delete_rule(self, rule_id: str) -> list[str]:
with self._database.transaction() as connection:
hashes = {
str(row[0])
for row in connection.execute(
"SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
)
}
before = _effective_for_hashes(connection, hashes)
connection.execute("DELETE FROM directory_rules WHERE id = ?", (rule_id,))
return _changed_paths(connection, hashes, before)
def add_direct(self, collection_id: str, content_hash: str) -> None:
now = datetime.now(UTC).isoformat()
with self._database.transaction() as connection:
connection.execute(
"DELETE FROM membership_exclusions "
"WHERE collection_id = ? AND document_hash = ?",
(collection_id, content_hash),
)
connection.execute(
"INSERT OR IGNORE INTO direct_memberships "
"(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
(collection_id, content_hash, now),
)
def has_direct(self, collection_id: str, content_hash: str) -> bool:
with self._database.connect() as connection:
return connection.execute(
"SELECT 1 FROM direct_memberships WHERE collection_id = ? AND document_hash = ?",
(collection_id, content_hash),
).fetchone() is not None
def exclude_document(self, collection_id: str, content_hash: str) -> None:
with self._database.transaction() as connection:
connection.execute(
"INSERT OR IGNORE INTO membership_exclusions "
"(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
(collection_id, content_hash, datetime.now(UTC).isoformat()),
)
def remove_document(self, collection_id: str, content_hash: str) -> None:
with self._database.transaction() as connection:
connection.execute(
"DELETE FROM direct_memberships "
"WHERE collection_id = ? AND document_hash = ?",
(collection_id, content_hash),
)
connection.execute(
"INSERT OR IGNORE INTO membership_exclusions "
"(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
(collection_id, content_hash, datetime.now(UTC).isoformat()),
)
def effective_collections(self, content_hash: str) -> list[str]:
with self._database.connect() as connection:
return _effective(connection, content_hash)
def _effective(connection: sqlite3.Connection, content_hash: str) -> list[str]:
return [
str(row[0])
for row in connection.execute(
"SELECT collection_id FROM direct_memberships WHERE document_hash = ? "
"UNION SELECT r.collection_id FROM rule_memberships AS rm "
"JOIN directory_rules AS r ON r.id = rm.rule_id "
"WHERE rm.document_hash = ? "
"AND NOT EXISTS (SELECT 1 FROM membership_exclusions AS e "
"WHERE e.collection_id = r.collection_id AND e.document_hash = rm.document_hash) "
"ORDER BY collection_id",
(content_hash, content_hash),
)
]
def _effective_for_hashes(
connection: sqlite3.Connection, hashes: set[str]
) -> dict[str, list[str]]:
return {content_hash: _effective(connection, content_hash) for content_hash in hashes}
def _changed_paths(
connection: sqlite3.Connection, hashes: set[str], before: dict[str, list[str]]
) -> list[str]:
paths: list[str] = []
for content_hash in hashes:
if before[content_hash] == _effective(connection, content_hash):
continue
row = connection.execute(
"SELECT relative_path FROM document_paths "
"WHERE document_hash = ? AND is_canonical = 1 AND missing_at IS NULL",
(content_hash,),
).fetchone()
if row is not None:
paths.append(str(row[0]))
return sorted(paths)
+42
View File
@@ -0,0 +1,42 @@
from pathlib import Path
import pytest
from recoll_modern.domain.membership import MembershipService
from recoll_modern.domain.models import CollectionCreate
from recoll_modern.indexing.discovery import Source
from recoll_modern.indexing.paths import PathPolicy
from recoll_modern.storage.collections import SQLiteCollectionRepository
from recoll_modern.storage.database import Database
from recoll_modern.storage.documents import DocumentRepository
from recoll_modern.storage.membership import MembershipRepository
def test_plan_add_rejects_exclusion_outside_rule(tmp_path: Path) -> None:
root = tmp_path / "docs"
root.mkdir()
database = Database(tmp_path / "state.sqlite3")
database.migrate()
collection_id = SQLiteCollectionRepository(database).create(CollectionCreate("Books")).id
service = MembershipService(
PathPolicy(root), DocumentRepository(database), MembershipRepository(database)
)
with pytest.raises(ValueError, match="descendant"):
service.plan_add(collection_id, [Source.directory(
"books", mode="persistent", excluded_directories=("other",)
)])
def test_plan_add_direct_file_keeps_existing_rule_membership(tmp_path: Path) -> None:
root = tmp_path / "docs"
root.mkdir()
(root / "a.txt").write_text("a")
database = Database(tmp_path / "state.sqlite3")
database.migrate()
collection_id = SQLiteCollectionRepository(database).create(CollectionCreate("Books")).id
membership = MembershipRepository(database)
service = MembershipService(PathPolicy(root), DocumentRepository(database), membership)
result = service.plan_add(collection_id, [Source.file("a.txt")])
content_hash = result.discovery.files[0].content_hash
assert membership.has_direct(collection_id, content_hash)
assert membership.effective_collections(content_hash) == [collection_id]
+54
View File
@@ -0,0 +1,54 @@
from pathlib import Path
from recoll_modern.domain.models import CollectionCreate
from recoll_modern.indexing.paths import PathPolicy
from recoll_modern.indexing.reconcile import reconcile_rule
from recoll_modern.storage.collections import SQLiteCollectionRepository
from recoll_modern.storage.database import Database
from recoll_modern.storage.documents import DocumentRepository
from recoll_modern.storage.membership import MembershipRepository
def test_rule_reconciliation_removes_departed_file_without_touching_direct(tmp_path: Path) -> None:
root = tmp_path / "docs"
root.mkdir()
books = root / "books"
books.mkdir()
(books / "keep.txt").write_text("keep")
(books / "gone.txt").write_text("gone")
database = Database(tmp_path / "state.sqlite3")
database.migrate()
collection_id = SQLiteCollectionRepository(database).create(CollectionCreate("Books")).id
membership = MembershipRepository(database)
documents = DocumentRepository(database)
rule = membership.create_rule(collection_id, "books", ())
first = reconcile_rule(rule.id, PathPolicy(root), documents, membership)
departed = next(
file for file in first.discovery.files if file.relative_path == "books/gone.txt"
)
membership.add_direct(collection_id, departed.content_hash)
(books / "gone.txt").unlink()
second = reconcile_rule(rule.id, PathPolicy(root), documents, membership)
assert departed.content_hash not in membership.rule_documents(rule.id)
assert membership.has_direct(collection_id, departed.content_hash)
assert second.discovery.errors == ()
def test_excluded_file_is_not_marked_missing(tmp_path: Path) -> None:
root = tmp_path / "docs"
skipped = root / "books" / "skip"
skipped.mkdir(parents=True)
(skipped / "a.txt").write_text("a")
database = Database(tmp_path / "state.sqlite3")
database.migrate()
collection_id = SQLiteCollectionRepository(database).create(CollectionCreate("Books")).id
membership = MembershipRepository(database)
documents = DocumentRepository(database)
rule = membership.create_rule(collection_id, "books", ())
first = reconcile_rule(rule.id, PathPolicy(root), documents, membership)
content_hash = first.discovery.files[0].content_hash
membership.patch_rule(rule.id, ("books/skip",))
reconcile_rule(rule.id, PathPolicy(root), documents, membership)
record = documents.get(content_hash)
assert record is not None and record.state == "present"
assert membership.rule_documents(rule.id) == set()
@@ -0,0 +1,59 @@
from pathlib import Path
from recoll_modern.domain.models import CollectionCreate
from recoll_modern.storage.collections import SQLiteCollectionRepository
from recoll_modern.storage.database import Database
from recoll_modern.storage.membership import MembershipRepository
HASH = "a" * 64
def _repo(tmp_path: Path) -> tuple[MembershipRepository, str]:
database = Database(tmp_path / "state.sqlite3")
database.migrate()
collection = SQLiteCollectionRepository(database).create(CollectionCreate("Books"))
with database.transaction() as connection:
connection.execute(
"INSERT INTO documents VALUES (?, 1, 'present', ?, ?)",
(HASH, "2026-01-01T00:00:00+00:00", "2026-01-01T00:00:00+00:00"),
)
connection.execute(
"INSERT INTO document_paths "
"(document_hash, relative_path, is_canonical, last_seen_at) VALUES (?, ?, 1, ?)",
(HASH, "books/a.txt", "2026-01-01T00:00:00+00:00"),
)
return MembershipRepository(database), collection.id
def test_direct_membership_wins_over_rule_exclusions(tmp_path: Path) -> None:
repo, collection_id = _repo(tmp_path)
repo.add_direct(collection_id, HASH)
repo.exclude_document(collection_id, HASH)
assert repo.effective_collections(HASH) == [collection_id]
def test_removing_one_overlapping_rule_keeps_other_source(tmp_path: Path) -> None:
repo, collection_id = _repo(tmp_path)
rule_a = repo.create_rule(collection_id, "books", ())
rule_b = repo.create_rule(collection_id, "other", ())
repo.replace_rule_members(rule_a.id, [HASH])
repo.replace_rule_members(rule_b.id, [HASH])
repo.delete_rule(rule_a.id)
assert repo.effective_collections(HASH) == [collection_id]
def test_removing_document_suppresses_rule_until_explicit_add(tmp_path: Path) -> None:
repo, collection_id = _repo(tmp_path)
rule = repo.create_rule(collection_id, "books", ())
repo.replace_rule_members(rule.id, [HASH])
repo.remove_document(collection_id, HASH)
assert repo.effective_collections(HASH) == []
repo.add_direct(collection_id, HASH)
assert repo.effective_collections(HASH) == [collection_id]
def test_rule_deletion_reports_changed_document_hashes(tmp_path: Path) -> None:
repo, collection_id = _repo(tmp_path)
rule = repo.create_rule(collection_id, "books", ())
repo.replace_rule_members(rule.id, [HASH])
assert repo.delete_rule(rule.id) == ["books/a.txt"]