feat: add membership provenance and rules

src/recoll_modern/domain/membership.py

@@ -0,0 +1,95 @@
+"""Validate and apply collection membership sources."""
+
+from dataclasses import dataclass
+
+from recoll_modern.indexing.discovery import DiscoveryReport, Source, discover_sources
+from recoll_modern.indexing.paths import PathPolicy
+from recoll_modern.indexing.reconcile import reconcile_rule
+from recoll_modern.storage.documents import DocumentRepository
+from recoll_modern.storage.membership import MembershipRepository
+
+
+@dataclass(frozen=True, slots=True)
+class MembershipPlanResult:
+    discovery: DiscoveryReport
+    changed_paths: tuple[str, ...]
+
+
+class MembershipService:
+    def __init__(
+        self,
+        policy: PathPolicy,
+        documents: DocumentRepository,
+        memberships: MembershipRepository,
+    ) -> None:
+        self._policy = policy
+        self._documents = documents
+        self._memberships = memberships
+
+    def plan_add(self, collection_id: str, sources: list[Source]) -> MembershipPlanResult:
+        for source in sources:
+            if source.type == "directory":
+                parent = self._policy.normalize(source.path)
+                if any(
+                    not self._policy.normalize(path).startswith(parent + "/")
+                    for path in source.excluded_directories
+                ):
+                    raise ValueError("excluded directory must be a descendant")
+        discovered = [(source, discover_sources(self._policy, [source])) for source in sources]
+        files = list({
+            file.relative_path: file
+            for _, report in discovered
+            for file in report.files
+        }.values())
+        observations = self._documents.observe_files(files)
+        changed = {
+            document.canonical_path
+            for document in observations.documents
+            if document.canonical_path is not None
+        }
+        for source, report in discovered:
+            if source.type == "directory" and source.mode == "persistent":
+                rule = self._memberships.create_rule(
+                    collection_id,
+                    self._policy.normalize(source.path),
+                    tuple(self._policy.normalize(path) for path in source.excluded_directories),
+                )
+                changed.update(self._memberships.replace_rule_members(
+                    rule.id, [file.content_hash for file in report.files]
+                ))
+            else:
+                for file in report.files:
+                    self._memberships.add_direct(collection_id, file.content_hash)
+                    record = self._documents.get(file.content_hash)
+                    if record is not None and record.canonical_path is not None:
+                        changed.add(record.canonical_path)
+        return MembershipPlanResult(
+            DiscoveryReport(
+                tuple(files), tuple(error for _, report in discovered for error in report.errors)
+            ),
+            tuple(sorted(changed)),
+        )
+
+    def plan_remove_document(self, collection_id: str, content_hash: str) -> tuple[str, ...]:
+        record = self._documents.get(content_hash)
+        self._memberships.remove_document(collection_id, content_hash)
+        return (record.canonical_path,) if record and record.canonical_path else ()
+
+    def plan_patch_rule(
+        self, rule_id: str, excluded_directories: tuple[str, ...]
+    ) -> MembershipPlanResult:
+        rule = self._memberships.get_rule(rule_id)
+        if rule is None:
+            raise ValueError("directory rule was not found")
+        parent = rule.relative_path
+        if any(
+            not self._policy.normalize(path).startswith(parent + "/")
+            for path in excluded_directories
+        ):
+            raise ValueError("excluded directory must be a descendant")
+        self._memberships.patch_rule(rule_id, excluded_directories)
+        result = reconcile_rule(rule_id, self._policy, self._documents, self._memberships)
+        return MembershipPlanResult(result.discovery, result.changed_paths)
+
+    def plan_delete_rule(self, rule_id: str) -> tuple[str, ...]:
+        return tuple(self._memberships.delete_rule(rule_id))

src/recoll_modern/indexing/reconcile.py

@@ -0,0 +1,45 @@
+"""Refresh one persistent directory rule from the read-only document tree."""
+
+from dataclasses import dataclass
+
+from recoll_modern.indexing.discovery import DiscoveryReport, Source, discover_sources
+from recoll_modern.indexing.paths import PathPolicy
+from recoll_modern.storage.documents import DocumentRepository
+from recoll_modern.storage.membership import MembershipRepository
+
+
+@dataclass(frozen=True, slots=True)
+class ReconcileResult:
+    discovery: DiscoveryReport
+    changed_paths: tuple[str, ...]
+
+
+def reconcile_rule(
+    rule_id: str,
+    policy: PathPolicy,
+    documents: DocumentRepository,
+    memberships: MembershipRepository,
+) -> ReconcileResult:
+    rule = memberships.get_rule(rule_id)
+    if rule is None:
+        raise ValueError("directory rule was not found")
+    report = discover_sources(policy, [Source.directory(
+        rule.relative_path,
+        mode="persistent",
+        excluded_directories=rule.excluded_directories,
+    )])
+    skipped = (*rule.excluded_directories, *(error.relative_path for error in report.errors))
+    known_paths = tuple(
+        path for path in documents.known_paths_under(rule.relative_path)
+        if not any(path == prefix or path.startswith(prefix + "/") for prefix in skipped)
+    )
+    observed = documents.observe_files(list(report.files), reconcile_paths=known_paths)
+    changed = memberships.replace_rule_members(
+        rule_id, [file.content_hash for file in report.files]
+    )
+    canonical_changes = [
+        document.canonical_path
+        for document in observed.documents
+        if document.canonical_path is not None
+    ]
+    return ReconcileResult(report, tuple(sorted(set(changed + canonical_changes))))

src/recoll_modern/storage/documents.py

@@ -32,6 +32,19 @@ class DocumentRepository:
         with self._database.connect() as connection:
             return _record(connection, content_hash)
 
+    def known_paths_under(self, relative_directory: str) -> tuple[str, ...]:
+        """Return known live paths beneath a directory for explicit reconciliation."""
+        prefix = relative_directory + "/"
+        with self._database.connect() as connection:
+            return tuple(
+                str(row[0])
+                for row in connection.execute(
+                    "SELECT relative_path FROM document_paths WHERE missing_at IS NULL "
+                    "ORDER BY relative_path"
+                )
+                if str(row[0]).startswith(prefix)
+            )
+
     def observe_files(
         self,
         files: list[ObservedFile],

src/recoll_modern/storage/membership.py

@@ -0,0 +1,203 @@
+"""SQLite provenance for direct and directory-rule membership."""
+
+import secrets
+import sqlite3
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+from recoll_modern.storage.database import Database
+
+
+@dataclass(frozen=True, slots=True)
+class DirectoryRule:
+    id: str
+    collection_id: str
+    relative_path: str
+    excluded_directories: tuple[str, ...]
+
+
+class MembershipRepository:
+    def __init__(self, database: Database) -> None:
+        self._database = database
+
+    def create_rule(
+        self, collection_id: str, relative_path: str, excluded_directories: tuple[str, ...]
+    ) -> DirectoryRule:
+        now = datetime.now(UTC).isoformat()
+        while True:
+            rule_id = f"r{secrets.token_hex(5)}"
+            try:
+                with self._database.transaction() as connection:
+                    connection.execute(
+                        "INSERT INTO directory_rules "
+                        "(id, collection_id, relative_path, created_at, updated_at) "
+                        "VALUES (?, ?, ?, ?, ?)",
+                        (rule_id, collection_id, relative_path, now, now),
+                    )
+                    connection.executemany(
+                        "INSERT INTO directory_rule_exclusions(rule_id, relative_path) "
+                        "VALUES (?, ?)",
+                        [(rule_id, path) for path in excluded_directories],
+                    )
+            except sqlite3.IntegrityError as error:
+                if "directory_rules.id" in str(error):
+                    continue
+                raise
+            return DirectoryRule(rule_id, collection_id, relative_path, excluded_directories)
+
+    def get_rule(self, rule_id: str) -> DirectoryRule | None:
+        with self._database.connect() as connection:
+            row = connection.execute(
+                "SELECT id, collection_id, relative_path FROM directory_rules WHERE id = ?",
+                (rule_id,),
+            ).fetchone()
+            if row is None:
+                return None
+            exclusions = tuple(
+                str(item[0])
+                for item in connection.execute(
+                    "SELECT relative_path FROM directory_rule_exclusions "
+                    "WHERE rule_id = ? ORDER BY relative_path",
+                    (rule_id,),
+                )
+            )
+            return DirectoryRule(str(row[0]), str(row[1]), str(row[2]), exclusions)
+
+    def patch_rule(self, rule_id: str, excluded_directories: tuple[str, ...]) -> None:
+        with self._database.transaction() as connection:
+            connection.execute(
+                "DELETE FROM directory_rule_exclusions WHERE rule_id = ?", (rule_id,)
+            )
+            connection.executemany(
+                "INSERT INTO directory_rule_exclusions(rule_id, relative_path) VALUES (?, ?)",
+                [(rule_id, path) for path in excluded_directories],
+            )
+            connection.execute(
+                "UPDATE directory_rules SET updated_at = ? WHERE id = ?",
+                (datetime.now(UTC).isoformat(), rule_id),
+            )
+
+    def rule_documents(self, rule_id: str) -> set[str]:
+        with self._database.connect() as connection:
+            return {
+                str(row[0])
+                for row in connection.execute(
+                    "SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
+                )
+            }
+
+    def replace_rule_members(self, rule_id: str, hashes: list[str]) -> list[str]:
+        with self._database.transaction() as connection:
+            old = {
+                str(row[0])
+                for row in connection.execute(
+                    "SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
+                )
+            }
+            affected = old | set(hashes)
+            before = _effective_for_hashes(connection, affected)
+            connection.execute("DELETE FROM rule_memberships WHERE rule_id = ?", (rule_id,))
+            now = datetime.now(UTC).isoformat()
+            connection.executemany(
+                "INSERT INTO rule_memberships(rule_id, document_hash, last_seen_at) "
+                "VALUES (?, ?, ?)",
+                [(rule_id, content_hash, now) for content_hash in set(hashes)],
+            )
+            return _changed_paths(connection, affected, before)
+
+    def delete_rule(self, rule_id: str) -> list[str]:
+        with self._database.transaction() as connection:
+            hashes = {
+                str(row[0])
+                for row in connection.execute(
+                    "SELECT document_hash FROM rule_memberships WHERE rule_id = ?", (rule_id,)
+                )
+            }
+            before = _effective_for_hashes(connection, hashes)
+            connection.execute("DELETE FROM directory_rules WHERE id = ?", (rule_id,))
+            return _changed_paths(connection, hashes, before)
+
+    def add_direct(self, collection_id: str, content_hash: str) -> None:
+        now = datetime.now(UTC).isoformat()
+        with self._database.transaction() as connection:
+            connection.execute(
+                "DELETE FROM membership_exclusions "
+                "WHERE collection_id = ? AND document_hash = ?",
+                (collection_id, content_hash),
+            )
+            connection.execute(
+                "INSERT OR IGNORE INTO direct_memberships "
+                "(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
+                (collection_id, content_hash, now),
+            )
+
+    def has_direct(self, collection_id: str, content_hash: str) -> bool:
+        with self._database.connect() as connection:
+            return connection.execute(
+                "SELECT 1 FROM direct_memberships WHERE collection_id = ? AND document_hash = ?",
+                (collection_id, content_hash),
+            ).fetchone() is not None
+
+    def exclude_document(self, collection_id: str, content_hash: str) -> None:
+        with self._database.transaction() as connection:
+            connection.execute(
+                "INSERT OR IGNORE INTO membership_exclusions "
+                "(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
+                (collection_id, content_hash, datetime.now(UTC).isoformat()),
+            )
+
+    def remove_document(self, collection_id: str, content_hash: str) -> None:
+        with self._database.transaction() as connection:
+            connection.execute(
+                "DELETE FROM direct_memberships "
+                "WHERE collection_id = ? AND document_hash = ?",
+                (collection_id, content_hash),
+            )
+            connection.execute(
+                "INSERT OR IGNORE INTO membership_exclusions "
+                "(collection_id, document_hash, created_at) VALUES (?, ?, ?)",
+                (collection_id, content_hash, datetime.now(UTC).isoformat()),
+            )
+
+    def effective_collections(self, content_hash: str) -> list[str]:
+        with self._database.connect() as connection:
+            return _effective(connection, content_hash)
+
+
+def _effective(connection: sqlite3.Connection, content_hash: str) -> list[str]:
+    return [
+        str(row[0])
+        for row in connection.execute(
+            "SELECT collection_id FROM direct_memberships WHERE document_hash = ? "
+            "UNION SELECT r.collection_id FROM rule_memberships AS rm "
+            "JOIN directory_rules AS r ON r.id = rm.rule_id "
+            "WHERE rm.document_hash = ? "
+            "AND NOT EXISTS (SELECT 1 FROM membership_exclusions AS e "
+            "WHERE e.collection_id = r.collection_id AND e.document_hash = rm.document_hash) "
+            "ORDER BY collection_id",
+            (content_hash, content_hash),
+        )
+    ]
+
+
+def _effective_for_hashes(
+    connection: sqlite3.Connection, hashes: set[str]
+) -> dict[str, list[str]]:
+    return {content_hash: _effective(connection, content_hash) for content_hash in hashes}
+
+
+def _changed_paths(
+    connection: sqlite3.Connection, hashes: set[str], before: dict[str, list[str]]
+) -> list[str]:
+    paths: list[str] = []
+    for content_hash in hashes:
+        if before[content_hash] == _effective(connection, content_hash):
+            continue
+        row = connection.execute(
+            "SELECT relative_path FROM document_paths "
+            "WHERE document_hash = ? AND is_canonical = 1 AND missing_at IS NULL",
+            (content_hash,),
+        ).fetchone()
+        if row is not None:
+            paths.append(str(row[0]))
+    return sorted(paths)