Archived
266 lines
12 KiB
Python
266 lines
12 KiB
Python
"""Exercise the complete HTTP lifecycle with durable state and a deterministic engine."""
|
|
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
import httpx
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
from recoll_modern.app import create_app
|
|
from recoll_modern.config import Settings
|
|
from recoll_modern.engine.fake import FakeEngine, FakeRecord
|
|
from recoll_modern.storage.database import Database
|
|
from recoll_modern.storage.jobs import JobRepository
|
|
|
|
|
|
def _live_wait(
|
|
client: httpx.Client, path: str, job_id: str, headers: dict[str, str]
|
|
) -> dict[str, object]:
|
|
deadline = time.monotonic() + 30
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = client.get(f"{path}/jobs/{job_id}", headers=headers)
|
|
except httpx.TransportError:
|
|
time.sleep(0.1)
|
|
continue
|
|
if response.status_code == 200:
|
|
job = cast(dict[str, object], response.json()["job"])
|
|
if job["state"] not in ("queued", "running"):
|
|
return job
|
|
time.sleep(0.1)
|
|
raise AssertionError(f"container job {job_id} did not finish")
|
|
|
|
|
|
def _wait(client: TestClient, job_id: str) -> dict[str, object]:
|
|
deadline = time.monotonic() + 5
|
|
while time.monotonic() < deadline:
|
|
job = client.get(f"/recoll/jobs/{job_id}").json()["job"]
|
|
if job["state"] not in ("queued", "running"):
|
|
return cast(dict[str, object], job)
|
|
time.sleep(0.01)
|
|
raise AssertionError(f"job {job_id} did not finish")
|
|
|
|
|
|
def test_complete_collection_lifecycle(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
root = tmp_path / "docs"
|
|
(root / "a" / ".calibre").mkdir(parents=True)
|
|
(root / "b").mkdir()
|
|
(root / "a" / "shared.txt").write_text("python duplicate")
|
|
(root / "b" / "shared.txt").write_text("python duplicate")
|
|
(root / "a" / "other.txt").write_text("python separate")
|
|
(root / "a" / ".calibre" / "hidden.txt").write_text("python hidden")
|
|
monkeypatch.setenv("RECOLL_MODERN_ADMIN_USERNAME", "admin")
|
|
monkeypatch.setenv("RECOLL_MODERN_ADMIN_PASSWORD", "secret-password")
|
|
settings = Settings.from_env({
|
|
"RECOLL_MODERN_DOCUMENT_ROOT": str(root),
|
|
"RECOLL_MODERN_STATE_ROOT": str(tmp_path / "state"),
|
|
"RECOLL_MODERN_RECOLL_CONFIG_DIR": str(tmp_path / "config"),
|
|
"RECOLL_MODERN_ROOT_PATH": "/recoll",
|
|
"RECOLL_MODERN_INLINE_TEXT_LIMIT": "12",
|
|
"RECOLL_MODERN_INLINE_TEXT_LIMIT_MAX": "12",
|
|
})
|
|
engine = FakeEngine([])
|
|
with TestClient(create_app(settings, engine), base_url="https://testserver") as client:
|
|
login = client.post("/recoll/auth/login", json={
|
|
"username": "admin", "password": "secret-password",
|
|
})
|
|
assert login.status_code == 200
|
|
assert "Path=/recoll" in login.headers["set-cookie"]
|
|
headers = {"X-CSRF-Token": login.json()["csrf_token"]}
|
|
first = client.post("/recoll/collections", json={"name": "First"}, headers=headers)
|
|
second = client.post("/recoll/collections", json={"name": "Second"}, headers=headers)
|
|
a, b = first.json()["id"], second.json()["id"]
|
|
for collection_id, sources in (
|
|
(a, [{"type": "directory", "path": "a", "mode": "persistent",
|
|
"excluded_directories": ["a/.calibre"]}]),
|
|
(b, [{"type": "directory", "path": "b", "mode": "snapshot"}]),
|
|
):
|
|
response = client.post(
|
|
f"/recoll/collections/{collection_id}/memberships",
|
|
headers=headers, json={"sources": sources},
|
|
)
|
|
assert _wait(client, response.json()["job"]["id"])["state"] == "completed"
|
|
direct = client.post(
|
|
f"/recoll/collections/{a}/memberships", headers=headers,
|
|
json={"sources": [
|
|
{"type": "file", "path": "a/other.txt"},
|
|
{"type": "file", "path": "a/shared.txt"},
|
|
]},
|
|
)
|
|
assert _wait(client, direct.json()["job"]["id"])["state"] == "completed"
|
|
members_a = client.get(f"/recoll/collections/{a}/documents").json()["items"]
|
|
members_b = client.get(f"/recoll/collections/{b}/documents").json()["items"]
|
|
assert len(members_a) == 2 and len(members_b) == 1
|
|
shared = next(item for item in members_a if item["path"] == "a/shared.txt")
|
|
assert shared["document_id"] == members_b[0]["document_id"]
|
|
assert all(item["path"] != "a/.calibre/hidden.txt" for item in members_a)
|
|
engine.records = [
|
|
FakeRecord(str(root / "a" / "shared.txt"), "Shared", "python duplicate", (a, b)),
|
|
FakeRecord(str(root / "a" / "other.txt"), "Other", "python separate", (a,)),
|
|
]
|
|
result = client.get("/recoll/search", params=[
|
|
("q", "python"), ("collections", a), ("collections", b),
|
|
("fields", "text"), ("text_limit", "6"),
|
|
])
|
|
assert result.status_code == 200
|
|
assert len(result.json()["hits"]) == 2
|
|
assert result.json()["hits"][0]["text"] == "python"
|
|
assert client.get(
|
|
f"/recoll/documents/{shared['document_id']}/text"
|
|
).json()["text"] == "python duplicate"
|
|
partial = client.get(
|
|
f"/recoll/documents/{shared['document_id']}/file", headers={"Range": "bytes=0-5"}
|
|
)
|
|
assert partial.status_code == 206 and partial.content == b"python"
|
|
|
|
(root / "a" / "shared.txt").unlink()
|
|
refresh = client.post("/recoll/jobs/reindex", headers=headers, json={})
|
|
assert _wait(client, refresh.json()["job"]["id"])["state"] == "completed"
|
|
promoted = cast(FastAPI, client.app).state.documents.get(shared["document_id"])
|
|
assert promoted.canonical_path == "b/shared.txt"
|
|
engine.records = [
|
|
FakeRecord(str(root / "b" / "shared.txt"), "Shared", "python duplicate", (a, b))
|
|
]
|
|
assert client.get("/recoll/search", params={
|
|
"q": "python", "collections": a,
|
|
}).json()["hits"][0]["document_id"] == shared["document_id"]
|
|
|
|
jobs = JobRepository(Database(settings.database_path))
|
|
interrupted = jobs.enqueue("reindex", {"version": 1, "paths": [], "collections": []}, False)
|
|
with Database(settings.database_path).transaction() as connection:
|
|
connection.execute("UPDATE jobs SET state = 'running' WHERE id = ?", (interrupted.id,))
|
|
with TestClient(create_app(settings, engine), base_url="https://testserver") as client:
|
|
client.post("/recoll/auth/login", json={
|
|
"username": "admin", "password": "secret-password",
|
|
})
|
|
assert _wait(client, interrupted.id)["state"] == "completed"
|
|
|
|
|
|
def test_real_container_collection_lifecycle(
|
|
compose_exec: Callable[[list[str]], subprocess.CompletedProcess[str]],
|
|
) -> None:
|
|
document_root = os.environ.get("RECOLL_MODERN_DOCUMENTS")
|
|
if document_root is None:
|
|
pytest.fail("RECOLL_MODERN_DOCUMENTS is required for real-container acceptance")
|
|
relative = Path(f"recoll-modern-test-{uuid.uuid4().hex}")
|
|
corpus = Path(document_root) / relative
|
|
(corpus / "a" / ".calibre").mkdir(parents=True)
|
|
(corpus / "b").mkdir()
|
|
marker = uuid.uuid4().hex
|
|
content = {
|
|
"a/shared.txt": f"zebraplatypus duplicate {marker}",
|
|
"b/shared.txt": f"zebraplatypus duplicate {marker}",
|
|
"a/other.txt": f"zebraplatypus first distinct {marker}",
|
|
"b/other.txt": f"zebraplatypus second distinct {marker}",
|
|
"a/special.txt": f"zebraplatypus second only {marker}",
|
|
"a/.calibre/hidden.txt": f"zebraplatypus excluded {marker}",
|
|
}
|
|
for name, text in content.items():
|
|
file = corpus / name
|
|
file.write_text(text)
|
|
prefix = os.environ.get("RECOLL_MODERN_ROOT_PATH", "/").rstrip("/")
|
|
port = os.environ.get("RECOLL_MODERN_PUBLISH_PORT", "8000")
|
|
with httpx.Client(base_url=f"http://127.0.0.1:{port}", timeout=15) as client:
|
|
login = client.post(f"{prefix}/auth/login", json={
|
|
"username": os.environ["RECOLL_MODERN_ADMIN_USERNAME"],
|
|
"password": os.environ["RECOLL_MODERN_ADMIN_PASSWORD"],
|
|
})
|
|
assert login.status_code == 200, login.text
|
|
cookie = login.cookies.get("recoll_modern_session")
|
|
assert cookie
|
|
headers = {
|
|
"Cookie": f"recoll_modern_session={cookie}",
|
|
"X-CSRF-Token": login.json()["csrf_token"],
|
|
}
|
|
a = client.post(
|
|
f"{prefix}/collections", headers=headers,
|
|
json={"name": f"Real A {uuid.uuid4().hex}"},
|
|
).json()["id"]
|
|
b = client.post(
|
|
f"{prefix}/collections", headers=headers,
|
|
json={"name": f"Real B {uuid.uuid4().hex}"},
|
|
).json()["id"]
|
|
def add(collection_id: str, sources: list[dict[str, object]]) -> str:
|
|
response = client.post(
|
|
f"{prefix}/collections/{collection_id}/memberships", headers=headers,
|
|
json={"sources": sources},
|
|
)
|
|
assert response.status_code == 202, response.text
|
|
job_id = response.json()["job"]["id"]
|
|
assert _live_wait(client, prefix, job_id, headers)["state"] == "completed"
|
|
return cast(str, job_id)
|
|
|
|
rule_job = add(a, [{
|
|
"type": "directory", "path": (relative / "a").as_posix(),
|
|
"mode": "persistent",
|
|
"excluded_directories": [(relative / "a" / ".calibre").as_posix()],
|
|
}])
|
|
add(b, [{"type": "directory", "path": (relative / "b").as_posix(),
|
|
"mode": "snapshot"}])
|
|
add(b, [{"type": "file", "path": (relative / "a" / "special.txt").as_posix()}])
|
|
docs_a = client.get(f"{prefix}/collections/{a}/documents", headers=headers).json()["items"]
|
|
docs_b = client.get(f"{prefix}/collections/{b}/documents", headers=headers).json()["items"]
|
|
assert len(docs_a) == 3 and len(docs_b) == 3
|
|
assert all(".calibre" not in item["path"] for item in docs_a)
|
|
shared = next(
|
|
item for item in docs_a
|
|
if item["document_id"] in {other["document_id"] for other in docs_b}
|
|
and item["path"].endswith("shared.txt")
|
|
)
|
|
assert shared["document_id"] in {item["document_id"] for item in docs_b}
|
|
combined = client.get(f"{prefix}/search", params=[
|
|
("q", "zebraplatypus"), ("collections", a), ("collections", b),
|
|
("fields", "text"), ("text_limit", "8"),
|
|
], headers=headers)
|
|
assert combined.status_code == 200, combined.text
|
|
assert len(combined.json()["hits"]) == 4, combined.json()
|
|
assert all(hit["text"] == "zebrapla" for hit in combined.json()["hits"])
|
|
text_response = client.get(
|
|
f"{prefix}/documents/{shared['document_id']}/text", headers=headers
|
|
)
|
|
assert text_response.status_code == 200 and "duplicate" in text_response.json()["text"]
|
|
partial = client.get(
|
|
f"{prefix}/documents/{shared['document_id']}/file",
|
|
headers={**headers, "Range": "bytes=0-7"},
|
|
)
|
|
assert partial.status_code == 206 and partial.content == b"zebrapla"
|
|
|
|
(corpus / "a" / "shared.txt").unlink()
|
|
refreshed = client.post(f"{prefix}/jobs/reindex", headers=headers, json={})
|
|
assert _live_wait(client, prefix, refreshed.json()["job"]["id"], headers)[
|
|
"state"
|
|
] == "completed"
|
|
promoted = client.get(f"{prefix}/collections/{b}/documents", headers=headers).json()[
|
|
"items"
|
|
]
|
|
assert next(item for item in promoted if item["document_id"] == shared["document_id"])[
|
|
"path"
|
|
] == (relative / "b" / "shared.txt").as_posix()
|
|
assert client.get(f"{prefix}/search", params={
|
|
"q": "zebraplatypus", "collections": b,
|
|
}, headers=headers).json()["hits"]
|
|
|
|
marked = compose_exec(["python", "-c", (
|
|
"from recoll_modern.storage.database import Database; "
|
|
"from pathlib import Path; "
|
|
"db=Database(Path('/var/lib/recoll-modern/recoll-modern.sqlite3')); "
|
|
"c=db.connect(); "
|
|
"c.execute(\"UPDATE jobs SET state='running' WHERE id=?\", "
|
|
"(__import__('sys').argv[1],)); "
|
|
"c.commit(); c.close()"
|
|
), rule_job])
|
|
assert marked.returncode == 0, marked.stderr
|
|
restarted = subprocess.run(
|
|
["docker", "compose", "restart", "app"], capture_output=True, text=True,
|
|
check=False, timeout=40,
|
|
)
|
|
assert restarted.returncode == 0, restarted.stderr
|
|
assert _live_wait(client, prefix, rule_job, headers)["state"] == "completed"
|