Similar fix for haiku.rag.ingester, and formatting

- Add regression tests
- Update Changelog
This commit is contained in:
Lawrence Akka 2026-08-12 15:20:11 +01:00
parent 5e6284fd02
commit d4d6414bc4
6 changed files with 59 additions and 20 deletions

View file

@ -4,6 +4,8 @@
### Fixed ### Fixed
- `cross-encoder` reranking no longer ties the scores of strongly-relevant candidates, which left their order to the sort. Scores remain 0-1. - `cross-encoder` reranking no longer ties the scores of strongly-relevant candidates, which left their order to the sort. Scores remain 0-1.
- haiku-rag CLI startup no longer imports `lancedb`, `pyarrow` and `pydantic_ai`.
- `haiku.rag.store` no longer re-exports `Store`; import it from `haiku.rag.store.engine`.
## [0.73.0] - 2026-08-06 ## [0.73.0] - 2026-08-06

View file

@ -4,6 +4,7 @@ from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING
import typer import typer
import yaml import yaml
@ -28,11 +29,6 @@ from haiku.rag.config import ( # noqa: E402
load_yaml_config, load_yaml_config,
set_config, set_config,
) )
from haiku.rag.ingester.app import ( # noqa: E402
BatchProgress,
BatchProgressCallback,
IngesterApp,
)
from haiku.rag.ingester.batch import BatchManifest # noqa: E402 from haiku.rag.ingester.batch import BatchManifest # noqa: E402
from haiku.rag.ingester.queue.migrations import open_queue # noqa: E402 from haiku.rag.ingester.queue.migrations import open_queue # noqa: E402
from haiku.rag.logging import configure_cli_logging # noqa: E402 from haiku.rag.logging import configure_cli_logging # noqa: E402
@ -41,6 +37,9 @@ from haiku.rag.store.exceptions import ( # noqa: E402
ReadOnlyError, ReadOnlyError,
) )
if TYPE_CHECKING:
from haiku.rag.ingester.app import BatchProgress, BatchProgressCallback
_cli = typer.Typer( _cli = typer.Typer(
name="haiku-ingester", name="haiku-ingester",
no_args_is_help=True, no_args_is_help=True,
@ -167,7 +166,7 @@ def _write_manifest(manifest: BatchManifest, path: Path) -> None:
@contextmanager @contextmanager
def _batch_progress( def _batch_progress(
description: str, description: str,
) -> Iterator[BatchProgressCallback | None]: # pragma: no cover ) -> "Iterator[BatchProgressCallback | None]": # pragma: no cover
console = Console(file=sys.stdout) console = Console(file=sys.stdout)
if not console.is_terminal: if not console.is_terminal:
yield None yield None
@ -184,7 +183,7 @@ def _batch_progress(
) )
task_id = None task_id = None
def _update(snapshot: BatchProgress) -> None: def _update(snapshot: "BatchProgress") -> None:
nonlocal task_id nonlocal task_id
task_description = ( task_description = (
f"{description} ({snapshot.succeeded} ok, {snapshot.dead} dead)" f"{description} ({snapshot.succeeded} ok, {snapshot.dead} dead)"
@ -244,6 +243,8 @@ def serve(
) -> None: ) -> None:
"""Run the production ingester: pollers + workers (and the HTTP API """Run the production ingester: pollers + workers (and the HTTP API
unless --no-api is set). Blocks until SIGINT/SIGTERM.""" unless --no-api is set). Blocks until SIGINT/SIGTERM."""
from haiku.rag.ingester.app import IngesterApp
app_config = get_config() app_config = get_config()
if host is not None: if host is not None:
app_config.ingester.api.host = host app_config.ingester.api.host = host
@ -309,6 +310,8 @@ async def _run_batch(
output: Path | None = None, output: Path | None = None,
manifest_path: Path | None = None, manifest_path: Path | None = None,
) -> None: ) -> None:
from haiku.rag.ingester.app import IngesterApp
db = _resolve_db_path(app_config, db_path) db = _resolve_db_path(app_config, db_path)
app = IngesterApp(config=app_config, db_path=db) app = IngesterApp(config=app_config, db_path=db)
if dry_run: if dry_run:

View file

@ -1,6 +1,8 @@
"""haiku-ingester CLI: exercises every subcommand via CliRunner with """haiku-ingester CLI: exercises every subcommand via CliRunner with
IngesterApp / open_queue patched out so no real ingestion runs.""" IngesterApp / open_queue patched out so no real ingestion runs."""
import subprocess
import sys
from contextlib import contextmanager from contextlib import contextmanager
from datetime import UTC, datetime from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock from unittest.mock import AsyncMock, MagicMock
@ -24,6 +26,22 @@ from haiku.rag.ingester.queue.models import JobOp
runner = CliRunner() runner = CliRunner()
def test_importing_ingester_cli_does_not_load_lancedb():
"""Test that lancedb is not imported automatically by the cli. Doing so is
expensive. Must be run in a subprocess because lancedb might be imported by
other tests in the same session."""
result = subprocess.run(
[
sys.executable,
"-c",
"import haiku.rag.ingester.cli, sys; assert 'lancedb' not in sys.modules",
],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
# --- helpers --- # --- helpers ---
@ -41,7 +59,7 @@ def _config_with_queue(queue: QueueConfig):
def _fake_app(report: BatchReport, monkeypatch) -> AsyncMock: def _fake_app(report: BatchReport, monkeypatch) -> AsyncMock:
fake = AsyncMock() fake = AsyncMock()
fake.run_batch.return_value = report fake.run_batch.return_value = report
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
return fake return fake
@ -78,14 +96,14 @@ def _manifest() -> BatchManifest:
def _fake_dry_run_app(report: BatchDryRunReport, monkeypatch) -> AsyncMock: def _fake_dry_run_app(report: BatchDryRunReport, monkeypatch) -> AsyncMock:
fake = AsyncMock() fake = AsyncMock()
fake.run_batch_dry_run.return_value = report fake.run_batch_dry_run.return_value = report
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
return fake return fake
def _fake_manifest_app(report: BatchReport, monkeypatch) -> AsyncMock: def _fake_manifest_app(report: BatchReport, monkeypatch) -> AsyncMock:
fake = AsyncMock() fake = AsyncMock()
fake.run_batch_from_manifest.return_value = report fake.run_batch_from_manifest.return_value = report
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
return fake return fake
@ -111,7 +129,7 @@ def test_run_batch_passes_progress_callback_when_enabled(monkeypatch):
return BatchReport(succeeded=1, dead=0) return BatchReport(succeeded=1, dead=0)
fake.run_batch.side_effect = _run_batch fake.run_batch.side_effect = _run_batch
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
monkeypatch.setattr( monkeypatch.setattr(
"haiku.rag.ingester.cli._batch_progress", "haiku.rag.ingester.cli._batch_progress",
lambda _: _progress_context(lambda snapshot: None), lambda _: _progress_context(lambda snapshot: None),
@ -227,7 +245,7 @@ def test_run_batch_manifest_passes_progress_callback(monkeypatch, tmp_path):
return BatchReport(succeeded=1, dead=0) return BatchReport(succeeded=1, dead=0)
fake.run_batch_from_manifest.side_effect = _run_manifest fake.run_batch_from_manifest.side_effect = _run_manifest
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
monkeypatch.setattr( monkeypatch.setattr(
"haiku.rag.ingester.cli._batch_progress", "haiku.rag.ingester.cli._batch_progress",
lambda _: _progress_context(lambda snapshot: None), lambda _: _progress_context(lambda snapshot: None),
@ -256,7 +274,7 @@ def test_run_batch_manifest_reports_validation_error(monkeypatch, tmp_path):
_write_manifest(manifest_path) _write_manifest(manifest_path)
fake = AsyncMock() fake = AsyncMock()
fake.run_batch_from_manifest.side_effect = ValueError("bad manifest") fake.run_batch_from_manifest.side_effect = ValueError("bad manifest")
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
result = runner.invoke(cli, ["run-batch", "--manifest", str(manifest_path)]) result = runner.invoke(cli, ["run-batch", "--manifest", str(manifest_path)])
@ -322,7 +340,7 @@ def test_resolve_queue_config_keeps_dburi_when_path_override_present(tmp_path):
def test_serve_invokes_app(monkeypatch): def test_serve_invokes_app(monkeypatch):
fake = AsyncMock() fake = AsyncMock()
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
result = runner.invoke(cli, ["serve", "--db", "x.lancedb", "--no-api"]) result = runner.invoke(cli, ["serve", "--db", "x.lancedb", "--no-api"])
@ -338,7 +356,7 @@ def test_serve_passes_host_and_port(monkeypatch):
captured.update(kwargs) captured.update(kwargs)
return fake return fake
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", _capture) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", _capture)
result = runner.invoke( result = runner.invoke(
cli, ["serve", "--db", "x.lancedb", "--host", "0.0.0.0", "--port", "9999"] cli, ["serve", "--db", "x.lancedb", "--host", "0.0.0.0", "--port", "9999"]
@ -357,7 +375,7 @@ def test_serve_passes_root_path(monkeypatch):
captured.update(kwargs) captured.update(kwargs)
return fake return fake
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", _capture) monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", _capture)
result = runner.invoke( result = runner.invoke(
cli, ["serve", "--db", "x.lancedb", "--root-path", "/ingester/"] cli, ["serve", "--db", "x.lancedb", "--root-path", "/ingester/"]

View file

@ -1,7 +1,6 @@
import pytest import pytest
from haiku.rag.store.engine import Store from haiku.rag.store.engine import Store, get_database_stats
from haiku.rag.store.engine import get_database_stats
class TestGetDatabaseStats: class TestGetDatabaseStats:

View file

@ -8,9 +8,8 @@ introduced by later migrations (``picture_data`` in v0.45.0,
import pytest import pytest
from haiku.rag.store.engine import Store
from haiku.rag.store.compression import compress_docling_split from haiku.rag.store.compression import compress_docling_split
from haiku.rag.store.engine import DocumentRecord from haiku.rag.store.engine import DocumentRecord, Store
from haiku.rag.store.upgrades.v0_40_0 import _apply_populate_document_items from haiku.rag.store.upgrades.v0_40_0 import _apply_populate_document_items

View file

@ -1,3 +1,5 @@
import subprocess
import sys
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
@ -12,6 +14,22 @@ from haiku.rag.store.exceptions import MigrationRequiredError
runner = CliRunner() runner = CliRunner()
def test_importing_cli_does_not_load_lancedb():
"""Test that lancedb is not imported automatically by the cli. Doing so is
expensive. Must be run in a subprocess because lancedb might be imported by
other tests in the same session."""
result = subprocess.run(
[
sys.executable,
"-c",
"import haiku.rag.cli, sys; assert 'lancedb' not in sys.modules",
],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
class TestParseMetaOptions: class TestParseMetaOptions:
def test_empty_input(self): def test_empty_input(self):
assert _parse_meta_options(None) == {} assert _parse_meta_options(None) == {}