Similar fix for haiku.rag.ingester, and formatting
- Add regression tests - Update Changelog
This commit is contained in:
parent
5e6284fd02
commit
d4d6414bc4
6 changed files with 59 additions and 20 deletions
|
|
@ -4,6 +4,8 @@
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- `cross-encoder` reranking no longer ties the scores of strongly-relevant candidates, which left their order to the sort. Scores remain 0-1.
|
- `cross-encoder` reranking no longer ties the scores of strongly-relevant candidates, which left their order to the sort. Scores remain 0-1.
|
||||||
|
- haiku-rag CLI startup no longer imports `lancedb`, `pyarrow` and `pydantic_ai`.
|
||||||
|
- `haiku.rag.store` no longer re-exports `Store`; import it from `haiku.rag.store.engine`.
|
||||||
|
|
||||||
## [0.73.0] - 2026-08-06
|
## [0.73.0] - 2026-08-06
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
import yaml
|
import yaml
|
||||||
|
|
@ -28,11 +29,6 @@ from haiku.rag.config import ( # noqa: E402
|
||||||
load_yaml_config,
|
load_yaml_config,
|
||||||
set_config,
|
set_config,
|
||||||
)
|
)
|
||||||
from haiku.rag.ingester.app import ( # noqa: E402
|
|
||||||
BatchProgress,
|
|
||||||
BatchProgressCallback,
|
|
||||||
IngesterApp,
|
|
||||||
)
|
|
||||||
from haiku.rag.ingester.batch import BatchManifest # noqa: E402
|
from haiku.rag.ingester.batch import BatchManifest # noqa: E402
|
||||||
from haiku.rag.ingester.queue.migrations import open_queue # noqa: E402
|
from haiku.rag.ingester.queue.migrations import open_queue # noqa: E402
|
||||||
from haiku.rag.logging import configure_cli_logging # noqa: E402
|
from haiku.rag.logging import configure_cli_logging # noqa: E402
|
||||||
|
|
@ -41,6 +37,9 @@ from haiku.rag.store.exceptions import ( # noqa: E402
|
||||||
ReadOnlyError,
|
ReadOnlyError,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from haiku.rag.ingester.app import BatchProgress, BatchProgressCallback
|
||||||
|
|
||||||
_cli = typer.Typer(
|
_cli = typer.Typer(
|
||||||
name="haiku-ingester",
|
name="haiku-ingester",
|
||||||
no_args_is_help=True,
|
no_args_is_help=True,
|
||||||
|
|
@ -167,7 +166,7 @@ def _write_manifest(manifest: BatchManifest, path: Path) -> None:
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _batch_progress(
|
def _batch_progress(
|
||||||
description: str,
|
description: str,
|
||||||
) -> Iterator[BatchProgressCallback | None]: # pragma: no cover
|
) -> "Iterator[BatchProgressCallback | None]": # pragma: no cover
|
||||||
console = Console(file=sys.stdout)
|
console = Console(file=sys.stdout)
|
||||||
if not console.is_terminal:
|
if not console.is_terminal:
|
||||||
yield None
|
yield None
|
||||||
|
|
@ -184,7 +183,7 @@ def _batch_progress(
|
||||||
)
|
)
|
||||||
task_id = None
|
task_id = None
|
||||||
|
|
||||||
def _update(snapshot: BatchProgress) -> None:
|
def _update(snapshot: "BatchProgress") -> None:
|
||||||
nonlocal task_id
|
nonlocal task_id
|
||||||
task_description = (
|
task_description = (
|
||||||
f"{description} ({snapshot.succeeded} ok, {snapshot.dead} dead)"
|
f"{description} ({snapshot.succeeded} ok, {snapshot.dead} dead)"
|
||||||
|
|
@ -244,6 +243,8 @@ def serve(
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Run the production ingester: pollers + workers (and the HTTP API
|
"""Run the production ingester: pollers + workers (and the HTTP API
|
||||||
unless --no-api is set). Blocks until SIGINT/SIGTERM."""
|
unless --no-api is set). Blocks until SIGINT/SIGTERM."""
|
||||||
|
from haiku.rag.ingester.app import IngesterApp
|
||||||
|
|
||||||
app_config = get_config()
|
app_config = get_config()
|
||||||
if host is not None:
|
if host is not None:
|
||||||
app_config.ingester.api.host = host
|
app_config.ingester.api.host = host
|
||||||
|
|
@ -309,6 +310,8 @@ async def _run_batch(
|
||||||
output: Path | None = None,
|
output: Path | None = None,
|
||||||
manifest_path: Path | None = None,
|
manifest_path: Path | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
from haiku.rag.ingester.app import IngesterApp
|
||||||
|
|
||||||
db = _resolve_db_path(app_config, db_path)
|
db = _resolve_db_path(app_config, db_path)
|
||||||
app = IngesterApp(config=app_config, db_path=db)
|
app = IngesterApp(config=app_config, db_path=db)
|
||||||
if dry_run:
|
if dry_run:
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,8 @@
|
||||||
"""haiku-ingester CLI: exercises every subcommand via CliRunner with
|
"""haiku-ingester CLI: exercises every subcommand via CliRunner with
|
||||||
IngesterApp / open_queue patched out so no real ingestion runs."""
|
IngesterApp / open_queue patched out so no real ingestion runs."""
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
@ -24,6 +26,22 @@ from haiku.rag.ingester.queue.models import JobOp
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
|
|
||||||
|
|
||||||
|
def test_importing_ingester_cli_does_not_load_lancedb():
|
||||||
|
"""Test that lancedb is not imported automatically by the cli. Doing so is
|
||||||
|
expensive. Must be run in a subprocess because lancedb might be imported by
|
||||||
|
other tests in the same session."""
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-c",
|
||||||
|
"import haiku.rag.ingester.cli, sys; assert 'lancedb' not in sys.modules",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
|
||||||
|
|
||||||
# --- helpers ---
|
# --- helpers ---
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -41,7 +59,7 @@ def _config_with_queue(queue: QueueConfig):
|
||||||
def _fake_app(report: BatchReport, monkeypatch) -> AsyncMock:
|
def _fake_app(report: BatchReport, monkeypatch) -> AsyncMock:
|
||||||
fake = AsyncMock()
|
fake = AsyncMock()
|
||||||
fake.run_batch.return_value = report
|
fake.run_batch.return_value = report
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
return fake
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -78,14 +96,14 @@ def _manifest() -> BatchManifest:
|
||||||
def _fake_dry_run_app(report: BatchDryRunReport, monkeypatch) -> AsyncMock:
|
def _fake_dry_run_app(report: BatchDryRunReport, monkeypatch) -> AsyncMock:
|
||||||
fake = AsyncMock()
|
fake = AsyncMock()
|
||||||
fake.run_batch_dry_run.return_value = report
|
fake.run_batch_dry_run.return_value = report
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
return fake
|
return fake
|
||||||
|
|
||||||
|
|
||||||
def _fake_manifest_app(report: BatchReport, monkeypatch) -> AsyncMock:
|
def _fake_manifest_app(report: BatchReport, monkeypatch) -> AsyncMock:
|
||||||
fake = AsyncMock()
|
fake = AsyncMock()
|
||||||
fake.run_batch_from_manifest.return_value = report
|
fake.run_batch_from_manifest.return_value = report
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
return fake
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -111,7 +129,7 @@ def test_run_batch_passes_progress_callback_when_enabled(monkeypatch):
|
||||||
return BatchReport(succeeded=1, dead=0)
|
return BatchReport(succeeded=1, dead=0)
|
||||||
|
|
||||||
fake.run_batch.side_effect = _run_batch
|
fake.run_batch.side_effect = _run_batch
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"haiku.rag.ingester.cli._batch_progress",
|
"haiku.rag.ingester.cli._batch_progress",
|
||||||
lambda _: _progress_context(lambda snapshot: None),
|
lambda _: _progress_context(lambda snapshot: None),
|
||||||
|
|
@ -227,7 +245,7 @@ def test_run_batch_manifest_passes_progress_callback(monkeypatch, tmp_path):
|
||||||
return BatchReport(succeeded=1, dead=0)
|
return BatchReport(succeeded=1, dead=0)
|
||||||
|
|
||||||
fake.run_batch_from_manifest.side_effect = _run_manifest
|
fake.run_batch_from_manifest.side_effect = _run_manifest
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"haiku.rag.ingester.cli._batch_progress",
|
"haiku.rag.ingester.cli._batch_progress",
|
||||||
lambda _: _progress_context(lambda snapshot: None),
|
lambda _: _progress_context(lambda snapshot: None),
|
||||||
|
|
@ -256,7 +274,7 @@ def test_run_batch_manifest_reports_validation_error(monkeypatch, tmp_path):
|
||||||
_write_manifest(manifest_path)
|
_write_manifest(manifest_path)
|
||||||
fake = AsyncMock()
|
fake = AsyncMock()
|
||||||
fake.run_batch_from_manifest.side_effect = ValueError("bad manifest")
|
fake.run_batch_from_manifest.side_effect = ValueError("bad manifest")
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
|
|
||||||
result = runner.invoke(cli, ["run-batch", "--manifest", str(manifest_path)])
|
result = runner.invoke(cli, ["run-batch", "--manifest", str(manifest_path)])
|
||||||
|
|
||||||
|
|
@ -322,7 +340,7 @@ def test_resolve_queue_config_keeps_dburi_when_path_override_present(tmp_path):
|
||||||
|
|
||||||
def test_serve_invokes_app(monkeypatch):
|
def test_serve_invokes_app(monkeypatch):
|
||||||
fake = AsyncMock()
|
fake = AsyncMock()
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", lambda **_: fake)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", lambda **_: fake)
|
||||||
|
|
||||||
result = runner.invoke(cli, ["serve", "--db", "x.lancedb", "--no-api"])
|
result = runner.invoke(cli, ["serve", "--db", "x.lancedb", "--no-api"])
|
||||||
|
|
||||||
|
|
@ -338,7 +356,7 @@ def test_serve_passes_host_and_port(monkeypatch):
|
||||||
captured.update(kwargs)
|
captured.update(kwargs)
|
||||||
return fake
|
return fake
|
||||||
|
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", _capture)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", _capture)
|
||||||
|
|
||||||
result = runner.invoke(
|
result = runner.invoke(
|
||||||
cli, ["serve", "--db", "x.lancedb", "--host", "0.0.0.0", "--port", "9999"]
|
cli, ["serve", "--db", "x.lancedb", "--host", "0.0.0.0", "--port", "9999"]
|
||||||
|
|
@ -357,7 +375,7 @@ def test_serve_passes_root_path(monkeypatch):
|
||||||
captured.update(kwargs)
|
captured.update(kwargs)
|
||||||
return fake
|
return fake
|
||||||
|
|
||||||
monkeypatch.setattr("haiku.rag.ingester.cli.IngesterApp", _capture)
|
monkeypatch.setattr("haiku.rag.ingester.app.IngesterApp", _capture)
|
||||||
|
|
||||||
result = runner.invoke(
|
result = runner.invoke(
|
||||||
cli, ["serve", "--db", "x.lancedb", "--root-path", "/ingester/"]
|
cli, ["serve", "--db", "x.lancedb", "--root-path", "/ingester/"]
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from haiku.rag.store.engine import Store
|
from haiku.rag.store.engine import Store, get_database_stats
|
||||||
from haiku.rag.store.engine import get_database_stats
|
|
||||||
|
|
||||||
|
|
||||||
class TestGetDatabaseStats:
|
class TestGetDatabaseStats:
|
||||||
|
|
|
||||||
|
|
@ -8,9 +8,8 @@ introduced by later migrations (``picture_data`` in v0.45.0,
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from haiku.rag.store.engine import Store
|
|
||||||
from haiku.rag.store.compression import compress_docling_split
|
from haiku.rag.store.compression import compress_docling_split
|
||||||
from haiku.rag.store.engine import DocumentRecord
|
from haiku.rag.store.engine import DocumentRecord, Store
|
||||||
from haiku.rag.store.upgrades.v0_40_0 import _apply_populate_document_items
|
from haiku.rag.store.upgrades.v0_40_0 import _apply_populate_document_items
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
@ -12,6 +14,22 @@ from haiku.rag.store.exceptions import MigrationRequiredError
|
||||||
runner = CliRunner()
|
runner = CliRunner()
|
||||||
|
|
||||||
|
|
||||||
|
def test_importing_cli_does_not_load_lancedb():
|
||||||
|
"""Test that lancedb is not imported automatically by the cli. Doing so is
|
||||||
|
expensive. Must be run in a subprocess because lancedb might be imported by
|
||||||
|
other tests in the same session."""
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-c",
|
||||||
|
"import haiku.rag.cli, sys; assert 'lancedb' not in sys.modules",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
|
||||||
|
|
||||||
class TestParseMetaOptions:
|
class TestParseMetaOptions:
|
||||||
def test_empty_input(self):
|
def test_empty_input(self):
|
||||||
assert _parse_meta_options(None) == {}
|
assert _parse_meta_options(None) == {}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue