diff --git a/CHANGELOG.md b/CHANGELOG.md index a67157bf..db856c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,16 @@ ### Added -- Database tags: `haiku-rag tag create/list/delete`, `--at TAG` time travel, tags shown in `history`. Vacuum retains versions back to the oldest tag. +- Database tags: `haiku-rag tag create/list/delete`, tags shown in `history`. Vacuum retains versions back to the oldest tag. ### Changed - `lancedb` bumped to 0.34.0. +### Removed + +- `--before` global flag. There is no read-only replacement; create tags prospectively before important changes and use `tag restore` during a maintenance window. + ### Fixed - `docling-local` text conversion no longer misroutes markdown/HTML content whose first bytes collide with a binary magic signature (e.g. `BM`, `ID3`) to an image or audio backend. diff --git a/README.md b/README.md index b0eae73f..d04300a5 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Agentic RAG built on [LanceDB](https://lancedb.com/), [Pydantic AI](https://ai.p - **MCP server** — Expose as tools for AI assistants (Claude Desktop, etc.) - **Visual grounding** — View chunks highlighted on original page images - **Production ingester** — Long-lived `haiku-ingester` service with persistent SQLite queue, async worker pool with retries and a dead-letter queue, FS / HTTP / S3 / WebDAV source adapters, FastAPI control plane, and a browser dashboard for operators. See [docs/ingester.md](docs/ingester.md). -- **Time travel** — Query the database at any historical point with `--before`, or tag states and query them with `--at` +- **Tags** — Name database states with `haiku-rag tag` and roll back to them - **Inspector** — TUI for browsing documents, chunks, and search results ## Installation diff --git a/docs/chat.md b/docs/chat.md index 966b6135..c0cdbc99 100644 --- a/docs/chat.md +++ b/docs/chat.md @@ -76,4 +76,4 @@ For everyday Q&A, the rag skill alone is faster and cheaper. Attaching both lets Run "Filter documents" from the command palette to restrict searches to a subset. The filter applies to every search the agent runs for the rest of the session. -Chat also honors the global `--read-only` and `--before` flags. See the [CLI reference](cli.md) for details. +Chat also honors the global `--read-only` flag. See the [CLI reference](cli.md) for details. diff --git a/docs/cli.md b/docs/cli.md index cb058673..760d88b9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -7,8 +7,6 @@ The `haiku-rag` CLI provides complete document management functionality. - `--config` - Specify custom configuration file - `--read-only` - Open database in read-only mode (blocks writes, skips upgrades) - - `--before` - Query database as it existed before a datetime (implies `--read-only`) - - `--at` - Query database at a tag (implies `--read-only`, mutually exclusive with `--before`) - `--version` / `-v` - Show version and exit Per-command options: @@ -21,7 +19,6 @@ The `haiku-rag` CLI provides complete document management functionality. haiku-rag --config /path/to/config.yaml list haiku-rag --config /path/to/config.yaml list --db /path/to/custom.db haiku-rag --read-only search "query" - haiku-rag --before "2025-01-15" search "query" haiku-rag add -h ``` @@ -543,39 +540,9 @@ haiku-skills chat --use-entrypoints --skill medic └── haiku.rag.yaml # Optional config ``` -## Time Travel +## Tags -LanceDB maintains version history for tables, enabling you to query the database as it existed at a previous point in time. This is useful for: - -- **Debugging**: Investigate data before a problematic change -- **Auditing**: Verify what knowledge was available when a support ticket was filed - -### Query Historical State - -Use `--before` to query the database as it existed before a specific datetime: - -```bash -# Query documents as of January 15, 2025 -haiku-rag --before "2025-01-15" list - -# Search historical state -haiku-rag --before "2025-01-15T14:30:00" search "machine learning" - -# Ask questions against historical data -haiku-rag --before "2025-01-15" ask "What documents existed?" -``` - -Supported datetime formats: - -- ISO 8601: `2025-01-15T14:30:00`, `2025-01-15T14:30:00Z`, `2025-01-15T14:30:00+00:00` -- Date only: `2025-01-15` (interpreted as start of day) - -!!! note - Time travel mode automatically enables read-only mode. You cannot modify the database while viewing historical state. - -### Tags - -Tags name the current database state so you can return to it without remembering timestamps. A tag covers every table in the database. Tagged versions survive `vacuum`; everything older than your oldest tag is retained until that tag is deleted, so remove tags you no longer need. +Tags name the current database state so you can return to it. A tag covers every table in the database. Tagged versions survive `vacuum`; everything older than your oldest tag is retained until that tag is deleted, so remove tags you no longer need. ```bash # Tag the current state, e.g. at deploy time or after an ingestion run @@ -588,16 +555,6 @@ haiku-rag tag list haiku-rag tag delete release-1 ``` -Query the database at a tag with `--at`: - -```bash -haiku-rag --at release-1 list -haiku-rag --at release-1 search "machine learning" -haiku-rag --at release-1 ask "What documents existed?" -``` - -`--at` implies read-only mode and is mutually exclusive with `--before`. - ### Version History View version history for database tables: @@ -628,5 +585,3 @@ chunks v7: 2025-01-14 10:00:00 ... ``` - -Use the timestamps from `history` to construct `--before` queries, or tag names with `--at`. diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index fc037eac..46bc557b 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -1,5 +1,4 @@ import logging -from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING @@ -36,14 +35,10 @@ class HaikuRAGApp: # pragma: no cover db_path: Path, config: AppConfig = Config, read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, ): self.db_path = db_path self.config = config self.read_only = read_only - self.before = before - self.at_tag = at_tag self.console = Console() from haiku.rag.store.engine import ConnectionMode @@ -71,11 +66,6 @@ class HaikuRAGApp: # pragma: no cover from haiku.rag.store.engine import gather_database_info - if self.before is not None or self.at_tag is not None: - self.console.print( - "[yellow]Note: --before/--at is not supported by info; showing current state.[/yellow]" - ) - # Basic: show path/URI self.console.print("[bold]haiku.rag database info[/bold]") self.console.print( @@ -292,8 +282,6 @@ class HaikuRAGApp: # pragma: no cover skip_validation=True, read_only=True, skip_migration_check=True, - before=self.before, - at_tag=self.at_tag, ) as store: tables = [ "documents", @@ -421,8 +409,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: documents = await self.client.list_documents(filter=filter) for doc in documents: @@ -435,8 +421,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as self.client: doc = await self.client.create_document( text, title=title, metadata=metadata @@ -453,8 +437,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as self.client: result = await self.client.create_document_from_source( source, title=title, metadata=metadata @@ -476,8 +458,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: doc = await self.client.get_document_by_id(doc_id) if doc is None: @@ -490,8 +470,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as self.client: deleted = await self.client.delete_document(doc_id) if deleted: @@ -535,8 +513,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: results = await self.client.search( search_input, @@ -558,8 +534,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: chunk = await self.client.get_chunk_by_id(chunk_id) if not chunk: @@ -603,8 +577,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: answer, citations = await self.client.ask(question, filter=filter) @@ -632,8 +604,6 @@ class HaikuRAGApp: # pragma: no cover db_path=self.db_path, config=self.config, read_only=True, - before=self.before, - at_tag=self.at_tag, ) as self.client: self.console.print(f"[bold blue]Question:[/bold blue] {question}") self.console.print() @@ -657,8 +627,6 @@ class HaikuRAGApp: # pragma: no cover config=self.config, skip_validation=True, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as client: if mode == RebuildMode.SET_EMBEDDER: async for _ in client.rebuild_database(mode=mode): @@ -702,8 +670,6 @@ class HaikuRAGApp: # pragma: no cover config=self.config, skip_validation=True, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as client: await client.vacuum() self.console.print("[bold green]Vacuum completed successfully.[/bold green]") @@ -732,8 +698,6 @@ class HaikuRAGApp: # pragma: no cover config=self.config, skip_validation=True, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) as client: row_count = await client.store.chunks_table.count_rows() self.console.print(f"Chunks in database: {row_count}") @@ -901,15 +865,9 @@ class HaikuRAGApp: # pragma: no cover self.db_path, config=self.config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ): server = create_mcp_server( - self.db_path, - config=self.config, - read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, + self.db_path, config=self.config, read_only=self.read_only ) try: if transport == "stdio": diff --git a/haiku_rag_slim/haiku/rag/chat/__init__.py b/haiku_rag_slim/haiku/rag/chat/__init__.py index b7e4ad18..6c94784b 100644 --- a/haiku_rag_slim/haiku/rag/chat/__init__.py +++ b/haiku_rag_slim/haiku/rag/chat/__init__.py @@ -1,12 +1,9 @@ -from datetime import datetime from pathlib import Path def run_chat( db_path: Path | None = None, read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, model: str | None = None, skills: list[str] | None = None, ) -> None: @@ -15,8 +12,6 @@ def run_chat( Args: db_path: Path to the LanceDB database. If None, uses default from config. read_only: Whether to open the database in read-only mode. - before: Query database as it existed before this datetime. - at_tag: Query database at this tag. model: Model to use for the chat. skills: Skills to enable ("rag", "analysis"). Defaults to ["rag"]. """ @@ -57,8 +52,6 @@ def run_chat( db_path, skills=skill_list, read_only=read_only, - before=before, - at_tag=at_tag, model=model or get_model(config.qa.model, config), ) app.run() diff --git a/haiku_rag_slim/haiku/rag/chat/app.py b/haiku_rag_slim/haiku/rag/chat/app.py index b1dc7d8a..c3464ebb 100644 --- a/haiku_rag_slim/haiku/rag/chat/app.py +++ b/haiku_rag_slim/haiku/rag/chat/app.py @@ -2,7 +2,6 @@ import asyncio import json import uuid from collections.abc import Iterable -from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any @@ -85,16 +84,12 @@ class ChatApp(App): db_path: Path, skills: list[Skill], read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, model: str | None = None, ) -> None: super().__init__() self.db_path = db_path self._skills = skills self.read_only = read_only - self.before = before - self.at_tag = at_tag self._model = model self.client: HaikuRAG | None = None self.config = get_config() @@ -152,8 +147,6 @@ class ChatApp(App): db_path=self.db_path, config=self.config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) await self.client.__aenter__() diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index 50796a7f..bb1f713b 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -2,7 +2,6 @@ import asyncio import json import sys import warnings -from datetime import datetime from importlib.metadata import version from pathlib import Path from typing import Any @@ -47,8 +46,6 @@ def cli(): # Module-level flags set by callback _read_only: bool = False -_before: datetime | None = None -_at_tag: str | None = None def create_app(db: Path | None = None) -> HaikuRAGApp: # pragma: no cover @@ -62,13 +59,7 @@ def create_app(db: Path | None = None) -> HaikuRAGApp: # pragma: no cover """ config = get_config() db_path = db if db else config.storage.data_dir / "haiku.rag.lancedb" - return HaikuRAGApp( - db_path=db_path, - config=config, - read_only=_read_only, - before=_before, - at_tag=_at_tag, - ) + return HaikuRAGApp(db_path=db_path, config=config, read_only=_read_only) async def check_version(): # pragma: no cover @@ -107,39 +98,10 @@ def main( "--read-only", help="Open database in read-only mode", ), - before: str | None = typer.Option( - None, - "--before", - help="Query database as it existed before this datetime (implies --read-only). " - "Accepts ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)", - ), - at: str | None = typer.Option( - None, - "--at", - help="Query database at this tag (implies --read-only). " - "Mutually exclusive with --before", - ), ): """haiku.rag CLI - Vector database RAG system""" - global _read_only, _before, _at_tag + global _read_only _read_only = read_only - - if before is not None and at is not None: # pragma: no cover - typer.echo("Error: --before and --at are mutually exclusive") - raise typer.Exit(1) - - # Parse and store before datetime - if before is not None: # pragma: no cover - from haiku.rag.utils import parse_datetime, to_utc - - try: - _before = to_utc(parse_datetime(before)) - except ValueError as e: - typer.echo(f"Error: {e}") - raise typer.Exit(1) - else: - _before = None - _at_tag = at # Load config from --config, local folder, or default directory config_path = find_config_file(cli_path=config) if config_path: @@ -658,15 +620,6 @@ tag_cli = typer.Typer( _cli.add_typer(tag_cli, name="tag") -def _reject_time_travel(operation: str) -> None: - """Writable tag operations act on the live database state; combining them - with a historical checkout would tag something other than what the user - sees.""" - if _before is not None or _at_tag is not None: - typer.echo(f"Error: --before/--at cannot be used with {operation}", err=True) - raise typer.Exit(1) - - @tag_cli.command("create", help="Tag the current database state") def tag_create( # pragma: no cover name: str = typer.Argument(help="Name of the tag to create"), @@ -676,7 +629,6 @@ def tag_create( # pragma: no cover help="Path to the LanceDB database file", ), ): - _reject_time_travel("tag create") app = create_app(db) try: asyncio.run(app.create_tag(name)) @@ -706,7 +658,6 @@ def tag_delete( # pragma: no cover help="Path to the LanceDB database file", ), ): - _reject_time_travel("tag delete") app = create_app(db) try: asyncio.run(app.delete_tag(name)) @@ -741,7 +692,7 @@ def inspect( # pragma: no cover raise typer.Exit(1) from e db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb" - run_inspector(db_path, read_only=True, before=_before, at_tag=_at_tag) + run_inspector(db_path, read_only=True) @_cli.command("chat", help="Launch interactive chat TUI for conversational RAG") @@ -772,8 +723,6 @@ def chat( # pragma: no cover run_chat( db_path, read_only=True, - before=_before, - at_tag=_at_tag, model=model, skills=skills, ) diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py index 0f7dccf9..bff6f32b 100644 --- a/haiku_rag_slim/haiku/rag/client/__init__.py +++ b/haiku_rag_slim/haiku/rag/client/__init__.py @@ -5,7 +5,6 @@ import logging import mimetypes import tempfile from collections.abc import AsyncGenerator, Sequence -from datetime import datetime from enum import Enum from functools import cached_property from pathlib import Path @@ -72,8 +71,6 @@ class HaikuRAG: skip_validation: bool = False, create: bool = False, read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, ): """Initialize the RAG client with a database path. @@ -83,10 +80,6 @@ class HaikuRAG: skip_validation: Whether to skip configuration validation on database load. create: Whether to create the database if it doesn't exist. read_only: Whether to open the database in read-only mode. - before: Query the database as it existed at this datetime. - Implies read_only=True. - at_tag: Query the database at this tag. Implies read_only=True; - mutually exclusive with before. """ self._config = config if db_path is None: @@ -96,8 +89,6 @@ class HaikuRAG: self._skip_validation = skip_validation self._create = create self._read_only = read_only - self._before = before - self._at_tag = at_tag self._vacuum_tasks: set[asyncio.Task] = set() self._last_vacuum_at: float | None = None self._vacuum_dirty = False @@ -129,8 +120,6 @@ class HaikuRAG: skip_validation=self._skip_validation, create=self._create, read_only=self._read_only, - before=self._before, - at_tag=self._at_tag, ) # If _initialize fails mid-way (e.g. migration check raises after # connect), close the store so we don't leak the LanceDB connection — diff --git a/haiku_rag_slim/haiku/rag/inspector/app.py b/haiku_rag_slim/haiku/rag/inspector/app.py index 4752d572..369e4f45 100644 --- a/haiku_rag_slim/haiku/rag/inspector/app.py +++ b/haiku_rag_slim/haiku/rag/inspector/app.py @@ -1,4 +1,3 @@ -from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING @@ -67,18 +66,10 @@ class InspectorApp(App): Binding("c", "show_context", "Context", show=True), ] - def __init__( - self, - db_path: Path, - read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, - ): + def __init__(self, db_path: Path, read_only: bool = False): super().__init__() self.db_path = db_path self.read_only = read_only - self.before = before - self.at_tag = at_tag self.client: HaikuRAG | None = None def compose(self) -> "ComposeResult": @@ -96,8 +87,6 @@ class InspectorApp(App): db_path=self.db_path, config=config, read_only=self.read_only, - before=self.before, - at_tag=self.at_tag, ) await self.client.__aenter__() @@ -240,20 +229,16 @@ class InspectorApp(App): def run_inspector( db_path: Path | None = None, read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, ) -> None: """Run the inspector TUI. Args: db_path: Path to the LanceDB database. If None, uses default from config. read_only: Whether to open the database in read-only mode. - before: Query database as it existed before this datetime. - at_tag: Query database at this tag. """ config = get_config() if db_path is None: db_path = config.storage.data_dir / "haiku.rag.lancedb" - app = InspectorApp(db_path, read_only=read_only, before=before, at_tag=at_tag) + app = InspectorApp(db_path, read_only=read_only) app.run() diff --git a/haiku_rag_slim/haiku/rag/mcp.py b/haiku_rag_slim/haiku/rag/mcp.py index 3f00ccc3..4ee94339 100644 --- a/haiku_rag_slim/haiku/rag/mcp.py +++ b/haiku_rag_slim/haiku/rag/mcp.py @@ -1,4 +1,3 @@ -from datetime import datetime from pathlib import Path from typing import Any @@ -12,11 +11,7 @@ from haiku.rag.utils import format_citations def create_mcp_server( - db_path: Path, - config: AppConfig = Config, - read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, + db_path: Path, config: AppConfig = Config, read_only: bool = False ) -> FastMCP: """Create an MCP server with the specified database path. @@ -24,10 +19,7 @@ def create_mcp_server( db_path: Path to the database file. config: Configuration to use. read_only: If True, write tools (add_document_*, delete_document) are not registered. - before: Serve the database as it existed at this datetime. Implies read_only. - at_tag: Serve the database at this tag. Implies read_only. """ - read_only = read_only or before is not None or at_tag is not None mcp = FastMCP("haiku-rag") # Write tools - only registered when not in read-only mode @@ -112,8 +104,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: return await rag.search( query, limit=limit, include_images=include_images @@ -153,8 +143,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: return await rag.search( raw, limit=limit, include_images=include_images @@ -170,8 +158,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: return await rag.get_document_by_id(document_id) except Exception: @@ -195,8 +181,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: documents = await rag.list_documents(limit, offset, filter) @@ -231,8 +215,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: answer, citations = await rag.ask(question) if cite and citations: @@ -264,8 +246,6 @@ def create_mcp_server( db_path, config=config, read_only=read_only, - before=before, - at_tag=at_tag, ) as rag: result = await rag.analyze(question, filter=filter) return result.answer diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 4fb4bf8a..833c0aad 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -370,18 +370,11 @@ class Store: skip_validation: bool = False, create: bool = False, read_only: bool = False, - before: datetime | None = None, - at_tag: str | None = None, skip_migration_check: bool = False, ): - if before is not None and at_tag is not None: - raise ValueError("before and at_tag are mutually exclusive") self.db_path: Path = db_path self._config = config - self._before = before - self._at_tag = at_tag - # Time-travel mode is always read-only - self._read_only = read_only or before is not None or at_tag is not None + self._read_only = read_only self._create = create self._skip_validation = skip_validation self._skip_migration_check = skip_migration_check @@ -439,12 +432,6 @@ class Store: # pending, before creating any newly-introduced table. await self._init_tables(is_new_db) - # Checkout tables to historical state if before or at_tag is specified - if self._before is not None: - await self._checkout_tables_before(self._before) - if self._at_tag is not None: - await self._checkout_tables_at_tag(self._at_tag) - # Set version for new databases. if is_new_db and not self._read_only: await self._set_initial_version() @@ -958,71 +945,6 @@ class Store: if not found: raise ValueError(f"Tag '{name}' does not exist") - async def _checkout_tables_before(self, before: datetime) -> None: - """Checkout all tables to their state at or before the given datetime. - - Args: - before: The datetime to checkout to - - Raises: - ValueError: If no version exists before the given datetime - """ - # LanceDB stores timestamps as naive datetimes in local time. - # Convert 'before' to naive local time for comparison. - if before.tzinfo is not None: - # Convert to local time and make naive - before_local = before.astimezone().replace(tzinfo=None) - else: - # Already naive, assume local time - before_local = before - - for table in self._tables().values(): - versions = await table.list_versions() - # Find the latest version at or before the target datetime - # Versions are sorted by version number, not timestamp, so we need to check all - best_version = None - best_timestamp = None - - for v in versions: - # LanceDB version timestamps are naive datetime objects in local time - v_timestamp = v["timestamp"] - # Make sure it's naive for comparison - if v_timestamp.tzinfo is not None: - v_timestamp = v_timestamp.replace(tzinfo=None) - - if v_timestamp <= before_local: - if best_timestamp is None or v_timestamp > best_timestamp: - best_version = v["version"] - best_timestamp = v_timestamp - - if best_version is None: - # Find the earliest version to report in error message - if versions: - earliest = min(versions, key=lambda v: v["timestamp"]) - earliest_ts = earliest["timestamp"] - raise ValueError( - f"No data exists before {before}. " - f"Database was created on {earliest_ts}" - ) - else: - raise ValueError( - f"No data exists before {before}. Table has no versions." - ) - - # Checkout to the found version - await table.checkout(best_version) - - async def _checkout_tables_at_tag(self, name: str) -> None: - """Checkout all tables at the version the tag points to. - - Raises: - ValueError: If any table is missing the tag. - """ - for table_name, table in self._tables().items(): - if name not in await table.tags.list(): - raise ValueError(f"Tag '{name}' does not exist on table '{table_name}'") - await table.checkout(name) - async def list_table_versions(self, table_name: str) -> list[dict[str, Any]]: """List version history for a table. diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index 94be9222..fa8f48e9 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -1,11 +1,9 @@ import math import sys -from datetime import UTC, datetime from importlib import metadata from pathlib import Path from typing import TYPE_CHECKING, Any, cast -from dateutil import parser as dateutil_parser from packaging.version import Version, parse if TYPE_CHECKING: @@ -38,55 +36,6 @@ def cosine_similarity(vec1: list[float], vec2: list[float]) -> float: return dot_product / (norm1 * norm2) -def parse_datetime(s: str) -> datetime: - """Parse a datetime string into a datetime object. - - Supports: - - ISO 8601 format: "2025-01-15T14:30:00", "2025-01-15T14:30:00Z", "2025-01-15T14:30:00+00:00" - - Date only: "2025-01-15" (interpreted as 00:00:00) - - Various other formats via dateutil - - Args: - s: String to parse - - Returns: - Parsed datetime object - - Raises: - ValueError: If the string cannot be parsed - """ - try: - return dateutil_parser.parse(s) - except (ValueError, TypeError) as e: - raise ValueError( - f"Could not parse datetime: {s}. " - "Use ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)" - ) from e - - -def to_utc(dt: datetime) -> datetime: - """Convert a datetime to UTC. - - - Naive datetimes are assumed to be local time and converted to UTC - - Datetimes with timezone info are converted to UTC - - UTC datetimes are returned as-is - - Args: - dt: Datetime to convert - - Returns: - Datetime in UTC timezone - """ - if dt.tzinfo is None: - # Naive datetime - assume local time - local_dt = dt.astimezone() # Adds local timezone - return local_dt.astimezone(UTC) - elif dt.tzinfo == UTC: - return dt - else: - return dt.astimezone(UTC) - - def apply_common_settings( settings: Any | None, settings_class: type[Any], diff --git a/tests/chat/test_chat_app.py b/tests/chat/test_chat_app.py index baf8eaa9..e225cc54 100644 --- a/tests/chat/test_chat_app.py +++ b/tests/chat/test_chat_app.py @@ -11,18 +11,14 @@ runner = CliRunner() def test_chat_command(): - """Test chat command launches chat TUI with the global time-travel flags.""" + """Test chat command launches chat TUI.""" with patch("haiku.rag.chat.run_chat") as mock_chat: mock_chat.return_value = None - result = runner.invoke(cli, ["--at", "release-1", "chat"]) + result = runner.invoke(cli, ["chat"]) assert result.exit_code == 0 mock_chat.assert_called_once() - kwargs = mock_chat.call_args.kwargs - assert kwargs["read_only"] is True - assert kwargs["before"] is None - assert kwargs["at_tag"] == "release-1" def test_run_chat_creates_app_and_runs(temp_db_path: Path): diff --git a/tests/store/test_tags.py b/tests/store/test_tags.py index 1f741814..f005195a 100644 --- a/tests/store/test_tags.py +++ b/tests/store/test_tags.py @@ -218,3 +218,27 @@ async def test_tag_writes_raise_when_read_only(temp_db_path): tags = await store.list_tags() assert tags["release-1"].complete is True + + +@pytest.mark.asyncio +async def test_current_table_versions_returns_versions(temp_db_path): + """current_table_versions returns dict of table versions.""" + async with Store(temp_db_path, create=True) as store: + versions = await store.current_table_versions() + + assert "documents" in versions + assert "chunks" in versions + assert "settings" in versions + assert all(isinstance(v, int) for v in versions.values()) + + +@pytest.mark.asyncio +async def test_list_table_versions_returns_history(temp_db_path): + """list_table_versions returns version history for a table.""" + async with Store(temp_db_path, create=True) as store: + versions = await store.list_table_versions("documents") + + assert len(versions) >= 1 + for v in versions: + assert "version" in v + assert "timestamp" in v diff --git a/tests/store/test_time_travel.py b/tests/store/test_time_travel.py deleted file mode 100644 index 35aeafab..00000000 --- a/tests/store/test_time_travel.py +++ /dev/null @@ -1,157 +0,0 @@ -import asyncio -from datetime import UTC, datetime, timedelta - -import pytest - -from haiku.rag.store import ReadOnlyError, Store -from haiku.rag.store.models import Document -from haiku.rag.store.repositories.document import DocumentRepository - - -class TestStoreTimeTravel: - @pytest.mark.asyncio - async def test_store_with_before_is_read_only(self, temp_db_path): - """Store with before parameter is automatically read-only.""" - async with Store(temp_db_path, create=True): - pass - - before = datetime.now(UTC) + timedelta(hours=1) - async with Store(temp_db_path, before=before) as store: - assert store.is_read_only is True - - @pytest.mark.asyncio - async def test_store_before_raises_on_write(self, temp_db_path): - """Store with before parameter raises on write operations.""" - async with Store(temp_db_path, create=True): - pass - - before = datetime.now(UTC) + timedelta(hours=1) - async with Store(temp_db_path, before=before) as store: - with pytest.raises(ReadOnlyError): - store._assert_writable() - - @pytest.mark.asyncio - async def test_store_before_checks_out_historical_state(self, temp_db_path): - """Store with before parameter checks out tables to historical state.""" - async with Store(temp_db_path, create=True) as store: - repo = DocumentRepository(store) - await repo.create(Document(content="First document")) - - versions_after_first = await store.list_table_versions("documents") - latest_version = max(versions_after_first, key=lambda v: v["version"]) - time_after_first = latest_version["timestamp"] - - await asyncio.sleep(0.5) - - await repo.create(Document(content="Second document")) - - versions_after_second = await store.list_table_versions("documents") - assert len(versions_after_second) > len(versions_after_first) - - async with Store(temp_db_path, before=time_after_first) as store: - repo = DocumentRepository(store) - - docs = await repo.list_all(include_content=True) - assert len(docs) == 1 - assert docs[0].content == "First document" - - async with Store(temp_db_path) as store: - repo = DocumentRepository(store) - - docs = await repo.list_all() - assert len(docs) == 2 - - @pytest.mark.asyncio - async def test_store_before_no_version_raises(self, temp_db_path): - """Store with before datetime before any version raises ValueError.""" - async with Store(temp_db_path, create=True): - pass - - before = datetime(2000, 1, 1, tzinfo=UTC) - with pytest.raises(ValueError) as exc_info: - async with Store(temp_db_path, before=before): - pass - assert "No data exists before" in str(exc_info.value) - - @pytest.mark.asyncio - async def test_store_with_at_tag_is_read_only(self, temp_db_path): - """Store with at_tag parameter is automatically read-only.""" - async with Store(temp_db_path, create=True) as store: - await store.create_tag("release-1") - - async with Store(temp_db_path, at_tag="release-1") as store: - assert store.is_read_only is True - with pytest.raises(ReadOnlyError): - store._assert_writable() - - @pytest.mark.asyncio - async def test_store_at_tag_checks_out_tagged_state(self, temp_db_path): - """Store with at_tag checks out every table at the tagged version.""" - async with Store(temp_db_path, create=True) as store: - repo = DocumentRepository(store) - await repo.create(Document(content="First document")) - await store.create_tag("release-1") - await repo.create(Document(content="Second document")) - - async with Store(temp_db_path, at_tag="release-1") as store: - docs = await DocumentRepository(store).list_all(include_content=True) - assert len(docs) == 1 - assert docs[0].content == "First document" - - async with Store(temp_db_path) as store: - docs = await DocumentRepository(store).list_all() - assert len(docs) == 2 - - @pytest.mark.asyncio - async def test_store_at_tag_unknown_raises(self, temp_db_path): - async with Store(temp_db_path, create=True): - pass - - with pytest.raises(ValueError, match="nope"): - async with Store(temp_db_path, at_tag="nope"): - pass - - @pytest.mark.asyncio - async def test_store_at_tag_partial_raises_naming_table(self, temp_db_path): - """A tag missing from some tables fails with the table named.""" - async with Store(temp_db_path, create=True) as store: - version = await store.chunks_table.version() - await store.chunks_table.tags.create("stale", version) - - with pytest.raises(ValueError, match="documents"): - async with Store(temp_db_path, at_tag="stale"): - pass - - @pytest.mark.asyncio - async def test_store_at_tag_and_before_mutually_exclusive(self, temp_db_path): - async with Store(temp_db_path, create=True) as store: - await store.create_tag("release-1") - - with pytest.raises(ValueError, match="mutually exclusive"): - Store( - temp_db_path, - at_tag="release-1", - before=datetime.now(UTC), - ) - - @pytest.mark.asyncio - async def test_current_table_versions_returns_versions(self, temp_db_path): - """current_table_versions returns dict of table versions.""" - async with Store(temp_db_path, create=True) as store: - versions = await store.current_table_versions() - - assert "documents" in versions - assert "chunks" in versions - assert "settings" in versions - assert all(isinstance(v, int) for v in versions.values()) - - @pytest.mark.asyncio - async def test_list_table_versions_returns_history(self, temp_db_path): - """list_table_versions returns version history for a table.""" - async with Store(temp_db_path, create=True) as store: - versions = await store.list_table_versions("documents") - - assert len(versions) >= 1 - for v in versions: - assert "version" in v - assert "timestamp" in v diff --git a/tests/test_cli.py b/tests/test_cli.py index 32a19d2b..4d770cb5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -153,21 +153,6 @@ class TestTagCommands: assert "document_meta" not in asyncio.run(_table_names()) - def test_tag_write_commands_reject_time_travel(self, temp_db_path): - db = str(temp_db_path) - result = runner.invoke(cli, ["init", "--db", db]) - assert result.exit_code == 0 - - result = runner.invoke(cli, ["--at", "x", "tag", "create", "r1", "--db", db]) - assert result.exit_code == 1 - assert "--at" in result.output - - result = runner.invoke( - cli, ["--before", "2025-01-01", "tag", "delete", "r1", "--db", db] - ) - assert result.exit_code == 1 - assert "--before" in result.output - def test_tag_create_invalid_name_fails_cleanly(self, temp_db_path): """lance restricts ref names to alphanumeric, '.', '-', '_'; the CLI surfaces that as a clean error instead of a traceback.""" diff --git a/tests/test_client.py b/tests/test_client.py index 0fb7d7b1..40884c0a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2264,32 +2264,3 @@ async def test_rebuild_rechunk_with_url_prefixed_stored_content( assert doc_after is not None assert "example.com" in doc_after.content assert "Stored" in doc_after.content - - -async def test_client_at_tag_opens_tagged_state_read_only(temp_db_path): - """HaikuRAG(at_tag=...) opens the database read-only at the tagged state.""" - dim = Config.embeddings.model.vector_dim - - def _doc(name: str, text: str) -> DoclingDocument: - doc = DoclingDocument(name=name) - doc.add_text(label=DocItemLabel.TEXT, text=text) - return doc - - async with HaikuRAG(temp_db_path, create=True) as client: - await client.import_document( - _doc("first", "First document"), - [Chunk(content="First document", embedding=[0.1] * dim, order=0)], - uri="mem://first", - ) - await client.store.create_tag("release-1") - await client.import_document( - _doc("second", "Second document"), - [Chunk(content="Second document", embedding=[0.1] * dim, order=0)], - uri="mem://second", - ) - - async with HaikuRAG(temp_db_path, at_tag="release-1") as client: - assert client.is_read_only is True - docs = await client.list_documents() - assert len(docs) == 1 - assert docs[0].uri == "mem://first" diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 76fa571a..74bd866a 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -267,47 +267,3 @@ class TestMCPImageQuery: # in search_documents_by_image rejects it. results = await search_by_image(image_base64="!!! not base64 !!!") assert results == [] - - -class TestMCPTimeTravel: - @pytest.mark.asyncio - async def test_at_tag_disables_write_tools(self, mcp_db): - async with HaikuRAG(mcp_db) as rag: - await rag.store.create_tag("release-1") - - mcp = create_mcp_server(mcp_db, at_tag="release-1") - tool_names = [t.name for t in await mcp.list_tools()] - assert "add_document_from_text" not in tool_names - assert "delete_document" not in tool_names - - @pytest.mark.asyncio - async def test_before_disables_write_tools(self, mcp_db): - from datetime import UTC, datetime - - mcp = create_mcp_server(mcp_db, before=datetime.now(UTC)) - tool_names = [t.name for t in await mcp.list_tools()] - assert "add_document_from_text" not in tool_names - assert "delete_document" not in tool_names - - @pytest.mark.asyncio - async def test_at_tag_serves_tagged_state(self, mcp_db): - """Read tools on a tagged server must see the tagged state, not the - live database.""" - async with HaikuRAG(mcp_db) as rag: - await rag.store.create_tag("release-1") - await rag.create_document( - "A document added after the tag.", - title="Post-tag Doc", - uri="test://post-tag", - ) - - mcp = create_mcp_server(mcp_db, at_tag="release-1") - list_docs = await _get_tool(mcp, "list_documents") - docs = await list_docs() - uris = {d.uri for d in docs} - assert "test://post-tag" not in uris - assert len(docs) == 2 - - mcp_live = create_mcp_server(mcp_db, read_only=True) - list_docs = await _get_tool(mcp_live, "list_documents") - assert len(await list_docs()) == 3 diff --git a/tests/test_utils.py b/tests/test_utils.py index 0d1944b6..4214784b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -382,79 +382,6 @@ def test_get_package_versions(): assert len(value) > 0 -# --- parse_datetime tests --- - - -def test_parse_datetime_iso8601(): - from haiku.rag.utils import parse_datetime - - dt = parse_datetime("2025-01-15T14:30:00") - assert dt.year == 2025 - assert dt.month == 1 - assert dt.day == 15 - assert dt.hour == 14 - assert dt.minute == 30 - - -def test_parse_datetime_date_only(): - from haiku.rag.utils import parse_datetime - - dt = parse_datetime("2025-01-15") - assert dt.year == 2025 - assert dt.month == 1 - assert dt.day == 15 - - -def test_parse_datetime_with_timezone(): - from haiku.rag.utils import parse_datetime - - dt = parse_datetime("2025-01-15T14:30:00+00:00") - assert dt.year == 2025 - assert dt.tzinfo is not None - - -def test_parse_datetime_invalid(): - from haiku.rag.utils import parse_datetime - - with pytest.raises(ValueError, match="Could not parse datetime"): - parse_datetime("not-a-date") - - -# --- to_utc tests --- - - -def test_to_utc_naive_datetime(): - from datetime import datetime - - from haiku.rag.utils import to_utc - - naive = datetime(2025, 6, 15, 12, 0, 0) - result = to_utc(naive) - assert result.tzinfo is not None - - -def test_to_utc_utc_datetime(): - from datetime import UTC, datetime - - from haiku.rag.utils import to_utc - - utc_dt = datetime(2025, 6, 15, 12, 0, 0, tzinfo=UTC) - result = to_utc(utc_dt) - assert result is utc_dt - - -def test_to_utc_aware_non_utc(): - from datetime import UTC, datetime, timedelta, timezone - - from haiku.rag.utils import to_utc - - eastern = timezone(timedelta(hours=-5)) - aware = datetime(2025, 6, 15, 12, 0, 0, tzinfo=eastern) - result = to_utc(aware) - assert result.tzinfo == UTC - assert result.hour == 17 - - # --- apply_common_settings tests --- diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py deleted file mode 100644 index 6d2f406b..00000000 --- a/tests/utils/test_datetime.py +++ /dev/null @@ -1,85 +0,0 @@ -from datetime import UTC, datetime, timezone - -import pytest - -from haiku.rag.utils import parse_datetime, to_utc - - -class TestParseDateTime: - def test_parse_iso8601_with_timezone(self): - """Parse ISO 8601 datetime with timezone.""" - result = parse_datetime("2025-01-15T14:30:00+00:00") - assert result.year == 2025 - assert result.month == 1 - assert result.day == 15 - assert result.hour == 14 - assert result.minute == 30 - assert result.second == 0 - assert result.tzinfo is not None - - def test_parse_iso8601_without_timezone(self): - """Parse ISO 8601 datetime without timezone (naive).""" - result = parse_datetime("2025-01-15T14:30:00") - assert result.year == 2025 - assert result.month == 1 - assert result.day == 15 - assert result.hour == 14 - assert result.minute == 30 - - def test_parse_date_only(self): - """Parse date-only string as start of day.""" - result = parse_datetime("2025-01-15") - assert result.year == 2025 - assert result.month == 1 - assert result.day == 15 - assert result.hour == 0 - assert result.minute == 0 - assert result.second == 0 - - def test_parse_various_formats(self): - """Parse various datetime formats.""" - # ISO with Z suffix - result = parse_datetime("2025-01-15T14:30:00Z") - assert result.year == 2025 - assert result.month == 1 - assert result.day == 15 - - # With milliseconds - result = parse_datetime("2025-01-15T14:30:00.123") - assert result.microsecond == 123000 - - def test_parse_invalid_raises_value_error(self): - """Invalid datetime string raises ValueError.""" - with pytest.raises(ValueError) as exc_info: - parse_datetime("not-a-datetime") - assert "Could not parse datetime" in str(exc_info.value) - - -class TestToUtc: - def test_naive_datetime_assumes_local_and_converts(self): - """Naive datetime is assumed local and converted to UTC.""" - naive = datetime(2025, 1, 15, 14, 30, 0) - result = to_utc(naive) - assert result.tzinfo == UTC - - def test_utc_datetime_unchanged(self): - """UTC datetime is returned as-is.""" - utc_dt = datetime(2025, 1, 15, 14, 30, 0, tzinfo=UTC) - result = to_utc(utc_dt) - assert result == utc_dt - assert result.tzinfo == UTC - - def test_other_timezone_converts_to_utc(self): - """Datetime with other timezone is converted to UTC.""" - from datetime import timedelta - - # Create a datetime at UTC+5 - tz_plus5 = timezone(timedelta(hours=5)) - dt_plus5 = datetime(2025, 1, 15, 19, 30, 0, tzinfo=tz_plus5) - - result = to_utc(dt_plus5) - - # 19:30 UTC+5 = 14:30 UTC - assert result.tzinfo == UTC - assert result.hour == 14 - assert result.minute == 30