Merge pull request #206 from ggozad/feat/time-travel
Time-travel, query the database as it existed at a previous point in time
This commit is contained in:
commit
54f7336904
11 changed files with 683 additions and 23 deletions
|
|
@ -8,6 +8,12 @@
|
|||
- Skips database upgrades and settings saves on open
|
||||
- Excludes write tools (`add_document_*`, `delete_document`) from MCP server
|
||||
- Disables file monitor with warning when `--read-only` is used with `serve --monitor`
|
||||
- **Time Travel**: Query the database as it existed at a previous point in time
|
||||
- Global `--before` CLI flag accepts datetime strings (ISO 8601 or date-only)
|
||||
- Automatically enables read-only mode when time-traveling
|
||||
- New `history` command shows version history for database tables
|
||||
- Useful for debugging and auditing
|
||||
- Supported throughout: CLI, Client, App, Inspector
|
||||
|
||||
### Fixed
|
||||
|
||||
|
|
|
|||
65
docs/cli.md
65
docs/cli.md
|
|
@ -7,6 +7,7 @@ The `haiku-rag` CLI provides complete document management functionality.
|
|||
|
||||
- `--config` - Specify custom configuration file
|
||||
- `--read-only` - Open database in read-only mode (blocks writes, skips upgrades)
|
||||
- `--before` - Query database as it existed before a datetime (implies `--read-only`)
|
||||
- `--version` / `-v` - Show version and exit
|
||||
|
||||
Per-command options:
|
||||
|
|
@ -19,6 +20,7 @@ The `haiku-rag` CLI provides complete document management functionality.
|
|||
haiku-rag --config /path/to/config.yaml list
|
||||
haiku-rag --config /path/to/config.yaml list --db /path/to/custom.db
|
||||
haiku-rag --read-only search "query"
|
||||
haiku-rag --before "2025-01-15" search "query"
|
||||
haiku-rag add -h
|
||||
```
|
||||
|
||||
|
|
@ -354,3 +356,66 @@ This command downloads:
|
|||
- Ollama models referenced in your configuration (embeddings, QA, research, rerank)
|
||||
|
||||
Progress is displayed in real-time with download status and progress bars for Ollama model pulls.
|
||||
|
||||
## Time Travel
|
||||
|
||||
LanceDB maintains version history for tables, enabling you to query the database as it existed at a previous point in time. This is useful for:
|
||||
|
||||
- **Debugging**: Investigate data before a problematic change
|
||||
- **Auditing**: Verify what knowledge was available when a support ticket was filed
|
||||
|
||||
### Query Historical State
|
||||
|
||||
Use `--before` to query the database as it existed before a specific datetime:
|
||||
|
||||
```bash
|
||||
# Query documents as of January 15, 2025
|
||||
haiku-rag --before "2025-01-15" list
|
||||
|
||||
# Search historical state
|
||||
haiku-rag --before "2025-01-15T14:30:00" search "machine learning"
|
||||
|
||||
# Ask questions against historical data
|
||||
haiku-rag --before "2025-01-15" ask "What documents existed?"
|
||||
```
|
||||
|
||||
Supported datetime formats:
|
||||
|
||||
- ISO 8601: `2025-01-15T14:30:00`, `2025-01-15T14:30:00Z`, `2025-01-15T14:30:00+00:00`
|
||||
- Date only: `2025-01-15` (interpreted as start of day)
|
||||
|
||||
!!! note
|
||||
Time travel mode automatically enables read-only mode. You cannot modify the database while viewing historical state.
|
||||
|
||||
### Version History
|
||||
|
||||
View version history for database tables:
|
||||
|
||||
```bash
|
||||
# Show history for all tables
|
||||
haiku-rag history
|
||||
|
||||
# Show history for a specific table
|
||||
haiku-rag history --table documents
|
||||
|
||||
# Limit number of versions shown
|
||||
haiku-rag history --limit 10
|
||||
```
|
||||
|
||||
Output shows version numbers and timestamps, sorted newest first:
|
||||
|
||||
```
|
||||
Version History
|
||||
|
||||
documents
|
||||
v5: 2025-01-15 14:30:00
|
||||
v4: 2025-01-14 10:00:00
|
||||
v3: 2025-01-13 09:15:00
|
||||
|
||||
chunks
|
||||
v8: 2025-01-15 14:30:00
|
||||
v7: 2025-01-14 10:00:00
|
||||
...
|
||||
```
|
||||
|
||||
Use the timestamps from `history` to construct `--before` queries.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from importlib.metadata import version as pkg_version
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
|
@ -36,11 +37,16 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class HaikuRAGApp:
|
||||
def __init__(
|
||||
self, db_path: Path, config: AppConfig = Config, read_only: bool = False
|
||||
self,
|
||||
db_path: Path,
|
||||
config: AppConfig = Config,
|
||||
read_only: bool = False,
|
||||
before: datetime | None = None,
|
||||
):
|
||||
self.db_path = db_path
|
||||
self.config = config
|
||||
self.read_only = read_only
|
||||
self.before = before
|
||||
self.console = Console()
|
||||
|
||||
async def init(self):
|
||||
|
|
@ -215,9 +221,64 @@ class HaikuRAGApp:
|
|||
f" [repr.attrib_name]docling[/repr.attrib_name]: {docling_version}"
|
||||
)
|
||||
|
||||
async def history(self, table: str | None = None, limit: int | None = None):
|
||||
"""Display version history for database tables.
|
||||
|
||||
Args:
|
||||
table: Specific table to show history for (documents, chunks, settings).
|
||||
If None, shows history for all tables.
|
||||
limit: Maximum number of versions to show per table.
|
||||
"""
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
if not self.db_path.exists():
|
||||
self.console.print("[red]Database path does not exist.[/red]")
|
||||
return
|
||||
|
||||
store = Store(self.db_path, config=self.config, skip_validation=True)
|
||||
|
||||
tables = ["documents", "chunks", "settings"]
|
||||
if table:
|
||||
if table not in tables:
|
||||
self.console.print(
|
||||
f"[red]Unknown table: {table}. Must be one of: {', '.join(tables)}[/red]"
|
||||
)
|
||||
store.close()
|
||||
return
|
||||
tables = [table]
|
||||
|
||||
self.console.print("[bold]Version History[/bold]")
|
||||
|
||||
for table_name in tables:
|
||||
versions = store.list_table_versions(table_name)
|
||||
|
||||
# Sort by version descending (newest first)
|
||||
versions = sorted(versions, key=lambda v: v["version"], reverse=True)
|
||||
|
||||
if limit:
|
||||
versions = versions[:limit]
|
||||
|
||||
self.console.print(f"\n[bold cyan]{table_name}[/bold cyan]")
|
||||
|
||||
if not versions:
|
||||
self.console.print(" [dim]No versions found[/dim]")
|
||||
continue
|
||||
|
||||
for v in versions:
|
||||
version_num = v["version"]
|
||||
timestamp = v["timestamp"]
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]v{version_num}[/repr.attrib_name]: {timestamp}"
|
||||
)
|
||||
|
||||
store.close()
|
||||
|
||||
async def list_documents(self, filter: str | None = None):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
documents = await self.client.list_documents(filter=filter)
|
||||
for doc in documents:
|
||||
|
|
@ -225,7 +286,10 @@ class HaikuRAGApp:
|
|||
|
||||
async def add_document_from_text(self, text: str, metadata: dict | None = None):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
doc = await self.client.create_document(text, metadata=metadata)
|
||||
self._rich_print_document(doc, truncate=True)
|
||||
|
|
@ -237,7 +301,10 @@ class HaikuRAGApp:
|
|||
self, source: str, title: str | None = None, metadata: dict | None = None
|
||||
):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
result = await self.client.create_document_from_source(
|
||||
source, title=title, metadata=metadata
|
||||
|
|
@ -256,7 +323,10 @@ class HaikuRAGApp:
|
|||
|
||||
async def get_document(self, doc_id: str):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
doc = await self.client.get_document_by_id(doc_id)
|
||||
if doc is None:
|
||||
|
|
@ -266,7 +336,10 @@ class HaikuRAGApp:
|
|||
|
||||
async def delete_document(self, doc_id: str):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
deleted = await self.client.delete_document(doc_id)
|
||||
if deleted:
|
||||
|
|
@ -282,7 +355,10 @@ class HaikuRAGApp:
|
|||
self, query: str, limit: int | None = None, filter: str | None = None
|
||||
):
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
results = await self.client.search(query, limit=limit, filter=filter)
|
||||
if not results:
|
||||
|
|
@ -296,7 +372,10 @@ class HaikuRAGApp:
|
|||
from textual_image.renderable import Image as RichImage
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
chunk = await self.client.chunk_repository.get_by_id(chunk_id)
|
||||
if not chunk:
|
||||
|
|
@ -343,7 +422,10 @@ class HaikuRAGApp:
|
|||
filter: SQL WHERE clause to filter documents
|
||||
"""
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as self.client:
|
||||
try:
|
||||
citations = []
|
||||
|
|
@ -414,7 +496,10 @@ class HaikuRAGApp:
|
|||
filter: SQL WHERE clause to filter documents
|
||||
"""
|
||||
async with HaikuRAG(
|
||||
db_path=self.db_path, config=self.config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as client:
|
||||
try:
|
||||
self.console.print("[bold cyan]Starting research[/bold cyan]")
|
||||
|
|
@ -510,6 +595,7 @@ class HaikuRAGApp:
|
|||
config=self.config,
|
||||
skip_validation=True,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as client:
|
||||
try:
|
||||
documents = await client.list_documents()
|
||||
|
|
@ -549,6 +635,7 @@ class HaikuRAGApp:
|
|||
config=self.config,
|
||||
skip_validation=True,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as client:
|
||||
await client.vacuum()
|
||||
self.console.print(
|
||||
|
|
@ -565,6 +652,7 @@ class HaikuRAGApp:
|
|||
config=self.config,
|
||||
skip_validation=True,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as client:
|
||||
row_count = client.store.chunks_table.count_rows()
|
||||
self.console.print(f"Chunks in database: {row_count}")
|
||||
|
|
@ -735,7 +823,10 @@ class HaikuRAGApp:
|
|||
):
|
||||
"""Start the server with selected services."""
|
||||
async with HaikuRAG(
|
||||
self.db_path, config=self.config, read_only=self.read_only
|
||||
self.db_path,
|
||||
config=self.config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
) as client:
|
||||
tasks = []
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import json
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
from importlib.metadata import version
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -26,8 +27,9 @@ cli = typer.Typer(
|
|||
context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
|
||||
)
|
||||
|
||||
# Module-level read-only flag set by callback
|
||||
# Module-level flags set by callback
|
||||
_read_only: bool = False
|
||||
_before: datetime | None = None
|
||||
|
||||
|
||||
def create_app(db: Path | None = None) -> HaikuRAGApp:
|
||||
|
|
@ -41,7 +43,9 @@ def create_app(db: Path | None = None) -> HaikuRAGApp:
|
|||
"""
|
||||
config = get_config()
|
||||
db_path = db if db else config.storage.data_dir / "haiku.rag.lancedb"
|
||||
return HaikuRAGApp(db_path=db_path, config=config, read_only=_read_only)
|
||||
return HaikuRAGApp(
|
||||
db_path=db_path, config=config, read_only=_read_only, before=_before
|
||||
)
|
||||
|
||||
|
||||
async def check_version():
|
||||
|
|
@ -80,10 +84,28 @@ def main(
|
|||
"--read-only",
|
||||
help="Open database in read-only mode",
|
||||
),
|
||||
before: str | None = typer.Option(
|
||||
None,
|
||||
"--before",
|
||||
help="Query database as it existed before this datetime (implies --read-only). "
|
||||
"Accepts ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)",
|
||||
),
|
||||
):
|
||||
"""haiku.rag CLI - Vector database RAG system"""
|
||||
global _read_only
|
||||
global _read_only, _before
|
||||
_read_only = read_only
|
||||
|
||||
# Parse and store before datetime
|
||||
if before is not None:
|
||||
from haiku.rag.utils import parse_datetime, to_utc
|
||||
|
||||
try:
|
||||
_before = to_utc(parse_datetime(before))
|
||||
except ValueError as e:
|
||||
typer.echo(f"Error: {e}")
|
||||
raise typer.Exit(1)
|
||||
else:
|
||||
_before = None
|
||||
# Load config from --config, local folder, or default directory
|
||||
config_path = find_config_file(cli_path=config)
|
||||
if config_path:
|
||||
|
|
@ -363,7 +385,9 @@ def research(
|
|||
from haiku.rag.cli_chat import interactive_research
|
||||
from haiku.rag.client import HaikuRAG
|
||||
|
||||
client = HaikuRAG(db_path=app.db_path, config=app.config, read_only=_read_only)
|
||||
client = HaikuRAG(
|
||||
db_path=app.db_path, config=app.config, read_only=_read_only, before=_before
|
||||
)
|
||||
try:
|
||||
interactive_research(
|
||||
client=client,
|
||||
|
|
@ -505,6 +529,30 @@ def info(
|
|||
asyncio.run(app.info())
|
||||
|
||||
|
||||
@cli.command("history", help="Show version history for database tables")
|
||||
def history(
|
||||
db: Path | None = typer.Option(
|
||||
None,
|
||||
"--db",
|
||||
help="Path to the LanceDB database file",
|
||||
),
|
||||
table: str | None = typer.Option(
|
||||
None,
|
||||
"--table",
|
||||
"-t",
|
||||
help="Specific table to show history for (documents, chunks, settings)",
|
||||
),
|
||||
limit: int | None = typer.Option(
|
||||
None,
|
||||
"--limit",
|
||||
"-l",
|
||||
help="Maximum number of versions to show per table",
|
||||
),
|
||||
):
|
||||
app = create_app(db)
|
||||
asyncio.run(app.history(table=table, limit=limit))
|
||||
|
||||
|
||||
@cli.command("download-models", help="Download Docling and Ollama models per config")
|
||||
def download_models_cmd():
|
||||
app = HaikuRAGApp(db_path=Path(), config=get_config())
|
||||
|
|
@ -531,7 +579,7 @@ def inspect(
|
|||
raise typer.Exit(1) from e
|
||||
|
||||
db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
|
||||
run_inspector(db_path, read_only=_read_only)
|
||||
run_inspector(db_path, read_only=_read_only, before=_before)
|
||||
|
||||
|
||||
@cli.command(
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ class HaikuRAG:
|
|||
skip_validation: bool = False,
|
||||
create: bool = False,
|
||||
read_only: bool = False,
|
||||
before: datetime | None = None,
|
||||
):
|
||||
"""Initialize the RAG client with a database path.
|
||||
|
||||
|
|
@ -70,16 +71,20 @@ class HaikuRAG:
|
|||
skip_validation: Whether to skip configuration validation on database load.
|
||||
create: Whether to create the database if it doesn't exist.
|
||||
read_only: Whether to open the database in read-only mode.
|
||||
before: Query the database as it existed at this datetime.
|
||||
Implies read_only=True.
|
||||
"""
|
||||
self._config = config
|
||||
if db_path is None:
|
||||
db_path = self._config.storage.data_dir / "haiku.rag.lancedb"
|
||||
|
||||
self.store = Store(
|
||||
db_path,
|
||||
config=self._config,
|
||||
skip_validation=skip_validation,
|
||||
create=create,
|
||||
read_only=read_only,
|
||||
before=before,
|
||||
)
|
||||
self.document_repository = DocumentRepository(self.store)
|
||||
self.chunk_repository = ChunkRepository(self.store)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# pyright: reportPossiblyUnboundVariable=false
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
|
|
@ -74,10 +75,13 @@ class InspectorApp(App): # type: ignore[misc] # pragma: no cover
|
|||
Binding("c", "show_context", "Context", show=True),
|
||||
]
|
||||
|
||||
def __init__(self, db_path: Path, read_only: bool = False):
|
||||
def __init__(
|
||||
self, db_path: Path, read_only: bool = False, before: datetime | None = None
|
||||
):
|
||||
super().__init__()
|
||||
self.db_path = db_path
|
||||
self.read_only = read_only
|
||||
self.before = before
|
||||
self.client: HaikuRAG | None = None
|
||||
|
||||
def compose(self) -> "ComposeResult":
|
||||
|
|
@ -92,7 +96,10 @@ class InspectorApp(App): # type: ignore[misc] # pragma: no cover
|
|||
"""Initialize the app when mounted."""
|
||||
config = get_config()
|
||||
self.client = HaikuRAG(
|
||||
db_path=self.db_path, config=config, read_only=self.read_only
|
||||
db_path=self.db_path,
|
||||
config=config,
|
||||
read_only=self.read_only,
|
||||
before=self.before,
|
||||
)
|
||||
await self.client.__aenter__()
|
||||
|
||||
|
|
@ -233,17 +240,20 @@ class InspectorApp(App): # type: ignore[misc] # pragma: no cover
|
|||
|
||||
|
||||
def run_inspector(
|
||||
db_path: Path | None = None, read_only: bool = False
|
||||
db_path: Path | None = None,
|
||||
read_only: bool = False,
|
||||
before: datetime | None = None,
|
||||
) -> None: # pragma: no cover
|
||||
"""Run the inspector TUI.
|
||||
|
||||
Args:
|
||||
db_path: Path to the LanceDB database. If None, uses default from config.
|
||||
read_only: Whether to open the database in read-only mode.
|
||||
before: Query database as it existed before this datetime.
|
||||
"""
|
||||
config = get_config()
|
||||
if db_path is None:
|
||||
db_path = config.storage.data_dir / "haiku.rag.lancedb"
|
||||
|
||||
app = InspectorApp(db_path, read_only=read_only)
|
||||
app = InspectorApp(db_path, read_only=read_only, before=before)
|
||||
app.run()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import lancedb
|
||||
|
|
@ -59,10 +60,13 @@ class Store:
|
|||
skip_validation: bool = False,
|
||||
create: bool = False,
|
||||
read_only: bool = False,
|
||||
before: datetime | None = None,
|
||||
):
|
||||
self.db_path: Path = db_path
|
||||
self._config = config
|
||||
self._read_only = read_only
|
||||
self._before = before
|
||||
# Time-travel mode is always read-only
|
||||
self._read_only = read_only or (before is not None)
|
||||
self.embedder = get_embedder(config=self._config)
|
||||
self._vacuum_lock = asyncio.Lock()
|
||||
|
||||
|
|
@ -89,9 +93,13 @@ class Store:
|
|||
# Initialize tables (creates them if they don't exist)
|
||||
self._init_tables()
|
||||
|
||||
# Checkout tables to historical state if before is specified
|
||||
if before is not None:
|
||||
self._checkout_tables_before(before)
|
||||
|
||||
# Run upgrades only on existing databases, set version for new ones
|
||||
# Skip upgrades in read-only mode (they would fail anyway)
|
||||
if not read_only:
|
||||
if not self._read_only:
|
||||
if is_new_db:
|
||||
self._set_initial_version()
|
||||
else:
|
||||
|
|
@ -418,3 +426,83 @@ class Store:
|
|||
def _connection(self):
|
||||
"""Compatibility property for repositories expecting _connection."""
|
||||
return self
|
||||
|
||||
def _checkout_tables_before(self, before: datetime) -> None:
|
||||
"""Checkout all tables to their state at or before the given datetime.
|
||||
|
||||
Args:
|
||||
before: The datetime to checkout to
|
||||
|
||||
Raises:
|
||||
ValueError: If no version exists before the given datetime
|
||||
"""
|
||||
# LanceDB stores timestamps as naive datetimes in local time.
|
||||
# Convert 'before' to naive local time for comparison.
|
||||
if before.tzinfo is not None:
|
||||
# Convert to local time and make naive
|
||||
before_local = before.astimezone().replace(tzinfo=None)
|
||||
else:
|
||||
# Already naive, assume local time
|
||||
before_local = before
|
||||
|
||||
tables = [
|
||||
("documents", self.documents_table),
|
||||
("chunks", self.chunks_table),
|
||||
("settings", self.settings_table),
|
||||
]
|
||||
|
||||
for table_name, table in tables:
|
||||
versions = table.list_versions()
|
||||
# Find the latest version at or before the target datetime
|
||||
# Versions are sorted by version number, not timestamp, so we need to check all
|
||||
best_version = None
|
||||
best_timestamp = None
|
||||
|
||||
for v in versions:
|
||||
# LanceDB version timestamps are naive datetime objects in local time
|
||||
v_timestamp = v["timestamp"]
|
||||
# Make sure it's naive for comparison
|
||||
if v_timestamp.tzinfo is not None:
|
||||
v_timestamp = v_timestamp.replace(tzinfo=None)
|
||||
|
||||
if v_timestamp <= before_local:
|
||||
if best_timestamp is None or v_timestamp > best_timestamp:
|
||||
best_version = v["version"]
|
||||
best_timestamp = v_timestamp
|
||||
|
||||
if best_version is None:
|
||||
# Find the earliest version to report in error message
|
||||
if versions:
|
||||
earliest = min(versions, key=lambda v: v["timestamp"])
|
||||
earliest_ts = earliest["timestamp"]
|
||||
raise ValueError(
|
||||
f"No data exists before {before}. "
|
||||
f"Database was created on {earliest_ts}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"No data exists before {before}. Table has no versions."
|
||||
)
|
||||
|
||||
# Checkout to the found version
|
||||
table.checkout(best_version)
|
||||
|
||||
def list_table_versions(self, table_name: str) -> list[dict[str, Any]]:
|
||||
"""List version history for a table.
|
||||
|
||||
Args:
|
||||
table_name: Name of the table ("documents", "chunks", or "settings")
|
||||
|
||||
Returns:
|
||||
List of version info dicts with "version" and "timestamp" keys
|
||||
"""
|
||||
table_map = {
|
||||
"documents": self.documents_table,
|
||||
"chunks": self.chunks_table,
|
||||
"settings": self.settings_table,
|
||||
}
|
||||
table = table_map.get(table_name)
|
||||
if table is None:
|
||||
raise ValueError(f"Unknown table: {table_name}")
|
||||
|
||||
return list(table.list_versions())
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from importlib import metadata
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from dateutil import parser as dateutil_parser
|
||||
from packaging.version import Version, parse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -12,6 +14,55 @@ if TYPE_CHECKING:
|
|||
from haiku.rag.graph.research.models import Citation
|
||||
|
||||
|
||||
def parse_datetime(s: str) -> datetime:
|
||||
"""Parse a datetime string into a datetime object.
|
||||
|
||||
Supports:
|
||||
- ISO 8601 format: "2025-01-15T14:30:00", "2025-01-15T14:30:00Z", "2025-01-15T14:30:00+00:00"
|
||||
- Date only: "2025-01-15" (interpreted as 00:00:00)
|
||||
- Various other formats via dateutil
|
||||
|
||||
Args:
|
||||
s: String to parse
|
||||
|
||||
Returns:
|
||||
Parsed datetime object
|
||||
|
||||
Raises:
|
||||
ValueError: If the string cannot be parsed
|
||||
"""
|
||||
try:
|
||||
return dateutil_parser.parse(s)
|
||||
except (ValueError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"Could not parse datetime: {s}. "
|
||||
"Use ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)"
|
||||
) from e
|
||||
|
||||
|
||||
def to_utc(dt: datetime) -> datetime:
|
||||
"""Convert a datetime to UTC.
|
||||
|
||||
- Naive datetimes are assumed to be local time and converted to UTC
|
||||
- Datetimes with timezone info are converted to UTC
|
||||
- UTC datetimes are returned as-is
|
||||
|
||||
Args:
|
||||
dt: Datetime to convert
|
||||
|
||||
Returns:
|
||||
Datetime in UTC timezone
|
||||
"""
|
||||
if dt.tzinfo is None:
|
||||
# Naive datetime - assume local time
|
||||
local_dt = dt.astimezone() # Adds local timezone
|
||||
return local_dt.astimezone(UTC)
|
||||
elif dt.tzinfo == UTC:
|
||||
return dt
|
||||
else:
|
||||
return dt.astimezone(UTC)
|
||||
|
||||
|
||||
def apply_common_settings(
|
||||
settings: Any | None,
|
||||
settings_class: type[Any],
|
||||
|
|
|
|||
112
tests/store/test_time_travel.py
Normal file
112
tests/store/test_time_travel.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from haiku.rag.store import ReadOnlyError, Store
|
||||
from haiku.rag.store.models import Document
|
||||
from haiku.rag.store.repositories.document import DocumentRepository
|
||||
|
||||
|
||||
class TestStoreTimeTravel:
|
||||
def test_store_with_before_is_read_only(self, temp_db_path):
|
||||
"""Store with before parameter is automatically read-only."""
|
||||
# Create a store first
|
||||
store = Store(temp_db_path, create=True)
|
||||
store.close()
|
||||
|
||||
# Open with before - should be read-only
|
||||
before = datetime.now(UTC) + timedelta(hours=1)
|
||||
store = Store(temp_db_path, before=before)
|
||||
assert store.is_read_only is True
|
||||
store.close()
|
||||
|
||||
def test_store_before_raises_on_write(self, temp_db_path):
|
||||
"""Store with before parameter raises on write operations."""
|
||||
store = Store(temp_db_path, create=True)
|
||||
store.close()
|
||||
|
||||
before = datetime.now(UTC) + timedelta(hours=1)
|
||||
store = Store(temp_db_path, before=before)
|
||||
|
||||
with pytest.raises(ReadOnlyError):
|
||||
store._assert_writable()
|
||||
store.close()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_before_checks_out_historical_state(self, temp_db_path):
|
||||
"""Store with before parameter checks out tables to historical state."""
|
||||
# Create store and add a document
|
||||
store = Store(temp_db_path, create=True)
|
||||
repo = DocumentRepository(store)
|
||||
await repo.create(Document(content="First document"))
|
||||
|
||||
# Get the version timestamp after first document
|
||||
versions_after_first = store.list_table_versions("documents")
|
||||
# Find the latest version timestamp
|
||||
latest_version = max(versions_after_first, key=lambda v: v["version"])
|
||||
time_after_first = latest_version["timestamp"]
|
||||
|
||||
# Wait a bit to ensure the next write gets a distinct timestamp
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Add second document
|
||||
await repo.create(Document(content="Second document"))
|
||||
|
||||
# Verify we have more versions now
|
||||
versions_after_second = store.list_table_versions("documents")
|
||||
assert len(versions_after_second) > len(versions_after_first)
|
||||
|
||||
store.close()
|
||||
|
||||
# Open at historical state (using the timestamp from after first write)
|
||||
store = Store(temp_db_path, before=time_after_first)
|
||||
repo = DocumentRepository(store)
|
||||
|
||||
# Should only see first document
|
||||
docs = await repo.list_all()
|
||||
assert len(docs) == 1
|
||||
assert docs[0].content == "First document"
|
||||
store.close()
|
||||
|
||||
# Open at current state
|
||||
store = Store(temp_db_path)
|
||||
repo = DocumentRepository(store)
|
||||
|
||||
# Should see both documents
|
||||
docs = await repo.list_all()
|
||||
assert len(docs) == 2
|
||||
store.close()
|
||||
|
||||
def test_store_before_no_version_raises(self, temp_db_path):
|
||||
"""Store with before datetime before any version raises ValueError."""
|
||||
store = Store(temp_db_path, create=True)
|
||||
store.close()
|
||||
|
||||
# Try to open before the database was created
|
||||
before = datetime(2000, 1, 1, tzinfo=UTC)
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
Store(temp_db_path, before=before)
|
||||
assert "No data exists before" in str(exc_info.value)
|
||||
|
||||
def test_current_table_versions_returns_versions(self, temp_db_path):
|
||||
"""current_table_versions returns dict of table versions."""
|
||||
store = Store(temp_db_path, create=True)
|
||||
versions = store.current_table_versions()
|
||||
|
||||
assert "documents" in versions
|
||||
assert "chunks" in versions
|
||||
assert "settings" in versions
|
||||
assert all(isinstance(v, int) for v in versions.values())
|
||||
store.close()
|
||||
|
||||
def test_list_table_versions_returns_history(self, temp_db_path):
|
||||
"""list_table_versions returns version history for a table."""
|
||||
store = Store(temp_db_path, create=True)
|
||||
versions = store.list_table_versions("documents")
|
||||
|
||||
assert len(versions) >= 1
|
||||
for v in versions:
|
||||
assert "version" in v
|
||||
assert "timestamp" in v
|
||||
store.close()
|
||||
|
|
@ -454,3 +454,102 @@ async def test_ask_with_deep_and_verbose(app: HaikuRAGApp, monkeypatch):
|
|||
# With verbose, it should use AGUIConsoleRenderer.render, not graph.run
|
||||
mock_renderer.render.assert_called_once()
|
||||
mock_graph.run.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_all_tables(tmp_path, monkeypatch):
|
||||
"""Test history command shows version history for all tables."""
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
# Create a real database with some data
|
||||
db_path = tmp_path / "test.lancedb"
|
||||
store = Store(db_path, create=True)
|
||||
store.close()
|
||||
|
||||
app = HaikuRAGApp(db_path=db_path)
|
||||
mock_print = MagicMock()
|
||||
monkeypatch.setattr(app.console, "print", mock_print)
|
||||
|
||||
await app.history()
|
||||
|
||||
# Should print header and at least one version for each table
|
||||
calls = [str(c) for c in mock_print.call_args_list]
|
||||
assert any("Version History" in c for c in calls)
|
||||
assert any("documents" in c for c in calls)
|
||||
assert any("chunks" in c for c in calls)
|
||||
assert any("settings" in c for c in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_specific_table(tmp_path, monkeypatch):
|
||||
"""Test history command for a specific table."""
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
db_path = tmp_path / "test.lancedb"
|
||||
store = Store(db_path, create=True)
|
||||
store.close()
|
||||
|
||||
app = HaikuRAGApp(db_path=db_path)
|
||||
mock_print = MagicMock()
|
||||
monkeypatch.setattr(app.console, "print", mock_print)
|
||||
|
||||
await app.history(table="documents")
|
||||
|
||||
calls = [str(c) for c in mock_print.call_args_list]
|
||||
assert any("Version History" in c for c in calls)
|
||||
assert any("documents" in c for c in calls)
|
||||
# Should not show other tables
|
||||
assert not any("chunks" in c and "documents" not in c for c in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_invalid_table(tmp_path, monkeypatch):
|
||||
"""Test history command with invalid table name."""
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
db_path = tmp_path / "test.lancedb"
|
||||
store = Store(db_path, create=True)
|
||||
store.close()
|
||||
|
||||
app = HaikuRAGApp(db_path=db_path)
|
||||
mock_print = MagicMock()
|
||||
monkeypatch.setattr(app.console, "print", mock_print)
|
||||
|
||||
await app.history(table="invalid_table")
|
||||
|
||||
calls = [str(c) for c in mock_print.call_args_list]
|
||||
assert any("Unknown table" in c for c in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_with_limit(tmp_path, monkeypatch):
|
||||
"""Test history command with limit."""
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
db_path = tmp_path / "test.lancedb"
|
||||
store = Store(db_path, create=True)
|
||||
store.close()
|
||||
|
||||
app = HaikuRAGApp(db_path=db_path)
|
||||
mock_print = MagicMock()
|
||||
monkeypatch.setattr(app.console, "print", mock_print)
|
||||
|
||||
await app.history(limit=1)
|
||||
|
||||
# Should still work with limit
|
||||
calls = [str(c) for c in mock_print.call_args_list]
|
||||
assert any("Version History" in c for c in calls)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_nonexistent_db(tmp_path, monkeypatch):
|
||||
"""Test history command when database doesn't exist."""
|
||||
db_path = tmp_path / "nonexistent.lancedb"
|
||||
app = HaikuRAGApp(db_path=db_path)
|
||||
mock_print = MagicMock()
|
||||
monkeypatch.setattr(app.console, "print", mock_print)
|
||||
|
||||
await app.history()
|
||||
|
||||
calls = [str(c) for c in mock_print.call_args_list]
|
||||
assert any("does not exist" in c for c in calls)
|
||||
|
|
|
|||
85
tests/utils/test_datetime.py
Normal file
85
tests/utils/test_datetime.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
from datetime import UTC, datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from haiku.rag.utils import parse_datetime, to_utc
|
||||
|
||||
|
||||
class TestParseDateTime:
|
||||
def test_parse_iso8601_with_timezone(self):
|
||||
"""Parse ISO 8601 datetime with timezone."""
|
||||
result = parse_datetime("2025-01-15T14:30:00+00:00")
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.day == 15
|
||||
assert result.hour == 14
|
||||
assert result.minute == 30
|
||||
assert result.second == 0
|
||||
assert result.tzinfo is not None
|
||||
|
||||
def test_parse_iso8601_without_timezone(self):
|
||||
"""Parse ISO 8601 datetime without timezone (naive)."""
|
||||
result = parse_datetime("2025-01-15T14:30:00")
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.day == 15
|
||||
assert result.hour == 14
|
||||
assert result.minute == 30
|
||||
|
||||
def test_parse_date_only(self):
|
||||
"""Parse date-only string as start of day."""
|
||||
result = parse_datetime("2025-01-15")
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.day == 15
|
||||
assert result.hour == 0
|
||||
assert result.minute == 0
|
||||
assert result.second == 0
|
||||
|
||||
def test_parse_various_formats(self):
|
||||
"""Parse various datetime formats."""
|
||||
# ISO with Z suffix
|
||||
result = parse_datetime("2025-01-15T14:30:00Z")
|
||||
assert result.year == 2025
|
||||
assert result.month == 1
|
||||
assert result.day == 15
|
||||
|
||||
# With milliseconds
|
||||
result = parse_datetime("2025-01-15T14:30:00.123")
|
||||
assert result.microsecond == 123000
|
||||
|
||||
def test_parse_invalid_raises_value_error(self):
|
||||
"""Invalid datetime string raises ValueError."""
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
parse_datetime("not-a-datetime")
|
||||
assert "Could not parse datetime" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestToUtc:
|
||||
def test_naive_datetime_assumes_local_and_converts(self):
|
||||
"""Naive datetime is assumed local and converted to UTC."""
|
||||
naive = datetime(2025, 1, 15, 14, 30, 0)
|
||||
result = to_utc(naive)
|
||||
assert result.tzinfo == UTC
|
||||
|
||||
def test_utc_datetime_unchanged(self):
|
||||
"""UTC datetime is returned as-is."""
|
||||
utc_dt = datetime(2025, 1, 15, 14, 30, 0, tzinfo=UTC)
|
||||
result = to_utc(utc_dt)
|
||||
assert result == utc_dt
|
||||
assert result.tzinfo == UTC
|
||||
|
||||
def test_other_timezone_converts_to_utc(self):
|
||||
"""Datetime with other timezone is converted to UTC."""
|
||||
from datetime import timedelta
|
||||
|
||||
# Create a datetime at UTC+5
|
||||
tz_plus5 = timezone(timedelta(hours=5))
|
||||
dt_plus5 = datetime(2025, 1, 15, 19, 30, 0, tzinfo=tz_plus5)
|
||||
|
||||
result = to_utc(dt_plus5)
|
||||
|
||||
# 19:30 UTC+5 = 14:30 UTC
|
||||
assert result.tzinfo == UTC
|
||||
assert result.hour == 14
|
||||
assert result.minute == 30
|
||||
Loading…
Reference in a new issue