inspector command that runs a textual app to browse documents and chunks in the database

This commit is contained in:
Yiorgis Gozadinos 2025-11-19 17:42:40 +02:00
parent ada2b66d5b
commit 89a2668c25
No known key found for this signature in database
11 changed files with 526 additions and 4 deletions

View file

@ -411,6 +411,21 @@ def download_models_cmd():
raise typer.Exit(1)
@cli.command("inspect", help="Launch interactive TUI to inspect database contents")
def inspect(
db: Path | None = typer.Option(
None,
"--db",
help="Path to the LanceDB database file",
),
):
"""Launch the inspector TUI for browsing documents and chunks."""
from haiku.rag.inspector import run_inspector
db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
run_inspector(db_path)
@cli.command(
"serve",
help="Start haiku.rag server. Use --monitor, --mcp, and/or --agui to enable services.",

View file

@ -0,0 +1,3 @@
from haiku.rag.inspector.app import run_inspector
__all__ = ["run_inspector"]

View file

@ -0,0 +1,167 @@
# pyright: reportPossiblyUnboundVariable=false
from pathlib import Path
from typing import TYPE_CHECKING
from haiku.rag.client import HaikuRAG
from haiku.rag.config import get_config
if TYPE_CHECKING:
from textual.app import ComposeResult
try:
from textual.app import App
from textual.binding import Binding
from textual.widgets import Footer, Header
from haiku.rag.inspector.widgets.chunk_list import ChunkList
from haiku.rag.inspector.widgets.detail_view import DetailView
from haiku.rag.inspector.widgets.document_list import DocumentList
TEXTUAL_AVAILABLE = True
except ImportError:
TEXTUAL_AVAILABLE = False
App = object # type: ignore
class InspectorApp(App): # type: ignore[misc]
"""Textual TUI for inspecting LanceDB data."""
CSS = """
Screen {
layout: grid;
grid-size: 2 2;
grid-columns: 1fr 2fr;
grid-rows: 1fr 1fr;
}
#document-list {
column-span: 1;
row-span: 2;
border: solid $primary;
}
#chunk-list {
column-span: 1;
row-span: 1;
border: solid $secondary;
}
#detail-view {
column-span: 1;
row-span: 1;
border: solid $accent;
}
ListItem {
overflow: hidden;
}
ListItem Static {
overflow: hidden;
text-overflow: ellipsis;
}
"""
BINDINGS = [
Binding("q", "quit", "Quit", show=True),
Binding("d", "focus_documents", "Documents", show=True),
Binding("c", "focus_chunks", "Chunks", show=True),
Binding("v", "focus_detail", "Detail", show=True),
Binding("/", "search", "Search", show=True),
]
def __init__(self, db_path: Path):
super().__init__()
self.db_path = db_path
self.client: HaikuRAG | None = None
def compose(self) -> "ComposeResult":
"""Compose the UI layout."""
yield Header()
yield DocumentList(id="document-list")
yield ChunkList(id="chunk-list")
yield DetailView(id="detail-view")
yield Footer()
async def on_mount(self) -> None:
"""Initialize the app when mounted."""
config = get_config()
self.client = HaikuRAG(db_path=self.db_path, config=config, allow_create=False)
await self.client.__aenter__()
# Load initial documents
doc_list = self.query_one(DocumentList)
await doc_list.load_documents(self.client)
async def on_unmount(self) -> None:
"""Clean up when unmounting."""
if self.client:
await self.client.__aexit__(None, None, None)
def action_focus_documents(self) -> None:
"""Focus the documents list."""
self.query_one(DocumentList).focus()
def action_focus_chunks(self) -> None:
"""Focus the chunks list."""
self.query_one(ChunkList).focus()
def action_focus_detail(self) -> None:
"""Focus the detail view."""
self.query_one(DetailView).focus()
def action_search(self) -> None:
"""Open search dialog."""
# TODO: Implement search dialog
pass
async def on_document_list_document_selected(
self, message: DocumentList.DocumentSelected
) -> None:
"""Handle document selection from document list.
Args:
message: Message containing selected document
"""
if not self.client:
return
# Show document details
detail_view = self.query_one(DetailView)
await detail_view.show_document(message.document)
# Load chunks for this document
if message.document.id:
chunk_list = self.query_one(ChunkList)
await chunk_list.load_chunks_for_document(self.client, message.document.id)
async def on_chunk_list_chunk_selected(
self, message: ChunkList.ChunkSelected
) -> None:
"""Handle chunk selection from chunk list.
Args:
message: Message containing selected chunk
"""
# Show chunk details
detail_view = self.query_one(DetailView)
await detail_view.show_chunk(message.chunk)
def run_inspector(db_path: Path | None = None) -> None:
"""Run the inspector TUI.
Args:
db_path: Path to the LanceDB database. If None, uses default from config.
"""
if not TEXTUAL_AVAILABLE:
raise ImportError(
"Textual is not installed. Install it with: pip install 'haiku.rag-slim[inspector]'"
)
config = get_config()
if db_path is None:
db_path = config.storage.data_dir / "haiku.rag.lancedb"
app = InspectorApp(db_path)
app.run()

View file

@ -0,0 +1,5 @@
from haiku.rag.inspector.widgets.chunk_list import ChunkList
from haiku.rag.inspector.widgets.detail_view import DetailView
from haiku.rag.inspector.widgets.document_list import DocumentList
__all__ = ["ChunkList", "DetailView", "DocumentList"]

View file

@ -0,0 +1,87 @@
from textual.app import ComposeResult
from textual.containers import VerticalScroll
from textual.message import Message
from textual.widgets import ListItem, ListView, Static
from haiku.rag.client import HaikuRAG
from haiku.rag.store.models import Chunk
class ChunkList(VerticalScroll):
"""Widget for displaying and browsing chunks."""
class ChunkSelected(Message):
"""Message sent when a chunk is selected."""
def __init__(self, chunk: Chunk) -> None:
super().__init__()
self.chunk = chunk
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.chunks: list[Chunk] = []
self.list_view: ListView | None = None
def compose(self) -> ComposeResult:
"""Compose the chunk list."""
yield Static("[bold]Chunks[/bold]", classes="title")
self.list_view = ListView()
yield self.list_view
async def load_chunks_for_document(
self, client: HaikuRAG, document_id: str
) -> None:
"""Load chunks for a specific document.
Args:
client: HaikuRAG client instance
document_id: ID of the document to load chunks for
"""
if self.list_view is None:
return
self.chunks = await client.chunk_repository.get_by_document_id(document_id)
# Clear existing items
await self.list_view.clear()
# Add chunk items
for chunk in self.chunks:
first_line = chunk.content.split("\n")[0]
item = ListItem(Static(f"[{chunk.order}] {first_line}"))
await self.list_view.append(item)
async def load_chunks_from_search(
self, client: HaikuRAG, query: str, limit: int = 20
) -> None:
"""Load chunks from search results.
Args:
client: HaikuRAG client instance
query: Search query
limit: Maximum number of results
"""
if self.list_view is None:
return
results = await client.chunk_repository.search(
query=query, limit=limit, search_type="hybrid"
)
self.chunks = [chunk for chunk, _score in results]
# Clear existing items
await self.list_view.clear()
# Add chunk items with scores
for chunk, score in results:
first_line = chunk.content.split("\n")[0]
item = ListItem(Static(f"[{score:.2f}] {first_line}"))
await self.list_view.append(item)
async def on_list_view_selected(self, event: ListView.Selected) -> None:
"""Handle chunk selection."""
if event.list_view == self.list_view:
idx = event.list_view.index
if idx is not None and 0 <= idx < len(self.chunks):
chunk = self.chunks[idx]
self.post_message(self.ChunkSelected(chunk))

View file

@ -0,0 +1,91 @@
from textual.app import ComposeResult
from textual.containers import VerticalScroll
from textual.widgets import Markdown, Static
from haiku.rag.store.models import Chunk, Document
class DetailView(VerticalScroll):
"""Widget for displaying detailed content of documents or chunks."""
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.title_widget: Static | None = None
self.content_widget: Markdown | None = None
def compose(self) -> ComposeResult:
"""Compose the detail view."""
self.title_widget = Static("[bold]Detail View[/bold]", classes="title")
yield self.title_widget
self.content_widget = Markdown("")
yield self.content_widget
async def show_document(self, document: Document) -> None:
"""Display document details.
Args:
document: Document to display
"""
if self.title_widget and self.content_widget:
title = document.title or document.uri or "Untitled Document"
self.title_widget.update(f"[bold]Document: {title}[/bold]")
# Build markdown content
content_parts = []
if document.id:
content_parts.append(f"**ID:** `{document.id}`")
if document.uri:
content_parts.append(f"**URI:** `{document.uri}`")
if document.metadata:
metadata_str = "\n".join(
f" - {k}: {v}" for k, v in document.metadata.items()
)
content_parts.append(f"**Metadata:**\n{metadata_str}")
if document.created_at:
content_parts.append(f"**Created:** {document.created_at}")
if document.updated_at:
content_parts.append(f"**Updated:** {document.updated_at}")
content_parts.append("\n---\n")
content_parts.append("**Content:**\n")
content_parts.append(f"```\n{document.content}\n```")
await self.content_widget.update("\n\n".join(content_parts))
async def show_chunk(self, chunk: Chunk) -> None:
"""Display chunk details.
Args:
chunk: Chunk to display
"""
if self.title_widget and self.content_widget:
self.title_widget.update(f"[bold]Chunk {chunk.order}[/bold]")
# Build markdown content
content_parts = []
if chunk.id:
content_parts.append(f"**ID:** `{chunk.id}`")
if chunk.document_id:
content_parts.append(f"**Document ID:** `{chunk.document_id}`")
if chunk.document_title:
content_parts.append(f"**Document Title:** {chunk.document_title}")
if chunk.document_uri:
content_parts.append(f"**Document URI:** `{chunk.document_uri}`")
content_parts.append(f"**Order:** {chunk.order}")
if chunk.metadata:
metadata_str = "\n".join(
f" - {k}: {v}" for k, v in chunk.metadata.items()
)
content_parts.append(f"**Metadata:**\n{metadata_str}")
if chunk.embedding:
content_parts.append(
f"**Embedding:** {len(chunk.embedding)} dimensions"
)
content_parts.append("\n---\n")
content_parts.append("**Content:**\n")
content_parts.append(f"```\n{chunk.content}\n```")
await self.content_widget.update("\n\n".join(content_parts))

View file

@ -0,0 +1,61 @@
from textual.app import ComposeResult
from textual.containers import VerticalScroll
from textual.message import Message
from textual.widgets import ListItem, ListView, Static
from haiku.rag.client import HaikuRAG
from haiku.rag.store.models import Document
class DocumentList(VerticalScroll):
"""Widget for displaying and browsing documents."""
class DocumentSelected(Message):
"""Message sent when a document is selected."""
def __init__(self, document: Document) -> None:
super().__init__()
self.document = document
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
self.documents: list[Document] = []
self.list_view: ListView | None = None
def compose(self) -> ComposeResult:
"""Compose the document list."""
yield Static("[bold]Documents[/bold]", classes="title")
self.list_view = ListView()
yield self.list_view
async def load_documents(
self, client: HaikuRAG, limit: int = 100, offset: int = 0
) -> None:
"""Load documents from the database.
Args:
client: HaikuRAG client instance
limit: Maximum number of documents to load
offset: Offset for pagination
"""
if self.list_view is None:
return
self.documents = await client.list_documents(limit=limit, offset=offset)
# Clear existing items
await self.list_view.clear()
# Add document items
for doc in self.documents:
title = doc.title or doc.uri or doc.id or "Untitled"
item = ListItem(Static(f"{title}"))
await self.list_view.append(item)
async def on_list_view_selected(self, event: ListView.Selected) -> None:
"""Handle document selection."""
if event.list_view == self.list_view:
idx = event.list_view.index
if idx is not None and 0 <= idx < len(self.documents):
document = self.documents[idx]
self.post_message(self.DocumentSelected(document))

View file

@ -44,6 +44,8 @@ voyageai = ["voyageai>=0.3.5"]
mxbai = ["mxbai-rerank>=0.1.6"]
cohere = ["cohere>=5.20.0"]
zeroentropy = ["zeroentropy>=0.1.0a6"]
# Inspector TUI
inspector = ["textual>=1.0.0"]
# Model providers (delegated to pydantic-ai-slim)
anthropic = ["pydantic-ai-slim[anthropic]"]
groq = ["pydantic-ai-slim[groq]"]

View file

@ -21,11 +21,16 @@ classifiers = [
"Typing :: Typed",
]
dependencies = ["haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy]"]
dependencies = ["haiku.rag-slim[docling,voyageai,mxbai,cohere,zeroentropy,inspector]"]
[project.scripts]
haiku-rag = "haiku.rag.cli:cli"
[project.optional-dependencies]
inspector = [
"textual>=1.0.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

18
tests/test_inspector.py Normal file
View file

@ -0,0 +1,18 @@
from unittest.mock import patch
from typer.testing import CliRunner
from haiku.rag.cli import cli
runner = CliRunner()
def test_inspect_command():
"""Test inspect command launches inspector TUI."""
with patch("haiku.rag.inspector.run_inspector") as mock_inspector:
mock_inspector.return_value = None
result = runner.invoke(cli, ["inspect"])
assert result.exit_code == 0
mock_inspector.assert_called_once()

74
uv.lock
View file

@ -1267,7 +1267,12 @@ name = "haiku-rag"
version = "0.18.0"
source = { editable = "." }
dependencies = [
{ name = "haiku-rag-slim", extra = ["cohere", "docling", "mxbai", "voyageai", "zeroentropy"] },
{ name = "haiku-rag-slim", extra = ["cohere", "docling", "inspector", "mxbai", "voyageai", "zeroentropy"] },
]
[package.optional-dependencies]
inspector = [
{ name = "textual" },
]
[package.dev-dependencies]
@ -1285,7 +1290,11 @@ dev = [
]
[package.metadata]
requires-dist = [{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy"], editable = "haiku_rag_slim" }]
requires-dist = [
{ name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "inspector"], editable = "haiku_rag_slim" },
{ name = "textual", marker = "extra == 'inspector'", specifier = ">=1.0.0" },
]
provides-extras = ["inspector"]
[package.metadata.requires-dev]
dev = [
@ -1359,6 +1368,9 @@ google = [
groq = [
{ name = "pydantic-ai-slim", extra = ["groq"] },
]
inspector = [
{ name = "textual" },
]
mistral = [
{ name = "pydantic-ai-slim", extra = ["mistral"] },
]
@ -1395,12 +1407,13 @@ requires-dist = [
{ name = "python-dotenv", specifier = ">=1.2.1" },
{ name = "pyyaml", specifier = ">=6.0.3" },
{ name = "rich", specifier = ">=14.2.0" },
{ name = "textual", marker = "extra == 'inspector'", specifier = ">=1.0.0" },
{ name = "typer", specifier = ">=0.19.2,<0.20.0" },
{ name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.5" },
{ name = "watchfiles", specifier = ">=1.1.1" },
{ name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a6" },
]
provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]
provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "inspector", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]
[[package]]
name = "hf-xet"
@ -1902,6 +1915,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc", size = 12097, upload-time = "2024-04-05T13:03:10.514Z" },
]
[[package]]
name = "linkify-it-py"
version = "2.0.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "uc-micro-py" },
]
sdist = { url = "https://files.pythonhosted.org/packages/2a/ae/bb56c6828e4797ba5a4821eec7c43b8bf40f69cda4d4f5f8c8a2810ec96a/linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048", size = 27946, upload-time = "2024-02-04T14:48:04.179Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/04/1e/b832de447dee8b582cac175871d2f6c3d5077cc56d5575cadba1fd1cccfa/linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79", size = 19820, upload-time = "2024-02-04T14:48:02.496Z" },
]
[[package]]
name = "logfire"
version = "4.14.2"
@ -2035,6 +2060,11 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
]
[package.optional-dependencies]
linkify = [
{ name = "linkify-it-py" },
]
[[package]]
name = "marko"
version = "2.2.1"
@ -2132,6 +2162,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/67/0f/669ecbe78a0ba192afcc0b026ae62d1005779e91bad27ab9d703401510bf/mcp-1.21.2-py3-none-any.whl", hash = "sha256:59413ef15db757a785e3859548c1a7ffc7be57bf162c3c24afc0e04fd9f4181c", size = 174854, upload-time = "2025-11-17T13:56:04.987Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.5.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "markdown-it-py" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/86/dd6e5db36df29e76c7a7699123569a4a18c1623ce68d826ed96c62643cae/mdit_py_plugins-0.5.0-py3-none-any.whl", hash = "sha256:07a08422fc1936a5d26d146759e9155ea466e842f5ab2f7d2266dd084c8dab1f", size = 57205, upload-time = "2025-08-11T07:25:47.597Z" },
]
[[package]]
name = "mdurl"
version = "0.1.2"
@ -4610,6 +4652,23 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" },
]
[[package]]
name = "textual"
version = "6.6.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "markdown-it-py", extra = ["linkify"] },
{ name = "mdit-py-plugins" },
{ name = "platformdirs" },
{ name = "pygments" },
{ name = "rich" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f6/2f/f0b408f227edca21d1996c1cd0b65309f0cbff44264aa40aded3ff9ce2e1/textual-6.6.0.tar.gz", hash = "sha256:53345166d6b0f9fd028ed0217d73b8f47c3a26679a18ba3b67616dcacb470eec", size = 1579327, upload-time = "2025-11-10T17:50:00.038Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/53/b3/95ab646b0c908823d71e49ab8b5949ec9f33346cee3897d1af6be28a8d91/textual-6.6.0-py3-none-any.whl", hash = "sha256:5a9484bd15ee8a6fd8ac4ed4849fb25ee56bed2cecc7b8a83c4cd7d5f19515e5", size = 712606, upload-time = "2025-11-10T17:49:58.391Z" },
]
[[package]]
name = "tifffile"
version = "2025.10.16"
@ -4939,6 +4998,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
]
[[package]]
name = "uc-micro-py"
version = "1.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/91/7a/146a99696aee0609e3712f2b44c6274566bc368dfe8375191278045186b8/uc-micro-py-1.0.3.tar.gz", hash = "sha256:d321b92cff673ec58027c04015fcaa8bb1e005478643ff4a500882eaab88c48a", size = 6043, upload-time = "2024-02-09T16:52:01.654Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/37/87/1f677586e8ac487e29672e4b17455758fce261de06a0d086167bb760361a/uc_micro_py-1.0.3-py3-none-any.whl", hash = "sha256:db1dffff340817673d7b466ec86114a9dc0e9d4d9b5ba229d9d60e5c12600cd5", size = 6229, upload-time = "2024-02-09T16:52:00.371Z" },
]
[[package]]
name = "urllib3"
version = "2.5.0"