Accept images on CLI ask/analyze and MCP tools

This commit is contained in:
Yiorgis Gozadinos 2026-07-25 10:35:25 +03:00
parent 4c5050d161
commit c62fd78c7a
No known key found for this signature in database
8 changed files with 162 additions and 6 deletions

View file

@ -4,6 +4,8 @@
### Added
- `HaikuRAG.ask` and `HaikuRAG.analyze` accept `images: Sequence[bytes]`, attached to the question as model input; requires `vision: true` on the driving model.
- `haiku-rag ask` and `haiku-rag analyze` accept `--image PATH` (repeatable).
- MCP `ask_question` and `analyze` tools accept `images_base64`.
## [0.69.0] - 2026-07-24

View file

@ -161,11 +161,17 @@ Filter to specific documents:
haiku-rag ask "What are the main findings?" --filter "uri LIKE '%paper%'"
```
Attach images to the question, for example to check an image against indexed documents:
```bash
haiku-rag ask "Does this photo satisfy the spec in the design document?" --image photo.jpg
```
`ask` runs the [RAG capability](capabilities/rag.md) and always renders citations under the answer. When available, citations use the document title, otherwise they fall back to the URI.
Flags:
- `--filter` / `-f`: Restrict searches to documents matching the filter (see [Filtering Search Results](python.md#filtering-search-results))
- `--image`: Path to an image attached to the question (repeatable). Retrieval stays text-based; the model must have `vision: true` configured.
## Analyze
@ -184,6 +190,7 @@ haiku-rag analyze "What is the total revenue?" --filter "title LIKE '%Financial%
Flags:
- `--filter` / `-f`: SQL WHERE clause to restrict document access
- `--image`: Path to an image attached to the question (repeatable). Requires `vision: true` on the analysis model.
See [Analysis capability](capabilities/analysis.md) for details and configuration.

View file

@ -107,12 +107,12 @@ After restarting Claude Desktop, you can ask Claude to search your documents, ad
- **`ask_question`** - Ask questions about your documents
- `question` (required): The question to ask
- `cite` (optional): Include source citations (default: false)
- `deep` (optional): Use multi-agent deep QA for complex questions (default: false)
- `images_base64` (optional): Base64-encoded images attached to the question (requires a vision-capable QA model)
- **`analyze`** - Answer complex analytical questions via code execution
- `question` (required): The question to answer
- `filter` (optional): SQL WHERE clause to restrict document access
- `document` (optional): Document title/ID to pre-load (can repeat)
- `images_base64` (optional): Base64-encoded images attached to the question (requires a vision-capable analysis model)
- Best for aggregation, computation, and multi-document analysis
## Continuous ingestion

View file

@ -592,19 +592,25 @@ class HaikuRAGApp: # pragma: no cover
self,
question: str,
filter: str | None = None,
images: list[Path] | None = None,
):
"""Ask a question using the RAG system.
Args:
question: The question to ask
filter: SQL WHERE clause to filter documents
images: Paths of images to attach to the question
"""
async with HaikuRAG(
db_path=self.db_path,
config=self.config,
read_only=True,
) as self.client:
answer, citations = await self.client.ask(question, filter=filter)
answer, citations = await self.client.ask(
question,
filter=filter,
images=[path.read_bytes() for path in images] if images else None,
)
self.console.print(f"[bold blue]Question:[/bold blue] {question}")
self.console.print()
@ -619,12 +625,14 @@ class HaikuRAGApp: # pragma: no cover
self,
question: str,
filter: str | None = None,
images: list[Path] | None = None,
):
"""Answer a question using the analysis capability.
Args:
question: The question to answer
filter: SQL WHERE clause to filter documents
images: Paths of images to attach to the question
"""
async with HaikuRAG(
db_path=self.db_path,
@ -638,7 +646,11 @@ class HaikuRAGApp: # pragma: no cover
)
self.console.print()
result = await self.client.analyze(question, filter=filter)
result = await self.client.analyze(
question,
filter=filter,
images=[path.read_bytes() for path in images] if images else None,
)
self.console.print("[bold green]Answer:[/bold green]")
self.console.print(Markdown(result.answer))

View file

@ -352,12 +352,18 @@ def ask( # pragma: no cover
"-f",
help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
),
image: list[Path] | None = typer.Option(
None,
"--image",
help="Path to an image to attach to the question (repeatable; requires a vision-capable model)",
),
):
app = create_app(db)
asyncio.run(
app.ask(
question=question,
filter=filter,
images=image,
)
)
@ -378,12 +384,18 @@ def analyze( # pragma: no cover
"-f",
help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
),
image: list[Path] | None = typer.Option(
None,
"--image",
help="Path to an image to attach to the question (repeatable; requires a vision-capable model)",
),
):
app = create_app(db)
asyncio.run(
app.analyze(
question=question,
filter=filter,
images=image,
)
)

View file

@ -10,6 +10,14 @@ from haiku.rag.tools.document import DocumentInfo
from haiku.rag.utils import format_citations
def _decode_images(images_base64: list[str] | None) -> list[bytes] | None:
if not images_base64:
return None
import base64
return [base64.b64decode(b64, validate=True) for b64 in images_base64]
def create_mcp_server(
db_path: Path, config: AppConfig = Config, read_only: bool = False
) -> FastMCP:
@ -186,19 +194,23 @@ def create_mcp_server(
async def ask_question(
question: str,
cite: bool = False,
images_base64: list[str] | None = None,
) -> str:
"""Ask a question using the QA agent.
Args:
question: The question to ask.
cite: Whether to include citations in the response.
images_base64: Base64-encoded images attached to the question
(requires a vision-capable QA model).
Returns:
The answer as a string.
"""
try:
images = _decode_images(images_base64)
async with HaikuRAG(db_path, config=config, read_only=read_only) as rag:
answer, citations = await rag.ask(question)
answer, citations = await rag.ask(question, images=images)
if cite and citations:
answer += "\n\n" + format_citations(citations)
return answer
@ -209,6 +221,7 @@ def create_mcp_server(
async def analyze(
question: str,
filter: str | None = None,
images_base64: list[str] | None = None,
) -> str:
"""Answer complex questions using the analysis capability.
@ -219,13 +232,16 @@ def create_mcp_server(
Args:
question: The question to answer.
filter: Optional SQL WHERE clause to filter documents.
images_base64: Base64-encoded images attached to the question
(requires a vision-capable analysis model).
Returns:
The answer as a string.
"""
try:
images = _decode_images(images_base64)
async with HaikuRAG(db_path, config=config, read_only=read_only) as rag:
result = await rag.analyze(question, filter=filter)
result = await rag.analyze(question, filter=filter, images=images)
return result.answer
except Exception as e:
return f"Error running analysis capability: {e!s}" # pragma: no cover

View file

@ -271,3 +271,61 @@ class TestTagRestore:
result = runner.invoke(cli, ["--help"])
assert "--before" not in result.output
assert "--at" not in result.output
class TestAskAnalyzeImageOption:
def test_ask_forwards_image_paths(self):
from unittest.mock import AsyncMock
from haiku.rag.app import HaikuRAGApp
with patch.object(HaikuRAGApp, "ask", new_callable=AsyncMock) as mock_ask:
result = runner.invoke(
cli,
["ask", "q", "--image", "/tmp/a.png", "--image", "/tmp/b.jpg"],
)
assert result.exit_code == 0
from pathlib import Path
assert mock_ask.call_args.kwargs["images"] == [
Path("/tmp/a.png"),
Path("/tmp/b.jpg"),
]
def test_analyze_forwards_image_paths(self):
from unittest.mock import AsyncMock
from haiku.rag.app import HaikuRAGApp
with patch.object(HaikuRAGApp, "analyze", new_callable=AsyncMock) as mock:
result = runner.invoke(cli, ["analyze", "q", "--image", "/tmp/a.png"])
assert result.exit_code == 0
from pathlib import Path
assert mock.call_args.kwargs["images"] == [Path("/tmp/a.png")]
@pytest.mark.asyncio
async def test_app_ask_reads_image_bytes(self, temp_db_path, tmp_path):
from io import BytesIO
from unittest.mock import AsyncMock
from PIL import Image as PILImage
from haiku.rag.app import HaikuRAGApp
from haiku.rag.client import HaikuRAG
buffer = BytesIO()
PILImage.new("RGB", (4, 4)).save(buffer, format="PNG")
img_path = tmp_path / "img.png"
img_path.write_bytes(buffer.getvalue())
async with HaikuRAG(temp_db_path, create=True):
pass
with patch.object(
HaikuRAG, "ask", new_callable=AsyncMock, return_value=("answer", [])
) as mock_ask:
app = HaikuRAGApp(db_path=temp_db_path)
await app.ask("q", images=[img_path])
assert mock_ask.call_args.kwargs["images"] == [buffer.getvalue()]

View file

@ -267,3 +267,52 @@ class TestMCPImageQuery:
# in search_documents_by_image rejects it.
results = await search_by_image(image_base64="!!! not base64 !!!")
assert results == []
class TestMCPImageInput:
@pytest.mark.asyncio
async def test_ask_question_decodes_images(self, mcp_db, monkeypatch):
from base64 import b64encode
captured = {}
async def fake_ask(self, question, filter=None, images=None):
captured["images"] = images
return ("answer", [])
monkeypatch.setattr(HaikuRAG, "ask", fake_ask)
mcp = create_mcp_server(mcp_db, read_only=True)
ask = await _get_tool(mcp, "ask_question")
png = b"fake image bytes"
result = await ask(question="q", images_base64=[b64encode(png).decode()])
assert result == "answer"
assert captured["images"] == [png]
@pytest.mark.asyncio
async def test_analyze_decodes_images(self, mcp_db, monkeypatch):
from base64 import b64encode
from types import SimpleNamespace
captured = {}
async def fake_analyze(self, question, filter=None, images=None):
captured["images"] = images
return SimpleNamespace(answer="answer")
monkeypatch.setattr(HaikuRAG, "analyze", fake_analyze)
mcp = create_mcp_server(mcp_db, read_only=True)
analyze = await _get_tool(mcp, "analyze")
jpeg = b"fake jpeg bytes"
result = await analyze(question="q", images_base64=[b64encode(jpeg).decode()])
assert result == "answer"
assert captured["images"] == [jpeg]
@pytest.mark.asyncio
async def test_ask_question_rejects_invalid_base64(self, mcp_db):
mcp = create_mcp_server(mcp_db, read_only=True)
ask = await _get_tool(mcp, "ask_question")
result = await ask(question="q", images_base64=["!!! not base64 !!!"])
assert "Error" in result