danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replace Docker sandbox with pydantic-monty

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-02-17 15:56:54 +02:00
parent 2cafb5c8cc
commit c61ded1271
No known key found for this signature in database
32 changed files with 6418 additions and 4437 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
CHANGELOG.md
View file

@ -1,6 +1,20 @@
# Changelog
## [Unreleased]
### Changed
- **RLM sandbox**: Replaced Docker-based code execution with [pydantic-monty](https://github.com/pydantic/monty), a minimal secure Python interpreter written in Rust. Eliminates Docker as a runtime dependency for RLM with sub-millisecond sandbox startup
- **RLM sandbox functions**: Replaced `get_docling_document()` with `get_chunk(chunk_id)` for retrieving chunk content and metadata from search results
- **`RLMConfig`**: Removed `docker_image` and `docker_memory_limit` fields
### Added
- **`HaikuRAG.get_chunk_by_id()`**: Public method for chunk lookup by ID
### Removed
- **`docker_sandbox.py`**, **`runner.py`**: Docker container plumbing replaced by `sandbox.py`
## [0.31.1] - 2026-02-20
### Fixed

4
haiku_rag_slim/haiku/rag/agents/rlm/__init__.py
View file

@ -1,16 +1,16 @@
from haiku.rag.agents.rlm.agent import create_rlm_agent
from haiku.rag.agents.rlm.dependencies import RLMContext, RLMDeps
from haiku.rag.agents.rlm.docker_sandbox import DockerSandbox, SandboxResult
from haiku.rag.agents.rlm.models import CodeExecution, RLMResult
from haiku.rag.agents.rlm.prompts import RLM_SYSTEM_PROMPT
from haiku.rag.agents.rlm.sandbox import Sandbox, SandboxResult
__all__ = [
"CodeExecution",
"DockerSandbox",
"RLMContext",
"RLMDeps",
"RLMResult",
"RLM_SYSTEM_PROMPT",
"Sandbox",
"SandboxResult",
"create_rlm_agent",
]

5
haiku_rag_slim/haiku/rag/agents/rlm/agent.py
View file

@ -32,11 +32,10 @@ def create_rlm_agent(config: AppConfig) -> Agent[RLMDeps, RLMResult]:
@agent.tool
async def execute_code(ctx: RunContext[RLMDeps], code: str) -> CodeExecution:
"""Execute Python code in a Docker-sandboxed environment.
"""Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.

4
haiku_rag_slim/haiku/rag/agents/rlm/dependencies.py
View file

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
from haiku.rag.store.models import Document
if TYPE_CHECKING:
from haiku.rag.agents.rlm.docker_sandbox import DockerSandbox
from haiku.rag.agents.rlm.sandbox import Sandbox
@dataclass
@ -19,5 +19,5 @@ class RLMContext:
class RLMDeps:
"""Dependencies for RLM agent."""
sandbox: "DockerSandbox"
sandbox: "Sandbox"
context: RLMContext = field(default_factory=RLMContext)

216
haiku_rag_slim/haiku/rag/agents/rlm/docker_sandbox.py
View file

@ -1,216 +0,0 @@
"""Docker-based sandboxed execution."""
import asyncio
import json
import os
import subprocess
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING
from haiku.rag.agents.rlm.dependencies import RLMContext
from haiku.rag.config.models import RLMConfig
if TYPE_CHECKING:
from haiku.rag.client import HaikuRAG
@dataclass
class SandboxResult:
"""Result of executing code in the sandbox."""
stdout: str
stderr: str
success: bool
class DockerSandbox: # pragma: no cover
"""Execute code in a persistent Docker container.
Use as an async context manager to manage container lifecycle:
async with DockerSandbox(client, config, context) as sandbox:
result = await sandbox.execute("print('hello')")
result = await sandbox.execute("print('world')")
"""
DEFAULT_IMAGE = "ghcr.io/ggozad/haiku.rag-slim:latest"
haiku_client: "HaikuRAG"
config: RLMConfig
context: RLMContext
image: str
_process: subprocess.Popen[bytes] | None
def __init__(
self,
client: "HaikuRAG",
config: RLMConfig,
context: RLMContext,
image: str | None = None,
):
self.haiku_client = client
self.config = config
self.context = context
self.image = image or self.DEFAULT_IMAGE
self._process = None
def _build_docker_cmd(self) -> list[str]:
"""Build the docker run command."""
db_path = str(self.haiku_client.store.db_path)
env_list = ["-e", "HAIKU_DB_PATH=/data/db.lancedb"]
if self.context.filter:
env_list.extend(["-e", f"HAIKU_FILTER={self.context.filter}"])
ollama_host = os.environ.get("OLLAMA_HOST", "")
ollama_base_url = os.environ.get("OLLAMA_BASE_URL", "")
if sys.platform == "darwin":
if not ollama_host or "localhost" in ollama_host:
ollama_host = "http://host.docker.internal:11434"
if not ollama_base_url or "localhost" in ollama_base_url:
ollama_base_url = "http://host.docker.internal:11434"
if ollama_host:
env_list.extend(["-e", f"OLLAMA_HOST={ollama_host}"])
if ollama_base_url:
env_list.extend(["-e", f"OLLAMA_BASE_URL={ollama_base_url}"])
for key in [
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"VOYAGE_API_KEY",
"COHERE_API_KEY",
]:
if value := os.environ.get(key):
env_list.extend(["-e", f"{key}={value}"])
return [
"docker",
"run",
"--rm",
"-i",
"-v",
f"{db_path}:/data/db.lancedb:ro",
f"--memory={self.config.docker_memory_limit}",
"--network=host",
*env_list,
self.image,
"python",
"-m",
"haiku.rag.agents.rlm.runner",
]
async def __aenter__(self) -> "DockerSandbox":
"""Start the container."""
loop = asyncio.get_running_loop()
await loop.run_in_executor(None, self._start_container)
return self
async def __aexit__(
self, exc_type: object, exc_val: object, exc_tb: object
) -> None:
"""Stop the container."""
loop = asyncio.get_running_loop()
await loop.run_in_executor(None, self._stop_container)
def _start_container(self) -> None:
"""Start the persistent container process."""
if self._process is not None:
return
cmd = self._build_docker_cmd()
self._process = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
def _stop_container(self) -> None:
"""Stop the container process."""
if self._process is None:
return
try:
if self._process.stdin:
try:
self._process.stdin.close()
except BrokenPipeError:
pass
self._process.terminate()
self._process.wait(timeout=5)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait()
finally:
self._process = None
async def execute(self, code: str) -> SandboxResult:
"""Execute code in the container."""
if self._process is None:
return SandboxResult(
stdout="",
stderr="Container not started. Use 'async with' context manager.",
success=False,
)
loop = asyncio.get_running_loop()
return await loop.run_in_executor(None, self._execute_sync, code)
def _execute_sync(self, code: str) -> SandboxResult:
"""Send code to container and read result."""
assert self._process is not None and self._process.stdin is not None
try:
message = json.dumps({"code": code})
length_line = f"{len(message)}\n".encode()
self._process.stdin.write(length_line)
self._process.stdin.write(message.encode())
self._process.stdin.flush()
if self._process.stdout is None:
return SandboxResult(
stdout="", stderr="No stdout from container.", success=False
)
length_line = self._process.stdout.readline()
if not length_line:
stderr = ""
if self._process.stderr:
stderr = self._process.stderr.read().decode()
return SandboxResult(
stdout="",
stderr=stderr or "Container closed unexpectedly.",
success=False,
)
length = int(length_line.strip())
response = self._process.stdout.read(length).decode()
result_data = json.loads(response)
return SandboxResult(
stdout=result_data.get("stdout", ""),
stderr=result_data.get("stderr", ""),
success=result_data.get("success", False),
)
except subprocess.TimeoutExpired:
return SandboxResult(
stdout="",
stderr=f"Execution timed out after {self.config.code_timeout} seconds",
success=False,
)
except json.JSONDecodeError as e:
return SandboxResult(
stdout="",
stderr=f"Invalid response from container: {e}",
success=False,
)
except Exception as e:
return SandboxResult(
stdout="",
stderr=f"Execution error: {e}",
success=False,
)

190
haiku_rag_slim/haiku/rag/agents/rlm/runner.py
View file

@ -1,190 +0,0 @@
"""Entry point for sandboxed code execution in Docker container."""
import asyncio
import json
import sys
import traceback
from io import StringIO
from typing import Any
def build_namespace( # pragma: no cover
client: Any, config: Any, context: Any, loop: asyncio.AbstractEventLoop
) -> dict[str, Any]:
"""Build execution namespace with haiku.rag functions injected."""
def run_async(coro: Any) -> Any:
"""Run async coroutine from sync context using thread-safe scheduling."""
future = asyncio.run_coroutine_threadsafe(coro, loop)
return future.result(timeout=config.rlm.code_timeout)
def search(query: str, limit: int = 10) -> list[dict]:
async def _search() -> Any:
return await client.search(query, limit=limit, filter=context.filter)
results = run_async(_search())
return [
{
"chunk_id": r.chunk_id,
"content": r.content,
"document_id": r.document_id,
"document_title": r.document_title,
"document_uri": r.document_uri,
"score": r.score,
"page_numbers": r.page_numbers,
"headings": r.headings,
}
for r in results
]
def list_documents(limit: int = 10, offset: int = 0) -> list[dict]:
async def _list() -> Any:
return await client.list_documents(
limit=limit, offset=offset, filter=context.filter
)
docs = run_async(_list())
return [
{
"id": d.id,
"title": d.title,
"uri": d.uri,
"created_at": str(d.created_at),
}
for d in docs
]
def get_document(id_or_title: str) -> str | None:
async def _get() -> str | None:
doc = await client.resolve_document(id_or_title)
return doc.content if doc else None
return run_async(_get())
def get_docling_document(id_or_title: str) -> Any:
async def _get() -> Any:
doc = await client.resolve_document(id_or_title)
return doc.get_docling_document() if doc else None
return run_async(_get())
def llm(prompt: str) -> str:
async def _llm() -> str:
from pydantic_ai import Agent
from haiku.rag.utils import get_model
model = get_model(config.rlm.model, config)
agent: Agent[None, str] = Agent(model, output_type=str)
result = await agent.run(prompt)
return result.output
return run_async(_llm())
namespace: dict[str, Any] = {
"search": search,
"list_documents": list_documents,
"get_document": get_document,
"get_docling_document": get_docling_document,
"llm": llm,
}
if context.documents:
namespace["documents"] = [
{"id": d.id, "title": d.title, "uri": d.uri, "content": d.content}
for d in context.documents
]
return namespace
def execute_code(
code: str, namespace: dict[str, Any], max_output_chars: int
) -> dict[str, Any]:
"""Execute code and capture output."""
stdout_capture = StringIO()
original_stdout = sys.stdout
try:
sys.stdout = stdout_capture
exec(code, namespace)
stdout = stdout_capture.getvalue()
if len(stdout) > max_output_chars:
stdout = stdout[:max_output_chars] + "\n... (output truncated)"
return {
"success": True,
"stdout": stdout,
"stderr": "",
}
except Exception:
return {
"success": False,
"stdout": stdout_capture.getvalue(),
"stderr": traceback.format_exc(),
}
finally:
sys.stdout = original_stdout
def send_response(result: dict[str, Any]) -> None:
"""Send length-prefixed JSON response."""
response = json.dumps(result)
sys.stdout.write(f"{len(response)}\n")
sys.stdout.write(response)
sys.stdout.flush()
async def main() -> None: # pragma: no cover
"""Main entry point for container execution.
Runs a loop reading length-prefixed JSON messages and executing code.
"""
import concurrent.futures
import os
from pathlib import Path
from haiku.rag.agents.rlm.dependencies import RLMContext
from haiku.rag.client import HaikuRAG
from haiku.rag.config import get_config
config = get_config()
db_path = Path(os.environ.get("HAIKU_DB_PATH", "/data/db.lancedb"))
filter_expr = os.environ.get("HAIKU_FILTER")
context = RLMContext(filter=filter_expr)
max_output_chars = config.rlm.max_output_chars
loop = asyncio.get_running_loop()
async with HaikuRAG(db_path, config=config, read_only=True) as client:
namespace = build_namespace(client, config, context, loop)
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
while True:
# Read length-prefixed message
length_line = sys.stdin.readline()
if not length_line:
break
try:
length = int(length_line.strip())
message = sys.stdin.read(length)
request = json.loads(message)
code = request.get("code", "")
result = await loop.run_in_executor(
executor, execute_code, code, namespace, max_output_chars
)
send_response(result)
except (ValueError, json.JSONDecodeError) as e:
send_response(
{
"success": False,
"stdout": "",
"stderr": f"Invalid request: {e}",
}
)
if __name__ == "__main__":
asyncio.run(main())

240
haiku_rag_slim/haiku/rag/agents/rlm/sandbox.py Normal file
View file

@ -0,0 +1,240 @@
import asyncio
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any, Literal
import pydantic_monty
from haiku.rag.agents.rlm.dependencies import RLMContext
from haiku.rag.config.models import AppConfig
if TYPE_CHECKING:
from haiku.rag.client import HaikuRAG
@dataclass
class SandboxResult:
"""Result of executing code in the sandbox."""
stdout: str
stderr: str
success: bool
class Sandbox:
"""Execute code in a sandboxed Python interpreter.
Uses pydantic-monty, a minimal secure Python interpreter written in Rust.
External functions (search, list_documents, etc.) are called by Monty code
and resolved asynchronously on the host.
Use as an async context manager:
async with Sandbox(client, config, context) as sandbox:
result = await sandbox.execute("print('hello')")
"""
_client: "HaikuRAG"
_config: AppConfig
_context: RLMContext
def __init__(
self,
client: "HaikuRAG",
config: AppConfig,
context: RLMContext,
):
self._client = client
self._config = config
self._context = context
async def __aenter__(self) -> "Sandbox":
return self
async def __aexit__(
self, exc_type: object, exc_val: object, exc_tb: object
) -> None:
pass
def _build_external_functions(self) -> dict[str, Any]:
"""Build async external functions for the Monty interpreter."""
client = self._client
config = self._config
context = self._context
async def search(query: str, limit: int = 10) -> list[dict[str, Any]]:
results = await client.search(query, limit=limit, filter=context.filter)
return [
{
"chunk_id": r.chunk_id,
"content": r.content,
"document_id": r.document_id,
"document_title": r.document_title,
"document_uri": r.document_uri,
"score": r.score,
"page_numbers": r.page_numbers,
"headings": r.headings,
}
for r in results
]
async def list_documents(
limit: int = 10, offset: int = 0
) -> list[dict[str, Any]]:
docs = await client.list_documents(
limit=limit, offset=offset, filter=context.filter
)
return [
{
"id": d.id,
"title": d.title,
"uri": d.uri,
"created_at": str(d.created_at),
}
for d in docs
]
async def get_document(id_or_title: str) -> str | None:
doc = await client.resolve_document(id_or_title)
return doc.content if doc else None
async def get_chunk(chunk_id: str) -> dict[str, Any] | None:
chunk = await client.get_chunk_by_id(chunk_id)
if not chunk:
return None
meta = chunk.get_chunk_metadata()
doc_title = chunk.document_title
if not doc_title and chunk.document_id:
doc = await client.get_document_by_id(chunk.document_id)
if doc:
doc_title = doc.title
return {
"chunk_id": chunk.id,
"content": chunk.content,
"document_id": chunk.document_id,
"document_title": doc_title,
"headings": meta.headings,
"page_numbers": meta.page_numbers,
"labels": meta.labels,
}
async def llm(prompt: str) -> str:
from pydantic_ai import Agent
from haiku.rag.utils import get_model
model = get_model(config.rlm.model, config)
agent: Agent[None, str] = Agent(model, output_type=str)
result = await agent.run(prompt)
return result.output
return {
"search": search,
"list_documents": list_documents,
"get_document": get_document,
"get_chunk": get_chunk,
"llm": llm,
}
async def execute(self, code: str) -> SandboxResult:
"""Execute Python code in the Monty interpreter.
Uses a manual start/resume loop so that async external functions
are awaited on the host while Monty code calls them synchronously
(without ``await``).
"""
external_fns = self._build_external_functions()
input_names: list[str] = []
inputs: dict[str, Any] | None = None
if self._context.documents:
input_names.append("documents")
inputs = {
"documents": [
{
"id": d.id,
"title": d.title,
"uri": d.uri,
"content": d.content,
}
for d in self._context.documents
]
}
try:
monty = pydantic_monty.Monty(
code,
inputs=input_names,
external_functions=list(external_fns.keys()),
)
except pydantic_monty.MontySyntaxError as e:
return SandboxResult(stdout="", stderr=str(e), success=False)
stdout_lines: list[str] = []
def print_callback(_stream: Literal["stdout"], text: str) -> None:
stdout_lines.append(text)
max_chars = self._config.rlm.max_output_chars
limits: pydantic_monty.ResourceLimits = {
"max_duration_secs": self._config.rlm.code_timeout,
}
loop = asyncio.get_running_loop()
try:
with ThreadPoolExecutor() as pool:
async def run_in_pool(func: Any) -> Any:
return await loop.run_in_executor(pool, func)
progress = await run_in_pool(
partial(
monty.start,
inputs=inputs,
limits=limits,
print_callback=print_callback,
)
)
while not isinstance(progress, pydantic_monty.MontyComplete):
assert isinstance(progress, pydantic_monty.MontySnapshot)
fn = external_fns.get(progress.function_name)
if fn is None:
exc = KeyError(f"Function {progress.function_name} not found")
progress = await run_in_pool(
partial(progress.resume, exception=exc)
)
continue
try:
result = await fn(*progress.args, **progress.kwargs)
except Exception as exc:
progress = await run_in_pool(
partial(progress.resume, exception=exc)
)
else:
progress = await run_in_pool(
partial(progress.resume, return_value=result)
)
output = progress.output
except pydantic_monty.MontyRuntimeError as e:
stdout = "".join(stdout_lines)
if len(stdout) > max_chars:
stdout = stdout[:max_chars] + "\n... (output truncated)"
return SandboxResult(stdout=stdout, stderr=str(e), success=False)
stdout = "".join(stdout_lines)
if output is not None:
stdout_with_output = f"{stdout}{output}" if stdout else str(output)
else:
stdout_with_output = stdout
if len(stdout_with_output) > max_chars:
stdout_with_output = (
stdout_with_output[:max_chars] + "\n... (output truncated)"
)
return SandboxResult(stdout=stdout_with_output, stderr="", success=True)

2
haiku_rag_slim/haiku/rag/app.py
View file

@ -354,7 +354,7 @@ class HaikuRAGApp: # pragma: no cover
read_only=self.read_only,
before=self.before,
) as self.client:
chunk = await self.client.chunk_repository.get_by_id(chunk_id)
chunk = await self.client.get_chunk_by_id(chunk_id)
if not chunk:
self.console.print(f"[red]Chunk with id {chunk_id} not found.[/red]")
return

2
haiku_rag_slim/haiku/rag/chat/app.py
View file

@ -344,7 +344,7 @@ class ChatApp(App):
return
citation = selected_widgets[0].citation
chunk = await self.client.chunk_repository.get_by_id(citation.chunk_id)
chunk = await self.client.get_chunk_by_id(citation.chunk_id)
if not chunk:
return

18
haiku_rag_slim/haiku/rag/client.py
View file

@ -732,6 +732,17 @@ class HaikuRAG:
"""
return await self.document_repository.get_by_id(document_id)
async def get_chunk_by_id(self, chunk_id: str) -> Chunk | None:
"""Get a chunk by its ID.
Args:
chunk_id: The unique identifier of the chunk.
Returns:
The Chunk instance if found, None otherwise.
"""
return await self.chunk_repository.get_by_id(chunk_id)
async def get_document_by_uri(self, uri: str) -> Document | None:
"""Get a document by its URI.
@ -1379,9 +1390,9 @@ class HaikuRAG:
RLMResult with the answer and the final consolidated program.
"""
from haiku.rag.agents.rlm import (
DockerSandbox,
RLMContext,
RLMDeps,
Sandbox,
create_rlm_agent,
)
@ -1395,11 +1406,10 @@ class HaikuRAG:
loaded_docs.append(doc)
context.documents = loaded_docs if loaded_docs else None
async with DockerSandbox(
async with Sandbox(
client=self,
config=self._config.rlm,
config=self._config,
context=context,
image=self._config.rlm.docker_image,
) as sandbox:
deps = RLMDeps(
sandbox=sandbox,

2
haiku_rag_slim/haiku/rag/config/models.py
View file

@ -104,8 +104,6 @@ class RLMConfig(BaseModel):
)
code_timeout: float = 60.0
max_output_chars: int = 50_000
docker_image: str = "ghcr.io/ggozad/haiku.rag-slim:latest"
docker_memory_limit: str = "512m"
class PictureDescriptionConfig(BaseModel):

4
haiku_rag_slim/haiku/rag/inspector/widgets/search_modal.py
View file

@ -108,9 +108,7 @@ class SearchModal(Screen):
self.chunks = []
for result in self.search_results:
if result.chunk_id:
chunk = await self.client.chunk_repository.get_by_id(
result.chunk_id
)
chunk = await self.client.get_chunk_by_id(result.chunk_id)
if chunk:
self.chunks.append(chunk)

7
haiku_rag_slim/haiku/rag/tools/analysis.py
View file

@ -3,7 +3,7 @@ from pydantic_ai import FunctionToolset, RunContext
from haiku.rag.agents.rlm.agent import create_rlm_agent
from haiku.rag.agents.rlm.dependencies import RLMContext, RLMDeps
from haiku.rag.agents.rlm.docker_sandbox import DockerSandbox
from haiku.rag.agents.rlm.sandbox import Sandbox
from haiku.rag.config.models import AppConfig
from haiku.rag.tools.context import RAGDeps
from haiku.rag.tools.filters import (
@ -62,11 +62,10 @@ def create_analysis_toolset(
rlm_context = RLMContext(filter=effective_filter)
async with DockerSandbox(
async with Sandbox(
client=client,
config=config.rlm,
config=config,
context=rlm_context,
image=config.rlm.docker_image,
) as sandbox:
deps = RLMDeps(
sandbox=sandbox,

1
haiku_rag_slim/pyproject.toml
View file

@ -31,6 +31,7 @@ dependencies = [
"pathspec>=1.0.3",
"pydantic>=2.12.5",
"pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]>=1.46.0",
"pydantic-monty>=0.0.6",
"python-dotenv>=1.2.1",
"pyyaml>=6.0.3",
"rich>=14.2.0",

44
tests/agents/rlm/conftest.py
View file

@ -1,39 +1,9 @@
import os
import subprocess
from pathlib import Path
import pytest
from haiku.rag.agents.rlm.dependencies import RLMContext
from haiku.rag.agents.rlm.docker_sandbox import DockerSandbox
from haiku.rag.agents.rlm.sandbox import Sandbox
from haiku.rag.client import HaikuRAG
from haiku.rag.config.models import RLMConfig
TEST_DOCKER_IMAGE = os.environ.get("HAIKU_TEST_DOCKER_IMAGE", "haiku-rag-slim:test")
@pytest.fixture(scope="session")
def test_docker_image():
"""Build and return the Docker image for testing."""
if os.environ.get("CI"):
return TEST_DOCKER_IMAGE
project_root = Path(__file__).parent.parent.parent.parent
dockerfile = project_root / "docker" / "Dockerfile.slim"
if not dockerfile.exists():
pytest.skip(f"Dockerfile.slim not found at {dockerfile}")
result = subprocess.run(
["docker", "build", "-t", TEST_DOCKER_IMAGE, "-f", str(dockerfile), "."],
cwd=project_root,
capture_output=True,
text=True,
)
if result.returncode != 0:
pytest.fail(f"Failed to build Docker image:\n{result.stderr}")
return TEST_DOCKER_IMAGE
from haiku.rag.config.models import AppConfig
@pytest.fixture
@ -44,11 +14,9 @@ async def empty_client(temp_db_path):
@pytest.fixture
async def docker_sandbox(empty_client, test_docker_image):
"""Create a Docker sandbox for testing."""
config = RLMConfig(docker_image=test_docker_image)
async def sandbox(empty_client):
"""Create a Monty sandbox for testing."""
config = AppConfig()
context = RLMContext()
async with DockerSandbox(
client=empty_client, config=config, context=context, image=test_docker_image
) as sandbox:
async with Sandbox(client=empty_client, config=config, context=context) as sandbox:
yield sandbox

67
tests/agents/rlm/test_agent.py
View file

@ -47,9 +47,7 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_count_documents(
self, allow_model_requests, temp_db_path, test_docker_image
):
async def test_rlm_count_documents(self, allow_model_requests, temp_db_path):
"""Test RLM agent can count documents.
Agent program:
@ -59,7 +57,7 @@ class TestClientRLMIntegration:
from haiku.rag.client import HaikuRAG
config = AppConfig()
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document("First document about cats.", title="Doc 1")
await client.create_document("Second document about dogs.", title="Doc 2")
@ -71,9 +69,7 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_aggregation(
self, allow_model_requests, temp_db_path, test_docker_image
):
async def test_rlm_aggregation(self, allow_model_requests, temp_db_path):
"""Test RLM agent can perform aggregation across documents.
Agent program:
@ -95,7 +91,7 @@ class TestClientRLMIntegration:
from haiku.rag.client import HaikuRAG
config = AppConfig()
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document(
"Sales report Q1: Revenue was $100,000.", title="Q1 Report"
@ -115,9 +111,7 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_with_filter(
self, allow_model_requests, temp_db_path, test_docker_image
):
async def test_rlm_with_filter(self, allow_model_requests, temp_db_path):
"""Test RLM agent respects filter parameter.
Agent program:
@ -130,7 +124,7 @@ class TestClientRLMIntegration:
from haiku.rag.client import HaikuRAG
config = AppConfig()
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document("Cat document.", title="Cats")
await client.create_document("Dog document.", title="Dogs")
@ -145,42 +139,36 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_docling_document_structure(
self, allow_model_requests, temp_db_path, test_docker_image
):
"""Test RLM agent can analyze document structure using DoclingDocument.
async def test_rlm_search_and_get_chunk(self, allow_model_requests, temp_db_path):
"""Test RLM agent can search and use get_chunk for citations.
Agent program:
docs = list_documents(limit=20)
print(docs)
doc = get_docling_document('<doc_id>')
print(doc.name)
print('tables:', len(doc.tables))
print('pictures:', len(doc.pictures))
results = search("content", limit=5)
for r in results:
chunk = get_chunk(r['chunk_id'])
print(chunk['document_title'], chunk['chunk_id'])
"""
from haiku.rag.client import HaikuRAG
pdf_path = Path("tests/data/doclaynet.pdf")
config = AppConfig()
config.processing.conversion_options.do_ocr = False
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document_from_source(pdf_path)
result = await client.rlm(
"How many tables are in the document? "
"Also tell me how many pictures/figures it contains."
await client.create_document(
"The quick brown fox jumps over the lazy dog.",
title="Animal Facts",
)
# The doclaynet.pdf has 1 table and 1 picture
assert "1" in result.answer
result = await client.rlm(
"Search for content about animals and tell me "
"which document it came from."
)
assert "Animal Facts" in result.answer
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_semantic_analysis_with_llm(
self, allow_model_requests, temp_db_path, test_docker_image
self, allow_model_requests, temp_db_path
):
"""Test RLM agent can use llm() for semantic analysis combined with computation.
@ -200,7 +188,7 @@ class TestClientRLMIntegration:
from haiku.rag.client import HaikuRAG
config = AppConfig()
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document(
"The new product launch exceeded expectations. Sales grew 40% "
@ -232,9 +220,7 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_search_and_extract(
self, allow_model_requests, temp_db_path, test_docker_image
):
async def test_rlm_search_and_extract(self, allow_model_requests, temp_db_path):
"""Test RLM agent can use search() to find content and extract information.
Agent program:
@ -252,7 +238,6 @@ class TestClientRLMIntegration:
pdf_path = Path("tests/data/doclaynet.pdf")
config = AppConfig()
config.processing.conversion_options.do_ocr = False
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document_from_source(pdf_path)
@ -293,7 +278,7 @@ class TestClientRLMIntegration:
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_rlm_with_preloaded_documents(
self, allow_model_requests, temp_db_path, test_docker_image
self, allow_model_requests, temp_db_path
):
"""Test RLM agent can use pre-loaded documents variable.
@ -307,7 +292,7 @@ class TestClientRLMIntegration:
from haiku.rag.client import HaikuRAG
config = AppConfig()
config.rlm.docker_image = test_docker_image
async with HaikuRAG(temp_db_path, config=config, create=True) as client:
await client.create_document(
"The company was founded in 1985 by Jane Smith.",

50
tests/agents/rlm/test_runner.py
View file

@ -1,50 +0,0 @@
import json
from io import StringIO
from haiku.rag.agents.rlm.runner import execute_code, send_response
def test_execute_code_success():
namespace: dict = {}
result = execute_code("x = 1 + 1", namespace, max_output_chars=1000)
assert result["success"] is True
assert result["stderr"] == ""
def test_execute_code_stdout_capture():
namespace: dict = {}
result = execute_code("print('hello')", namespace, max_output_chars=1000)
assert result["success"] is True
assert "hello" in result["stdout"]
def test_execute_code_exception():
namespace: dict = {}
result = execute_code("raise ValueError('boom')", namespace, max_output_chars=1000)
assert result["success"] is False
assert "ValueError" in result["stderr"]
assert "boom" in result["stderr"]
def test_execute_code_output_truncation():
namespace: dict = {}
code = "print('x' * 100)"
result = execute_code(code, namespace, max_output_chars=10)
assert result["success"] is True
assert "truncated" in result["stdout"]
assert len(result["stdout"]) < 100
def test_send_response(monkeypatch):
buf = StringIO()
monkeypatch.setattr("sys.stdout", buf)
payload = {"success": True, "stdout": "hi", "stderr": ""}
send_response(payload)
output = buf.getvalue()
lines = output.split("\n", 1)
length = int(lines[0])
body = lines[1]
assert json.loads(body) == payload
assert length == len(json.dumps(payload))

214
tests/agents/rlm/test_sandbox.py
View file

@ -1,12 +1,11 @@
import os
from pathlib import Path
import pytest
from haiku.rag.agents.rlm.dependencies import RLMContext
from haiku.rag.agents.rlm.docker_sandbox import DockerSandbox, SandboxResult
from haiku.rag.agents.rlm.sandbox import Sandbox, SandboxResult
from haiku.rag.client import HaikuRAG
from haiku.rag.config.models import RLMConfig
from haiku.rag.config.models import AppConfig
@pytest.fixture(scope="module")
@ -14,103 +13,76 @@ def vcr_cassette_dir():
return str(Path(__file__).parent.parent.parent / "cassettes" / "test_sandbox")
def is_docker_available() -> bool:
"""Check if Docker daemon is available."""
try:
import subprocess
class TestSandboxBasics:
"""Test basic sandbox functionality."""
result = subprocess.run(["docker", "info"], capture_output=True, timeout=5)
return result.returncode == 0
except Exception:
return False
docker_required = pytest.mark.skipif(
not is_docker_available(),
reason="Docker daemon not available",
)
@pytest.mark.integration
class TestDockerSandboxBasics:
"""Test basic Docker sandbox functionality."""
@docker_required
@pytest.mark.asyncio
async def test_execute_simple_code(self, docker_sandbox):
async def test_execute_simple_code(self, sandbox):
"""Test executing simple code in the sandbox."""
result = await docker_sandbox.execute("print('hello world')")
result = await sandbox.execute("print('hello world')")
assert isinstance(result, SandboxResult)
assert result.success
assert "hello world" in result.stdout
assert result.stderr == ""
@pytest.mark.integration
class TestDockerSandboxErrors:
"""Test error handling in Docker sandbox."""
@docker_required
@pytest.mark.asyncio
async def test_syntax_error(self, docker_sandbox):
async def test_execute_expression_output(self, sandbox):
"""Test that expression values are captured."""
result = await sandbox.execute("1 + 2")
assert result.success
assert "3" in result.stdout
@pytest.mark.asyncio
async def test_execute_print_and_expression(self, sandbox):
"""Test print output combined with expression value."""
result = await sandbox.execute("print('hello')\n42")
assert result.success
assert "hello" in result.stdout
assert "42" in result.stdout
class TestSandboxErrors:
"""Test error handling in sandbox."""
@pytest.mark.asyncio
async def test_syntax_error(self, sandbox):
"""Test that syntax errors are reported."""
result = await docker_sandbox.execute("def foo(")
result = await sandbox.execute("def foo(")
assert not result.success
assert "SyntaxError" in result.stderr
assert result.stderr != ""
@docker_required
@pytest.mark.asyncio
async def test_runtime_error(self, docker_sandbox):
async def test_runtime_error(self, sandbox):
"""Test that runtime errors are reported."""
result = await docker_sandbox.execute("x = 1/0")
result = await sandbox.execute("x = 1/0")
assert not result.success
assert "ZeroDivisionError" in result.stderr
@docker_required
@pytest.mark.asyncio
async def test_name_error(self, docker_sandbox):
async def test_name_error(self, sandbox):
"""Test that name errors are reported."""
result = await docker_sandbox.execute("print(undefined_variable)")
result = await sandbox.execute("print(undefined_variable)")
assert not result.success
assert "NameError" in result.stderr
@docker_required
class TestSandboxHaikuRAG:
"""Test haiku.rag functions in sandbox."""
@pytest.mark.asyncio
async def test_missing_image(self, temp_db_path):
"""Test error when Docker image is not found."""
async with HaikuRAG(temp_db_path, create=True) as client:
config = RLMConfig(docker_image="nonexistent-image:v999.999.999")
context = RLMContext()
async with DockerSandbox(
client=client, config=config, context=context, image=config.docker_image
) as sandbox:
result = await sandbox.execute("print('hello')")
assert not result.success
assert (
"not found" in result.stderr.lower()
or "error" in result.stderr.lower()
)
@pytest.mark.integration
class TestDockerSandboxHaikuRAG:
"""Test haiku.rag functions in Docker sandbox."""
@docker_required
@pytest.mark.asyncio
async def test_list_documents_empty(self, docker_sandbox):
async def test_list_documents_empty(self, sandbox):
"""Test list_documents returns empty list for empty database."""
result = await docker_sandbox.execute(
result = await sandbox.execute(
"docs = list_documents()\nprint(type(docs).__name__, len(docs))"
)
assert result.success
assert "list 0" in result.stdout
@docker_required
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_list_documents_with_data(self, temp_db_path, test_docker_image):
async def test_list_documents_with_data(self, temp_db_path):
"""Test list_documents returns documents when populated."""
config = AppConfig()
async with HaikuRAG(temp_db_path, create=True) as client:
await client.create_document(
content="Test content",
@ -118,27 +90,20 @@ class TestDockerSandboxHaikuRAG:
title="Test Document",
)
config = RLMConfig(docker_image=test_docker_image)
context = RLMContext()
async with DockerSandbox(
client=client, config=config, context=context, image=test_docker_image
) as sandbox:
result = await sandbox.execute(
async with Sandbox(client=client, config=config, context=context) as sb:
result = await sb.execute(
"docs = list_documents()\nprint(len(docs))\nprint(docs[0]['title'])"
)
assert result.success
assert "1" in result.stdout
assert "Test Document" in result.stdout
@docker_required
@pytest.mark.asyncio
@pytest.mark.vcr()
@pytest.mark.skipif(
os.environ.get("CI") == "true",
reason="Requires Ollama - VCR can't capture calls from inside Docker",
)
async def test_search_with_data(self, temp_db_path, test_docker_image):
async def test_search_with_data(self, temp_db_path):
"""Test search function works."""
config = AppConfig()
async with HaikuRAG(temp_db_path, create=True) as client:
await client.create_document(
content="The quick brown fox jumps over the lazy dog.",
@ -146,26 +111,22 @@ class TestDockerSandboxHaikuRAG:
title="Animals",
)
config = RLMConfig(docker_image=test_docker_image)
context = RLMContext()
async with DockerSandbox(
client=client, config=config, context=context, image=test_docker_image
) as sandbox:
result = await sandbox.execute(
async with Sandbox(client=client, config=config, context=context) as sb:
result = await sb.execute(
"results = search('fox', limit=5)\n"
"print(len(results))\n"
"if results:\n"
" print('fox' in results[0]['content'].lower())"
)
assert result.success
# Search should return at least one result
assert "True" in result.stdout or "1" in result.stdout
@docker_required
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_get_document(self, temp_db_path, test_docker_image):
async def test_get_document(self, temp_db_path):
"""Test get_document function."""
config = AppConfig()
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document(
content="Content about foxes and dogs.",
@ -173,40 +134,68 @@ class TestDockerSandboxHaikuRAG:
title="Fox Document",
)
config = RLMConfig(docker_image=test_docker_image)
context = RLMContext()
async with DockerSandbox(
client=client, config=config, context=context, image=test_docker_image
) as sandbox:
result = await sandbox.execute(
async with Sandbox(client=client, config=config, context=context) as sb:
result = await sb.execute(
f"content = get_document('{doc.id}')\n"
"print('foxes' in content.lower() if content else 'None')"
)
assert result.success
assert "True" in result.stdout
@docker_required
@pytest.mark.asyncio
async def test_get_document_not_found(self, docker_sandbox):
async def test_get_document_not_found(self, sandbox):
"""Test get_document returns None for missing document."""
result = await docker_sandbox.execute(
result = await sandbox.execute(
"content = get_document('nonexistent-id')\nprint(content is None)"
)
assert result.success
assert "True" in result.stdout
@pytest.mark.integration
class TestDockerSandboxContextFilter:
"""Test context filter is applied."""
@docker_required
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_filter_applied_to_list_documents(
self, temp_db_path, test_docker_image
):
async def test_get_chunk(self, temp_db_path):
"""Test get_chunk function returns chunk with metadata."""
config = AppConfig()
async with HaikuRAG(temp_db_path, create=True) as client:
await client.create_document(
content="Content about foxes and dogs.",
uri="test://doc",
title="Fox Document",
)
context = RLMContext()
async with Sandbox(client=client, config=config, context=context) as sb:
# First search to get a chunk_id
result = await sb.execute(
"results = search('foxes', limit=1)\n"
"chunk_id = results[0]['chunk_id']\n"
"chunk = get_chunk(chunk_id)\n"
"print(chunk['document_title'])\n"
"print('content' in chunk)"
)
assert result.success
assert "Fox Document" in result.stdout
assert "True" in result.stdout
@pytest.mark.asyncio
async def test_get_chunk_not_found(self, sandbox):
"""Test get_chunk returns None for missing chunk."""
result = await sandbox.execute(
"chunk = get_chunk('nonexistent-id')\nprint(chunk is None)"
)
assert result.success
assert "True" in result.stdout
class TestSandboxContextFilter:
"""Test context filter is applied."""
@pytest.mark.asyncio
@pytest.mark.vcr()
async def test_filter_applied_to_list_documents(self, temp_db_path):
"""Test that context filter is passed to list_documents."""
config = AppConfig()
async with HaikuRAG(temp_db_path, create=True) as client:
await client.create_document(
content="Public content",
@ -219,12 +208,9 @@ class TestDockerSandboxContextFilter:
title="Private Doc",
)
config = RLMConfig(docker_image=test_docker_image)
context = RLMContext(filter="uri LIKE 'public://%'")
async with DockerSandbox(
client=client, config=config, context=context, image=test_docker_image
) as sandbox:
result = await sandbox.execute(
async with Sandbox(client=client, config=config, context=context) as sb:
result = await sb.execute(
"docs = list_documents()\n"
"print(len(docs))\n"
"if docs:\n"
@ -236,16 +222,12 @@ class TestDockerSandboxContextFilter:
assert "Private Doc" not in result.stdout
@pytest.mark.integration
class TestDockerSandboxPreloadedDocuments:
class TestSandboxPreloadedDocuments:
"""Test pre-loaded documents context variable."""
@docker_required
@pytest.mark.asyncio
async def test_documents_variable_not_available_without_preload(
self, docker_sandbox
):
async def test_documents_variable_not_available_without_preload(self, sandbox):
"""documents variable is not available when context.documents is None."""
result = await docker_sandbox.execute("print(documents)")
result = await sandbox.execute("print(documents)")
assert not result.success
assert "NameError" in result.stderr

1548
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_aggregation.yaml
View file

File diff suppressed because one or more lines are too long

72
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_count_documents.yaml
View file

@ -128,7 +128,7 @@ interactions:
connection:
- keep-alive
content-length:
- '7774'
- '7719'
content-type:
- application/json
host:
@ -189,7 +189,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -300,11 +300,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -344,7 +343,7 @@ interactions:
response:
headers:
content-length:
- '523'
- '585'
content-type:
- application/json
parsed_body:
@ -353,24 +352,24 @@ interactions:
index: 0
message:
content: ''
reasoning: We need to list documents.
reasoning: We need to list documents via list_documents to count.
role: assistant
tool_calls:
- function:
arguments: '{"code":"# list documents\nimport json\nprint(list_documents())\n"}'
arguments: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))\nfor d in docs:\n print(d[''title''])"}'
name: execute_code
id: call_d8xhmimu
id: call_1he6vvcy
index: 0
type: function
created: 1770373335
id: chatcmpl-184
created: 1771336260
id: chatcmpl-619
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 42
prompt_tokens: 1747
total_tokens: 1789
completion_tokens: 63
prompt_tokens: 1734
total_tokens: 1797
status:
code: 200
message: OK
@ -383,7 +382,7 @@ interactions:
connection:
- keep-alive
content-length:
- '8588'
- '8283'
content-type:
- application/json
host:
@ -444,7 +443,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -548,23 +547,19 @@ interactions:
role: system
- content: How many documents are in the database?
role: user
- content: |-
<think>
We need to list documents.
</think>
- content: null
reasoning: We need to list documents via list_documents to count.
role: assistant
tool_calls:
- function:
arguments: '{"code":"# list documents\nimport json\nprint(list_documents())\n"}'
arguments: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))\nfor d in docs:\n print(d[''title''])"}'
name: execute_code
id: call_d8xhmimu
id: call_1he6vvcy
type: function
- content: '{"code":"# list documents\nimport json\nprint(list_documents())\n","stdout":"[{''id'': ''b73f8a17-4328-475c-84db-3d81ce52adce'',
''title'': ''Doc 1'', ''uri'': None, ''created_at'': ''2026-02-06 11:21:55.843558''}, {''id'': ''accb877b-f04e-4bf2-ba4c-2d90339fa875'',
''title'': ''Doc 2'', ''uri'': None, ''created_at'': ''2026-02-06 11:21:57.397026''}, {''id'': ''afdb966f-5e9d-4759-a08f-28eb5108c80f'',
''title'': ''Doc 3'', ''uri'': None, ''created_at'': ''2026-02-06 11:21:58.988378''}]\n","stderr":"","success":true}'
- content: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))\nfor d in docs:\n print(d[''title''])","stdout":"3\nDoc
1\nDoc 2\nDoc 3\n","stderr":"","success":true}'
role: tool
tool_call_id: call_d8xhmimu
tool_call_id: call_1he6vvcy
model: gpt-oss
reasoning_effort: low
stream: false
@ -572,11 +567,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -616,7 +610,7 @@ interactions:
response:
headers:
content-length:
- '523'
- '577'
content-type:
- application/json
parsed_body:
@ -624,19 +618,19 @@ interactions:
- finish_reason: stop
index: 0
message:
content: '{"answer":"There are 3 documents in the database.","program":"# List and count documents\nimport json\n\ndocs
= list_documents()\nprint(f\"Number of documents: {len(docs)}\")\n"}'
reasoning: Count is 3. Provide answer.
content: '{"answer":"There are 3 documents in the database. They are titled: Doc 1, Doc 2, and Doc 3.","program":"#
Count documents in the database\n\ndocs = list_documents(limit=1000)\nprint(f\"Number of documents: {len(docs)}\")\nfor
d in docs:\n print(f\"- {d[''title'']}\")"}'
role: assistant
created: 1770373336
id: chatcmpl-441
created: 1771336262
id: chatcmpl-256
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 68
prompt_tokens: 2019
total_tokens: 2087
completion_tokens: 91
prompt_tokens: 1864
total_tokens: 1955
status:
code: 200
message: OK

988
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_docling_document_structure.yaml
View file

File diff suppressed because one or more lines are too long

3759
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_search_and_extract.yaml
View file

File diff suppressed because one or more lines are too long

600
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_search_and_get_chunk.yaml Normal file
View file

File diff suppressed because one or more lines are too long

2132
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_semantic_analysis_with_llm.yaml
View file

File diff suppressed because one or more lines are too long

52
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_with_filter.yaml
View file

@ -128,7 +128,7 @@ interactions:
connection:
- keep-alive
content-length:
- '7768'
- '7713'
content-type:
- application/json
host:
@ -189,7 +189,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -300,11 +300,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -344,7 +343,7 @@ interactions:
response:
headers:
content-length:
- '517'
- '527'
content-type:
- application/json
parsed_body:
@ -353,24 +352,24 @@ interactions:
index: 0
message:
content: ''
reasoning: Need to get list_documents.
reasoning: Need to call list_documents to count.
role: assistant
tool_calls:
- function:
arguments: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))"}'
name: execute_code
id: call_ly3bn3y0
id: call_9nd3m1g0
index: 0
type: function
created: 1770373376
id: chatcmpl-196
created: 1771336314
id: chatcmpl-255
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 44
prompt_tokens: 1745
total_tokens: 1789
completion_tokens: 46
prompt_tokens: 1732
total_tokens: 1778
status:
code: 200
message: OK
@ -383,7 +382,7 @@ interactions:
connection:
- keep-alive
content-length:
- '8201'
- '8154'
content-type:
- application/json
host:
@ -444,7 +443,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -548,20 +547,18 @@ interactions:
role: system
- content: How many documents are available?
role: user
- content: |-
<think>
Need to get list_documents.
</think>
- content: null
reasoning: Need to call list_documents to count.
role: assistant
tool_calls:
- function:
arguments: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))"}'
name: execute_code
id: call_ly3bn3y0
id: call_9nd3m1g0
type: function
- content: '{"code":"docs = list_documents(limit=1000)\nprint(len(docs))","stdout":"1\n","stderr":"","success":true}'
role: tool
tool_call_id: call_ly3bn3y0
tool_call_id: call_9nd3m1g0
model: gpt-oss
reasoning_effort: low
stream: false
@ -569,11 +566,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -623,15 +619,15 @@ interactions:
message:
content: '{"answer":"There is 1 document available in the knowledge base.","program":"docs = list_documents(limit=1000)\nprint(len(docs))"}'
role: assistant
created: 1770373377
id: chatcmpl-195
created: 1771336315
id: chatcmpl-500
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 39
prompt_tokens: 1836
total_tokens: 1875
prompt_tokens: 1819
total_tokens: 1858
status:
code: 200
message: OK

452
tests/cassettes/test_rlm/TestClientRLMIntegration.test_rlm_with_preloaded_documents.yaml
View file

@ -88,7 +88,7 @@ interactions:
connection:
- keep-alive
content-length:
- '7839'
- '7784'
content-type:
- application/json
host:
@ -149,7 +149,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -260,11 +260,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -304,7 +303,7 @@ interactions:
response:
headers:
content-length:
- '622'
- '640'
content-type:
- application/json
parsed_body:
@ -313,25 +312,25 @@ interactions:
index: 0
message:
content: ''
reasoning: Need to inspect documents variable.
reasoning: We need to check if 'documents' variable exists.
role: assistant
tool_calls:
- function:
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''No
preloaded documents'')"}'
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')"}'
name: execute_code
id: call_dk6bzypo
id: call_j4wr5u97
index: 0
type: function
created: 1770373463
id: chatcmpl-869
created: 1771336411
id: chatcmpl-46
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 69
prompt_tokens: 1759
total_tokens: 1828
completion_tokens: 74
prompt_tokens: 1746
total_tokens: 1820
status:
code: 200
message: OK
@ -344,7 +343,7 @@ interactions:
connection:
- keep-alive
content-length:
- '8495'
- '8475'
content-type:
- application/json
host:
@ -405,7 +404,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -509,22 +508,20 @@ interactions:
role: system
- content: Using the pre-loaded documents variable, tell me when was the company founded and what is their mission?
role: user
- content: |-
<think>
Need to inspect documents variable.
</think>
- content: null
reasoning: We need to check if 'documents' variable exists.
role: assistant
tool_calls:
- function:
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''No
preloaded documents'')"}'
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')"}'
name: execute_code
id: call_dk6bzypo
id: call_j4wr5u97
type: function
- content: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''No
preloaded documents'')","stdout":"No preloaded documents\n","stderr":"","success":true}'
- content: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')","stdout":"","stderr":"NameError: name ''dir'' is not defined","success":false}'
role: tool
tool_call_id: call_dk6bzypo
tool_call_id: call_j4wr5u97
model: gpt-oss
reasoning_effort: low
stream: false
@ -532,11 +529,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -576,7 +572,7 @@ interactions:
response:
headers:
content-length:
- '661'
- '699'
content-type:
- application/json
parsed_body:
@ -585,25 +581,26 @@ interactions:
index: 0
message:
content: ''
reasoning: No preloaded docs. Need to search.
reasoning: It says dir not defined, maybe need to import builtins. Use __builtins__. Let's just try list(dir())?
But we can just check if variable exists via try/except.
role: assistant
tool_calls:
- function:
arguments: '{"code":"results = search(''company founded mission'', limit=10)\nfor r in results:\n print(''Title:'',
r[''document_title''], ''Score:'', r[''score''])\n print(r[''content''][:200])\n print(''---'')"}'
arguments: '{"code":"try:\n docs=documents\n print(''exists'', len(docs))\nexcept NameError:\n print(''not
exist'')"}'
name: execute_code
id: call_zf1l4x9e
id: call_bclp2lct
index: 0
type: function
created: 1770373465
id: chatcmpl-682
created: 1771336414
id: chatcmpl-351
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 86
prompt_tokens: 1903
total_tokens: 1989
completion_tokens: 91
prompt_tokens: 1894
total_tokens: 1985
status:
code: 200
message: OK
@ -616,7 +613,7 @@ interactions:
connection:
- keep-alive
content-length:
- '9434'
- '9146'
content-type:
- application/json
host:
@ -677,7 +674,7 @@ interactions:
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The REPL captures stdout - print intermediate results to see what you're working with.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
@ -781,40 +778,35 @@ interactions:
role: system
- content: Using the pre-loaded documents variable, tell me when was the company founded and what is their mission?
role: user
- content: |-
<think>
Need to inspect documents variable.
</think>
- content: null
reasoning: We need to check if 'documents' variable exists.
role: assistant
tool_calls:
- function:
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''No
preloaded documents'')"}'
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')"}'
name: execute_code
id: call_dk6bzypo
id: call_j4wr5u97
type: function
- content: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''No
preloaded documents'')","stdout":"No preloaded documents\n","stderr":"","success":true}'
- content: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')","stdout":"","stderr":"NameError: name ''dir'' is not defined","success":false}'
role: tool
tool_call_id: call_dk6bzypo
- content: |-
<think>
No preloaded docs. Need to search.
</think>
tool_call_id: call_j4wr5u97
- content: null
reasoning: It says dir not defined, maybe need to import builtins. Use __builtins__. Let's just try list(dir())? But
we can just check if variable exists via try/except.
role: assistant
tool_calls:
- function:
arguments: '{"code":"results = search(''company founded mission'', limit=10)\nfor r in results:\n print(''Title:'',
r[''document_title''], ''Score:'', r[''score''])\n print(r[''content''][:200])\n print(''---'')"}'
arguments: '{"code":"try:\n docs=documents\n print(''exists'', len(docs))\nexcept NameError:\n print(''not
exist'')"}'
name: execute_code
id: call_zf1l4x9e
id: call_bclp2lct
type: function
- content: '{"code":"results = search(''company founded mission'', limit=10)\nfor r in results:\n print(''Title:'',
r[''document_title''], ''Score:'', r[''score''])\n print(r[''content''][:200])\n print(''---'')","stdout":"Title:
Mission Statement Score: 0.032522473484277725\nOur mission is to make technology accessible to everyone.\n---\nTitle:
Company History Score: 0.032522473484277725\nThe company was founded in 1985 by Jane Smith.\n---\n","stderr":"","success":true}'
- content: '{"code":"try:\n docs=documents\n print(''exists'', len(docs))\nexcept NameError:\n print(''not
exist'')","stdout":"exists 2\n","stderr":"","success":true}'
role: tool
tool_call_id: call_zf1l4x9e
tool_call_id: call_bclp2lct
model: gpt-oss
reasoning_effort: low
stream: false
@ -822,11 +814,10 @@ interactions:
tools:
- function:
description: |-
<summary>Execute Python code in a Docker-sandboxed environment.
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_docling_document, llm) and any Python standard
library module.
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
@ -866,7 +857,306 @@ interactions:
response:
headers:
content-length:
- '1256'
- '607'
content-type:
- application/json
parsed_body:
choices:
- finish_reason: tool_calls
index: 0
message:
content: ''
reasoning: So two docs loaded. Let's view titles.
role: assistant
tool_calls:
- function:
arguments: '{"code":"for doc in documents:\n print(''Title:'', doc[''title''])\n print(''Content snippet:'',
doc[''content''][:200])\n print(''-''*40)"}'
name: execute_code
id: call_jzdp8w33
index: 0
type: function
created: 1771336416
id: chatcmpl-830
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 70
prompt_tokens: 2043
total_tokens: 2113
status:
code: 200
message: OK
- request:
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-length:
- '10023'
content-type:
- application/json
host:
- localhost:11434
method: POST
parsed_body:
messages:
- content: |-
You are a Recursive Language Model (RLM) agent that solves complex research questions by writing and executing Python code.
IMPORTANT: You MUST use the `execute_code` tool to run Python code. The functions described below are ONLY available inside the execute_code tool - you cannot access them any other way. Always execute code to answer questions; do not just describe what code would do.
CRITICAL: Inside execute_code, these functions are ALREADY available in the namespace. Do NOT import them - just use them directly:
- search("query") ✓ CORRECT
- from haiku.rag import search ✗ WRONG - will fail
You have access to a sandboxed Python environment with these haiku.rag functions (use them directly, no imports needed):
## Available Functions
### search(query, limit=10) -> list[dict]
Search the knowledge base using hybrid search (vector + full-text).
Returns list of dicts with keys: chunk_id, content, document_id, document_title, document_uri, score, page_numbers, headings
### list_documents(limit=10, offset=0) -> list[dict]
List available documents in the knowledge base.
Returns list of dicts with keys: id, title, uri, created_at
### get_document(id_or_title) -> str | None
Get the full text content of a document by ID, title, or URI.
Returns the document content as a string, or None if not found.
### get_docling_document(id_or_title) -> DoclingDocument | None
Get the structured DoclingDocument object for advanced analysis.
Returns a DoclingDocument object, or None if not found.
See "DoclingDocument API" section below for how to use it.
### llm(prompt) -> str
Call an LLM directly with the given prompt. Returns the response as a string.
Use this for classification, summarization, extraction, or any task where you
already have the content and just need LLM reasoning.
## Pre-loaded Documents Variable
If documents were pre-loaded for this session, a `documents` variable is available:
```python
# documents is a list of dicts with keys: id, title, uri, content
for doc in documents:
print(doc['title'], len(doc['content']))
```
Check if it exists with: `if 'documents' in dir(): ...`
## Standard Library Modules
You can import any Python standard library module.
## Strategy Guide
1. **Explore First**: Start by listing documents or searching to understand what's available. Document names may differ from filenames (e.g., "tbmed593.pdf" might be stored as "TB MED 593" or similar).
2. **If get_document returns None**: Use `list_documents()` to see actual document titles, or `search()` to find relevant content.
3. **Iterative Refinement**: Run code, examine results, adjust your approach based on what you find.
4. **Use print() Liberally**: The sandbox captures stdout - print intermediate results to see what you're working with.
5. **Aggregate with Code**: For counting, averaging, or comparing across documents, write loops and use collections.
6. **Use llm() for Classification/Extraction**: When you need to classify, summarize, or extract structured data from content you already have, use llm().
7. **Cite Your Sources**: Track which documents/chunks informed your answer for citation.
## DoclingDocument API
When you call `get_docling_document(id_or_title)`, you get a DoclingDocument object for structured document analysis.
### Properties
- `doc.texts` - List of all text items (paragraphs, headings, etc.)
- `doc.tables` - List of all tables
- `doc.pictures` - List of all pictures/figures
- `doc.name` - Document name
### Methods
- `doc.iterate_items(with_groups=False)` - Iterate all items with hierarchy level
Returns tuples of (item, level) where level is nesting depth
- `doc.export_to_markdown()` - Export entire document as markdown string
### Text Item Properties
- `item.text` - The text content
- `item.label` - Type: title, paragraph, section_header, list_item, etc. (lowercase enum values)
- `item.prov` - Provenance (page numbers, bounding boxes)
### Table Access
- `table.data.num_rows`, `table.data.num_cols` - Dimensions
- `table.data.table_cells` - List of TableCell objects
- `cell.text`, `cell.start_row_offset_idx`, `cell.start_col_offset_idx`
### Example Usage
```python
doc = get_docling_document("My Document")
# Get all headings
headings = [t.text for t in doc.texts if "header" in str(t.label)]
# Iterate with structure
for item, level in doc.iterate_items():
print(" " * level + item.text[:50])
# Extract table data
for table in doc.tables:
for cell in table.data.table_cells:
print(f"Row {cell.start_row_offset_idx}, Col {cell.start_col_offset_idx}: {cell.text}")
```
## Example Patterns
### Counting documents matching a condition
```python
docs = list_documents(limit=100)
count = 0
for doc in docs:
content = get_document(doc['id'])
if content and 'keyword' in content.lower():
count += 1
print(f"Found in: {doc['title']}")
print(f"Total: {count}")
```
### Aggregating data across documents
```python
import re
numbers = []
results = search("financial data", limit=20)
for r in results:
matches = re.findall(r'\$([\d,]+)', r['content'])
for m in matches:
numbers.append(int(m.replace(',', '')))
print(f"Average: ${sum(numbers)/len(numbers):,.2f}")
```
### Using llm() for classification
```python
# Get document content
content = get_document("Q1 Report")
# Use llm() to classify sentiment
sentiment = llm(f"Classify the sentiment as positive, negative, or mixed: {content}")
print(sentiment)
```
## Workflow
1. **ALWAYS start by using execute_code** to explore the knowledge base
2. Run multiple code blocks as needed to gather information
3. After collecting data, provide your final answer
## Output Format
CRITICAL: Your final response MUST be valid JSON matching this exact schema:
```json
{"answer": "Your complete answer here as a string", "program": "Your final consolidated program here as a string"}
```
- `answer`: A clear answer to the user's question with key findings and references to specific documents/chunks.
- `program`: A single, self-contained Python program that produces the answer. Consolidate your exploratory code executions into one clean script.
Do NOT return arbitrary JSON structures. Always use the exact format: {"answer": "...", "program": "..."}
CRITICAL: You MUST call execute_code at least once before providing your answer. Never give up without trying to execute code first.
role: system
- content: Using the pre-loaded documents variable, tell me when was the company founded and what is their mission?
role: user
- content: null
reasoning: We need to check if 'documents' variable exists.
role: assistant
tool_calls:
- function:
arguments: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')"}'
name: execute_code
id: call_j4wr5u97
type: function
- content: '{"code":"if ''documents'' in dir():\n for doc in documents:\n print(doc[''title''], len(doc[''content'']))\nelse:\n print(''documents
variable not found'')","stdout":"","stderr":"NameError: name ''dir'' is not defined","success":false}'
role: tool
tool_call_id: call_j4wr5u97
- content: null
reasoning: It says dir not defined, maybe need to import builtins. Use __builtins__. Let's just try list(dir())? But
we can just check if variable exists via try/except.
role: assistant
tool_calls:
- function:
arguments: '{"code":"try:\n docs=documents\n print(''exists'', len(docs))\nexcept NameError:\n print(''not
exist'')"}'
name: execute_code
id: call_bclp2lct
type: function
- content: '{"code":"try:\n docs=documents\n print(''exists'', len(docs))\nexcept NameError:\n print(''not
exist'')","stdout":"exists 2\n","stderr":"","success":true}'
role: tool
tool_call_id: call_bclp2lct
- content: null
reasoning: So two docs loaded. Let's view titles.
role: assistant
tool_calls:
- function:
arguments: '{"code":"for doc in documents:\n print(''Title:'', doc[''title''])\n print(''Content snippet:'',
doc[''content''][:200])\n print(''-''*40)"}'
name: execute_code
id: call_jzdp8w33
type: function
- content: '{"code":"for doc in documents:\n print(''Title:'', doc[''title''])\n print(''Content snippet:'', doc[''content''][:200])\n print(''-''*40)","stdout":"Title:
Company History\nContent snippet: The company was founded in 1985 by Jane Smith.\n----------------------------------------\nTitle:
Mission Statement\nContent snippet: Our mission is to make technology accessible to everyone.\n----------------------------------------\n","stderr":"","success":true}'
role: tool
tool_call_id: call_jzdp8w33
model: gpt-oss
reasoning_effort: low
stream: false
tool_choice: auto
tools:
- function:
description: |-
<summary>Execute Python code in a sandboxed interpreter.
The code has access to haiku.rag functions (search, list_documents,
get_document, get_chunk, llm).
Use print() to output results.</summary>
<returns>
<description>Structured result with success status, stdout, and stderr.</description>
</returns>
name: execute_code
parameters:
additionalProperties: false
properties:
code:
description: Python code to execute.
type: string
required:
- code
type: object
strict: true
type: function
- function:
description: Result from RLM agent execution.
name: final_result
parameters:
additionalProperties: false
properties:
answer:
description: The answer to the user's question
type: string
program:
description: The final consolidated program
type: string
required:
- answer
- program
type: object
strict: true
type: function
uri: http://localhost:11434/v1/chat/completions
response:
headers:
content-length:
- '1186'
content-type:
- application/json
parsed_body:
@ -874,24 +1164,24 @@ interactions:
- finish_reason: stop
index: 0
message:
content: '{"answer":"The company was founded in 1985. Its mission is to make technology accessible to everyone.","program":"#
Get relevant documents via search\nresults = search(''company founded mission'', limit=10)\n# Find founded year
and mission statement\nfounded_year = None\nmission = None\nfor r in results:\n text = r[''content'']\n if
''founded'' in text.lower() and founded_year is None:\n import re\n m = re.search(r''\\b(19\\d{2}|20\\d{2})\\b'',
text)\n if m:\n founded_year = m.group(1)\n if ''mission'' in text.lower() and mission is
None:\n # simple extraction of the sentence containing \"mission\"\n sentences = re.split(r''(?<=[.!?])\\s+'',
text)\n for s in sentences:\n if ''mission'' in s.lower():\n mission = s.strip()\n break\n#
Output\nprint(f\"Founded year: {founded_year}\")\nprint(f\"Mission: {mission}\")"}'
content: '{"answer":"The company was founded in **1985**. Its mission is to **make technology accessible to everyone**.","program":"#
Consolidated script\n# Assume pre-loaded variable ''documents''\n\n# Find founding year and mission statements\nfounded_year
= None\nmission = None\nfor doc in documents:\n title = doc[''title''].lower()\n content = doc[''content'']\n if
''history'' in title:\n # Search for a year pattern\n import re\n match = re.search(r''\\b(19\\d{2})\\b'',
content)\n if match:\n founded_year = int(match.group(1))\n if ''mission'' in title:\n mission
= content.strip()\n\n# Output results\nprint(f\"Founded year: {founded_year}\")\nprint(f\"Mission: {mission}\")"}'
reasoning: 'So answers: founded 1985, mission to make technology accessible to everyone. Provide answer. And program
consolidating.'
role: assistant
created: 1770373471
id: chatcmpl-972
created: 1771336422
id: chatcmpl-444
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 246
prompt_tokens: 2135
total_tokens: 2381
completion_tokens: 236
prompt_tokens: 2225
total_tokens: 2461
status:
code: 200
message: OK

0
tests/cassettes/test_sandbox/TestDockerSandboxContextFilter.test_filter_applied_to_list_documents.yaml → tests/cassettes/test_sandbox/TestSandboxContextFilter.test_filter_applied_to_list_documents.yaml
View file

82
tests/cassettes/test_sandbox/TestSandboxHaikuRAG.test_get_chunk.yaml Normal file
View file

File diff suppressed because one or more lines are too long

0
tests/cassettes/test_sandbox/TestDockerSandboxHaikuRAG.test_get_document.yaml → tests/cassettes/test_sandbox/TestSandboxHaikuRAG.test_get_document.yaml
View file

0
tests/cassettes/test_sandbox/TestDockerSandboxHaikuRAG.test_list_documents_with_data.yaml → tests/cassettes/test_sandbox/TestSandboxHaikuRAG.test_list_documents_with_data.yaml
View file

40
tests/cassettes/test_sandbox/TestDockerSandboxHaikuRAG.test_search_with_data.yaml → tests/cassettes/test_sandbox/TestSandboxHaikuRAG.test_search_with_data.yaml
View file

File diff suppressed because one or more lines are too long

46
uv.lock
View file

@ -1456,6 +1456,7 @@ dependencies = [
{ name = "pathspec" },
{ name = "pydantic" },
{ name = "pydantic-ai-slim", extra = ["ag-ui", "fastmcp", "logfire", "openai"] },
{ name = "pydantic-monty" },
{ name = "python-dotenv" },
{ name = "pyyaml" },
{ name = "rich" },
@ -1529,6 +1530,7 @@ requires-dist = [
{ name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire", "ag-ui"], specifier = ">=1.46.0" },
{ name = "pydantic-ai-slim", extras = ["vertexai"], marker = "extra == 'vertexai'" },
{ name = "pydantic-ai-slim", extras = ["voyageai"], marker = "extra == 'voyageai'" },
{ name = "pydantic-monty", specifier = ">=0.0.6" },
{ name = "python-dotenv", specifier = ">=1.2.1" },
{ name = "pyyaml", specifier = ">=6.0.3" },
{ name = "rich", specifier = ">=14.2.0" },
@ -3787,6 +3789,50 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d8/6c/ce1c0eca77c6efbf7c7168c05a244c2756bde876a52575f750471d522024/pydantic_graph-1.60.0-py3-none-any.whl", hash = "sha256:741fa1e48424b0def86079a01100ad0652e75882f0352cd157232b75ace468a5", size = 72345, upload-time = "2026-02-17T00:33:25.077Z" },
]
[[package]]
name = "pydantic-monty"
version = "0.0.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/6f/22/8925f8016ba33968cb95f9647eba90d821f94ce8c36ef28ecfeed80f586a/pydantic_monty-0.0.6.tar.gz", hash = "sha256:338cc3264b15fb541c631790e7cbe63e556d901b296362797f0b92d2576300c8", size = 667966, upload-time = "2026-02-16T11:58:05.052Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cc/80/5581932d49b501336fa7280df2d682412207e1cbcb4ef2a93a6416c96949/pydantic_monty-0.0.6-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3e3d6a9c22f96e2679c72588fd04b2d9a7759016b665379f8026bba18836096f", size = 6342276, upload-time = "2026-02-16T11:58:54.637Z" },
{ url = "https://files.pythonhosted.org/packages/c6/4f/2f1262c6ebf8cfe36d8eabd670774bd1d41baca2486f3ac299755186cd5e/pydantic_monty-0.0.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cf72be2893b6f1dbbfafdb04aa91efc79ce8e48f6fea771b2548e5ece5b87eb0", size = 6197620, upload-time = "2026-02-16T11:59:18.327Z" },
{ url = "https://files.pythonhosted.org/packages/dc/bc/de41d61c3921af9f2ab77cb007a5c95f9004e6845ee0d5f9c519451c635e/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16854c0650cf576a478bd17bf1633b8cf84c6bde9e4eb6405ed204f332c15ee2", size = 6136071, upload-time = "2026-02-16T11:58:12.347Z" },
{ url = "https://files.pythonhosted.org/packages/de/2c/e719ecc8f90e3a58fa29ed9b0d3f0682b65b349e1e72f0abacb2c0a93513/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e19a90847b9e35129049803d739544759986265e27a5999bc156505da1934c7", size = 6397946, upload-time = "2026-02-16T11:59:11.703Z" },
{ url = "https://files.pythonhosted.org/packages/41/65/6983f9d9e066e7c5e7cb65197399ca03b87d5f922cd986294af527939331/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b641e165b2b5724c2e609f09ff28bed3a9149d61393b6877c8e9e914273c5d9d", size = 6918298, upload-time = "2026-02-16T11:58:52.747Z" },
{ url = "https://files.pythonhosted.org/packages/78/fc/597f26a4b8498f142dd989526d11cc5343020e1a25e35709db43f57402ae/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77e41c75b042cae1f91a66294f6aa8518a873006cee6d088248bbbab09dbfc5f", size = 6969576, upload-time = "2026-02-16T11:58:50.872Z" },
{ url = "https://files.pythonhosted.org/packages/4d/1e/bc76a10215c28fcd5277318cd642047d41f7df0b7dbd5ae165c8eb7c9143/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:710d78c9e08369ee7f1705a2295363926425535ceb4c87c43a4e38cf421c648f", size = 6681558, upload-time = "2026-02-16T11:58:43.507Z" },
{ url = "https://files.pythonhosted.org/packages/24/dc/42e12b097f5fbfc10bfcae60891664b87deb35d947f41e742001fcd12dd9/pydantic_monty-0.0.6-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5f021d2479546eddf46c2eb3f753b73f8e3a9828ac3f4e2207768dc9a500c74b", size = 6782625, upload-time = "2026-02-16T11:58:06.576Z" },
{ url = "https://files.pythonhosted.org/packages/94/70/b6e64572744261f0cce6d8c1606edcb44958e92573135f551a10ec5a0cd7/pydantic_monty-0.0.6-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:df4ee9a2bf132bb773dc905bd6ab6a16e13a89468e883b65014cdc25dcf1c18f", size = 6307895, upload-time = "2026-02-16T11:58:45.28Z" },
{ url = "https://files.pythonhosted.org/packages/a7/3c/4debe5cc7b7bf167e2a57e0da30d89d65f50955e75c9ea61a7a63e9dbb44/pydantic_monty-0.0.6-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4c019e678aef18af3754422c1a202861e166e0c7a3acc7af08b50b03ddac28ac", size = 6747057, upload-time = "2026-02-16T11:57:44.413Z" },
{ url = "https://files.pythonhosted.org/packages/87/60/24eda6b192975be42907f88a7f9bc9d29f9ff0098dfe7fe14dd78056d007/pydantic_monty-0.0.6-cp312-cp312-win32.whl", hash = "sha256:ea14f1eb10927edde6bfa29596260d581b2718379d1a8a6062db47217fe302e0", size = 6225199, upload-time = "2026-02-16T11:58:08.789Z" },
{ url = "https://files.pythonhosted.org/packages/4c/e2/2ebf6f3d189373bbe7227c13d232fdd3c8ad6678f8ce5d88002e285d092a/pydantic_monty-0.0.6-cp312-cp312-win_amd64.whl", hash = "sha256:4f08efbdaba3c89d74a501ef5fa53272a707d26d38cfb0d84625be2f8c0eae82", size = 6737909, upload-time = "2026-02-16T11:57:49.151Z" },
{ url = "https://files.pythonhosted.org/packages/e0/08/27c05ea6e213eef670cb7eca75a8aca18b28cc0ed63e8bf804beca2c7ab6/pydantic_monty-0.0.6-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3d0744aacbaf75abc3c3450225d3f08bc4a9f51a700be3d16fd7e1885efd53dd", size = 6341795, upload-time = "2026-02-16T11:58:56.401Z" },
{ url = "https://files.pythonhosted.org/packages/e3/c6/c3ac0e3d5e8440edcc17f166e0889f0921aba83307ab769e8caf8d0cbe2a/pydantic_monty-0.0.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c3f0412b8ab77fbea3eba6d91c60485be5a9fd2f411c6817c0af0da79c4391c5", size = 6197593, upload-time = "2026-02-16T11:57:40.613Z" },
{ url = "https://files.pythonhosted.org/packages/f7/d6/495bfb141305d3669e9e25e767631f19acca281138b102216fa3adb255bc/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3f66a3d405f6c8da8675e723f169dd6509e20f48d4ff1e6b037310b88a10b1c", size = 6135697, upload-time = "2026-02-16T11:58:58.22Z" },
{ url = "https://files.pythonhosted.org/packages/33/e5/905dde95087190d7151aa00448f6735fa689401758669c64f5be36fb3c24/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a8d95eb1416562160824ebb36c437313656ec85b6802925445d131317c013ee", size = 6397248, upload-time = "2026-02-16T11:58:19.073Z" },
{ url = "https://files.pythonhosted.org/packages/94/20/acf43f7c3d4af1c4d095e03fa40a135d28d7e12f4605742ba50f8b274cfd/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9b9a568f866c89358b7c93fad2c96d38a9e591c6aa57bfee4ba0984c7bc8aa1c", size = 6918359, upload-time = "2026-02-16T11:58:14.913Z" },
{ url = "https://files.pythonhosted.org/packages/15/1f/1c547cb7d3c609ced4e497c68b246adbe239e9b34c8f791d2f96f8d17930/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28254e0f339a320f5a34c2933f03acacb33ef150f147a77f5c4578af397e3f09", size = 6968524, upload-time = "2026-02-16T11:57:15.583Z" },
{ url = "https://files.pythonhosted.org/packages/fb/b1/99f7df5edc200ace9ac46faedd0253a5db3c31a91c0c633c305c5d6d32e2/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1963ab53a306f8af9b412ae51e4e54fe807fa68b6129060415c422eb8ea1f67", size = 6681553, upload-time = "2026-02-16T11:57:52.804Z" },
{ url = "https://files.pythonhosted.org/packages/31/e2/008c63a4b10358fe663138c3b97c624f75fa89e71072452a7fbc365a5cf3/pydantic_monty-0.0.6-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d3ca80d150293f32d9abc7aac189cf2604832090be2197dcbb097b46c5faaad5", size = 6782531, upload-time = "2026-02-16T11:57:46.796Z" },
{ url = "https://files.pythonhosted.org/packages/8b/2a/365ffe13eeafadc9e3687c703ff6b86364e83a29c6e1a653d3d8d597a17d/pydantic_monty-0.0.6-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41345c5f3b1e050adbf5a803717349f4084085d6ddb2aaad49bbb7fea245fee4", size = 6307831, upload-time = "2026-02-16T11:58:24.978Z" },
{ url = "https://files.pythonhosted.org/packages/cd/e3/16ddd81f8ae74da35b1f9ac763f914a413ad5f7a737ab94d0476c99ffeb9/pydantic_monty-0.0.6-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:30052d593c1affb4706e5b70f415a291d4158c4e45bbf2e51311f35173c0d56b", size = 6746734, upload-time = "2026-02-16T11:58:03.208Z" },
{ url = "https://files.pythonhosted.org/packages/75/a9/beb4587a68e799e86a2ea993a3582650e2b1ef246fb2651d719020491e0d/pydantic_monty-0.0.6-cp313-cp313-win32.whl", hash = "sha256:408cd43b4b152c5129fbab5267993649456f7a9001281fa4fcdc05a25d94ee73", size = 6224481, upload-time = "2026-02-16T11:59:13.424Z" },
{ url = "https://files.pythonhosted.org/packages/81/c5/e767ea0f79da8cd98d03d0bae137bf7426a61e9f9f7b3f378551e73fa972/pydantic_monty-0.0.6-cp313-cp313-win_amd64.whl", hash = "sha256:5febecdc86675105c5bfadfdcb00a3773be948c41d186f2892ab848c69eabf89", size = 6737388, upload-time = "2026-02-16T11:58:48.884Z" },
{ url = "https://files.pythonhosted.org/packages/85/be/9419c06ac88a15cd4002c7001419687ffc1824ae46e83abe124e1f9af2d1/pydantic_monty-0.0.6-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:be4e754c0d81339c29c7c0484416e9251e7dc8375ca1c98fe78d34b1918280e9", size = 6343417, upload-time = "2026-02-16T11:57:59.432Z" },
{ url = "https://files.pythonhosted.org/packages/d2/8e/d6875e7f9fa7ae09aa8e352fb67993a6671b05ab5ceeed889f3174249fed/pydantic_monty-0.0.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:36bf33caa1b59b56bdf82235f050a6c7ac2fdce0039c6e284a5d9ea75b2d7bd5", size = 6218595, upload-time = "2026-02-16T11:58:16.734Z" },
{ url = "https://files.pythonhosted.org/packages/10/b6/12d294a0113ac61fec979816a09d49516f4dfe49e0a09b7c1a04ed54835b/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:563586485bf024c824353d3394c64068a71791bb2599251180e4065e1f349dec", size = 6136987, upload-time = "2026-02-16T11:57:25.002Z" },
{ url = "https://files.pythonhosted.org/packages/0b/cc/3e328e2ca8178d9db37dbd8d2fd9a696bafdb9899ee69be6559ba43eb1ad/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:65bf08f0934cc2d1200ecfbf11d4c4cbfa8f3cd2b3a8af5294ac4392bb2764b8", size = 6398186, upload-time = "2026-02-16T11:59:04.898Z" },
{ url = "https://files.pythonhosted.org/packages/ff/04/aab363a93472397f76e62b91c3822d3fbd8c26cf02bbc564aa27f259aecc/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:221dfef1555272ce1a3a0f0fceafd5361830f404921dc6eb01c49c78656815b5", size = 6919986, upload-time = "2026-02-16T11:59:07.249Z" },
{ url = "https://files.pythonhosted.org/packages/45/8c/b1144a0cc8e5966e8d0f80bfa4acc7a924b909524c6f555d085b90a6b1c7/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e1e688dc4a9a6899c993c7fc432221d74c5b2d4a02f7d31d795b42a0d57400d", size = 6969856, upload-time = "2026-02-16T11:58:20.973Z" },
{ url = "https://files.pythonhosted.org/packages/8c/c6/1e21d850001bf1a34ef3bfd15244debf838ce48940aa2e5c068a40a59c82/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1af60236cf88949a097172bb271afddc41854a91f62541b0eef8ce9d985388c9", size = 6707086, upload-time = "2026-02-16T11:58:01.323Z" },
{ url = "https://files.pythonhosted.org/packages/0c/35/3f13c88a2cee69d7181d08b713bd45bc0bb942315c49a94f930f41dd7755/pydantic_monty-0.0.6-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f35a5f26ae76ddf4766021d481f34a856e75585668f9de2e57725778c0b94fe6", size = 6783791, upload-time = "2026-02-16T11:59:00.153Z" },
{ url = "https://files.pythonhosted.org/packages/24/84/eba4a2ae6b74f8aba52a84d046a2b412b285a8f7c35ebfe4a186d3fe6950/pydantic_monty-0.0.6-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:15a4fa6bc8a032e75d1ca3b02be0f8f8a9006af4c52c05becf3496fa9edf5fc3", size = 6309095, upload-time = "2026-02-16T11:57:26.685Z" },
{ url = "https://files.pythonhosted.org/packages/c6/77/2ea62b3c5c5977e9da358744d90d2c455832b3c35deac2290814da13298e/pydantic_monty-0.0.6-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e4868421b7a239148076da06cf33590cf7d1c5f165903a74da1fb7fa3466068f", size = 6748092, upload-time = "2026-02-16T11:57:42.693Z" },
{ url = "https://files.pythonhosted.org/packages/5b/47/7d29f6a31a603300e39db79e7f70c4c52d819d387a0f1cef07368ae1fa6a/pydantic_monty-0.0.6-cp314-cp314-win32.whl", hash = "sha256:c7366f79c95b22440d6f7e4c5f78503bda432a4a69704bac2a26e923ac511986", size = 6227205, upload-time = "2026-02-16T11:58:36.05Z" },
{ url = "https://files.pythonhosted.org/packages/56/6c/3544b35f2d72d1415f243ec1641551e2f128f5bd5f63ff8a7f75c58b41cd/pydantic_monty-0.0.6-cp314-cp314-win_amd64.whl", hash = "sha256:342e580e39cbb23b32572a2f1e0fc725ca893e40ba2d45cee18da0723c5e759a", size = 6763621, upload-time = "2026-02-16T11:57:28.472Z" },
]
[[package]]
name = "pydantic-settings"
version = "2.13.0"