S3 storage tests
This commit is contained in:
parent
8e876ef28d
commit
98641c24e4
3 changed files with 185 additions and 3 deletions
|
|
@ -62,8 +62,9 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
# Create the database
|
# Create the database
|
||||||
client = HaikuRAG(db_path=self.db_path, config=self.config, create=True)
|
client = HaikuRAG(db_path=self.db_path, config=self.config, create=True)
|
||||||
client.close()
|
client.close()
|
||||||
|
display_path = self.config.lancedb.uri if not self._is_local else self.db_path
|
||||||
self.console.print(
|
self.console.print(
|
||||||
f"[bold green]Database initialized at {self.db_path}[/bold green]"
|
f"[bold green]Database initialized at {display_path}[/bold green]"
|
||||||
)
|
)
|
||||||
|
|
||||||
async def info(self):
|
async def info(self):
|
||||||
|
|
@ -71,10 +72,11 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
|
|
||||||
from haiku.rag.store.engine import Store, connect_lancedb
|
from haiku.rag.store.engine import Store, connect_lancedb
|
||||||
|
|
||||||
# Basic: show path
|
# Basic: show path/URI
|
||||||
self.console.print("[bold]haiku.rag database info[/bold]")
|
self.console.print("[bold]haiku.rag database info[/bold]")
|
||||||
|
display_path = self.config.lancedb.uri if not self._is_local else self.db_path
|
||||||
self.console.print(
|
self.console.print(
|
||||||
f" [repr.attrib_name]path[/repr.attrib_name]: {self.db_path}"
|
f" [repr.attrib_name]path[/repr.attrib_name]: {display_path}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if self._is_local and not self.db_path.exists():
|
if self._is_local and not self.db_path.exists():
|
||||||
|
|
@ -84,6 +86,12 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
# Connect without going through Store to avoid upgrades/validation writes
|
# Connect without going through Store to avoid upgrades/validation writes
|
||||||
db = connect_lancedb(self.config, self.db_path)
|
db = connect_lancedb(self.config, self.db_path)
|
||||||
|
|
||||||
|
if not db.list_tables().tables:
|
||||||
|
self.console.print(
|
||||||
|
"[red]Database is empty. Use 'haiku-rag init' to initialize.[/red]"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
versions = get_package_versions()
|
versions = get_package_versions()
|
||||||
|
|
||||||
store = Store(
|
store = Store(
|
||||||
|
|
|
||||||
22
tests/docker/docker-compose.minio.yml
Normal file
22
tests/docker/docker-compose.minio.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
services:
|
||||||
|
minio:
|
||||||
|
image: minio/minio
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER: minioadmin
|
||||||
|
MINIO_ROOT_PASSWORD: minioadmin
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
|
||||||
|
createbucket:
|
||||||
|
image: minio/mc
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
entrypoint: >
|
||||||
|
/bin/sh -c "
|
||||||
|
sleep 2;
|
||||||
|
mc alias set myminio http://minio:9000 minioadmin minioadmin;
|
||||||
|
mc mb myminio/test-bucket;
|
||||||
|
exit 0;
|
||||||
|
"
|
||||||
152
tests/test_s3_integration.py
Normal file
152
tests/test_s3_integration.py
Normal file
|
|
@ -0,0 +1,152 @@
|
||||||
|
# Start MinIO before running:
|
||||||
|
# docker compose -f tests/docker/docker-compose.minio.yml up -d
|
||||||
|
# Stop after:
|
||||||
|
# docker compose -f tests/docker/docker-compose.minio.yml down -v
|
||||||
|
|
||||||
|
import socket
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from haiku.rag.app import HaikuRAGApp
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
from haiku.rag.config.models import AppConfig, LanceDBConfig
|
||||||
|
from haiku.rag.store.engine import Store
|
||||||
|
|
||||||
|
MINIO_ENDPOINT = "http://localhost:9000"
|
||||||
|
MINIO_BUCKET = "test-bucket"
|
||||||
|
MINIO_STORAGE_OPTIONS = {
|
||||||
|
"aws_access_key_id": "minioadmin",
|
||||||
|
"aws_secret_access_key": "minioadmin",
|
||||||
|
"endpoint": MINIO_ENDPOINT,
|
||||||
|
"region": "us-east-1",
|
||||||
|
"allow_http": "true",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _minio_available() -> bool:
|
||||||
|
try:
|
||||||
|
s = socket.create_connection(("localhost", 9000), timeout=1)
|
||||||
|
s.close()
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
pytestmark = [
|
||||||
|
pytest.mark.integration,
|
||||||
|
pytest.mark.skipif(
|
||||||
|
not _minio_available(), reason="MinIO not running on localhost:9000"
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _make_config() -> AppConfig:
|
||||||
|
unique_prefix = uuid4().hex[:8]
|
||||||
|
return AppConfig(
|
||||||
|
lancedb=LanceDBConfig(
|
||||||
|
uri=f"s3://{MINIO_BUCKET}/test-{unique_prefix}",
|
||||||
|
storage_options=MINIO_STORAGE_OPTIONS,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_connect_and_create(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
store = Store(tmp_path / "unused", config=config, create=True)
|
||||||
|
stats = store.get_stats()
|
||||||
|
assert stats["documents"]["exists"]
|
||||||
|
assert stats["chunks"]["exists"]
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_store_vacuum(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
store = Store(tmp_path / "unused", config=config, create=True)
|
||||||
|
await store.vacuum()
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_add_document(tmp_path):
|
||||||
|
from haiku.rag.store.engine import DocumentRecord
|
||||||
|
|
||||||
|
config = _make_config()
|
||||||
|
store = Store(tmp_path / "unused", config=config, create=True)
|
||||||
|
|
||||||
|
doc = DocumentRecord(content="The quick brown fox jumps over the lazy dog.")
|
||||||
|
store.documents_table.add([doc])
|
||||||
|
|
||||||
|
stats = store.get_stats()
|
||||||
|
assert stats["documents"]["num_rows"] == 1
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_client_create_document(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
async with HaikuRAG(tmp_path / "unused", config=config, create=True) as rag:
|
||||||
|
doc = await rag.create_document(
|
||||||
|
"Python is a programming language.", uri="test://python"
|
||||||
|
)
|
||||||
|
assert doc.id
|
||||||
|
assert doc.uri == "test://python"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_client_list_documents(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
async with HaikuRAG(tmp_path / "unused", config=config, create=True) as rag:
|
||||||
|
await rag.create_document("First document.", uri="test://first")
|
||||||
|
await rag.create_document("Second document.", uri="test://second")
|
||||||
|
|
||||||
|
docs = await rag.list_documents()
|
||||||
|
assert len(docs) == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_client_search(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
async with HaikuRAG(tmp_path / "unused", config=config, create=True) as rag:
|
||||||
|
await rag.create_document(
|
||||||
|
"The Eiffel Tower is located in Paris, France.", uri="test://eiffel"
|
||||||
|
)
|
||||||
|
results = await rag.search("Eiffel Tower")
|
||||||
|
assert len(results) > 0
|
||||||
|
assert "Eiffel" in results[0].content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_client_delete_document(tmp_path):
|
||||||
|
config = _make_config()
|
||||||
|
async with HaikuRAG(tmp_path / "unused", config=config, create=True) as rag:
|
||||||
|
doc = await rag.create_document("Temporary document.", uri="test://temp")
|
||||||
|
await rag.delete_document(doc.id)
|
||||||
|
docs = await rag.list_documents()
|
||||||
|
assert len(docs) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_app_info(tmp_path, capsys):
|
||||||
|
config = _make_config()
|
||||||
|
# Initialize the database first
|
||||||
|
async with HaikuRAG(tmp_path / "unused", config=config, create=True) as rag:
|
||||||
|
await rag.create_document("Info test document.", uri="test://info")
|
||||||
|
|
||||||
|
app = HaikuRAGApp(db_path=tmp_path / "unused", config=config)
|
||||||
|
await app.info()
|
||||||
|
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "path:" in out
|
||||||
|
assert config.lancedb.uri in out
|
||||||
|
assert "documents: 1" in out
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_app_info_empty_db(tmp_path, capsys):
|
||||||
|
config = _make_config()
|
||||||
|
app = HaikuRAGApp(db_path=tmp_path / "unused", config=config)
|
||||||
|
await app.info()
|
||||||
|
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "Database is empty" in out
|
||||||
Loading…
Reference in a new issue