From c69daecf5371509f0a45f6023a695ad166c49e1f Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 29 Aug 2025 16:07:36 +0300 Subject: [PATCH] update docs --- README.md | 12 ++++++------ docs/cli.md | 2 +- docs/index.md | 10 +++++----- docs/python.md | 28 +++++++++++++++++++++------- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b7dcc337..75ba3d1e 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -# Haiku SQLite RAG +# Haiku LanceDB RAG -Retrieval-Augmented Generation (RAG) library on SQLite. +Retrieval-Augmented Generation (RAG) library built on LanceDB. -`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. +`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. ## Features -- **Local SQLite**: No external servers required +- **Local LanceDB**: No external servers required - **Multiple embedding providers**: Ollama, VoyageAI, OpenAI - **Multiple QA providers**: Any provider/model supported by Pydantic AI -- **Hybrid search**: Vector + full-text search with Reciprocal Rank Fusion +- **Native hybrid search**: Vector + full-text search with native LanceDB RRF reranking - **Reranking**: Default search result reranking with MixedBread AI or Cohere - **Question answering**: Built-in QA agents on your documents - **File monitoring**: Auto-index files when run as server @@ -49,7 +49,7 @@ haiku-rag serve ```python from haiku.rag.client import HaikuRAG -async with HaikuRAG("database.db") as client: +async with HaikuRAG("database.lancedb") as client: # Add document doc = await client.create_document("Your content") diff --git a/docs/cli.md b/docs/cli.md index 0dd19b0b..164d0efe 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -54,7 +54,7 @@ haiku-rag search "machine learning" With options: ```bash -haiku-rag search "python programming" --limit 10 --k 100 +haiku-rag search "python programming" --limit 10 ``` ## Question Answering diff --git a/docs/index.md b/docs/index.md index 22f2e485..0b616ab9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,12 +1,12 @@ # haiku.rag -`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work on SQLite alone without the need for external vector databases. It uses [sqlite-vec](https://github.com/asg017/sqlite-vec) for storing the embeddings and performs semantic (vector) search as well as full-text search combined through Reciprocal Rank Fusion. Both open-source (Ollama, MixedBread AI) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. +`haiku.rag` is a Retrieval-Augmented Generation (RAG) library built to work with LanceDB as a local vector database. It uses LanceDB for storing embeddings and performs semantic (vector) search as well as full-text search combined through native hybrid search with Reciprocal Rank Fusion. Both open-source (Ollama, MixedBread AI) as well as commercial (OpenAI, VoyageAI) embedding providers are supported. ## Features -- **Local SQLite**: No need to run additional servers +- **Local LanceDB**: No need to run additional servers - **Support for various embedding providers**: Ollama, VoyageAI, OpenAI or add your own -- **Hybrid Search**: Vector search using `sqlite-vec` combined with full-text search `FTS5`, using Reciprocal Rank Fusion +- **Native Hybrid Search**: Vector search combined with full-text search using native LanceDB RRF reranking - **Reranking**: Optional result reranking with MixedBread AI or Cohere - **Question Answering**: Built-in QA agents using Ollama, OpenAI, or Anthropic. - **File monitoring**: Automatically index files when run as a server @@ -26,7 +26,7 @@ Use from Python: ```python from haiku.rag.client import HaikuRAG -async with HaikuRAG("database.db") as client: +async with HaikuRAG("database.lancedb") as client: # Add a document doc = await client.create_document("Your content here") @@ -34,7 +34,7 @@ async with HaikuRAG("database.db") as client: results = await client.search("query") # Ask questions - answer = await client.ask("Who is the author of haiku.rag?", rerank=False) + answer = await client.ask("Who is the author of haiku.rag?") ``` Or use the CLI: diff --git a/docs/python.md b/docs/python.md index 37374488..fa3f5f03 100644 --- a/docs/python.md +++ b/docs/python.md @@ -9,7 +9,7 @@ from pathlib import Path from haiku.rag.client import HaikuRAG # Use as async context manager (recommended) -async with HaikuRAG("path/to/database.db") as client: +async with HaikuRAG("path/to/database.lancedb") as client: # Your code here pass ``` @@ -101,9 +101,9 @@ async for doc_id in client.rebuild_database(): ## Searching Documents -The search method performs hybrid search (vector + full-text) with **reranking enabled by default** for improved relevance: +The search method performs native hybrid search (vector + full-text) using LanceDB with **reranking enabled by default** for improved relevance: -Basic search (with reranking): +Basic hybrid search (default, with reranking): ```python results = await client.search("machine learning algorithms", limit=5) for chunk, score in results: @@ -112,13 +112,27 @@ for chunk, score in results: print(f"Document ID: {chunk.document_id}") ``` -With options: +Search with different search types: ```python +# Vector search only results = await client.search( query="machine learning", - limit=5, # Maximum results to return - k=60, # RRF parameter for reciprocal rank fusion - rerank=False # Disable reranking for faster search + limit=5, + search_type="vector" +) + +# Full-text search only +results = await client.search( + query="machine learning", + limit=5, + search_type="fts" +) + +# Hybrid search (default - combines vector + fts with native LanceDB RRF) +results = await client.search( + query="machine learning", + limit=5, + search_type="hybrid" ) # Process results