diff --git a/docs/configuration.md b/docs/configuration.md
index 5e78ee8a..3f42a369 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -107,9 +107,19 @@ ANTHROPIC_API_KEY="your-api-key"
 
 Reranking improves search quality by re-ordering the initial search results using specialized models. When enabled, the system retrieves more candidates (3x the requested limit) and then reranks them to return the most relevant results.
 
-Reranking is **automatically enabled** if you install the appropriate reranking provider package.
+Reranking is **automatically enabled** by default using Ollama, or if you install the appropriate reranking provider package.
 
-### MixedBread AI (Default)
+### Ollama (Default)
+
+Ollama reranking uses LLMs with structured output to rank documents by relevance:
+
+```bash
+RERANK_PROVIDER="ollama"
+RERANK_MODEL="qwen3:1.7b"  # or any model that supports structured output
+OLLAMA_BASE_URL="http://localhost:11434"
+```
+
+### MixedBread AI
 
 For MxBAI reranking, install with mxbai extras:
 
diff --git a/src/haiku/rag/config.py b/src/haiku/rag/config.py
index 251d4a46..e4732f48 100644
--- a/src/haiku/rag/config.py
+++ b/src/haiku/rag/config.py
@@ -19,8 +19,8 @@ class AppConfig(BaseModel):
     EMBEDDINGS_MODEL: str = "mxbai-embed-large"
     EMBEDDINGS_VECTOR_DIM: int = 1024
 
-    RERANK_PROVIDER: str = "mxbai"
-    RERANK_MODEL: str = "mixedbread-ai/mxbai-rerank-base-v2"
+    RERANK_PROVIDER: str = "ollama"
+    RERANK_MODEL: str = "qwen3"
 
     QA_PROVIDER: str = "ollama"
     QA_MODEL: str = "qwen3"
diff --git a/src/haiku/rag/reranking/__init__.py b/src/haiku/rag/reranking/__init__.py
index 80df6bcb..449d35c3 100644
--- a/src/haiku/rag/reranking/__init__.py
+++ b/src/haiku/rag/reranking/__init__.py
@@ -35,4 +35,10 @@ def get_reranker() -> RerankerBase | None:
         except ImportError:
             return None
 
+    if Config.RERANK_PROVIDER == "ollama":
+        from haiku.rag.reranking.ollama import OllamaReranker
+
+        _reranker = OllamaReranker()
+        return _reranker
+
     return None
diff --git a/src/haiku/rag/reranking/ollama.py b/src/haiku/rag/reranking/ollama.py
new file mode 100644
index 00000000..727c546b
--- /dev/null
+++ b/src/haiku/rag/reranking/ollama.py
@@ -0,0 +1,84 @@
+import json
+
+from ollama import AsyncClient
+from pydantic import BaseModel
+
+from haiku.rag.config import Config
+from haiku.rag.reranking.base import RerankerBase
+from haiku.rag.store.models.chunk import Chunk
+
+OLLAMA_OPTIONS = {"temperature": 0.0, "seed": 42, "num_ctx": 16384}
+
+
+class RerankResult(BaseModel):
+    """Individual rerank result with index and relevance score."""
+
+    index: int
+    relevance_score: float
+
+
+class RerankResponse(BaseModel):
+    """Response from the reranking model containing ranked results."""
+
+    results: list[RerankResult]
+
+
+class OllamaReranker(RerankerBase):
+    def __init__(self, model: str = Config.RERANK_MODEL):
+        self._model = model
+        self._client = AsyncClient(host=Config.OLLAMA_BASE_URL)
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        if not chunks:
+            return []
+
+        documents = []
+        for i, chunk in enumerate(chunks):
+            documents.append({"index": i, "content": chunk.content})
+
+        # Create the prompt for reranking
+        system_prompt = """You are a document reranking assistant. Given a query and a list of document chunks, you must rank them by relevance to the query.
+
+Return your response as a JSON object with a "results" array. Each result should have:
+- "index": the original index of the document (integer)
+- "relevance_score": a score between 0.0 and 1.0 indicating relevance (float, where 1.0 is most relevant)
+
+Only return the top documents up to the requested limit, ordered by decreasing relevance score."""
+
+        documents_text = ""
+        for doc in documents:
+            documents_text += f"Index {doc['index']}: {doc['content']}\n\n"
+
+        user_prompt = f"""Query: {query}
+
+Documents to rerank:
+{documents_text.strip()}
+
+Please rank these documents by relevance to the query and return the top {top_n} results as JSON."""
+
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ]
+
+        try:
+            response = await self._client.chat(
+                model=self._model,
+                messages=messages,
+                format=RerankResponse.model_json_schema(),
+                options=OLLAMA_OPTIONS,
+            )
+
+            content = response["message"]["content"]
+
+            parsed_response = RerankResponse.model_validate(json.loads(content))
+            return [
+                (chunks[result.index], result.relevance_score)
+                for result in parsed_response.results[:top_n]
+            ]
+
+        except Exception:
+            # Fallback: return chunks in original order with same score
+            return [(chunks[i], 1.0) for i in range(min(top_n, len(chunks)))]
diff --git a/tests/test_reranker.py b/tests/test_reranker.py
index 00e28b3b..5ce13156 100644
--- a/tests/test_reranker.py
+++ b/tests/test_reranker.py
@@ -21,7 +21,7 @@ chunks = [
 @pytest.mark.asyncio
 async def test_reranker_base():
     reranker = RerankerBase()
-    assert reranker._model == "mixedbread-ai/mxbai-rerank-base-v2"
+    assert reranker._model == "qwen3"
 
     with pytest.raises(NotImplementedError):
         await reranker.rerank("query", [])
@@ -58,3 +58,16 @@ async def test_cohere_reranker():
 
     except ImportError:
         pytest.skip("Cohere package not installed")
+
+
+@pytest.mark.asyncio
+async def test_ollama_reranker():
+    from haiku.rag.reranking.ollama import OllamaReranker
+
+    reranker = OllamaReranker()
+    reranked = await reranker.rerank(
+        "Who wrote 'To Kill a Mockingbird'?", chunks, top_n=2
+    )
+
+    assert [chunk.document_id for chunk, score in reranked] == [0, 2]
+    assert all(isinstance(score, float) for chunk, score in reranked)