From 2d89f4ebbedf062bb64d467c55da5c75c53c4c40 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 1 Jun 2026 10:42:36 +0300
Subject: [PATCH] Build the docs database with haiku-ingester run-batch

---
 .github/haiku.rag.yaml             |  8 ++++++++
 .github/workflows/build-rag-db.yml | 21 ++++++++++-----------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml
index c0f4d1ba..9d227260 100644
--- a/.github/haiku.rag.yaml
+++ b/.github/haiku.rag.yaml
@@ -9,3 +9,11 @@ embeddings:
 processing:
   conversion_options:
     do_ocr: false
+
+ingester:
+  queue:
+    path: ingester.db
+  sources:
+    - type: fs
+      id: docs
+      root: docs
diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml
index 3c847996..b91a6525 100644
--- a/.github/workflows/build-rag-db.yml
+++ b/.github/workflows/build-rag-db.yml
@@ -18,7 +18,7 @@ jobs:
         with:
           python-version-file: "pyproject.toml"
       - name: Install dependencies
-        run: uv sync --package haiku.rag-slim --extra docling --no-dev
+        run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev
       - name: Install Ollama
         run: curl -fsSL https://ollama.com/install.sh | sh
       - name: Wait for Ollama
@@ -47,35 +47,34 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do
-            if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.lancedb.tar.gz"; then
+            if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then
               echo "$tag"
               break
             fi
           done)
           if [ -n "$LATEST" ]; then
-            gh release download "$LATEST" --pattern "haiku-rag-docs.lancedb.tar.gz"
-            tar xzf haiku-rag-docs.lancedb.tar.gz
-            rm haiku-rag-docs.lancedb.tar.gz
+            gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz"
+            tar xzf haiku-rag-docs.tar.gz
+            rm haiku-rag-docs.tar.gz
             echo "Restored database from $LATEST"
           else
             echo "No previous database found, starting fresh"
-            uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml init --db rag.lancedb
           fi
       - name: Build RAG database
-        run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb
+        run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb
       - name: Vacuum database
         run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb
       - name: Package RAG database
-        run: tar czf haiku-rag-docs.lancedb.tar.gz rag.lancedb/
+        run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db*
       - name: Upload to release
         if: github.event_name == 'release'
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.lancedb.tar.gz --clobber
+        run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber
       - name: Upload workflow artifact
         if: github.event_name == 'workflow_dispatch'
         uses: actions/upload-artifact@v4
         with:
-          name: haiku-rag-docs.lancedb.tar.gz
-          path: haiku-rag-docs.lancedb.tar.gz
+          name: haiku-rag-docs.tar.gz
+          path: haiku-rag-docs.tar.gz
           retention-days: 1