From 076916a8cceabf67326c9db01794fee2bc90de73 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 2 Apr 2026 15:07:37 +0300 Subject: [PATCH 1/6] Add CI workflow to build RAG database on release --- .github/haiku.rag.yaml | 8 +++++ .github/workflows/build-rag-db.yml | 58 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 .github/haiku.rag.yaml create mode 100644 .github/workflows/build-rag-db.yml diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml new file mode 100644 index 00000000..1619ec0f --- /dev/null +++ b/.github/haiku.rag.yaml @@ -0,0 +1,8 @@ +embeddings: + model: + provider: ollama + name: qwen3-embedding:4b + +processing: + conversion_options: + do_ocr: false diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml new file mode 100644 index 00000000..06f3664d --- /dev/null +++ b/.github/workflows/build-rag-db.yml @@ -0,0 +1,58 @@ +name: Build RAG database +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: "pyproject.toml" + - name: Install dependencies + run: uv sync + - name: Restore previous RAG database + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do + if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.lancedb.tar.gz"; then + echo "$tag" + break + fi + done) + if [ -n "$LATEST" ]; then + gh release download "$LATEST" --pattern "haiku-rag-docs.lancedb.tar.gz" + tar xzf haiku-rag-docs.lancedb.tar.gz + rm haiku-rag-docs.lancedb.tar.gz + echo "Restored database from $LATEST" + else + echo "No previous database found, starting fresh" + uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml init --db rag.lancedb + fi + - name: Build RAG database + env: + OLLAMA_BASE_URL: ${{ secrets.OLLAMA_BASE_URL }} + OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} + run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb + - name: Package RAG database + run: tar czf haiku-rag-docs.lancedb.tar.gz rag.lancedb/ + - name: Upload to release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.lancedb.tar.gz --clobber + - name: Upload workflow artifact + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: haiku-rag-docs.lancedb.tar.gz + path: haiku-rag-docs.lancedb.tar.gz + retention-days: 1 From 11555e37878df230d1665eba5a860df659372e28 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 2 Apr 2026 16:48:25 +0300 Subject: [PATCH 2/6] Vacuum the db after ingestion --- .github/haiku.rag.yaml | 3 +++ .github/workflows/build-rag-db.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml index 1619ec0f..c0f4d1ba 100644 --- a/.github/haiku.rag.yaml +++ b/.github/haiku.rag.yaml @@ -1,3 +1,6 @@ +storage: + vacuum_retention_seconds: 0 + embeddings: model: provider: ollama diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml index 06f3664d..f1f687f2 100644 --- a/.github/workflows/build-rag-db.yml +++ b/.github/workflows/build-rag-db.yml @@ -42,6 +42,8 @@ jobs: OLLAMA_BASE_URL: ${{ secrets.OLLAMA_BASE_URL }} OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb + - name: Vacuum database + run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb - name: Package RAG database run: tar czf haiku-rag-docs.lancedb.tar.gz rag.lancedb/ - name: Upload to release From 83a0d7f5b9cd859fcee0b541f78c30a099cdd2c1 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 2 Apr 2026 16:55:46 +0300 Subject: [PATCH 3/6] Limit installation extras --- .github/workflows/build-rag-db.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml index f1f687f2..7ebab8ce 100644 --- a/.github/workflows/build-rag-db.yml +++ b/.github/workflows/build-rag-db.yml @@ -17,7 +17,7 @@ jobs: with: python-version-file: "pyproject.toml" - name: Install dependencies - run: uv sync + run: uv sync --package haiku.rag-slim --extra docling --no-dev - name: Restore previous RAG database env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 22fc406abbac5ab77b38cf35248021a3895345a4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 17:37:14 +0300 Subject: [PATCH 4/6] Add Ollama to the mix --- .github/workflows/build-rag-db.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml index 7ebab8ce..3c847996 100644 --- a/.github/workflows/build-rag-db.yml +++ b/.github/workflows/build-rag-db.yml @@ -7,6 +7,7 @@ on: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v4 @@ -18,6 +19,29 @@ jobs: python-version-file: "pyproject.toml" - name: Install dependencies run: uv sync --package haiku.rag-slim --extra docling --no-dev + - name: Install Ollama + run: curl -fsSL https://ollama.com/install.sh | sh + - name: Wait for Ollama + run: | + for i in $(seq 1 30); do + if curl -sf http://localhost:11434/api/version >/dev/null; then + exit 0 + fi + sleep 1 + done + echo "Ollama did not become ready within 30s" >&2 + exit 1 + - name: Cache Ollama models + uses: actions/cache@v4 + with: + path: ~/.ollama/models + key: ollama-models-qwen3-embedding-4b-v1 + - name: Pull embedding model + run: ollama pull qwen3-embedding:4b + - name: Warm embedding model + run: | + curl -sf -X POST http://localhost:11434/api/embed \ + -d '{"model":"qwen3-embedding:4b","input":"warmup"}' >/dev/null - name: Restore previous RAG database env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,9 +62,6 @@ jobs: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml init --db rag.lancedb fi - name: Build RAG database - env: - OLLAMA_BASE_URL: ${{ secrets.OLLAMA_BASE_URL }} - OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb - name: Vacuum database run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb From 2d89f4ebbedf062bb64d467c55da5c75c53c4c40 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 1 Jun 2026 10:42:36 +0300 Subject: [PATCH 5/6] Build the docs database with haiku-ingester run-batch --- .github/haiku.rag.yaml | 8 ++++++++ .github/workflows/build-rag-db.yml | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml index c0f4d1ba..9d227260 100644 --- a/.github/haiku.rag.yaml +++ b/.github/haiku.rag.yaml @@ -9,3 +9,11 @@ embeddings: processing: conversion_options: do_ocr: false + +ingester: + queue: + path: ingester.db + sources: + - type: fs + id: docs + root: docs diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml index 3c847996..b91a6525 100644 --- a/.github/workflows/build-rag-db.yml +++ b/.github/workflows/build-rag-db.yml @@ -18,7 +18,7 @@ jobs: with: python-version-file: "pyproject.toml" - name: Install dependencies - run: uv sync --package haiku.rag-slim --extra docling --no-dev + run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev - name: Install Ollama run: curl -fsSL https://ollama.com/install.sh | sh - name: Wait for Ollama @@ -47,35 +47,34 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do - if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.lancedb.tar.gz"; then + if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then echo "$tag" break fi done) if [ -n "$LATEST" ]; then - gh release download "$LATEST" --pattern "haiku-rag-docs.lancedb.tar.gz" - tar xzf haiku-rag-docs.lancedb.tar.gz - rm haiku-rag-docs.lancedb.tar.gz + gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz" + tar xzf haiku-rag-docs.tar.gz + rm haiku-rag-docs.tar.gz echo "Restored database from $LATEST" else echo "No previous database found, starting fresh" - uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml init --db rag.lancedb fi - name: Build RAG database - run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb + run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb - name: Vacuum database run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb - name: Package RAG database - run: tar czf haiku-rag-docs.lancedb.tar.gz rag.lancedb/ + run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db* - name: Upload to release if: github.event_name == 'release' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.lancedb.tar.gz --clobber + run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber - name: Upload workflow artifact if: github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v4 with: - name: haiku-rag-docs.lancedb.tar.gz - path: haiku-rag-docs.lancedb.tar.gz + name: haiku-rag-docs.tar.gz + path: haiku-rag-docs.tar.gz retention-days: 1 From cb18133dba470c40d5d2fca22d60d29370871e97 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 1 Jun 2026 12:28:44 +0300 Subject: [PATCH 6/6] Use a single worker --- .github/haiku.rag.yaml | 4 ++++ .github/workflows/build-rag-db.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml index 9d227260..3f7d59cd 100644 --- a/.github/haiku.rag.yaml +++ b/.github/haiku.rag.yaml @@ -13,6 +13,10 @@ processing: ingester: queue: path: ingester.db + workers: + # One worker: the runner's single CPU-bound Ollama serialises embeds, so + # concurrent workers thrash rather than parallelise. + worker_count: 1 sources: - type: fs id: docs diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml index b91a6525..13b1a460 100644 --- a/.github/workflows/build-rag-db.yml +++ b/.github/workflows/build-rag-db.yml @@ -7,7 +7,7 @@ on: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v4