Merge cb18133dba into f63dec5f75

.github/haiku.rag.yaml

@@ -0,0 +1,23 @@
+storage:
+  vacuum_retention_seconds: 0
+
+embeddings:
+  model:
+    provider: ollama
+    name: qwen3-embedding:4b
+
+processing:
+  conversion_options:
+    do_ocr: false
+
+ingester:
+  queue:
+    path: ingester.db
+  workers:
+    # One worker: the runner's single CPU-bound Ollama serialises embeds, so
+    # concurrent workers thrash rather than parallelise.
+    worker_count: 1
+  sources:
+    - type: fs
+      id: docs
+      root: docs

.github/workflows/build-rag-db.yml

@@ -0,0 +1,80 @@
+name: Build RAG database
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: "pyproject.toml"
+      - name: Install dependencies
+        run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev
+      - name: Install Ollama
+        run: curl -fsSL https://ollama.com/install.sh | sh
+      - name: Wait for Ollama
+        run: |
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:11434/api/version >/dev/null; then
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "Ollama did not become ready within 30s" >&2
+          exit 1
+      - name: Cache Ollama models
+        uses: actions/cache@v4
+        with:
+          path: ~/.ollama/models
+          key: ollama-models-qwen3-embedding-4b-v1
+      - name: Pull embedding model
+        run: ollama pull qwen3-embedding:4b
+      - name: Warm embedding model
+        run: |
+          curl -sf -X POST http://localhost:11434/api/embed \
+            -d '{"model":"qwen3-embedding:4b","input":"warmup"}' >/dev/null
+      - name: Restore previous RAG database
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do
+            if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then
+              echo "$tag"
+              break
+            fi
+          done)
+          if [ -n "$LATEST" ]; then
+            gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz"
+            tar xzf haiku-rag-docs.tar.gz
+            rm haiku-rag-docs.tar.gz
+            echo "Restored database from $LATEST"
+          else
+            echo "No previous database found, starting fresh"
+          fi
+      - name: Build RAG database
+        run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb
+      - name: Vacuum database
+        run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb
+      - name: Package RAG database
+        run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db*
+      - name: Upload to release
+        if: github.event_name == 'release'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber
+      - name: Upload workflow artifact
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/upload-artifact@v4
+        with:
+          name: haiku-rag-docs.tar.gz
+          path: haiku-rag-docs.tar.gz
+          retention-days: 1