diff --git a/.github/haiku.rag.yaml b/.github/haiku.rag.yaml new file mode 100644 index 00000000..3f7d59cd --- /dev/null +++ b/.github/haiku.rag.yaml @@ -0,0 +1,23 @@ +storage: + vacuum_retention_seconds: 0 + +embeddings: + model: + provider: ollama + name: qwen3-embedding:4b + +processing: + conversion_options: + do_ocr: false + +ingester: + queue: + path: ingester.db + workers: + # One worker: the runner's single CPU-bound Ollama serialises embeds, so + # concurrent workers thrash rather than parallelise. + worker_count: 1 + sources: + - type: fs + id: docs + root: docs diff --git a/.github/workflows/build-rag-db.yml b/.github/workflows/build-rag-db.yml new file mode 100644 index 00000000..13b1a460 --- /dev/null +++ b/.github/workflows/build-rag-db.yml @@ -0,0 +1,80 @@ +name: Build RAG database +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: "pyproject.toml" + - name: Install dependencies + run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev + - name: Install Ollama + run: curl -fsSL https://ollama.com/install.sh | sh + - name: Wait for Ollama + run: | + for i in $(seq 1 30); do + if curl -sf http://localhost:11434/api/version >/dev/null; then + exit 0 + fi + sleep 1 + done + echo "Ollama did not become ready within 30s" >&2 + exit 1 + - name: Cache Ollama models + uses: actions/cache@v4 + with: + path: ~/.ollama/models + key: ollama-models-qwen3-embedding-4b-v1 + - name: Pull embedding model + run: ollama pull qwen3-embedding:4b + - name: Warm embedding model + run: | + curl -sf -X POST http://localhost:11434/api/embed \ + -d '{"model":"qwen3-embedding:4b","input":"warmup"}' >/dev/null + - name: Restore previous RAG database + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do + if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then + echo "$tag" + break + fi + done) + if [ -n "$LATEST" ]; then + gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz" + tar xzf haiku-rag-docs.tar.gz + rm haiku-rag-docs.tar.gz + echo "Restored database from $LATEST" + else + echo "No previous database found, starting fresh" + fi + - name: Build RAG database + run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb + - name: Vacuum database + run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb + - name: Package RAG database + run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db* + - name: Upload to release + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber + - name: Upload workflow artifact + if: github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: haiku-rag-docs.tar.gz + path: haiku-rag-docs.tar.gz + retention-days: 1