danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Build the docs database with haiku-ingester run-batch

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-06-01 10:42:36 +03:00
parent 22fc406abb
commit 2d89f4ebbe
No known key found for this signature in database
2 changed files with 18 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/haiku.rag.yaml vendored
View file

@ -9,3 +9,11 @@ embeddings:
processing: processing:
conversion_options: conversion_options:
do_ocr: false do_ocr: false
ingester:
queue:
path: ingester.db
sources:
- type: fs
id: docs
root: docs

21
.github/workflows/build-rag-db.yml vendored
View file

@ -18,7 +18,7 @@ jobs:
with: with:
python-version-file: "pyproject.toml" python-version-file: "pyproject.toml"
- name: Install dependencies - name: Install dependencies
run: uv sync --package haiku.rag-slim --extra docling --no-dev run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev
- name: Install Ollama - name: Install Ollama
run: curl -fsSL https://ollama.com/install.sh | sh run: curl -fsSL https://ollama.com/install.sh | sh
- name: Wait for Ollama - name: Wait for Ollama
@ -47,35 +47,34 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: | run: |
LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do
if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.lancedb.tar.gz"; then if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then
echo "$tag" echo "$tag"
break break
fi fi
done) done)
if [ -n "$LATEST" ]; then if [ -n "$LATEST" ]; then
gh release download "$LATEST" --pattern "haiku-rag-docs.lancedb.tar.gz" gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz"
tar xzf haiku-rag-docs.lancedb.tar.gz tar xzf haiku-rag-docs.tar.gz
rm haiku-rag-docs.lancedb.tar.gz rm haiku-rag-docs.tar.gz
echo "Restored database from $LATEST" echo "Restored database from $LATEST"
else else
echo "No previous database found, starting fresh" echo "No previous database found, starting fresh"
uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml init --db rag.lancedb
fi fi
- name: Build RAG database - name: Build RAG database
run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml add-src docs/ --db rag.lancedb run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb
- name: Vacuum database - name: Vacuum database
run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb
- name: Package RAG database - name: Package RAG database
run: tar czf haiku-rag-docs.lancedb.tar.gz rag.lancedb/ run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db*
- name: Upload to release - name: Upload to release
if: github.event_name == 'release' if: github.event_name == 'release'
env: env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.lancedb.tar.gz --clobber run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber
- name: Upload workflow artifact - name: Upload workflow artifact
if: github.event_name == 'workflow_dispatch' if: github.event_name == 'workflow_dispatch'
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: haiku-rag-docs.lancedb.tar.gz name: haiku-rag-docs.tar.gz
path: haiku-rag-docs.lancedb.tar.gz path: haiku-rag-docs.tar.gz
retention-days: 1 retention-days: 1