80 lines
2.9 KiB
YAML
80 lines
2.9 KiB
YAML
name: Build RAG database
|
|
on:
|
|
release:
|
|
types: [published]
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 90
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: astral-sh/setup-uv@v4
|
|
with:
|
|
enable-cache: true
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version-file: "pyproject.toml"
|
|
- name: Install dependencies
|
|
run: uv sync --package haiku.rag-slim --extra docling --extra ingester --no-dev
|
|
- name: Install Ollama
|
|
run: curl -fsSL https://ollama.com/install.sh | sh
|
|
- name: Wait for Ollama
|
|
run: |
|
|
for i in $(seq 1 30); do
|
|
if curl -sf http://localhost:11434/api/version >/dev/null; then
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "Ollama did not become ready within 30s" >&2
|
|
exit 1
|
|
- name: Cache Ollama models
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.ollama/models
|
|
key: ollama-models-qwen3-embedding-4b-v1
|
|
- name: Pull embedding model
|
|
run: ollama pull qwen3-embedding:4b
|
|
- name: Warm embedding model
|
|
run: |
|
|
curl -sf -X POST http://localhost:11434/api/embed \
|
|
-d '{"model":"qwen3-embedding:4b","input":"warmup"}' >/dev/null
|
|
- name: Restore previous RAG database
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
LATEST=$(gh release list --limit 10 --json tagName --jq '.[].tagName' | while read -r tag; do
|
|
if gh release view "$tag" --json assets --jq '.assets[].name' 2>/dev/null | grep -q "haiku-rag-docs.tar.gz"; then
|
|
echo "$tag"
|
|
break
|
|
fi
|
|
done)
|
|
if [ -n "$LATEST" ]; then
|
|
gh release download "$LATEST" --pattern "haiku-rag-docs.tar.gz"
|
|
tar xzf haiku-rag-docs.tar.gz
|
|
rm haiku-rag-docs.tar.gz
|
|
echo "Restored database from $LATEST"
|
|
else
|
|
echo "No previous database found, starting fresh"
|
|
fi
|
|
- name: Build RAG database
|
|
run: uv run haiku-ingester --config ${{ github.workspace }}/.github/haiku.rag.yaml run-batch --db rag.lancedb
|
|
- name: Vacuum database
|
|
run: uv run haiku-rag --config ${{ github.workspace }}/.github/haiku.rag.yaml vacuum --db rag.lancedb
|
|
- name: Package RAG database
|
|
run: tar czf haiku-rag-docs.tar.gz rag.lancedb/ ingester.db*
|
|
- name: Upload to release
|
|
if: github.event_name == 'release'
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: gh release upload ${{ github.event.release.tag_name }} haiku-rag-docs.tar.gz --clobber
|
|
- name: Upload workflow artifact
|
|
if: github.event_name == 'workflow_dispatch'
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: haiku-rag-docs.tar.gz
|
|
path: haiku-rag-docs.tar.gz
|
|
retention-days: 1
|