openreader/.github/workflows/docker-publish.yml
Richard R f1aa1c3e3b feat(compute): add external compute worker backend and integration
Introduce support for external compute worker mode (`COMPUTE_MODE=worker`) using a new `WorkerComputeBackend`. This enables offloading heavy ONNX Whisper alignment and PDF layout parsing to a standalone worker service (Redis + BullMQ), improving scalability and compatibility with serverless/limited environments.

- Add `@openreader/compute-core` as a shared package for ONNX inference and PDF parsing logic.
- Implement `WorkerComputeBackend` and worker contract/types for remote job execution.
- Update compute backend selection logic and remove previous worker mode guards.
- Extend `WhisperAlignInput` and `PdfLayoutInput` types to support object keys for remote data access.
- Refactor local compute backend to use `@openreader/compute-core` and support both buffer and object key inputs.
- Update job runner, TTS segment alignment, and PDF layout parsing flows to use new compute backend APIs.
- Add scripts, Docker workflow, and documentation for deploying and running the compute worker.
- Update environment variable docs and examples for worker mode, including storage requirements and configuration.
- Document published images and stack changes to reflect the new compute worker architecture.

BREAKING CHANGE: `COMPUTE_MODE=worker` now requires an external compute worker service and S3-compatible object storage. Embedded SeaweedFS (`weed mini`) is not supported in worker mode. See the new documentation for deployment and configuration details.
2026-05-19 15:21:25 -06:00

205 lines
7.1 KiB
YAML

name: Create and publish Docker images
on:
push:
tags:
- 'v*.*.*'
workflow_dispatch:
inputs:
use_latest_tag:
description: 'Tag as latest instead of branch name'
required: false
type: boolean
default: false
env:
REGISTRY: ghcr.io
permissions:
contents: read
jobs:
prepare:
runs-on: ubuntu-24.04
outputs:
web_image_name: ${{ steps.image-name.outputs.web_image_name }}
web_legacy_image_name: ${{ steps.image-name.outputs.web_legacy_image_name }}
compute_worker_image_name: ${{ steps.image-name.outputs.compute_worker_image_name }}
steps:
- name: Compute Docker image names
id: image-name
run: |
owner="${GITHUB_REPOSITORY_OWNER,,}"
echo "web_image_name=${owner}/openreader" >> "$GITHUB_OUTPUT"
echo "web_legacy_image_name=${owner}/openreader-webui" >> "$GITHUB_OUTPUT"
echo "compute_worker_image_name=${owner}/openreader-compute-worker" >> "$GITHUB_OUTPUT"
build:
needs: prepare
runs-on: ${{ matrix.runner }}
permissions:
actions: write
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- image_target: web
arch: amd64
platform: linux/amd64
runner: ubuntu-24.04
context: .
dockerfile: ./Dockerfile
- image_target: web
arch: arm64
platform: linux/arm64
runner: ubuntu-24.04-arm
context: .
dockerfile: ./Dockerfile
- image_target: compute-worker
arch: amd64
platform: linux/amd64
runner: ubuntu-24.04
context: .
dockerfile: ./compute/worker/Dockerfile
- image_target: compute-worker
arch: arm64
platform: linux/arm64
runner: ubuntu-24.04-arm
context: .
dockerfile: ./compute/worker/Dockerfile
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
ref: ${{ github.sha }}
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Select image name
id: image
run: |
if [ "${{ matrix.image_target }}" = "web" ]; then
echo "image_name=${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT"
else
echo "image_name=${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push Docker image (${{ matrix.image_target }} / ${{ matrix.arch }})
id: build
uses: docker/build-push-action@v7
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
cache-from: type=gha,scope=${{ matrix.image_target }}-${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.image_target }}-${{ matrix.arch }}
provenance: false
outputs: type=image,name=${{ env.REGISTRY }}/${{ steps.image.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true
- name: Export digest
run: |
mkdir -p /tmp/digests/${{ matrix.image_target }}
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${{ matrix.image_target }}/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v7
with:
name: digests-${{ matrix.image_target }}-${{ matrix.arch }}
path: /tmp/digests/${{ matrix.image_target }}
if-no-files-found: error
retention-days: 1
merge:
needs: [prepare, build]
runs-on: ${{ matrix.runner }}
permissions:
actions: read
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- image_target: web
runner: ubuntu-24.04
- image_target: compute-worker
runner: ubuntu-24.04
steps:
- name: Download digests
uses: actions/download-artifact@v8
with:
path: /tmp/digests/${{ matrix.image_target }}
pattern: digests-${{ matrix.image_target }}-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Select image names
id: image
run: |
if [ "${{ matrix.image_target }}" = "web" ]; then
echo "image_name=${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT"
echo "metadata_images<<EOF" >> "$GITHUB_OUTPUT"
echo "${{ env.REGISTRY }}/${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT"
echo "${{ env.REGISTRY }}/${{ needs.prepare.outputs.web_legacy_image_name }}" >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
else
echo "image_name=${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT"
echo "metadata_images=${{ env.REGISTRY }}/${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v6
with:
images: ${{ steps.image.outputs.metadata_images }}
tags: |
type=raw,value=latest,enable=${{ (github.event_name == 'push' && !contains(github.ref, '-pre')) || (github.event_name == 'workflow_dispatch' && inputs.use_latest_tag == true) }},priority=1000
type=semver,pattern={{version}}
type=ref,event=tag
type=ref,event=branch,enable=${{ github.event_name == 'workflow_dispatch' && inputs.use_latest_tag != true }}
- name: Create manifest list and push
working-directory: /tmp/digests/${{ matrix.image_target }}
run: |
docker buildx imagetools create \
$(echo "$TAGS" | xargs -I {} echo -t {}) \
$(printf '${{ env.REGISTRY }}/${{ steps.image.outputs.image_name }}@sha256:%s ' *)
env:
TAGS: ${{ steps.meta.outputs.tags }}
- name: Output build information
run: |
echo "✅ Docker image built and pushed successfully!"
echo "🐋 Target: ${{ matrix.image_target }}"
echo "🐋 Images:"
echo '${{ steps.meta.outputs.tags }}' | sed 's/^/ - /'
echo "📝 Event: ${{ github.event_name }}"
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
echo "📝 Triggered by manual workflow dispatch on branch: ${{ github.ref_name }}"
echo "📝 use_latest_tag: ${{ inputs.use_latest_tag }}"
else
echo "📝 Triggered by tag: ${{ github.ref_name }}"
fi