From 291b21015fec2c43aaba8bfa3e36f2012490ba7f Mon Sep 17 00:00:00 2001 From: qaz741wsd856 Date: Thu, 15 Jan 2026 17:07:06 +0000 Subject: [PATCH] feat: enhance backup workflow to support WebDAV uploads --- .github/workflows/backup-d1.yaml | 162 ++++++++++++++++++++++++++++++- docs/db-backup-recovery.md | 49 ++++++++-- 2 files changed, 197 insertions(+), 14 deletions(-) diff --git a/.github/workflows/backup-d1.yaml b/.github/workflows/backup-d1.yaml index daffcea..55abd83 100644 --- a/.github/workflows/backup-d1.yaml +++ b/.github/workflows/backup-d1.yaml @@ -1,4 +1,4 @@ -name: Backup D1 Database to S3 +name: Backup D1 Database (S3/WebDAV) on: schedule: @@ -16,7 +16,7 @@ on: - dev env: - BACKUP_RETENTION_DAYS: 30 + BACKUP_RETENTION_DAYS: ${{ secrets.BACKUP_RETENTION_DAYS || 30 }} jobs: backup-production: @@ -25,6 +25,35 @@ jobs: if: github.event_name == 'schedule' || github.event.inputs.environment == 'production' steps: + - name: Check backup backends availability + id: backends + env: + S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} + S3_REGION: ${{ secrets.S3_REGION }} + WEBDAV_URL: ${{ secrets.WEBDAV_URL }} + WEBDAV_USER: ${{ secrets.WEBDAV_USER }} + WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }} + run: | + # Check S3 backend + if [[ -n "$S3_ACCESS_KEY_ID" && -n "$S3_SECRET_ACCESS_KEY" && -n "$S3_BUCKET" && -n "$S3_REGION" ]]; then + echo "S3_ENABLED=true" >> $GITHUB_OUTPUT + echo "✅ S3 backend: enabled" + else + echo "S3_ENABLED=false" >> $GITHUB_OUTPUT + echo "⚠️ S3 backend: disabled (missing credentials)" + fi + + # Check WebDAV backend + if [[ -n "$WEBDAV_URL" && -n "$WEBDAV_USER" && -n "$WEBDAV_PASSWORD" ]]; then + echo "WEBDAV_ENABLED=true" >> $GITHUB_OUTPUT + echo "✅ WebDAV backend: enabled" + else + echo "WEBDAV_ENABLED=false" >> $GITHUB_OUTPUT + echo "⚠️ WebDAV backend: disabled (missing credentials)" + fi + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT @@ -83,6 +112,7 @@ jobs: fi - name: Upload to S3 + if: steps.backends.outputs.S3_ENABLED == 'true' env: AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} @@ -108,7 +138,8 @@ jobs: echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$BACKUP_FILE" - - name: Cleanup old backups + - name: Cleanup old S3 backups + if: steps.backends.outputs.S3_ENABLED == 'true' env: AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} @@ -134,12 +165,86 @@ jobs: echo "✅ Cleanup completed" + - name: Setup rclone + if: steps.backends.outputs.WEBDAV_ENABLED == 'true' + uses: AnimMouse/setup-rclone@v1 + + - name: Configure rclone (WebDAV) + if: steps.backends.outputs.WEBDAV_ENABLED == 'true' + env: + WEBDAV_URL: ${{ secrets.WEBDAV_URL }} + WEBDAV_VENDOR: ${{ secrets.WEBDAV_VENDOR || 'other' }} + WEBDAV_USER: ${{ secrets.WEBDAV_USER }} + WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }} + run: | + mkdir -p ~/.config/rclone + RCLONE_WEBDAV_PASS="$(rclone obscure "$WEBDAV_PASSWORD")" + + cat > ~/.config/rclone/rclone.conf <> $GITHUB_OUTPUT + echo "✅ S3 backend: enabled" + else + echo "S3_ENABLED=false" >> $GITHUB_OUTPUT + echo "⚠️ S3 backend: disabled (missing credentials)" + fi + + # Check WebDAV backend + if [[ -n "$WEBDAV_URL" && -n "$WEBDAV_USER" && -n "$WEBDAV_PASSWORD" ]]; then + echo "WEBDAV_ENABLED=true" >> $GITHUB_OUTPUT + echo "✅ WebDAV backend: enabled" + else + echo "WEBDAV_ENABLED=false" >> $GITHUB_OUTPUT + echo "⚠️ WebDAV backend: disabled (missing credentials)" + fi + - name: Get current date id: date run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT @@ -155,7 +260,7 @@ jobs: run: | DB_NAME=$(npx wrangler d1 list --json | jq -r '.[] | select(.uuid == "${{ secrets.D1_DATABASE_ID_DEV }}") | .name') if [ -z "$DB_NAME" ]; then - echo "❌ Error: Could not find database with D1_DATABASE_ID_DEV secret + echo "❌ Error: Could not find database with D1_DATABASE_ID_DEV secret" exit 1 fi echo "Found database name" @@ -197,6 +302,7 @@ jobs: fi - name: Upload to S3 + if: steps.backends.outputs.S3_ENABLED == 'true' env: AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} @@ -221,7 +327,8 @@ jobs: echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$BACKUP_FILE" - - name: Cleanup old backups + - name: Cleanup old S3 backups + if: steps.backends.outputs.S3_ENABLED == 'true' env: AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} @@ -245,3 +352,48 @@ jobs: done echo "✅ Cleanup completed" + + - name: Setup rclone + if: steps.backends.outputs.WEBDAV_ENABLED == 'true' + uses: AnimMouse/setup-rclone@v1 + + - name: Configure rclone (WebDAV) + if: steps.backends.outputs.WEBDAV_ENABLED == 'true' + env: + WEBDAV_URL: ${{ secrets.WEBDAV_URL }} + WEBDAV_VENDOR: ${{ secrets.WEBDAV_VENDOR || 'other' }} + WEBDAV_USER: ${{ secrets.WEBDAV_USER }} + WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }} + run: | + mkdir -p ~/.config/rclone + RCLONE_WEBDAV_PASS="$(rclone obscure "$WEBDAV_PASSWORD")" + + cat > ~/.config/rclone/rclone.conf < [!NOTE] To use this backup feature, you must fork this repository and configure the same three required secrets as described in the [CI/CD deployment](deployment.md#cicd-deployment-with-github-actions) section in advance: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and `D1_DATABASE_ID`. +> [!NOTE] To use this backup feature, you must fork this repository and configure the required Cloudflare secrets as described in the [CI/CD deployment](deployment.md#cicd-deployment-with-github-actions) section in advance: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and `D1_DATABASE_ID` (and `D1_DATABASE_ID_DEV` if you want to backup `dev`). -This project includes a GitHub Action workflow that automatically backs up your D1 database to S3-compatible storage daily. The backup runs at 04:00 UTC (1 hour after the cleanup task). +This project includes a GitHub Action workflow that automatically exports your D1 database and uploads the backup to one or more destinations (S3-compatible storage and/or WebDAV) daily. The backup runs at 04:00 UTC (1 hour after the cleanup task). > [!NOTE] **Important Notes:** -> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database to S3 → Run workflow) before scheduled backups will run automatically. +> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database (S3/WebDAV) → Run workflow) before scheduled backups will run automatically. > - **Ensure your S3 bucket is set to private access** to prevent data leaks and avoid unnecessary public traffic costs. > - **⚠️ CRITICAL: Do NOT use R2 from the same Cloudflare account as your Worker** for backups. If your Cloudflare account gets suspended or banned, you will lose access to both your Worker and your backup storage, resulting in complete data loss. Always use a separate Cloudflare account or a different S3-compatible storage provider (AWS S3, Backblaze B2, MinIO, etc.) for backups to ensure redundancy and disaster recovery. +> - **Destinations are opt-in:** Upload steps run only when the corresponding secrets are configured. If you configure neither S3 nor WebDAV, the workflow will still export/compress/encrypt the backup but will not upload it anywhere. -### Required Secrets for Backup +### Backup Destination Secrets Add the following secrets to your GitHub repository (`Settings > Secrets and variables > Actions`): +#### S3-compatible storage (optional) + | Secret | Required | Description | |--------|----------|-------------| -| `S3_ACCESS_KEY_ID` | yes | Your S3 access key ID | -| `S3_SECRET_ACCESS_KEY` | yes | Your S3 secret access key | -| `S3_BUCKET` | yes | The S3 bucket name for storing backups | -| `S3_REGION` | yes | The S3 region (e.g., `us-east-1`). If unsure, use `auto` | +| `S3_ACCESS_KEY_ID` | yes (for S3) | Your S3 access key ID | +| `S3_SECRET_ACCESS_KEY` | yes (for S3) | Your S3 secret access key | +| `S3_BUCKET` | yes (for S3) | The S3 bucket name for storing backups | +| `S3_REGION` | yes (for S3) | The S3 region (e.g., `us-east-1`). If unsure, use `auto` | | `S3_ENDPOINT` | no | Custom S3 endpoint URL. Defaults to AWS S3 if not set. Required for S3-compatible services (MinIO, Cloudflare R2, Backblaze B2, etc.) | + +#### WebDAV (optional) + +| Secret | Required | Description | +|--------|----------|-------------| +| `WEBDAV_URL` | yes (for WebDAV) | WebDAV endpoint URL (e.g., Nextcloud: `https://example.com/remote.php/dav/files//`) | +| `WEBDAV_USER` | yes (for WebDAV) | WebDAV username | +| `WEBDAV_PASSWORD` | yes (for WebDAV) | WebDAV password | +| `WEBDAV_VENDOR` | no | WebDAV vendor for rclone (`nextcloud`, `owncloud`, or `other`). Defaults to `other` | +| `WEBDAV_BASE_PATH` | no | Base path for backups on the remote. Defaults to `warden-worker` | + +#### Common (optional) + +| Secret | Required | Description | +|--------|----------|-------------| | `BACKUP_ENCRYPTION_KEY` | no | Optional encryption passphrase. If set, backups will be encrypted with AES-256. **Strongly recommended** since the database contains unencrypted user metadata (emails, item counts) | +| `BACKUP_RETENTION_DAYS` | no | Number of days to keep backups. Defaults to 30 | ### Backup Features @@ -34,11 +53,13 @@ Add the following secrets to your GitHub repository (`Settings > Secrets and var * **Compression:** Backups are compressed using gzip to save storage space * **Optional Encryption:** If `BACKUP_ENCRYPTION_KEY` is set, backups are encrypted with AES-256-CBC (PBKDF2 key derivation, 100k iterations) * **Automatic Cleanup:** Old backups older than 30 days are automatically deleted +* **Destination-based uploads:** Upload steps run only when destination secrets are configured * **S3-Compatible:** Works with AWS S3, Cloudflare R2, MinIO, Backblaze B2, and any S3-compatible storage +* **WebDAV:** Works with most WebDAV servers (including Nextcloud/ownCloud) ### Backup File Location -Backups are stored in your S3 bucket with the following structure: +Backups are stored with the following structure: ``` # Unencrypted backups @@ -46,6 +67,10 @@ s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz # Encrypted backups (when BACKUP_ENCRYPTION_KEY is set) s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc + +# WebDAV backups (WEBDAV_BASE_PATH defaults to warden-worker) +/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz +/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ``` ### Decrypting Backups @@ -75,6 +100,12 @@ gunzip backup.sql.gz --endpoint-url https://your-s3-endpoint.com ``` + Or from WebDAV (using rclone): + + ```bash + rclone copy webdav:warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./ + ``` + 2. **Decrypt the backup (if encrypted):** ```bash