feat: enhance backup workflow to support WebDAV uploads
This commit is contained in:
qaz741wsd856
parent
fe298c9246
commit
291b21015f
2 changed files with 197 additions and 14 deletions
162
.github/workflows/backup-d1.yaml
vendored
162
.github/workflows/backup-d1.yaml
vendored
|
|
@ -1,4 +1,4 @@
|
|||
name: Backup D1 Database to S3
|
||||
name: Backup D1 Database (S3/WebDAV)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
|
|
@ -16,7 +16,7 @@ on:
|
|||
- dev
|
||||
|
||||
env:
|
||||
BACKUP_RETENTION_DAYS: 30
|
||||
BACKUP_RETENTION_DAYS: ${{ secrets.BACKUP_RETENTION_DAYS || 30 }}
|
||||
|
||||
jobs:
|
||||
backup-production:
|
||||
|
|
@ -25,6 +25,35 @@ jobs:
|
|||
if: github.event_name == 'schedule' || github.event.inputs.environment == 'production'
|
||||
|
||||
steps:
|
||||
- name: Check backup backends availability
|
||||
id: backends
|
||||
env:
|
||||
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
S3_REGION: ${{ secrets.S3_REGION }}
|
||||
WEBDAV_URL: ${{ secrets.WEBDAV_URL }}
|
||||
WEBDAV_USER: ${{ secrets.WEBDAV_USER }}
|
||||
WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }}
|
||||
run: |
|
||||
# Check S3 backend
|
||||
if [[ -n "$S3_ACCESS_KEY_ID" && -n "$S3_SECRET_ACCESS_KEY" && -n "$S3_BUCKET" && -n "$S3_REGION" ]]; then
|
||||
echo "S3_ENABLED=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ S3 backend: enabled"
|
||||
else
|
||||
echo "S3_ENABLED=false" >> $GITHUB_OUTPUT
|
||||
echo "⚠️ S3 backend: disabled (missing credentials)"
|
||||
fi
|
||||
|
||||
# Check WebDAV backend
|
||||
if [[ -n "$WEBDAV_URL" && -n "$WEBDAV_USER" && -n "$WEBDAV_PASSWORD" ]]; then
|
||||
echo "WEBDAV_ENABLED=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ WebDAV backend: enabled"
|
||||
else
|
||||
echo "WEBDAV_ENABLED=false" >> $GITHUB_OUTPUT
|
||||
echo "⚠️ WebDAV backend: disabled (missing credentials)"
|
||||
fi
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
|
||||
|
|
@ -83,6 +112,7 @@ jobs:
|
|||
fi
|
||||
|
||||
- name: Upload to S3
|
||||
if: steps.backends.outputs.S3_ENABLED == 'true'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
|
|
@ -108,7 +138,8 @@ jobs:
|
|||
|
||||
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$BACKUP_FILE"
|
||||
|
||||
- name: Cleanup old backups
|
||||
- name: Cleanup old S3 backups
|
||||
if: steps.backends.outputs.S3_ENABLED == 'true'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
|
|
@ -134,12 +165,86 @@ jobs:
|
|||
|
||||
echo "✅ Cleanup completed"
|
||||
|
||||
- name: Setup rclone
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
uses: AnimMouse/setup-rclone@v1
|
||||
|
||||
- name: Configure rclone (WebDAV)
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
env:
|
||||
WEBDAV_URL: ${{ secrets.WEBDAV_URL }}
|
||||
WEBDAV_VENDOR: ${{ secrets.WEBDAV_VENDOR || 'other' }}
|
||||
WEBDAV_USER: ${{ secrets.WEBDAV_USER }}
|
||||
WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }}
|
||||
run: |
|
||||
mkdir -p ~/.config/rclone
|
||||
RCLONE_WEBDAV_PASS="$(rclone obscure "$WEBDAV_PASSWORD")"
|
||||
|
||||
cat > ~/.config/rclone/rclone.conf <<EOF
|
||||
[webdav]
|
||||
type = webdav
|
||||
url = $WEBDAV_URL
|
||||
vendor = $WEBDAV_VENDOR
|
||||
user = $WEBDAV_USER
|
||||
pass = $RCLONE_WEBDAV_PASS
|
||||
EOF
|
||||
|
||||
- name: Upload to WebDAV
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
env:
|
||||
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
||||
WEBDAV_BASE_PATH: ${{ secrets.WEBDAV_BASE_PATH || 'warden-worker' }}
|
||||
run: |
|
||||
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
||||
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc"
|
||||
else
|
||||
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
|
||||
fi
|
||||
|
||||
rclone copyto "$BACKUP_FILE" "webdav:${WEBDAV_BASE_PATH}/production/$BACKUP_FILE"
|
||||
|
||||
echo "✅ Backup uploaded to WebDAV: webdav:${WEBDAV_BASE_PATH}/production/$BACKUP_FILE"
|
||||
|
||||
# Cleanup old backups
|
||||
rclone delete "webdav:${WEBDAV_BASE_PATH}/production/" -v --min-age ${{ env.BACKUP_RETENTION_DAYS }}d
|
||||
|
||||
echo "✅ Cleanup completed"
|
||||
|
||||
backup-dev:
|
||||
name: Backup Dev D1 Database
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.inputs.environment == 'dev'
|
||||
|
||||
steps:
|
||||
- name: Check backup backends availability
|
||||
id: backends
|
||||
env:
|
||||
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
S3_REGION: ${{ secrets.S3_REGION }}
|
||||
WEBDAV_URL: ${{ secrets.WEBDAV_URL }}
|
||||
WEBDAV_USER: ${{ secrets.WEBDAV_USER }}
|
||||
WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }}
|
||||
run: |
|
||||
# Check S3 backend
|
||||
if [[ -n "$S3_ACCESS_KEY_ID" && -n "$S3_SECRET_ACCESS_KEY" && -n "$S3_BUCKET" && -n "$S3_REGION" ]]; then
|
||||
echo "S3_ENABLED=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ S3 backend: enabled"
|
||||
else
|
||||
echo "S3_ENABLED=false" >> $GITHUB_OUTPUT
|
||||
echo "⚠️ S3 backend: disabled (missing credentials)"
|
||||
fi
|
||||
|
||||
# Check WebDAV backend
|
||||
if [[ -n "$WEBDAV_URL" && -n "$WEBDAV_USER" && -n "$WEBDAV_PASSWORD" ]]; then
|
||||
echo "WEBDAV_ENABLED=true" >> $GITHUB_OUTPUT
|
||||
echo "✅ WebDAV backend: enabled"
|
||||
else
|
||||
echo "WEBDAV_ENABLED=false" >> $GITHUB_OUTPUT
|
||||
echo "⚠️ WebDAV backend: disabled (missing credentials)"
|
||||
fi
|
||||
|
||||
- name: Get current date
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
|
||||
|
|
@ -155,7 +260,7 @@ jobs:
|
|||
run: |
|
||||
DB_NAME=$(npx wrangler d1 list --json | jq -r '.[] | select(.uuid == "${{ secrets.D1_DATABASE_ID_DEV }}") | .name')
|
||||
if [ -z "$DB_NAME" ]; then
|
||||
echo "❌ Error: Could not find database with D1_DATABASE_ID_DEV secret
|
||||
echo "❌ Error: Could not find database with D1_DATABASE_ID_DEV secret"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found database name"
|
||||
|
|
@ -197,6 +302,7 @@ jobs:
|
|||
fi
|
||||
|
||||
- name: Upload to S3
|
||||
if: steps.backends.outputs.S3_ENABLED == 'true'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
|
|
@ -221,7 +327,8 @@ jobs:
|
|||
|
||||
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$BACKUP_FILE"
|
||||
|
||||
- name: Cleanup old backups
|
||||
- name: Cleanup old S3 backups
|
||||
if: steps.backends.outputs.S3_ENABLED == 'true'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
||||
|
|
@ -245,3 +352,48 @@ jobs:
|
|||
done
|
||||
|
||||
echo "✅ Cleanup completed"
|
||||
|
||||
- name: Setup rclone
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
uses: AnimMouse/setup-rclone@v1
|
||||
|
||||
- name: Configure rclone (WebDAV)
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
env:
|
||||
WEBDAV_URL: ${{ secrets.WEBDAV_URL }}
|
||||
WEBDAV_VENDOR: ${{ secrets.WEBDAV_VENDOR || 'other' }}
|
||||
WEBDAV_USER: ${{ secrets.WEBDAV_USER }}
|
||||
WEBDAV_PASSWORD: ${{ secrets.WEBDAV_PASSWORD }}
|
||||
run: |
|
||||
mkdir -p ~/.config/rclone
|
||||
RCLONE_WEBDAV_PASS="$(rclone obscure "$WEBDAV_PASSWORD")"
|
||||
|
||||
cat > ~/.config/rclone/rclone.conf <<EOF
|
||||
[webdav]
|
||||
type = webdav
|
||||
url = $WEBDAV_URL
|
||||
vendor = $WEBDAV_VENDOR
|
||||
user = $WEBDAV_USER
|
||||
pass = $RCLONE_WEBDAV_PASS
|
||||
EOF
|
||||
|
||||
- name: Upload to WebDAV
|
||||
if: steps.backends.outputs.WEBDAV_ENABLED == 'true'
|
||||
env:
|
||||
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
||||
WEBDAV_BASE_PATH: ${{ secrets.WEBDAV_BASE_PATH || 'warden-worker' }}
|
||||
run: |
|
||||
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
||||
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc"
|
||||
else
|
||||
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
|
||||
fi
|
||||
|
||||
rclone copyto "$BACKUP_FILE" "webdav:${WEBDAV_BASE_PATH}/dev/$BACKUP_FILE"
|
||||
|
||||
echo "✅ Backup uploaded to WebDAV: webdav:${WEBDAV_BASE_PATH}/dev/$BACKUP_FILE"
|
||||
|
||||
# Cleanup old backups
|
||||
rclone delete "webdav:${WEBDAV_BASE_PATH}/dev/" -v --min-age ${{ env.BACKUP_RETENTION_DAYS }}d
|
||||
|
||||
echo "✅ Cleanup completed"
|
||||
|
|
|
|||
|
|
@ -4,27 +4,46 @@ Centralize your D1 operational playbooks here: backup automation, restore flows,
|
|||
|
||||
## GitHub Actions Backups
|
||||
|
||||
> [!NOTE] To use this backup feature, you must fork this repository and configure the same three required secrets as described in the [CI/CD deployment](deployment.md#cicd-deployment-with-github-actions) section in advance: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and `D1_DATABASE_ID`.
|
||||
> [!NOTE] To use this backup feature, you must fork this repository and configure the required Cloudflare secrets as described in the [CI/CD deployment](deployment.md#cicd-deployment-with-github-actions) section in advance: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`, and `D1_DATABASE_ID` (and `D1_DATABASE_ID_DEV` if you want to backup `dev`).
|
||||
|
||||
This project includes a GitHub Action workflow that automatically backs up your D1 database to S3-compatible storage daily. The backup runs at 04:00 UTC (1 hour after the cleanup task).
|
||||
This project includes a GitHub Action workflow that automatically exports your D1 database and uploads the backup to one or more destinations (S3-compatible storage and/or WebDAV) daily. The backup runs at 04:00 UTC (1 hour after the cleanup task).
|
||||
|
||||
> [!NOTE] **Important Notes:**
|
||||
> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database to S3 → Run workflow) before scheduled backups will run automatically.
|
||||
> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database (S3/WebDAV) → Run workflow) before scheduled backups will run automatically.
|
||||
> - **Ensure your S3 bucket is set to private access** to prevent data leaks and avoid unnecessary public traffic costs.
|
||||
> - **⚠️ CRITICAL: Do NOT use R2 from the same Cloudflare account as your Worker** for backups. If your Cloudflare account gets suspended or banned, you will lose access to both your Worker and your backup storage, resulting in complete data loss. Always use a separate Cloudflare account or a different S3-compatible storage provider (AWS S3, Backblaze B2, MinIO, etc.) for backups to ensure redundancy and disaster recovery.
|
||||
> - **Destinations are opt-in:** Upload steps run only when the corresponding secrets are configured. If you configure neither S3 nor WebDAV, the workflow will still export/compress/encrypt the backup but will not upload it anywhere.
|
||||
|
||||
### Required Secrets for Backup
|
||||
### Backup Destination Secrets
|
||||
|
||||
Add the following secrets to your GitHub repository (`Settings > Secrets and variables > Actions`):
|
||||
|
||||
#### S3-compatible storage (optional)
|
||||
|
||||
| Secret | Required | Description |
|
||||
|--------|----------|-------------|
|
||||
| `S3_ACCESS_KEY_ID` | yes | Your S3 access key ID |
|
||||
| `S3_SECRET_ACCESS_KEY` | yes | Your S3 secret access key |
|
||||
| `S3_BUCKET` | yes | The S3 bucket name for storing backups |
|
||||
| `S3_REGION` | yes | The S3 region (e.g., `us-east-1`). If unsure, use `auto` |
|
||||
| `S3_ACCESS_KEY_ID` | yes (for S3) | Your S3 access key ID |
|
||||
| `S3_SECRET_ACCESS_KEY` | yes (for S3) | Your S3 secret access key |
|
||||
| `S3_BUCKET` | yes (for S3) | The S3 bucket name for storing backups |
|
||||
| `S3_REGION` | yes (for S3) | The S3 region (e.g., `us-east-1`). If unsure, use `auto` |
|
||||
| `S3_ENDPOINT` | no | Custom S3 endpoint URL. Defaults to AWS S3 if not set. Required for S3-compatible services (MinIO, Cloudflare R2, Backblaze B2, etc.) |
|
||||
|
||||
#### WebDAV (optional)
|
||||
|
||||
| Secret | Required | Description |
|
||||
|--------|----------|-------------|
|
||||
| `WEBDAV_URL` | yes (for WebDAV) | WebDAV endpoint URL (e.g., Nextcloud: `https://example.com/remote.php/dav/files/<user>/`) |
|
||||
| `WEBDAV_USER` | yes (for WebDAV) | WebDAV username |
|
||||
| `WEBDAV_PASSWORD` | yes (for WebDAV) | WebDAV password |
|
||||
| `WEBDAV_VENDOR` | no | WebDAV vendor for rclone (`nextcloud`, `owncloud`, or `other`). Defaults to `other` |
|
||||
| `WEBDAV_BASE_PATH` | no | Base path for backups on the remote. Defaults to `warden-worker` |
|
||||
|
||||
#### Common (optional)
|
||||
|
||||
| Secret | Required | Description |
|
||||
|--------|----------|-------------|
|
||||
| `BACKUP_ENCRYPTION_KEY` | no | Optional encryption passphrase. If set, backups will be encrypted with AES-256. **Strongly recommended** since the database contains unencrypted user metadata (emails, item counts) |
|
||||
| `BACKUP_RETENTION_DAYS` | no | Number of days to keep backups. Defaults to 30 |
|
||||
|
||||
### Backup Features
|
||||
|
||||
|
|
@ -34,11 +53,13 @@ Add the following secrets to your GitHub repository (`Settings > Secrets and var
|
|||
* **Compression:** Backups are compressed using gzip to save storage space
|
||||
* **Optional Encryption:** If `BACKUP_ENCRYPTION_KEY` is set, backups are encrypted with AES-256-CBC (PBKDF2 key derivation, 100k iterations)
|
||||
* **Automatic Cleanup:** Old backups older than 30 days are automatically deleted
|
||||
* **Destination-based uploads:** Upload steps run only when destination secrets are configured
|
||||
* **S3-Compatible:** Works with AWS S3, Cloudflare R2, MinIO, Backblaze B2, and any S3-compatible storage
|
||||
* **WebDAV:** Works with most WebDAV servers (including Nextcloud/ownCloud)
|
||||
|
||||
### Backup File Location
|
||||
|
||||
Backups are stored in your S3 bucket with the following structure:
|
||||
Backups are stored with the following structure:
|
||||
|
||||
```
|
||||
# Unencrypted backups
|
||||
|
|
@ -46,6 +67,10 @@ s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz
|
|||
|
||||
# Encrypted backups (when BACKUP_ENCRYPTION_KEY is set)
|
||||
s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc
|
||||
|
||||
# WebDAV backups (WEBDAV_BASE_PATH defaults to warden-worker)
|
||||
<WEBDAV_BASE_PATH>/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz
|
||||
<WEBDAV_BASE_PATH>/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc
|
||||
```
|
||||
|
||||
### Decrypting Backups
|
||||
|
|
@ -75,6 +100,12 @@ gunzip backup.sql.gz
|
|||
--endpoint-url https://your-s3-endpoint.com
|
||||
```
|
||||
|
||||
Or from WebDAV (using rclone):
|
||||
|
||||
```bash
|
||||
rclone copy webdav:warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./
|
||||
```
|
||||
|
||||
2. **Decrypt the backup (if encrypted):**
|
||||
|
||||
```bash
|
||||
|
|
|
|||
Loading…
Reference in a new issue