danvics/warden-worker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat: add GitHub Actions workflow for automated D1 database backups to S3

Browse source
This commit is contained in:
qaz741wsd856 2025-11-29 04:02:12 +00:00
parent 3bd68ababf
commit ec4a851951
2 changed files with 371 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

219
.github/workflows/backup-d1.yaml vendored Normal file
View file

@ -0,0 +1,219 @@
name: Backup D1 Database to S3
on:
schedule:
# Run daily at UTC 04:00 (1 hour after the auto cleanup task)
- cron: "0 4 * * *"
workflow_dispatch:
inputs:
environment:
description: "Select the environment to backup"
required: true
default: "production"
type: choice
options:
- production
- dev
env:
BACKUP_RETENTION_DAYS: 30
jobs:
backup-production:
name: Backup Production D1 Database
runs-on: ubuntu-latest
if: github.event_name == 'schedule' || github.event.inputs.environment == 'production'
steps:
- name: Get current date
id: date
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
- name: Install wrangler
run: npm install -g wrangler
- name: Export D1 Database
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
run: |
echo "Exporting D1 database..."
npx wrangler d1 export ${{ secrets.D1_DATABASE_ID }} \
--remote \
--output=backup.sql
# Compress backup file
gzip backup.sql
echo "Backup compressed: backup.sql.gz"
ls -lh backup.sql.gz
- name: Encrypt backup (optional)
env:
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
run: |
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
echo "Encrypting backup with AES-256..."
openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \
-in backup.sql.gz \
-out "vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc" \
-pass pass:"$BACKUP_ENCRYPTION_KEY"
rm backup.sql.gz
echo "✅ Backup encrypted: vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc"
ls -lh *.enc
else
echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption"
mv backup.sql.gz "vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
echo "Backup file: vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
ls -lh *.sql.gz
fi
- name: Upload to S3
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
run: |
# Set custom S3 endpoint if configured (for S3-compatible services like MinIO, R2, etc.)
ENDPOINT_FLAG=""
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
fi
# Choose file extension based on encryption status
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc"
else
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
fi
aws s3 cp "$BACKUP_FILE" \
"s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" \
$ENDPOINT_FLAG
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$BACKUP_FILE"
- name: Cleanup old backups
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
run: |
ENDPOINT_FLAG=""
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
fi
# Find and delete backup files older than retention period
CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d)
echo "Cleaning up backups older than $CUTOFF_DATE..."
aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" $ENDPOINT_FLAG | while read -r line; do
FILE_DATE=$(echo "$line" | awk '{print $1}')
FILE_NAME=$(echo "$line" | awk '{print $4}')
if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then
echo "Deleting old backup: $FILE_NAME"
aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$FILE_NAME" $ENDPOINT_FLAG
fi
done
echo "✅ Cleanup completed"
backup-dev:
name: Backup Dev D1 Database
runs-on: ubuntu-latest
if: github.event.inputs.environment == 'dev'
steps:
- name: Get current date
id: date
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
- name: Install wrangler
run: npm install -g wrangler
- name: Export D1 Database (Dev)
env:
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
run: |
echo "Exporting D1 dev database..."
npx wrangler d1 export ${{ secrets.D1_DATABASE_ID_DEV }} \
--remote \
--output=backup.sql
gzip backup.sql
echo "Backup compressed: backup.sql.gz"
ls -lh backup.sql.gz
- name: Encrypt backup (optional)
env:
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
run: |
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
echo "Encrypting backup with AES-256..."
openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \
-in backup.sql.gz \
-out "vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc" \
-pass pass:"$BACKUP_ENCRYPTION_KEY"
rm backup.sql.gz
echo "✅ Backup encrypted: vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc"
ls -lh *.enc
else
echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption"
mv backup.sql.gz "vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
echo "Backup file: vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
ls -lh *.sql.gz
fi
- name: Upload to S3
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
run: |
ENDPOINT_FLAG=""
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
fi
# Choose file extension based on encryption status
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc"
else
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
fi
aws s3 cp "$BACKUP_FILE" \
"s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" \
$ENDPOINT_FLAG
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$BACKUP_FILE"
- name: Cleanup old backups
env:
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
run: |
ENDPOINT_FLAG=""
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
fi
CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d)
echo "Cleaning up backups older than $CUTOFF_DATE..."
aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" $ENDPOINT_FLAG | while read -r line; do
FILE_DATE=$(echo "$line" | awk '{print $1}')
FILE_NAME=$(echo "$line" | awk '{print $4}')
if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then
echo "Deleting old backup: $FILE_NAME"
aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$FILE_NAME" $ENDPOINT_FLAG
fi
done
echo "✅ Cleanup completed"

152
README.md
View file

@ -133,6 +133,158 @@ The worker includes a scheduled task that runs automatically to clean up soft-de
You can modify the cron schedule in `wrangler.toml` if you want to run the cleanup task at a different time or frequency. See [Cloudflare Cron Triggers documentation](https://developers.cloudflare.com/workers/configuration/cron-triggers/) for cron expression syntax.
### Database Backup (GitHub Actions)
This project includes a GitHub Action workflow that automatically backs up your D1 database to S3-compatible storage daily. The backup runs at 04:00 UTC (1 hour after the cleanup task).
> ⚠️ **Important Notes:**
> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database to S3 → Run workflow) before scheduled backups will run automatically.
> - **Ensure your S3 bucket is set to private access** to prevent data leaks and avoid unnecessary public traffic costs.
> - **⚠️ CRITICAL: Do NOT use R2 from the same Cloudflare account as your Worker** for backups. If your Cloudflare account gets suspended or banned, you will lose access to both your Worker and your backup storage, resulting in complete data loss. Always use a separate Cloudflare account or a different S3-compatible storage provider (AWS S3, Backblaze B2, MinIO, etc.) for backups to ensure redundancy and disaster recovery.
#### Required Secrets for Backup
Add the following secrets to your GitHub repository (`Settings > Secrets and variables > Actions`):
| Secret | Required | Description |
|--------|----------|-------------|
| `S3_ACCESS_KEY_ID` | yes | Your S3 access key ID |
| `S3_SECRET_ACCESS_KEY` | yes | Your S3 secret access key |
| `S3_BUCKET` | yes | The S3 bucket name for storing backups |
| `S3_REGION` | yes | The S3 region (e.g., `us-east-1`). If unsure, use `auto` |
| `S3_ENDPOINT` | no | Custom S3 endpoint URL. Defaults to AWS S3 if not set. Required for S3-compatible services (MinIO, Cloudflare R2, Backblaze B2, etc.) |
| `BACKUP_ENCRYPTION_KEY` | no | Optional encryption passphrase. If set, backups will be encrypted with AES-256. **Strongly recommended** since the database contains unencrypted user metadata (emails, item counts) |
#### Backup Features
* **Automatic Daily Backups:** Production database is backed up daily at 04:00 UTC
* **Manual Trigger:** You can manually trigger a backup from the GitHub Actions tab
* **Environment Selection:** When triggering manually, you can choose to backup either `production` or `dev` database
* **Compression:** Backups are compressed using gzip to save storage space
* **Optional Encryption:** If `BACKUP_ENCRYPTION_KEY` is set, backups are encrypted with AES-256-CBC (PBKDF2 key derivation, 100k iterations)
* **Automatic Cleanup:** Old backups older than 30 days are automatically deleted
* **S3-Compatible:** Works with AWS S3, Cloudflare R2, MinIO, Backblaze B2, and any S3-compatible storage
#### Backup File Location
Backups are stored in your S3 bucket with the following structure:
```
# Unencrypted backups
s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz
# Encrypted backups (when BACKUP_ENCRYPTION_KEY is set)
s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc
```
#### Decrypting Backups
If you enabled encryption, use the following command to decrypt a backup:
```bash
openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
-in vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc \
-out backup.sql.gz \
-pass pass:"YOUR_ENCRYPTION_KEY"
# Then decompress
gunzip backup.sql.gz
```
#### Restoring Database to Cloudflare D1
To restore your D1 database from a backup:
1. **Download the backup from S3:**
```bash
# Using AWS CLI
aws s3 cp s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./
# Or with custom endpoint (e.g., R2, MinIO)
aws s3 cp s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./ \
--endpoint-url https://your-s3-endpoint.com
```
2. **Decrypt the backup (if encrypted):**
```bash
openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \
-in vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc \
-out backup.sql.gz \
-pass pass:"YOUR_ENCRYPTION_KEY"
```
3. **Decompress the backup:**
```bash
gunzip backup.sql.gz
```
4. **Restore to Cloudflare D1:**
```bash
# First, you may want to clear the existing database (optional, use with caution!)
# wrangler d1 execute vault1 --remote --command "DELETE FROM ciphers; DELETE FROM folders; DELETE FROM users;"
# Import the backup
wrangler d1 execute vault1 --remote --file=backup.sql
```
> **Note:** The `--remote` flag is required to execute against your production D1 database. Without it, the command will run against the local development database.
**Alternative: Using Database ID directly**
If you don't have `vault1` binding configured in your local `wrangler.toml`, you can use the database ID (UUID) directly instead of the binding name:
```bash
# Replace YOUR_DATABASE_ID with your actual D1 database UUID
# wrangler d1 execute YOUR_DATABASE_ID --remote --command "DELETE FROM ciphers; DELETE FROM folders; DELETE FROM users;"
wrangler d1 execute YOUR_DATABASE_ID --remote --file=backup.sql
```
You can find your database ID in the Cloudflare dashboard under Storage & databases > D1 SQL database, or by running `wrangler d1 list`.
### Local Development with D1
You can run this Worker locally with full D1 database support using Wrangler. This is useful for development, testing, or as a temporary fallback when Cloudflare services are unavailable.
To run locally with your production data (useful as emergency fallback):
1. **Download and decrypt your backup** (follow the steps above)
2. **Import the backup to local D1:**
```bash
# Without --remote flag, this imports to local database
wrangler d1 execute vault1 --file=backup.sql
```
3. **Start the local server with persistence:**
```bash
wrangler dev --persist
```
4. **Configure your Bitwarden client** to use `http://localhost:8787` (or your local network IP for mobile devices)
#### Accessing Local SQLite Database Directly
The local D1 database is stored as a SQLite file. You can access it directly:
```bash
# Find the database file
ls .wrangler/state/v3/d1/
# Open with SQLite CLI
sqlite3 .wrangler/state/v3/d1/miniflare-D1DatabaseObject/*.sqlite
# Example: List all users
sqlite> SELECT email FROM users;
```
> **Note:** The local development environment requires Node.js and Wrangler installed. The Worker runs in a simulated environment using [workerd](https://github.com/cloudflare/workerd), Cloudflare's open-source Workers runtime.
## Contributing
Contributions are welcome! If you find a bug, have a feature request, or want to improve the code, please open an issue or submit a pull request.