From ec4a851951cc3acfa3a0af27ea7a82cb001189c9 Mon Sep 17 00:00:00 2001 From: qaz741wsd856 Date: Sat, 29 Nov 2025 04:02:12 +0000 Subject: [PATCH] feat: add GitHub Actions workflow for automated D1 database backups to S3 --- .github/workflows/backup-d1.yaml | 219 +++++++++++++++++++++++++++++++ README.md | 152 +++++++++++++++++++++ 2 files changed, 371 insertions(+) create mode 100644 .github/workflows/backup-d1.yaml diff --git a/.github/workflows/backup-d1.yaml b/.github/workflows/backup-d1.yaml new file mode 100644 index 0000000..f8c8c8f --- /dev/null +++ b/.github/workflows/backup-d1.yaml @@ -0,0 +1,219 @@ +name: Backup D1 Database to S3 + +on: + schedule: + # Run daily at UTC 04:00 (1 hour after the auto cleanup task) + - cron: "0 4 * * *" + workflow_dispatch: + inputs: + environment: + description: "Select the environment to backup" + required: true + default: "production" + type: choice + options: + - production + - dev + +env: + BACKUP_RETENTION_DAYS: 30 + +jobs: + backup-production: + name: Backup Production D1 Database + runs-on: ubuntu-latest + if: github.event_name == 'schedule' || github.event.inputs.environment == 'production' + + steps: + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT + + - name: Install wrangler + run: npm install -g wrangler + + - name: Export D1 Database + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + echo "Exporting D1 database..." + npx wrangler d1 export ${{ secrets.D1_DATABASE_ID }} \ + --remote \ + --output=backup.sql + + # Compress backup file + gzip backup.sql + + echo "Backup compressed: backup.sql.gz" + ls -lh backup.sql.gz + + - name: Encrypt backup (optional) + env: + BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }} + run: | + if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then + echo "Encrypting backup with AES-256..." + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \ + -in backup.sql.gz \ + -out "vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc" \ + -pass pass:"$BACKUP_ENCRYPTION_KEY" + rm backup.sql.gz + echo "✅ Backup encrypted: vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc" + ls -lh *.enc + else + echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption" + mv backup.sql.gz "vault1_prod_${{ steps.date.outputs.date }}.sql.gz" + echo "Backup file: vault1_prod_${{ steps.date.outputs.date }}.sql.gz" + ls -lh *.sql.gz + fi + + - name: Upload to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }} + run: | + # Set custom S3 endpoint if configured (for S3-compatible services like MinIO, R2, etc.) + ENDPOINT_FLAG="" + if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then + ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}" + fi + + # Choose file extension based on encryption status + if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then + BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc" + else + BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz" + fi + + aws s3 cp "$BACKUP_FILE" \ + "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" \ + $ENDPOINT_FLAG + + echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$BACKUP_FILE" + + - name: Cleanup old backups + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + run: | + ENDPOINT_FLAG="" + if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then + ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}" + fi + + # Find and delete backup files older than retention period + CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d) + echo "Cleaning up backups older than $CUTOFF_DATE..." + + aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" $ENDPOINT_FLAG | while read -r line; do + FILE_DATE=$(echo "$line" | awk '{print $1}') + FILE_NAME=$(echo "$line" | awk '{print $4}') + if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then + echo "Deleting old backup: $FILE_NAME" + aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$FILE_NAME" $ENDPOINT_FLAG + fi + done + + echo "✅ Cleanup completed" + + backup-dev: + name: Backup Dev D1 Database + runs-on: ubuntu-latest + if: github.event.inputs.environment == 'dev' + + steps: + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT + + - name: Install wrangler + run: npm install -g wrangler + + - name: Export D1 Database (Dev) + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + echo "Exporting D1 dev database..." + npx wrangler d1 export ${{ secrets.D1_DATABASE_ID_DEV }} \ + --remote \ + --output=backup.sql + + gzip backup.sql + + echo "Backup compressed: backup.sql.gz" + ls -lh backup.sql.gz + + - name: Encrypt backup (optional) + env: + BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }} + run: | + if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then + echo "Encrypting backup with AES-256..." + openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \ + -in backup.sql.gz \ + -out "vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc" \ + -pass pass:"$BACKUP_ENCRYPTION_KEY" + rm backup.sql.gz + echo "✅ Backup encrypted: vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc" + ls -lh *.enc + else + echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption" + mv backup.sql.gz "vault1_dev_${{ steps.date.outputs.date }}.sql.gz" + echo "Backup file: vault1_dev_${{ steps.date.outputs.date }}.sql.gz" + ls -lh *.sql.gz + fi + + - name: Upload to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }} + run: | + ENDPOINT_FLAG="" + if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then + ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}" + fi + + # Choose file extension based on encryption status + if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then + BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc" + else + BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz" + fi + + aws s3 cp "$BACKUP_FILE" \ + "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" \ + $ENDPOINT_FLAG + + echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$BACKUP_FILE" + + - name: Cleanup old backups + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} + run: | + ENDPOINT_FLAG="" + if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then + ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}" + fi + + CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d) + echo "Cleaning up backups older than $CUTOFF_DATE..." + + aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" $ENDPOINT_FLAG | while read -r line; do + FILE_DATE=$(echo "$line" | awk '{print $1}') + FILE_NAME=$(echo "$line" | awk '{print $4}') + if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then + echo "Deleting old backup: $FILE_NAME" + aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$FILE_NAME" $ENDPOINT_FLAG + fi + done + + echo "✅ Cleanup completed" diff --git a/README.md b/README.md index 680ab26..4bd62ce 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,158 @@ The worker includes a scheduled task that runs automatically to clean up soft-de You can modify the cron schedule in `wrangler.toml` if you want to run the cleanup task at a different time or frequency. See [Cloudflare Cron Triggers documentation](https://developers.cloudflare.com/workers/configuration/cron-triggers/) for cron expression syntax. +### Database Backup (GitHub Actions) + +This project includes a GitHub Action workflow that automatically backs up your D1 database to S3-compatible storage daily. The backup runs at 04:00 UTC (1 hour after the cleanup task). + +> ⚠️ **Important Notes:** +> - **Manual trigger required for first run:** You must manually trigger the Action once (GitHub Actions → Backup D1 Database to S3 → Run workflow) before scheduled backups will run automatically. +> - **Ensure your S3 bucket is set to private access** to prevent data leaks and avoid unnecessary public traffic costs. +> - **⚠️ CRITICAL: Do NOT use R2 from the same Cloudflare account as your Worker** for backups. If your Cloudflare account gets suspended or banned, you will lose access to both your Worker and your backup storage, resulting in complete data loss. Always use a separate Cloudflare account or a different S3-compatible storage provider (AWS S3, Backblaze B2, MinIO, etc.) for backups to ensure redundancy and disaster recovery. + +#### Required Secrets for Backup + +Add the following secrets to your GitHub repository (`Settings > Secrets and variables > Actions`): + +| Secret | Required | Description | +|--------|----------|-------------| +| `S3_ACCESS_KEY_ID` | yes | Your S3 access key ID | +| `S3_SECRET_ACCESS_KEY` | yes | Your S3 secret access key | +| `S3_BUCKET` | yes | The S3 bucket name for storing backups | +| `S3_REGION` | yes | The S3 region (e.g., `us-east-1`). If unsure, use `auto` | +| `S3_ENDPOINT` | no | Custom S3 endpoint URL. Defaults to AWS S3 if not set. Required for S3-compatible services (MinIO, Cloudflare R2, Backblaze B2, etc.) | +| `BACKUP_ENCRYPTION_KEY` | no | Optional encryption passphrase. If set, backups will be encrypted with AES-256. **Strongly recommended** since the database contains unencrypted user metadata (emails, item counts) | + +#### Backup Features + +* **Automatic Daily Backups:** Production database is backed up daily at 04:00 UTC +* **Manual Trigger:** You can manually trigger a backup from the GitHub Actions tab +* **Environment Selection:** When triggering manually, you can choose to backup either `production` or `dev` database +* **Compression:** Backups are compressed using gzip to save storage space +* **Optional Encryption:** If `BACKUP_ENCRYPTION_KEY` is set, backups are encrypted with AES-256-CBC (PBKDF2 key derivation, 100k iterations) +* **Automatic Cleanup:** Old backups older than 30 days are automatically deleted +* **S3-Compatible:** Works with AWS S3, Cloudflare R2, MinIO, Backblaze B2, and any S3-compatible storage + +#### Backup File Location + +Backups are stored in your S3 bucket with the following structure: + +``` +# Unencrypted backups +s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz + +# Encrypted backups (when BACKUP_ENCRYPTION_KEY is set) +s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc +``` + +#### Decrypting Backups + +If you enabled encryption, use the following command to decrypt a backup: + +```bash +openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \ + -in vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc \ + -out backup.sql.gz \ + -pass pass:"YOUR_ENCRYPTION_KEY" + +# Then decompress +gunzip backup.sql.gz +``` + +#### Restoring Database to Cloudflare D1 + +To restore your D1 database from a backup: + +1. **Download the backup from S3:** + + ```bash + # Using AWS CLI + aws s3 cp s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./ + + # Or with custom endpoint (e.g., R2, MinIO) + aws s3 cp s3://your-bucket/warden-worker/production/vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc ./ \ + --endpoint-url https://your-s3-endpoint.com + ``` + +2. **Decrypt the backup (if encrypted):** + + ```bash + openssl enc -aes-256-cbc -d -pbkdf2 -iter 100000 \ + -in vault1_prod_YYYY-MM-DD_HH-MM-SS.sql.gz.enc \ + -out backup.sql.gz \ + -pass pass:"YOUR_ENCRYPTION_KEY" + ``` + +3. **Decompress the backup:** + + ```bash + gunzip backup.sql.gz + ``` + +4. **Restore to Cloudflare D1:** + + ```bash + # First, you may want to clear the existing database (optional, use with caution!) + # wrangler d1 execute vault1 --remote --command "DELETE FROM ciphers; DELETE FROM folders; DELETE FROM users;" + + # Import the backup + wrangler d1 execute vault1 --remote --file=backup.sql + ``` + + > **Note:** The `--remote` flag is required to execute against your production D1 database. Without it, the command will run against the local development database. + + **Alternative: Using Database ID directly** + + If you don't have `vault1` binding configured in your local `wrangler.toml`, you can use the database ID (UUID) directly instead of the binding name: + + ```bash + # Replace YOUR_DATABASE_ID with your actual D1 database UUID + # wrangler d1 execute YOUR_DATABASE_ID --remote --command "DELETE FROM ciphers; DELETE FROM folders; DELETE FROM users;" + wrangler d1 execute YOUR_DATABASE_ID --remote --file=backup.sql + ``` + + You can find your database ID in the Cloudflare dashboard under Storage & databases > D1 SQL database, or by running `wrangler d1 list`. + +### Local Development with D1 + +You can run this Worker locally with full D1 database support using Wrangler. This is useful for development, testing, or as a temporary fallback when Cloudflare services are unavailable. + +To run locally with your production data (useful as emergency fallback): + +1. **Download and decrypt your backup** (follow the steps above) + +2. **Import the backup to local D1:** + + ```bash + # Without --remote flag, this imports to local database + wrangler d1 execute vault1 --file=backup.sql + ``` + +3. **Start the local server with persistence:** + + ```bash + wrangler dev --persist + ``` + +4. **Configure your Bitwarden client** to use `http://localhost:8787` (or your local network IP for mobile devices) + +#### Accessing Local SQLite Database Directly + +The local D1 database is stored as a SQLite file. You can access it directly: + +```bash +# Find the database file +ls .wrangler/state/v3/d1/ + +# Open with SQLite CLI +sqlite3 .wrangler/state/v3/d1/miniflare-D1DatabaseObject/*.sqlite + +# Example: List all users +sqlite> SELECT email FROM users; +``` + +> **Note:** The local development environment requires Node.js and Wrangler installed. The Worker runs in a simulated environment using [workerd](https://github.com/cloudflare/workerd), Cloudflare's open-source Workers runtime. + ## Contributing Contributions are welcome! If you find a bug, have a feature request, or want to improve the code, please open an issue or submit a pull request.