247 lines
11 KiB
YAML
247 lines
11 KiB
YAML
name: Backup D1 Database to S3
|
|
|
|
on:
|
|
schedule:
|
|
# Run daily at UTC 04:00 (1 hour after the auto cleanup task)
|
|
- cron: "0 4 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
environment:
|
|
description: "Select the environment to backup"
|
|
required: true
|
|
default: "production"
|
|
type: choice
|
|
options:
|
|
- production
|
|
- dev
|
|
|
|
env:
|
|
BACKUP_RETENTION_DAYS: 30
|
|
|
|
jobs:
|
|
backup-production:
|
|
name: Backup Production D1 Database
|
|
runs-on: ubuntu-latest
|
|
if: github.event_name == 'schedule' || github.event.inputs.environment == 'production'
|
|
|
|
steps:
|
|
- name: Get current date
|
|
id: date
|
|
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Install wrangler
|
|
run: npm install -g wrangler
|
|
|
|
- name: Get D1 Database Name
|
|
id: db_name
|
|
env:
|
|
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
run: |
|
|
DB_NAME=$(npx wrangler d1 list --json | jq -r '.[] | select(.uuid == "${{ secrets.D1_DATABASE_ID }}") | .name')
|
|
if [ -z "$DB_NAME" ]; then
|
|
echo "❌ Error: Could not find database with D1_DATABASE_ID secret"
|
|
exit 1
|
|
fi
|
|
echo "Found database name
|
|
echo "db_name=$DB_NAME" >> $GITHUB_OUTPUT
|
|
|
|
- name: Export D1 Database
|
|
env:
|
|
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
run: |
|
|
echo "Exporting D1 database: ${{ steps.db_name.outputs.db_name }}..."
|
|
npx wrangler d1 export "${{ steps.db_name.outputs.db_name }}" \
|
|
--remote \
|
|
--output=backup.sql
|
|
|
|
# Compress backup file
|
|
gzip backup.sql
|
|
|
|
echo "Backup compressed: backup.sql.gz"
|
|
ls -lh backup.sql.gz
|
|
|
|
- name: Encrypt backup (optional)
|
|
env:
|
|
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
|
run: |
|
|
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
|
echo "Encrypting backup with AES-256..."
|
|
openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \
|
|
-in backup.sql.gz \
|
|
-out "vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc" \
|
|
-pass pass:"$BACKUP_ENCRYPTION_KEY"
|
|
rm backup.sql.gz
|
|
echo "✅ Backup encrypted: vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc"
|
|
ls -lh *.enc
|
|
else
|
|
echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption"
|
|
mv backup.sql.gz "vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
|
|
echo "Backup file: vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
|
|
ls -lh *.sql.gz
|
|
fi
|
|
|
|
- name: Upload to S3
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
|
|
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
|
run: |
|
|
# Set custom S3 endpoint if configured (for S3-compatible services like MinIO, R2, etc.)
|
|
ENDPOINT_FLAG=""
|
|
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
|
|
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
|
|
fi
|
|
|
|
# Choose file extension based on encryption status
|
|
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
|
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz.enc"
|
|
else
|
|
BACKUP_FILE="vault1_prod_${{ steps.date.outputs.date }}.sql.gz"
|
|
fi
|
|
|
|
aws s3 cp "$BACKUP_FILE" \
|
|
"s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" \
|
|
$ENDPOINT_FLAG
|
|
|
|
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$BACKUP_FILE"
|
|
|
|
- name: Cleanup old backups
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
|
|
run: |
|
|
ENDPOINT_FLAG=""
|
|
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
|
|
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
|
|
fi
|
|
|
|
# Find and delete backup files older than retention period
|
|
CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d)
|
|
echo "Cleaning up backups older than $CUTOFF_DATE..."
|
|
|
|
aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/" $ENDPOINT_FLAG | while read -r line; do
|
|
FILE_DATE=$(echo "$line" | awk '{print $1}')
|
|
FILE_NAME=$(echo "$line" | awk '{print $4}')
|
|
if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then
|
|
echo "Deleting old backup: $FILE_NAME"
|
|
aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/production/$FILE_NAME" $ENDPOINT_FLAG
|
|
fi
|
|
done
|
|
|
|
echo "✅ Cleanup completed"
|
|
|
|
backup-dev:
|
|
name: Backup Dev D1 Database
|
|
runs-on: ubuntu-latest
|
|
if: github.event.inputs.environment == 'dev'
|
|
|
|
steps:
|
|
- name: Get current date
|
|
id: date
|
|
run: echo "date=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Install wrangler
|
|
run: npm install -g wrangler
|
|
|
|
- name: Get D1 Database Name (Dev)
|
|
id: db_name
|
|
env:
|
|
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
run: |
|
|
DB_NAME=$(npx wrangler d1 list --json | jq -r '.[] | select(.uuid == "${{ secrets.D1_DATABASE_ID_DEV }}") | .name')
|
|
if [ -z "$DB_NAME" ]; then
|
|
echo "❌ Error: Could not find database with D1_DATABASE_ID_DEV secret
|
|
exit 1
|
|
fi
|
|
echo "Found database name
|
|
echo "db_name=$DB_NAME" >> $GITHUB_OUTPUT
|
|
|
|
- name: Export D1 Database (Dev)
|
|
env:
|
|
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
run: |
|
|
echo "Exporting D1 dev database: ${{ steps.db_name.outputs.db_name }}..."
|
|
npx wrangler d1 export "${{ steps.db_name.outputs.db_name }}" \
|
|
--remote \
|
|
--output=backup.sql
|
|
|
|
gzip backup.sql
|
|
|
|
echo "Backup compressed: backup.sql.gz"
|
|
ls -lh backup.sql.gz
|
|
|
|
- name: Encrypt backup (optional)
|
|
env:
|
|
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
|
run: |
|
|
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
|
echo "Encrypting backup with AES-256..."
|
|
openssl enc -aes-256-cbc -salt -pbkdf2 -iter 100000 \
|
|
-in backup.sql.gz \
|
|
-out "vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc" \
|
|
-pass pass:"$BACKUP_ENCRYPTION_KEY"
|
|
rm backup.sql.gz
|
|
echo "✅ Backup encrypted: vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc"
|
|
ls -lh *.enc
|
|
else
|
|
echo "⚠️ BACKUP_ENCRYPTION_KEY not set, skipping encryption"
|
|
mv backup.sql.gz "vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
|
|
echo "Backup file: vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
|
|
ls -lh *.sql.gz
|
|
fi
|
|
|
|
- name: Upload to S3
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
|
|
BACKUP_ENCRYPTION_KEY: ${{ secrets.BACKUP_ENCRYPTION_KEY }}
|
|
run: |
|
|
ENDPOINT_FLAG=""
|
|
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
|
|
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
|
|
fi
|
|
|
|
# Choose file extension based on encryption status
|
|
if [ -n "$BACKUP_ENCRYPTION_KEY" ]; then
|
|
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz.enc"
|
|
else
|
|
BACKUP_FILE="vault1_dev_${{ steps.date.outputs.date }}.sql.gz"
|
|
fi
|
|
|
|
aws s3 cp "$BACKUP_FILE" \
|
|
"s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" \
|
|
$ENDPOINT_FLAG
|
|
|
|
echo "✅ Backup uploaded to S3: s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$BACKUP_FILE"
|
|
|
|
- name: Cleanup old backups
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
|
|
run: |
|
|
ENDPOINT_FLAG=""
|
|
if [ -n "${{ secrets.S3_ENDPOINT }}" ]; then
|
|
ENDPOINT_FLAG="--endpoint-url ${{ secrets.S3_ENDPOINT }}"
|
|
fi
|
|
|
|
CUTOFF_DATE=$(date -d "-${{ env.BACKUP_RETENTION_DAYS }} days" +%Y-%m-%d)
|
|
echo "Cleaning up backups older than $CUTOFF_DATE..."
|
|
|
|
aws s3 ls "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/" $ENDPOINT_FLAG | while read -r line; do
|
|
FILE_DATE=$(echo "$line" | awk '{print $1}')
|
|
FILE_NAME=$(echo "$line" | awk '{print $4}')
|
|
if [[ "$FILE_DATE" < "$CUTOFF_DATE" ]] && [[ -n "$FILE_NAME" ]]; then
|
|
echo "Deleting old backup: $FILE_NAME"
|
|
aws s3 rm "s3://${{ secrets.S3_BUCKET }}/warden-worker/dev/$FILE_NAME" $ENDPOINT_FLAG
|
|
fi
|
|
done
|
|
|
|
echo "✅ Cleanup completed"
|