From 5edd72b6cfb58029ba45a4b8471b29802ab3cc90 Mon Sep 17 00:00:00 2001 From: JohnBaumb <80135794+JohnBaumb@users.noreply.github.com> Date: Mon, 20 Apr 2026 14:16:57 -0700 Subject: [PATCH] Add dev nightly builds, ruff linting, Docker layer caching, and GHCR cleanup --- .github/workflows/build-and-test.yml | 6 ++ .github/workflows/cleanup-dev-images.yml | 24 +++++ .github/workflows/dev-nightly.yml | 108 +++++++++++++++++++++++ .github/workflows/docker-publish.yml | 12 ++- pyproject.toml | 18 ++++ requirements-dev.txt | 1 + web_server.py | 24 ++++- 7 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cleanup-dev-images.yml create mode 100644 .github/workflows/dev-nightly.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f94a3123..8c9b1a1c 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,6 +2,9 @@ name: Compile the app and run tests on: push: + branches-ignore: + - main + - dev jobs: sanity-check: @@ -21,6 +24,9 @@ jobs: - name: Install dependencies run: python -m pip install --upgrade pip && python -m pip install -r requirements-dev.txt + - name: Lint with ruff + run: python -m ruff check --output-format=github . + - name: Build app run: python -m compileall api core database services scripts web_server.py wsgi.py beatport_unified_scraper.py diff --git a/.github/workflows/cleanup-dev-images.yml b/.github/workflows/cleanup-dev-images.yml new file mode 100644 index 00000000..8edc5b9d --- /dev/null +++ b/.github/workflows/cleanup-dev-images.yml @@ -0,0 +1,24 @@ +name: Cleanup old dev images + +on: + schedule: + # Weekly on Sunday at 6 AM UTC + - cron: '0 6 * * 0' + workflow_dispatch: + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + packages: write + + steps: + - name: Delete old dev image tags + uses: actions/delete-package-versions@v5 + with: + package-name: soulsync + package-type: container + min-versions-to-keep: 10 + delete-only-pre-release-versions: false + # Only prune tags matching the dev nightly pattern (keep :dev rolling tag) + ignore-versions: '^(dev|latest|\\d+\\.\\d+)$' diff --git a/.github/workflows/dev-nightly.yml b/.github/workflows/dev-nightly.yml new file mode 100644 index 00000000..ed3d6ed9 --- /dev/null +++ b/.github/workflows/dev-nightly.yml @@ -0,0 +1,108 @@ +name: Dev Nightly Build + +on: + # Nightly at 4:00 AM UTC — only if dev branch has new commits + schedule: + - cron: '0 4 * * *' + # Also build on every push to dev for immediate feedback + push: + branches: + - dev + # Manual trigger for testing + workflow_dispatch: + +jobs: + nightly: + runs-on: ubuntu-latest + # Skip scheduled runs if dev branch has no new commits in the last 24h + # (pushes and manual triggers always run) + permissions: + contents: read + packages: write + + steps: + - name: Checkout dev branch + uses: actions/checkout@v6 + with: + ref: dev + + - name: Check for recent commits (scheduled only) + if: github.event_name == 'schedule' + id: recent + run: | + LAST_COMMIT=$(git log -1 --format=%ct) + NOW=$(date +%s) + DIFF=$(( NOW - LAST_COMMIT )) + if [ "$DIFF" -gt 86400 ]; then + echo "No commits in the last 24h, skipping build" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Python + if: steps.recent.outputs.skip != 'true' + uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Install dependencies and run tests + if: steps.recent.outputs.skip != 'true' + env: + PYTHONPATH: ${{ github.workspace }} + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-dev.txt + python -m compileall api core database services scripts web_server.py wsgi.py beatport_unified_scraper.py + python -m pytest + + - name: Generate version tag + if: steps.recent.outputs.skip != 'true' + id: version + run: | + # Read base version from web_server.py + BASE=$(python -c " + import re + with open('web_server.py') as f: + m = re.search(r'_SOULSYNC_BASE_VERSION\s*=\s*\"(.+?)\"', f.read()) + print(m.group(1) if m else 'dev') + ") + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + DATE=$(date -u +%Y%m%d) + TAG="${BASE}-dev.${DATE}.${SHORT_SHA}" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "Image tag: $TAG" + + - name: Set up QEMU + if: steps.recent.outputs.skip != 'true' + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + if: steps.recent.outputs.skip != 'true' + uses: docker/setup-buildx-action@v4 + + - name: Log in to GHCR + if: steps.recent.outputs.skip != 'true' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push to GHCR + if: steps.recent.outputs.skip != 'true' + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + pull: true + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + COMMIT_SHA=${{ github.sha }} + tags: | + ghcr.io/${{ github.repository_owner }}/soulsync:dev + ghcr.io/${{ github.repository_owner }}/soulsync:${{ steps.version.outputs.tag }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index f12467cf..e7ffa8ca 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,6 +1,11 @@ name: Build and Push Docker Image on: + # Auto-build :latest on every push to main + push: + branches: + - main + # Manual trigger for tagged releases (e.g. 2.33) workflow_dispatch: inputs: version_tag: @@ -34,16 +39,17 @@ jobs: context: . platforms: linux/amd64,linux/arm64 push: true - no-cache: true pull: true + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | COMMIT_SHA=${{ github.sha }} tags: | boulderbadgedad/soulsync:latest - boulderbadgedad/soulsync:${{ inputs.version_tag }} + ${{ inputs.version_tag && format('boulderbadgedad/soulsync:{0}', inputs.version_tag) || '' }} - name: Announce release to Discord - if: success() + if: success() && inputs.version_tag env: DISCORD_WEBHOOK: ${{ secrets.DISCORD_ANNOUNCEMENTS_WEBHOOK }} run: | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..79ca43ed --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[tool.ruff] +target-version = "py311" +line-length = 160 + +[tool.ruff.lint] +# Start lenient — catch real bugs, not style nits +# E: pycodestyle errors, F: pyflakes, UP: pyupgrade, B: bugbear +select = ["F", "E9", "W6", "B"] +# Ignore rules that will flag too much in an existing codebase +ignore = [ + "F401", # unused imports (too noisy initially) + "F841", # unused variables + "E501", # line length (handled by line-length setting) +] + +[tool.ruff.lint.per-file-ignores] +# Tests can use assert, magic values, etc. +"tests/**" = ["B", "F"] diff --git a/requirements-dev.txt b/requirements-dev.txt index a0c7797e..587022df 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,6 +2,7 @@ # Runtime web dependencies + test runner -r requirements.txt +ruff # Test runner pytest>=9.0.0 diff --git a/web_server.py b/web_server.py index b9c69e4d..82caa7f8 100644 --- a/web_server.py +++ b/web_server.py @@ -37,7 +37,24 @@ _log_dir = Path(_log_path).parent logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, version-info endpoint, etc. -SOULSYNC_VERSION = "2.35" +_SOULSYNC_BASE_VERSION = "2.35" + +def _build_version_string(): + """Append short commit hash to version when available (e.g. 2.35+abc1234).""" + sha = os.environ.get('SOULSYNC_COMMIT_SHA', '').strip() + if not sha: + try: + import subprocess + result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.') + if result.returncode == 0: + sha = result.stdout.strip() + except Exception: + pass + if sha: + return f"{_SOULSYNC_BASE_VERSION}+{sha[:7]}" + return _SOULSYNC_BASE_VERSION + +SOULSYNC_VERSION = _build_version_string() # Dedicated source reuse logger — writes alongside app.log in the configured log directory import logging as _logging @@ -25925,7 +25942,10 @@ def restore_backup_endpoint(filename): pass version_warning = None - if backup_version and backup_version != SOULSYNC_VERSION: + # Compare base versions only (strip +commit suffix) to avoid false mismatches + _backup_base = backup_version.split('+')[0] if backup_version else None + _current_base = SOULSYNC_VERSION.split('+')[0] + if _backup_base and _backup_base != _current_base: # Allow restore but warn — the caller must pass force=true to confirm force = request.json.get('force', False) if request.is_json else False if not force: